kernel/sched/wait_bit.c

   1 /*
   2  * The implementation of the wait_bit*() and related waiting APIs:
   3  */
   4 #include <linux/wait_bit.h>
   5 #include <linux/sched/signal.h>
   6 #include <linux/sched/debug.h>
   7 #include <linux/hash.h>
   8
   9 #define WAIT_TABLE_BITS 8
  10 #define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)
  11
  12 static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned;
  13
  14 wait_queue_head_t *bit_waitqueue(void *word, int bit)
  15 {
  16         const int shift = BITS_PER_LONG == 32 ? 5 : 6;
  17         unsigned long val = (unsigned long)word << shift | bit;
  18
  19         return bit_wait_table + hash_long(val, WAIT_TABLE_BITS);
  20 }
  21 EXPORT_SYMBOL(bit_waitqueue);
  22
  23 int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg)
  24 {
  25         struct wait_bit_key *key = arg;
  26         struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
  27
  28         if (wait_bit->key.flags != key->flags ||
  29                         wait_bit->key.bit_nr != key->bit_nr ||
  30                         test_bit(key->bit_nr, key->flags))
  31                 return 0;
  32
  33         return autoremove_wake_function(wq_entry, mode, sync, key);
  34 }
  35 EXPORT_SYMBOL(wake_bit_function);
  36
  37 /*
  38  * To allow interruptible waiting and asynchronous (i.e. nonblocking)
  39  * waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are
  40  * permitted return codes. Nonzero return codes halt waiting and return.
  41  */
  42 int __sched
  43 __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
  44               wait_bit_action_f *action, unsigned mode)
  45 {
  46         int ret = 0;
  47
  48         do {
  49                 prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
  50                 if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags))
  51                         ret = (*action)(&wbq_entry->key, mode);
  52         } while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret);
  53
  54         finish_wait(wq_head, &wbq_entry->wq_entry);
  55
  56         return ret;
  57 }
  58 EXPORT_SYMBOL(__wait_on_bit);
  59
  60 int __sched out_of_line_wait_on_bit(void *word, int bit,
  61                                     wait_bit_action_f *action, unsigned mode)
  62 {
  63         struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
  64         DEFINE_WAIT_BIT(wq_entry, word, bit);
  65
  66         return __wait_on_bit(wq_head, &wq_entry, action, mode);
  67 }
  68 EXPORT_SYMBOL(out_of_line_wait_on_bit);
  69
  70 int __sched out_of_line_wait_on_bit_timeout(
  71         void *word, int bit, wait_bit_action_f *action,
  72         unsigned mode, unsigned long timeout)
  73 {
  74         struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
  75         DEFINE_WAIT_BIT(wq_entry, word, bit);
  76
  77         wq_entry.key.timeout = jiffies + timeout;
  78
  79         return __wait_on_bit(wq_head, &wq_entry, action, mode);
  80 }
  81 EXPORT_SYMBOL_GPL(out_of_line_wait_on_bit_timeout);
  82
  83 int __sched
  84 __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
  85                         wait_bit_action_f *action, unsigned mode)
  86 {
  87         int ret = 0;
  88
  89         for (;;) {
  90                 prepare_to_wait_exclusive(wq_head, &wbq_entry->wq_entry, mode);
  91                 if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
  92                         ret = action(&wbq_entry->key, mode);
  93                         /*
  94                          * See the comment in prepare_to_wait_event().
  95                          * finish_wait() does not necessarily takes wwq_head->lock,
  96                          * but test_and_set_bit() implies mb() which pairs with
  97                          * smp_mb__after_atomic() before wake_up_page().
  98                          */
  99                         if (ret)
 100                                 finish_wait(wq_head, &wbq_entry->wq_entry);
 101                 }
 102                 if (!test_and_set_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
 103                         if (!ret)
 104                                 finish_wait(wq_head, &wbq_entry->wq_entry);
 105                         return 0;
 106                 } else if (ret) {
 107                         return ret;
 108                 }
 109         }
 110 }
 111 EXPORT_SYMBOL(__wait_on_bit_lock);
 112
 113 int __sched out_of_line_wait_on_bit_lock(void *word, int bit,
 114                                          wait_bit_action_f *action, unsigned mode)
 115 {
 116         struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
 117         DEFINE_WAIT_BIT(wq_entry, word, bit);
 118
 119         return __wait_on_bit_lock(wq_head, &wq_entry, action, mode);
 120 }
 121 EXPORT_SYMBOL(out_of_line_wait_on_bit_lock);
 122
 123 void __wake_up_bit(struct wait_queue_head *wq_head, void *word, int bit)
 124 {
 125         struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit);
 126
 127         if (waitqueue_active(wq_head))
 128                 __wake_up(wq_head, TASK_NORMAL, 1, &key);
 129 }
 130 EXPORT_SYMBOL(__wake_up_bit);
 131
 132 /**
 133  * wake_up_bit - wake up a waiter on a bit
 134  * @word: the word being waited on, a kernel virtual address
 135  * @bit: the bit of the word being waited on
 136  *
 137  * There is a standard hashed waitqueue table for generic use. This
 138  * is the part of the hashtable's accessor API that wakes up waiters
 139  * on a bit. For instance, if one were to have waiters on a bitflag,
 140  * one would call wake_up_bit() after clearing the bit.
 141  *
 142  * In order for this to function properly, as it uses waitqueue_active()
 143  * internally, some kind of memory barrier must be done prior to calling
 144  * this. Typically, this will be smp_mb__after_atomic(), but in some
 145  * cases where bitflags are manipulated non-atomically under a lock, one
 146  * may need to use a less regular barrier, such fs/inode.c's smp_mb(),
 147  * because spin_unlock() does not guarantee a memory barrier.
 148  */
 149 void wake_up_bit(void *word, int bit)
 150 {
 151         __wake_up_bit(bit_waitqueue(word, bit), word, bit);
 152 }
 153 EXPORT_SYMBOL(wake_up_bit);
 154
 155 /*
 156  * Manipulate the atomic_t address to produce a better bit waitqueue table hash
 157  * index (we're keying off bit -1, but that would produce a horrible hash
 158  * value).
 159  */
 160 static inline wait_queue_head_t *atomic_t_waitqueue(atomic_t *p)
 161 {
 162         if (BITS_PER_LONG == 64) {
 163                 unsigned long q = (unsigned long)p;
 164
 165                 return bit_waitqueue((void *)(q & ~1), q & 1);
 166         }
 167         return bit_waitqueue(p, 0);
 168 }
 169
 170 static int wake_atomic_t_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync,
 171                                   void *arg)
 172 {
 173         struct wait_bit_key *key = arg;
 174         struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
 175         atomic_t *val = key->flags;
 176
 177         if (wait_bit->key.flags != key->flags ||
 178             wait_bit->key.bit_nr != key->bit_nr ||
 179             atomic_read(val) != 0)
 180                 return 0;
 181
 182         return autoremove_wake_function(wq_entry, mode, sync, key);
 183 }
 184
 185 /*
 186  * To allow interruptible waiting and asynchronous (i.e. nonblocking) waiting,
 187  * the actions of __wait_on_atomic_t() are permitted return codes.  Nonzero
 188  * return codes halt waiting and return.
 189  */
 190 static __sched
 191 int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
 192                        wait_atomic_t_action_f action, unsigned int mode)
 193 {
 194         atomic_t *val;
 195         int ret = 0;
 196
 197         do {
 198                 prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
 199                 val = wbq_entry->key.flags;
 200                 if (atomic_read(val) == 0)
 201                         break;
 202                 ret = (*action)(val, mode);
 203         } while (!ret && atomic_read(val) != 0);
 204         finish_wait(wq_head, &wbq_entry->wq_entry);
 205
 206         return ret;
 207 }
 208
 209 #define DEFINE_WAIT_ATOMIC_T(name, p)                                   \
 210         struct wait_bit_queue_entry name = {                            \
 211                 .key = __WAIT_ATOMIC_T_KEY_INITIALIZER(p),              \
 212                 .wq_entry = {                                           \
 213                         .private        = current,                      \
 214                         .func           = wake_atomic_t_function,       \
 215                         .entry          =                               \
 216                                 LIST_HEAD_INIT((name).wq_entry.entry),  \
 217                 },                                                      \
 218         }
 219
 220 __sched int out_of_line_wait_on_atomic_t(atomic_t *p,
 221                                          wait_atomic_t_action_f action,
 222                                          unsigned int mode)
 223 {
 224         struct wait_queue_head *wq_head = atomic_t_waitqueue(p);
 225         DEFINE_WAIT_ATOMIC_T(wq_entry, p);
 226
 227         return __wait_on_atomic_t(wq_head, &wq_entry, action, mode);
 228 }
 229 EXPORT_SYMBOL(out_of_line_wait_on_atomic_t);
 230
 231 __sched int atomic_t_wait(atomic_t *counter, unsigned int mode)
 232 {
 233         schedule();
 234         if (signal_pending_state(mode, current))
 235                 return -EINTR;
 236
 237         return 0;
 238 }
 239 EXPORT_SYMBOL(atomic_t_wait);
 240
 241 /**
 242  * wake_up_atomic_t - Wake up a waiter on a atomic_t
 243  * @p: The atomic_t being waited on, a kernel virtual address
 244  *
 245  * Wake up anyone waiting for the atomic_t to go to zero.
 246  *
 247  * Abuse the bit-waker function and its waitqueue hash table set (the atomic_t
 248  * check is done by the waiter's wake function, not the by the waker itself).
 249  */
 250 void wake_up_atomic_t(atomic_t *p)
 251 {
 252         __wake_up_bit(atomic_t_waitqueue(p), p, WAIT_ATOMIC_T_BIT_NR);
 253 }
 254 EXPORT_SYMBOL(wake_up_atomic_t);
 255
 256 __sched int bit_wait(struct wait_bit_key *word, int mode)
 257 {
 258         schedule();
 259         if (signal_pending_state(mode, current))
 260                 return -EINTR;
 261
 262         return 0;
 263 }
 264 EXPORT_SYMBOL(bit_wait);
 265
 266 __sched int bit_wait_io(struct wait_bit_key *word, int mode)
 267 {
 268         io_schedule();
 269         if (signal_pending_state(mode, current))
 270                 return -EINTR;
 271
 272         return 0;
 273 }
 274 EXPORT_SYMBOL(bit_wait_io);
 275
 276 __sched int bit_wait_timeout(struct wait_bit_key *word, int mode)
 277 {
 278         unsigned long now = READ_ONCE(jiffies);
 279
 280         if (time_after_eq(now, word->timeout))
 281                 return -EAGAIN;
 282         schedule_timeout(word->timeout - now);
 283         if (signal_pending_state(mode, current))
 284                 return -EINTR;
 285
 286         return 0;
 287 }
 288 EXPORT_SYMBOL_GPL(bit_wait_timeout);
 289
 290 __sched int bit_wait_io_timeout(struct wait_bit_key *word, int mode)
 291 {
 292         unsigned long now = READ_ONCE(jiffies);
 293
 294         if (time_after_eq(now, word->timeout))
 295                 return -EAGAIN;
 296         io_schedule_timeout(word->timeout - now);
 297         if (signal_pending_state(mode, current))
 298                 return -EINTR;
 299
 300         return 0;
 301 }
 302 EXPORT_SYMBOL_GPL(bit_wait_io_timeout);
 303
 304 void __init wait_bit_init(void)
 305 {
 306         int i;
 307
 308         for (i = 0; i < WAIT_TABLE_SIZE; i++)
 309                 init_waitqueue_head(bit_wait_table + i);
 310 }