kernel/sched/completion.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Generic wait-for-completion handler;
   4  *
   5  * It differs from semaphores in that their default case is the opposite,
   6  * wait_for_completion default blocks whereas semaphore default non-block. The
   7  * interface also makes it easy to 'complete' multiple waiting threads,
   8  * something which isn't entirely natural for semaphores.
   9  *
  10  * But more importantly, the primitive documents the usage. Semaphores would
  11  * typically be used for exclusion which gives rise to priority inversion.
  12  * Waiting for completion is a typically sync point, but not an exclusion point.
  13  */
  14
  15 #include <linux/sched/signal.h>
  16 #include <linux/sched/debug.h>
  17 #include <linux/completion.h>
  18
  19 /**
  20  * complete: - signals a single thread waiting on this completion
  21  * @x:  holds the state of this particular completion
  22  *
  23  * This will wake up a single thread waiting on this completion. Threads will be
  24  * awakened in the same order in which they were queued.
  25  *
  26  * See also complete_all(), wait_for_completion() and related routines.
  27  *
  28  * It may be assumed that this function implies a write memory barrier before
  29  * changing the task state if and only if any tasks are woken up.
  30  */
  31 void complete(struct completion *x)
  32 {
  33         unsigned long flags;
  34
  35         spin_lock_irqsave(&x->wait.lock, flags);
  36
  37         if (x->done != UINT_MAX)
  38                 x->done++;
  39         __wake_up_locked(&x->wait, TASK_NORMAL, 1);
  40         spin_unlock_irqrestore(&x->wait.lock, flags);
  41 }
  42 EXPORT_SYMBOL(complete);
  43
  44 /**
  45  * complete_all: - signals all threads waiting on this completion
  46  * @x:  holds the state of this particular completion
  47  *
  48  * This will wake up all threads waiting on this particular completion event.
  49  *
  50  * It may be assumed that this function implies a write memory barrier before
  51  * changing the task state if and only if any tasks are woken up.
  52  *
  53  * Since complete_all() sets the completion of @x permanently to done
  54  * to allow multiple waiters to finish, a call to reinit_completion()
  55  * must be used on @x if @x is to be used again. The code must make
  56  * sure that all waiters have woken and finished before reinitializing
  57  * @x. Also note that the function completion_done() can not be used
  58  * to know if there are still waiters after complete_all() has been called.
  59  */
  60 void complete_all(struct completion *x)
  61 {
  62         unsigned long flags;
  63
  64         spin_lock_irqsave(&x->wait.lock, flags);
  65         x->done = UINT_MAX;
  66         __wake_up_locked(&x->wait, TASK_NORMAL, 0);
  67         spin_unlock_irqrestore(&x->wait.lock, flags);
  68 }
  69 EXPORT_SYMBOL(complete_all);
  70
  71 static inline long __sched
  72 do_wait_for_common(struct completion *x,
  73                    long (*action)(long), long timeout, int state)
  74 {
  75         if (!x->done) {
  76                 DECLARE_WAITQUEUE(wait, current);
  77
  78                 __add_wait_queue_entry_tail_exclusive(&x->wait, &wait);
  79                 do {
  80                         if (signal_pending_state(state, current)) {
  81                                 timeout = -ERESTARTSYS;
  82                                 break;
  83                         }
  84                         __set_current_state(state);
  85                         spin_unlock_irq(&x->wait.lock);
  86                         timeout = action(timeout);
  87                         spin_lock_irq(&x->wait.lock);
  88                 } while (!x->done && timeout);
  89                 __remove_wait_queue(&x->wait, &wait);
  90                 if (!x->done)
  91                         return timeout;
  92         }
  93         if (x->done != UINT_MAX)
  94                 x->done--;
  95         return timeout ?: 1;
  96 }
  97
  98 static inline long __sched
  99 __wait_for_common(struct completion *x,
 100                   long (*action)(long), long timeout, int state)
 101 {
 102         might_sleep();
 103
 104         complete_acquire(x);
 105
 106         spin_lock_irq(&x->wait.lock);
 107         timeout = do_wait_for_common(x, action, timeout, state);
 108         spin_unlock_irq(&x->wait.lock);
 109
 110         complete_release(x);
 111
 112         return timeout;
 113 }
 114
 115 static long __sched
 116 wait_for_common(struct completion *x, long timeout, int state)
 117 {
 118         return __wait_for_common(x, schedule_timeout, timeout, state);
 119 }
 120
 121 static long __sched
 122 wait_for_common_io(struct completion *x, long timeout, int state)
 123 {
 124         return __wait_for_common(x, io_schedule_timeout, timeout, state);
 125 }
 126
 127 /**
 128  * wait_for_completion: - waits for completion of a task
 129  * @x:  holds the state of this particular completion
 130  *
 131  * This waits to be signaled for completion of a specific task. It is NOT
 132  * interruptible and there is no timeout.
 133  *
 134  * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
 135  * and interrupt capability. Also see complete().
 136  */
 137 void __sched wait_for_completion(struct completion *x)
 138 {
 139         wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
 140 }
 141 EXPORT_SYMBOL(wait_for_completion);
 142
 143 /**
 144  * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
 145  * @x:  holds the state of this particular completion
 146  * @timeout:  timeout value in jiffies
 147  *
 148  * This waits for either a completion of a specific task to be signaled or for a
 149  * specified timeout to expire. The timeout is in jiffies. It is not
 150  * interruptible.
 151  *
 152  * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
 153  * till timeout) if completed.
 154  */
 155 unsigned long __sched
 156 wait_for_completion_timeout(struct completion *x, unsigned long timeout)
 157 {
 158         return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
 159 }
 160 EXPORT_SYMBOL(wait_for_completion_timeout);
 161
 162 /**
 163  * wait_for_completion_io: - waits for completion of a task
 164  * @x:  holds the state of this particular completion
 165  *
 166  * This waits to be signaled for completion of a specific task. It is NOT
 167  * interruptible and there is no timeout. The caller is accounted as waiting
 168  * for IO (which traditionally means blkio only).
 169  */
 170 void __sched wait_for_completion_io(struct completion *x)
 171 {
 172         wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
 173 }
 174 EXPORT_SYMBOL(wait_for_completion_io);
 175
 176 /**
 177  * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
 178  * @x:  holds the state of this particular completion
 179  * @timeout:  timeout value in jiffies
 180  *
 181  * This waits for either a completion of a specific task to be signaled or for a
 182  * specified timeout to expire. The timeout is in jiffies. It is not
 183  * interruptible. The caller is accounted as waiting for IO (which traditionally
 184  * means blkio only).
 185  *
 186  * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
 187  * till timeout) if completed.
 188  */
 189 unsigned long __sched
 190 wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
 191 {
 192         return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
 193 }
 194 EXPORT_SYMBOL(wait_for_completion_io_timeout);
 195
 196 /**
 197  * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
 198  * @x:  holds the state of this particular completion
 199  *
 200  * This waits for completion of a specific task to be signaled. It is
 201  * interruptible.
 202  *
 203  * Return: -ERESTARTSYS if interrupted, 0 if completed.
 204  */
 205 int __sched wait_for_completion_interruptible(struct completion *x)
 206 {
 207         long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
 208         if (t == -ERESTARTSYS)
 209                 return t;
 210         return 0;
 211 }
 212 EXPORT_SYMBOL(wait_for_completion_interruptible);
 213
 214 /**
 215  * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
 216  * @x:  holds the state of this particular completion
 217  * @timeout:  timeout value in jiffies
 218  *
 219  * This waits for either a completion of a specific task to be signaled or for a
 220  * specified timeout to expire. It is interruptible. The timeout is in jiffies.
 221  *
 222  * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
 223  * or number of jiffies left till timeout) if completed.
 224  */
 225 long __sched
 226 wait_for_completion_interruptible_timeout(struct completion *x,
 227                                           unsigned long timeout)
 228 {
 229         return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
 230 }
 231 EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
 232
 233 /**
 234  * wait_for_completion_killable: - waits for completion of a task (killable)
 235  * @x:  holds the state of this particular completion
 236  *
 237  * This waits to be signaled for completion of a specific task. It can be
 238  * interrupted by a kill signal.
 239  *
 240  * Return: -ERESTARTSYS if interrupted, 0 if completed.
 241  */
 242 int __sched wait_for_completion_killable(struct completion *x)
 243 {
 244         long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
 245         if (t == -ERESTARTSYS)
 246                 return t;
 247         return 0;
 248 }
 249 EXPORT_SYMBOL(wait_for_completion_killable);
 250
 251 /**
 252  * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
 253  * @x:  holds the state of this particular completion
 254  * @timeout:  timeout value in jiffies
 255  *
 256  * This waits for either a completion of a specific task to be
 257  * signaled or for a specified timeout to expire. It can be
 258  * interrupted by a kill signal. The timeout is in jiffies.
 259  *
 260  * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
 261  * or number of jiffies left till timeout) if completed.
 262  */
 263 long __sched
 264 wait_for_completion_killable_timeout(struct completion *x,
 265                                      unsigned long timeout)
 266 {
 267         return wait_for_common(x, timeout, TASK_KILLABLE);
 268 }
 269 EXPORT_SYMBOL(wait_for_completion_killable_timeout);
 270
 271 /**
 272  *      try_wait_for_completion - try to decrement a completion without blocking
 273  *      @x:     completion structure
 274  *
 275  *      Return: 0 if a decrement cannot be done without blocking
 276  *               1 if a decrement succeeded.
 277  *
 278  *      If a completion is being used as a counting completion,
 279  *      attempt to decrement the counter without blocking. This
 280  *      enables us to avoid waiting if the resource the completion
 281  *      is protecting is not available.
 282  */
 283 bool try_wait_for_completion(struct completion *x)
 284 {
 285         unsigned long flags;
 286         int ret = 1;
 287
 288         /*
 289          * Since x->done will need to be locked only
 290          * in the non-blocking case, we check x->done
 291          * first without taking the lock so we can
 292          * return early in the blocking case.
 293          */
 294         if (!READ_ONCE(x->done))
 295                 return 0;
 296
 297         spin_lock_irqsave(&x->wait.lock, flags);
 298         if (!x->done)
 299                 ret = 0;
 300         else if (x->done != UINT_MAX)
 301                 x->done--;
 302         spin_unlock_irqrestore(&x->wait.lock, flags);
 303         return ret;
 304 }
 305 EXPORT_SYMBOL(try_wait_for_completion);
 306
 307 /**
 308  *      completion_done - Test to see if a completion has any waiters
 309  *      @x:     completion structure
 310  *
 311  *      Return: 0 if there are waiters (wait_for_completion() in progress)
 312  *               1 if there are no waiters.
 313  *
 314  *      Note, this will always return true if complete_all() was called on @X.
 315  */
 316 bool completion_done(struct completion *x)
 317 {
 318         unsigned long flags;
 319
 320         if (!READ_ONCE(x->done))
 321                 return false;
 322
 323         /*
 324          * If ->done, we need to wait for complete() to release ->wait.lock
 325          * otherwise we can end up freeing the completion before complete()
 326          * is done referencing it.
 327          */
 328         spin_lock_irqsave(&x->wait.lock, flags);
 329         spin_unlock_irqrestore(&x->wait.lock, flags);
 330         return true;
 331 }
 332 EXPORT_SYMBOL(completion_done);