]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
e73f8959 ON |
2 | #include <linux/spinlock.h> |
3 | #include <linux/task_work.h> | |
4 | #include <linux/tracehook.h> | |
5 | ||
9da33de6 ON |
6 | static struct callback_head work_exited; /* all we need is ->next == NULL */ |
7 | ||
892f6668 ON |
8 | /** |
9 | * task_work_add - ask the @task to execute @work->func() | |
10 | * @task: the task which should run the callback | |
11 | * @work: the callback to run | |
12 | * @notify: send the notification if true | |
13 | * | |
14 | * Queue @work for task_work_run() below and notify the @task if @notify. | |
15 | * Fails if the @task is exiting/exited and thus it can't process this @work. | |
16 | * Otherwise @work->func() will be called when the @task returns from kernel | |
17 | * mode or exits. | |
18 | * | |
19 | * This is like the signal handler which runs in kernel mode, but it doesn't | |
20 | * try to wake up the @task. | |
21 | * | |
c8219906 ED |
22 | * Note: there is no ordering guarantee on works queued here. |
23 | * | |
892f6668 ON |
24 | * RETURNS: |
25 | * 0 if succeeds or -ESRCH. | |
26 | */ | |
e73f8959 | 27 | int |
ac3d0da8 | 28 | task_work_add(struct task_struct *task, struct callback_head *work, bool notify) |
e73f8959 | 29 | { |
ac3d0da8 | 30 | struct callback_head *head; |
9da33de6 | 31 | |
ac3d0da8 | 32 | do { |
61e96496 | 33 | head = READ_ONCE(task->task_works); |
9da33de6 ON |
34 | if (unlikely(head == &work_exited)) |
35 | return -ESRCH; | |
ac3d0da8 ON |
36 | work->next = head; |
37 | } while (cmpxchg(&task->task_works, head, work) != head); | |
e73f8959 | 38 | |
ed3e694d | 39 | if (notify) |
e73f8959 | 40 | set_notify_resume(task); |
ed3e694d | 41 | return 0; |
e73f8959 | 42 | } |
ceaf03b4 | 43 | EXPORT_SYMBOL(task_work_add); |
e73f8959 | 44 | |
892f6668 ON |
45 | /** |
46 | * task_work_cancel - cancel a pending work added by task_work_add() | |
47 | * @task: the task which should execute the work | |
48 | * @func: identifies the work to remove | |
49 | * | |
50 | * Find the last queued pending work with ->func == @func and remove | |
51 | * it from queue. | |
52 | * | |
53 | * RETURNS: | |
54 | * The found work or NULL if not found. | |
55 | */ | |
67d12145 | 56 | struct callback_head * |
e73f8959 ON |
57 | task_work_cancel(struct task_struct *task, task_work_func_t func) |
58 | { | |
ac3d0da8 | 59 | struct callback_head **pprev = &task->task_works; |
205e550a | 60 | struct callback_head *work; |
e73f8959 | 61 | unsigned long flags; |
61e96496 ON |
62 | |
63 | if (likely(!task->task_works)) | |
64 | return NULL; | |
ac3d0da8 ON |
65 | /* |
66 | * If cmpxchg() fails we continue without updating pprev. | |
67 | * Either we raced with task_work_add() which added the | |
68 | * new entry before this work, we will find it again. Or | |
9da33de6 | 69 | * we raced with task_work_run(), *pprev == NULL/exited. |
ac3d0da8 | 70 | */ |
e73f8959 | 71 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
506458ef | 72 | while ((work = READ_ONCE(*pprev))) { |
ac3d0da8 ON |
73 | if (work->func != func) |
74 | pprev = &work->next; | |
75 | else if (cmpxchg(pprev, work, work->next) == work) | |
76 | break; | |
e73f8959 | 77 | } |
e73f8959 | 78 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
ac3d0da8 ON |
79 | |
80 | return work; | |
e73f8959 ON |
81 | } |
82 | ||
892f6668 ON |
83 | /** |
84 | * task_work_run - execute the works added by task_work_add() | |
85 | * | |
86 | * Flush the pending works. Should be used by the core kernel code. | |
87 | * Called before the task returns to the user-mode or stops, or when | |
88 | * it exits. In the latter case task_work_add() can no longer add the | |
89 | * new work after task_work_run() returns. | |
90 | */ | |
e73f8959 ON |
91 | void task_work_run(void) |
92 | { | |
93 | struct task_struct *task = current; | |
ac3d0da8 | 94 | struct callback_head *work, *head, *next; |
e73f8959 | 95 | |
ac3d0da8 | 96 | for (;;) { |
9da33de6 ON |
97 | /* |
98 | * work->func() can do task_work_add(), do not set | |
99 | * work_exited unless the list is empty. | |
100 | */ | |
f274f1e7 | 101 | raw_spin_lock_irq(&task->pi_lock); |
9da33de6 | 102 | do { |
61e96496 | 103 | work = READ_ONCE(task->task_works); |
9da33de6 ON |
104 | head = !work && (task->flags & PF_EXITING) ? |
105 | &work_exited : NULL; | |
106 | } while (cmpxchg(&task->task_works, work, head) != work); | |
f274f1e7 | 107 | raw_spin_unlock_irq(&task->pi_lock); |
9da33de6 | 108 | |
ac3d0da8 ON |
109 | if (!work) |
110 | break; | |
e73f8959 | 111 | |
ac3d0da8 ON |
112 | do { |
113 | next = work->next; | |
114 | work->func(work); | |
115 | work = next; | |
f341861f | 116 | cond_resched(); |
ac3d0da8 | 117 | } while (work); |
e73f8959 ON |
118 | } |
119 | } | |
a3a49a17 | 120 | EXPORT_SYMBOL_GPL(task_work_run); |