]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - kernel/workqueue.c
NVMe: Remove unused sq_head read in completion path
[mirror_ubuntu-artful-kernel.git] / kernel / workqueue.c
CommitLineData
1da177e4 1/*
c54fce6e 2 * kernel/workqueue.c - generic async execution with shared worker pool
1da177e4 3 *
c54fce6e 4 * Copyright (C) 2002 Ingo Molnar
1da177e4 5 *
c54fce6e
TH
6 * Derived from the taskqueue/keventd code by:
7 * David Woodhouse <dwmw2@infradead.org>
8 * Andrew Morton
9 * Kai Petzke <wpp@marie.physik.tu-berlin.de>
10 * Theodore Ts'o <tytso@mit.edu>
1da177e4 11 *
c54fce6e 12 * Made to use alloc_percpu by Christoph Lameter.
1da177e4 13 *
c54fce6e
TH
14 * Copyright (C) 2010 SUSE Linux Products GmbH
15 * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
89ada679 16 *
c54fce6e
TH
17 * This is the generic async execution mechanism. Work items as are
18 * executed in process context. The worker pool is shared and
b11895c4
L
19 * automatically managed. There are two worker pools for each CPU (one for
20 * normal work items and the other for high priority ones) and some extra
21 * pools for workqueues which are not bound to any specific CPU - the
22 * number of these backing pools is dynamic.
c54fce6e
TH
23 *
24 * Please read Documentation/workqueue.txt for details.
1da177e4
LT
25 */
26
9984de1a 27#include <linux/export.h>
1da177e4
LT
28#include <linux/kernel.h>
29#include <linux/sched.h>
30#include <linux/init.h>
31#include <linux/signal.h>
32#include <linux/completion.h>
33#include <linux/workqueue.h>
34#include <linux/slab.h>
35#include <linux/cpu.h>
36#include <linux/notifier.h>
37#include <linux/kthread.h>
1fa44eca 38#include <linux/hardirq.h>
46934023 39#include <linux/mempolicy.h>
341a5958 40#include <linux/freezer.h>
d5abe669
PZ
41#include <linux/kallsyms.h>
42#include <linux/debug_locks.h>
4e6045f1 43#include <linux/lockdep.h>
c34056a3 44#include <linux/idr.h>
29c91e99 45#include <linux/jhash.h>
42f8570f 46#include <linux/hashtable.h>
76af4d93 47#include <linux/rculist.h>
bce90380 48#include <linux/nodemask.h>
4c16bd32 49#include <linux/moduleparam.h>
3d1cb205 50#include <linux/uaccess.h>
e22bee78 51
ea138446 52#include "workqueue_internal.h"
1da177e4 53
c8e55f36 54enum {
24647570
TH
55 /*
56 * worker_pool flags
bc2ae0f5 57 *
24647570 58 * A bound pool is either associated or disassociated with its CPU.
bc2ae0f5
TH
59 * While associated (!DISASSOCIATED), all workers are bound to the
60 * CPU and none has %WORKER_UNBOUND set and concurrency management
61 * is in effect.
62 *
63 * While DISASSOCIATED, the cpu may be offline and all workers have
64 * %WORKER_UNBOUND set and concurrency management disabled, and may
24647570 65 * be executing on any CPU. The pool behaves as an unbound one.
bc2ae0f5 66 *
bc3a1afc 67 * Note that DISASSOCIATED should be flipped only while holding
92f9c5c4 68 * attach_mutex to avoid changing binding state while
4736cbf7 69 * worker_attach_to_pool() is in progress.
bc2ae0f5 70 */
24647570 71 POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
db7bccf4 72
c8e55f36 73 /* worker flags */
c8e55f36
TH
74 WORKER_DIE = 1 << 1, /* die die die */
75 WORKER_IDLE = 1 << 2, /* is idle */
e22bee78 76 WORKER_PREP = 1 << 3, /* preparing to run works */
fb0e7beb 77 WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
f3421797 78 WORKER_UNBOUND = 1 << 7, /* worker is unbound */
a9ab775b 79 WORKER_REBOUND = 1 << 8, /* worker was rebound */
e22bee78 80
a9ab775b
TH
81 WORKER_NOT_RUNNING = WORKER_PREP | WORKER_CPU_INTENSIVE |
82 WORKER_UNBOUND | WORKER_REBOUND,
db7bccf4 83
e34cdddb 84 NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */
4ce62e9e 85
29c91e99 86 UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
c8e55f36 87 BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
db7bccf4 88
e22bee78
TH
89 MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */
90 IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */
91
3233cdbd
TH
92 MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
93 /* call for help after 10ms
94 (min two ticks) */
e22bee78
TH
95 MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */
96 CREATE_COOLDOWN = HZ, /* time to breath after fail */
e22bee78
TH
97
98 /*
99 * Rescue workers are used only on emergencies and shared by
8698a745 100 * all cpus. Give MIN_NICE.
e22bee78 101 */
8698a745
DY
102 RESCUER_NICE_LEVEL = MIN_NICE,
103 HIGHPRI_NICE_LEVEL = MIN_NICE,
ecf6881f
TH
104
105 WQ_NAME_LEN = 24,
c8e55f36 106};
1da177e4
LT
107
108/*
4690c4ab
TH
109 * Structure fields follow one of the following exclusion rules.
110 *
e41e704b
TH
111 * I: Modifiable by initialization/destruction paths and read-only for
112 * everyone else.
4690c4ab 113 *
e22bee78
TH
114 * P: Preemption protected. Disabling preemption is enough and should
115 * only be modified and accessed from the local cpu.
116 *
d565ed63 117 * L: pool->lock protected. Access with pool->lock held.
4690c4ab 118 *
d565ed63
TH
119 * X: During normal operation, modification requires pool->lock and should
120 * be done only from local cpu. Either disabling preemption on local
121 * cpu or grabbing pool->lock is enough for read access. If
122 * POOL_DISASSOCIATED is set, it's identical to L.
e22bee78 123 *
92f9c5c4 124 * A: pool->attach_mutex protected.
822d8405 125 *
68e13a67 126 * PL: wq_pool_mutex protected.
5bcab335 127 *
68e13a67 128 * PR: wq_pool_mutex protected for writes. Sched-RCU protected for reads.
76af4d93 129 *
5b95e1af
LJ
130 * PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
131 *
132 * PWR: wq_pool_mutex and wq->mutex protected for writes. Either or
133 * sched-RCU for reads.
134 *
3c25a55d
LJ
135 * WQ: wq->mutex protected.
136 *
b5927605 137 * WR: wq->mutex protected for writes. Sched-RCU protected for reads.
2e109a28
TH
138 *
139 * MD: wq_mayday_lock protected.
1da177e4 140 */
1da177e4 141
2eaebdb3 142/* struct worker is defined in workqueue_internal.h */
c34056a3 143
bd7bdd43 144struct worker_pool {
d565ed63 145 spinlock_t lock; /* the pool lock */
d84ff051 146 int cpu; /* I: the associated cpu */
f3f90ad4 147 int node; /* I: the associated node ID */
9daf9e67 148 int id; /* I: pool ID */
11ebea50 149 unsigned int flags; /* X: flags */
bd7bdd43 150
82607adc
TH
151 unsigned long watchdog_ts; /* L: watchdog timestamp */
152
bd7bdd43
TH
153 struct list_head worklist; /* L: list of pending works */
154 int nr_workers; /* L: total number of workers */
ea1abd61
LJ
155
156 /* nr_idle includes the ones off idle_list for rebinding */
bd7bdd43
TH
157 int nr_idle; /* L: currently idle ones */
158
159 struct list_head idle_list; /* X: list of idle workers */
160 struct timer_list idle_timer; /* L: worker idle timeout */
161 struct timer_list mayday_timer; /* L: SOS timer for workers */
162
c5aa87bb 163 /* a workers is either on busy_hash or idle_list, or the manager */
c9e7cf27
TH
164 DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
165 /* L: hash of busy workers */
166
bc3a1afc 167 /* see manage_workers() for details on the two manager mutexes */
34a06bd6 168 struct mutex manager_arb; /* manager arbitration */
2607d7a6 169 struct worker *manager; /* L: purely informational */
92f9c5c4
LJ
170 struct mutex attach_mutex; /* attach/detach exclusion */
171 struct list_head workers; /* A: attached workers */
60f5a4bc 172 struct completion *detach_completion; /* all workers detached */
e19e397a 173
7cda9aae 174 struct ida worker_ida; /* worker IDs for task name */
e19e397a 175
7a4e344c 176 struct workqueue_attrs *attrs; /* I: worker attributes */
68e13a67
LJ
177 struct hlist_node hash_node; /* PL: unbound_pool_hash node */
178 int refcnt; /* PL: refcnt for unbound pools */
7a4e344c 179
e19e397a
TH
180 /*
181 * The current concurrency level. As it's likely to be accessed
182 * from other CPUs during try_to_wake_up(), put it in a separate
183 * cacheline.
184 */
185 atomic_t nr_running ____cacheline_aligned_in_smp;
29c91e99
TH
186
187 /*
188 * Destruction of pool is sched-RCU protected to allow dereferences
189 * from get_work_pool().
190 */
191 struct rcu_head rcu;
8b03ae3c
TH
192} ____cacheline_aligned_in_smp;
193
1da177e4 194/*
112202d9
TH
195 * The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
196 * of work_struct->data are used for flags and the remaining high bits
197 * point to the pwq; thus, pwqs need to be aligned at two's power of the
198 * number of flag bits.
1da177e4 199 */
112202d9 200struct pool_workqueue {
bd7bdd43 201 struct worker_pool *pool; /* I: the associated pool */
4690c4ab 202 struct workqueue_struct *wq; /* I: the owning workqueue */
73f53c4a
TH
203 int work_color; /* L: current color */
204 int flush_color; /* L: flushing color */
8864b4e5 205 int refcnt; /* L: reference count */
73f53c4a
TH
206 int nr_in_flight[WORK_NR_COLORS];
207 /* L: nr of in_flight works */
1e19ffc6 208 int nr_active; /* L: nr of active works */
a0a1a5fd 209 int max_active; /* L: max active works */
1e19ffc6 210 struct list_head delayed_works; /* L: delayed works */
3c25a55d 211 struct list_head pwqs_node; /* WR: node on wq->pwqs */
2e109a28 212 struct list_head mayday_node; /* MD: node on wq->maydays */
8864b4e5
TH
213
214 /*
215 * Release of unbound pwq is punted to system_wq. See put_pwq()
216 * and pwq_unbound_release_workfn() for details. pool_workqueue
217 * itself is also sched-RCU protected so that the first pwq can be
b09f4fd3 218 * determined without grabbing wq->mutex.
8864b4e5
TH
219 */
220 struct work_struct unbound_release_work;
221 struct rcu_head rcu;
e904e6c2 222} __aligned(1 << WORK_STRUCT_FLAG_BITS);
1da177e4 223
73f53c4a
TH
224/*
225 * Structure used to wait for workqueue flush.
226 */
227struct wq_flusher {
3c25a55d
LJ
228 struct list_head list; /* WQ: list of flushers */
229 int flush_color; /* WQ: flush color waiting for */
73f53c4a
TH
230 struct completion done; /* flush completion */
231};
232
226223ab
TH
233struct wq_device;
234
1da177e4 235/*
c5aa87bb
TH
236 * The externally visible workqueue. It relays the issued work items to
237 * the appropriate worker_pool through its pool_workqueues.
1da177e4
LT
238 */
239struct workqueue_struct {
3c25a55d 240 struct list_head pwqs; /* WR: all pwqs of this wq */
e2dca7ad 241 struct list_head list; /* PR: list of all workqueues */
73f53c4a 242
3c25a55d
LJ
243 struct mutex mutex; /* protects this wq */
244 int work_color; /* WQ: current work color */
245 int flush_color; /* WQ: current flush color */
112202d9 246 atomic_t nr_pwqs_to_flush; /* flush in progress */
3c25a55d
LJ
247 struct wq_flusher *first_flusher; /* WQ: first flusher */
248 struct list_head flusher_queue; /* WQ: flush waiters */
249 struct list_head flusher_overflow; /* WQ: flush overflow list */
73f53c4a 250
2e109a28 251 struct list_head maydays; /* MD: pwqs requesting rescue */
e22bee78
TH
252 struct worker *rescuer; /* I: rescue worker */
253
87fc741e 254 int nr_drainers; /* WQ: drain in progress */
a357fc03 255 int saved_max_active; /* WQ: saved pwq max_active */
226223ab 256
5b95e1af
LJ
257 struct workqueue_attrs *unbound_attrs; /* PW: only for unbound wqs */
258 struct pool_workqueue *dfl_pwq; /* PW: only for unbound wqs */
6029a918 259
226223ab
TH
260#ifdef CONFIG_SYSFS
261 struct wq_device *wq_dev; /* I: for sysfs interface */
262#endif
4e6045f1 263#ifdef CONFIG_LOCKDEP
4690c4ab 264 struct lockdep_map lockdep_map;
4e6045f1 265#endif
ecf6881f 266 char name[WQ_NAME_LEN]; /* I: workqueue name */
2728fd2f 267
e2dca7ad
TH
268 /*
269 * Destruction of workqueue_struct is sched-RCU protected to allow
270 * walking the workqueues list without grabbing wq_pool_mutex.
271 * This is used to dump all workqueues from sysrq.
272 */
273 struct rcu_head rcu;
274
2728fd2f
TH
275 /* hot fields used during command issue, aligned to cacheline */
276 unsigned int flags ____cacheline_aligned; /* WQ: WQ_* flags */
277 struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwqs */
5b95e1af 278 struct pool_workqueue __rcu *numa_pwq_tbl[]; /* PWR: unbound pwqs indexed by node */
1da177e4
LT
279};
280
e904e6c2
TH
281static struct kmem_cache *pwq_cache;
282
bce90380
TH
283static cpumask_var_t *wq_numa_possible_cpumask;
284 /* possible CPUs of each node */
285
d55262c4
TH
286static bool wq_disable_numa;
287module_param_named(disable_numa, wq_disable_numa, bool, 0444);
288
cee22a15 289/* see the comment above the definition of WQ_POWER_EFFICIENT */
552f530c 290static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
cee22a15
VK
291module_param_named(power_efficient, wq_power_efficient, bool, 0444);
292
bce90380
TH
293static bool wq_numa_enabled; /* unbound NUMA affinity enabled */
294
4c16bd32
TH
295/* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */
296static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf;
297
68e13a67 298static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
2e109a28 299static DEFINE_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
5bcab335 300
e2dca7ad 301static LIST_HEAD(workqueues); /* PR: list of all workqueues */
68e13a67 302static bool workqueue_freezing; /* PL: have wqs started freezing? */
7d19c5ce 303
042f7df1 304static cpumask_var_t wq_unbound_cpumask; /* PL: low level cpumask for all unbound wqs */
b05a7928 305
7d19c5ce
TH
306/* the per-cpu worker pools */
307static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
308 cpu_worker_pools);
309
68e13a67 310static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */
7d19c5ce 311
68e13a67 312/* PL: hash of all unbound pools keyed by pool->attrs */
29c91e99
TH
313static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
314
c5aa87bb 315/* I: attributes used when instantiating standard unbound pools on demand */
29c91e99
TH
316static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
317
8a2b7538
TH
318/* I: attributes used when instantiating ordered pools on demand */
319static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
320
d320c038 321struct workqueue_struct *system_wq __read_mostly;
ad7b1f84 322EXPORT_SYMBOL(system_wq);
044c782c 323struct workqueue_struct *system_highpri_wq __read_mostly;
1aabe902 324EXPORT_SYMBOL_GPL(system_highpri_wq);
044c782c 325struct workqueue_struct *system_long_wq __read_mostly;
d320c038 326EXPORT_SYMBOL_GPL(system_long_wq);
044c782c 327struct workqueue_struct *system_unbound_wq __read_mostly;
f3421797 328EXPORT_SYMBOL_GPL(system_unbound_wq);
044c782c 329struct workqueue_struct *system_freezable_wq __read_mostly;
24d51add 330EXPORT_SYMBOL_GPL(system_freezable_wq);
0668106c
VK
331struct workqueue_struct *system_power_efficient_wq __read_mostly;
332EXPORT_SYMBOL_GPL(system_power_efficient_wq);
333struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;
334EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
d320c038 335
7d19c5ce 336static int worker_thread(void *__worker);
6ba94429 337static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
7d19c5ce 338
97bd2347
TH
339#define CREATE_TRACE_POINTS
340#include <trace/events/workqueue.h>
341
68e13a67 342#define assert_rcu_or_pool_mutex() \
f78f5b90
PM
343 RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
344 !lockdep_is_held(&wq_pool_mutex), \
345 "sched RCU or wq_pool_mutex should be held")
5bcab335 346
b09f4fd3 347#define assert_rcu_or_wq_mutex(wq) \
f78f5b90
PM
348 RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
349 !lockdep_is_held(&wq->mutex), \
350 "sched RCU or wq->mutex should be held")
76af4d93 351
5b95e1af 352#define assert_rcu_or_wq_mutex_or_pool_mutex(wq) \
f78f5b90
PM
353 RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
354 !lockdep_is_held(&wq->mutex) && \
355 !lockdep_is_held(&wq_pool_mutex), \
356 "sched RCU, wq->mutex or wq_pool_mutex should be held")
5b95e1af 357
f02ae73a
TH
358#define for_each_cpu_worker_pool(pool, cpu) \
359 for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
360 (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
7a62c2c8 361 (pool)++)
4ce62e9e 362
17116969
TH
363/**
364 * for_each_pool - iterate through all worker_pools in the system
365 * @pool: iteration cursor
611c92a0 366 * @pi: integer used for iteration
fa1b54e6 367 *
68e13a67
LJ
368 * This must be called either with wq_pool_mutex held or sched RCU read
369 * locked. If the pool needs to be used beyond the locking in effect, the
370 * caller is responsible for guaranteeing that the pool stays online.
fa1b54e6
TH
371 *
372 * The if/else clause exists only for the lockdep assertion and can be
373 * ignored.
17116969 374 */
611c92a0
TH
375#define for_each_pool(pool, pi) \
376 idr_for_each_entry(&worker_pool_idr, pool, pi) \
68e13a67 377 if (({ assert_rcu_or_pool_mutex(); false; })) { } \
fa1b54e6 378 else
17116969 379
822d8405
TH
380/**
381 * for_each_pool_worker - iterate through all workers of a worker_pool
382 * @worker: iteration cursor
822d8405
TH
383 * @pool: worker_pool to iterate workers of
384 *
92f9c5c4 385 * This must be called with @pool->attach_mutex.
822d8405
TH
386 *
387 * The if/else clause exists only for the lockdep assertion and can be
388 * ignored.
389 */
da028469
LJ
390#define for_each_pool_worker(worker, pool) \
391 list_for_each_entry((worker), &(pool)->workers, node) \
92f9c5c4 392 if (({ lockdep_assert_held(&pool->attach_mutex); false; })) { } \
822d8405
TH
393 else
394
49e3cf44
TH
395/**
396 * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
397 * @pwq: iteration cursor
398 * @wq: the target workqueue
76af4d93 399 *
b09f4fd3 400 * This must be called either with wq->mutex held or sched RCU read locked.
794b18bc
TH
401 * If the pwq needs to be used beyond the locking in effect, the caller is
402 * responsible for guaranteeing that the pwq stays online.
76af4d93
TH
403 *
404 * The if/else clause exists only for the lockdep assertion and can be
405 * ignored.
49e3cf44
TH
406 */
407#define for_each_pwq(pwq, wq) \
76af4d93 408 list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node) \
b09f4fd3 409 if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \
76af4d93 410 else
f3421797 411
dc186ad7
TG
412#ifdef CONFIG_DEBUG_OBJECTS_WORK
413
414static struct debug_obj_descr work_debug_descr;
415
99777288
SG
416static void *work_debug_hint(void *addr)
417{
418 return ((struct work_struct *) addr)->func;
419}
420
dc186ad7
TG
421/*
422 * fixup_init is called when:
423 * - an active object is initialized
424 */
425static int work_fixup_init(void *addr, enum debug_obj_state state)
426{
427 struct work_struct *work = addr;
428
429 switch (state) {
430 case ODEBUG_STATE_ACTIVE:
431 cancel_work_sync(work);
432 debug_object_init(work, &work_debug_descr);
433 return 1;
434 default:
435 return 0;
436 }
437}
438
439/*
440 * fixup_activate is called when:
441 * - an active object is activated
442 * - an unknown object is activated (might be a statically initialized object)
443 */
444static int work_fixup_activate(void *addr, enum debug_obj_state state)
445{
446 struct work_struct *work = addr;
447
448 switch (state) {
449
450 case ODEBUG_STATE_NOTAVAILABLE:
451 /*
452 * This is not really a fixup. The work struct was
453 * statically initialized. We just make sure that it
454 * is tracked in the object tracker.
455 */
22df02bb 456 if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) {
dc186ad7
TG
457 debug_object_init(work, &work_debug_descr);
458 debug_object_activate(work, &work_debug_descr);
459 return 0;
460 }
461 WARN_ON_ONCE(1);
462 return 0;
463
464 case ODEBUG_STATE_ACTIVE:
465 WARN_ON(1);
466
467 default:
468 return 0;
469 }
470}
471
472/*
473 * fixup_free is called when:
474 * - an active object is freed
475 */
476static int work_fixup_free(void *addr, enum debug_obj_state state)
477{
478 struct work_struct *work = addr;
479
480 switch (state) {
481 case ODEBUG_STATE_ACTIVE:
482 cancel_work_sync(work);
483 debug_object_free(work, &work_debug_descr);
484 return 1;
485 default:
486 return 0;
487 }
488}
489
490static struct debug_obj_descr work_debug_descr = {
491 .name = "work_struct",
99777288 492 .debug_hint = work_debug_hint,
dc186ad7
TG
493 .fixup_init = work_fixup_init,
494 .fixup_activate = work_fixup_activate,
495 .fixup_free = work_fixup_free,
496};
497
498static inline void debug_work_activate(struct work_struct *work)
499{
500 debug_object_activate(work, &work_debug_descr);
501}
502
503static inline void debug_work_deactivate(struct work_struct *work)
504{
505 debug_object_deactivate(work, &work_debug_descr);
506}
507
508void __init_work(struct work_struct *work, int onstack)
509{
510 if (onstack)
511 debug_object_init_on_stack(work, &work_debug_descr);
512 else
513 debug_object_init(work, &work_debug_descr);
514}
515EXPORT_SYMBOL_GPL(__init_work);
516
517void destroy_work_on_stack(struct work_struct *work)
518{
519 debug_object_free(work, &work_debug_descr);
520}
521EXPORT_SYMBOL_GPL(destroy_work_on_stack);
522
ea2e64f2
TG
523void destroy_delayed_work_on_stack(struct delayed_work *work)
524{
525 destroy_timer_on_stack(&work->timer);
526 debug_object_free(&work->work, &work_debug_descr);
527}
528EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack);
529
dc186ad7
TG
530#else
531static inline void debug_work_activate(struct work_struct *work) { }
532static inline void debug_work_deactivate(struct work_struct *work) { }
533#endif
534
4e8b22bd
LB
535/**
536 * worker_pool_assign_id - allocate ID and assing it to @pool
537 * @pool: the pool pointer of interest
538 *
539 * Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned
540 * successfully, -errno on failure.
541 */
9daf9e67
TH
542static int worker_pool_assign_id(struct worker_pool *pool)
543{
544 int ret;
545
68e13a67 546 lockdep_assert_held(&wq_pool_mutex);
5bcab335 547
4e8b22bd
LB
548 ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,
549 GFP_KERNEL);
229641a6 550 if (ret >= 0) {
e68035fb 551 pool->id = ret;
229641a6
TH
552 return 0;
553 }
fa1b54e6 554 return ret;
7c3eed5c
TH
555}
556
df2d5ae4
TH
557/**
558 * unbound_pwq_by_node - return the unbound pool_workqueue for the given node
559 * @wq: the target workqueue
560 * @node: the node ID
561 *
5b95e1af
LJ
562 * This must be called with any of wq_pool_mutex, wq->mutex or sched RCU
563 * read locked.
df2d5ae4
TH
564 * If the pwq needs to be used beyond the locking in effect, the caller is
565 * responsible for guaranteeing that the pwq stays online.
d185af30
YB
566 *
567 * Return: The unbound pool_workqueue for @node.
df2d5ae4
TH
568 */
569static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,
570 int node)
571{
5b95e1af 572 assert_rcu_or_wq_mutex_or_pool_mutex(wq);
df2d5ae4
TH
573 return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
574}
575
73f53c4a
TH
576static unsigned int work_color_to_flags(int color)
577{
578 return color << WORK_STRUCT_COLOR_SHIFT;
579}
580
581static int get_work_color(struct work_struct *work)
582{
583 return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
584 ((1 << WORK_STRUCT_COLOR_BITS) - 1);
585}
586
587static int work_next_color(int color)
588{
589 return (color + 1) % WORK_NR_COLORS;
590}
1da177e4 591
14441960 592/*
112202d9
TH
593 * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
594 * contain the pointer to the queued pwq. Once execution starts, the flag
7c3eed5c 595 * is cleared and the high bits contain OFFQ flags and pool ID.
7a22ad75 596 *
112202d9
TH
597 * set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
598 * and clear_work_data() can be used to set the pwq, pool or clear
bbb68dfa
TH
599 * work->data. These functions should only be called while the work is
600 * owned - ie. while the PENDING bit is set.
7a22ad75 601 *
112202d9 602 * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
7c3eed5c 603 * corresponding to a work. Pool is available once the work has been
112202d9 604 * queued anywhere after initialization until it is sync canceled. pwq is
7c3eed5c 605 * available only while the work item is queued.
7a22ad75 606 *
bbb68dfa
TH
607 * %WORK_OFFQ_CANCELING is used to mark a work item which is being
608 * canceled. While being canceled, a work item may have its PENDING set
609 * but stay off timer and worklist for arbitrarily long and nobody should
610 * try to steal the PENDING bit.
14441960 611 */
7a22ad75
TH
612static inline void set_work_data(struct work_struct *work, unsigned long data,
613 unsigned long flags)
365970a1 614{
6183c009 615 WARN_ON_ONCE(!work_pending(work));
7a22ad75
TH
616 atomic_long_set(&work->data, data | flags | work_static(work));
617}
365970a1 618
112202d9 619static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
7a22ad75
TH
620 unsigned long extra_flags)
621{
112202d9
TH
622 set_work_data(work, (unsigned long)pwq,
623 WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags);
365970a1
DH
624}
625
4468a00f
LJ
626static void set_work_pool_and_keep_pending(struct work_struct *work,
627 int pool_id)
628{
629 set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
630 WORK_STRUCT_PENDING);
631}
632
7c3eed5c
TH
633static void set_work_pool_and_clear_pending(struct work_struct *work,
634 int pool_id)
7a22ad75 635{
23657bb1
TH
636 /*
637 * The following wmb is paired with the implied mb in
638 * test_and_set_bit(PENDING) and ensures all updates to @work made
639 * here are visible to and precede any updates by the next PENDING
640 * owner.
641 */
642 smp_wmb();
7c3eed5c 643 set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
7a22ad75 644}
f756d5e2 645
7a22ad75 646static void clear_work_data(struct work_struct *work)
1da177e4 647{
7c3eed5c
TH
648 smp_wmb(); /* see set_work_pool_and_clear_pending() */
649 set_work_data(work, WORK_STRUCT_NO_POOL, 0);
1da177e4
LT
650}
651
112202d9 652static struct pool_workqueue *get_work_pwq(struct work_struct *work)
b1f4ec17 653{
e120153d 654 unsigned long data = atomic_long_read(&work->data);
7a22ad75 655
112202d9 656 if (data & WORK_STRUCT_PWQ)
e120153d
TH
657 return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
658 else
659 return NULL;
4d707b9f
ON
660}
661
7c3eed5c
TH
662/**
663 * get_work_pool - return the worker_pool a given work was associated with
664 * @work: the work item of interest
665 *
68e13a67
LJ
666 * Pools are created and destroyed under wq_pool_mutex, and allows read
667 * access under sched-RCU read lock. As such, this function should be
668 * called under wq_pool_mutex or with preemption disabled.
fa1b54e6
TH
669 *
670 * All fields of the returned pool are accessible as long as the above
671 * mentioned locking is in effect. If the returned pool needs to be used
672 * beyond the critical section, the caller is responsible for ensuring the
673 * returned pool is and stays online.
d185af30
YB
674 *
675 * Return: The worker_pool @work was last associated with. %NULL if none.
7c3eed5c
TH
676 */
677static struct worker_pool *get_work_pool(struct work_struct *work)
365970a1 678{
e120153d 679 unsigned long data = atomic_long_read(&work->data);
7c3eed5c 680 int pool_id;
7a22ad75 681
68e13a67 682 assert_rcu_or_pool_mutex();
fa1b54e6 683
112202d9
TH
684 if (data & WORK_STRUCT_PWQ)
685 return ((struct pool_workqueue *)
7c3eed5c 686 (data & WORK_STRUCT_WQ_DATA_MASK))->pool;
7a22ad75 687
7c3eed5c
TH
688 pool_id = data >> WORK_OFFQ_POOL_SHIFT;
689 if (pool_id == WORK_OFFQ_POOL_NONE)
7a22ad75
TH
690 return NULL;
691
fa1b54e6 692 return idr_find(&worker_pool_idr, pool_id);
7c3eed5c
TH
693}
694
695/**
696 * get_work_pool_id - return the worker pool ID a given work is associated with
697 * @work: the work item of interest
698 *
d185af30 699 * Return: The worker_pool ID @work was last associated with.
7c3eed5c
TH
700 * %WORK_OFFQ_POOL_NONE if none.
701 */
702static int get_work_pool_id(struct work_struct *work)
703{
54d5b7d0
LJ
704 unsigned long data = atomic_long_read(&work->data);
705
112202d9
TH
706 if (data & WORK_STRUCT_PWQ)
707 return ((struct pool_workqueue *)
54d5b7d0 708 (data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
7c3eed5c 709
54d5b7d0 710 return data >> WORK_OFFQ_POOL_SHIFT;
7c3eed5c
TH
711}
712
bbb68dfa
TH
713static void mark_work_canceling(struct work_struct *work)
714{
7c3eed5c 715 unsigned long pool_id = get_work_pool_id(work);
bbb68dfa 716
7c3eed5c
TH
717 pool_id <<= WORK_OFFQ_POOL_SHIFT;
718 set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
bbb68dfa
TH
719}
720
721static bool work_is_canceling(struct work_struct *work)
722{
723 unsigned long data = atomic_long_read(&work->data);
724
112202d9 725 return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
bbb68dfa
TH
726}
727
e22bee78 728/*
3270476a
TH
729 * Policy functions. These define the policies on how the global worker
730 * pools are managed. Unless noted otherwise, these functions assume that
d565ed63 731 * they're being called with pool->lock held.
e22bee78
TH
732 */
733
63d95a91 734static bool __need_more_worker(struct worker_pool *pool)
a848e3b6 735{
e19e397a 736 return !atomic_read(&pool->nr_running);
a848e3b6
ON
737}
738
4594bf15 739/*
e22bee78
TH
740 * Need to wake up a worker? Called from anything but currently
741 * running workers.
974271c4
TH
742 *
743 * Note that, because unbound workers never contribute to nr_running, this
706026c2 744 * function will always return %true for unbound pools as long as the
974271c4 745 * worklist isn't empty.
4594bf15 746 */
63d95a91 747static bool need_more_worker(struct worker_pool *pool)
365970a1 748{
63d95a91 749 return !list_empty(&pool->worklist) && __need_more_worker(pool);
e22bee78 750}
4594bf15 751
e22bee78 752/* Can I start working? Called from busy but !running workers. */
63d95a91 753static bool may_start_working(struct worker_pool *pool)
e22bee78 754{
63d95a91 755 return pool->nr_idle;
e22bee78
TH
756}
757
758/* Do I need to keep working? Called from currently running workers. */
63d95a91 759static bool keep_working(struct worker_pool *pool)
e22bee78 760{
e19e397a
TH
761 return !list_empty(&pool->worklist) &&
762 atomic_read(&pool->nr_running) <= 1;
e22bee78
TH
763}
764
765/* Do we need a new worker? Called from manager. */
63d95a91 766static bool need_to_create_worker(struct worker_pool *pool)
e22bee78 767{
63d95a91 768 return need_more_worker(pool) && !may_start_working(pool);
e22bee78 769}
365970a1 770
e22bee78 771/* Do we have too many workers and should some go away? */
63d95a91 772static bool too_many_workers(struct worker_pool *pool)
e22bee78 773{
34a06bd6 774 bool managing = mutex_is_locked(&pool->manager_arb);
63d95a91
TH
775 int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
776 int nr_busy = pool->nr_workers - nr_idle;
e22bee78
TH
777
778 return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
365970a1
DH
779}
780
4d707b9f 781/*
e22bee78
TH
782 * Wake up functions.
783 */
784
1037de36
LJ
785/* Return the first idle worker. Safe with preemption disabled */
786static struct worker *first_idle_worker(struct worker_pool *pool)
7e11629d 787{
63d95a91 788 if (unlikely(list_empty(&pool->idle_list)))
7e11629d
TH
789 return NULL;
790
63d95a91 791 return list_first_entry(&pool->idle_list, struct worker, entry);
7e11629d
TH
792}
793
794/**
795 * wake_up_worker - wake up an idle worker
63d95a91 796 * @pool: worker pool to wake worker from
7e11629d 797 *
63d95a91 798 * Wake up the first idle worker of @pool.
7e11629d
TH
799 *
800 * CONTEXT:
d565ed63 801 * spin_lock_irq(pool->lock).
7e11629d 802 */
63d95a91 803static void wake_up_worker(struct worker_pool *pool)
7e11629d 804{
1037de36 805 struct worker *worker = first_idle_worker(pool);
7e11629d
TH
806
807 if (likely(worker))
808 wake_up_process(worker->task);
809}
810
d302f017 811/**
e22bee78
TH
812 * wq_worker_waking_up - a worker is waking up
813 * @task: task waking up
814 * @cpu: CPU @task is waking up to
815 *
816 * This function is called during try_to_wake_up() when a worker is
817 * being awoken.
818 *
819 * CONTEXT:
820 * spin_lock_irq(rq->lock)
821 */
d84ff051 822void wq_worker_waking_up(struct task_struct *task, int cpu)
e22bee78
TH
823{
824 struct worker *worker = kthread_data(task);
825
36576000 826 if (!(worker->flags & WORKER_NOT_RUNNING)) {
ec22ca5e 827 WARN_ON_ONCE(worker->pool->cpu != cpu);
e19e397a 828 atomic_inc(&worker->pool->nr_running);
36576000 829 }
e22bee78
TH
830}
831
832/**
833 * wq_worker_sleeping - a worker is going to sleep
834 * @task: task going to sleep
835 * @cpu: CPU in question, must be the current CPU number
836 *
837 * This function is called during schedule() when a busy worker is
838 * going to sleep. Worker on the same cpu can be woken up by
839 * returning pointer to its task.
840 *
841 * CONTEXT:
842 * spin_lock_irq(rq->lock)
843 *
d185af30 844 * Return:
e22bee78
TH
845 * Worker task on @cpu to wake up, %NULL if none.
846 */
d84ff051 847struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu)
e22bee78
TH
848{
849 struct worker *worker = kthread_data(task), *to_wakeup = NULL;
111c225a 850 struct worker_pool *pool;
e22bee78 851
111c225a
TH
852 /*
853 * Rescuers, which may not have all the fields set up like normal
854 * workers, also reach here, let's not access anything before
855 * checking NOT_RUNNING.
856 */
2d64672e 857 if (worker->flags & WORKER_NOT_RUNNING)
e22bee78
TH
858 return NULL;
859
111c225a 860 pool = worker->pool;
111c225a 861
e22bee78 862 /* this can only happen on the local cpu */
92b69f50 863 if (WARN_ON_ONCE(cpu != raw_smp_processor_id() || pool->cpu != cpu))
6183c009 864 return NULL;
e22bee78
TH
865
866 /*
867 * The counterpart of the following dec_and_test, implied mb,
868 * worklist not empty test sequence is in insert_work().
869 * Please read comment there.
870 *
628c78e7
TH
871 * NOT_RUNNING is clear. This means that we're bound to and
872 * running on the local cpu w/ rq lock held and preemption
873 * disabled, which in turn means that none else could be
d565ed63 874 * manipulating idle_list, so dereferencing idle_list without pool
628c78e7 875 * lock is safe.
e22bee78 876 */
e19e397a
TH
877 if (atomic_dec_and_test(&pool->nr_running) &&
878 !list_empty(&pool->worklist))
1037de36 879 to_wakeup = first_idle_worker(pool);
e22bee78
TH
880 return to_wakeup ? to_wakeup->task : NULL;
881}
882
883/**
884 * worker_set_flags - set worker flags and adjust nr_running accordingly
cb444766 885 * @worker: self
d302f017 886 * @flags: flags to set
d302f017 887 *
228f1d00 888 * Set @flags in @worker->flags and adjust nr_running accordingly.
d302f017 889 *
cb444766 890 * CONTEXT:
d565ed63 891 * spin_lock_irq(pool->lock)
d302f017 892 */
228f1d00 893static inline void worker_set_flags(struct worker *worker, unsigned int flags)
d302f017 894{
bd7bdd43 895 struct worker_pool *pool = worker->pool;
e22bee78 896
cb444766
TH
897 WARN_ON_ONCE(worker->task != current);
898
228f1d00 899 /* If transitioning into NOT_RUNNING, adjust nr_running. */
e22bee78
TH
900 if ((flags & WORKER_NOT_RUNNING) &&
901 !(worker->flags & WORKER_NOT_RUNNING)) {
228f1d00 902 atomic_dec(&pool->nr_running);
e22bee78
TH
903 }
904
d302f017
TH
905 worker->flags |= flags;
906}
907
908/**
e22bee78 909 * worker_clr_flags - clear worker flags and adjust nr_running accordingly
cb444766 910 * @worker: self
d302f017
TH
911 * @flags: flags to clear
912 *
e22bee78 913 * Clear @flags in @worker->flags and adjust nr_running accordingly.
d302f017 914 *
cb444766 915 * CONTEXT:
d565ed63 916 * spin_lock_irq(pool->lock)
d302f017
TH
917 */
918static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
919{
63d95a91 920 struct worker_pool *pool = worker->pool;
e22bee78
TH
921 unsigned int oflags = worker->flags;
922
cb444766
TH
923 WARN_ON_ONCE(worker->task != current);
924
d302f017 925 worker->flags &= ~flags;
e22bee78 926
42c025f3
TH
927 /*
928 * If transitioning out of NOT_RUNNING, increment nr_running. Note
929 * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
930 * of multiple flags, not a single flag.
931 */
e22bee78
TH
932 if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
933 if (!(worker->flags & WORKER_NOT_RUNNING))
e19e397a 934 atomic_inc(&pool->nr_running);
d302f017
TH
935}
936
8cca0eea
TH
937/**
938 * find_worker_executing_work - find worker which is executing a work
c9e7cf27 939 * @pool: pool of interest
8cca0eea
TH
940 * @work: work to find worker for
941 *
c9e7cf27
TH
942 * Find a worker which is executing @work on @pool by searching
943 * @pool->busy_hash which is keyed by the address of @work. For a worker
a2c1c57b
TH
944 * to match, its current execution should match the address of @work and
945 * its work function. This is to avoid unwanted dependency between
946 * unrelated work executions through a work item being recycled while still
947 * being executed.
948 *
949 * This is a bit tricky. A work item may be freed once its execution
950 * starts and nothing prevents the freed area from being recycled for
951 * another work item. If the same work item address ends up being reused
952 * before the original execution finishes, workqueue will identify the
953 * recycled work item as currently executing and make it wait until the
954 * current execution finishes, introducing an unwanted dependency.
955 *
c5aa87bb
TH
956 * This function checks the work item address and work function to avoid
957 * false positives. Note that this isn't complete as one may construct a
958 * work function which can introduce dependency onto itself through a
959 * recycled work item. Well, if somebody wants to shoot oneself in the
960 * foot that badly, there's only so much we can do, and if such deadlock
961 * actually occurs, it should be easy to locate the culprit work function.
8cca0eea
TH
962 *
963 * CONTEXT:
d565ed63 964 * spin_lock_irq(pool->lock).
8cca0eea 965 *
d185af30
YB
966 * Return:
967 * Pointer to worker which is executing @work if found, %NULL
8cca0eea 968 * otherwise.
4d707b9f 969 */
c9e7cf27 970static struct worker *find_worker_executing_work(struct worker_pool *pool,
8cca0eea 971 struct work_struct *work)
4d707b9f 972{
42f8570f 973 struct worker *worker;
42f8570f 974
b67bfe0d 975 hash_for_each_possible(pool->busy_hash, worker, hentry,
a2c1c57b
TH
976 (unsigned long)work)
977 if (worker->current_work == work &&
978 worker->current_func == work->func)
42f8570f
SL
979 return worker;
980
981 return NULL;
4d707b9f
ON
982}
983
bf4ede01
TH
984/**
985 * move_linked_works - move linked works to a list
986 * @work: start of series of works to be scheduled
987 * @head: target list to append @work to
402dd89d 988 * @nextp: out parameter for nested worklist walking
bf4ede01
TH
989 *
990 * Schedule linked works starting from @work to @head. Work series to
991 * be scheduled starts at @work and includes any consecutive work with
992 * WORK_STRUCT_LINKED set in its predecessor.
993 *
994 * If @nextp is not NULL, it's updated to point to the next work of
995 * the last scheduled work. This allows move_linked_works() to be
996 * nested inside outer list_for_each_entry_safe().
997 *
998 * CONTEXT:
d565ed63 999 * spin_lock_irq(pool->lock).
bf4ede01
TH
1000 */
1001static void move_linked_works(struct work_struct *work, struct list_head *head,
1002 struct work_struct **nextp)
1003{
1004 struct work_struct *n;
1005
1006 /*
1007 * Linked worklist will always end before the end of the list,
1008 * use NULL for list head.
1009 */
1010 list_for_each_entry_safe_from(work, n, NULL, entry) {
1011 list_move_tail(&work->entry, head);
1012 if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
1013 break;
1014 }
1015
1016 /*
1017 * If we're already inside safe list traversal and have moved
1018 * multiple works to the scheduled queue, the next position
1019 * needs to be updated.
1020 */
1021 if (nextp)
1022 *nextp = n;
1023}
1024
8864b4e5
TH
1025/**
1026 * get_pwq - get an extra reference on the specified pool_workqueue
1027 * @pwq: pool_workqueue to get
1028 *
1029 * Obtain an extra reference on @pwq. The caller should guarantee that
1030 * @pwq has positive refcnt and be holding the matching pool->lock.
1031 */
1032static void get_pwq(struct pool_workqueue *pwq)
1033{
1034 lockdep_assert_held(&pwq->pool->lock);
1035 WARN_ON_ONCE(pwq->refcnt <= 0);
1036 pwq->refcnt++;
1037}
1038
1039/**
1040 * put_pwq - put a pool_workqueue reference
1041 * @pwq: pool_workqueue to put
1042 *
1043 * Drop a reference of @pwq. If its refcnt reaches zero, schedule its
1044 * destruction. The caller should be holding the matching pool->lock.
1045 */
1046static void put_pwq(struct pool_workqueue *pwq)
1047{
1048 lockdep_assert_held(&pwq->pool->lock);
1049 if (likely(--pwq->refcnt))
1050 return;
1051 if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
1052 return;
1053 /*
1054 * @pwq can't be released under pool->lock, bounce to
1055 * pwq_unbound_release_workfn(). This never recurses on the same
1056 * pool->lock as this path is taken only for unbound workqueues and
1057 * the release work item is scheduled on a per-cpu workqueue. To
1058 * avoid lockdep warning, unbound pool->locks are given lockdep
1059 * subclass of 1 in get_unbound_pool().
1060 */
1061 schedule_work(&pwq->unbound_release_work);
1062}
1063
dce90d47
TH
1064/**
1065 * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
1066 * @pwq: pool_workqueue to put (can be %NULL)
1067 *
1068 * put_pwq() with locking. This function also allows %NULL @pwq.
1069 */
1070static void put_pwq_unlocked(struct pool_workqueue *pwq)
1071{
1072 if (pwq) {
1073 /*
1074 * As both pwqs and pools are sched-RCU protected, the
1075 * following lock operations are safe.
1076 */
1077 spin_lock_irq(&pwq->pool->lock);
1078 put_pwq(pwq);
1079 spin_unlock_irq(&pwq->pool->lock);
1080 }
1081}
1082
112202d9 1083static void pwq_activate_delayed_work(struct work_struct *work)
bf4ede01 1084{
112202d9 1085 struct pool_workqueue *pwq = get_work_pwq(work);
bf4ede01
TH
1086
1087 trace_workqueue_activate_work(work);
82607adc
TH
1088 if (list_empty(&pwq->pool->worklist))
1089 pwq->pool->watchdog_ts = jiffies;
112202d9 1090 move_linked_works(work, &pwq->pool->worklist, NULL);
bf4ede01 1091 __clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
112202d9 1092 pwq->nr_active++;
bf4ede01
TH
1093}
1094
112202d9 1095static void pwq_activate_first_delayed(struct pool_workqueue *pwq)
3aa62497 1096{
112202d9 1097 struct work_struct *work = list_first_entry(&pwq->delayed_works,
3aa62497
LJ
1098 struct work_struct, entry);
1099
112202d9 1100 pwq_activate_delayed_work(work);
3aa62497
LJ
1101}
1102
bf4ede01 1103/**
112202d9
TH
1104 * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
1105 * @pwq: pwq of interest
bf4ede01 1106 * @color: color of work which left the queue
bf4ede01
TH
1107 *
1108 * A work either has completed or is removed from pending queue,
112202d9 1109 * decrement nr_in_flight of its pwq and handle workqueue flushing.
bf4ede01
TH
1110 *
1111 * CONTEXT:
d565ed63 1112 * spin_lock_irq(pool->lock).
bf4ede01 1113 */
112202d9 1114static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
bf4ede01 1115{
8864b4e5 1116 /* uncolored work items don't participate in flushing or nr_active */
bf4ede01 1117 if (color == WORK_NO_COLOR)
8864b4e5 1118 goto out_put;
bf4ede01 1119
112202d9 1120 pwq->nr_in_flight[color]--;
bf4ede01 1121
112202d9
TH
1122 pwq->nr_active--;
1123 if (!list_empty(&pwq->delayed_works)) {
b3f9f405 1124 /* one down, submit a delayed one */
112202d9
TH
1125 if (pwq->nr_active < pwq->max_active)
1126 pwq_activate_first_delayed(pwq);
bf4ede01
TH
1127 }
1128
1129 /* is flush in progress and are we at the flushing tip? */
112202d9 1130 if (likely(pwq->flush_color != color))
8864b4e5 1131 goto out_put;
bf4ede01
TH
1132
1133 /* are there still in-flight works? */
112202d9 1134 if (pwq->nr_in_flight[color])
8864b4e5 1135 goto out_put;
bf4ede01 1136
112202d9
TH
1137 /* this pwq is done, clear flush_color */
1138 pwq->flush_color = -1;
bf4ede01
TH
1139
1140 /*
112202d9 1141 * If this was the last pwq, wake up the first flusher. It
bf4ede01
TH
1142 * will handle the rest.
1143 */
112202d9
TH
1144 if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
1145 complete(&pwq->wq->first_flusher->done);
8864b4e5
TH
1146out_put:
1147 put_pwq(pwq);
bf4ede01
TH
1148}
1149
36e227d2 1150/**
bbb68dfa 1151 * try_to_grab_pending - steal work item from worklist and disable irq
36e227d2
TH
1152 * @work: work item to steal
1153 * @is_dwork: @work is a delayed_work
bbb68dfa 1154 * @flags: place to store irq state
36e227d2
TH
1155 *
1156 * Try to grab PENDING bit of @work. This function can handle @work in any
d185af30 1157 * stable state - idle, on timer or on worklist.
36e227d2 1158 *
d185af30 1159 * Return:
36e227d2
TH
1160 * 1 if @work was pending and we successfully stole PENDING
1161 * 0 if @work was idle and we claimed PENDING
1162 * -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
bbb68dfa
TH
1163 * -ENOENT if someone else is canceling @work, this state may persist
1164 * for arbitrarily long
36e227d2 1165 *
d185af30 1166 * Note:
bbb68dfa 1167 * On >= 0 return, the caller owns @work's PENDING bit. To avoid getting
e0aecdd8
TH
1168 * interrupted while holding PENDING and @work off queue, irq must be
1169 * disabled on entry. This, combined with delayed_work->timer being
1170 * irqsafe, ensures that we return -EAGAIN for finite short period of time.
bbb68dfa
TH
1171 *
1172 * On successful return, >= 0, irq is disabled and the caller is
1173 * responsible for releasing it using local_irq_restore(*@flags).
1174 *
e0aecdd8 1175 * This function is safe to call from any context including IRQ handler.
bf4ede01 1176 */
bbb68dfa
TH
1177static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
1178 unsigned long *flags)
bf4ede01 1179{
d565ed63 1180 struct worker_pool *pool;
112202d9 1181 struct pool_workqueue *pwq;
bf4ede01 1182
bbb68dfa
TH
1183 local_irq_save(*flags);
1184
36e227d2
TH
1185 /* try to steal the timer if it exists */
1186 if (is_dwork) {
1187 struct delayed_work *dwork = to_delayed_work(work);
1188
e0aecdd8
TH
1189 /*
1190 * dwork->timer is irqsafe. If del_timer() fails, it's
1191 * guaranteed that the timer is not queued anywhere and not
1192 * running on the local CPU.
1193 */
36e227d2
TH
1194 if (likely(del_timer(&dwork->timer)))
1195 return 1;
1196 }
1197
1198 /* try to claim PENDING the normal way */
bf4ede01
TH
1199 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
1200 return 0;
1201
1202 /*
1203 * The queueing is in progress, or it is already queued. Try to
1204 * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
1205 */
d565ed63
TH
1206 pool = get_work_pool(work);
1207 if (!pool)
bbb68dfa 1208 goto fail;
bf4ede01 1209
d565ed63 1210 spin_lock(&pool->lock);
0b3dae68 1211 /*
112202d9
TH
1212 * work->data is guaranteed to point to pwq only while the work
1213 * item is queued on pwq->wq, and both updating work->data to point
1214 * to pwq on queueing and to pool on dequeueing are done under
1215 * pwq->pool->lock. This in turn guarantees that, if work->data
1216 * points to pwq which is associated with a locked pool, the work
0b3dae68
LJ
1217 * item is currently queued on that pool.
1218 */
112202d9
TH
1219 pwq = get_work_pwq(work);
1220 if (pwq && pwq->pool == pool) {
16062836
TH
1221 debug_work_deactivate(work);
1222
1223 /*
1224 * A delayed work item cannot be grabbed directly because
1225 * it might have linked NO_COLOR work items which, if left
112202d9 1226 * on the delayed_list, will confuse pwq->nr_active
16062836
TH
1227 * management later on and cause stall. Make sure the work
1228 * item is activated before grabbing.
1229 */
1230 if (*work_data_bits(work) & WORK_STRUCT_DELAYED)
112202d9 1231 pwq_activate_delayed_work(work);
16062836
TH
1232
1233 list_del_init(&work->entry);
9c34a704 1234 pwq_dec_nr_in_flight(pwq, get_work_color(work));
16062836 1235
112202d9 1236 /* work->data points to pwq iff queued, point to pool */
16062836
TH
1237 set_work_pool_and_keep_pending(work, pool->id);
1238
1239 spin_unlock(&pool->lock);
1240 return 1;
bf4ede01 1241 }
d565ed63 1242 spin_unlock(&pool->lock);
bbb68dfa
TH
1243fail:
1244 local_irq_restore(*flags);
1245 if (work_is_canceling(work))
1246 return -ENOENT;
1247 cpu_relax();
36e227d2 1248 return -EAGAIN;
bf4ede01
TH
1249}
1250
4690c4ab 1251/**
706026c2 1252 * insert_work - insert a work into a pool
112202d9 1253 * @pwq: pwq @work belongs to
4690c4ab
TH
1254 * @work: work to insert
1255 * @head: insertion point
1256 * @extra_flags: extra WORK_STRUCT_* flags to set
1257 *
112202d9 1258 * Insert @work which belongs to @pwq after @head. @extra_flags is or'd to
706026c2 1259 * work_struct flags.
4690c4ab
TH
1260 *
1261 * CONTEXT:
d565ed63 1262 * spin_lock_irq(pool->lock).
4690c4ab 1263 */
112202d9
TH
1264static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
1265 struct list_head *head, unsigned int extra_flags)
b89deed3 1266{
112202d9 1267 struct worker_pool *pool = pwq->pool;
e22bee78 1268
4690c4ab 1269 /* we own @work, set data and link */
112202d9 1270 set_work_pwq(work, pwq, extra_flags);
1a4d9b0a 1271 list_add_tail(&work->entry, head);
8864b4e5 1272 get_pwq(pwq);
e22bee78
TH
1273
1274 /*
c5aa87bb
TH
1275 * Ensure either wq_worker_sleeping() sees the above
1276 * list_add_tail() or we see zero nr_running to avoid workers lying
1277 * around lazily while there are works to be processed.
e22bee78
TH
1278 */
1279 smp_mb();
1280
63d95a91
TH
1281 if (__need_more_worker(pool))
1282 wake_up_worker(pool);
b89deed3
ON
1283}
1284
c8efcc25
TH
1285/*
1286 * Test whether @work is being queued from another work executing on the
8d03ecfe 1287 * same workqueue.
c8efcc25
TH
1288 */
1289static bool is_chained_work(struct workqueue_struct *wq)
1290{
8d03ecfe
TH
1291 struct worker *worker;
1292
1293 worker = current_wq_worker();
1294 /*
1295 * Return %true iff I'm a worker execuing a work item on @wq. If
1296 * I'm @worker, it's safe to dereference it without locking.
1297 */
112202d9 1298 return worker && worker->current_pwq->wq == wq;
c8efcc25
TH
1299}
1300
d84ff051 1301static void __queue_work(int cpu, struct workqueue_struct *wq,
1da177e4
LT
1302 struct work_struct *work)
1303{
112202d9 1304 struct pool_workqueue *pwq;
c9178087 1305 struct worker_pool *last_pool;
1e19ffc6 1306 struct list_head *worklist;
8a2e8e5d 1307 unsigned int work_flags;
b75cac93 1308 unsigned int req_cpu = cpu;
8930caba
TH
1309
1310 /*
1311 * While a work item is PENDING && off queue, a task trying to
1312 * steal the PENDING will busy-loop waiting for it to either get
1313 * queued or lose PENDING. Grabbing PENDING and queueing should
1314 * happen with IRQ disabled.
1315 */
1316 WARN_ON_ONCE(!irqs_disabled());
1da177e4 1317
dc186ad7 1318 debug_work_activate(work);
1e19ffc6 1319
9ef28a73 1320 /* if draining, only works from the same workqueue are allowed */
618b01eb 1321 if (unlikely(wq->flags & __WQ_DRAINING) &&
c8efcc25 1322 WARN_ON_ONCE(!is_chained_work(wq)))
e41e704b 1323 return;
9e8cd2f5 1324retry:
df2d5ae4
TH
1325 if (req_cpu == WORK_CPU_UNBOUND)
1326 cpu = raw_smp_processor_id();
1327
c9178087 1328 /* pwq which will be used unless @work is executing elsewhere */
df2d5ae4 1329 if (!(wq->flags & WQ_UNBOUND))
7fb98ea7 1330 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
df2d5ae4
TH
1331 else
1332 pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
dbf2576e 1333
c9178087
TH
1334 /*
1335 * If @work was previously on a different pool, it might still be
1336 * running there, in which case the work needs to be queued on that
1337 * pool to guarantee non-reentrancy.
1338 */
1339 last_pool = get_work_pool(work);
1340 if (last_pool && last_pool != pwq->pool) {
1341 struct worker *worker;
18aa9eff 1342
c9178087 1343 spin_lock(&last_pool->lock);
18aa9eff 1344
c9178087 1345 worker = find_worker_executing_work(last_pool, work);
18aa9eff 1346
c9178087
TH
1347 if (worker && worker->current_pwq->wq == wq) {
1348 pwq = worker->current_pwq;
8930caba 1349 } else {
c9178087
TH
1350 /* meh... not running there, queue here */
1351 spin_unlock(&last_pool->lock);
112202d9 1352 spin_lock(&pwq->pool->lock);
8930caba 1353 }
f3421797 1354 } else {
112202d9 1355 spin_lock(&pwq->pool->lock);
502ca9d8
TH
1356 }
1357
9e8cd2f5
TH
1358 /*
1359 * pwq is determined and locked. For unbound pools, we could have
1360 * raced with pwq release and it could already be dead. If its
1361 * refcnt is zero, repeat pwq selection. Note that pwqs never die
df2d5ae4
TH
1362 * without another pwq replacing it in the numa_pwq_tbl or while
1363 * work items are executing on it, so the retrying is guaranteed to
9e8cd2f5
TH
1364 * make forward-progress.
1365 */
1366 if (unlikely(!pwq->refcnt)) {
1367 if (wq->flags & WQ_UNBOUND) {
1368 spin_unlock(&pwq->pool->lock);
1369 cpu_relax();
1370 goto retry;
1371 }
1372 /* oops */
1373 WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
1374 wq->name, cpu);
1375 }
1376
112202d9
TH
1377 /* pwq determined, queue */
1378 trace_workqueue_queue_work(req_cpu, pwq, work);
502ca9d8 1379
f5b2552b 1380 if (WARN_ON(!list_empty(&work->entry))) {
112202d9 1381 spin_unlock(&pwq->pool->lock);
f5b2552b
DC
1382 return;
1383 }
1e19ffc6 1384
112202d9
TH
1385 pwq->nr_in_flight[pwq->work_color]++;
1386 work_flags = work_color_to_flags(pwq->work_color);
1e19ffc6 1387
112202d9 1388 if (likely(pwq->nr_active < pwq->max_active)) {
cdadf009 1389 trace_workqueue_activate_work(work);
112202d9
TH
1390 pwq->nr_active++;
1391 worklist = &pwq->pool->worklist;
82607adc
TH
1392 if (list_empty(worklist))
1393 pwq->pool->watchdog_ts = jiffies;
8a2e8e5d
TH
1394 } else {
1395 work_flags |= WORK_STRUCT_DELAYED;
112202d9 1396 worklist = &pwq->delayed_works;
8a2e8e5d 1397 }
1e19ffc6 1398
112202d9 1399 insert_work(pwq, work, worklist, work_flags);
1e19ffc6 1400
112202d9 1401 spin_unlock(&pwq->pool->lock);
1da177e4
LT
1402}
1403
0fcb78c2 1404/**
c1a220e7
ZR
1405 * queue_work_on - queue work on specific cpu
1406 * @cpu: CPU number to execute work on
0fcb78c2
REB
1407 * @wq: workqueue to use
1408 * @work: work to queue
1409 *
c1a220e7
ZR
1410 * We queue the work to a specific CPU, the caller must ensure it
1411 * can't go away.
d185af30
YB
1412 *
1413 * Return: %false if @work was already on a queue, %true otherwise.
1da177e4 1414 */
d4283e93
TH
1415bool queue_work_on(int cpu, struct workqueue_struct *wq,
1416 struct work_struct *work)
1da177e4 1417{
d4283e93 1418 bool ret = false;
8930caba 1419 unsigned long flags;
ef1ca236 1420
8930caba 1421 local_irq_save(flags);
c1a220e7 1422
22df02bb 1423 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
4690c4ab 1424 __queue_work(cpu, wq, work);
d4283e93 1425 ret = true;
c1a220e7 1426 }
ef1ca236 1427
8930caba 1428 local_irq_restore(flags);
1da177e4
LT
1429 return ret;
1430}
ad7b1f84 1431EXPORT_SYMBOL(queue_work_on);
1da177e4 1432
d8e794df 1433void delayed_work_timer_fn(unsigned long __data)
1da177e4 1434{
52bad64d 1435 struct delayed_work *dwork = (struct delayed_work *)__data;
1da177e4 1436
e0aecdd8 1437 /* should have been called from irqsafe timer with irq already off */
60c057bc 1438 __queue_work(dwork->cpu, dwork->wq, &dwork->work);
1da177e4 1439}
1438ade5 1440EXPORT_SYMBOL(delayed_work_timer_fn);
1da177e4 1441
7beb2edf
TH
1442static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
1443 struct delayed_work *dwork, unsigned long delay)
1da177e4 1444{
7beb2edf
TH
1445 struct timer_list *timer = &dwork->timer;
1446 struct work_struct *work = &dwork->work;
7beb2edf
TH
1447
1448 WARN_ON_ONCE(timer->function != delayed_work_timer_fn ||
1449 timer->data != (unsigned long)dwork);
fc4b514f
TH
1450 WARN_ON_ONCE(timer_pending(timer));
1451 WARN_ON_ONCE(!list_empty(&work->entry));
7beb2edf 1452
8852aac2
TH
1453 /*
1454 * If @delay is 0, queue @dwork->work immediately. This is for
1455 * both optimization and correctness. The earliest @timer can
1456 * expire is on the closest next tick and delayed_work users depend
1457 * on that there's no such delay when @delay is 0.
1458 */
1459 if (!delay) {
1460 __queue_work(cpu, wq, &dwork->work);
1461 return;
1462 }
1463
7beb2edf 1464 timer_stats_timer_set_start_info(&dwork->timer);
1da177e4 1465
60c057bc 1466 dwork->wq = wq;
874bbfe6
SL
1467 /* timer isn't guaranteed to run in this cpu, record earlier */
1468 if (cpu == WORK_CPU_UNBOUND)
1469 cpu = raw_smp_processor_id();
1265057f 1470 dwork->cpu = cpu;
7beb2edf
TH
1471 timer->expires = jiffies + delay;
1472
874bbfe6 1473 add_timer_on(timer, cpu);
1da177e4
LT
1474}
1475
0fcb78c2
REB
1476/**
1477 * queue_delayed_work_on - queue work on specific CPU after delay
1478 * @cpu: CPU number to execute work on
1479 * @wq: workqueue to use
af9997e4 1480 * @dwork: work to queue
0fcb78c2
REB
1481 * @delay: number of jiffies to wait before queueing
1482 *
d185af30 1483 * Return: %false if @work was already on a queue, %true otherwise. If
715f1300
TH
1484 * @delay is zero and @dwork is idle, it will be scheduled for immediate
1485 * execution.
0fcb78c2 1486 */
d4283e93
TH
1487bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1488 struct delayed_work *dwork, unsigned long delay)
7a6bc1cd 1489{
52bad64d 1490 struct work_struct *work = &dwork->work;
d4283e93 1491 bool ret = false;
8930caba 1492 unsigned long flags;
7a6bc1cd 1493
8930caba
TH
1494 /* read the comment in __queue_work() */
1495 local_irq_save(flags);
7a6bc1cd 1496
22df02bb 1497 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
7beb2edf 1498 __queue_delayed_work(cpu, wq, dwork, delay);
d4283e93 1499 ret = true;
7a6bc1cd 1500 }
8a3e77cc 1501
8930caba 1502 local_irq_restore(flags);
7a6bc1cd
VP
1503 return ret;
1504}
ad7b1f84 1505EXPORT_SYMBOL(queue_delayed_work_on);
c7fc77f7 1506
8376fe22
TH
1507/**
1508 * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
1509 * @cpu: CPU number to execute work on
1510 * @wq: workqueue to use
1511 * @dwork: work to queue
1512 * @delay: number of jiffies to wait before queueing
1513 *
1514 * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
1515 * modify @dwork's timer so that it expires after @delay. If @delay is
1516 * zero, @work is guaranteed to be scheduled immediately regardless of its
1517 * current state.
1518 *
d185af30 1519 * Return: %false if @dwork was idle and queued, %true if @dwork was
8376fe22
TH
1520 * pending and its timer was modified.
1521 *
e0aecdd8 1522 * This function is safe to call from any context including IRQ handler.
8376fe22
TH
1523 * See try_to_grab_pending() for details.
1524 */
1525bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
1526 struct delayed_work *dwork, unsigned long delay)
1527{
1528 unsigned long flags;
1529 int ret;
c7fc77f7 1530
8376fe22
TH
1531 do {
1532 ret = try_to_grab_pending(&dwork->work, true, &flags);
1533 } while (unlikely(ret == -EAGAIN));
63bc0362 1534
8376fe22
TH
1535 if (likely(ret >= 0)) {
1536 __queue_delayed_work(cpu, wq, dwork, delay);
1537 local_irq_restore(flags);
7a6bc1cd 1538 }
8376fe22
TH
1539
1540 /* -ENOENT from try_to_grab_pending() becomes %true */
7a6bc1cd
VP
1541 return ret;
1542}
8376fe22
TH
1543EXPORT_SYMBOL_GPL(mod_delayed_work_on);
1544
c8e55f36
TH
1545/**
1546 * worker_enter_idle - enter idle state
1547 * @worker: worker which is entering idle state
1548 *
1549 * @worker is entering idle state. Update stats and idle timer if
1550 * necessary.
1551 *
1552 * LOCKING:
d565ed63 1553 * spin_lock_irq(pool->lock).
c8e55f36
TH
1554 */
1555static void worker_enter_idle(struct worker *worker)
1da177e4 1556{
bd7bdd43 1557 struct worker_pool *pool = worker->pool;
c8e55f36 1558
6183c009
TH
1559 if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
1560 WARN_ON_ONCE(!list_empty(&worker->entry) &&
1561 (worker->hentry.next || worker->hentry.pprev)))
1562 return;
c8e55f36 1563
051e1850 1564 /* can't use worker_set_flags(), also called from create_worker() */
cb444766 1565 worker->flags |= WORKER_IDLE;
bd7bdd43 1566 pool->nr_idle++;
e22bee78 1567 worker->last_active = jiffies;
c8e55f36
TH
1568
1569 /* idle_list is LIFO */
bd7bdd43 1570 list_add(&worker->entry, &pool->idle_list);
db7bccf4 1571
628c78e7
TH
1572 if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1573 mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
cb444766 1574
544ecf31 1575 /*
706026c2 1576 * Sanity check nr_running. Because wq_unbind_fn() releases
d565ed63 1577 * pool->lock between setting %WORKER_UNBOUND and zapping
628c78e7
TH
1578 * nr_running, the warning may trigger spuriously. Check iff
1579 * unbind is not in progress.
544ecf31 1580 */
24647570 1581 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
bd7bdd43 1582 pool->nr_workers == pool->nr_idle &&
e19e397a 1583 atomic_read(&pool->nr_running));
c8e55f36
TH
1584}
1585
1586/**
1587 * worker_leave_idle - leave idle state
1588 * @worker: worker which is leaving idle state
1589 *
1590 * @worker is leaving idle state. Update stats.
1591 *
1592 * LOCKING:
d565ed63 1593 * spin_lock_irq(pool->lock).
c8e55f36
TH
1594 */
1595static void worker_leave_idle(struct worker *worker)
1596{
bd7bdd43 1597 struct worker_pool *pool = worker->pool;
c8e55f36 1598
6183c009
TH
1599 if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
1600 return;
d302f017 1601 worker_clr_flags(worker, WORKER_IDLE);
bd7bdd43 1602 pool->nr_idle--;
c8e55f36
TH
1603 list_del_init(&worker->entry);
1604}
1605
f7537df5 1606static struct worker *alloc_worker(int node)
c34056a3
TH
1607{
1608 struct worker *worker;
1609
f7537df5 1610 worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);
c8e55f36
TH
1611 if (worker) {
1612 INIT_LIST_HEAD(&worker->entry);
affee4b2 1613 INIT_LIST_HEAD(&worker->scheduled);
da028469 1614 INIT_LIST_HEAD(&worker->node);
e22bee78
TH
1615 /* on creation a worker is in !idle && prep state */
1616 worker->flags = WORKER_PREP;
c8e55f36 1617 }
c34056a3
TH
1618 return worker;
1619}
1620
4736cbf7
LJ
1621/**
1622 * worker_attach_to_pool() - attach a worker to a pool
1623 * @worker: worker to be attached
1624 * @pool: the target pool
1625 *
1626 * Attach @worker to @pool. Once attached, the %WORKER_UNBOUND flag and
1627 * cpu-binding of @worker are kept coordinated with the pool across
1628 * cpu-[un]hotplugs.
1629 */
1630static void worker_attach_to_pool(struct worker *worker,
1631 struct worker_pool *pool)
1632{
1633 mutex_lock(&pool->attach_mutex);
1634
1635 /*
1636 * set_cpus_allowed_ptr() will fail if the cpumask doesn't have any
1637 * online CPUs. It'll be re-applied when any of the CPUs come up.
1638 */
1639 set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
1640
1641 /*
1642 * The pool->attach_mutex ensures %POOL_DISASSOCIATED remains
1643 * stable across this function. See the comments above the
1644 * flag definition for details.
1645 */
1646 if (pool->flags & POOL_DISASSOCIATED)
1647 worker->flags |= WORKER_UNBOUND;
1648
1649 list_add_tail(&worker->node, &pool->workers);
1650
1651 mutex_unlock(&pool->attach_mutex);
1652}
1653
60f5a4bc
LJ
1654/**
1655 * worker_detach_from_pool() - detach a worker from its pool
1656 * @worker: worker which is attached to its pool
1657 * @pool: the pool @worker is attached to
1658 *
4736cbf7
LJ
1659 * Undo the attaching which had been done in worker_attach_to_pool(). The
1660 * caller worker shouldn't access to the pool after detached except it has
1661 * other reference to the pool.
60f5a4bc
LJ
1662 */
1663static void worker_detach_from_pool(struct worker *worker,
1664 struct worker_pool *pool)
1665{
1666 struct completion *detach_completion = NULL;
1667
92f9c5c4 1668 mutex_lock(&pool->attach_mutex);
da028469
LJ
1669 list_del(&worker->node);
1670 if (list_empty(&pool->workers))
60f5a4bc 1671 detach_completion = pool->detach_completion;
92f9c5c4 1672 mutex_unlock(&pool->attach_mutex);
60f5a4bc 1673
b62c0751
LJ
1674 /* clear leftover flags without pool->lock after it is detached */
1675 worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND);
1676
60f5a4bc
LJ
1677 if (detach_completion)
1678 complete(detach_completion);
1679}
1680
c34056a3
TH
1681/**
1682 * create_worker - create a new workqueue worker
63d95a91 1683 * @pool: pool the new worker will belong to
c34056a3 1684 *
051e1850 1685 * Create and start a new worker which is attached to @pool.
c34056a3
TH
1686 *
1687 * CONTEXT:
1688 * Might sleep. Does GFP_KERNEL allocations.
1689 *
d185af30 1690 * Return:
c34056a3
TH
1691 * Pointer to the newly created worker.
1692 */
bc2ae0f5 1693static struct worker *create_worker(struct worker_pool *pool)
c34056a3 1694{
c34056a3 1695 struct worker *worker = NULL;
f3421797 1696 int id = -1;
e3c916a4 1697 char id_buf[16];
c34056a3 1698
7cda9aae
LJ
1699 /* ID is needed to determine kthread name */
1700 id = ida_simple_get(&pool->worker_ida, 0, 0, GFP_KERNEL);
822d8405
TH
1701 if (id < 0)
1702 goto fail;
c34056a3 1703
f7537df5 1704 worker = alloc_worker(pool->node);
c34056a3
TH
1705 if (!worker)
1706 goto fail;
1707
bd7bdd43 1708 worker->pool = pool;
c34056a3
TH
1709 worker->id = id;
1710
29c91e99 1711 if (pool->cpu >= 0)
e3c916a4
TH
1712 snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
1713 pool->attrs->nice < 0 ? "H" : "");
f3421797 1714 else
e3c916a4
TH
1715 snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);
1716
f3f90ad4 1717 worker->task = kthread_create_on_node(worker_thread, worker, pool->node,
e3c916a4 1718 "kworker/%s", id_buf);
c34056a3
TH
1719 if (IS_ERR(worker->task))
1720 goto fail;
1721
91151228 1722 set_user_nice(worker->task, pool->attrs->nice);
25834c73 1723 kthread_bind_mask(worker->task, pool->attrs->cpumask);
91151228 1724
da028469 1725 /* successful, attach the worker to the pool */
4736cbf7 1726 worker_attach_to_pool(worker, pool);
822d8405 1727
051e1850
LJ
1728 /* start the newly created worker */
1729 spin_lock_irq(&pool->lock);
1730 worker->pool->nr_workers++;
1731 worker_enter_idle(worker);
1732 wake_up_process(worker->task);
1733 spin_unlock_irq(&pool->lock);
1734
c34056a3 1735 return worker;
822d8405 1736
c34056a3 1737fail:
9625ab17 1738 if (id >= 0)
7cda9aae 1739 ida_simple_remove(&pool->worker_ida, id);
c34056a3
TH
1740 kfree(worker);
1741 return NULL;
1742}
1743
c34056a3
TH
1744/**
1745 * destroy_worker - destroy a workqueue worker
1746 * @worker: worker to be destroyed
1747 *
73eb7fe7
LJ
1748 * Destroy @worker and adjust @pool stats accordingly. The worker should
1749 * be idle.
c8e55f36
TH
1750 *
1751 * CONTEXT:
60f5a4bc 1752 * spin_lock_irq(pool->lock).
c34056a3
TH
1753 */
1754static void destroy_worker(struct worker *worker)
1755{
bd7bdd43 1756 struct worker_pool *pool = worker->pool;
c34056a3 1757
cd549687
TH
1758 lockdep_assert_held(&pool->lock);
1759
c34056a3 1760 /* sanity check frenzy */
6183c009 1761 if (WARN_ON(worker->current_work) ||
73eb7fe7
LJ
1762 WARN_ON(!list_empty(&worker->scheduled)) ||
1763 WARN_ON(!(worker->flags & WORKER_IDLE)))
6183c009 1764 return;
c34056a3 1765
73eb7fe7
LJ
1766 pool->nr_workers--;
1767 pool->nr_idle--;
5bdfff96 1768
c8e55f36 1769 list_del_init(&worker->entry);
cb444766 1770 worker->flags |= WORKER_DIE;
60f5a4bc 1771 wake_up_process(worker->task);
c34056a3
TH
1772}
1773
63d95a91 1774static void idle_worker_timeout(unsigned long __pool)
e22bee78 1775{
63d95a91 1776 struct worker_pool *pool = (void *)__pool;
e22bee78 1777
d565ed63 1778 spin_lock_irq(&pool->lock);
e22bee78 1779
3347fc9f 1780 while (too_many_workers(pool)) {
e22bee78
TH
1781 struct worker *worker;
1782 unsigned long expires;
1783
1784 /* idle_list is kept in LIFO order, check the last one */
63d95a91 1785 worker = list_entry(pool->idle_list.prev, struct worker, entry);
e22bee78
TH
1786 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1787
3347fc9f 1788 if (time_before(jiffies, expires)) {
63d95a91 1789 mod_timer(&pool->idle_timer, expires);
3347fc9f 1790 break;
d5abe669 1791 }
3347fc9f
LJ
1792
1793 destroy_worker(worker);
e22bee78
TH
1794 }
1795
d565ed63 1796 spin_unlock_irq(&pool->lock);
e22bee78 1797}
d5abe669 1798
493a1724 1799static void send_mayday(struct work_struct *work)
e22bee78 1800{
112202d9
TH
1801 struct pool_workqueue *pwq = get_work_pwq(work);
1802 struct workqueue_struct *wq = pwq->wq;
493a1724 1803
2e109a28 1804 lockdep_assert_held(&wq_mayday_lock);
e22bee78 1805
493008a8 1806 if (!wq->rescuer)
493a1724 1807 return;
e22bee78
TH
1808
1809 /* mayday mayday mayday */
493a1724 1810 if (list_empty(&pwq->mayday_node)) {
77668c8b
LJ
1811 /*
1812 * If @pwq is for an unbound wq, its base ref may be put at
1813 * any time due to an attribute change. Pin @pwq until the
1814 * rescuer is done with it.
1815 */
1816 get_pwq(pwq);
493a1724 1817 list_add_tail(&pwq->mayday_node, &wq->maydays);
e22bee78 1818 wake_up_process(wq->rescuer->task);
493a1724 1819 }
e22bee78
TH
1820}
1821
706026c2 1822static void pool_mayday_timeout(unsigned long __pool)
e22bee78 1823{
63d95a91 1824 struct worker_pool *pool = (void *)__pool;
e22bee78
TH
1825 struct work_struct *work;
1826
b2d82909
TH
1827 spin_lock_irq(&pool->lock);
1828 spin_lock(&wq_mayday_lock); /* for wq->maydays */
e22bee78 1829
63d95a91 1830 if (need_to_create_worker(pool)) {
e22bee78
TH
1831 /*
1832 * We've been trying to create a new worker but
1833 * haven't been successful. We might be hitting an
1834 * allocation deadlock. Send distress signals to
1835 * rescuers.
1836 */
63d95a91 1837 list_for_each_entry(work, &pool->worklist, entry)
e22bee78 1838 send_mayday(work);
1da177e4 1839 }
e22bee78 1840
b2d82909
TH
1841 spin_unlock(&wq_mayday_lock);
1842 spin_unlock_irq(&pool->lock);
e22bee78 1843
63d95a91 1844 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
1da177e4
LT
1845}
1846
e22bee78
TH
1847/**
1848 * maybe_create_worker - create a new worker if necessary
63d95a91 1849 * @pool: pool to create a new worker for
e22bee78 1850 *
63d95a91 1851 * Create a new worker for @pool if necessary. @pool is guaranteed to
e22bee78
TH
1852 * have at least one idle worker on return from this function. If
1853 * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
63d95a91 1854 * sent to all rescuers with works scheduled on @pool to resolve
e22bee78
TH
1855 * possible allocation deadlock.
1856 *
c5aa87bb
TH
1857 * On return, need_to_create_worker() is guaranteed to be %false and
1858 * may_start_working() %true.
e22bee78
TH
1859 *
1860 * LOCKING:
d565ed63 1861 * spin_lock_irq(pool->lock) which may be released and regrabbed
e22bee78
TH
1862 * multiple times. Does GFP_KERNEL allocations. Called only from
1863 * manager.
e22bee78 1864 */
29187a9e 1865static void maybe_create_worker(struct worker_pool *pool)
d565ed63
TH
1866__releases(&pool->lock)
1867__acquires(&pool->lock)
1da177e4 1868{
e22bee78 1869restart:
d565ed63 1870 spin_unlock_irq(&pool->lock);
9f9c2364 1871
e22bee78 1872 /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
63d95a91 1873 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
e22bee78
TH
1874
1875 while (true) {
051e1850 1876 if (create_worker(pool) || !need_to_create_worker(pool))
e22bee78 1877 break;
1da177e4 1878
e212f361 1879 schedule_timeout_interruptible(CREATE_COOLDOWN);
9f9c2364 1880
63d95a91 1881 if (!need_to_create_worker(pool))
e22bee78
TH
1882 break;
1883 }
1884
63d95a91 1885 del_timer_sync(&pool->mayday_timer);
d565ed63 1886 spin_lock_irq(&pool->lock);
051e1850
LJ
1887 /*
1888 * This is necessary even after a new worker was just successfully
1889 * created as @pool->lock was dropped and the new worker might have
1890 * already become busy.
1891 */
63d95a91 1892 if (need_to_create_worker(pool))
e22bee78 1893 goto restart;
e22bee78
TH
1894}
1895
73f53c4a 1896/**
e22bee78
TH
1897 * manage_workers - manage worker pool
1898 * @worker: self
73f53c4a 1899 *
706026c2 1900 * Assume the manager role and manage the worker pool @worker belongs
e22bee78 1901 * to. At any given time, there can be only zero or one manager per
706026c2 1902 * pool. The exclusion is handled automatically by this function.
e22bee78
TH
1903 *
1904 * The caller can safely start processing works on false return. On
1905 * true return, it's guaranteed that need_to_create_worker() is false
1906 * and may_start_working() is true.
73f53c4a
TH
1907 *
1908 * CONTEXT:
d565ed63 1909 * spin_lock_irq(pool->lock) which may be released and regrabbed
e22bee78
TH
1910 * multiple times. Does GFP_KERNEL allocations.
1911 *
d185af30 1912 * Return:
29187a9e
TH
1913 * %false if the pool doesn't need management and the caller can safely
1914 * start processing works, %true if management function was performed and
1915 * the conditions that the caller verified before calling the function may
1916 * no longer be true.
73f53c4a 1917 */
e22bee78 1918static bool manage_workers(struct worker *worker)
73f53c4a 1919{
63d95a91 1920 struct worker_pool *pool = worker->pool;
73f53c4a 1921
bc3a1afc 1922 /*
bc3a1afc
TH
1923 * Anyone who successfully grabs manager_arb wins the arbitration
1924 * and becomes the manager. mutex_trylock() on pool->manager_arb
1925 * failure while holding pool->lock reliably indicates that someone
1926 * else is managing the pool and the worker which failed trylock
1927 * can proceed to executing work items. This means that anyone
1928 * grabbing manager_arb is responsible for actually performing
1929 * manager duties. If manager_arb is grabbed and released without
1930 * actual management, the pool may stall indefinitely.
bc3a1afc 1931 */
34a06bd6 1932 if (!mutex_trylock(&pool->manager_arb))
29187a9e 1933 return false;
2607d7a6 1934 pool->manager = worker;
1e19ffc6 1935
29187a9e 1936 maybe_create_worker(pool);
e22bee78 1937
2607d7a6 1938 pool->manager = NULL;
34a06bd6 1939 mutex_unlock(&pool->manager_arb);
29187a9e 1940 return true;
73f53c4a
TH
1941}
1942
a62428c0
TH
1943/**
1944 * process_one_work - process single work
c34056a3 1945 * @worker: self
a62428c0
TH
1946 * @work: work to process
1947 *
1948 * Process @work. This function contains all the logics necessary to
1949 * process a single work including synchronization against and
1950 * interaction with other workers on the same cpu, queueing and
1951 * flushing. As long as context requirement is met, any worker can
1952 * call this function to process a work.
1953 *
1954 * CONTEXT:
d565ed63 1955 * spin_lock_irq(pool->lock) which is released and regrabbed.
a62428c0 1956 */
c34056a3 1957static void process_one_work(struct worker *worker, struct work_struct *work)
d565ed63
TH
1958__releases(&pool->lock)
1959__acquires(&pool->lock)
a62428c0 1960{
112202d9 1961 struct pool_workqueue *pwq = get_work_pwq(work);
bd7bdd43 1962 struct worker_pool *pool = worker->pool;
112202d9 1963 bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
73f53c4a 1964 int work_color;
7e11629d 1965 struct worker *collision;
a62428c0
TH
1966#ifdef CONFIG_LOCKDEP
1967 /*
1968 * It is permissible to free the struct work_struct from
1969 * inside the function that is called from it, this we need to
1970 * take into account for lockdep too. To avoid bogus "held
1971 * lock freed" warnings as well as problems when looking into
1972 * work->lockdep_map, make a copy and use that here.
1973 */
4d82a1de
PZ
1974 struct lockdep_map lockdep_map;
1975
1976 lockdep_copy_map(&lockdep_map, &work->lockdep_map);
a62428c0 1977#endif
807407c0 1978 /* ensure we're on the correct CPU */
85327af6 1979 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
ec22ca5e 1980 raw_smp_processor_id() != pool->cpu);
25511a47 1981
7e11629d
TH
1982 /*
1983 * A single work shouldn't be executed concurrently by
1984 * multiple workers on a single cpu. Check whether anyone is
1985 * already processing the work. If so, defer the work to the
1986 * currently executing one.
1987 */
c9e7cf27 1988 collision = find_worker_executing_work(pool, work);
7e11629d
TH
1989 if (unlikely(collision)) {
1990 move_linked_works(work, &collision->scheduled, NULL);
1991 return;
1992 }
1993
8930caba 1994 /* claim and dequeue */
a62428c0 1995 debug_work_deactivate(work);
c9e7cf27 1996 hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
c34056a3 1997 worker->current_work = work;
a2c1c57b 1998 worker->current_func = work->func;
112202d9 1999 worker->current_pwq = pwq;
73f53c4a 2000 work_color = get_work_color(work);
7a22ad75 2001
a62428c0
TH
2002 list_del_init(&work->entry);
2003
fb0e7beb 2004 /*
228f1d00
LJ
2005 * CPU intensive works don't participate in concurrency management.
2006 * They're the scheduler's responsibility. This takes @worker out
2007 * of concurrency management and the next code block will chain
2008 * execution of the pending work items.
fb0e7beb
TH
2009 */
2010 if (unlikely(cpu_intensive))
228f1d00 2011 worker_set_flags(worker, WORKER_CPU_INTENSIVE);
fb0e7beb 2012
974271c4 2013 /*
a489a03e
LJ
2014 * Wake up another worker if necessary. The condition is always
2015 * false for normal per-cpu workers since nr_running would always
2016 * be >= 1 at this point. This is used to chain execution of the
2017 * pending work items for WORKER_NOT_RUNNING workers such as the
228f1d00 2018 * UNBOUND and CPU_INTENSIVE ones.
974271c4 2019 */
a489a03e 2020 if (need_more_worker(pool))
63d95a91 2021 wake_up_worker(pool);
974271c4 2022
8930caba 2023 /*
7c3eed5c 2024 * Record the last pool and clear PENDING which should be the last
d565ed63 2025 * update to @work. Also, do this inside @pool->lock so that
23657bb1
TH
2026 * PENDING and queued state changes happen together while IRQ is
2027 * disabled.
8930caba 2028 */
7c3eed5c 2029 set_work_pool_and_clear_pending(work, pool->id);
a62428c0 2030
d565ed63 2031 spin_unlock_irq(&pool->lock);
a62428c0 2032
112202d9 2033 lock_map_acquire_read(&pwq->wq->lockdep_map);
a62428c0 2034 lock_map_acquire(&lockdep_map);
e36c886a 2035 trace_workqueue_execute_start(work);
a2c1c57b 2036 worker->current_func(work);
e36c886a
AV
2037 /*
2038 * While we must be careful to not use "work" after this, the trace
2039 * point will only record its address.
2040 */
2041 trace_workqueue_execute_end(work);
a62428c0 2042 lock_map_release(&lockdep_map);
112202d9 2043 lock_map_release(&pwq->wq->lockdep_map);
a62428c0
TH
2044
2045 if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
044c782c
VI
2046 pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
2047 " last function: %pf\n",
a2c1c57b
TH
2048 current->comm, preempt_count(), task_pid_nr(current),
2049 worker->current_func);
a62428c0
TH
2050 debug_show_held_locks(current);
2051 dump_stack();
2052 }
2053
b22ce278
TH
2054 /*
2055 * The following prevents a kworker from hogging CPU on !PREEMPT
2056 * kernels, where a requeueing work item waiting for something to
2057 * happen could deadlock with stop_machine as such work item could
2058 * indefinitely requeue itself while all other CPUs are trapped in
789cbbec
JL
2059 * stop_machine. At the same time, report a quiescent RCU state so
2060 * the same condition doesn't freeze RCU.
b22ce278 2061 */
3e28e377 2062 cond_resched_rcu_qs();
b22ce278 2063
d565ed63 2064 spin_lock_irq(&pool->lock);
a62428c0 2065
fb0e7beb
TH
2066 /* clear cpu intensive status */
2067 if (unlikely(cpu_intensive))
2068 worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2069
a62428c0 2070 /* we're done with it, release */
42f8570f 2071 hash_del(&worker->hentry);
c34056a3 2072 worker->current_work = NULL;
a2c1c57b 2073 worker->current_func = NULL;
112202d9 2074 worker->current_pwq = NULL;
3d1cb205 2075 worker->desc_valid = false;
112202d9 2076 pwq_dec_nr_in_flight(pwq, work_color);
a62428c0
TH
2077}
2078
affee4b2
TH
2079/**
2080 * process_scheduled_works - process scheduled works
2081 * @worker: self
2082 *
2083 * Process all scheduled works. Please note that the scheduled list
2084 * may change while processing a work, so this function repeatedly
2085 * fetches a work from the top and executes it.
2086 *
2087 * CONTEXT:
d565ed63 2088 * spin_lock_irq(pool->lock) which may be released and regrabbed
affee4b2
TH
2089 * multiple times.
2090 */
2091static void process_scheduled_works(struct worker *worker)
1da177e4 2092{
affee4b2
TH
2093 while (!list_empty(&worker->scheduled)) {
2094 struct work_struct *work = list_first_entry(&worker->scheduled,
1da177e4 2095 struct work_struct, entry);
c34056a3 2096 process_one_work(worker, work);
1da177e4 2097 }
1da177e4
LT
2098}
2099
4690c4ab
TH
2100/**
2101 * worker_thread - the worker thread function
c34056a3 2102 * @__worker: self
4690c4ab 2103 *
c5aa87bb
TH
2104 * The worker thread function. All workers belong to a worker_pool -
2105 * either a per-cpu one or dynamic unbound one. These workers process all
2106 * work items regardless of their specific target workqueue. The only
2107 * exception is work items which belong to workqueues with a rescuer which
2108 * will be explained in rescuer_thread().
d185af30
YB
2109 *
2110 * Return: 0
4690c4ab 2111 */
c34056a3 2112static int worker_thread(void *__worker)
1da177e4 2113{
c34056a3 2114 struct worker *worker = __worker;
bd7bdd43 2115 struct worker_pool *pool = worker->pool;
1da177e4 2116
e22bee78
TH
2117 /* tell the scheduler that this is a workqueue worker */
2118 worker->task->flags |= PF_WQ_WORKER;
c8e55f36 2119woke_up:
d565ed63 2120 spin_lock_irq(&pool->lock);
1da177e4 2121
a9ab775b
TH
2122 /* am I supposed to die? */
2123 if (unlikely(worker->flags & WORKER_DIE)) {
d565ed63 2124 spin_unlock_irq(&pool->lock);
a9ab775b
TH
2125 WARN_ON_ONCE(!list_empty(&worker->entry));
2126 worker->task->flags &= ~PF_WQ_WORKER;
60f5a4bc
LJ
2127
2128 set_task_comm(worker->task, "kworker/dying");
7cda9aae 2129 ida_simple_remove(&pool->worker_ida, worker->id);
60f5a4bc
LJ
2130 worker_detach_from_pool(worker, pool);
2131 kfree(worker);
a9ab775b 2132 return 0;
c8e55f36 2133 }
affee4b2 2134
c8e55f36 2135 worker_leave_idle(worker);
db7bccf4 2136recheck:
e22bee78 2137 /* no more worker necessary? */
63d95a91 2138 if (!need_more_worker(pool))
e22bee78
TH
2139 goto sleep;
2140
2141 /* do we need to manage? */
63d95a91 2142 if (unlikely(!may_start_working(pool)) && manage_workers(worker))
e22bee78
TH
2143 goto recheck;
2144
c8e55f36
TH
2145 /*
2146 * ->scheduled list can only be filled while a worker is
2147 * preparing to process a work or actually processing it.
2148 * Make sure nobody diddled with it while I was sleeping.
2149 */
6183c009 2150 WARN_ON_ONCE(!list_empty(&worker->scheduled));
c8e55f36 2151
e22bee78 2152 /*
a9ab775b
TH
2153 * Finish PREP stage. We're guaranteed to have at least one idle
2154 * worker or that someone else has already assumed the manager
2155 * role. This is where @worker starts participating in concurrency
2156 * management if applicable and concurrency management is restored
2157 * after being rebound. See rebind_workers() for details.
e22bee78 2158 */
a9ab775b 2159 worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);
e22bee78
TH
2160
2161 do {
c8e55f36 2162 struct work_struct *work =
bd7bdd43 2163 list_first_entry(&pool->worklist,
c8e55f36
TH
2164 struct work_struct, entry);
2165
82607adc
TH
2166 pool->watchdog_ts = jiffies;
2167
c8e55f36
TH
2168 if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
2169 /* optimization path, not strictly necessary */
2170 process_one_work(worker, work);
2171 if (unlikely(!list_empty(&worker->scheduled)))
affee4b2 2172 process_scheduled_works(worker);
c8e55f36
TH
2173 } else {
2174 move_linked_works(work, &worker->scheduled, NULL);
2175 process_scheduled_works(worker);
affee4b2 2176 }
63d95a91 2177 } while (keep_working(pool));
e22bee78 2178
228f1d00 2179 worker_set_flags(worker, WORKER_PREP);
d313dd85 2180sleep:
c8e55f36 2181 /*
d565ed63
TH
2182 * pool->lock is held and there's no work to process and no need to
2183 * manage, sleep. Workers are woken up only while holding
2184 * pool->lock or from local cpu, so setting the current state
2185 * before releasing pool->lock is enough to prevent losing any
2186 * event.
c8e55f36
TH
2187 */
2188 worker_enter_idle(worker);
2189 __set_current_state(TASK_INTERRUPTIBLE);
d565ed63 2190 spin_unlock_irq(&pool->lock);
c8e55f36
TH
2191 schedule();
2192 goto woke_up;
1da177e4
LT
2193}
2194
e22bee78
TH
2195/**
2196 * rescuer_thread - the rescuer thread function
111c225a 2197 * @__rescuer: self
e22bee78
TH
2198 *
2199 * Workqueue rescuer thread function. There's one rescuer for each
493008a8 2200 * workqueue which has WQ_MEM_RECLAIM set.
e22bee78 2201 *
706026c2 2202 * Regular work processing on a pool may block trying to create a new
e22bee78
TH
2203 * worker which uses GFP_KERNEL allocation which has slight chance of
2204 * developing into deadlock if some works currently on the same queue
2205 * need to be processed to satisfy the GFP_KERNEL allocation. This is
2206 * the problem rescuer solves.
2207 *
706026c2
TH
2208 * When such condition is possible, the pool summons rescuers of all
2209 * workqueues which have works queued on the pool and let them process
e22bee78
TH
2210 * those works so that forward progress can be guaranteed.
2211 *
2212 * This should happen rarely.
d185af30
YB
2213 *
2214 * Return: 0
e22bee78 2215 */
111c225a 2216static int rescuer_thread(void *__rescuer)
e22bee78 2217{
111c225a
TH
2218 struct worker *rescuer = __rescuer;
2219 struct workqueue_struct *wq = rescuer->rescue_wq;
e22bee78 2220 struct list_head *scheduled = &rescuer->scheduled;
4d595b86 2221 bool should_stop;
e22bee78
TH
2222
2223 set_user_nice(current, RESCUER_NICE_LEVEL);
111c225a
TH
2224
2225 /*
2226 * Mark rescuer as worker too. As WORKER_PREP is never cleared, it
2227 * doesn't participate in concurrency management.
2228 */
2229 rescuer->task->flags |= PF_WQ_WORKER;
e22bee78
TH
2230repeat:
2231 set_current_state(TASK_INTERRUPTIBLE);
2232
4d595b86
LJ
2233 /*
2234 * By the time the rescuer is requested to stop, the workqueue
2235 * shouldn't have any work pending, but @wq->maydays may still have
2236 * pwq(s) queued. This can happen by non-rescuer workers consuming
2237 * all the work items before the rescuer got to them. Go through
2238 * @wq->maydays processing before acting on should_stop so that the
2239 * list is always empty on exit.
2240 */
2241 should_stop = kthread_should_stop();
e22bee78 2242
493a1724 2243 /* see whether any pwq is asking for help */
2e109a28 2244 spin_lock_irq(&wq_mayday_lock);
493a1724
TH
2245
2246 while (!list_empty(&wq->maydays)) {
2247 struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
2248 struct pool_workqueue, mayday_node);
112202d9 2249 struct worker_pool *pool = pwq->pool;
e22bee78 2250 struct work_struct *work, *n;
82607adc 2251 bool first = true;
e22bee78
TH
2252
2253 __set_current_state(TASK_RUNNING);
493a1724
TH
2254 list_del_init(&pwq->mayday_node);
2255
2e109a28 2256 spin_unlock_irq(&wq_mayday_lock);
e22bee78 2257
51697d39
LJ
2258 worker_attach_to_pool(rescuer, pool);
2259
2260 spin_lock_irq(&pool->lock);
b3104104 2261 rescuer->pool = pool;
e22bee78
TH
2262
2263 /*
2264 * Slurp in all works issued via this workqueue and
2265 * process'em.
2266 */
0479c8c5 2267 WARN_ON_ONCE(!list_empty(scheduled));
82607adc
TH
2268 list_for_each_entry_safe(work, n, &pool->worklist, entry) {
2269 if (get_work_pwq(work) == pwq) {
2270 if (first)
2271 pool->watchdog_ts = jiffies;
e22bee78 2272 move_linked_works(work, scheduled, &n);
82607adc
TH
2273 }
2274 first = false;
2275 }
e22bee78 2276
008847f6
N
2277 if (!list_empty(scheduled)) {
2278 process_scheduled_works(rescuer);
2279
2280 /*
2281 * The above execution of rescued work items could
2282 * have created more to rescue through
2283 * pwq_activate_first_delayed() or chained
2284 * queueing. Let's put @pwq back on mayday list so
2285 * that such back-to-back work items, which may be
2286 * being used to relieve memory pressure, don't
2287 * incur MAYDAY_INTERVAL delay inbetween.
2288 */
2289 if (need_to_create_worker(pool)) {
2290 spin_lock(&wq_mayday_lock);
2291 get_pwq(pwq);
2292 list_move_tail(&pwq->mayday_node, &wq->maydays);
2293 spin_unlock(&wq_mayday_lock);
2294 }
2295 }
7576958a 2296
77668c8b
LJ
2297 /*
2298 * Put the reference grabbed by send_mayday(). @pool won't
13b1d625 2299 * go away while we're still attached to it.
77668c8b
LJ
2300 */
2301 put_pwq(pwq);
2302
7576958a 2303 /*
d8ca83e6 2304 * Leave this pool. If need_more_worker() is %true, notify a
7576958a
TH
2305 * regular worker; otherwise, we end up with 0 concurrency
2306 * and stalling the execution.
2307 */
d8ca83e6 2308 if (need_more_worker(pool))
63d95a91 2309 wake_up_worker(pool);
7576958a 2310
b3104104 2311 rescuer->pool = NULL;
13b1d625
LJ
2312 spin_unlock_irq(&pool->lock);
2313
2314 worker_detach_from_pool(rescuer, pool);
2315
2316 spin_lock_irq(&wq_mayday_lock);
e22bee78
TH
2317 }
2318
2e109a28 2319 spin_unlock_irq(&wq_mayday_lock);
493a1724 2320
4d595b86
LJ
2321 if (should_stop) {
2322 __set_current_state(TASK_RUNNING);
2323 rescuer->task->flags &= ~PF_WQ_WORKER;
2324 return 0;
2325 }
2326
111c225a
TH
2327 /* rescuers should never participate in concurrency management */
2328 WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
e22bee78
TH
2329 schedule();
2330 goto repeat;
1da177e4
LT
2331}
2332
fca839c0
TH
2333/**
2334 * check_flush_dependency - check for flush dependency sanity
2335 * @target_wq: workqueue being flushed
2336 * @target_work: work item being flushed (NULL for workqueue flushes)
2337 *
2338 * %current is trying to flush the whole @target_wq or @target_work on it.
2339 * If @target_wq doesn't have %WQ_MEM_RECLAIM, verify that %current is not
2340 * reclaiming memory or running on a workqueue which doesn't have
2341 * %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to
2342 * a deadlock.
2343 */
2344static void check_flush_dependency(struct workqueue_struct *target_wq,
2345 struct work_struct *target_work)
2346{
2347 work_func_t target_func = target_work ? target_work->func : NULL;
2348 struct worker *worker;
2349
2350 if (target_wq->flags & WQ_MEM_RECLAIM)
2351 return;
2352
2353 worker = current_wq_worker();
2354
2355 WARN_ONCE(current->flags & PF_MEMALLOC,
2356 "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%pf",
2357 current->pid, current->comm, target_wq->name, target_func);
2358 WARN_ONCE(worker && (worker->current_pwq->wq->flags & WQ_MEM_RECLAIM),
2359 "workqueue: WQ_MEM_RECLAIM %s:%pf is flushing !WQ_MEM_RECLAIM %s:%pf",
2360 worker->current_pwq->wq->name, worker->current_func,
2361 target_wq->name, target_func);
2362}
2363
fc2e4d70
ON
2364struct wq_barrier {
2365 struct work_struct work;
2366 struct completion done;
2607d7a6 2367 struct task_struct *task; /* purely informational */
fc2e4d70
ON
2368};
2369
2370static void wq_barrier_func(struct work_struct *work)
2371{
2372 struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
2373 complete(&barr->done);
2374}
2375
4690c4ab
TH
2376/**
2377 * insert_wq_barrier - insert a barrier work
112202d9 2378 * @pwq: pwq to insert barrier into
4690c4ab 2379 * @barr: wq_barrier to insert
affee4b2
TH
2380 * @target: target work to attach @barr to
2381 * @worker: worker currently executing @target, NULL if @target is not executing
4690c4ab 2382 *
affee4b2
TH
2383 * @barr is linked to @target such that @barr is completed only after
2384 * @target finishes execution. Please note that the ordering
2385 * guarantee is observed only with respect to @target and on the local
2386 * cpu.
2387 *
2388 * Currently, a queued barrier can't be canceled. This is because
2389 * try_to_grab_pending() can't determine whether the work to be
2390 * grabbed is at the head of the queue and thus can't clear LINKED
2391 * flag of the previous work while there must be a valid next work
2392 * after a work with LINKED flag set.
2393 *
2394 * Note that when @worker is non-NULL, @target may be modified
112202d9 2395 * underneath us, so we can't reliably determine pwq from @target.
4690c4ab
TH
2396 *
2397 * CONTEXT:
d565ed63 2398 * spin_lock_irq(pool->lock).
4690c4ab 2399 */
112202d9 2400static void insert_wq_barrier(struct pool_workqueue *pwq,
affee4b2
TH
2401 struct wq_barrier *barr,
2402 struct work_struct *target, struct worker *worker)
fc2e4d70 2403{
affee4b2
TH
2404 struct list_head *head;
2405 unsigned int linked = 0;
2406
dc186ad7 2407 /*
d565ed63 2408 * debugobject calls are safe here even with pool->lock locked
dc186ad7
TG
2409 * as we know for sure that this will not trigger any of the
2410 * checks and call back into the fixup functions where we
2411 * might deadlock.
2412 */
ca1cab37 2413 INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
22df02bb 2414 __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
fc2e4d70 2415 init_completion(&barr->done);
2607d7a6 2416 barr->task = current;
83c22520 2417
affee4b2
TH
2418 /*
2419 * If @target is currently being executed, schedule the
2420 * barrier to the worker; otherwise, put it after @target.
2421 */
2422 if (worker)
2423 head = worker->scheduled.next;
2424 else {
2425 unsigned long *bits = work_data_bits(target);
2426
2427 head = target->entry.next;
2428 /* there can already be other linked works, inherit and set */
2429 linked = *bits & WORK_STRUCT_LINKED;
2430 __set_bit(WORK_STRUCT_LINKED_BIT, bits);
2431 }
2432
dc186ad7 2433 debug_work_activate(&barr->work);
112202d9 2434 insert_work(pwq, &barr->work, head,
affee4b2 2435 work_color_to_flags(WORK_NO_COLOR) | linked);
fc2e4d70
ON
2436}
2437
73f53c4a 2438/**
112202d9 2439 * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
73f53c4a
TH
2440 * @wq: workqueue being flushed
2441 * @flush_color: new flush color, < 0 for no-op
2442 * @work_color: new work color, < 0 for no-op
2443 *
112202d9 2444 * Prepare pwqs for workqueue flushing.
73f53c4a 2445 *
112202d9
TH
2446 * If @flush_color is non-negative, flush_color on all pwqs should be
2447 * -1. If no pwq has in-flight commands at the specified color, all
2448 * pwq->flush_color's stay at -1 and %false is returned. If any pwq
2449 * has in flight commands, its pwq->flush_color is set to
2450 * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
73f53c4a
TH
2451 * wakeup logic is armed and %true is returned.
2452 *
2453 * The caller should have initialized @wq->first_flusher prior to
2454 * calling this function with non-negative @flush_color. If
2455 * @flush_color is negative, no flush color update is done and %false
2456 * is returned.
2457 *
112202d9 2458 * If @work_color is non-negative, all pwqs should have the same
73f53c4a
TH
2459 * work_color which is previous to @work_color and all will be
2460 * advanced to @work_color.
2461 *
2462 * CONTEXT:
3c25a55d 2463 * mutex_lock(wq->mutex).
73f53c4a 2464 *
d185af30 2465 * Return:
73f53c4a
TH
2466 * %true if @flush_color >= 0 and there's something to flush. %false
2467 * otherwise.
2468 */
112202d9 2469static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
73f53c4a 2470 int flush_color, int work_color)
1da177e4 2471{
73f53c4a 2472 bool wait = false;
49e3cf44 2473 struct pool_workqueue *pwq;
1da177e4 2474
73f53c4a 2475 if (flush_color >= 0) {
6183c009 2476 WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
112202d9 2477 atomic_set(&wq->nr_pwqs_to_flush, 1);
1da177e4 2478 }
2355b70f 2479
49e3cf44 2480 for_each_pwq(pwq, wq) {
112202d9 2481 struct worker_pool *pool = pwq->pool;
fc2e4d70 2482
b09f4fd3 2483 spin_lock_irq(&pool->lock);
83c22520 2484
73f53c4a 2485 if (flush_color >= 0) {
6183c009 2486 WARN_ON_ONCE(pwq->flush_color != -1);
fc2e4d70 2487
112202d9
TH
2488 if (pwq->nr_in_flight[flush_color]) {
2489 pwq->flush_color = flush_color;
2490 atomic_inc(&wq->nr_pwqs_to_flush);
73f53c4a
TH
2491 wait = true;
2492 }
2493 }
1da177e4 2494
73f53c4a 2495 if (work_color >= 0) {
6183c009 2496 WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
112202d9 2497 pwq->work_color = work_color;
73f53c4a 2498 }
1da177e4 2499
b09f4fd3 2500 spin_unlock_irq(&pool->lock);
1da177e4 2501 }
2355b70f 2502
112202d9 2503 if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
73f53c4a 2504 complete(&wq->first_flusher->done);
14441960 2505
73f53c4a 2506 return wait;
1da177e4
LT
2507}
2508
0fcb78c2 2509/**
1da177e4 2510 * flush_workqueue - ensure that any scheduled work has run to completion.
0fcb78c2 2511 * @wq: workqueue to flush
1da177e4 2512 *
c5aa87bb
TH
2513 * This function sleeps until all work items which were queued on entry
2514 * have finished execution, but it is not livelocked by new incoming ones.
1da177e4 2515 */
7ad5b3a5 2516void flush_workqueue(struct workqueue_struct *wq)
1da177e4 2517{
73f53c4a
TH
2518 struct wq_flusher this_flusher = {
2519 .list = LIST_HEAD_INIT(this_flusher.list),
2520 .flush_color = -1,
2521 .done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done),
2522 };
2523 int next_color;
1da177e4 2524
3295f0ef
IM
2525 lock_map_acquire(&wq->lockdep_map);
2526 lock_map_release(&wq->lockdep_map);
73f53c4a 2527
3c25a55d 2528 mutex_lock(&wq->mutex);
73f53c4a
TH
2529
2530 /*
2531 * Start-to-wait phase
2532 */
2533 next_color = work_next_color(wq->work_color);
2534
2535 if (next_color != wq->flush_color) {
2536 /*
2537 * Color space is not full. The current work_color
2538 * becomes our flush_color and work_color is advanced
2539 * by one.
2540 */
6183c009 2541 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
73f53c4a
TH
2542 this_flusher.flush_color = wq->work_color;
2543 wq->work_color = next_color;
2544
2545 if (!wq->first_flusher) {
2546 /* no flush in progress, become the first flusher */
6183c009 2547 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
73f53c4a
TH
2548
2549 wq->first_flusher = &this_flusher;
2550
112202d9 2551 if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
73f53c4a
TH
2552 wq->work_color)) {
2553 /* nothing to flush, done */
2554 wq->flush_color = next_color;
2555 wq->first_flusher = NULL;
2556 goto out_unlock;
2557 }
2558 } else {
2559 /* wait in queue */
6183c009 2560 WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
73f53c4a 2561 list_add_tail(&this_flusher.list, &wq->flusher_queue);
112202d9 2562 flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
73f53c4a
TH
2563 }
2564 } else {
2565 /*
2566 * Oops, color space is full, wait on overflow queue.
2567 * The next flush completion will assign us
2568 * flush_color and transfer to flusher_queue.
2569 */
2570 list_add_tail(&this_flusher.list, &wq->flusher_overflow);
2571 }
2572
fca839c0
TH
2573 check_flush_dependency(wq, NULL);
2574
3c25a55d 2575 mutex_unlock(&wq->mutex);
73f53c4a
TH
2576
2577 wait_for_completion(&this_flusher.done);
2578
2579 /*
2580 * Wake-up-and-cascade phase
2581 *
2582 * First flushers are responsible for cascading flushes and
2583 * handling overflow. Non-first flushers can simply return.
2584 */
2585 if (wq->first_flusher != &this_flusher)
2586 return;
2587
3c25a55d 2588 mutex_lock(&wq->mutex);
73f53c4a 2589
4ce48b37
TH
2590 /* we might have raced, check again with mutex held */
2591 if (wq->first_flusher != &this_flusher)
2592 goto out_unlock;
2593
73f53c4a
TH
2594 wq->first_flusher = NULL;
2595
6183c009
TH
2596 WARN_ON_ONCE(!list_empty(&this_flusher.list));
2597 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
73f53c4a
TH
2598
2599 while (true) {
2600 struct wq_flusher *next, *tmp;
2601
2602 /* complete all the flushers sharing the current flush color */
2603 list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
2604 if (next->flush_color != wq->flush_color)
2605 break;
2606 list_del_init(&next->list);
2607 complete(&next->done);
2608 }
2609
6183c009
TH
2610 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
2611 wq->flush_color != work_next_color(wq->work_color));
73f53c4a
TH
2612
2613 /* this flush_color is finished, advance by one */
2614 wq->flush_color = work_next_color(wq->flush_color);
2615
2616 /* one color has been freed, handle overflow queue */
2617 if (!list_empty(&wq->flusher_overflow)) {
2618 /*
2619 * Assign the same color to all overflowed
2620 * flushers, advance work_color and append to
2621 * flusher_queue. This is the start-to-wait
2622 * phase for these overflowed flushers.
2623 */
2624 list_for_each_entry(tmp, &wq->flusher_overflow, list)
2625 tmp->flush_color = wq->work_color;
2626
2627 wq->work_color = work_next_color(wq->work_color);
2628
2629 list_splice_tail_init(&wq->flusher_overflow,
2630 &wq->flusher_queue);
112202d9 2631 flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
73f53c4a
TH
2632 }
2633
2634 if (list_empty(&wq->flusher_queue)) {
6183c009 2635 WARN_ON_ONCE(wq->flush_color != wq->work_color);
73f53c4a
TH
2636 break;
2637 }
2638
2639 /*
2640 * Need to flush more colors. Make the next flusher
112202d9 2641 * the new first flusher and arm pwqs.
73f53c4a 2642 */
6183c009
TH
2643 WARN_ON_ONCE(wq->flush_color == wq->work_color);
2644 WARN_ON_ONCE(wq->flush_color != next->flush_color);
73f53c4a
TH
2645
2646 list_del_init(&next->list);
2647 wq->first_flusher = next;
2648
112202d9 2649 if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
73f53c4a
TH
2650 break;
2651
2652 /*
2653 * Meh... this color is already done, clear first
2654 * flusher and repeat cascading.
2655 */
2656 wq->first_flusher = NULL;
2657 }
2658
2659out_unlock:
3c25a55d 2660 mutex_unlock(&wq->mutex);
1da177e4 2661}
1dadafa8 2662EXPORT_SYMBOL(flush_workqueue);
1da177e4 2663
9c5a2ba7
TH
2664/**
2665 * drain_workqueue - drain a workqueue
2666 * @wq: workqueue to drain
2667 *
2668 * Wait until the workqueue becomes empty. While draining is in progress,
2669 * only chain queueing is allowed. IOW, only currently pending or running
2670 * work items on @wq can queue further work items on it. @wq is flushed
b749b1b6 2671 * repeatedly until it becomes empty. The number of flushing is determined
9c5a2ba7
TH
2672 * by the depth of chaining and should be relatively short. Whine if it
2673 * takes too long.
2674 */
2675void drain_workqueue(struct workqueue_struct *wq)
2676{
2677 unsigned int flush_cnt = 0;
49e3cf44 2678 struct pool_workqueue *pwq;
9c5a2ba7
TH
2679
2680 /*
2681 * __queue_work() needs to test whether there are drainers, is much
2682 * hotter than drain_workqueue() and already looks at @wq->flags.
618b01eb 2683 * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
9c5a2ba7 2684 */
87fc741e 2685 mutex_lock(&wq->mutex);
9c5a2ba7 2686 if (!wq->nr_drainers++)
618b01eb 2687 wq->flags |= __WQ_DRAINING;
87fc741e 2688 mutex_unlock(&wq->mutex);
9c5a2ba7
TH
2689reflush:
2690 flush_workqueue(wq);
2691
b09f4fd3 2692 mutex_lock(&wq->mutex);
76af4d93 2693
49e3cf44 2694 for_each_pwq(pwq, wq) {
fa2563e4 2695 bool drained;
9c5a2ba7 2696
b09f4fd3 2697 spin_lock_irq(&pwq->pool->lock);
112202d9 2698 drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
b09f4fd3 2699 spin_unlock_irq(&pwq->pool->lock);
fa2563e4
TT
2700
2701 if (drained)
9c5a2ba7
TH
2702 continue;
2703
2704 if (++flush_cnt == 10 ||
2705 (flush_cnt % 100 == 0 && flush_cnt <= 1000))
c5aa87bb 2706 pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n",
044c782c 2707 wq->name, flush_cnt);
76af4d93 2708
b09f4fd3 2709 mutex_unlock(&wq->mutex);
9c5a2ba7
TH
2710 goto reflush;
2711 }
2712
9c5a2ba7 2713 if (!--wq->nr_drainers)
618b01eb 2714 wq->flags &= ~__WQ_DRAINING;
87fc741e 2715 mutex_unlock(&wq->mutex);
9c5a2ba7
TH
2716}
2717EXPORT_SYMBOL_GPL(drain_workqueue);
2718
606a5020 2719static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr)
db700897 2720{
affee4b2 2721 struct worker *worker = NULL;
c9e7cf27 2722 struct worker_pool *pool;
112202d9 2723 struct pool_workqueue *pwq;
db700897
ON
2724
2725 might_sleep();
fa1b54e6
TH
2726
2727 local_irq_disable();
c9e7cf27 2728 pool = get_work_pool(work);
fa1b54e6
TH
2729 if (!pool) {
2730 local_irq_enable();
baf59022 2731 return false;
fa1b54e6 2732 }
db700897 2733
fa1b54e6 2734 spin_lock(&pool->lock);
0b3dae68 2735 /* see the comment in try_to_grab_pending() with the same code */
112202d9
TH
2736 pwq = get_work_pwq(work);
2737 if (pwq) {
2738 if (unlikely(pwq->pool != pool))
4690c4ab 2739 goto already_gone;
606a5020 2740 } else {
c9e7cf27 2741 worker = find_worker_executing_work(pool, work);
affee4b2 2742 if (!worker)
4690c4ab 2743 goto already_gone;
112202d9 2744 pwq = worker->current_pwq;
606a5020 2745 }
db700897 2746
fca839c0
TH
2747 check_flush_dependency(pwq->wq, work);
2748
112202d9 2749 insert_wq_barrier(pwq, barr, work, worker);
d565ed63 2750 spin_unlock_irq(&pool->lock);
7a22ad75 2751
e159489b
TH
2752 /*
2753 * If @max_active is 1 or rescuer is in use, flushing another work
2754 * item on the same workqueue may lead to deadlock. Make sure the
2755 * flusher is not running on the same workqueue by verifying write
2756 * access.
2757 */
493008a8 2758 if (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)
112202d9 2759 lock_map_acquire(&pwq->wq->lockdep_map);
e159489b 2760 else
112202d9
TH
2761 lock_map_acquire_read(&pwq->wq->lockdep_map);
2762 lock_map_release(&pwq->wq->lockdep_map);
e159489b 2763
401a8d04 2764 return true;
4690c4ab 2765already_gone:
d565ed63 2766 spin_unlock_irq(&pool->lock);
401a8d04 2767 return false;
db700897 2768}
baf59022
TH
2769
2770/**
2771 * flush_work - wait for a work to finish executing the last queueing instance
2772 * @work: the work to flush
2773 *
606a5020
TH
2774 * Wait until @work has finished execution. @work is guaranteed to be idle
2775 * on return if it hasn't been requeued since flush started.
baf59022 2776 *
d185af30 2777 * Return:
baf59022
TH
2778 * %true if flush_work() waited for the work to finish execution,
2779 * %false if it was already idle.
2780 */
2781bool flush_work(struct work_struct *work)
2782{
12997d1a
BH
2783 struct wq_barrier barr;
2784
0976dfc1
SB
2785 lock_map_acquire(&work->lockdep_map);
2786 lock_map_release(&work->lockdep_map);
2787
12997d1a
BH
2788 if (start_flush_work(work, &barr)) {
2789 wait_for_completion(&barr.done);
2790 destroy_work_on_stack(&barr.work);
2791 return true;
2792 } else {
2793 return false;
2794 }
6e84d644 2795}
606a5020 2796EXPORT_SYMBOL_GPL(flush_work);
6e84d644 2797
8603e1b3
TH
2798struct cwt_wait {
2799 wait_queue_t wait;
2800 struct work_struct *work;
2801};
2802
2803static int cwt_wakefn(wait_queue_t *wait, unsigned mode, int sync, void *key)
2804{
2805 struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait);
2806
2807 if (cwait->work != key)
2808 return 0;
2809 return autoremove_wake_function(wait, mode, sync, key);
2810}
2811
36e227d2 2812static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
1f1f642e 2813{
8603e1b3 2814 static DECLARE_WAIT_QUEUE_HEAD(cancel_waitq);
bbb68dfa 2815 unsigned long flags;
1f1f642e
ON
2816 int ret;
2817
2818 do {
bbb68dfa
TH
2819 ret = try_to_grab_pending(work, is_dwork, &flags);
2820 /*
8603e1b3
TH
2821 * If someone else is already canceling, wait for it to
2822 * finish. flush_work() doesn't work for PREEMPT_NONE
2823 * because we may get scheduled between @work's completion
2824 * and the other canceling task resuming and clearing
2825 * CANCELING - flush_work() will return false immediately
2826 * as @work is no longer busy, try_to_grab_pending() will
2827 * return -ENOENT as @work is still being canceled and the
2828 * other canceling task won't be able to clear CANCELING as
2829 * we're hogging the CPU.
2830 *
2831 * Let's wait for completion using a waitqueue. As this
2832 * may lead to the thundering herd problem, use a custom
2833 * wake function which matches @work along with exclusive
2834 * wait and wakeup.
bbb68dfa 2835 */
8603e1b3
TH
2836 if (unlikely(ret == -ENOENT)) {
2837 struct cwt_wait cwait;
2838
2839 init_wait(&cwait.wait);
2840 cwait.wait.func = cwt_wakefn;
2841 cwait.work = work;
2842
2843 prepare_to_wait_exclusive(&cancel_waitq, &cwait.wait,
2844 TASK_UNINTERRUPTIBLE);
2845 if (work_is_canceling(work))
2846 schedule();
2847 finish_wait(&cancel_waitq, &cwait.wait);
2848 }
1f1f642e
ON
2849 } while (unlikely(ret < 0));
2850
bbb68dfa
TH
2851 /* tell other tasks trying to grab @work to back off */
2852 mark_work_canceling(work);
2853 local_irq_restore(flags);
2854
606a5020 2855 flush_work(work);
7a22ad75 2856 clear_work_data(work);
8603e1b3
TH
2857
2858 /*
2859 * Paired with prepare_to_wait() above so that either
2860 * waitqueue_active() is visible here or !work_is_canceling() is
2861 * visible there.
2862 */
2863 smp_mb();
2864 if (waitqueue_active(&cancel_waitq))
2865 __wake_up(&cancel_waitq, TASK_NORMAL, 1, work);
2866
1f1f642e
ON
2867 return ret;
2868}
2869
6e84d644 2870/**
401a8d04
TH
2871 * cancel_work_sync - cancel a work and wait for it to finish
2872 * @work: the work to cancel
6e84d644 2873 *
401a8d04
TH
2874 * Cancel @work and wait for its execution to finish. This function
2875 * can be used even if the work re-queues itself or migrates to
2876 * another workqueue. On return from this function, @work is
2877 * guaranteed to be not pending or executing on any CPU.
1f1f642e 2878 *
401a8d04
TH
2879 * cancel_work_sync(&delayed_work->work) must not be used for
2880 * delayed_work's. Use cancel_delayed_work_sync() instead.
6e84d644 2881 *
401a8d04 2882 * The caller must ensure that the workqueue on which @work was last
6e84d644 2883 * queued can't be destroyed before this function returns.
401a8d04 2884 *
d185af30 2885 * Return:
401a8d04 2886 * %true if @work was pending, %false otherwise.
6e84d644 2887 */
401a8d04 2888bool cancel_work_sync(struct work_struct *work)
6e84d644 2889{
36e227d2 2890 return __cancel_work_timer(work, false);
b89deed3 2891}
28e53bdd 2892EXPORT_SYMBOL_GPL(cancel_work_sync);
b89deed3 2893
6e84d644 2894/**
401a8d04
TH
2895 * flush_delayed_work - wait for a dwork to finish executing the last queueing
2896 * @dwork: the delayed work to flush
6e84d644 2897 *
401a8d04
TH
2898 * Delayed timer is cancelled and the pending work is queued for
2899 * immediate execution. Like flush_work(), this function only
2900 * considers the last queueing instance of @dwork.
1f1f642e 2901 *
d185af30 2902 * Return:
401a8d04
TH
2903 * %true if flush_work() waited for the work to finish execution,
2904 * %false if it was already idle.
6e84d644 2905 */
401a8d04
TH
2906bool flush_delayed_work(struct delayed_work *dwork)
2907{
8930caba 2908 local_irq_disable();
401a8d04 2909 if (del_timer_sync(&dwork->timer))
60c057bc 2910 __queue_work(dwork->cpu, dwork->wq, &dwork->work);
8930caba 2911 local_irq_enable();
401a8d04
TH
2912 return flush_work(&dwork->work);
2913}
2914EXPORT_SYMBOL(flush_delayed_work);
2915
09383498 2916/**
57b30ae7
TH
2917 * cancel_delayed_work - cancel a delayed work
2918 * @dwork: delayed_work to cancel
09383498 2919 *
d185af30
YB
2920 * Kill off a pending delayed_work.
2921 *
2922 * Return: %true if @dwork was pending and canceled; %false if it wasn't
2923 * pending.
2924 *
2925 * Note:
2926 * The work callback function may still be running on return, unless
2927 * it returns %true and the work doesn't re-arm itself. Explicitly flush or
2928 * use cancel_delayed_work_sync() to wait on it.
09383498 2929 *
57b30ae7 2930 * This function is safe to call from any context including IRQ handler.
09383498 2931 */
57b30ae7 2932bool cancel_delayed_work(struct delayed_work *dwork)
09383498 2933{
57b30ae7
TH
2934 unsigned long flags;
2935 int ret;
2936
2937 do {
2938 ret = try_to_grab_pending(&dwork->work, true, &flags);
2939 } while (unlikely(ret == -EAGAIN));
2940
2941 if (unlikely(ret < 0))
2942 return false;
2943
7c3eed5c
TH
2944 set_work_pool_and_clear_pending(&dwork->work,
2945 get_work_pool_id(&dwork->work));
57b30ae7 2946 local_irq_restore(flags);
c0158ca6 2947 return ret;
09383498 2948}
57b30ae7 2949EXPORT_SYMBOL(cancel_delayed_work);
09383498 2950
401a8d04
TH
2951/**
2952 * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
2953 * @dwork: the delayed work cancel
2954 *
2955 * This is cancel_work_sync() for delayed works.
2956 *
d185af30 2957 * Return:
401a8d04
TH
2958 * %true if @dwork was pending, %false otherwise.
2959 */
2960bool cancel_delayed_work_sync(struct delayed_work *dwork)
6e84d644 2961{
36e227d2 2962 return __cancel_work_timer(&dwork->work, true);
6e84d644 2963}
f5a421a4 2964EXPORT_SYMBOL(cancel_delayed_work_sync);
1da177e4 2965
b6136773 2966/**
31ddd871 2967 * schedule_on_each_cpu - execute a function synchronously on each online CPU
b6136773 2968 * @func: the function to call
b6136773 2969 *
31ddd871
TH
2970 * schedule_on_each_cpu() executes @func on each online CPU using the
2971 * system workqueue and blocks until all CPUs have completed.
b6136773 2972 * schedule_on_each_cpu() is very slow.
31ddd871 2973 *
d185af30 2974 * Return:
31ddd871 2975 * 0 on success, -errno on failure.
b6136773 2976 */
65f27f38 2977int schedule_on_each_cpu(work_func_t func)
15316ba8
CL
2978{
2979 int cpu;
38f51568 2980 struct work_struct __percpu *works;
15316ba8 2981
b6136773
AM
2982 works = alloc_percpu(struct work_struct);
2983 if (!works)
15316ba8 2984 return -ENOMEM;
b6136773 2985
93981800
TH
2986 get_online_cpus();
2987
15316ba8 2988 for_each_online_cpu(cpu) {
9bfb1839
IM
2989 struct work_struct *work = per_cpu_ptr(works, cpu);
2990
2991 INIT_WORK(work, func);
b71ab8c2 2992 schedule_work_on(cpu, work);
65a64464 2993 }
93981800
TH
2994
2995 for_each_online_cpu(cpu)
2996 flush_work(per_cpu_ptr(works, cpu));
2997
95402b38 2998 put_online_cpus();
b6136773 2999 free_percpu(works);
15316ba8
CL
3000 return 0;
3001}
3002
1fa44eca
JB
3003/**
3004 * execute_in_process_context - reliably execute the routine with user context
3005 * @fn: the function to execute
1fa44eca
JB
3006 * @ew: guaranteed storage for the execute work structure (must
3007 * be available when the work executes)
3008 *
3009 * Executes the function immediately if process context is available,
3010 * otherwise schedules the function for delayed execution.
3011 *
d185af30 3012 * Return: 0 - function was executed
1fa44eca
JB
3013 * 1 - function was scheduled for execution
3014 */
65f27f38 3015int execute_in_process_context(work_func_t fn, struct execute_work *ew)
1fa44eca
JB
3016{
3017 if (!in_interrupt()) {
65f27f38 3018 fn(&ew->work);
1fa44eca
JB
3019 return 0;
3020 }
3021
65f27f38 3022 INIT_WORK(&ew->work, fn);
1fa44eca
JB
3023 schedule_work(&ew->work);
3024
3025 return 1;
3026}
3027EXPORT_SYMBOL_GPL(execute_in_process_context);
3028
6ba94429
FW
3029/**
3030 * free_workqueue_attrs - free a workqueue_attrs
3031 * @attrs: workqueue_attrs to free
226223ab 3032 *
6ba94429 3033 * Undo alloc_workqueue_attrs().
226223ab 3034 */
6ba94429 3035void free_workqueue_attrs(struct workqueue_attrs *attrs)
226223ab 3036{
6ba94429
FW
3037 if (attrs) {
3038 free_cpumask_var(attrs->cpumask);
3039 kfree(attrs);
3040 }
226223ab
TH
3041}
3042
6ba94429
FW
3043/**
3044 * alloc_workqueue_attrs - allocate a workqueue_attrs
3045 * @gfp_mask: allocation mask to use
3046 *
3047 * Allocate a new workqueue_attrs, initialize with default settings and
3048 * return it.
3049 *
3050 * Return: The allocated new workqueue_attr on success. %NULL on failure.
3051 */
3052struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask)
226223ab 3053{
6ba94429 3054 struct workqueue_attrs *attrs;
226223ab 3055
6ba94429
FW
3056 attrs = kzalloc(sizeof(*attrs), gfp_mask);
3057 if (!attrs)
3058 goto fail;
3059 if (!alloc_cpumask_var(&attrs->cpumask, gfp_mask))
3060 goto fail;
3061
3062 cpumask_copy(attrs->cpumask, cpu_possible_mask);
3063 return attrs;
3064fail:
3065 free_workqueue_attrs(attrs);
3066 return NULL;
226223ab
TH
3067}
3068
6ba94429
FW
3069static void copy_workqueue_attrs(struct workqueue_attrs *to,
3070 const struct workqueue_attrs *from)
226223ab 3071{
6ba94429
FW
3072 to->nice = from->nice;
3073 cpumask_copy(to->cpumask, from->cpumask);
3074 /*
3075 * Unlike hash and equality test, this function doesn't ignore
3076 * ->no_numa as it is used for both pool and wq attrs. Instead,
3077 * get_unbound_pool() explicitly clears ->no_numa after copying.
3078 */
3079 to->no_numa = from->no_numa;
226223ab
TH
3080}
3081
6ba94429
FW
3082/* hash value of the content of @attr */
3083static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
226223ab 3084{
6ba94429 3085 u32 hash = 0;
226223ab 3086
6ba94429
FW
3087 hash = jhash_1word(attrs->nice, hash);
3088 hash = jhash(cpumask_bits(attrs->cpumask),
3089 BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
3090 return hash;
226223ab 3091}
226223ab 3092
6ba94429
FW
3093/* content equality test */
3094static bool wqattrs_equal(const struct workqueue_attrs *a,
3095 const struct workqueue_attrs *b)
226223ab 3096{
6ba94429
FW
3097 if (a->nice != b->nice)
3098 return false;
3099 if (!cpumask_equal(a->cpumask, b->cpumask))
3100 return false;
3101 return true;
226223ab
TH
3102}
3103
6ba94429
FW
3104/**
3105 * init_worker_pool - initialize a newly zalloc'd worker_pool
3106 * @pool: worker_pool to initialize
3107 *
402dd89d 3108 * Initialize a newly zalloc'd @pool. It also allocates @pool->attrs.
6ba94429
FW
3109 *
3110 * Return: 0 on success, -errno on failure. Even on failure, all fields
3111 * inside @pool proper are initialized and put_unbound_pool() can be called
3112 * on @pool safely to release it.
3113 */
3114static int init_worker_pool(struct worker_pool *pool)
226223ab 3115{
6ba94429
FW
3116 spin_lock_init(&pool->lock);
3117 pool->id = -1;
3118 pool->cpu = -1;
3119 pool->node = NUMA_NO_NODE;
3120 pool->flags |= POOL_DISASSOCIATED;
82607adc 3121 pool->watchdog_ts = jiffies;
6ba94429
FW
3122 INIT_LIST_HEAD(&pool->worklist);
3123 INIT_LIST_HEAD(&pool->idle_list);
3124 hash_init(pool->busy_hash);
226223ab 3125
6ba94429
FW
3126 init_timer_deferrable(&pool->idle_timer);
3127 pool->idle_timer.function = idle_worker_timeout;
3128 pool->idle_timer.data = (unsigned long)pool;
226223ab 3129
6ba94429
FW
3130 setup_timer(&pool->mayday_timer, pool_mayday_timeout,
3131 (unsigned long)pool);
226223ab 3132
6ba94429
FW
3133 mutex_init(&pool->manager_arb);
3134 mutex_init(&pool->attach_mutex);
3135 INIT_LIST_HEAD(&pool->workers);
226223ab 3136
6ba94429
FW
3137 ida_init(&pool->worker_ida);
3138 INIT_HLIST_NODE(&pool->hash_node);
3139 pool->refcnt = 1;
226223ab 3140
6ba94429
FW
3141 /* shouldn't fail above this point */
3142 pool->attrs = alloc_workqueue_attrs(GFP_KERNEL);
3143 if (!pool->attrs)
3144 return -ENOMEM;
3145 return 0;
226223ab
TH
3146}
3147
6ba94429 3148static void rcu_free_wq(struct rcu_head *rcu)
226223ab 3149{
6ba94429
FW
3150 struct workqueue_struct *wq =
3151 container_of(rcu, struct workqueue_struct, rcu);
226223ab 3152
6ba94429
FW
3153 if (!(wq->flags & WQ_UNBOUND))
3154 free_percpu(wq->cpu_pwqs);
226223ab 3155 else
6ba94429 3156 free_workqueue_attrs(wq->unbound_attrs);
226223ab 3157
6ba94429
FW
3158 kfree(wq->rescuer);
3159 kfree(wq);
226223ab
TH
3160}
3161
6ba94429 3162static void rcu_free_pool(struct rcu_head *rcu)
226223ab 3163{
6ba94429 3164 struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
226223ab 3165
6ba94429
FW
3166 ida_destroy(&pool->worker_ida);
3167 free_workqueue_attrs(pool->attrs);
3168 kfree(pool);
226223ab
TH
3169}
3170
6ba94429
FW
3171/**
3172 * put_unbound_pool - put a worker_pool
3173 * @pool: worker_pool to put
3174 *
3175 * Put @pool. If its refcnt reaches zero, it gets destroyed in sched-RCU
3176 * safe manner. get_unbound_pool() calls this function on its failure path
3177 * and this function should be able to release pools which went through,
3178 * successfully or not, init_worker_pool().
3179 *
3180 * Should be called with wq_pool_mutex held.
3181 */
3182static void put_unbound_pool(struct worker_pool *pool)
226223ab 3183{
6ba94429
FW
3184 DECLARE_COMPLETION_ONSTACK(detach_completion);
3185 struct worker *worker;
226223ab 3186
6ba94429 3187 lockdep_assert_held(&wq_pool_mutex);
226223ab 3188
6ba94429
FW
3189 if (--pool->refcnt)
3190 return;
226223ab 3191
6ba94429
FW
3192 /* sanity checks */
3193 if (WARN_ON(!(pool->cpu < 0)) ||
3194 WARN_ON(!list_empty(&pool->worklist)))
3195 return;
226223ab 3196
6ba94429
FW
3197 /* release id and unhash */
3198 if (pool->id >= 0)
3199 idr_remove(&worker_pool_idr, pool->id);
3200 hash_del(&pool->hash_node);
d55262c4 3201
6ba94429
FW
3202 /*
3203 * Become the manager and destroy all workers. Grabbing
3204 * manager_arb prevents @pool's workers from blocking on
3205 * attach_mutex.
3206 */
3207 mutex_lock(&pool->manager_arb);
d55262c4 3208
6ba94429
FW
3209 spin_lock_irq(&pool->lock);
3210 while ((worker = first_idle_worker(pool)))
3211 destroy_worker(worker);
3212 WARN_ON(pool->nr_workers || pool->nr_idle);
3213 spin_unlock_irq(&pool->lock);
d55262c4 3214
6ba94429
FW
3215 mutex_lock(&pool->attach_mutex);
3216 if (!list_empty(&pool->workers))
3217 pool->detach_completion = &detach_completion;
3218 mutex_unlock(&pool->attach_mutex);
226223ab 3219
6ba94429
FW
3220 if (pool->detach_completion)
3221 wait_for_completion(pool->detach_completion);
226223ab 3222
6ba94429 3223 mutex_unlock(&pool->manager_arb);
226223ab 3224
6ba94429
FW
3225 /* shut down the timers */
3226 del_timer_sync(&pool->idle_timer);
3227 del_timer_sync(&pool->mayday_timer);
226223ab 3228
6ba94429
FW
3229 /* sched-RCU protected to allow dereferences from get_work_pool() */
3230 call_rcu_sched(&pool->rcu, rcu_free_pool);
226223ab
TH
3231}
3232
3233/**
6ba94429
FW
3234 * get_unbound_pool - get a worker_pool with the specified attributes
3235 * @attrs: the attributes of the worker_pool to get
226223ab 3236 *
6ba94429
FW
3237 * Obtain a worker_pool which has the same attributes as @attrs, bump the
3238 * reference count and return it. If there already is a matching
3239 * worker_pool, it will be used; otherwise, this function attempts to
3240 * create a new one.
226223ab 3241 *
6ba94429 3242 * Should be called with wq_pool_mutex held.
226223ab 3243 *
6ba94429
FW
3244 * Return: On success, a worker_pool with the same attributes as @attrs.
3245 * On failure, %NULL.
226223ab 3246 */
6ba94429 3247static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
226223ab 3248{
6ba94429
FW
3249 u32 hash = wqattrs_hash(attrs);
3250 struct worker_pool *pool;
3251 int node;
e2273584 3252 int target_node = NUMA_NO_NODE;
226223ab 3253
6ba94429 3254 lockdep_assert_held(&wq_pool_mutex);
226223ab 3255
6ba94429
FW
3256 /* do we already have a matching pool? */
3257 hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
3258 if (wqattrs_equal(pool->attrs, attrs)) {
3259 pool->refcnt++;
3260 return pool;
3261 }
3262 }
226223ab 3263
e2273584
XP
3264 /* if cpumask is contained inside a NUMA node, we belong to that node */
3265 if (wq_numa_enabled) {
3266 for_each_node(node) {
3267 if (cpumask_subset(attrs->cpumask,
3268 wq_numa_possible_cpumask[node])) {
3269 target_node = node;
3270 break;
3271 }
3272 }
3273 }
3274
6ba94429 3275 /* nope, create a new one */
e2273584 3276 pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, target_node);
6ba94429
FW
3277 if (!pool || init_worker_pool(pool) < 0)
3278 goto fail;
3279
3280 lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */
3281 copy_workqueue_attrs(pool->attrs, attrs);
e2273584 3282 pool->node = target_node;
226223ab
TH
3283
3284 /*
6ba94429
FW
3285 * no_numa isn't a worker_pool attribute, always clear it. See
3286 * 'struct workqueue_attrs' comments for detail.
226223ab 3287 */
6ba94429 3288 pool->attrs->no_numa = false;
226223ab 3289
6ba94429
FW
3290 if (worker_pool_assign_id(pool) < 0)
3291 goto fail;
226223ab 3292
6ba94429
FW
3293 /* create and start the initial worker */
3294 if (!create_worker(pool))
3295 goto fail;
226223ab 3296
6ba94429
FW
3297 /* install */
3298 hash_add(unbound_pool_hash, &pool->hash_node, hash);
226223ab 3299
6ba94429
FW
3300 return pool;
3301fail:
3302 if (pool)
3303 put_unbound_pool(pool);
3304 return NULL;
226223ab 3305}
226223ab 3306
6ba94429 3307static void rcu_free_pwq(struct rcu_head *rcu)
7a4e344c 3308{
6ba94429
FW
3309 kmem_cache_free(pwq_cache,
3310 container_of(rcu, struct pool_workqueue, rcu));
7a4e344c
TH
3311}
3312
6ba94429
FW
3313/*
3314 * Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt
3315 * and needs to be destroyed.
7a4e344c 3316 */
6ba94429 3317static void pwq_unbound_release_workfn(struct work_struct *work)
7a4e344c 3318{
6ba94429
FW
3319 struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
3320 unbound_release_work);
3321 struct workqueue_struct *wq = pwq->wq;
3322 struct worker_pool *pool = pwq->pool;
3323 bool is_last;
7a4e344c 3324
6ba94429
FW
3325 if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
3326 return;
7a4e344c 3327
6ba94429
FW
3328 mutex_lock(&wq->mutex);
3329 list_del_rcu(&pwq->pwqs_node);
3330 is_last = list_empty(&wq->pwqs);
3331 mutex_unlock(&wq->mutex);
3332
3333 mutex_lock(&wq_pool_mutex);
3334 put_unbound_pool(pool);
3335 mutex_unlock(&wq_pool_mutex);
3336
3337 call_rcu_sched(&pwq->rcu, rcu_free_pwq);
7a4e344c 3338
2865a8fb 3339 /*
6ba94429
FW
3340 * If we're the last pwq going away, @wq is already dead and no one
3341 * is gonna access it anymore. Schedule RCU free.
2865a8fb 3342 */
6ba94429
FW
3343 if (is_last)
3344 call_rcu_sched(&wq->rcu, rcu_free_wq);
29c91e99
TH
3345}
3346
7a4e344c 3347/**
6ba94429
FW
3348 * pwq_adjust_max_active - update a pwq's max_active to the current setting
3349 * @pwq: target pool_workqueue
d185af30 3350 *
6ba94429
FW
3351 * If @pwq isn't freezing, set @pwq->max_active to the associated
3352 * workqueue's saved_max_active and activate delayed work items
3353 * accordingly. If @pwq is freezing, clear @pwq->max_active to zero.
7a4e344c 3354 */
6ba94429 3355static void pwq_adjust_max_active(struct pool_workqueue *pwq)
4e1a1f9a 3356{
6ba94429
FW
3357 struct workqueue_struct *wq = pwq->wq;
3358 bool freezable = wq->flags & WQ_FREEZABLE;
4e1a1f9a 3359
6ba94429
FW
3360 /* for @wq->saved_max_active */
3361 lockdep_assert_held(&wq->mutex);
4e1a1f9a 3362
6ba94429
FW
3363 /* fast exit for non-freezable wqs */
3364 if (!freezable && pwq->max_active == wq->saved_max_active)
3365 return;
7a4e344c 3366
6ba94429 3367 spin_lock_irq(&pwq->pool->lock);
29c91e99 3368
6ba94429
FW
3369 /*
3370 * During [un]freezing, the caller is responsible for ensuring that
3371 * this function is called at least once after @workqueue_freezing
3372 * is updated and visible.
3373 */
3374 if (!freezable || !workqueue_freezing) {
3375 pwq->max_active = wq->saved_max_active;
4e1a1f9a 3376
6ba94429
FW
3377 while (!list_empty(&pwq->delayed_works) &&
3378 pwq->nr_active < pwq->max_active)
3379 pwq_activate_first_delayed(pwq);
e2dca7ad 3380
6ba94429
FW
3381 /*
3382 * Need to kick a worker after thawed or an unbound wq's
3383 * max_active is bumped. It's a slow path. Do it always.
3384 */
3385 wake_up_worker(pwq->pool);
3386 } else {
3387 pwq->max_active = 0;
3388 }
e2dca7ad 3389
6ba94429 3390 spin_unlock_irq(&pwq->pool->lock);
e2dca7ad
TH
3391}
3392
6ba94429
FW
3393/* initialize newly alloced @pwq which is associated with @wq and @pool */
3394static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
3395 struct worker_pool *pool)
29c91e99 3396{
6ba94429 3397 BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
29c91e99 3398
6ba94429
FW
3399 memset(pwq, 0, sizeof(*pwq));
3400
3401 pwq->pool = pool;
3402 pwq->wq = wq;
3403 pwq->flush_color = -1;
3404 pwq->refcnt = 1;
3405 INIT_LIST_HEAD(&pwq->delayed_works);
3406 INIT_LIST_HEAD(&pwq->pwqs_node);
3407 INIT_LIST_HEAD(&pwq->mayday_node);
3408 INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
29c91e99
TH
3409}
3410
6ba94429
FW
3411/* sync @pwq with the current state of its associated wq and link it */
3412static void link_pwq(struct pool_workqueue *pwq)
29c91e99 3413{
6ba94429 3414 struct workqueue_struct *wq = pwq->wq;
29c91e99 3415
6ba94429 3416 lockdep_assert_held(&wq->mutex);
a892cacc 3417
6ba94429
FW
3418 /* may be called multiple times, ignore if already linked */
3419 if (!list_empty(&pwq->pwqs_node))
29c91e99 3420 return;
29c91e99 3421
6ba94429
FW
3422 /* set the matching work_color */
3423 pwq->work_color = wq->work_color;
29c91e99 3424
6ba94429
FW
3425 /* sync max_active to the current setting */
3426 pwq_adjust_max_active(pwq);
29c91e99 3427
6ba94429
FW
3428 /* link in @pwq */
3429 list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
3430}
29c91e99 3431
6ba94429
FW
3432/* obtain a pool matching @attr and create a pwq associating the pool and @wq */
3433static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
3434 const struct workqueue_attrs *attrs)
3435{
3436 struct worker_pool *pool;
3437 struct pool_workqueue *pwq;
60f5a4bc 3438
6ba94429 3439 lockdep_assert_held(&wq_pool_mutex);
60f5a4bc 3440
6ba94429
FW
3441 pool = get_unbound_pool(attrs);
3442 if (!pool)
3443 return NULL;
60f5a4bc 3444
6ba94429
FW
3445 pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
3446 if (!pwq) {
3447 put_unbound_pool(pool);
3448 return NULL;
3449 }
29c91e99 3450
6ba94429
FW
3451 init_pwq(pwq, wq, pool);
3452 return pwq;
3453}
29c91e99 3454
29c91e99 3455/**
30186c6f 3456 * wq_calc_node_cpumask - calculate a wq_attrs' cpumask for the specified node
042f7df1 3457 * @attrs: the wq_attrs of the default pwq of the target workqueue
6ba94429
FW
3458 * @node: the target NUMA node
3459 * @cpu_going_down: if >= 0, the CPU to consider as offline
3460 * @cpumask: outarg, the resulting cpumask
29c91e99 3461 *
6ba94429
FW
3462 * Calculate the cpumask a workqueue with @attrs should use on @node. If
3463 * @cpu_going_down is >= 0, that cpu is considered offline during
3464 * calculation. The result is stored in @cpumask.
a892cacc 3465 *
6ba94429
FW
3466 * If NUMA affinity is not enabled, @attrs->cpumask is always used. If
3467 * enabled and @node has online CPUs requested by @attrs, the returned
3468 * cpumask is the intersection of the possible CPUs of @node and
3469 * @attrs->cpumask.
d185af30 3470 *
6ba94429
FW
3471 * The caller is responsible for ensuring that the cpumask of @node stays
3472 * stable.
3473 *
3474 * Return: %true if the resulting @cpumask is different from @attrs->cpumask,
3475 * %false if equal.
29c91e99 3476 */
6ba94429
FW
3477static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,
3478 int cpu_going_down, cpumask_t *cpumask)
29c91e99 3479{
6ba94429
FW
3480 if (!wq_numa_enabled || attrs->no_numa)
3481 goto use_dfl;
29c91e99 3482
6ba94429
FW
3483 /* does @node have any online CPUs @attrs wants? */
3484 cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);
3485 if (cpu_going_down >= 0)
3486 cpumask_clear_cpu(cpu_going_down, cpumask);
29c91e99 3487
6ba94429
FW
3488 if (cpumask_empty(cpumask))
3489 goto use_dfl;
4c16bd32
TH
3490
3491 /* yeap, return possible CPUs in @node that @attrs wants */
3492 cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]);
3493 return !cpumask_equal(cpumask, attrs->cpumask);
3494
3495use_dfl:
3496 cpumask_copy(cpumask, attrs->cpumask);
3497 return false;
3498}
3499
1befcf30
TH
3500/* install @pwq into @wq's numa_pwq_tbl[] for @node and return the old pwq */
3501static struct pool_workqueue *numa_pwq_tbl_install(struct workqueue_struct *wq,
3502 int node,
3503 struct pool_workqueue *pwq)
3504{
3505 struct pool_workqueue *old_pwq;
3506
5b95e1af 3507 lockdep_assert_held(&wq_pool_mutex);
1befcf30
TH
3508 lockdep_assert_held(&wq->mutex);
3509
3510 /* link_pwq() can handle duplicate calls */
3511 link_pwq(pwq);
3512
3513 old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
3514 rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);
3515 return old_pwq;
3516}
3517
2d5f0764
LJ
3518/* context to store the prepared attrs & pwqs before applying */
3519struct apply_wqattrs_ctx {
3520 struct workqueue_struct *wq; /* target workqueue */
3521 struct workqueue_attrs *attrs; /* attrs to apply */
042f7df1 3522 struct list_head list; /* queued for batching commit */
2d5f0764
LJ
3523 struct pool_workqueue *dfl_pwq;
3524 struct pool_workqueue *pwq_tbl[];
3525};
3526
3527/* free the resources after success or abort */
3528static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx)
3529{
3530 if (ctx) {
3531 int node;
3532
3533 for_each_node(node)
3534 put_pwq_unlocked(ctx->pwq_tbl[node]);
3535 put_pwq_unlocked(ctx->dfl_pwq);
3536
3537 free_workqueue_attrs(ctx->attrs);
3538
3539 kfree(ctx);
3540 }
3541}
3542
3543/* allocate the attrs and pwqs for later installation */
3544static struct apply_wqattrs_ctx *
3545apply_wqattrs_prepare(struct workqueue_struct *wq,
3546 const struct workqueue_attrs *attrs)
9e8cd2f5 3547{
2d5f0764 3548 struct apply_wqattrs_ctx *ctx;
4c16bd32 3549 struct workqueue_attrs *new_attrs, *tmp_attrs;
2d5f0764 3550 int node;
9e8cd2f5 3551
2d5f0764 3552 lockdep_assert_held(&wq_pool_mutex);
9e8cd2f5 3553
2d5f0764
LJ
3554 ctx = kzalloc(sizeof(*ctx) + nr_node_ids * sizeof(ctx->pwq_tbl[0]),
3555 GFP_KERNEL);
8719dcea 3556
13e2e556 3557 new_attrs = alloc_workqueue_attrs(GFP_KERNEL);
4c16bd32 3558 tmp_attrs = alloc_workqueue_attrs(GFP_KERNEL);
2d5f0764
LJ
3559 if (!ctx || !new_attrs || !tmp_attrs)
3560 goto out_free;
13e2e556 3561
042f7df1
LJ
3562 /*
3563 * Calculate the attrs of the default pwq.
3564 * If the user configured cpumask doesn't overlap with the
3565 * wq_unbound_cpumask, we fallback to the wq_unbound_cpumask.
3566 */
13e2e556 3567 copy_workqueue_attrs(new_attrs, attrs);
b05a7928 3568 cpumask_and(new_attrs->cpumask, new_attrs->cpumask, wq_unbound_cpumask);
042f7df1
LJ
3569 if (unlikely(cpumask_empty(new_attrs->cpumask)))
3570 cpumask_copy(new_attrs->cpumask, wq_unbound_cpumask);
13e2e556 3571
4c16bd32
TH
3572 /*
3573 * We may create multiple pwqs with differing cpumasks. Make a
3574 * copy of @new_attrs which will be modified and used to obtain
3575 * pools.
3576 */
3577 copy_workqueue_attrs(tmp_attrs, new_attrs);
3578
4c16bd32
TH
3579 /*
3580 * If something goes wrong during CPU up/down, we'll fall back to
3581 * the default pwq covering whole @attrs->cpumask. Always create
3582 * it even if we don't use it immediately.
3583 */
2d5f0764
LJ
3584 ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
3585 if (!ctx->dfl_pwq)
3586 goto out_free;
4c16bd32
TH
3587
3588 for_each_node(node) {
042f7df1 3589 if (wq_calc_node_cpumask(new_attrs, node, -1, tmp_attrs->cpumask)) {
2d5f0764
LJ
3590 ctx->pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);
3591 if (!ctx->pwq_tbl[node])
3592 goto out_free;
4c16bd32 3593 } else {
2d5f0764
LJ
3594 ctx->dfl_pwq->refcnt++;
3595 ctx->pwq_tbl[node] = ctx->dfl_pwq;
4c16bd32
TH
3596 }
3597 }
3598
042f7df1
LJ
3599 /* save the user configured attrs and sanitize it. */
3600 copy_workqueue_attrs(new_attrs, attrs);
3601 cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
2d5f0764 3602 ctx->attrs = new_attrs;
042f7df1 3603
2d5f0764
LJ
3604 ctx->wq = wq;
3605 free_workqueue_attrs(tmp_attrs);
3606 return ctx;
3607
3608out_free:
3609 free_workqueue_attrs(tmp_attrs);
3610 free_workqueue_attrs(new_attrs);
3611 apply_wqattrs_cleanup(ctx);
3612 return NULL;
3613}
3614
3615/* set attrs and install prepared pwqs, @ctx points to old pwqs on return */
3616static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
3617{
3618 int node;
9e8cd2f5 3619
4c16bd32 3620 /* all pwqs have been created successfully, let's install'em */
2d5f0764 3621 mutex_lock(&ctx->wq->mutex);
a892cacc 3622
2d5f0764 3623 copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
4c16bd32
TH
3624
3625 /* save the previous pwq and install the new one */
f147f29e 3626 for_each_node(node)
2d5f0764
LJ
3627 ctx->pwq_tbl[node] = numa_pwq_tbl_install(ctx->wq, node,
3628 ctx->pwq_tbl[node]);
4c16bd32
TH
3629
3630 /* @dfl_pwq might not have been used, ensure it's linked */
2d5f0764
LJ
3631 link_pwq(ctx->dfl_pwq);
3632 swap(ctx->wq->dfl_pwq, ctx->dfl_pwq);
f147f29e 3633
2d5f0764
LJ
3634 mutex_unlock(&ctx->wq->mutex);
3635}
9e8cd2f5 3636
a0111cf6
LJ
3637static void apply_wqattrs_lock(void)
3638{
3639 /* CPUs should stay stable across pwq creations and installations */
3640 get_online_cpus();
3641 mutex_lock(&wq_pool_mutex);
3642}
3643
3644static void apply_wqattrs_unlock(void)
3645{
3646 mutex_unlock(&wq_pool_mutex);
3647 put_online_cpus();
3648}
3649
3650static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
3651 const struct workqueue_attrs *attrs)
2d5f0764
LJ
3652{
3653 struct apply_wqattrs_ctx *ctx;
4c16bd32 3654
2d5f0764
LJ
3655 /* only unbound workqueues can change attributes */
3656 if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
3657 return -EINVAL;
13e2e556 3658
2d5f0764
LJ
3659 /* creating multiple pwqs breaks ordering guarantee */
3660 if (WARN_ON((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs)))
3661 return -EINVAL;
3662
2d5f0764 3663 ctx = apply_wqattrs_prepare(wq, attrs);
6201171e 3664 if (!ctx)
3665 return -ENOMEM;
2d5f0764
LJ
3666
3667 /* the ctx has been prepared successfully, let's commit it */
6201171e 3668 apply_wqattrs_commit(ctx);
2d5f0764
LJ
3669 apply_wqattrs_cleanup(ctx);
3670
6201171e 3671 return 0;
9e8cd2f5
TH
3672}
3673
a0111cf6
LJ
3674/**
3675 * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
3676 * @wq: the target workqueue
3677 * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
3678 *
3679 * Apply @attrs to an unbound workqueue @wq. Unless disabled, on NUMA
3680 * machines, this function maps a separate pwq to each NUMA node with
3681 * possibles CPUs in @attrs->cpumask so that work items are affine to the
3682 * NUMA node it was issued on. Older pwqs are released as in-flight work
3683 * items finish. Note that a work item which repeatedly requeues itself
3684 * back-to-back will stay on its current pwq.
3685 *
3686 * Performs GFP_KERNEL allocations.
3687 *
3688 * Return: 0 on success and -errno on failure.
3689 */
3690int apply_workqueue_attrs(struct workqueue_struct *wq,
3691 const struct workqueue_attrs *attrs)
3692{
3693 int ret;
3694
3695 apply_wqattrs_lock();
3696 ret = apply_workqueue_attrs_locked(wq, attrs);
3697 apply_wqattrs_unlock();
3698
3699 return ret;
3700}
3701
4c16bd32
TH
3702/**
3703 * wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
3704 * @wq: the target workqueue
3705 * @cpu: the CPU coming up or going down
3706 * @online: whether @cpu is coming up or going down
3707 *
3708 * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
3709 * %CPU_DOWN_FAILED. @cpu is being hot[un]plugged, update NUMA affinity of
3710 * @wq accordingly.
3711 *
3712 * If NUMA affinity can't be adjusted due to memory allocation failure, it
3713 * falls back to @wq->dfl_pwq which may not be optimal but is always
3714 * correct.
3715 *
3716 * Note that when the last allowed CPU of a NUMA node goes offline for a
3717 * workqueue with a cpumask spanning multiple nodes, the workers which were
3718 * already executing the work items for the workqueue will lose their CPU
3719 * affinity and may execute on any CPU. This is similar to how per-cpu
3720 * workqueues behave on CPU_DOWN. If a workqueue user wants strict
3721 * affinity, it's the user's responsibility to flush the work item from
3722 * CPU_DOWN_PREPARE.
3723 */
3724static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
3725 bool online)
3726{
3727 int node = cpu_to_node(cpu);
3728 int cpu_off = online ? -1 : cpu;
3729 struct pool_workqueue *old_pwq = NULL, *pwq;
3730 struct workqueue_attrs *target_attrs;
3731 cpumask_t *cpumask;
3732
3733 lockdep_assert_held(&wq_pool_mutex);
3734
f7142ed4
LJ
3735 if (!wq_numa_enabled || !(wq->flags & WQ_UNBOUND) ||
3736 wq->unbound_attrs->no_numa)
4c16bd32
TH
3737 return;
3738
3739 /*
3740 * We don't wanna alloc/free wq_attrs for each wq for each CPU.
3741 * Let's use a preallocated one. The following buf is protected by
3742 * CPU hotplug exclusion.
3743 */
3744 target_attrs = wq_update_unbound_numa_attrs_buf;
3745 cpumask = target_attrs->cpumask;
3746
4c16bd32
TH
3747 copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
3748 pwq = unbound_pwq_by_node(wq, node);
3749
3750 /*
3751 * Let's determine what needs to be done. If the target cpumask is
042f7df1
LJ
3752 * different from the default pwq's, we need to compare it to @pwq's
3753 * and create a new one if they don't match. If the target cpumask
3754 * equals the default pwq's, the default pwq should be used.
4c16bd32 3755 */
042f7df1 3756 if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, cpu_off, cpumask)) {
4c16bd32 3757 if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
f7142ed4 3758 return;
4c16bd32 3759 } else {
534a3fbb 3760 goto use_dfl_pwq;
4c16bd32
TH
3761 }
3762
4c16bd32
TH
3763 /* create a new pwq */
3764 pwq = alloc_unbound_pwq(wq, target_attrs);
3765 if (!pwq) {
2d916033
FF
3766 pr_warn("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
3767 wq->name);
77f300b1 3768 goto use_dfl_pwq;
4c16bd32
TH
3769 }
3770
f7142ed4 3771 /* Install the new pwq. */
4c16bd32
TH
3772 mutex_lock(&wq->mutex);
3773 old_pwq = numa_pwq_tbl_install(wq, node, pwq);
3774 goto out_unlock;
3775
3776use_dfl_pwq:
f7142ed4 3777 mutex_lock(&wq->mutex);
4c16bd32
TH
3778 spin_lock_irq(&wq->dfl_pwq->pool->lock);
3779 get_pwq(wq->dfl_pwq);
3780 spin_unlock_irq(&wq->dfl_pwq->pool->lock);
3781 old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
3782out_unlock:
3783 mutex_unlock(&wq->mutex);
3784 put_pwq_unlocked(old_pwq);
3785}
3786
30cdf249 3787static int alloc_and_link_pwqs(struct workqueue_struct *wq)
0f900049 3788{
49e3cf44 3789 bool highpri = wq->flags & WQ_HIGHPRI;
8a2b7538 3790 int cpu, ret;
30cdf249
TH
3791
3792 if (!(wq->flags & WQ_UNBOUND)) {
420c0ddb
TH
3793 wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
3794 if (!wq->cpu_pwqs)
30cdf249
TH
3795 return -ENOMEM;
3796
3797 for_each_possible_cpu(cpu) {
7fb98ea7
TH
3798 struct pool_workqueue *pwq =
3799 per_cpu_ptr(wq->cpu_pwqs, cpu);
7a62c2c8 3800 struct worker_pool *cpu_pools =
f02ae73a 3801 per_cpu(cpu_worker_pools, cpu);
f3421797 3802
f147f29e
TH
3803 init_pwq(pwq, wq, &cpu_pools[highpri]);
3804
3805 mutex_lock(&wq->mutex);
1befcf30 3806 link_pwq(pwq);
f147f29e 3807 mutex_unlock(&wq->mutex);
30cdf249 3808 }
9e8cd2f5 3809 return 0;
8a2b7538
TH
3810 } else if (wq->flags & __WQ_ORDERED) {
3811 ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
3812 /* there should only be single pwq for ordering guarantee */
3813 WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node ||
3814 wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
3815 "ordering guarantee broken for workqueue %s\n", wq->name);
3816 return ret;
30cdf249 3817 } else {
9e8cd2f5 3818 return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
30cdf249 3819 }
0f900049
TH
3820}
3821
f3421797
TH
3822static int wq_clamp_max_active(int max_active, unsigned int flags,
3823 const char *name)
b71ab8c2 3824{
f3421797
TH
3825 int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
3826
3827 if (max_active < 1 || max_active > lim)
044c782c
VI
3828 pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
3829 max_active, name, 1, lim);
b71ab8c2 3830
f3421797 3831 return clamp_val(max_active, 1, lim);
b71ab8c2
TH
3832}
3833
b196be89 3834struct workqueue_struct *__alloc_workqueue_key(const char *fmt,
d320c038
TH
3835 unsigned int flags,
3836 int max_active,
3837 struct lock_class_key *key,
b196be89 3838 const char *lock_name, ...)
1da177e4 3839{
df2d5ae4 3840 size_t tbl_size = 0;
ecf6881f 3841 va_list args;
1da177e4 3842 struct workqueue_struct *wq;
49e3cf44 3843 struct pool_workqueue *pwq;
b196be89 3844
cee22a15
VK
3845 /* see the comment above the definition of WQ_POWER_EFFICIENT */
3846 if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
3847 flags |= WQ_UNBOUND;
3848
ecf6881f 3849 /* allocate wq and format name */
df2d5ae4 3850 if (flags & WQ_UNBOUND)
ddcb57e2 3851 tbl_size = nr_node_ids * sizeof(wq->numa_pwq_tbl[0]);
df2d5ae4
TH
3852
3853 wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
b196be89 3854 if (!wq)
d2c1d404 3855 return NULL;
b196be89 3856
6029a918
TH
3857 if (flags & WQ_UNBOUND) {
3858 wq->unbound_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3859 if (!wq->unbound_attrs)
3860 goto err_free_wq;
3861 }
3862
ecf6881f
TH
3863 va_start(args, lock_name);
3864 vsnprintf(wq->name, sizeof(wq->name), fmt, args);
b196be89 3865 va_end(args);
1da177e4 3866
d320c038 3867 max_active = max_active ?: WQ_DFL_ACTIVE;
b196be89 3868 max_active = wq_clamp_max_active(max_active, flags, wq->name);
3af24433 3869
b196be89 3870 /* init wq */
97e37d7b 3871 wq->flags = flags;
a0a1a5fd 3872 wq->saved_max_active = max_active;
3c25a55d 3873 mutex_init(&wq->mutex);
112202d9 3874 atomic_set(&wq->nr_pwqs_to_flush, 0);
30cdf249 3875 INIT_LIST_HEAD(&wq->pwqs);
73f53c4a
TH
3876 INIT_LIST_HEAD(&wq->flusher_queue);
3877 INIT_LIST_HEAD(&wq->flusher_overflow);
493a1724 3878 INIT_LIST_HEAD(&wq->maydays);
502ca9d8 3879
eb13ba87 3880 lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
cce1a165 3881 INIT_LIST_HEAD(&wq->list);
3af24433 3882
30cdf249 3883 if (alloc_and_link_pwqs(wq) < 0)
d2c1d404 3884 goto err_free_wq;
1537663f 3885
493008a8
TH
3886 /*
3887 * Workqueues which may be used during memory reclaim should
3888 * have a rescuer to guarantee forward progress.
3889 */
3890 if (flags & WQ_MEM_RECLAIM) {
e22bee78
TH
3891 struct worker *rescuer;
3892
f7537df5 3893 rescuer = alloc_worker(NUMA_NO_NODE);
e22bee78 3894 if (!rescuer)
d2c1d404 3895 goto err_destroy;
e22bee78 3896
111c225a
TH
3897 rescuer->rescue_wq = wq;
3898 rescuer->task = kthread_create(rescuer_thread, rescuer, "%s",
b196be89 3899 wq->name);
d2c1d404
TH
3900 if (IS_ERR(rescuer->task)) {
3901 kfree(rescuer);
3902 goto err_destroy;
3903 }
e22bee78 3904
d2c1d404 3905 wq->rescuer = rescuer;
25834c73 3906 kthread_bind_mask(rescuer->task, cpu_possible_mask);
e22bee78 3907 wake_up_process(rescuer->task);
3af24433
ON
3908 }
3909
226223ab
TH
3910 if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
3911 goto err_destroy;
3912
a0a1a5fd 3913 /*
68e13a67
LJ
3914 * wq_pool_mutex protects global freeze state and workqueues list.
3915 * Grab it, adjust max_active and add the new @wq to workqueues
3916 * list.
a0a1a5fd 3917 */
68e13a67 3918 mutex_lock(&wq_pool_mutex);
a0a1a5fd 3919
a357fc03 3920 mutex_lock(&wq->mutex);
699ce097
TH
3921 for_each_pwq(pwq, wq)
3922 pwq_adjust_max_active(pwq);
a357fc03 3923 mutex_unlock(&wq->mutex);
a0a1a5fd 3924
e2dca7ad 3925 list_add_tail_rcu(&wq->list, &workqueues);
a0a1a5fd 3926
68e13a67 3927 mutex_unlock(&wq_pool_mutex);
1537663f 3928
3af24433 3929 return wq;
d2c1d404
TH
3930
3931err_free_wq:
6029a918 3932 free_workqueue_attrs(wq->unbound_attrs);
d2c1d404
TH
3933 kfree(wq);
3934 return NULL;
3935err_destroy:
3936 destroy_workqueue(wq);
4690c4ab 3937 return NULL;
3af24433 3938}
d320c038 3939EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
1da177e4 3940
3af24433
ON
3941/**
3942 * destroy_workqueue - safely terminate a workqueue
3943 * @wq: target workqueue
3944 *
3945 * Safely destroy a workqueue. All work currently pending will be done first.
3946 */
3947void destroy_workqueue(struct workqueue_struct *wq)
3948{
49e3cf44 3949 struct pool_workqueue *pwq;
4c16bd32 3950 int node;
3af24433 3951
9c5a2ba7
TH
3952 /* drain it before proceeding with destruction */
3953 drain_workqueue(wq);
c8efcc25 3954
6183c009 3955 /* sanity checks */
b09f4fd3 3956 mutex_lock(&wq->mutex);
49e3cf44 3957 for_each_pwq(pwq, wq) {
6183c009
TH
3958 int i;
3959
76af4d93
TH
3960 for (i = 0; i < WORK_NR_COLORS; i++) {
3961 if (WARN_ON(pwq->nr_in_flight[i])) {
b09f4fd3 3962 mutex_unlock(&wq->mutex);
6183c009 3963 return;
76af4d93
TH
3964 }
3965 }
3966
5c529597 3967 if (WARN_ON((pwq != wq->dfl_pwq) && (pwq->refcnt > 1)) ||
8864b4e5 3968 WARN_ON(pwq->nr_active) ||
76af4d93 3969 WARN_ON(!list_empty(&pwq->delayed_works))) {
b09f4fd3 3970 mutex_unlock(&wq->mutex);
6183c009 3971 return;
76af4d93 3972 }
6183c009 3973 }
b09f4fd3 3974 mutex_unlock(&wq->mutex);
6183c009 3975
a0a1a5fd
TH
3976 /*
3977 * wq list is used to freeze wq, remove from list after
3978 * flushing is complete in case freeze races us.
3979 */
68e13a67 3980 mutex_lock(&wq_pool_mutex);
e2dca7ad 3981 list_del_rcu(&wq->list);
68e13a67 3982 mutex_unlock(&wq_pool_mutex);
3af24433 3983
226223ab
TH
3984 workqueue_sysfs_unregister(wq);
3985
e2dca7ad 3986 if (wq->rescuer)
e22bee78 3987 kthread_stop(wq->rescuer->task);
e22bee78 3988
8864b4e5
TH
3989 if (!(wq->flags & WQ_UNBOUND)) {
3990 /*
3991 * The base ref is never dropped on per-cpu pwqs. Directly
e2dca7ad 3992 * schedule RCU free.
8864b4e5 3993 */
e2dca7ad 3994 call_rcu_sched(&wq->rcu, rcu_free_wq);
8864b4e5
TH
3995 } else {
3996 /*
3997 * We're the sole accessor of @wq at this point. Directly
4c16bd32
TH
3998 * access numa_pwq_tbl[] and dfl_pwq to put the base refs.
3999 * @wq will be freed when the last pwq is released.
8864b4e5 4000 */
4c16bd32
TH
4001 for_each_node(node) {
4002 pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
4003 RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL);
4004 put_pwq_unlocked(pwq);
4005 }
4006
4007 /*
4008 * Put dfl_pwq. @wq may be freed any time after dfl_pwq is
4009 * put. Don't access it afterwards.
4010 */
4011 pwq = wq->dfl_pwq;
4012 wq->dfl_pwq = NULL;
dce90d47 4013 put_pwq_unlocked(pwq);
29c91e99 4014 }
3af24433
ON
4015}
4016EXPORT_SYMBOL_GPL(destroy_workqueue);
4017
dcd989cb
TH
4018/**
4019 * workqueue_set_max_active - adjust max_active of a workqueue
4020 * @wq: target workqueue
4021 * @max_active: new max_active value.
4022 *
4023 * Set max_active of @wq to @max_active.
4024 *
4025 * CONTEXT:
4026 * Don't call from IRQ context.
4027 */
4028void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
4029{
49e3cf44 4030 struct pool_workqueue *pwq;
dcd989cb 4031
8719dcea
TH
4032 /* disallow meddling with max_active for ordered workqueues */
4033 if (WARN_ON(wq->flags & __WQ_ORDERED))
4034 return;
4035
f3421797 4036 max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
dcd989cb 4037
a357fc03 4038 mutex_lock(&wq->mutex);
dcd989cb
TH
4039
4040 wq->saved_max_active = max_active;
4041
699ce097
TH
4042 for_each_pwq(pwq, wq)
4043 pwq_adjust_max_active(pwq);
93981800 4044
a357fc03 4045 mutex_unlock(&wq->mutex);
15316ba8 4046}
dcd989cb 4047EXPORT_SYMBOL_GPL(workqueue_set_max_active);
15316ba8 4048
e6267616
TH
4049/**
4050 * current_is_workqueue_rescuer - is %current workqueue rescuer?
4051 *
4052 * Determine whether %current is a workqueue rescuer. Can be used from
4053 * work functions to determine whether it's being run off the rescuer task.
d185af30
YB
4054 *
4055 * Return: %true if %current is a workqueue rescuer. %false otherwise.
e6267616
TH
4056 */
4057bool current_is_workqueue_rescuer(void)
4058{
4059 struct worker *worker = current_wq_worker();
4060
6a092dfd 4061 return worker && worker->rescue_wq;
e6267616
TH
4062}
4063
eef6a7d5 4064/**
dcd989cb
TH
4065 * workqueue_congested - test whether a workqueue is congested
4066 * @cpu: CPU in question
4067 * @wq: target workqueue
eef6a7d5 4068 *
dcd989cb
TH
4069 * Test whether @wq's cpu workqueue for @cpu is congested. There is
4070 * no synchronization around this function and the test result is
4071 * unreliable and only useful as advisory hints or for debugging.
eef6a7d5 4072 *
d3251859
TH
4073 * If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
4074 * Note that both per-cpu and unbound workqueues may be associated with
4075 * multiple pool_workqueues which have separate congested states. A
4076 * workqueue being congested on one CPU doesn't mean the workqueue is also
4077 * contested on other CPUs / NUMA nodes.
4078 *
d185af30 4079 * Return:
dcd989cb 4080 * %true if congested, %false otherwise.
eef6a7d5 4081 */
d84ff051 4082bool workqueue_congested(int cpu, struct workqueue_struct *wq)
1da177e4 4083{
7fb98ea7 4084 struct pool_workqueue *pwq;
76af4d93
TH
4085 bool ret;
4086
88109453 4087 rcu_read_lock_sched();
7fb98ea7 4088
d3251859
TH
4089 if (cpu == WORK_CPU_UNBOUND)
4090 cpu = smp_processor_id();
4091
7fb98ea7
TH
4092 if (!(wq->flags & WQ_UNBOUND))
4093 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
4094 else
df2d5ae4 4095 pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
dcd989cb 4096
76af4d93 4097 ret = !list_empty(&pwq->delayed_works);
88109453 4098 rcu_read_unlock_sched();
76af4d93
TH
4099
4100 return ret;
1da177e4 4101}
dcd989cb 4102EXPORT_SYMBOL_GPL(workqueue_congested);
1da177e4 4103
dcd989cb
TH
4104/**
4105 * work_busy - test whether a work is currently pending or running
4106 * @work: the work to be tested
4107 *
4108 * Test whether @work is currently pending or running. There is no
4109 * synchronization around this function and the test result is
4110 * unreliable and only useful as advisory hints or for debugging.
dcd989cb 4111 *
d185af30 4112 * Return:
dcd989cb
TH
4113 * OR'd bitmask of WORK_BUSY_* bits.
4114 */
4115unsigned int work_busy(struct work_struct *work)
1da177e4 4116{
fa1b54e6 4117 struct worker_pool *pool;
dcd989cb
TH
4118 unsigned long flags;
4119 unsigned int ret = 0;
1da177e4 4120
dcd989cb
TH
4121 if (work_pending(work))
4122 ret |= WORK_BUSY_PENDING;
1da177e4 4123
fa1b54e6
TH
4124 local_irq_save(flags);
4125 pool = get_work_pool(work);
038366c5 4126 if (pool) {
fa1b54e6 4127 spin_lock(&pool->lock);
038366c5
LJ
4128 if (find_worker_executing_work(pool, work))
4129 ret |= WORK_BUSY_RUNNING;
fa1b54e6 4130 spin_unlock(&pool->lock);
038366c5 4131 }
fa1b54e6 4132 local_irq_restore(flags);
1da177e4 4133
dcd989cb 4134 return ret;
1da177e4 4135}
dcd989cb 4136EXPORT_SYMBOL_GPL(work_busy);
1da177e4 4137
3d1cb205
TH
4138/**
4139 * set_worker_desc - set description for the current work item
4140 * @fmt: printf-style format string
4141 * @...: arguments for the format string
4142 *
4143 * This function can be called by a running work function to describe what
4144 * the work item is about. If the worker task gets dumped, this
4145 * information will be printed out together to help debugging. The
4146 * description can be at most WORKER_DESC_LEN including the trailing '\0'.
4147 */
4148void set_worker_desc(const char *fmt, ...)
4149{
4150 struct worker *worker = current_wq_worker();
4151 va_list args;
4152
4153 if (worker) {
4154 va_start(args, fmt);
4155 vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
4156 va_end(args);
4157 worker->desc_valid = true;
4158 }
4159}
4160
4161/**
4162 * print_worker_info - print out worker information and description
4163 * @log_lvl: the log level to use when printing
4164 * @task: target task
4165 *
4166 * If @task is a worker and currently executing a work item, print out the
4167 * name of the workqueue being serviced and worker description set with
4168 * set_worker_desc() by the currently executing work item.
4169 *
4170 * This function can be safely called on any task as long as the
4171 * task_struct itself is accessible. While safe, this function isn't
4172 * synchronized and may print out mixups or garbages of limited length.
4173 */
4174void print_worker_info(const char *log_lvl, struct task_struct *task)
4175{
4176 work_func_t *fn = NULL;
4177 char name[WQ_NAME_LEN] = { };
4178 char desc[WORKER_DESC_LEN] = { };
4179 struct pool_workqueue *pwq = NULL;
4180 struct workqueue_struct *wq = NULL;
4181 bool desc_valid = false;
4182 struct worker *worker;
4183
4184 if (!(task->flags & PF_WQ_WORKER))
4185 return;
4186
4187 /*
4188 * This function is called without any synchronization and @task
4189 * could be in any state. Be careful with dereferences.
4190 */
4191 worker = probe_kthread_data(task);
4192
4193 /*
4194 * Carefully copy the associated workqueue's workfn and name. Keep
4195 * the original last '\0' in case the original contains garbage.
4196 */
4197 probe_kernel_read(&fn, &worker->current_func, sizeof(fn));
4198 probe_kernel_read(&pwq, &worker->current_pwq, sizeof(pwq));
4199 probe_kernel_read(&wq, &pwq->wq, sizeof(wq));
4200 probe_kernel_read(name, wq->name, sizeof(name) - 1);
4201
4202 /* copy worker description */
4203 probe_kernel_read(&desc_valid, &worker->desc_valid, sizeof(desc_valid));
4204 if (desc_valid)
4205 probe_kernel_read(desc, worker->desc, sizeof(desc) - 1);
4206
4207 if (fn || name[0] || desc[0]) {
4208 printk("%sWorkqueue: %s %pf", log_lvl, name, fn);
4209 if (desc[0])
4210 pr_cont(" (%s)", desc);
4211 pr_cont("\n");
4212 }
4213}
4214
3494fc30
TH
4215static void pr_cont_pool_info(struct worker_pool *pool)
4216{
4217 pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask);
4218 if (pool->node != NUMA_NO_NODE)
4219 pr_cont(" node=%d", pool->node);
4220 pr_cont(" flags=0x%x nice=%d", pool->flags, pool->attrs->nice);
4221}
4222
4223static void pr_cont_work(bool comma, struct work_struct *work)
4224{
4225 if (work->func == wq_barrier_func) {
4226 struct wq_barrier *barr;
4227
4228 barr = container_of(work, struct wq_barrier, work);
4229
4230 pr_cont("%s BAR(%d)", comma ? "," : "",
4231 task_pid_nr(barr->task));
4232 } else {
4233 pr_cont("%s %pf", comma ? "," : "", work->func);
4234 }
4235}
4236
4237static void show_pwq(struct pool_workqueue *pwq)
4238{
4239 struct worker_pool *pool = pwq->pool;
4240 struct work_struct *work;
4241 struct worker *worker;
4242 bool has_in_flight = false, has_pending = false;
4243 int bkt;
4244
4245 pr_info(" pwq %d:", pool->id);
4246 pr_cont_pool_info(pool);
4247
4248 pr_cont(" active=%d/%d%s\n", pwq->nr_active, pwq->max_active,
4249 !list_empty(&pwq->mayday_node) ? " MAYDAY" : "");
4250
4251 hash_for_each(pool->busy_hash, bkt, worker, hentry) {
4252 if (worker->current_pwq == pwq) {
4253 has_in_flight = true;
4254 break;
4255 }
4256 }
4257 if (has_in_flight) {
4258 bool comma = false;
4259
4260 pr_info(" in-flight:");
4261 hash_for_each(pool->busy_hash, bkt, worker, hentry) {
4262 if (worker->current_pwq != pwq)
4263 continue;
4264
4265 pr_cont("%s %d%s:%pf", comma ? "," : "",
4266 task_pid_nr(worker->task),
4267 worker == pwq->wq->rescuer ? "(RESCUER)" : "",
4268 worker->current_func);
4269 list_for_each_entry(work, &worker->scheduled, entry)
4270 pr_cont_work(false, work);
4271 comma = true;
4272 }
4273 pr_cont("\n");
4274 }
4275
4276 list_for_each_entry(work, &pool->worklist, entry) {
4277 if (get_work_pwq(work) == pwq) {
4278 has_pending = true;
4279 break;
4280 }
4281 }
4282 if (has_pending) {
4283 bool comma = false;
4284
4285 pr_info(" pending:");
4286 list_for_each_entry(work, &pool->worklist, entry) {
4287 if (get_work_pwq(work) != pwq)
4288 continue;
4289
4290 pr_cont_work(comma, work);
4291 comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
4292 }
4293 pr_cont("\n");
4294 }
4295
4296 if (!list_empty(&pwq->delayed_works)) {
4297 bool comma = false;
4298
4299 pr_info(" delayed:");
4300 list_for_each_entry(work, &pwq->delayed_works, entry) {
4301 pr_cont_work(comma, work);
4302 comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
4303 }
4304 pr_cont("\n");
4305 }
4306}
4307
4308/**
4309 * show_workqueue_state - dump workqueue state
4310 *
4311 * Called from a sysrq handler and prints out all busy workqueues and
4312 * pools.
4313 */
4314void show_workqueue_state(void)
4315{
4316 struct workqueue_struct *wq;
4317 struct worker_pool *pool;
4318 unsigned long flags;
4319 int pi;
4320
4321 rcu_read_lock_sched();
4322
4323 pr_info("Showing busy workqueues and worker pools:\n");
4324
4325 list_for_each_entry_rcu(wq, &workqueues, list) {
4326 struct pool_workqueue *pwq;
4327 bool idle = true;
4328
4329 for_each_pwq(pwq, wq) {
4330 if (pwq->nr_active || !list_empty(&pwq->delayed_works)) {
4331 idle = false;
4332 break;
4333 }
4334 }
4335 if (idle)
4336 continue;
4337
4338 pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);
4339
4340 for_each_pwq(pwq, wq) {
4341 spin_lock_irqsave(&pwq->pool->lock, flags);
4342 if (pwq->nr_active || !list_empty(&pwq->delayed_works))
4343 show_pwq(pwq);
4344 spin_unlock_irqrestore(&pwq->pool->lock, flags);
4345 }
4346 }
4347
4348 for_each_pool(pool, pi) {
4349 struct worker *worker;
4350 bool first = true;
4351
4352 spin_lock_irqsave(&pool->lock, flags);
4353 if (pool->nr_workers == pool->nr_idle)
4354 goto next_pool;
4355
4356 pr_info("pool %d:", pool->id);
4357 pr_cont_pool_info(pool);
82607adc
TH
4358 pr_cont(" hung=%us workers=%d",
4359 jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000,
4360 pool->nr_workers);
3494fc30
TH
4361 if (pool->manager)
4362 pr_cont(" manager: %d",
4363 task_pid_nr(pool->manager->task));
4364 list_for_each_entry(worker, &pool->idle_list, entry) {
4365 pr_cont(" %s%d", first ? "idle: " : "",
4366 task_pid_nr(worker->task));
4367 first = false;
4368 }
4369 pr_cont("\n");
4370 next_pool:
4371 spin_unlock_irqrestore(&pool->lock, flags);
4372 }
4373
4374 rcu_read_unlock_sched();
4375}
4376
db7bccf4
TH
4377/*
4378 * CPU hotplug.
4379 *
e22bee78 4380 * There are two challenges in supporting CPU hotplug. Firstly, there
112202d9 4381 * are a lot of assumptions on strong associations among work, pwq and
706026c2 4382 * pool which make migrating pending and scheduled works very
e22bee78 4383 * difficult to implement without impacting hot paths. Secondly,
94cf58bb 4384 * worker pools serve mix of short, long and very long running works making
e22bee78
TH
4385 * blocked draining impractical.
4386 *
24647570 4387 * This is solved by allowing the pools to be disassociated from the CPU
628c78e7
TH
4388 * running as an unbound one and allowing it to be reattached later if the
4389 * cpu comes back online.
db7bccf4 4390 */
1da177e4 4391
706026c2 4392static void wq_unbind_fn(struct work_struct *work)
3af24433 4393{
38db41d9 4394 int cpu = smp_processor_id();
4ce62e9e 4395 struct worker_pool *pool;
db7bccf4 4396 struct worker *worker;
3af24433 4397
f02ae73a 4398 for_each_cpu_worker_pool(pool, cpu) {
92f9c5c4 4399 mutex_lock(&pool->attach_mutex);
94cf58bb 4400 spin_lock_irq(&pool->lock);
3af24433 4401
94cf58bb 4402 /*
92f9c5c4 4403 * We've blocked all attach/detach operations. Make all workers
94cf58bb
TH
4404 * unbound and set DISASSOCIATED. Before this, all workers
4405 * except for the ones which are still executing works from
4406 * before the last CPU down must be on the cpu. After
4407 * this, they may become diasporas.
4408 */
da028469 4409 for_each_pool_worker(worker, pool)
c9e7cf27 4410 worker->flags |= WORKER_UNBOUND;
06ba38a9 4411
24647570 4412 pool->flags |= POOL_DISASSOCIATED;
f2d5a0ee 4413
94cf58bb 4414 spin_unlock_irq(&pool->lock);
92f9c5c4 4415 mutex_unlock(&pool->attach_mutex);
628c78e7 4416
eb283428
LJ
4417 /*
4418 * Call schedule() so that we cross rq->lock and thus can
4419 * guarantee sched callbacks see the %WORKER_UNBOUND flag.
4420 * This is necessary as scheduler callbacks may be invoked
4421 * from other cpus.
4422 */
4423 schedule();
06ba38a9 4424
eb283428
LJ
4425 /*
4426 * Sched callbacks are disabled now. Zap nr_running.
4427 * After this, nr_running stays zero and need_more_worker()
4428 * and keep_working() are always true as long as the
4429 * worklist is not empty. This pool now behaves as an
4430 * unbound (in terms of concurrency management) pool which
4431 * are served by workers tied to the pool.
4432 */
e19e397a 4433 atomic_set(&pool->nr_running, 0);
eb283428
LJ
4434
4435 /*
4436 * With concurrency management just turned off, a busy
4437 * worker blocking could lead to lengthy stalls. Kick off
4438 * unbound chain execution of currently pending work items.
4439 */
4440 spin_lock_irq(&pool->lock);
4441 wake_up_worker(pool);
4442 spin_unlock_irq(&pool->lock);
4443 }
3af24433 4444}
3af24433 4445
bd7c089e
TH
4446/**
4447 * rebind_workers - rebind all workers of a pool to the associated CPU
4448 * @pool: pool of interest
4449 *
a9ab775b 4450 * @pool->cpu is coming online. Rebind all workers to the CPU.
bd7c089e
TH
4451 */
4452static void rebind_workers(struct worker_pool *pool)
4453{
a9ab775b 4454 struct worker *worker;
bd7c089e 4455
92f9c5c4 4456 lockdep_assert_held(&pool->attach_mutex);
bd7c089e 4457
a9ab775b
TH
4458 /*
4459 * Restore CPU affinity of all workers. As all idle workers should
4460 * be on the run-queue of the associated CPU before any local
402dd89d 4461 * wake-ups for concurrency management happen, restore CPU affinity
a9ab775b
TH
4462 * of all workers first and then clear UNBOUND. As we're called
4463 * from CPU_ONLINE, the following shouldn't fail.
4464 */
da028469 4465 for_each_pool_worker(worker, pool)
a9ab775b
TH
4466 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4467 pool->attrs->cpumask) < 0);
bd7c089e 4468
a9ab775b 4469 spin_lock_irq(&pool->lock);
3de5e884 4470 pool->flags &= ~POOL_DISASSOCIATED;
bd7c089e 4471
da028469 4472 for_each_pool_worker(worker, pool) {
a9ab775b 4473 unsigned int worker_flags = worker->flags;
bd7c089e
TH
4474
4475 /*
a9ab775b
TH
4476 * A bound idle worker should actually be on the runqueue
4477 * of the associated CPU for local wake-ups targeting it to
4478 * work. Kick all idle workers so that they migrate to the
4479 * associated CPU. Doing this in the same loop as
4480 * replacing UNBOUND with REBOUND is safe as no worker will
4481 * be bound before @pool->lock is released.
bd7c089e 4482 */
a9ab775b
TH
4483 if (worker_flags & WORKER_IDLE)
4484 wake_up_process(worker->task);
bd7c089e 4485
a9ab775b
TH
4486 /*
4487 * We want to clear UNBOUND but can't directly call
4488 * worker_clr_flags() or adjust nr_running. Atomically
4489 * replace UNBOUND with another NOT_RUNNING flag REBOUND.
4490 * @worker will clear REBOUND using worker_clr_flags() when
4491 * it initiates the next execution cycle thus restoring
4492 * concurrency management. Note that when or whether
4493 * @worker clears REBOUND doesn't affect correctness.
4494 *
4495 * ACCESS_ONCE() is necessary because @worker->flags may be
4496 * tested without holding any lock in
4497 * wq_worker_waking_up(). Without it, NOT_RUNNING test may
4498 * fail incorrectly leading to premature concurrency
4499 * management operations.
4500 */
4501 WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
4502 worker_flags |= WORKER_REBOUND;
4503 worker_flags &= ~WORKER_UNBOUND;
4504 ACCESS_ONCE(worker->flags) = worker_flags;
bd7c089e 4505 }
a9ab775b
TH
4506
4507 spin_unlock_irq(&pool->lock);
bd7c089e
TH
4508}
4509
7dbc725e
TH
4510/**
4511 * restore_unbound_workers_cpumask - restore cpumask of unbound workers
4512 * @pool: unbound pool of interest
4513 * @cpu: the CPU which is coming up
4514 *
4515 * An unbound pool may end up with a cpumask which doesn't have any online
4516 * CPUs. When a worker of such pool get scheduled, the scheduler resets
4517 * its cpus_allowed. If @cpu is in @pool's cpumask which didn't have any
4518 * online CPU before, cpus_allowed of all its workers should be restored.
4519 */
4520static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
4521{
4522 static cpumask_t cpumask;
4523 struct worker *worker;
7dbc725e 4524
92f9c5c4 4525 lockdep_assert_held(&pool->attach_mutex);
7dbc725e
TH
4526
4527 /* is @cpu allowed for @pool? */
4528 if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
4529 return;
4530
4531 /* is @cpu the only online CPU? */
4532 cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
4533 if (cpumask_weight(&cpumask) != 1)
4534 return;
4535
4536 /* as we're called from CPU_ONLINE, the following shouldn't fail */
da028469 4537 for_each_pool_worker(worker, pool)
7dbc725e
TH
4538 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4539 pool->attrs->cpumask) < 0);
4540}
4541
8db25e78
TH
4542/*
4543 * Workqueues should be brought up before normal priority CPU notifiers.
4544 * This will be registered high priority CPU notifier.
4545 */
0db0628d 4546static int workqueue_cpu_up_callback(struct notifier_block *nfb,
8db25e78
TH
4547 unsigned long action,
4548 void *hcpu)
3af24433 4549{
d84ff051 4550 int cpu = (unsigned long)hcpu;
4ce62e9e 4551 struct worker_pool *pool;
4c16bd32 4552 struct workqueue_struct *wq;
7dbc725e 4553 int pi;
3ce63377 4554
8db25e78 4555 switch (action & ~CPU_TASKS_FROZEN) {
3af24433 4556 case CPU_UP_PREPARE:
f02ae73a 4557 for_each_cpu_worker_pool(pool, cpu) {
3ce63377
TH
4558 if (pool->nr_workers)
4559 continue;
051e1850 4560 if (!create_worker(pool))
3ce63377 4561 return NOTIFY_BAD;
3af24433 4562 }
8db25e78 4563 break;
3af24433 4564
db7bccf4
TH
4565 case CPU_DOWN_FAILED:
4566 case CPU_ONLINE:
68e13a67 4567 mutex_lock(&wq_pool_mutex);
7dbc725e
TH
4568
4569 for_each_pool(pool, pi) {
92f9c5c4 4570 mutex_lock(&pool->attach_mutex);
94cf58bb 4571
f05b558d 4572 if (pool->cpu == cpu)
7dbc725e 4573 rebind_workers(pool);
f05b558d 4574 else if (pool->cpu < 0)
7dbc725e 4575 restore_unbound_workers_cpumask(pool, cpu);
94cf58bb 4576
6ba94429
FW
4577 mutex_unlock(&pool->attach_mutex);
4578 }
4579
4580 /* update NUMA affinity of unbound workqueues */
4581 list_for_each_entry(wq, &workqueues, list)
4582 wq_update_unbound_numa(wq, cpu, true);
4583
4584 mutex_unlock(&wq_pool_mutex);
4585 break;
4586 }
4587 return NOTIFY_OK;
4588}
4589
4590/*
4591 * Workqueues should be brought down after normal priority CPU notifiers.
4592 * This will be registered as low priority CPU notifier.
4593 */
4594static int workqueue_cpu_down_callback(struct notifier_block *nfb,
4595 unsigned long action,
4596 void *hcpu)
4597{
4598 int cpu = (unsigned long)hcpu;
4599 struct work_struct unbind_work;
4600 struct workqueue_struct *wq;
4601
4602 switch (action & ~CPU_TASKS_FROZEN) {
4603 case CPU_DOWN_PREPARE:
4604 /* unbinding per-cpu workers should happen on the local CPU */
4605 INIT_WORK_ONSTACK(&unbind_work, wq_unbind_fn);
4606 queue_work_on(cpu, system_highpri_wq, &unbind_work);
4607
4608 /* update NUMA affinity of unbound workqueues */
4609 mutex_lock(&wq_pool_mutex);
4610 list_for_each_entry(wq, &workqueues, list)
4611 wq_update_unbound_numa(wq, cpu, false);
4612 mutex_unlock(&wq_pool_mutex);
4613
4614 /* wait for per-cpu unbinding to finish */
4615 flush_work(&unbind_work);
4616 destroy_work_on_stack(&unbind_work);
4617 break;
4618 }
4619 return NOTIFY_OK;
4620}
4621
4622#ifdef CONFIG_SMP
4623
4624struct work_for_cpu {
4625 struct work_struct work;
4626 long (*fn)(void *);
4627 void *arg;
4628 long ret;
4629};
4630
4631static void work_for_cpu_fn(struct work_struct *work)
4632{
4633 struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
4634
4635 wfc->ret = wfc->fn(wfc->arg);
4636}
4637
4638/**
4639 * work_on_cpu - run a function in user context on a particular cpu
4640 * @cpu: the cpu to run on
4641 * @fn: the function to run
4642 * @arg: the function arg
4643 *
4644 * It is up to the caller to ensure that the cpu doesn't go offline.
4645 * The caller must not hold any locks which would prevent @fn from completing.
4646 *
4647 * Return: The value @fn returns.
4648 */
4649long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
4650{
4651 struct work_for_cpu wfc = { .fn = fn, .arg = arg };
4652
4653 INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
4654 schedule_work_on(cpu, &wfc.work);
4655 flush_work(&wfc.work);
4656 destroy_work_on_stack(&wfc.work);
4657 return wfc.ret;
4658}
4659EXPORT_SYMBOL_GPL(work_on_cpu);
4660#endif /* CONFIG_SMP */
4661
4662#ifdef CONFIG_FREEZER
4663
4664/**
4665 * freeze_workqueues_begin - begin freezing workqueues
4666 *
4667 * Start freezing workqueues. After this function returns, all freezable
4668 * workqueues will queue new works to their delayed_works list instead of
4669 * pool->worklist.
4670 *
4671 * CONTEXT:
4672 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
4673 */
4674void freeze_workqueues_begin(void)
4675{
4676 struct workqueue_struct *wq;
4677 struct pool_workqueue *pwq;
4678
4679 mutex_lock(&wq_pool_mutex);
4680
4681 WARN_ON_ONCE(workqueue_freezing);
4682 workqueue_freezing = true;
4683
4684 list_for_each_entry(wq, &workqueues, list) {
4685 mutex_lock(&wq->mutex);
4686 for_each_pwq(pwq, wq)
4687 pwq_adjust_max_active(pwq);
4688 mutex_unlock(&wq->mutex);
4689 }
4690
4691 mutex_unlock(&wq_pool_mutex);
4692}
4693
4694/**
4695 * freeze_workqueues_busy - are freezable workqueues still busy?
4696 *
4697 * Check whether freezing is complete. This function must be called
4698 * between freeze_workqueues_begin() and thaw_workqueues().
4699 *
4700 * CONTEXT:
4701 * Grabs and releases wq_pool_mutex.
4702 *
4703 * Return:
4704 * %true if some freezable workqueues are still busy. %false if freezing
4705 * is complete.
4706 */
4707bool freeze_workqueues_busy(void)
4708{
4709 bool busy = false;
4710 struct workqueue_struct *wq;
4711 struct pool_workqueue *pwq;
4712
4713 mutex_lock(&wq_pool_mutex);
4714
4715 WARN_ON_ONCE(!workqueue_freezing);
4716
4717 list_for_each_entry(wq, &workqueues, list) {
4718 if (!(wq->flags & WQ_FREEZABLE))
4719 continue;
4720 /*
4721 * nr_active is monotonically decreasing. It's safe
4722 * to peek without lock.
4723 */
4724 rcu_read_lock_sched();
4725 for_each_pwq(pwq, wq) {
4726 WARN_ON_ONCE(pwq->nr_active < 0);
4727 if (pwq->nr_active) {
4728 busy = true;
4729 rcu_read_unlock_sched();
4730 goto out_unlock;
4731 }
4732 }
4733 rcu_read_unlock_sched();
4734 }
4735out_unlock:
4736 mutex_unlock(&wq_pool_mutex);
4737 return busy;
4738}
4739
4740/**
4741 * thaw_workqueues - thaw workqueues
4742 *
4743 * Thaw workqueues. Normal queueing is restored and all collected
4744 * frozen works are transferred to their respective pool worklists.
4745 *
4746 * CONTEXT:
4747 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
4748 */
4749void thaw_workqueues(void)
4750{
4751 struct workqueue_struct *wq;
4752 struct pool_workqueue *pwq;
4753
4754 mutex_lock(&wq_pool_mutex);
4755
4756 if (!workqueue_freezing)
4757 goto out_unlock;
4758
4759 workqueue_freezing = false;
4760
4761 /* restore max_active and repopulate worklist */
4762 list_for_each_entry(wq, &workqueues, list) {
4763 mutex_lock(&wq->mutex);
4764 for_each_pwq(pwq, wq)
4765 pwq_adjust_max_active(pwq);
4766 mutex_unlock(&wq->mutex);
4767 }
4768
4769out_unlock:
4770 mutex_unlock(&wq_pool_mutex);
4771}
4772#endif /* CONFIG_FREEZER */
4773
042f7df1
LJ
4774static int workqueue_apply_unbound_cpumask(void)
4775{
4776 LIST_HEAD(ctxs);
4777 int ret = 0;
4778 struct workqueue_struct *wq;
4779 struct apply_wqattrs_ctx *ctx, *n;
4780
4781 lockdep_assert_held(&wq_pool_mutex);
4782
4783 list_for_each_entry(wq, &workqueues, list) {
4784 if (!(wq->flags & WQ_UNBOUND))
4785 continue;
4786 /* creating multiple pwqs breaks ordering guarantee */
4787 if (wq->flags & __WQ_ORDERED)
4788 continue;
4789
4790 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs);
4791 if (!ctx) {
4792 ret = -ENOMEM;
4793 break;
4794 }
4795
4796 list_add_tail(&ctx->list, &ctxs);
4797 }
4798
4799 list_for_each_entry_safe(ctx, n, &ctxs, list) {
4800 if (!ret)
4801 apply_wqattrs_commit(ctx);
4802 apply_wqattrs_cleanup(ctx);
4803 }
4804
4805 return ret;
4806}
4807
4808/**
4809 * workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
4810 * @cpumask: the cpumask to set
4811 *
4812 * The low-level workqueues cpumask is a global cpumask that limits
4813 * the affinity of all unbound workqueues. This function check the @cpumask
4814 * and apply it to all unbound workqueues and updates all pwqs of them.
4815 *
4816 * Retun: 0 - Success
4817 * -EINVAL - Invalid @cpumask
4818 * -ENOMEM - Failed to allocate memory for attrs or pwqs.
4819 */
4820int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
4821{
4822 int ret = -EINVAL;
4823 cpumask_var_t saved_cpumask;
4824
4825 if (!zalloc_cpumask_var(&saved_cpumask, GFP_KERNEL))
4826 return -ENOMEM;
4827
042f7df1
LJ
4828 cpumask_and(cpumask, cpumask, cpu_possible_mask);
4829 if (!cpumask_empty(cpumask)) {
a0111cf6 4830 apply_wqattrs_lock();
042f7df1
LJ
4831
4832 /* save the old wq_unbound_cpumask. */
4833 cpumask_copy(saved_cpumask, wq_unbound_cpumask);
4834
4835 /* update wq_unbound_cpumask at first and apply it to wqs. */
4836 cpumask_copy(wq_unbound_cpumask, cpumask);
4837 ret = workqueue_apply_unbound_cpumask();
4838
4839 /* restore the wq_unbound_cpumask when failed. */
4840 if (ret < 0)
4841 cpumask_copy(wq_unbound_cpumask, saved_cpumask);
4842
a0111cf6 4843 apply_wqattrs_unlock();
042f7df1 4844 }
042f7df1
LJ
4845
4846 free_cpumask_var(saved_cpumask);
4847 return ret;
4848}
4849
6ba94429
FW
4850#ifdef CONFIG_SYSFS
4851/*
4852 * Workqueues with WQ_SYSFS flag set is visible to userland via
4853 * /sys/bus/workqueue/devices/WQ_NAME. All visible workqueues have the
4854 * following attributes.
4855 *
4856 * per_cpu RO bool : whether the workqueue is per-cpu or unbound
4857 * max_active RW int : maximum number of in-flight work items
4858 *
4859 * Unbound workqueues have the following extra attributes.
4860 *
4861 * id RO int : the associated pool ID
4862 * nice RW int : nice value of the workers
4863 * cpumask RW mask : bitmask of allowed CPUs for the workers
4864 */
4865struct wq_device {
4866 struct workqueue_struct *wq;
4867 struct device dev;
4868};
4869
4870static struct workqueue_struct *dev_to_wq(struct device *dev)
4871{
4872 struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
4873
4874 return wq_dev->wq;
4875}
4876
4877static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr,
4878 char *buf)
4879{
4880 struct workqueue_struct *wq = dev_to_wq(dev);
4881
4882 return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
4883}
4884static DEVICE_ATTR_RO(per_cpu);
4885
4886static ssize_t max_active_show(struct device *dev,
4887 struct device_attribute *attr, char *buf)
4888{
4889 struct workqueue_struct *wq = dev_to_wq(dev);
4890
4891 return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
4892}
4893
4894static ssize_t max_active_store(struct device *dev,
4895 struct device_attribute *attr, const char *buf,
4896 size_t count)
4897{
4898 struct workqueue_struct *wq = dev_to_wq(dev);
4899 int val;
4900
4901 if (sscanf(buf, "%d", &val) != 1 || val <= 0)
4902 return -EINVAL;
4903
4904 workqueue_set_max_active(wq, val);
4905 return count;
4906}
4907static DEVICE_ATTR_RW(max_active);
4908
4909static struct attribute *wq_sysfs_attrs[] = {
4910 &dev_attr_per_cpu.attr,
4911 &dev_attr_max_active.attr,
4912 NULL,
4913};
4914ATTRIBUTE_GROUPS(wq_sysfs);
4915
4916static ssize_t wq_pool_ids_show(struct device *dev,
4917 struct device_attribute *attr, char *buf)
4918{
4919 struct workqueue_struct *wq = dev_to_wq(dev);
4920 const char *delim = "";
4921 int node, written = 0;
4922
4923 rcu_read_lock_sched();
4924 for_each_node(node) {
4925 written += scnprintf(buf + written, PAGE_SIZE - written,
4926 "%s%d:%d", delim, node,
4927 unbound_pwq_by_node(wq, node)->pool->id);
4928 delim = " ";
4929 }
4930 written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
4931 rcu_read_unlock_sched();
4932
4933 return written;
4934}
4935
4936static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
4937 char *buf)
4938{
4939 struct workqueue_struct *wq = dev_to_wq(dev);
4940 int written;
4941
4942 mutex_lock(&wq->mutex);
4943 written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
4944 mutex_unlock(&wq->mutex);
4945
4946 return written;
4947}
4948
4949/* prepare workqueue_attrs for sysfs store operations */
4950static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
4951{
4952 struct workqueue_attrs *attrs;
4953
899a94fe
LJ
4954 lockdep_assert_held(&wq_pool_mutex);
4955
6ba94429
FW
4956 attrs = alloc_workqueue_attrs(GFP_KERNEL);
4957 if (!attrs)
4958 return NULL;
4959
6ba94429 4960 copy_workqueue_attrs(attrs, wq->unbound_attrs);
6ba94429
FW
4961 return attrs;
4962}
4963
4964static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
4965 const char *buf, size_t count)
4966{
4967 struct workqueue_struct *wq = dev_to_wq(dev);
4968 struct workqueue_attrs *attrs;
d4d3e257
LJ
4969 int ret = -ENOMEM;
4970
4971 apply_wqattrs_lock();
6ba94429
FW
4972
4973 attrs = wq_sysfs_prep_attrs(wq);
4974 if (!attrs)
d4d3e257 4975 goto out_unlock;
6ba94429
FW
4976
4977 if (sscanf(buf, "%d", &attrs->nice) == 1 &&
4978 attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE)
d4d3e257 4979 ret = apply_workqueue_attrs_locked(wq, attrs);
6ba94429
FW
4980 else
4981 ret = -EINVAL;
4982
d4d3e257
LJ
4983out_unlock:
4984 apply_wqattrs_unlock();
6ba94429
FW
4985 free_workqueue_attrs(attrs);
4986 return ret ?: count;
4987}
4988
4989static ssize_t wq_cpumask_show(struct device *dev,
4990 struct device_attribute *attr, char *buf)
4991{
4992 struct workqueue_struct *wq = dev_to_wq(dev);
4993 int written;
4994
4995 mutex_lock(&wq->mutex);
4996 written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
4997 cpumask_pr_args(wq->unbound_attrs->cpumask));
4998 mutex_unlock(&wq->mutex);
4999 return written;
5000}
5001
5002static ssize_t wq_cpumask_store(struct device *dev,
5003 struct device_attribute *attr,
5004 const char *buf, size_t count)
5005{
5006 struct workqueue_struct *wq = dev_to_wq(dev);
5007 struct workqueue_attrs *attrs;
d4d3e257
LJ
5008 int ret = -ENOMEM;
5009
5010 apply_wqattrs_lock();
6ba94429
FW
5011
5012 attrs = wq_sysfs_prep_attrs(wq);
5013 if (!attrs)
d4d3e257 5014 goto out_unlock;
6ba94429
FW
5015
5016 ret = cpumask_parse(buf, attrs->cpumask);
5017 if (!ret)
d4d3e257 5018 ret = apply_workqueue_attrs_locked(wq, attrs);
6ba94429 5019
d4d3e257
LJ
5020out_unlock:
5021 apply_wqattrs_unlock();
6ba94429
FW
5022 free_workqueue_attrs(attrs);
5023 return ret ?: count;
5024}
5025
5026static ssize_t wq_numa_show(struct device *dev, struct device_attribute *attr,
5027 char *buf)
5028{
5029 struct workqueue_struct *wq = dev_to_wq(dev);
5030 int written;
7dbc725e 5031
6ba94429
FW
5032 mutex_lock(&wq->mutex);
5033 written = scnprintf(buf, PAGE_SIZE, "%d\n",
5034 !wq->unbound_attrs->no_numa);
5035 mutex_unlock(&wq->mutex);
4c16bd32 5036
6ba94429 5037 return written;
65758202
TH
5038}
5039
6ba94429
FW
5040static ssize_t wq_numa_store(struct device *dev, struct device_attribute *attr,
5041 const char *buf, size_t count)
65758202 5042{
6ba94429
FW
5043 struct workqueue_struct *wq = dev_to_wq(dev);
5044 struct workqueue_attrs *attrs;
d4d3e257
LJ
5045 int v, ret = -ENOMEM;
5046
5047 apply_wqattrs_lock();
4c16bd32 5048
6ba94429
FW
5049 attrs = wq_sysfs_prep_attrs(wq);
5050 if (!attrs)
d4d3e257 5051 goto out_unlock;
4c16bd32 5052
6ba94429
FW
5053 ret = -EINVAL;
5054 if (sscanf(buf, "%d", &v) == 1) {
5055 attrs->no_numa = !v;
d4d3e257 5056 ret = apply_workqueue_attrs_locked(wq, attrs);
65758202 5057 }
6ba94429 5058
d4d3e257
LJ
5059out_unlock:
5060 apply_wqattrs_unlock();
6ba94429
FW
5061 free_workqueue_attrs(attrs);
5062 return ret ?: count;
65758202
TH
5063}
5064
6ba94429
FW
5065static struct device_attribute wq_sysfs_unbound_attrs[] = {
5066 __ATTR(pool_ids, 0444, wq_pool_ids_show, NULL),
5067 __ATTR(nice, 0644, wq_nice_show, wq_nice_store),
5068 __ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
5069 __ATTR(numa, 0644, wq_numa_show, wq_numa_store),
5070 __ATTR_NULL,
5071};
8ccad40d 5072
6ba94429
FW
5073static struct bus_type wq_subsys = {
5074 .name = "workqueue",
5075 .dev_groups = wq_sysfs_groups,
2d3854a3
RR
5076};
5077
b05a7928
FW
5078static ssize_t wq_unbound_cpumask_show(struct device *dev,
5079 struct device_attribute *attr, char *buf)
5080{
5081 int written;
5082
042f7df1 5083 mutex_lock(&wq_pool_mutex);
b05a7928
FW
5084 written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
5085 cpumask_pr_args(wq_unbound_cpumask));
042f7df1 5086 mutex_unlock(&wq_pool_mutex);
b05a7928
FW
5087
5088 return written;
5089}
5090
042f7df1
LJ
5091static ssize_t wq_unbound_cpumask_store(struct device *dev,
5092 struct device_attribute *attr, const char *buf, size_t count)
5093{
5094 cpumask_var_t cpumask;
5095 int ret;
5096
5097 if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
5098 return -ENOMEM;
5099
5100 ret = cpumask_parse(buf, cpumask);
5101 if (!ret)
5102 ret = workqueue_set_unbound_cpumask(cpumask);
5103
5104 free_cpumask_var(cpumask);
5105 return ret ? ret : count;
5106}
5107
b05a7928 5108static struct device_attribute wq_sysfs_cpumask_attr =
042f7df1
LJ
5109 __ATTR(cpumask, 0644, wq_unbound_cpumask_show,
5110 wq_unbound_cpumask_store);
b05a7928 5111
6ba94429 5112static int __init wq_sysfs_init(void)
2d3854a3 5113{
b05a7928
FW
5114 int err;
5115
5116 err = subsys_virtual_register(&wq_subsys, NULL);
5117 if (err)
5118 return err;
5119
5120 return device_create_file(wq_subsys.dev_root, &wq_sysfs_cpumask_attr);
2d3854a3 5121}
6ba94429 5122core_initcall(wq_sysfs_init);
2d3854a3 5123
6ba94429 5124static void wq_device_release(struct device *dev)
2d3854a3 5125{
6ba94429 5126 struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
6b44003e 5127
6ba94429 5128 kfree(wq_dev);
2d3854a3 5129}
a0a1a5fd
TH
5130
5131/**
6ba94429
FW
5132 * workqueue_sysfs_register - make a workqueue visible in sysfs
5133 * @wq: the workqueue to register
a0a1a5fd 5134 *
6ba94429
FW
5135 * Expose @wq in sysfs under /sys/bus/workqueue/devices.
5136 * alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
5137 * which is the preferred method.
a0a1a5fd 5138 *
6ba94429
FW
5139 * Workqueue user should use this function directly iff it wants to apply
5140 * workqueue_attrs before making the workqueue visible in sysfs; otherwise,
5141 * apply_workqueue_attrs() may race against userland updating the
5142 * attributes.
5143 *
5144 * Return: 0 on success, -errno on failure.
a0a1a5fd 5145 */
6ba94429 5146int workqueue_sysfs_register(struct workqueue_struct *wq)
a0a1a5fd 5147{
6ba94429
FW
5148 struct wq_device *wq_dev;
5149 int ret;
a0a1a5fd 5150
6ba94429 5151 /*
402dd89d 5152 * Adjusting max_active or creating new pwqs by applying
6ba94429
FW
5153 * attributes breaks ordering guarantee. Disallow exposing ordered
5154 * workqueues.
5155 */
5156 if (WARN_ON(wq->flags & __WQ_ORDERED))
5157 return -EINVAL;
a0a1a5fd 5158
6ba94429
FW
5159 wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
5160 if (!wq_dev)
5161 return -ENOMEM;
5bcab335 5162
6ba94429
FW
5163 wq_dev->wq = wq;
5164 wq_dev->dev.bus = &wq_subsys;
5165 wq_dev->dev.init_name = wq->name;
5166 wq_dev->dev.release = wq_device_release;
a0a1a5fd 5167
6ba94429
FW
5168 /*
5169 * unbound_attrs are created separately. Suppress uevent until
5170 * everything is ready.
5171 */
5172 dev_set_uevent_suppress(&wq_dev->dev, true);
a0a1a5fd 5173
6ba94429
FW
5174 ret = device_register(&wq_dev->dev);
5175 if (ret) {
5176 kfree(wq_dev);
5177 wq->wq_dev = NULL;
5178 return ret;
5179 }
a0a1a5fd 5180
6ba94429
FW
5181 if (wq->flags & WQ_UNBOUND) {
5182 struct device_attribute *attr;
a0a1a5fd 5183
6ba94429
FW
5184 for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
5185 ret = device_create_file(&wq_dev->dev, attr);
5186 if (ret) {
5187 device_unregister(&wq_dev->dev);
5188 wq->wq_dev = NULL;
5189 return ret;
a0a1a5fd
TH
5190 }
5191 }
5192 }
6ba94429
FW
5193
5194 dev_set_uevent_suppress(&wq_dev->dev, false);
5195 kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
5196 return 0;
a0a1a5fd
TH
5197}
5198
5199/**
6ba94429
FW
5200 * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
5201 * @wq: the workqueue to unregister
a0a1a5fd 5202 *
6ba94429 5203 * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
a0a1a5fd 5204 */
6ba94429 5205static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
a0a1a5fd 5206{
6ba94429 5207 struct wq_device *wq_dev = wq->wq_dev;
8b03ae3c 5208
6ba94429
FW
5209 if (!wq->wq_dev)
5210 return;
a0a1a5fd 5211
6ba94429
FW
5212 wq->wq_dev = NULL;
5213 device_unregister(&wq_dev->dev);
a0a1a5fd 5214}
6ba94429
FW
5215#else /* CONFIG_SYSFS */
5216static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
5217#endif /* CONFIG_SYSFS */
a0a1a5fd 5218
82607adc
TH
5219/*
5220 * Workqueue watchdog.
5221 *
5222 * Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal
5223 * flush dependency, a concurrency managed work item which stays RUNNING
5224 * indefinitely. Workqueue stalls can be very difficult to debug as the
5225 * usual warning mechanisms don't trigger and internal workqueue state is
5226 * largely opaque.
5227 *
5228 * Workqueue watchdog monitors all worker pools periodically and dumps
5229 * state if some pools failed to make forward progress for a while where
5230 * forward progress is defined as the first item on ->worklist changing.
5231 *
5232 * This mechanism is controlled through the kernel parameter
5233 * "workqueue.watchdog_thresh" which can be updated at runtime through the
5234 * corresponding sysfs parameter file.
5235 */
5236#ifdef CONFIG_WQ_WATCHDOG
5237
5238static void wq_watchdog_timer_fn(unsigned long data);
5239
5240static unsigned long wq_watchdog_thresh = 30;
5241static struct timer_list wq_watchdog_timer =
5242 TIMER_DEFERRED_INITIALIZER(wq_watchdog_timer_fn, 0, 0);
5243
5244static unsigned long wq_watchdog_touched = INITIAL_JIFFIES;
5245static DEFINE_PER_CPU(unsigned long, wq_watchdog_touched_cpu) = INITIAL_JIFFIES;
5246
5247static void wq_watchdog_reset_touched(void)
5248{
5249 int cpu;
5250
5251 wq_watchdog_touched = jiffies;
5252 for_each_possible_cpu(cpu)
5253 per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
5254}
5255
5256static void wq_watchdog_timer_fn(unsigned long data)
5257{
5258 unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
5259 bool lockup_detected = false;
5260 struct worker_pool *pool;
5261 int pi;
5262
5263 if (!thresh)
5264 return;
5265
5266 rcu_read_lock();
5267
5268 for_each_pool(pool, pi) {
5269 unsigned long pool_ts, touched, ts;
5270
5271 if (list_empty(&pool->worklist))
5272 continue;
5273
5274 /* get the latest of pool and touched timestamps */
5275 pool_ts = READ_ONCE(pool->watchdog_ts);
5276 touched = READ_ONCE(wq_watchdog_touched);
5277
5278 if (time_after(pool_ts, touched))
5279 ts = pool_ts;
5280 else
5281 ts = touched;
5282
5283 if (pool->cpu >= 0) {
5284 unsigned long cpu_touched =
5285 READ_ONCE(per_cpu(wq_watchdog_touched_cpu,
5286 pool->cpu));
5287 if (time_after(cpu_touched, ts))
5288 ts = cpu_touched;
5289 }
5290
5291 /* did we stall? */
5292 if (time_after(jiffies, ts + thresh)) {
5293 lockup_detected = true;
5294 pr_emerg("BUG: workqueue lockup - pool");
5295 pr_cont_pool_info(pool);
5296 pr_cont(" stuck for %us!\n",
5297 jiffies_to_msecs(jiffies - pool_ts) / 1000);
5298 }
5299 }
5300
5301 rcu_read_unlock();
5302
5303 if (lockup_detected)
5304 show_workqueue_state();
5305
5306 wq_watchdog_reset_touched();
5307 mod_timer(&wq_watchdog_timer, jiffies + thresh);
5308}
5309
5310void wq_watchdog_touch(int cpu)
5311{
5312 if (cpu >= 0)
5313 per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
5314 else
5315 wq_watchdog_touched = jiffies;
5316}
5317
5318static void wq_watchdog_set_thresh(unsigned long thresh)
5319{
5320 wq_watchdog_thresh = 0;
5321 del_timer_sync(&wq_watchdog_timer);
5322
5323 if (thresh) {
5324 wq_watchdog_thresh = thresh;
5325 wq_watchdog_reset_touched();
5326 mod_timer(&wq_watchdog_timer, jiffies + thresh * HZ);
5327 }
5328}
5329
5330static int wq_watchdog_param_set_thresh(const char *val,
5331 const struct kernel_param *kp)
5332{
5333 unsigned long thresh;
5334 int ret;
5335
5336 ret = kstrtoul(val, 0, &thresh);
5337 if (ret)
5338 return ret;
5339
5340 if (system_wq)
5341 wq_watchdog_set_thresh(thresh);
5342 else
5343 wq_watchdog_thresh = thresh;
5344
5345 return 0;
5346}
5347
5348static const struct kernel_param_ops wq_watchdog_thresh_ops = {
5349 .set = wq_watchdog_param_set_thresh,
5350 .get = param_get_ulong,
5351};
5352
5353module_param_cb(watchdog_thresh, &wq_watchdog_thresh_ops, &wq_watchdog_thresh,
5354 0644);
5355
5356static void wq_watchdog_init(void)
5357{
5358 wq_watchdog_set_thresh(wq_watchdog_thresh);
5359}
5360
5361#else /* CONFIG_WQ_WATCHDOG */
5362
5363static inline void wq_watchdog_init(void) { }
5364
5365#endif /* CONFIG_WQ_WATCHDOG */
5366
bce90380
TH
5367static void __init wq_numa_init(void)
5368{
5369 cpumask_var_t *tbl;
5370 int node, cpu;
5371
bce90380
TH
5372 if (num_possible_nodes() <= 1)
5373 return;
5374
d55262c4
TH
5375 if (wq_disable_numa) {
5376 pr_info("workqueue: NUMA affinity support disabled\n");
5377 return;
5378 }
5379
4c16bd32
TH
5380 wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs(GFP_KERNEL);
5381 BUG_ON(!wq_update_unbound_numa_attrs_buf);
5382
bce90380
TH
5383 /*
5384 * We want masks of possible CPUs of each node which isn't readily
5385 * available. Build one from cpu_to_node() which should have been
5386 * fully initialized by now.
5387 */
ddcb57e2 5388 tbl = kzalloc(nr_node_ids * sizeof(tbl[0]), GFP_KERNEL);
bce90380
TH
5389 BUG_ON(!tbl);
5390
5391 for_each_node(node)
5a6024f1 5392 BUG_ON(!zalloc_cpumask_var_node(&tbl[node], GFP_KERNEL,
1be0c25d 5393 node_online(node) ? node : NUMA_NO_NODE));
bce90380
TH
5394
5395 for_each_possible_cpu(cpu) {
5396 node = cpu_to_node(cpu);
5397 if (WARN_ON(node == NUMA_NO_NODE)) {
5398 pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
5399 /* happens iff arch is bonkers, let's just proceed */
5400 return;
5401 }
5402 cpumask_set_cpu(cpu, tbl[node]);
5403 }
5404
5405 wq_numa_possible_cpumask = tbl;
5406 wq_numa_enabled = true;
5407}
5408
6ee0578b 5409static int __init init_workqueues(void)
1da177e4 5410{
7a4e344c
TH
5411 int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
5412 int i, cpu;
c34056a3 5413
e904e6c2
TH
5414 WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
5415
b05a7928
FW
5416 BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
5417 cpumask_copy(wq_unbound_cpumask, cpu_possible_mask);
5418
e904e6c2
TH
5419 pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
5420
65758202 5421 cpu_notifier(workqueue_cpu_up_callback, CPU_PRI_WORKQUEUE_UP);
a5b4e57d 5422 hotcpu_notifier(workqueue_cpu_down_callback, CPU_PRI_WORKQUEUE_DOWN);
8b03ae3c 5423
bce90380
TH
5424 wq_numa_init();
5425
706026c2 5426 /* initialize CPU pools */
29c91e99 5427 for_each_possible_cpu(cpu) {
4ce62e9e 5428 struct worker_pool *pool;
8b03ae3c 5429
7a4e344c 5430 i = 0;
f02ae73a 5431 for_each_cpu_worker_pool(pool, cpu) {
7a4e344c 5432 BUG_ON(init_worker_pool(pool));
ec22ca5e 5433 pool->cpu = cpu;
29c91e99 5434 cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
7a4e344c 5435 pool->attrs->nice = std_nice[i++];
f3f90ad4 5436 pool->node = cpu_to_node(cpu);
7a4e344c 5437
9daf9e67 5438 /* alloc pool ID */
68e13a67 5439 mutex_lock(&wq_pool_mutex);
9daf9e67 5440 BUG_ON(worker_pool_assign_id(pool));
68e13a67 5441 mutex_unlock(&wq_pool_mutex);
4ce62e9e 5442 }
8b03ae3c
TH
5443 }
5444
e22bee78 5445 /* create the initial worker */
29c91e99 5446 for_each_online_cpu(cpu) {
4ce62e9e 5447 struct worker_pool *pool;
e22bee78 5448
f02ae73a 5449 for_each_cpu_worker_pool(pool, cpu) {
29c91e99 5450 pool->flags &= ~POOL_DISASSOCIATED;
051e1850 5451 BUG_ON(!create_worker(pool));
4ce62e9e 5452 }
e22bee78
TH
5453 }
5454
8a2b7538 5455 /* create default unbound and ordered wq attrs */
29c91e99
TH
5456 for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
5457 struct workqueue_attrs *attrs;
5458
5459 BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
29c91e99 5460 attrs->nice = std_nice[i];
29c91e99 5461 unbound_std_wq_attrs[i] = attrs;
8a2b7538
TH
5462
5463 /*
5464 * An ordered wq should have only one pwq as ordering is
5465 * guaranteed by max_active which is enforced by pwqs.
5466 * Turn off NUMA so that dfl_pwq is used for all nodes.
5467 */
5468 BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
5469 attrs->nice = std_nice[i];
5470 attrs->no_numa = true;
5471 ordered_wq_attrs[i] = attrs;
29c91e99
TH
5472 }
5473
d320c038 5474 system_wq = alloc_workqueue("events", 0, 0);
1aabe902 5475 system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
d320c038 5476 system_long_wq = alloc_workqueue("events_long", 0, 0);
f3421797
TH
5477 system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
5478 WQ_UNBOUND_MAX_ACTIVE);
24d51add
TH
5479 system_freezable_wq = alloc_workqueue("events_freezable",
5480 WQ_FREEZABLE, 0);
0668106c
VK
5481 system_power_efficient_wq = alloc_workqueue("events_power_efficient",
5482 WQ_POWER_EFFICIENT, 0);
5483 system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_power_efficient",
5484 WQ_FREEZABLE | WQ_POWER_EFFICIENT,
5485 0);
1aabe902 5486 BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||
0668106c
VK
5487 !system_unbound_wq || !system_freezable_wq ||
5488 !system_power_efficient_wq ||
5489 !system_freezable_power_efficient_wq);
82607adc
TH
5490
5491 wq_watchdog_init();
5492
6ee0578b 5493 return 0;
1da177e4 5494}
6ee0578b 5495early_initcall(init_workqueues);