]>
Commit | Line | Data |
---|---|---|
38498a67 TG |
1 | /* |
2 | * Common SMP CPU bringup/teardown functions | |
3 | */ | |
f97f8f06 | 4 | #include <linux/cpu.h> |
29d5e047 TG |
5 | #include <linux/err.h> |
6 | #include <linux/smp.h> | |
8038dad7 | 7 | #include <linux/delay.h> |
38498a67 | 8 | #include <linux/init.h> |
f97f8f06 TG |
9 | #include <linux/list.h> |
10 | #include <linux/slab.h> | |
29d5e047 | 11 | #include <linux/sched.h> |
f97f8f06 | 12 | #include <linux/export.h> |
29d5e047 | 13 | #include <linux/percpu.h> |
f97f8f06 TG |
14 | #include <linux/kthread.h> |
15 | #include <linux/smpboot.h> | |
38498a67 TG |
16 | |
17 | #include "smpboot.h" | |
18 | ||
3180d89b PM |
19 | #ifdef CONFIG_SMP |
20 | ||
29d5e047 | 21 | #ifdef CONFIG_GENERIC_SMP_IDLE_THREAD |
29d5e047 TG |
22 | /* |
23 | * For the hotplug case we keep the task structs around and reuse | |
24 | * them. | |
25 | */ | |
26 | static DEFINE_PER_CPU(struct task_struct *, idle_threads); | |
27 | ||
0db0628d | 28 | struct task_struct *idle_thread_get(unsigned int cpu) |
29d5e047 TG |
29 | { |
30 | struct task_struct *tsk = per_cpu(idle_threads, cpu); | |
31 | ||
32 | if (!tsk) | |
3bb5d2ee | 33 | return ERR_PTR(-ENOMEM); |
29d5e047 TG |
34 | init_idle(tsk, cpu); |
35 | return tsk; | |
36 | } | |
37 | ||
3bb5d2ee | 38 | void __init idle_thread_set_boot_cpu(void) |
29d5e047 | 39 | { |
3bb5d2ee | 40 | per_cpu(idle_threads, smp_processor_id()) = current; |
29d5e047 TG |
41 | } |
42 | ||
4a70d2d9 SB |
43 | /** |
44 | * idle_init - Initialize the idle thread for a cpu | |
45 | * @cpu: The cpu for which the idle thread should be initialized | |
46 | * | |
47 | * Creates the thread if it does not exist. | |
48 | */ | |
3bb5d2ee | 49 | static inline void idle_init(unsigned int cpu) |
29d5e047 | 50 | { |
3bb5d2ee SS |
51 | struct task_struct *tsk = per_cpu(idle_threads, cpu); |
52 | ||
53 | if (!tsk) { | |
54 | tsk = fork_idle(cpu); | |
55 | if (IS_ERR(tsk)) | |
56 | pr_err("SMP: fork_idle() failed for CPU %u\n", cpu); | |
57 | else | |
58 | per_cpu(idle_threads, cpu) = tsk; | |
59 | } | |
29d5e047 TG |
60 | } |
61 | ||
62 | /** | |
4a70d2d9 | 63 | * idle_threads_init - Initialize idle threads for all cpus |
29d5e047 | 64 | */ |
3bb5d2ee | 65 | void __init idle_threads_init(void) |
29d5e047 | 66 | { |
ee74d132 SB |
67 | unsigned int cpu, boot_cpu; |
68 | ||
69 | boot_cpu = smp_processor_id(); | |
29d5e047 | 70 | |
3bb5d2ee | 71 | for_each_possible_cpu(cpu) { |
ee74d132 | 72 | if (cpu != boot_cpu) |
3bb5d2ee | 73 | idle_init(cpu); |
29d5e047 | 74 | } |
29d5e047 | 75 | } |
29d5e047 | 76 | #endif |
f97f8f06 | 77 | |
3180d89b PM |
78 | #endif /* #ifdef CONFIG_SMP */ |
79 | ||
f97f8f06 TG |
80 | static LIST_HEAD(hotplug_threads); |
81 | static DEFINE_MUTEX(smpboot_threads_lock); | |
82 | ||
83 | struct smpboot_thread_data { | |
84 | unsigned int cpu; | |
85 | unsigned int status; | |
86 | struct smp_hotplug_thread *ht; | |
87 | }; | |
88 | ||
89 | enum { | |
90 | HP_THREAD_NONE = 0, | |
91 | HP_THREAD_ACTIVE, | |
92 | HP_THREAD_PARKED, | |
93 | }; | |
94 | ||
95 | /** | |
96 | * smpboot_thread_fn - percpu hotplug thread loop function | |
97 | * @data: thread data pointer | |
98 | * | |
99 | * Checks for thread stop and park conditions. Calls the necessary | |
100 | * setup, cleanup, park and unpark functions for the registered | |
101 | * thread. | |
102 | * | |
103 | * Returns 1 when the thread should exit, 0 otherwise. | |
104 | */ | |
105 | static int smpboot_thread_fn(void *data) | |
106 | { | |
107 | struct smpboot_thread_data *td = data; | |
108 | struct smp_hotplug_thread *ht = td->ht; | |
109 | ||
110 | while (1) { | |
111 | set_current_state(TASK_INTERRUPTIBLE); | |
112 | preempt_disable(); | |
113 | if (kthread_should_stop()) { | |
7d4d2696 | 114 | __set_current_state(TASK_RUNNING); |
f97f8f06 | 115 | preempt_enable(); |
3dd08c0c FW |
116 | /* cleanup must mirror setup */ |
117 | if (ht->cleanup && td->status != HP_THREAD_NONE) | |
f97f8f06 TG |
118 | ht->cleanup(td->cpu, cpu_online(td->cpu)); |
119 | kfree(td); | |
120 | return 0; | |
121 | } | |
122 | ||
123 | if (kthread_should_park()) { | |
124 | __set_current_state(TASK_RUNNING); | |
125 | preempt_enable(); | |
126 | if (ht->park && td->status == HP_THREAD_ACTIVE) { | |
127 | BUG_ON(td->cpu != smp_processor_id()); | |
128 | ht->park(td->cpu); | |
129 | td->status = HP_THREAD_PARKED; | |
130 | } | |
131 | kthread_parkme(); | |
132 | /* We might have been woken for stop */ | |
133 | continue; | |
134 | } | |
135 | ||
dc893e19 | 136 | BUG_ON(td->cpu != smp_processor_id()); |
f97f8f06 TG |
137 | |
138 | /* Check for state change setup */ | |
139 | switch (td->status) { | |
140 | case HP_THREAD_NONE: | |
7d4d2696 | 141 | __set_current_state(TASK_RUNNING); |
f97f8f06 TG |
142 | preempt_enable(); |
143 | if (ht->setup) | |
144 | ht->setup(td->cpu); | |
145 | td->status = HP_THREAD_ACTIVE; | |
7d4d2696 PZ |
146 | continue; |
147 | ||
f97f8f06 | 148 | case HP_THREAD_PARKED: |
7d4d2696 | 149 | __set_current_state(TASK_RUNNING); |
f97f8f06 TG |
150 | preempt_enable(); |
151 | if (ht->unpark) | |
152 | ht->unpark(td->cpu); | |
153 | td->status = HP_THREAD_ACTIVE; | |
7d4d2696 | 154 | continue; |
f97f8f06 TG |
155 | } |
156 | ||
157 | if (!ht->thread_should_run(td->cpu)) { | |
7d4d2696 | 158 | preempt_enable_no_resched(); |
f97f8f06 TG |
159 | schedule(); |
160 | } else { | |
7d4d2696 | 161 | __set_current_state(TASK_RUNNING); |
f97f8f06 TG |
162 | preempt_enable(); |
163 | ht->thread_fn(td->cpu); | |
164 | } | |
165 | } | |
166 | } | |
167 | ||
168 | static int | |
169 | __smpboot_create_thread(struct smp_hotplug_thread *ht, unsigned int cpu) | |
170 | { | |
171 | struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); | |
172 | struct smpboot_thread_data *td; | |
173 | ||
174 | if (tsk) | |
175 | return 0; | |
176 | ||
177 | td = kzalloc_node(sizeof(*td), GFP_KERNEL, cpu_to_node(cpu)); | |
178 | if (!td) | |
179 | return -ENOMEM; | |
180 | td->cpu = cpu; | |
181 | td->ht = ht; | |
182 | ||
183 | tsk = kthread_create_on_cpu(smpboot_thread_fn, td, cpu, | |
184 | ht->thread_comm); | |
185 | if (IS_ERR(tsk)) { | |
186 | kfree(td); | |
187 | return PTR_ERR(tsk); | |
188 | } | |
f97f8f06 TG |
189 | get_task_struct(tsk); |
190 | *per_cpu_ptr(ht->store, cpu) = tsk; | |
f2530dc7 TG |
191 | if (ht->create) { |
192 | /* | |
193 | * Make sure that the task has actually scheduled out | |
194 | * into park position, before calling the create | |
195 | * callback. At least the migration thread callback | |
196 | * requires that the task is off the runqueue. | |
197 | */ | |
198 | if (!wait_task_inactive(tsk, TASK_PARKED)) | |
199 | WARN_ON(1); | |
200 | else | |
201 | ht->create(cpu); | |
202 | } | |
f97f8f06 TG |
203 | return 0; |
204 | } | |
205 | ||
206 | int smpboot_create_threads(unsigned int cpu) | |
207 | { | |
208 | struct smp_hotplug_thread *cur; | |
209 | int ret = 0; | |
210 | ||
211 | mutex_lock(&smpboot_threads_lock); | |
212 | list_for_each_entry(cur, &hotplug_threads, list) { | |
213 | ret = __smpboot_create_thread(cur, cpu); | |
214 | if (ret) | |
215 | break; | |
216 | } | |
217 | mutex_unlock(&smpboot_threads_lock); | |
218 | return ret; | |
219 | } | |
220 | ||
221 | static void smpboot_unpark_thread(struct smp_hotplug_thread *ht, unsigned int cpu) | |
222 | { | |
223 | struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); | |
224 | ||
c00166d8 ON |
225 | if (!ht->selfparking) |
226 | kthread_unpark(tsk); | |
f97f8f06 TG |
227 | } |
228 | ||
229 | void smpboot_unpark_threads(unsigned int cpu) | |
230 | { | |
231 | struct smp_hotplug_thread *cur; | |
232 | ||
233 | mutex_lock(&smpboot_threads_lock); | |
234 | list_for_each_entry(cur, &hotplug_threads, list) | |
b5242e98 CM |
235 | if (cpumask_test_cpu(cpu, cur->cpumask)) |
236 | smpboot_unpark_thread(cur, cpu); | |
f97f8f06 TG |
237 | mutex_unlock(&smpboot_threads_lock); |
238 | } | |
239 | ||
240 | static void smpboot_park_thread(struct smp_hotplug_thread *ht, unsigned int cpu) | |
241 | { | |
242 | struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); | |
243 | ||
7d7e499f | 244 | if (tsk && !ht->selfparking) |
f97f8f06 TG |
245 | kthread_park(tsk); |
246 | } | |
247 | ||
248 | void smpboot_park_threads(unsigned int cpu) | |
249 | { | |
250 | struct smp_hotplug_thread *cur; | |
251 | ||
252 | mutex_lock(&smpboot_threads_lock); | |
253 | list_for_each_entry_reverse(cur, &hotplug_threads, list) | |
254 | smpboot_park_thread(cur, cpu); | |
255 | mutex_unlock(&smpboot_threads_lock); | |
256 | } | |
257 | ||
258 | static void smpboot_destroy_threads(struct smp_hotplug_thread *ht) | |
259 | { | |
260 | unsigned int cpu; | |
261 | ||
262 | /* We need to destroy also the parked threads of offline cpus */ | |
263 | for_each_possible_cpu(cpu) { | |
264 | struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); | |
265 | ||
266 | if (tsk) { | |
267 | kthread_stop(tsk); | |
268 | put_task_struct(tsk); | |
269 | *per_cpu_ptr(ht->store, cpu) = NULL; | |
270 | } | |
271 | } | |
272 | } | |
273 | ||
274 | /** | |
230ec939 FW |
275 | * smpboot_register_percpu_thread_cpumask - Register a per_cpu thread related |
276 | * to hotplug | |
f97f8f06 | 277 | * @plug_thread: Hotplug thread descriptor |
230ec939 | 278 | * @cpumask: The cpumask where threads run |
f97f8f06 TG |
279 | * |
280 | * Creates and starts the threads on all online cpus. | |
281 | */ | |
230ec939 FW |
282 | int smpboot_register_percpu_thread_cpumask(struct smp_hotplug_thread *plug_thread, |
283 | const struct cpumask *cpumask) | |
f97f8f06 TG |
284 | { |
285 | unsigned int cpu; | |
286 | int ret = 0; | |
287 | ||
b5242e98 CM |
288 | if (!alloc_cpumask_var(&plug_thread->cpumask, GFP_KERNEL)) |
289 | return -ENOMEM; | |
230ec939 | 290 | cpumask_copy(plug_thread->cpumask, cpumask); |
b5242e98 | 291 | |
4bee9686 | 292 | get_online_cpus(); |
f97f8f06 TG |
293 | mutex_lock(&smpboot_threads_lock); |
294 | for_each_online_cpu(cpu) { | |
295 | ret = __smpboot_create_thread(plug_thread, cpu); | |
296 | if (ret) { | |
297 | smpboot_destroy_threads(plug_thread); | |
5869b506 | 298 | free_cpumask_var(plug_thread->cpumask); |
f97f8f06 TG |
299 | goto out; |
300 | } | |
230ec939 FW |
301 | if (cpumask_test_cpu(cpu, cpumask)) |
302 | smpboot_unpark_thread(plug_thread, cpu); | |
f97f8f06 TG |
303 | } |
304 | list_add(&plug_thread->list, &hotplug_threads); | |
305 | out: | |
306 | mutex_unlock(&smpboot_threads_lock); | |
4bee9686 | 307 | put_online_cpus(); |
f97f8f06 TG |
308 | return ret; |
309 | } | |
230ec939 | 310 | EXPORT_SYMBOL_GPL(smpboot_register_percpu_thread_cpumask); |
f97f8f06 TG |
311 | |
312 | /** | |
313 | * smpboot_unregister_percpu_thread - Unregister a per_cpu thread related to hotplug | |
314 | * @plug_thread: Hotplug thread descriptor | |
315 | * | |
316 | * Stops all threads on all possible cpus. | |
317 | */ | |
318 | void smpboot_unregister_percpu_thread(struct smp_hotplug_thread *plug_thread) | |
319 | { | |
320 | get_online_cpus(); | |
321 | mutex_lock(&smpboot_threads_lock); | |
322 | list_del(&plug_thread->list); | |
323 | smpboot_destroy_threads(plug_thread); | |
324 | mutex_unlock(&smpboot_threads_lock); | |
325 | put_online_cpus(); | |
b5242e98 | 326 | free_cpumask_var(plug_thread->cpumask); |
f97f8f06 TG |
327 | } |
328 | EXPORT_SYMBOL_GPL(smpboot_unregister_percpu_thread); | |
8038dad7 | 329 | |
b5242e98 CM |
330 | /** |
331 | * smpboot_update_cpumask_percpu_thread - Adjust which per_cpu hotplug threads stay parked | |
332 | * @plug_thread: Hotplug thread descriptor | |
333 | * @new: Revised mask to use | |
334 | * | |
335 | * The cpumask field in the smp_hotplug_thread must not be updated directly | |
336 | * by the client, but only by calling this function. | |
fe4ba3c3 | 337 | * This function can only be called on a registered smp_hotplug_thread. |
b5242e98 CM |
338 | */ |
339 | int smpboot_update_cpumask_percpu_thread(struct smp_hotplug_thread *plug_thread, | |
340 | const struct cpumask *new) | |
341 | { | |
342 | struct cpumask *old = plug_thread->cpumask; | |
343 | cpumask_var_t tmp; | |
344 | unsigned int cpu; | |
345 | ||
346 | if (!alloc_cpumask_var(&tmp, GFP_KERNEL)) | |
347 | return -ENOMEM; | |
348 | ||
349 | get_online_cpus(); | |
350 | mutex_lock(&smpboot_threads_lock); | |
351 | ||
352 | /* Park threads that were exclusively enabled on the old mask. */ | |
353 | cpumask_andnot(tmp, old, new); | |
354 | for_each_cpu_and(cpu, tmp, cpu_online_mask) | |
355 | smpboot_park_thread(plug_thread, cpu); | |
356 | ||
357 | /* Unpark threads that are exclusively enabled on the new mask. */ | |
358 | cpumask_andnot(tmp, new, old); | |
359 | for_each_cpu_and(cpu, tmp, cpu_online_mask) | |
360 | smpboot_unpark_thread(plug_thread, cpu); | |
361 | ||
362 | cpumask_copy(old, new); | |
363 | ||
364 | mutex_unlock(&smpboot_threads_lock); | |
365 | put_online_cpus(); | |
366 | ||
367 | free_cpumask_var(tmp); | |
368 | ||
369 | return 0; | |
370 | } | |
371 | EXPORT_SYMBOL_GPL(smpboot_update_cpumask_percpu_thread); | |
372 | ||
8038dad7 PM |
373 | static DEFINE_PER_CPU(atomic_t, cpu_hotplug_state) = ATOMIC_INIT(CPU_POST_DEAD); |
374 | ||
375 | /* | |
376 | * Called to poll specified CPU's state, for example, when waiting for | |
377 | * a CPU to come online. | |
378 | */ | |
379 | int cpu_report_state(int cpu) | |
380 | { | |
381 | return atomic_read(&per_cpu(cpu_hotplug_state, cpu)); | |
382 | } | |
383 | ||
384 | /* | |
385 | * If CPU has died properly, set its state to CPU_UP_PREPARE and | |
386 | * return success. Otherwise, return -EBUSY if the CPU died after | |
387 | * cpu_wait_death() timed out. And yet otherwise again, return -EAGAIN | |
388 | * if cpu_wait_death() timed out and the CPU still hasn't gotten around | |
389 | * to dying. In the latter two cases, the CPU might not be set up | |
390 | * properly, but it is up to the arch-specific code to decide. | |
391 | * Finally, -EIO indicates an unanticipated problem. | |
392 | * | |
393 | * Note that it is permissible to omit this call entirely, as is | |
394 | * done in architectures that do no CPU-hotplug error checking. | |
395 | */ | |
396 | int cpu_check_up_prepare(int cpu) | |
397 | { | |
398 | if (!IS_ENABLED(CONFIG_HOTPLUG_CPU)) { | |
399 | atomic_set(&per_cpu(cpu_hotplug_state, cpu), CPU_UP_PREPARE); | |
400 | return 0; | |
401 | } | |
402 | ||
403 | switch (atomic_read(&per_cpu(cpu_hotplug_state, cpu))) { | |
404 | ||
405 | case CPU_POST_DEAD: | |
406 | ||
407 | /* The CPU died properly, so just start it up again. */ | |
408 | atomic_set(&per_cpu(cpu_hotplug_state, cpu), CPU_UP_PREPARE); | |
409 | return 0; | |
410 | ||
411 | case CPU_DEAD_FROZEN: | |
412 | ||
413 | /* | |
414 | * Timeout during CPU death, so let caller know. | |
415 | * The outgoing CPU completed its processing, but after | |
416 | * cpu_wait_death() timed out and reported the error. The | |
417 | * caller is free to proceed, in which case the state | |
418 | * will be reset properly by cpu_set_state_online(). | |
419 | * Proceeding despite this -EBUSY return makes sense | |
420 | * for systems where the outgoing CPUs take themselves | |
421 | * offline, with no post-death manipulation required from | |
422 | * a surviving CPU. | |
423 | */ | |
424 | return -EBUSY; | |
425 | ||
426 | case CPU_BROKEN: | |
427 | ||
428 | /* | |
429 | * The most likely reason we got here is that there was | |
430 | * a timeout during CPU death, and the outgoing CPU never | |
431 | * did complete its processing. This could happen on | |
432 | * a virtualized system if the outgoing VCPU gets preempted | |
433 | * for more than five seconds, and the user attempts to | |
434 | * immediately online that same CPU. Trying again later | |
435 | * might return -EBUSY above, hence -EAGAIN. | |
436 | */ | |
437 | return -EAGAIN; | |
438 | ||
439 | default: | |
440 | ||
441 | /* Should not happen. Famous last words. */ | |
442 | return -EIO; | |
443 | } | |
444 | } | |
445 | ||
446 | /* | |
447 | * Mark the specified CPU online. | |
448 | * | |
449 | * Note that it is permissible to omit this call entirely, as is | |
450 | * done in architectures that do no CPU-hotplug error checking. | |
451 | */ | |
452 | void cpu_set_state_online(int cpu) | |
453 | { | |
454 | (void)atomic_xchg(&per_cpu(cpu_hotplug_state, cpu), CPU_ONLINE); | |
455 | } | |
456 | ||
457 | #ifdef CONFIG_HOTPLUG_CPU | |
458 | ||
459 | /* | |
460 | * Wait for the specified CPU to exit the idle loop and die. | |
461 | */ | |
462 | bool cpu_wait_death(unsigned int cpu, int seconds) | |
463 | { | |
464 | int jf_left = seconds * HZ; | |
465 | int oldstate; | |
466 | bool ret = true; | |
467 | int sleep_jf = 1; | |
468 | ||
469 | might_sleep(); | |
470 | ||
471 | /* The outgoing CPU will normally get done quite quickly. */ | |
472 | if (atomic_read(&per_cpu(cpu_hotplug_state, cpu)) == CPU_DEAD) | |
473 | goto update_state; | |
474 | udelay(5); | |
475 | ||
476 | /* But if the outgoing CPU dawdles, wait increasingly long times. */ | |
477 | while (atomic_read(&per_cpu(cpu_hotplug_state, cpu)) != CPU_DEAD) { | |
478 | schedule_timeout_uninterruptible(sleep_jf); | |
479 | jf_left -= sleep_jf; | |
480 | if (jf_left <= 0) | |
481 | break; | |
482 | sleep_jf = DIV_ROUND_UP(sleep_jf * 11, 10); | |
483 | } | |
484 | update_state: | |
485 | oldstate = atomic_read(&per_cpu(cpu_hotplug_state, cpu)); | |
486 | if (oldstate == CPU_DEAD) { | |
487 | /* Outgoing CPU died normally, update state. */ | |
488 | smp_mb(); /* atomic_read() before update. */ | |
489 | atomic_set(&per_cpu(cpu_hotplug_state, cpu), CPU_POST_DEAD); | |
490 | } else { | |
491 | /* Outgoing CPU still hasn't died, set state accordingly. */ | |
492 | if (atomic_cmpxchg(&per_cpu(cpu_hotplug_state, cpu), | |
493 | oldstate, CPU_BROKEN) != oldstate) | |
494 | goto update_state; | |
495 | ret = false; | |
496 | } | |
497 | return ret; | |
498 | } | |
499 | ||
500 | /* | |
501 | * Called by the outgoing CPU to report its successful death. Return | |
502 | * false if this report follows the surviving CPU's timing out. | |
503 | * | |
504 | * A separate "CPU_DEAD_FROZEN" is used when the surviving CPU | |
505 | * timed out. This approach allows architectures to omit calls to | |
506 | * cpu_check_up_prepare() and cpu_set_state_online() without defeating | |
507 | * the next cpu_wait_death()'s polling loop. | |
508 | */ | |
509 | bool cpu_report_death(void) | |
510 | { | |
511 | int oldstate; | |
512 | int newstate; | |
513 | int cpu = smp_processor_id(); | |
514 | ||
515 | do { | |
516 | oldstate = atomic_read(&per_cpu(cpu_hotplug_state, cpu)); | |
517 | if (oldstate != CPU_BROKEN) | |
518 | newstate = CPU_DEAD; | |
519 | else | |
520 | newstate = CPU_DEAD_FROZEN; | |
521 | } while (atomic_cmpxchg(&per_cpu(cpu_hotplug_state, cpu), | |
522 | oldstate, newstate) != oldstate); | |
523 | return newstate == CPU_DEAD; | |
524 | } | |
525 | ||
526 | #endif /* #ifdef CONFIG_HOTPLUG_CPU */ |