2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
4 * This code is licenced under the GPL.
6 #include <linux/proc_fs.h>
8 #include <linux/init.h>
9 #include <linux/notifier.h>
10 #include <linux/sched/signal.h>
11 #include <linux/sched/hotplug.h>
12 #include <linux/sched/task.h>
13 #include <linux/unistd.h>
14 #include <linux/cpu.h>
15 #include <linux/oom.h>
16 #include <linux/rcupdate.h>
17 #include <linux/export.h>
18 #include <linux/bug.h>
19 #include <linux/kthread.h>
20 #include <linux/stop_machine.h>
21 #include <linux/mutex.h>
22 #include <linux/gfp.h>
23 #include <linux/suspend.h>
24 #include <linux/lockdep.h>
25 #include <linux/tick.h>
26 #include <linux/irq.h>
27 #include <linux/nmi.h>
28 #include <linux/smpboot.h>
29 #include <linux/relay.h>
30 #include <linux/slab.h>
31 #include <linux/percpu-rwsem.h>
33 #include <trace/events/power.h>
34 #define CREATE_TRACE_POINTS
35 #include <trace/events/cpuhp.h>
40 * cpuhp_cpu_state - Per cpu hotplug state storage
41 * @state: The current cpu state
42 * @target: The target state
43 * @thread: Pointer to the hotplug thread
44 * @should_run: Thread should execute
45 * @rollback: Perform a rollback
46 * @single: Single callback invocation
47 * @bringup: Single callback bringup or teardown selector
48 * @cb_state: The state for a single callback (install/uninstall)
49 * @result: Result of the operation
50 * @done_up: Signal completion to the issuer of the task for cpu-up
51 * @done_down: Signal completion to the issuer of the task for cpu-down
53 struct cpuhp_cpu_state
{
54 enum cpuhp_state state
;
55 enum cpuhp_state target
;
56 enum cpuhp_state fail
;
58 struct task_struct
*thread
;
63 struct hlist_node
*node
;
64 struct hlist_node
*last
;
65 enum cpuhp_state cb_state
;
67 struct completion done_up
;
68 struct completion done_down
;
72 static DEFINE_PER_CPU(struct cpuhp_cpu_state
, cpuhp_state
) = {
73 .fail
= CPUHP_INVALID
,
76 #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
77 static struct lockdep_map cpuhp_state_up_map
=
78 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map
);
79 static struct lockdep_map cpuhp_state_down_map
=
80 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map
);
83 static inline void cpuhp_lock_acquire(bool bringup
)
85 lock_map_acquire(bringup
? &cpuhp_state_up_map
: &cpuhp_state_down_map
);
88 static inline void cpuhp_lock_release(bool bringup
)
90 lock_map_release(bringup
? &cpuhp_state_up_map
: &cpuhp_state_down_map
);
94 static inline void cpuhp_lock_acquire(bool bringup
) { }
95 static inline void cpuhp_lock_release(bool bringup
) { }
100 * cpuhp_step - Hotplug state machine step
101 * @name: Name of the step
102 * @startup: Startup function of the step
103 * @teardown: Teardown function of the step
104 * @skip_onerr: Do not invoke the functions on error rollback
105 * Will go away once the notifiers are gone
106 * @cant_stop: Bringup/teardown can't be stopped at this step
111 int (*single
)(unsigned int cpu
);
112 int (*multi
)(unsigned int cpu
,
113 struct hlist_node
*node
);
116 int (*single
)(unsigned int cpu
);
117 int (*multi
)(unsigned int cpu
,
118 struct hlist_node
*node
);
120 struct hlist_head list
;
126 static DEFINE_MUTEX(cpuhp_state_mutex
);
127 static struct cpuhp_step cpuhp_hp_states
[];
129 static struct cpuhp_step
*cpuhp_get_step(enum cpuhp_state state
)
131 return cpuhp_hp_states
+ state
;
135 * cpuhp_invoke_callback _ Invoke the callbacks for a given state
136 * @cpu: The cpu for which the callback should be invoked
137 * @state: The state to do callbacks for
138 * @bringup: True if the bringup callback should be invoked
139 * @node: For multi-instance, do a single entry callback for install/remove
140 * @lastp: For multi-instance rollback, remember how far we got
142 * Called from cpu hotplug and from the state register machinery.
144 static int cpuhp_invoke_callback(unsigned int cpu
, enum cpuhp_state state
,
145 bool bringup
, struct hlist_node
*node
,
146 struct hlist_node
**lastp
)
148 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
149 struct cpuhp_step
*step
= cpuhp_get_step(state
);
150 int (*cbm
)(unsigned int cpu
, struct hlist_node
*node
);
151 int (*cb
)(unsigned int cpu
);
154 if (st
->fail
== state
) {
155 st
->fail
= CPUHP_INVALID
;
157 if (!(bringup
? step
->startup
.single
: step
->teardown
.single
))
163 if (!step
->multi_instance
) {
164 WARN_ON_ONCE(lastp
&& *lastp
);
165 cb
= bringup
? step
->startup
.single
: step
->teardown
.single
;
168 trace_cpuhp_enter(cpu
, st
->target
, state
, cb
);
170 trace_cpuhp_exit(cpu
, st
->state
, state
, ret
);
173 cbm
= bringup
? step
->startup
.multi
: step
->teardown
.multi
;
177 /* Single invocation for instance add/remove */
179 WARN_ON_ONCE(lastp
&& *lastp
);
180 trace_cpuhp_multi_enter(cpu
, st
->target
, state
, cbm
, node
);
181 ret
= cbm(cpu
, node
);
182 trace_cpuhp_exit(cpu
, st
->state
, state
, ret
);
186 /* State transition. Invoke on all instances */
188 hlist_for_each(node
, &step
->list
) {
189 if (lastp
&& node
== *lastp
)
192 trace_cpuhp_multi_enter(cpu
, st
->target
, state
, cbm
, node
);
193 ret
= cbm(cpu
, node
);
194 trace_cpuhp_exit(cpu
, st
->state
, state
, ret
);
208 /* Rollback the instances if one failed */
209 cbm
= !bringup
? step
->startup
.multi
: step
->teardown
.multi
;
213 hlist_for_each(node
, &step
->list
) {
217 trace_cpuhp_multi_enter(cpu
, st
->target
, state
, cbm
, node
);
218 ret
= cbm(cpu
, node
);
219 trace_cpuhp_exit(cpu
, st
->state
, state
, ret
);
221 * Rollback must not fail,
229 static bool cpuhp_is_ap_state(enum cpuhp_state state
)
232 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
233 * purposes as that state is handled explicitly in cpu_down.
235 return state
> CPUHP_BRINGUP_CPU
&& state
!= CPUHP_TEARDOWN_CPU
;
238 static inline void wait_for_ap_thread(struct cpuhp_cpu_state
*st
, bool bringup
)
240 struct completion
*done
= bringup
? &st
->done_up
: &st
->done_down
;
241 wait_for_completion(done
);
244 static inline void complete_ap_thread(struct cpuhp_cpu_state
*st
, bool bringup
)
246 struct completion
*done
= bringup
? &st
->done_up
: &st
->done_down
;
251 * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
253 static bool cpuhp_is_atomic_state(enum cpuhp_state state
)
255 return CPUHP_AP_IDLE_DEAD
<= state
&& state
< CPUHP_AP_ONLINE
;
258 /* Serializes the updates to cpu_online_mask, cpu_present_mask */
259 static DEFINE_MUTEX(cpu_add_remove_lock
);
260 bool cpuhp_tasks_frozen
;
261 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen
);
264 * The following two APIs (cpu_maps_update_begin/done) must be used when
265 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
267 void cpu_maps_update_begin(void)
269 mutex_lock(&cpu_add_remove_lock
);
272 void cpu_maps_update_done(void)
274 mutex_unlock(&cpu_add_remove_lock
);
278 * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
279 * Should always be manipulated under cpu_add_remove_lock
281 static int cpu_hotplug_disabled
;
283 #ifdef CONFIG_HOTPLUG_CPU
285 DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock
);
287 void cpus_read_lock(void)
289 percpu_down_read(&cpu_hotplug_lock
);
291 EXPORT_SYMBOL_GPL(cpus_read_lock
);
293 void cpus_read_unlock(void)
295 percpu_up_read(&cpu_hotplug_lock
);
297 EXPORT_SYMBOL_GPL(cpus_read_unlock
);
299 void cpus_write_lock(void)
301 percpu_down_write(&cpu_hotplug_lock
);
304 void cpus_write_unlock(void)
306 percpu_up_write(&cpu_hotplug_lock
);
309 void lockdep_assert_cpus_held(void)
311 percpu_rwsem_assert_held(&cpu_hotplug_lock
);
315 * Wait for currently running CPU hotplug operations to complete (if any) and
316 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
317 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
318 * hotplug path before performing hotplug operations. So acquiring that lock
319 * guarantees mutual exclusion from any currently running hotplug operations.
321 void cpu_hotplug_disable(void)
323 cpu_maps_update_begin();
324 cpu_hotplug_disabled
++;
325 cpu_maps_update_done();
327 EXPORT_SYMBOL_GPL(cpu_hotplug_disable
);
329 static void __cpu_hotplug_enable(void)
331 if (WARN_ONCE(!cpu_hotplug_disabled
, "Unbalanced cpu hotplug enable\n"))
333 cpu_hotplug_disabled
--;
336 void cpu_hotplug_enable(void)
338 cpu_maps_update_begin();
339 __cpu_hotplug_enable();
340 cpu_maps_update_done();
342 EXPORT_SYMBOL_GPL(cpu_hotplug_enable
);
343 #endif /* CONFIG_HOTPLUG_CPU */
345 static inline enum cpuhp_state
346 cpuhp_set_state(struct cpuhp_cpu_state
*st
, enum cpuhp_state target
)
348 enum cpuhp_state prev_state
= st
->state
;
350 st
->rollback
= false;
355 st
->bringup
= st
->state
< target
;
361 cpuhp_reset_state(struct cpuhp_cpu_state
*st
, enum cpuhp_state prev_state
)
366 * If we have st->last we need to undo partial multi_instance of this
367 * state first. Otherwise start undo at the previous state.
376 st
->target
= prev_state
;
377 st
->bringup
= !st
->bringup
;
380 /* Regular hotplug invocation of the AP hotplug thread */
381 static void __cpuhp_kick_ap(struct cpuhp_cpu_state
*st
)
383 if (!st
->single
&& st
->state
== st
->target
)
388 * Make sure the above stores are visible before should_run becomes
389 * true. Paired with the mb() above in cpuhp_thread_fun()
392 st
->should_run
= true;
393 wake_up_process(st
->thread
);
394 wait_for_ap_thread(st
, st
->bringup
);
397 static int cpuhp_kick_ap(struct cpuhp_cpu_state
*st
, enum cpuhp_state target
)
399 enum cpuhp_state prev_state
;
402 prev_state
= cpuhp_set_state(st
, target
);
404 if ((ret
= st
->result
)) {
405 cpuhp_reset_state(st
, prev_state
);
412 static int bringup_wait_for_ap(unsigned int cpu
)
414 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
416 /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
417 wait_for_ap_thread(st
, true);
418 if (WARN_ON_ONCE((!cpu_online(cpu
))))
421 /* Unpark the stopper thread and the hotplug thread of the target cpu */
422 stop_machine_unpark(cpu
);
423 kthread_unpark(st
->thread
);
425 if (st
->target
<= CPUHP_AP_ONLINE_IDLE
)
428 return cpuhp_kick_ap(st
, st
->target
);
431 static int bringup_cpu(unsigned int cpu
)
433 struct task_struct
*idle
= idle_thread_get(cpu
);
437 * Some architectures have to walk the irq descriptors to
438 * setup the vector space for the cpu which comes online.
439 * Prevent irq alloc/free across the bringup.
443 /* Arch-specific enabling code. */
444 ret
= __cpu_up(cpu
, idle
);
448 return bringup_wait_for_ap(cpu
);
452 * Hotplug state machine related functions
455 static void undo_cpu_up(unsigned int cpu
, struct cpuhp_cpu_state
*st
)
457 for (st
->state
--; st
->state
> st
->target
; st
->state
--) {
458 struct cpuhp_step
*step
= cpuhp_get_step(st
->state
);
460 if (!step
->skip_onerr
)
461 cpuhp_invoke_callback(cpu
, st
->state
, false, NULL
, NULL
);
465 static int cpuhp_up_callbacks(unsigned int cpu
, struct cpuhp_cpu_state
*st
,
466 enum cpuhp_state target
)
468 enum cpuhp_state prev_state
= st
->state
;
471 while (st
->state
< target
) {
473 ret
= cpuhp_invoke_callback(cpu
, st
->state
, true, NULL
, NULL
);
475 st
->target
= prev_state
;
476 undo_cpu_up(cpu
, st
);
484 * The cpu hotplug threads manage the bringup and teardown of the cpus
486 static void cpuhp_create(unsigned int cpu
)
488 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
490 init_completion(&st
->done_up
);
491 init_completion(&st
->done_down
);
494 static int cpuhp_should_run(unsigned int cpu
)
496 struct cpuhp_cpu_state
*st
= this_cpu_ptr(&cpuhp_state
);
498 return st
->should_run
;
502 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
503 * callbacks when a state gets [un]installed at runtime.
505 * Each invocation of this function by the smpboot thread does a single AP
508 * It has 3 modes of operation:
509 * - single: runs st->cb_state
510 * - up: runs ++st->state, while st->state < st->target
511 * - down: runs st->state--, while st->state > st->target
513 * When complete or on error, should_run is cleared and the completion is fired.
515 static void cpuhp_thread_fun(unsigned int cpu
)
517 struct cpuhp_cpu_state
*st
= this_cpu_ptr(&cpuhp_state
);
518 bool bringup
= st
->bringup
;
519 enum cpuhp_state state
;
522 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
523 * that if we see ->should_run we also see the rest of the state.
527 if (WARN_ON_ONCE(!st
->should_run
))
530 cpuhp_lock_acquire(bringup
);
533 state
= st
->cb_state
;
534 st
->should_run
= false;
539 st
->should_run
= (st
->state
< st
->target
);
540 WARN_ON_ONCE(st
->state
> st
->target
);
544 st
->should_run
= (st
->state
> st
->target
);
545 WARN_ON_ONCE(st
->state
< st
->target
);
549 WARN_ON_ONCE(!cpuhp_is_ap_state(state
));
552 struct cpuhp_step
*step
= cpuhp_get_step(state
);
553 if (step
->skip_onerr
)
557 if (cpuhp_is_atomic_state(state
)) {
559 st
->result
= cpuhp_invoke_callback(cpu
, state
, bringup
, st
->node
, &st
->last
);
563 * STARTING/DYING must not fail!
565 WARN_ON_ONCE(st
->result
);
567 st
->result
= cpuhp_invoke_callback(cpu
, state
, bringup
, st
->node
, &st
->last
);
572 * If we fail on a rollback, we're up a creek without no
573 * paddle, no way forward, no way back. We loose, thanks for
576 WARN_ON_ONCE(st
->rollback
);
577 st
->should_run
= false;
581 cpuhp_lock_release(bringup
);
584 complete_ap_thread(st
, bringup
);
587 /* Invoke a single callback on a remote cpu */
589 cpuhp_invoke_ap_callback(int cpu
, enum cpuhp_state state
, bool bringup
,
590 struct hlist_node
*node
)
592 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
595 if (!cpu_online(cpu
))
598 cpuhp_lock_acquire(false);
599 cpuhp_lock_release(false);
601 cpuhp_lock_acquire(true);
602 cpuhp_lock_release(true);
605 * If we are up and running, use the hotplug thread. For early calls
606 * we invoke the thread function directly.
609 return cpuhp_invoke_callback(cpu
, state
, bringup
, node
, NULL
);
611 st
->rollback
= false;
615 st
->bringup
= bringup
;
616 st
->cb_state
= state
;
622 * If we failed and did a partial, do a rollback.
624 if ((ret
= st
->result
) && st
->last
) {
626 st
->bringup
= !bringup
;
632 * Clean up the leftovers so the next hotplug operation wont use stale
635 st
->node
= st
->last
= NULL
;
639 static int cpuhp_kick_ap_work(unsigned int cpu
)
641 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
642 enum cpuhp_state prev_state
= st
->state
;
645 cpuhp_lock_acquire(false);
646 cpuhp_lock_release(false);
648 cpuhp_lock_acquire(true);
649 cpuhp_lock_release(true);
651 trace_cpuhp_enter(cpu
, st
->target
, prev_state
, cpuhp_kick_ap_work
);
652 ret
= cpuhp_kick_ap(st
, st
->target
);
653 trace_cpuhp_exit(cpu
, st
->state
, prev_state
, ret
);
658 static struct smp_hotplug_thread cpuhp_threads
= {
659 .store
= &cpuhp_state
.thread
,
660 .create
= &cpuhp_create
,
661 .thread_should_run
= cpuhp_should_run
,
662 .thread_fn
= cpuhp_thread_fun
,
663 .thread_comm
= "cpuhp/%u",
667 void __init
cpuhp_threads_init(void)
669 BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads
));
670 kthread_unpark(this_cpu_read(cpuhp_state
.thread
));
673 #ifdef CONFIG_HOTPLUG_CPU
675 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
678 * This function walks all processes, finds a valid mm struct for each one and
679 * then clears a corresponding bit in mm's cpumask. While this all sounds
680 * trivial, there are various non-obvious corner cases, which this function
681 * tries to solve in a safe manner.
683 * Also note that the function uses a somewhat relaxed locking scheme, so it may
684 * be called only for an already offlined CPU.
686 void clear_tasks_mm_cpumask(int cpu
)
688 struct task_struct
*p
;
691 * This function is called after the cpu is taken down and marked
692 * offline, so its not like new tasks will ever get this cpu set in
693 * their mm mask. -- Peter Zijlstra
694 * Thus, we may use rcu_read_lock() here, instead of grabbing
695 * full-fledged tasklist_lock.
697 WARN_ON(cpu_online(cpu
));
699 for_each_process(p
) {
700 struct task_struct
*t
;
703 * Main thread might exit, but other threads may still have
704 * a valid mm. Find one.
706 t
= find_lock_task_mm(p
);
709 cpumask_clear_cpu(cpu
, mm_cpumask(t
->mm
));
715 /* Take this CPU down. */
716 static int take_cpu_down(void *_param
)
718 struct cpuhp_cpu_state
*st
= this_cpu_ptr(&cpuhp_state
);
719 enum cpuhp_state target
= max((int)st
->target
, CPUHP_AP_OFFLINE
);
720 int err
, cpu
= smp_processor_id();
723 /* Ensure this CPU doesn't handle any more interrupts. */
724 err
= __cpu_disable();
729 * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
730 * do this step again.
732 WARN_ON(st
->state
!= CPUHP_TEARDOWN_CPU
);
734 /* Invoke the former CPU_DYING callbacks */
735 for (; st
->state
> target
; st
->state
--) {
736 ret
= cpuhp_invoke_callback(cpu
, st
->state
, false, NULL
, NULL
);
738 * DYING must not fail!
743 /* Give up timekeeping duties */
744 tick_handover_do_timer();
745 /* Park the stopper thread */
746 stop_machine_park(cpu
);
750 static int takedown_cpu(unsigned int cpu
)
752 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
755 /* Park the smpboot threads */
756 kthread_park(per_cpu_ptr(&cpuhp_state
, cpu
)->thread
);
759 * Prevent irq alloc/free while the dying cpu reorganizes the
760 * interrupt affinities.
765 * So now all preempt/rcu users must observe !cpu_active().
767 err
= stop_machine_cpuslocked(take_cpu_down
, NULL
, cpumask_of(cpu
));
769 /* CPU refused to die */
771 /* Unpark the hotplug thread so we can rollback there */
772 kthread_unpark(per_cpu_ptr(&cpuhp_state
, cpu
)->thread
);
775 BUG_ON(cpu_online(cpu
));
778 * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
779 * all runnable tasks from the CPU, there's only the idle task left now
780 * that the migration thread is done doing the stop_machine thing.
782 * Wait for the stop thread to go away.
784 wait_for_ap_thread(st
, false);
785 BUG_ON(st
->state
!= CPUHP_AP_IDLE_DEAD
);
787 /* Interrupts are moved away from the dying cpu, reenable alloc/free */
790 hotplug_cpu__broadcast_tick_pull(cpu
);
791 /* This actually kills the CPU. */
794 tick_cleanup_dead_cpu(cpu
);
795 rcutree_migrate_callbacks(cpu
);
799 static void cpuhp_complete_idle_dead(void *arg
)
801 struct cpuhp_cpu_state
*st
= arg
;
803 complete_ap_thread(st
, false);
806 void cpuhp_report_idle_dead(void)
808 struct cpuhp_cpu_state
*st
= this_cpu_ptr(&cpuhp_state
);
810 BUG_ON(st
->state
!= CPUHP_AP_OFFLINE
);
811 rcu_report_dead(smp_processor_id());
812 st
->state
= CPUHP_AP_IDLE_DEAD
;
814 * We cannot call complete after rcu_report_dead() so we delegate it
817 smp_call_function_single(cpumask_first(cpu_online_mask
),
818 cpuhp_complete_idle_dead
, st
, 0);
821 static void undo_cpu_down(unsigned int cpu
, struct cpuhp_cpu_state
*st
)
823 for (st
->state
++; st
->state
< st
->target
; st
->state
++) {
824 struct cpuhp_step
*step
= cpuhp_get_step(st
->state
);
826 if (!step
->skip_onerr
)
827 cpuhp_invoke_callback(cpu
, st
->state
, true, NULL
, NULL
);
831 static int cpuhp_down_callbacks(unsigned int cpu
, struct cpuhp_cpu_state
*st
,
832 enum cpuhp_state target
)
834 enum cpuhp_state prev_state
= st
->state
;
837 for (; st
->state
> target
; st
->state
--) {
838 ret
= cpuhp_invoke_callback(cpu
, st
->state
, false, NULL
, NULL
);
840 st
->target
= prev_state
;
841 undo_cpu_down(cpu
, st
);
848 /* Requires cpu_add_remove_lock to be held */
849 static int __ref
_cpu_down(unsigned int cpu
, int tasks_frozen
,
850 enum cpuhp_state target
)
852 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
853 int prev_state
, ret
= 0;
855 if (num_online_cpus() == 1)
858 if (!cpu_present(cpu
))
863 cpuhp_tasks_frozen
= tasks_frozen
;
865 prev_state
= cpuhp_set_state(st
, target
);
867 * If the current CPU state is in the range of the AP hotplug thread,
868 * then we need to kick the thread.
870 if (st
->state
> CPUHP_TEARDOWN_CPU
) {
871 st
->target
= max((int)target
, CPUHP_TEARDOWN_CPU
);
872 ret
= cpuhp_kick_ap_work(cpu
);
874 * The AP side has done the error rollback already. Just
875 * return the error code..
881 * We might have stopped still in the range of the AP hotplug
882 * thread. Nothing to do anymore.
884 if (st
->state
> CPUHP_TEARDOWN_CPU
)
890 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
891 * to do the further cleanups.
893 ret
= cpuhp_down_callbacks(cpu
, st
, target
);
894 if (ret
&& st
->state
> CPUHP_TEARDOWN_CPU
&& st
->state
< prev_state
) {
895 cpuhp_reset_state(st
, prev_state
);
902 * Do post unplug cleanup. This is still protected against
903 * concurrent CPU hotplug via cpu_add_remove_lock.
905 lockup_detector_cleanup();
909 static int do_cpu_down(unsigned int cpu
, enum cpuhp_state target
)
913 cpu_maps_update_begin();
915 if (cpu_hotplug_disabled
) {
920 err
= _cpu_down(cpu
, 0, target
);
923 cpu_maps_update_done();
927 int cpu_down(unsigned int cpu
)
929 return do_cpu_down(cpu
, CPUHP_OFFLINE
);
931 EXPORT_SYMBOL(cpu_down
);
934 #define takedown_cpu NULL
935 #endif /*CONFIG_HOTPLUG_CPU*/
938 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
939 * @cpu: cpu that just started
941 * It must be called by the arch code on the new cpu, before the new cpu
942 * enables interrupts and before the "boot" cpu returns from __cpu_up().
944 void notify_cpu_starting(unsigned int cpu
)
946 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
947 enum cpuhp_state target
= min((int)st
->target
, CPUHP_AP_ONLINE
);
950 rcu_cpu_starting(cpu
); /* Enables RCU usage on this CPU. */
951 while (st
->state
< target
) {
953 ret
= cpuhp_invoke_callback(cpu
, st
->state
, true, NULL
, NULL
);
955 * STARTING must not fail!
962 * Called from the idle task. Wake up the controlling task which brings the
963 * stopper and the hotplug thread of the upcoming CPU up and then delegates
964 * the rest of the online bringup to the hotplug thread.
966 void cpuhp_online_idle(enum cpuhp_state state
)
968 struct cpuhp_cpu_state
*st
= this_cpu_ptr(&cpuhp_state
);
970 /* Happens for the boot cpu */
971 if (state
!= CPUHP_AP_ONLINE_IDLE
)
974 st
->state
= CPUHP_AP_ONLINE_IDLE
;
975 complete_ap_thread(st
, true);
978 /* Requires cpu_add_remove_lock to be held */
979 static int _cpu_up(unsigned int cpu
, int tasks_frozen
, enum cpuhp_state target
)
981 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
982 struct task_struct
*idle
;
987 if (!cpu_present(cpu
)) {
993 * The caller of do_cpu_up might have raced with another
994 * caller. Ignore it for now.
996 if (st
->state
>= target
)
999 if (st
->state
== CPUHP_OFFLINE
) {
1000 /* Let it fail before we try to bring the cpu up */
1001 idle
= idle_thread_get(cpu
);
1003 ret
= PTR_ERR(idle
);
1008 cpuhp_tasks_frozen
= tasks_frozen
;
1010 cpuhp_set_state(st
, target
);
1012 * If the current CPU state is in the range of the AP hotplug thread,
1013 * then we need to kick the thread once more.
1015 if (st
->state
> CPUHP_BRINGUP_CPU
) {
1016 ret
= cpuhp_kick_ap_work(cpu
);
1018 * The AP side has done the error rollback already. Just
1019 * return the error code..
1026 * Try to reach the target state. We max out on the BP at
1027 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1028 * responsible for bringing it up to the target state.
1030 target
= min((int)target
, CPUHP_BRINGUP_CPU
);
1031 ret
= cpuhp_up_callbacks(cpu
, st
, target
);
1033 cpus_write_unlock();
1037 static int do_cpu_up(unsigned int cpu
, enum cpuhp_state target
)
1041 if (!cpu_possible(cpu
)) {
1042 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1044 #if defined(CONFIG_IA64)
1045 pr_err("please check additional_cpus= boot parameter\n");
1050 err
= try_online_node(cpu_to_node(cpu
));
1054 cpu_maps_update_begin();
1056 if (cpu_hotplug_disabled
) {
1061 err
= _cpu_up(cpu
, 0, target
);
1063 cpu_maps_update_done();
1067 int cpu_up(unsigned int cpu
)
1069 return do_cpu_up(cpu
, CPUHP_ONLINE
);
1071 EXPORT_SYMBOL_GPL(cpu_up
);
1073 #ifdef CONFIG_PM_SLEEP_SMP
1074 static cpumask_var_t frozen_cpus
;
1076 int freeze_secondary_cpus(int primary
)
1080 cpu_maps_update_begin();
1081 if (!cpu_online(primary
))
1082 primary
= cpumask_first(cpu_online_mask
);
1084 * We take down all of the non-boot CPUs in one shot to avoid races
1085 * with the userspace trying to use the CPU hotplug at the same time
1087 cpumask_clear(frozen_cpus
);
1089 pr_info("Disabling non-boot CPUs ...\n");
1090 for_each_online_cpu(cpu
) {
1093 trace_suspend_resume(TPS("CPU_OFF"), cpu
, true);
1094 error
= _cpu_down(cpu
, 1, CPUHP_OFFLINE
);
1095 trace_suspend_resume(TPS("CPU_OFF"), cpu
, false);
1097 cpumask_set_cpu(cpu
, frozen_cpus
);
1099 pr_err("Error taking CPU%d down: %d\n", cpu
, error
);
1105 BUG_ON(num_online_cpus() > 1);
1107 pr_err("Non-boot CPUs are not disabled\n");
1110 * Make sure the CPUs won't be enabled by someone else. We need to do
1111 * this even in case of failure as all disable_nonboot_cpus() users are
1112 * supposed to do enable_nonboot_cpus() on the failure path.
1114 cpu_hotplug_disabled
++;
1116 cpu_maps_update_done();
1120 void __weak
arch_enable_nonboot_cpus_begin(void)
1124 void __weak
arch_enable_nonboot_cpus_end(void)
1128 void enable_nonboot_cpus(void)
1132 /* Allow everyone to use the CPU hotplug again */
1133 cpu_maps_update_begin();
1134 __cpu_hotplug_enable();
1135 if (cpumask_empty(frozen_cpus
))
1138 pr_info("Enabling non-boot CPUs ...\n");
1140 arch_enable_nonboot_cpus_begin();
1142 for_each_cpu(cpu
, frozen_cpus
) {
1143 trace_suspend_resume(TPS("CPU_ON"), cpu
, true);
1144 error
= _cpu_up(cpu
, 1, CPUHP_ONLINE
);
1145 trace_suspend_resume(TPS("CPU_ON"), cpu
, false);
1147 pr_info("CPU%d is up\n", cpu
);
1150 pr_warn("Error taking CPU%d up: %d\n", cpu
, error
);
1153 arch_enable_nonboot_cpus_end();
1155 cpumask_clear(frozen_cpus
);
1157 cpu_maps_update_done();
1160 static int __init
alloc_frozen_cpus(void)
1162 if (!alloc_cpumask_var(&frozen_cpus
, GFP_KERNEL
|__GFP_ZERO
))
1166 core_initcall(alloc_frozen_cpus
);
1169 * When callbacks for CPU hotplug notifications are being executed, we must
1170 * ensure that the state of the system with respect to the tasks being frozen
1171 * or not, as reported by the notification, remains unchanged *throughout the
1172 * duration* of the execution of the callbacks.
1173 * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1175 * This synchronization is implemented by mutually excluding regular CPU
1176 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1177 * Hibernate notifications.
1180 cpu_hotplug_pm_callback(struct notifier_block
*nb
,
1181 unsigned long action
, void *ptr
)
1185 case PM_SUSPEND_PREPARE
:
1186 case PM_HIBERNATION_PREPARE
:
1187 cpu_hotplug_disable();
1190 case PM_POST_SUSPEND
:
1191 case PM_POST_HIBERNATION
:
1192 cpu_hotplug_enable();
1203 static int __init
cpu_hotplug_pm_sync_init(void)
1206 * cpu_hotplug_pm_callback has higher priority than x86
1207 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1208 * to disable cpu hotplug to avoid cpu hotplug race.
1210 pm_notifier(cpu_hotplug_pm_callback
, 0);
1213 core_initcall(cpu_hotplug_pm_sync_init
);
1215 #endif /* CONFIG_PM_SLEEP_SMP */
1219 #endif /* CONFIG_SMP */
1221 /* Boot processor state steps */
1222 static struct cpuhp_step cpuhp_hp_states
[] = {
1225 .startup
.single
= NULL
,
1226 .teardown
.single
= NULL
,
1229 [CPUHP_CREATE_THREADS
]= {
1230 .name
= "threads:prepare",
1231 .startup
.single
= smpboot_create_threads
,
1232 .teardown
.single
= NULL
,
1235 [CPUHP_PERF_PREPARE
] = {
1236 .name
= "perf:prepare",
1237 .startup
.single
= perf_event_init_cpu
,
1238 .teardown
.single
= perf_event_exit_cpu
,
1240 [CPUHP_WORKQUEUE_PREP
] = {
1241 .name
= "workqueue:prepare",
1242 .startup
.single
= workqueue_prepare_cpu
,
1243 .teardown
.single
= NULL
,
1245 [CPUHP_HRTIMERS_PREPARE
] = {
1246 .name
= "hrtimers:prepare",
1247 .startup
.single
= hrtimers_prepare_cpu
,
1248 .teardown
.single
= hrtimers_dead_cpu
,
1250 [CPUHP_SMPCFD_PREPARE
] = {
1251 .name
= "smpcfd:prepare",
1252 .startup
.single
= smpcfd_prepare_cpu
,
1253 .teardown
.single
= smpcfd_dead_cpu
,
1255 [CPUHP_RELAY_PREPARE
] = {
1256 .name
= "relay:prepare",
1257 .startup
.single
= relay_prepare_cpu
,
1258 .teardown
.single
= NULL
,
1260 [CPUHP_SLAB_PREPARE
] = {
1261 .name
= "slab:prepare",
1262 .startup
.single
= slab_prepare_cpu
,
1263 .teardown
.single
= slab_dead_cpu
,
1265 [CPUHP_RCUTREE_PREP
] = {
1266 .name
= "RCU/tree:prepare",
1267 .startup
.single
= rcutree_prepare_cpu
,
1268 .teardown
.single
= rcutree_dead_cpu
,
1271 * On the tear-down path, timers_dead_cpu() must be invoked
1272 * before blk_mq_queue_reinit_notify() from notify_dead(),
1273 * otherwise a RCU stall occurs.
1275 [CPUHP_TIMERS_PREPARE
] = {
1276 .name
= "timers:dead",
1277 .startup
.single
= timers_prepare_cpu
,
1278 .teardown
.single
= timers_dead_cpu
,
1280 /* Kicks the plugged cpu into life */
1281 [CPUHP_BRINGUP_CPU
] = {
1282 .name
= "cpu:bringup",
1283 .startup
.single
= bringup_cpu
,
1284 .teardown
.single
= NULL
,
1287 /* Final state before CPU kills itself */
1288 [CPUHP_AP_IDLE_DEAD
] = {
1289 .name
= "idle:dead",
1292 * Last state before CPU enters the idle loop to die. Transient state
1293 * for synchronization.
1295 [CPUHP_AP_OFFLINE
] = {
1296 .name
= "ap:offline",
1299 /* First state is scheduler control. Interrupts are disabled */
1300 [CPUHP_AP_SCHED_STARTING
] = {
1301 .name
= "sched:starting",
1302 .startup
.single
= sched_cpu_starting
,
1303 .teardown
.single
= sched_cpu_dying
,
1305 [CPUHP_AP_RCUTREE_DYING
] = {
1306 .name
= "RCU/tree:dying",
1307 .startup
.single
= NULL
,
1308 .teardown
.single
= rcutree_dying_cpu
,
1310 [CPUHP_AP_SMPCFD_DYING
] = {
1311 .name
= "smpcfd:dying",
1312 .startup
.single
= NULL
,
1313 .teardown
.single
= smpcfd_dying_cpu
,
1315 /* Entry state on starting. Interrupts enabled from here on. Transient
1316 * state for synchronsization */
1317 [CPUHP_AP_ONLINE
] = {
1318 .name
= "ap:online",
1321 * Handled on controll processor until the plugged processor manages
1324 [CPUHP_TEARDOWN_CPU
] = {
1325 .name
= "cpu:teardown",
1326 .startup
.single
= NULL
,
1327 .teardown
.single
= takedown_cpu
,
1330 /* Handle smpboot threads park/unpark */
1331 [CPUHP_AP_SMPBOOT_THREADS
] = {
1332 .name
= "smpboot/threads:online",
1333 .startup
.single
= smpboot_unpark_threads
,
1334 .teardown
.single
= smpboot_park_threads
,
1336 [CPUHP_AP_IRQ_AFFINITY_ONLINE
] = {
1337 .name
= "irq/affinity:online",
1338 .startup
.single
= irq_affinity_online_cpu
,
1339 .teardown
.single
= NULL
,
1341 [CPUHP_AP_PERF_ONLINE
] = {
1342 .name
= "perf:online",
1343 .startup
.single
= perf_event_init_cpu
,
1344 .teardown
.single
= perf_event_exit_cpu
,
1346 [CPUHP_AP_WORKQUEUE_ONLINE
] = {
1347 .name
= "workqueue:online",
1348 .startup
.single
= workqueue_online_cpu
,
1349 .teardown
.single
= workqueue_offline_cpu
,
1351 [CPUHP_AP_RCUTREE_ONLINE
] = {
1352 .name
= "RCU/tree:online",
1353 .startup
.single
= rcutree_online_cpu
,
1354 .teardown
.single
= rcutree_offline_cpu
,
1358 * The dynamically registered state space is here
1362 /* Last state is scheduler control setting the cpu active */
1363 [CPUHP_AP_ACTIVE
] = {
1364 .name
= "sched:active",
1365 .startup
.single
= sched_cpu_activate
,
1366 .teardown
.single
= sched_cpu_deactivate
,
1370 /* CPU is fully up and running. */
1373 .startup
.single
= NULL
,
1374 .teardown
.single
= NULL
,
1378 /* Sanity check for callbacks */
1379 static int cpuhp_cb_check(enum cpuhp_state state
)
1381 if (state
<= CPUHP_OFFLINE
|| state
>= CPUHP_ONLINE
)
1387 * Returns a free for dynamic slot assignment of the Online state. The states
1388 * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1389 * by having no name assigned.
1391 static int cpuhp_reserve_state(enum cpuhp_state state
)
1393 enum cpuhp_state i
, end
;
1394 struct cpuhp_step
*step
;
1397 case CPUHP_AP_ONLINE_DYN
:
1398 step
= cpuhp_hp_states
+ CPUHP_AP_ONLINE_DYN
;
1399 end
= CPUHP_AP_ONLINE_DYN_END
;
1401 case CPUHP_BP_PREPARE_DYN
:
1402 step
= cpuhp_hp_states
+ CPUHP_BP_PREPARE_DYN
;
1403 end
= CPUHP_BP_PREPARE_DYN_END
;
1409 for (i
= state
; i
<= end
; i
++, step
++) {
1413 WARN(1, "No more dynamic states available for CPU hotplug\n");
1417 static int cpuhp_store_callbacks(enum cpuhp_state state
, const char *name
,
1418 int (*startup
)(unsigned int cpu
),
1419 int (*teardown
)(unsigned int cpu
),
1420 bool multi_instance
)
1422 /* (Un)Install the callbacks for further cpu hotplug operations */
1423 struct cpuhp_step
*sp
;
1427 * If name is NULL, then the state gets removed.
1429 * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1430 * the first allocation from these dynamic ranges, so the removal
1431 * would trigger a new allocation and clear the wrong (already
1432 * empty) state, leaving the callbacks of the to be cleared state
1433 * dangling, which causes wreckage on the next hotplug operation.
1435 if (name
&& (state
== CPUHP_AP_ONLINE_DYN
||
1436 state
== CPUHP_BP_PREPARE_DYN
)) {
1437 ret
= cpuhp_reserve_state(state
);
1442 sp
= cpuhp_get_step(state
);
1443 if (name
&& sp
->name
)
1446 sp
->startup
.single
= startup
;
1447 sp
->teardown
.single
= teardown
;
1449 sp
->multi_instance
= multi_instance
;
1450 INIT_HLIST_HEAD(&sp
->list
);
1454 static void *cpuhp_get_teardown_cb(enum cpuhp_state state
)
1456 return cpuhp_get_step(state
)->teardown
.single
;
1460 * Call the startup/teardown function for a step either on the AP or
1461 * on the current CPU.
1463 static int cpuhp_issue_call(int cpu
, enum cpuhp_state state
, bool bringup
,
1464 struct hlist_node
*node
)
1466 struct cpuhp_step
*sp
= cpuhp_get_step(state
);
1470 * If there's nothing to do, we done.
1471 * Relies on the union for multi_instance.
1473 if ((bringup
&& !sp
->startup
.single
) ||
1474 (!bringup
&& !sp
->teardown
.single
))
1477 * The non AP bound callbacks can fail on bringup. On teardown
1478 * e.g. module removal we crash for now.
1481 if (cpuhp_is_ap_state(state
))
1482 ret
= cpuhp_invoke_ap_callback(cpu
, state
, bringup
, node
);
1484 ret
= cpuhp_invoke_callback(cpu
, state
, bringup
, node
, NULL
);
1486 ret
= cpuhp_invoke_callback(cpu
, state
, bringup
, node
, NULL
);
1488 BUG_ON(ret
&& !bringup
);
1493 * Called from __cpuhp_setup_state on a recoverable failure.
1495 * Note: The teardown callbacks for rollback are not allowed to fail!
1497 static void cpuhp_rollback_install(int failedcpu
, enum cpuhp_state state
,
1498 struct hlist_node
*node
)
1502 /* Roll back the already executed steps on the other cpus */
1503 for_each_present_cpu(cpu
) {
1504 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1505 int cpustate
= st
->state
;
1507 if (cpu
>= failedcpu
)
1510 /* Did we invoke the startup call on that cpu ? */
1511 if (cpustate
>= state
)
1512 cpuhp_issue_call(cpu
, state
, false, node
);
1516 int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state
,
1517 struct hlist_node
*node
,
1520 struct cpuhp_step
*sp
;
1524 lockdep_assert_cpus_held();
1526 sp
= cpuhp_get_step(state
);
1527 if (sp
->multi_instance
== false)
1530 mutex_lock(&cpuhp_state_mutex
);
1532 if (!invoke
|| !sp
->startup
.multi
)
1536 * Try to call the startup callback for each present cpu
1537 * depending on the hotplug state of the cpu.
1539 for_each_present_cpu(cpu
) {
1540 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1541 int cpustate
= st
->state
;
1543 if (cpustate
< state
)
1546 ret
= cpuhp_issue_call(cpu
, state
, true, node
);
1548 if (sp
->teardown
.multi
)
1549 cpuhp_rollback_install(cpu
, state
, node
);
1555 hlist_add_head(node
, &sp
->list
);
1557 mutex_unlock(&cpuhp_state_mutex
);
1561 int __cpuhp_state_add_instance(enum cpuhp_state state
, struct hlist_node
*node
,
1567 ret
= __cpuhp_state_add_instance_cpuslocked(state
, node
, invoke
);
1571 EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance
);
1574 * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
1575 * @state: The state to setup
1576 * @invoke: If true, the startup function is invoked for cpus where
1577 * cpu state >= @state
1578 * @startup: startup callback function
1579 * @teardown: teardown callback function
1580 * @multi_instance: State is set up for multiple instances which get
1583 * The caller needs to hold cpus read locked while calling this function.
1586 * Positive state number if @state is CPUHP_AP_ONLINE_DYN
1587 * 0 for all other states
1588 * On failure: proper (negative) error code
1590 int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state
,
1591 const char *name
, bool invoke
,
1592 int (*startup
)(unsigned int cpu
),
1593 int (*teardown
)(unsigned int cpu
),
1594 bool multi_instance
)
1599 lockdep_assert_cpus_held();
1601 if (cpuhp_cb_check(state
) || !name
)
1604 mutex_lock(&cpuhp_state_mutex
);
1606 ret
= cpuhp_store_callbacks(state
, name
, startup
, teardown
,
1609 dynstate
= state
== CPUHP_AP_ONLINE_DYN
;
1610 if (ret
> 0 && dynstate
) {
1615 if (ret
|| !invoke
|| !startup
)
1619 * Try to call the startup callback for each present cpu
1620 * depending on the hotplug state of the cpu.
1622 for_each_present_cpu(cpu
) {
1623 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1624 int cpustate
= st
->state
;
1626 if (cpustate
< state
)
1629 ret
= cpuhp_issue_call(cpu
, state
, true, NULL
);
1632 cpuhp_rollback_install(cpu
, state
, NULL
);
1633 cpuhp_store_callbacks(state
, NULL
, NULL
, NULL
, false);
1638 mutex_unlock(&cpuhp_state_mutex
);
1640 * If the requested state is CPUHP_AP_ONLINE_DYN, return the
1641 * dynamically allocated state in case of success.
1643 if (!ret
&& dynstate
)
1647 EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked
);
1649 int __cpuhp_setup_state(enum cpuhp_state state
,
1650 const char *name
, bool invoke
,
1651 int (*startup
)(unsigned int cpu
),
1652 int (*teardown
)(unsigned int cpu
),
1653 bool multi_instance
)
1658 ret
= __cpuhp_setup_state_cpuslocked(state
, name
, invoke
, startup
,
1659 teardown
, multi_instance
);
1663 EXPORT_SYMBOL(__cpuhp_setup_state
);
1665 int __cpuhp_state_remove_instance(enum cpuhp_state state
,
1666 struct hlist_node
*node
, bool invoke
)
1668 struct cpuhp_step
*sp
= cpuhp_get_step(state
);
1671 BUG_ON(cpuhp_cb_check(state
));
1673 if (!sp
->multi_instance
)
1677 mutex_lock(&cpuhp_state_mutex
);
1679 if (!invoke
|| !cpuhp_get_teardown_cb(state
))
1682 * Call the teardown callback for each present cpu depending
1683 * on the hotplug state of the cpu. This function is not
1684 * allowed to fail currently!
1686 for_each_present_cpu(cpu
) {
1687 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1688 int cpustate
= st
->state
;
1690 if (cpustate
>= state
)
1691 cpuhp_issue_call(cpu
, state
, false, node
);
1696 mutex_unlock(&cpuhp_state_mutex
);
1701 EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance
);
1704 * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
1705 * @state: The state to remove
1706 * @invoke: If true, the teardown function is invoked for cpus where
1707 * cpu state >= @state
1709 * The caller needs to hold cpus read locked while calling this function.
1710 * The teardown callback is currently not allowed to fail. Think
1711 * about module removal!
1713 void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state
, bool invoke
)
1715 struct cpuhp_step
*sp
= cpuhp_get_step(state
);
1718 BUG_ON(cpuhp_cb_check(state
));
1720 lockdep_assert_cpus_held();
1722 mutex_lock(&cpuhp_state_mutex
);
1723 if (sp
->multi_instance
) {
1724 WARN(!hlist_empty(&sp
->list
),
1725 "Error: Removing state %d which has instances left.\n",
1730 if (!invoke
|| !cpuhp_get_teardown_cb(state
))
1734 * Call the teardown callback for each present cpu depending
1735 * on the hotplug state of the cpu. This function is not
1736 * allowed to fail currently!
1738 for_each_present_cpu(cpu
) {
1739 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1740 int cpustate
= st
->state
;
1742 if (cpustate
>= state
)
1743 cpuhp_issue_call(cpu
, state
, false, NULL
);
1746 cpuhp_store_callbacks(state
, NULL
, NULL
, NULL
, false);
1747 mutex_unlock(&cpuhp_state_mutex
);
1749 EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked
);
1751 void __cpuhp_remove_state(enum cpuhp_state state
, bool invoke
)
1754 __cpuhp_remove_state_cpuslocked(state
, invoke
);
1757 EXPORT_SYMBOL(__cpuhp_remove_state
);
1759 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
1760 static ssize_t
show_cpuhp_state(struct device
*dev
,
1761 struct device_attribute
*attr
, char *buf
)
1763 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, dev
->id
);
1765 return sprintf(buf
, "%d\n", st
->state
);
1767 static DEVICE_ATTR(state
, 0444, show_cpuhp_state
, NULL
);
1769 static ssize_t
write_cpuhp_target(struct device
*dev
,
1770 struct device_attribute
*attr
,
1771 const char *buf
, size_t count
)
1773 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, dev
->id
);
1774 struct cpuhp_step
*sp
;
1777 ret
= kstrtoint(buf
, 10, &target
);
1781 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
1782 if (target
< CPUHP_OFFLINE
|| target
> CPUHP_ONLINE
)
1785 if (target
!= CPUHP_OFFLINE
&& target
!= CPUHP_ONLINE
)
1789 ret
= lock_device_hotplug_sysfs();
1793 mutex_lock(&cpuhp_state_mutex
);
1794 sp
= cpuhp_get_step(target
);
1795 ret
= !sp
->name
|| sp
->cant_stop
? -EINVAL
: 0;
1796 mutex_unlock(&cpuhp_state_mutex
);
1800 if (st
->state
< target
)
1801 ret
= do_cpu_up(dev
->id
, target
);
1803 ret
= do_cpu_down(dev
->id
, target
);
1805 unlock_device_hotplug();
1806 return ret
? ret
: count
;
1809 static ssize_t
show_cpuhp_target(struct device
*dev
,
1810 struct device_attribute
*attr
, char *buf
)
1812 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, dev
->id
);
1814 return sprintf(buf
, "%d\n", st
->target
);
1816 static DEVICE_ATTR(target
, 0644, show_cpuhp_target
, write_cpuhp_target
);
1819 static ssize_t
write_cpuhp_fail(struct device
*dev
,
1820 struct device_attribute
*attr
,
1821 const char *buf
, size_t count
)
1823 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, dev
->id
);
1824 struct cpuhp_step
*sp
;
1827 ret
= kstrtoint(buf
, 10, &fail
);
1832 * Cannot fail STARTING/DYING callbacks.
1834 if (cpuhp_is_atomic_state(fail
))
1838 * Cannot fail anything that doesn't have callbacks.
1840 mutex_lock(&cpuhp_state_mutex
);
1841 sp
= cpuhp_get_step(fail
);
1842 if (!sp
->startup
.single
&& !sp
->teardown
.single
)
1844 mutex_unlock(&cpuhp_state_mutex
);
1853 static ssize_t
show_cpuhp_fail(struct device
*dev
,
1854 struct device_attribute
*attr
, char *buf
)
1856 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, dev
->id
);
1858 return sprintf(buf
, "%d\n", st
->fail
);
1861 static DEVICE_ATTR(fail
, 0644, show_cpuhp_fail
, write_cpuhp_fail
);
1863 static struct attribute
*cpuhp_cpu_attrs
[] = {
1864 &dev_attr_state
.attr
,
1865 &dev_attr_target
.attr
,
1866 &dev_attr_fail
.attr
,
1870 static const struct attribute_group cpuhp_cpu_attr_group
= {
1871 .attrs
= cpuhp_cpu_attrs
,
1876 static ssize_t
show_cpuhp_states(struct device
*dev
,
1877 struct device_attribute
*attr
, char *buf
)
1879 ssize_t cur
, res
= 0;
1882 mutex_lock(&cpuhp_state_mutex
);
1883 for (i
= CPUHP_OFFLINE
; i
<= CPUHP_ONLINE
; i
++) {
1884 struct cpuhp_step
*sp
= cpuhp_get_step(i
);
1887 cur
= sprintf(buf
, "%3d: %s\n", i
, sp
->name
);
1892 mutex_unlock(&cpuhp_state_mutex
);
1895 static DEVICE_ATTR(states
, 0444, show_cpuhp_states
, NULL
);
1897 static struct attribute
*cpuhp_cpu_root_attrs
[] = {
1898 &dev_attr_states
.attr
,
1902 static const struct attribute_group cpuhp_cpu_root_attr_group
= {
1903 .attrs
= cpuhp_cpu_root_attrs
,
1908 static int __init
cpuhp_sysfs_init(void)
1912 ret
= sysfs_create_group(&cpu_subsys
.dev_root
->kobj
,
1913 &cpuhp_cpu_root_attr_group
);
1917 for_each_possible_cpu(cpu
) {
1918 struct device
*dev
= get_cpu_device(cpu
);
1922 ret
= sysfs_create_group(&dev
->kobj
, &cpuhp_cpu_attr_group
);
1928 device_initcall(cpuhp_sysfs_init
);
1932 * cpu_bit_bitmap[] is a special, "compressed" data structure that
1933 * represents all NR_CPUS bits binary values of 1<<nr.
1935 * It is used by cpumask_of() to get a constant address to a CPU
1936 * mask value that has a single bit set only.
1939 /* cpu_bit_bitmap[0] is empty - so we can back into it */
1940 #define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
1941 #define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
1942 #define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
1943 #define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
1945 const unsigned long cpu_bit_bitmap
[BITS_PER_LONG
+1][BITS_TO_LONGS(NR_CPUS
)] = {
1947 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
1948 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
1949 #if BITS_PER_LONG > 32
1950 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
1951 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
1954 EXPORT_SYMBOL_GPL(cpu_bit_bitmap
);
1956 const DECLARE_BITMAP(cpu_all_bits
, NR_CPUS
) = CPU_BITS_ALL
;
1957 EXPORT_SYMBOL(cpu_all_bits
);
1959 #ifdef CONFIG_INIT_ALL_POSSIBLE
1960 struct cpumask __cpu_possible_mask __read_mostly
1963 struct cpumask __cpu_possible_mask __read_mostly
;
1965 EXPORT_SYMBOL(__cpu_possible_mask
);
1967 struct cpumask __cpu_online_mask __read_mostly
;
1968 EXPORT_SYMBOL(__cpu_online_mask
);
1970 struct cpumask __cpu_present_mask __read_mostly
;
1971 EXPORT_SYMBOL(__cpu_present_mask
);
1973 struct cpumask __cpu_active_mask __read_mostly
;
1974 EXPORT_SYMBOL(__cpu_active_mask
);
1976 void init_cpu_present(const struct cpumask
*src
)
1978 cpumask_copy(&__cpu_present_mask
, src
);
1981 void init_cpu_possible(const struct cpumask
*src
)
1983 cpumask_copy(&__cpu_possible_mask
, src
);
1986 void init_cpu_online(const struct cpumask
*src
)
1988 cpumask_copy(&__cpu_online_mask
, src
);
1992 * Activate the first processor.
1994 void __init
boot_cpu_init(void)
1996 int cpu
= smp_processor_id();
1998 /* Mark the boot cpu "present", "online" etc for SMP and UP case */
1999 set_cpu_online(cpu
, true);
2000 set_cpu_active(cpu
, true);
2001 set_cpu_present(cpu
, true);
2002 set_cpu_possible(cpu
, true);
2005 __boot_cpu_id
= cpu
;
2010 * Must be called _AFTER_ setting up the per_cpu areas
2012 void __init
boot_cpu_state_init(void)
2014 per_cpu_ptr(&cpuhp_state
, smp_processor_id())->state
= CPUHP_ONLINE
;