]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - kernel/smp.c
block: provide plug based way of signaling forced no-wait semantics
[mirror_ubuntu-jammy-kernel.git] / kernel / smp.c
CommitLineData
457c8996 1// SPDX-License-Identifier: GPL-2.0-only
3d442233
JA
2/*
3 * Generic helpers for smp ipi calls
4 *
5 * (C) Jens Axboe <jens.axboe@oracle.com> 2008
3d442233 6 */
ca7dfdbb
ME
7
8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
47885016 10#include <linux/irq_work.h>
3d442233 11#include <linux/rcupdate.h>
59190f42 12#include <linux/rculist.h>
641cd4cf 13#include <linux/kernel.h>
9984de1a 14#include <linux/export.h>
0b13fda1
IM
15#include <linux/percpu.h>
16#include <linux/init.h>
5a0e3ad6 17#include <linux/gfp.h>
3d442233 18#include <linux/smp.h>
8969a5ed 19#include <linux/cpu.h>
c6f4459f 20#include <linux/sched.h>
4c822698 21#include <linux/sched/idle.h>
47ae4b05 22#include <linux/hypervisor.h>
3d442233 23
3bb5d2ee 24#include "smpboot.h"
1f8db415 25#include "sched/smp.h"
3bb5d2ee 26
4b44a21d 27#define CSD_TYPE(_csd) ((_csd)->flags & CSD_FLAG_TYPE_MASK)
3d442233
JA
28
29struct call_function_data {
966a9671 30 call_single_data_t __percpu *csd;
0b13fda1 31 cpumask_var_t cpumask;
3fc5b3b6 32 cpumask_var_t cpumask_ipi;
3d442233
JA
33};
34
a22793c7 35static DEFINE_PER_CPU_ALIGNED(struct call_function_data, cfd_data);
e03bcb68 36
6897fc22 37static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue);
8969a5ed 38
8d056c48
SB
39static void flush_smp_call_function_queue(bool warn_cpu_offline);
40
31487f83 41int smpcfd_prepare_cpu(unsigned int cpu)
8969a5ed 42{
8969a5ed
PZ
43 struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
44
31487f83
RW
45 if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL,
46 cpu_to_node(cpu)))
47 return -ENOMEM;
3fc5b3b6
AL
48 if (!zalloc_cpumask_var_node(&cfd->cpumask_ipi, GFP_KERNEL,
49 cpu_to_node(cpu))) {
50 free_cpumask_var(cfd->cpumask);
51 return -ENOMEM;
52 }
966a9671 53 cfd->csd = alloc_percpu(call_single_data_t);
31487f83 54 if (!cfd->csd) {
8969a5ed 55 free_cpumask_var(cfd->cpumask);
3fc5b3b6 56 free_cpumask_var(cfd->cpumask_ipi);
31487f83
RW
57 return -ENOMEM;
58 }
59
60 return 0;
8969a5ed
PZ
61}
62
31487f83
RW
63int smpcfd_dead_cpu(unsigned int cpu)
64{
65 struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
66
67 free_cpumask_var(cfd->cpumask);
3fc5b3b6 68 free_cpumask_var(cfd->cpumask_ipi);
31487f83
RW
69 free_percpu(cfd->csd);
70 return 0;
71}
72
73int smpcfd_dying_cpu(unsigned int cpu)
74{
75 /*
76 * The IPIs for the smp-call-function callbacks queued by other
77 * CPUs might arrive late, either due to hardware latencies or
78 * because this CPU disabled interrupts (inside stop-machine)
79 * before the IPIs were sent. So flush out any pending callbacks
80 * explicitly (without waiting for the IPIs to arrive), to
81 * ensure that the outgoing CPU doesn't go offline with work
82 * still pending.
83 */
84 flush_smp_call_function_queue(false);
afaa653c 85 irq_work_run();
31487f83
RW
86 return 0;
87}
8969a5ed 88
d8ad7d11 89void __init call_function_init(void)
3d442233
JA
90{
91 int i;
92
6897fc22
CH
93 for_each_possible_cpu(i)
94 init_llist_head(&per_cpu(call_single_queue, i));
8969a5ed 95
31487f83 96 smpcfd_prepare_cpu(smp_processor_id());
3d442233
JA
97}
98
8969a5ed
PZ
99/*
100 * csd_lock/csd_unlock used to serialize access to per-cpu csd resources
101 *
0b13fda1
IM
102 * For non-synchronous ipi calls the csd can still be in use by the
103 * previous function call. For multi-cpu calls its even more interesting
104 * as we'll have to ensure no other cpu is observing our csd.
8969a5ed 105 */
966a9671 106static __always_inline void csd_lock_wait(call_single_data_t *csd)
8969a5ed 107{
1f03e8d2 108 smp_cond_load_acquire(&csd->flags, !(VAL & CSD_FLAG_LOCK));
6e275637
PZ
109}
110
966a9671 111static __always_inline void csd_lock(call_single_data_t *csd)
6e275637 112{
e1d12f32
AM
113 csd_lock_wait(csd);
114 csd->flags |= CSD_FLAG_LOCK;
8969a5ed
PZ
115
116 /*
0b13fda1
IM
117 * prevent CPU from reordering the above assignment
118 * to ->flags with any subsequent assignments to other
966a9671 119 * fields of the specified call_single_data_t structure:
8969a5ed 120 */
8053871d 121 smp_wmb();
8969a5ed
PZ
122}
123
966a9671 124static __always_inline void csd_unlock(call_single_data_t *csd)
8969a5ed 125{
8053871d 126 WARN_ON(!(csd->flags & CSD_FLAG_LOCK));
0b13fda1 127
8969a5ed 128 /*
0b13fda1 129 * ensure we're all done before releasing data:
8969a5ed 130 */
8053871d 131 smp_store_release(&csd->flags, 0);
3d442233
JA
132}
133
966a9671 134static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data);
8b28499a 135
4b44a21d
PZ
136void __smp_call_single_queue(int cpu, struct llist_node *node)
137{
138 /*
139 * The list addition should be visible before sending the IPI
140 * handler locks the list to pull the entry off it because of
141 * normal cache coherency rules implied by spinlocks.
142 *
143 * If IPIs can go out of order to the cache coherency protocol
144 * in an architecture, sufficient synchronisation should be added
145 * to arch code to make it appear to obey cache coherency WRT
146 * locking and barrier primitives. Generic code isn't really
147 * equipped to do the right thing...
148 */
149 if (llist_add(node, &per_cpu(call_single_queue, cpu)))
150 send_call_function_single_ipi(cpu);
151}
152
3d442233 153/*
966a9671 154 * Insert a previously allocated call_single_data_t element
0b13fda1
IM
155 * for execution on the given CPU. data must already have
156 * ->func, ->info, and ->flags set.
3d442233 157 */
4b44a21d 158static int generic_exec_single(int cpu, call_single_data_t *csd)
3d442233 159{
8b28499a 160 if (cpu == smp_processor_id()) {
4b44a21d
PZ
161 smp_call_func_t func = csd->func;
162 void *info = csd->info;
8053871d
LT
163 unsigned long flags;
164
165 /*
166 * We can unlock early even for the synchronous on-stack case,
167 * since we're doing this from the same CPU..
168 */
169 csd_unlock(csd);
8b28499a
FW
170 local_irq_save(flags);
171 func(info);
172 local_irq_restore(flags);
173 return 0;
174 }
175
5224b961
LT
176 if ((unsigned)cpu >= nr_cpu_ids || !cpu_online(cpu)) {
177 csd_unlock(csd);
8b28499a 178 return -ENXIO;
5224b961 179 }
8b28499a 180
4b44a21d 181 __smp_call_single_queue(cpu, &csd->llist);
3d442233 182
8b28499a 183 return 0;
3d442233
JA
184}
185
8d056c48
SB
186/**
187 * generic_smp_call_function_single_interrupt - Execute SMP IPI callbacks
188 *
189 * Invoked by arch to handle an IPI for call function single.
190 * Must be called with interrupts disabled.
3d442233
JA
191 */
192void generic_smp_call_function_single_interrupt(void)
193{
8d056c48
SB
194 flush_smp_call_function_queue(true);
195}
196
197/**
198 * flush_smp_call_function_queue - Flush pending smp-call-function callbacks
199 *
200 * @warn_cpu_offline: If set to 'true', warn if callbacks were queued on an
201 * offline CPU. Skip this check if set to 'false'.
202 *
203 * Flush any pending smp-call-function callbacks queued on this CPU. This is
204 * invoked by the generic IPI handler, as well as by a CPU about to go offline,
205 * to ensure that all pending IPI callbacks are run before it goes completely
206 * offline.
207 *
208 * Loop through the call_single_queue and run all the queued callbacks.
209 * Must be called with interrupts disabled.
210 */
211static void flush_smp_call_function_queue(bool warn_cpu_offline)
212{
966a9671 213 call_single_data_t *csd, *csd_next;
52103be0
PZ
214 struct llist_node *entry, *prev;
215 struct llist_head *head;
a219ccf4
SB
216 static bool warned;
217
83efcbd0 218 lockdep_assert_irqs_disabled();
8d056c48 219
bb964a92 220 head = this_cpu_ptr(&call_single_queue);
8d056c48 221 entry = llist_del_all(head);
a219ccf4 222 entry = llist_reverse_order(entry);
3d442233 223
8d056c48
SB
224 /* There shouldn't be any pending callbacks on an offline CPU. */
225 if (unlikely(warn_cpu_offline && !cpu_online(smp_processor_id()) &&
226 !warned && !llist_empty(head))) {
a219ccf4
SB
227 warned = true;
228 WARN(1, "IPI on offline CPU %d\n", smp_processor_id());
229
230 /*
231 * We don't have to use the _safe() variant here
232 * because we are not invoking the IPI handlers yet.
233 */
4b44a21d
PZ
234 llist_for_each_entry(csd, entry, llist) {
235 switch (CSD_TYPE(csd)) {
236 case CSD_TYPE_ASYNC:
237 case CSD_TYPE_SYNC:
238 case CSD_TYPE_IRQ_WORK:
239 pr_warn("IPI callback %pS sent to offline CPU\n",
240 csd->func);
241 break;
242
a1488664
PZ
243 case CSD_TYPE_TTWU:
244 pr_warn("IPI task-wakeup sent to offline CPU\n");
245 break;
246
4b44a21d
PZ
247 default:
248 pr_warn("IPI callback, unknown type %d, sent to offline CPU\n",
249 CSD_TYPE(csd));
250 break;
251 }
252 }
a219ccf4 253 }
3d442233 254
52103be0
PZ
255 /*
256 * First; run all SYNC callbacks, people are waiting for us.
257 */
258 prev = NULL;
5fd77595 259 llist_for_each_entry_safe(csd, csd_next, entry, llist) {
8053871d 260 /* Do we wait until *after* callback? */
4b44a21d
PZ
261 if (CSD_TYPE(csd) == CSD_TYPE_SYNC) {
262 smp_call_func_t func = csd->func;
263 void *info = csd->info;
264
52103be0
PZ
265 if (prev) {
266 prev->next = &csd_next->llist;
267 } else {
268 entry = &csd_next->llist;
269 }
4b44a21d 270
8053871d
LT
271 func(info);
272 csd_unlock(csd);
273 } else {
52103be0 274 prev = &csd->llist;
8053871d 275 }
3d442233 276 }
47885016 277
a1488664
PZ
278 if (!entry)
279 return;
280
47885016 281 /*
52103be0 282 * Second; run all !SYNC callbacks.
47885016 283 */
a1488664 284 prev = NULL;
52103be0 285 llist_for_each_entry_safe(csd, csd_next, entry, llist) {
4b44a21d 286 int type = CSD_TYPE(csd);
52103be0 287
a1488664
PZ
288 if (type != CSD_TYPE_TTWU) {
289 if (prev) {
290 prev->next = &csd_next->llist;
291 } else {
292 entry = &csd_next->llist;
293 }
4b44a21d 294
a1488664
PZ
295 if (type == CSD_TYPE_ASYNC) {
296 smp_call_func_t func = csd->func;
297 void *info = csd->info;
298
299 csd_unlock(csd);
300 func(info);
301 } else if (type == CSD_TYPE_IRQ_WORK) {
302 irq_work_single(csd);
303 }
304
305 } else {
306 prev = &csd->llist;
4b44a21d 307 }
52103be0 308 }
a1488664
PZ
309
310 /*
311 * Third; only CSD_TYPE_TTWU is left, issue those.
312 */
313 if (entry)
314 sched_ttwu_pending(entry);
3d442233
JA
315}
316
b2a02fc4
PZ
317void flush_smp_call_function_from_idle(void)
318{
319 unsigned long flags;
320
321 if (llist_empty(this_cpu_ptr(&call_single_queue)))
322 return;
323
324 local_irq_save(flags);
325 flush_smp_call_function_queue(true);
326 local_irq_restore(flags);
3d442233
JA
327}
328
329/*
330 * smp_call_function_single - Run a function on a specific CPU
331 * @func: The function to run. This must be fast and non-blocking.
332 * @info: An arbitrary pointer to pass to the function.
3d442233
JA
333 * @wait: If true, wait until function has completed on other CPUs.
334 *
72f279b2 335 * Returns 0 on success, else a negative status code.
3d442233 336 */
3a5f65df 337int smp_call_function_single(int cpu, smp_call_func_t func, void *info,
8691e5a8 338 int wait)
3d442233 339{
966a9671
YH
340 call_single_data_t *csd;
341 call_single_data_t csd_stack = {
4b44a21d 342 .flags = CSD_FLAG_LOCK | CSD_TYPE_SYNC,
966a9671 343 };
0b13fda1 344 int this_cpu;
8b28499a 345 int err;
3d442233 346
0b13fda1
IM
347 /*
348 * prevent preemption and reschedule on another processor,
349 * as well as CPU removal
350 */
351 this_cpu = get_cpu();
352
269c861b
SS
353 /*
354 * Can deadlock when called with interrupts disabled.
355 * We allow cpu's that are not yet online though, as no one else can
356 * send smp call function interrupt to this cpu and as such deadlocks
357 * can't happen.
358 */
359 WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
360 && !oops_in_progress);
3d442233 361
19dbdcb8
PZ
362 /*
363 * When @wait we can deadlock when we interrupt between llist_add() and
364 * arch_send_call_function_ipi*(); when !@wait we can deadlock due to
365 * csd_lock() on because the interrupt context uses the same csd
366 * storage.
367 */
368 WARN_ON_ONCE(!in_task());
369
8053871d
LT
370 csd = &csd_stack;
371 if (!wait) {
372 csd = this_cpu_ptr(&csd_data);
373 csd_lock(csd);
374 }
375
4b44a21d
PZ
376 csd->func = func;
377 csd->info = info;
378
379 err = generic_exec_single(cpu, csd);
8053871d
LT
380
381 if (wait)
382 csd_lock_wait(csd);
3d442233
JA
383
384 put_cpu();
0b13fda1 385
f73be6de 386 return err;
3d442233
JA
387}
388EXPORT_SYMBOL(smp_call_function_single);
389
d7877c03 390/**
c46fff2a
FW
391 * smp_call_function_single_async(): Run an asynchronous function on a
392 * specific CPU.
d7877c03
FW
393 * @cpu: The CPU to run on.
394 * @csd: Pre-allocated and setup data structure
d7877c03 395 *
c46fff2a
FW
396 * Like smp_call_function_single(), but the call is asynchonous and
397 * can thus be done from contexts with disabled interrupts.
398 *
399 * The caller passes his own pre-allocated data structure
400 * (ie: embedded in an object) and is responsible for synchronizing it
401 * such that the IPIs performed on the @csd are strictly serialized.
402 *
5a18ceca
PX
403 * If the function is called with one csd which has not yet been
404 * processed by previous call to smp_call_function_single_async(), the
405 * function will return immediately with -EBUSY showing that the csd
406 * object is still in progress.
407 *
c46fff2a
FW
408 * NOTE: Be careful, there is unfortunately no current debugging facility to
409 * validate the correctness of this serialization.
d7877c03 410 */
966a9671 411int smp_call_function_single_async(int cpu, call_single_data_t *csd)
d7877c03
FW
412{
413 int err = 0;
d7877c03 414
fce8ad15 415 preempt_disable();
8053871d 416
5a18ceca
PX
417 if (csd->flags & CSD_FLAG_LOCK) {
418 err = -EBUSY;
419 goto out;
420 }
8053871d
LT
421
422 csd->flags = CSD_FLAG_LOCK;
423 smp_wmb();
424
4b44a21d 425 err = generic_exec_single(cpu, csd);
5a18ceca
PX
426
427out:
fce8ad15 428 preempt_enable();
d7877c03
FW
429
430 return err;
431}
c46fff2a 432EXPORT_SYMBOL_GPL(smp_call_function_single_async);
d7877c03 433
2ea6dec4
RR
434/*
435 * smp_call_function_any - Run a function on any of the given cpus
436 * @mask: The mask of cpus it can run on.
437 * @func: The function to run. This must be fast and non-blocking.
438 * @info: An arbitrary pointer to pass to the function.
439 * @wait: If true, wait until function has completed.
440 *
441 * Returns 0 on success, else a negative status code (if no cpus were online).
2ea6dec4
RR
442 *
443 * Selection preference:
444 * 1) current cpu if in @mask
445 * 2) any cpu of current node if in @mask
446 * 3) any other online cpu in @mask
447 */
448int smp_call_function_any(const struct cpumask *mask,
3a5f65df 449 smp_call_func_t func, void *info, int wait)
2ea6dec4
RR
450{
451 unsigned int cpu;
452 const struct cpumask *nodemask;
453 int ret;
454
455 /* Try for same CPU (cheapest) */
456 cpu = get_cpu();
457 if (cpumask_test_cpu(cpu, mask))
458 goto call;
459
460 /* Try for same node. */
af2422c4 461 nodemask = cpumask_of_node(cpu_to_node(cpu));
2ea6dec4
RR
462 for (cpu = cpumask_first_and(nodemask, mask); cpu < nr_cpu_ids;
463 cpu = cpumask_next_and(cpu, nodemask, mask)) {
464 if (cpu_online(cpu))
465 goto call;
466 }
467
468 /* Any online will do: smp_call_function_single handles nr_cpu_ids. */
469 cpu = cpumask_any_and(mask, cpu_online_mask);
470call:
471 ret = smp_call_function_single(cpu, func, info, wait);
472 put_cpu();
473 return ret;
474}
475EXPORT_SYMBOL_GPL(smp_call_function_any);
476
67719ef2
SAS
477static void smp_call_function_many_cond(const struct cpumask *mask,
478 smp_call_func_t func, void *info,
479 bool wait, smp_cond_func_t cond_func)
3d442233 480{
e1d12f32 481 struct call_function_data *cfd;
9a46ad6d 482 int cpu, next_cpu, this_cpu = smp_processor_id();
3d442233 483
269c861b
SS
484 /*
485 * Can deadlock when called with interrupts disabled.
486 * We allow cpu's that are not yet online though, as no one else can
487 * send smp call function interrupt to this cpu and as such deadlocks
488 * can't happen.
489 */
490 WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
bd924e8c 491 && !oops_in_progress && !early_boot_irqs_disabled);
3d442233 492
19dbdcb8
PZ
493 /*
494 * When @wait we can deadlock when we interrupt between llist_add() and
495 * arch_send_call_function_ipi*(); when !@wait we can deadlock due to
496 * csd_lock() on because the interrupt context uses the same csd
497 * storage.
498 */
499 WARN_ON_ONCE(!in_task());
500
723aae25 501 /* Try to fastpath. So, what's a CPU they want? Ignoring this one. */
54b11e6d 502 cpu = cpumask_first_and(mask, cpu_online_mask);
0b13fda1 503 if (cpu == this_cpu)
54b11e6d 504 cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
0b13fda1 505
54b11e6d
RR
506 /* No online cpus? We're done. */
507 if (cpu >= nr_cpu_ids)
508 return;
509
510 /* Do we have another CPU which isn't us? */
511 next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
0b13fda1 512 if (next_cpu == this_cpu)
54b11e6d
RR
513 next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask);
514
515 /* Fastpath: do that cpu by itself. */
516 if (next_cpu >= nr_cpu_ids) {
25a3a154 517 if (!cond_func || cond_func(cpu, info))
67719ef2 518 smp_call_function_single(cpu, func, info, wait);
54b11e6d 519 return;
3d442233
JA
520 }
521
bb964a92 522 cfd = this_cpu_ptr(&cfd_data);
45a57919 523
e1d12f32 524 cpumask_and(cfd->cpumask, mask, cpu_online_mask);
6c8557bd 525 __cpumask_clear_cpu(this_cpu, cfd->cpumask);
723aae25
MM
526
527 /* Some callers race with other cpus changing the passed mask */
e1d12f32 528 if (unlikely(!cpumask_weight(cfd->cpumask)))
723aae25 529 return;
3d442233 530
3fc5b3b6 531 cpumask_clear(cfd->cpumask_ipi);
e1d12f32 532 for_each_cpu(cpu, cfd->cpumask) {
966a9671 533 call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu);
9a46ad6d 534
67719ef2
SAS
535 if (cond_func && !cond_func(cpu, info))
536 continue;
537
9a46ad6d 538 csd_lock(csd);
8053871d 539 if (wait)
4b44a21d 540 csd->flags |= CSD_TYPE_SYNC;
9a46ad6d
SL
541 csd->func = func;
542 csd->info = info;
3fc5b3b6 543 if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))
6c8557bd 544 __cpumask_set_cpu(cpu, cfd->cpumask_ipi);
9a46ad6d 545 }
561920a0 546
3d442233 547 /* Send a message to all CPUs in the map */
3fc5b3b6 548 arch_send_call_function_ipi_mask(cfd->cpumask_ipi);
3d442233 549
9a46ad6d 550 if (wait) {
e1d12f32 551 for_each_cpu(cpu, cfd->cpumask) {
966a9671 552 call_single_data_t *csd;
e1d12f32
AM
553
554 csd = per_cpu_ptr(cfd->csd, cpu);
9a46ad6d
SL
555 csd_lock_wait(csd);
556 }
557 }
3d442233 558}
67719ef2
SAS
559
560/**
561 * smp_call_function_many(): Run a function on a set of other CPUs.
562 * @mask: The set of cpus to run on (only runs on online subset).
563 * @func: The function to run. This must be fast and non-blocking.
564 * @info: An arbitrary pointer to pass to the function.
565 * @wait: If true, wait (atomically) until function has completed
566 * on other CPUs.
567 *
568 * If @wait is true, then returns once @func has returned.
569 *
570 * You must not call this function with disabled interrupts or from a
571 * hardware interrupt handler or from a bottom half handler. Preemption
572 * must be disabled when calling this function.
573 */
574void smp_call_function_many(const struct cpumask *mask,
575 smp_call_func_t func, void *info, bool wait)
576{
577 smp_call_function_many_cond(mask, func, info, wait, NULL);
578}
54b11e6d 579EXPORT_SYMBOL(smp_call_function_many);
3d442233
JA
580
581/**
582 * smp_call_function(): Run a function on all other CPUs.
583 * @func: The function to run. This must be fast and non-blocking.
584 * @info: An arbitrary pointer to pass to the function.
0b13fda1
IM
585 * @wait: If true, wait (atomically) until function has completed
586 * on other CPUs.
3d442233 587 *
54b11e6d 588 * Returns 0.
3d442233
JA
589 *
590 * If @wait is true, then returns once @func has returned; otherwise
72f279b2 591 * it returns just before the target cpu calls @func.
3d442233
JA
592 *
593 * You must not call this function with disabled interrupts or from a
594 * hardware interrupt handler or from a bottom half handler.
595 */
caa75932 596void smp_call_function(smp_call_func_t func, void *info, int wait)
3d442233 597{
3d442233 598 preempt_disable();
54b11e6d 599 smp_call_function_many(cpu_online_mask, func, info, wait);
3d442233 600 preempt_enable();
3d442233
JA
601}
602EXPORT_SYMBOL(smp_call_function);
351f8f8e 603
34db18a0
AW
604/* Setup configured maximum number of CPUs to activate */
605unsigned int setup_max_cpus = NR_CPUS;
606EXPORT_SYMBOL(setup_max_cpus);
607
608
609/*
610 * Setup routine for controlling SMP activation
611 *
612 * Command-line option of "nosmp" or "maxcpus=0" will disable SMP
613 * activation entirely (the MPS table probe still happens, though).
614 *
615 * Command-line option of "maxcpus=<NUM>", where <NUM> is an integer
616 * greater than 0, limits the maximum number of CPUs activated in
617 * SMP mode to <NUM>.
618 */
619
620void __weak arch_disable_smp_support(void) { }
621
622static int __init nosmp(char *str)
623{
624 setup_max_cpus = 0;
625 arch_disable_smp_support();
626
627 return 0;
628}
629
630early_param("nosmp", nosmp);
631
632/* this is hard limit */
633static int __init nrcpus(char *str)
634{
635 int nr_cpus;
636
637 get_option(&str, &nr_cpus);
638 if (nr_cpus > 0 && nr_cpus < nr_cpu_ids)
639 nr_cpu_ids = nr_cpus;
640
641 return 0;
642}
643
644early_param("nr_cpus", nrcpus);
645
646static int __init maxcpus(char *str)
647{
648 get_option(&str, &setup_max_cpus);
649 if (setup_max_cpus == 0)
650 arch_disable_smp_support();
651
652 return 0;
653}
654
655early_param("maxcpus", maxcpus);
656
657/* Setup number of possible processor ids */
9b130ad5 658unsigned int nr_cpu_ids __read_mostly = NR_CPUS;
34db18a0
AW
659EXPORT_SYMBOL(nr_cpu_ids);
660
661/* An arch may set nr_cpu_ids earlier if needed, so this would be redundant */
662void __init setup_nr_cpu_ids(void)
663{
664 nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1;
665}
666
667/* Called by boot processor to activate the rest. */
668void __init smp_init(void)
669{
92b23278 670 int num_nodes, num_cpus;
34db18a0 671
4b44a21d
PZ
672 /*
673 * Ensure struct irq_work layout matches so that
674 * flush_smp_call_function_queue() can do horrible things.
675 */
676 BUILD_BUG_ON(offsetof(struct irq_work, llnode) !=
677 offsetof(struct __call_single_data, llist));
678 BUILD_BUG_ON(offsetof(struct irq_work, func) !=
679 offsetof(struct __call_single_data, func));
680 BUILD_BUG_ON(offsetof(struct irq_work, flags) !=
681 offsetof(struct __call_single_data, flags));
682
a1488664
PZ
683 /*
684 * Assert the CSD_TYPE_TTWU layout is similar enough
685 * for task_struct to be on the @call_single_queue.
686 */
687 BUILD_BUG_ON(offsetof(struct task_struct, wake_entry_type) - offsetof(struct task_struct, wake_entry) !=
688 offsetof(struct __call_single_data, flags) - offsetof(struct __call_single_data, llist));
689
3bb5d2ee 690 idle_threads_init();
4cb28ced 691 cpuhp_threads_init();
3bb5d2ee 692
51111dce
ME
693 pr_info("Bringing up secondary CPUs ...\n");
694
b99a2659 695 bringup_nonboot_cpus(setup_max_cpus);
34db18a0 696
92b23278
ME
697 num_nodes = num_online_nodes();
698 num_cpus = num_online_cpus();
699 pr_info("Brought up %d node%s, %d CPU%s\n",
700 num_nodes, (num_nodes > 1 ? "s" : ""),
701 num_cpus, (num_cpus > 1 ? "s" : ""));
702
34db18a0 703 /* Any cleanup work */
34db18a0
AW
704 smp_cpus_done(setup_max_cpus);
705}
706
351f8f8e 707/*
bd924e8c
TH
708 * Call a function on all processors. May be used during early boot while
709 * early_boot_irqs_disabled is set. Use local_irq_save/restore() instead
710 * of local_irq_disable/enable().
351f8f8e 711 */
58eb7b77 712void on_each_cpu(smp_call_func_t func, void *info, int wait)
351f8f8e 713{
bd924e8c 714 unsigned long flags;
351f8f8e
AW
715
716 preempt_disable();
caa75932 717 smp_call_function(func, info, wait);
bd924e8c 718 local_irq_save(flags);
351f8f8e 719 func(info);
bd924e8c 720 local_irq_restore(flags);
351f8f8e 721 preempt_enable();
351f8f8e
AW
722}
723EXPORT_SYMBOL(on_each_cpu);
3fc498f1
GBY
724
725/**
726 * on_each_cpu_mask(): Run a function on processors specified by
727 * cpumask, which may include the local processor.
728 * @mask: The set of cpus to run on (only runs on online subset).
729 * @func: The function to run. This must be fast and non-blocking.
730 * @info: An arbitrary pointer to pass to the function.
731 * @wait: If true, wait (atomically) until function has completed
732 * on other CPUs.
733 *
734 * If @wait is true, then returns once @func has returned.
735 *
202da400
DD
736 * You must not call this function with disabled interrupts or from a
737 * hardware interrupt handler or from a bottom half handler. The
738 * exception is that it may be used during early boot while
739 * early_boot_irqs_disabled is set.
3fc498f1
GBY
740 */
741void on_each_cpu_mask(const struct cpumask *mask, smp_call_func_t func,
742 void *info, bool wait)
743{
744 int cpu = get_cpu();
745
746 smp_call_function_many(mask, func, info, wait);
747 if (cpumask_test_cpu(cpu, mask)) {
202da400
DD
748 unsigned long flags;
749 local_irq_save(flags);
3fc498f1 750 func(info);
202da400 751 local_irq_restore(flags);
3fc498f1
GBY
752 }
753 put_cpu();
754}
755EXPORT_SYMBOL(on_each_cpu_mask);
b3a7e98e
GBY
756
757/*
758 * on_each_cpu_cond(): Call a function on each processor for which
759 * the supplied function cond_func returns true, optionally waiting
760 * for all the required CPUs to finish. This may include the local
761 * processor.
762 * @cond_func: A callback function that is passed a cpu id and
763 * the the info parameter. The function is called
764 * with preemption disabled. The function should
765 * return a blooean value indicating whether to IPI
766 * the specified CPU.
767 * @func: The function to run on all applicable CPUs.
768 * This must be fast and non-blocking.
769 * @info: An arbitrary pointer to pass to both functions.
770 * @wait: If true, wait (atomically) until function has
771 * completed on other CPUs.
b3a7e98e
GBY
772 *
773 * Preemption is disabled to protect against CPUs going offline but not online.
774 * CPUs going online during the call will not be seen or sent an IPI.
775 *
776 * You must not call this function with disabled interrupts or
777 * from a hardware interrupt handler or from a bottom half handler.
778 */
5671d814 779void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func,
cb923159 780 void *info, bool wait, const struct cpumask *mask)
b3a7e98e 781{
67719ef2
SAS
782 int cpu = get_cpu();
783
784 smp_call_function_many_cond(mask, func, info, wait, cond_func);
785 if (cpumask_test_cpu(cpu, mask) && cond_func(cpu, info)) {
786 unsigned long flags;
787
788 local_irq_save(flags);
789 func(info);
790 local_irq_restore(flags);
b3a7e98e 791 }
67719ef2 792 put_cpu();
b3a7e98e 793}
7d49b28a
RR
794EXPORT_SYMBOL(on_each_cpu_cond_mask);
795
5671d814 796void on_each_cpu_cond(smp_cond_func_t cond_func, smp_call_func_t func,
cb923159 797 void *info, bool wait)
7d49b28a 798{
cb923159 799 on_each_cpu_cond_mask(cond_func, func, info, wait, cpu_online_mask);
7d49b28a 800}
b3a7e98e 801EXPORT_SYMBOL(on_each_cpu_cond);
f37f435f
TG
802
803static void do_nothing(void *unused)
804{
805}
806
807/**
808 * kick_all_cpus_sync - Force all cpus out of idle
809 *
810 * Used to synchronize the update of pm_idle function pointer. It's
811 * called after the pointer is updated and returns after the dummy
812 * callback function has been executed on all cpus. The execution of
813 * the function can only happen on the remote cpus after they have
814 * left the idle function which had been called via pm_idle function
815 * pointer. So it's guaranteed that nothing uses the previous pointer
816 * anymore.
817 */
818void kick_all_cpus_sync(void)
819{
820 /* Make sure the change is visible before we kick the cpus */
821 smp_mb();
822 smp_call_function(do_nothing, NULL, 1);
823}
824EXPORT_SYMBOL_GPL(kick_all_cpus_sync);
c6f4459f
CL
825
826/**
827 * wake_up_all_idle_cpus - break all cpus out of idle
828 * wake_up_all_idle_cpus try to break all cpus which is in idle state even
829 * including idle polling cpus, for non-idle cpus, we will do nothing
830 * for them.
831 */
832void wake_up_all_idle_cpus(void)
833{
834 int cpu;
835
836 preempt_disable();
837 for_each_online_cpu(cpu) {
838 if (cpu == smp_processor_id())
839 continue;
840
841 wake_up_if_idle(cpu);
842 }
843 preempt_enable();
844}
845EXPORT_SYMBOL_GPL(wake_up_all_idle_cpus);
df8ce9d7
JG
846
847/**
848 * smp_call_on_cpu - Call a function on a specific cpu
849 *
850 * Used to call a function on a specific cpu and wait for it to return.
851 * Optionally make sure the call is done on a specified physical cpu via vcpu
852 * pinning in order to support virtualized environments.
853 */
854struct smp_call_on_cpu_struct {
855 struct work_struct work;
856 struct completion done;
857 int (*func)(void *);
858 void *data;
859 int ret;
860 int cpu;
861};
862
863static void smp_call_on_cpu_callback(struct work_struct *work)
864{
865 struct smp_call_on_cpu_struct *sscs;
866
867 sscs = container_of(work, struct smp_call_on_cpu_struct, work);
868 if (sscs->cpu >= 0)
869 hypervisor_pin_vcpu(sscs->cpu);
870 sscs->ret = sscs->func(sscs->data);
871 if (sscs->cpu >= 0)
872 hypervisor_pin_vcpu(-1);
873
874 complete(&sscs->done);
875}
876
877int smp_call_on_cpu(unsigned int cpu, int (*func)(void *), void *par, bool phys)
878{
879 struct smp_call_on_cpu_struct sscs = {
df8ce9d7
JG
880 .done = COMPLETION_INITIALIZER_ONSTACK(sscs.done),
881 .func = func,
882 .data = par,
883 .cpu = phys ? cpu : -1,
884 };
885
8db54949
PZ
886 INIT_WORK_ONSTACK(&sscs.work, smp_call_on_cpu_callback);
887
df8ce9d7
JG
888 if (cpu >= nr_cpu_ids || !cpu_online(cpu))
889 return -ENXIO;
890
891 queue_work_on(cpu, system_wq, &sscs.work);
892 wait_for_completion(&sscs.done);
893
894 return sscs.ret;
895}
896EXPORT_SYMBOL_GPL(smp_call_on_cpu);