]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - kernel/smp.c
Merge branch 'akpm-incoming-2'
[mirror_ubuntu-artful-kernel.git] / kernel / smp.c
CommitLineData
3d442233
JA
1/*
2 * Generic helpers for smp ipi calls
3 *
4 * (C) Jens Axboe <jens.axboe@oracle.com> 2008
3d442233 5 */
3d442233 6#include <linux/rcupdate.h>
59190f42 7#include <linux/rculist.h>
641cd4cf 8#include <linux/kernel.h>
0b13fda1
IM
9#include <linux/module.h>
10#include <linux/percpu.h>
11#include <linux/init.h>
5a0e3ad6 12#include <linux/gfp.h>
3d442233 13#include <linux/smp.h>
8969a5ed 14#include <linux/cpu.h>
3d442233 15
8969a5ed
PZ
16static struct {
17 struct list_head queue;
9f5a5621 18 raw_spinlock_t lock;
0b13fda1
IM
19} call_function __cacheline_aligned_in_smp =
20 {
21 .queue = LIST_HEAD_INIT(call_function.queue),
9f5a5621 22 .lock = __RAW_SPIN_LOCK_UNLOCKED(call_function.lock),
0b13fda1 23 };
3d442233
JA
24
25enum {
6e275637 26 CSD_FLAG_LOCK = 0x01,
3d442233
JA
27};
28
29struct call_function_data {
0b13fda1 30 struct call_single_data csd;
54fdade1 31 atomic_t refs;
0b13fda1 32 cpumask_var_t cpumask;
3d442233
JA
33};
34
e03bcb68
MM
35static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_function_data, cfd_data);
36
3d442233 37struct call_single_queue {
0b13fda1 38 struct list_head list;
9f5a5621 39 raw_spinlock_t lock;
3d442233
JA
40};
41
e03bcb68 42static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_queue, call_single_queue);
8969a5ed
PZ
43
44static int
45hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)
46{
47 long cpu = (long)hcpu;
48 struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
49
50 switch (action) {
51 case CPU_UP_PREPARE:
52 case CPU_UP_PREPARE_FROZEN:
eaa95840 53 if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL,
8969a5ed 54 cpu_to_node(cpu)))
80b5184c 55 return notifier_from_errno(-ENOMEM);
8969a5ed
PZ
56 break;
57
69dd647f 58#ifdef CONFIG_HOTPLUG_CPU
8969a5ed
PZ
59 case CPU_UP_CANCELED:
60 case CPU_UP_CANCELED_FROZEN:
61
62 case CPU_DEAD:
63 case CPU_DEAD_FROZEN:
64 free_cpumask_var(cfd->cpumask);
65 break;
66#endif
67 };
68
69 return NOTIFY_OK;
70}
71
72static struct notifier_block __cpuinitdata hotplug_cfd_notifier = {
0b13fda1 73 .notifier_call = hotplug_cfd,
8969a5ed
PZ
74};
75
7babe8db 76static int __cpuinit init_call_single_data(void)
3d442233 77{
8969a5ed 78 void *cpu = (void *)(long)smp_processor_id();
3d442233
JA
79 int i;
80
81 for_each_possible_cpu(i) {
82 struct call_single_queue *q = &per_cpu(call_single_queue, i);
83
9f5a5621 84 raw_spin_lock_init(&q->lock);
3d442233
JA
85 INIT_LIST_HEAD(&q->list);
86 }
8969a5ed
PZ
87
88 hotplug_cfd(&hotplug_cfd_notifier, CPU_UP_PREPARE, cpu);
89 register_cpu_notifier(&hotplug_cfd_notifier);
90
7babe8db 91 return 0;
3d442233 92}
7babe8db 93early_initcall(init_call_single_data);
3d442233 94
8969a5ed
PZ
95/*
96 * csd_lock/csd_unlock used to serialize access to per-cpu csd resources
97 *
0b13fda1
IM
98 * For non-synchronous ipi calls the csd can still be in use by the
99 * previous function call. For multi-cpu calls its even more interesting
100 * as we'll have to ensure no other cpu is observing our csd.
8969a5ed 101 */
6e275637 102static void csd_lock_wait(struct call_single_data *data)
8969a5ed
PZ
103{
104 while (data->flags & CSD_FLAG_LOCK)
105 cpu_relax();
6e275637
PZ
106}
107
108static void csd_lock(struct call_single_data *data)
109{
110 csd_lock_wait(data);
8969a5ed
PZ
111 data->flags = CSD_FLAG_LOCK;
112
113 /*
0b13fda1
IM
114 * prevent CPU from reordering the above assignment
115 * to ->flags with any subsequent assignments to other
116 * fields of the specified call_single_data structure:
8969a5ed 117 */
8969a5ed
PZ
118 smp_mb();
119}
120
121static void csd_unlock(struct call_single_data *data)
122{
123 WARN_ON(!(data->flags & CSD_FLAG_LOCK));
0b13fda1 124
8969a5ed 125 /*
0b13fda1 126 * ensure we're all done before releasing data:
8969a5ed
PZ
127 */
128 smp_mb();
0b13fda1 129
8969a5ed 130 data->flags &= ~CSD_FLAG_LOCK;
3d442233
JA
131}
132
133/*
0b13fda1
IM
134 * Insert a previously allocated call_single_data element
135 * for execution on the given CPU. data must already have
136 * ->func, ->info, and ->flags set.
3d442233 137 */
6e275637
PZ
138static
139void generic_exec_single(int cpu, struct call_single_data *data, int wait)
3d442233
JA
140{
141 struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);
3d442233 142 unsigned long flags;
6e275637 143 int ipi;
3d442233 144
9f5a5621 145 raw_spin_lock_irqsave(&dst->lock, flags);
3d442233
JA
146 ipi = list_empty(&dst->list);
147 list_add_tail(&data->list, &dst->list);
9f5a5621 148 raw_spin_unlock_irqrestore(&dst->lock, flags);
3d442233 149
561920a0 150 /*
15d0d3b3
NP
151 * The list addition should be visible before sending the IPI
152 * handler locks the list to pull the entry off it because of
153 * normal cache coherency rules implied by spinlocks.
154 *
155 * If IPIs can go out of order to the cache coherency protocol
156 * in an architecture, sufficient synchronisation should be added
157 * to arch code to make it appear to obey cache coherency WRT
0b13fda1
IM
158 * locking and barrier primitives. Generic code isn't really
159 * equipped to do the right thing...
561920a0 160 */
3d442233
JA
161 if (ipi)
162 arch_send_call_function_single_ipi(cpu);
163
164 if (wait)
6e275637 165 csd_lock_wait(data);
3d442233
JA
166}
167
168/*
169 * Invoked by arch to handle an IPI for call function. Must be called with
170 * interrupts disabled.
171 */
172void generic_smp_call_function_interrupt(void)
173{
174 struct call_function_data *data;
c0f68c2f 175 int cpu = smp_processor_id();
3d442233 176
269c861b
SS
177 /*
178 * Shouldn't receive this interrupt on a cpu that is not yet online.
179 */
180 WARN_ON_ONCE(!cpu_online(cpu));
181
15d0d3b3
NP
182 /*
183 * Ensure entry is visible on call_function_queue after we have
184 * entered the IPI. See comment in smp_call_function_many.
185 * If we don't have this, then we may miss an entry on the list
186 * and never get another IPI to process it.
187 */
188 smp_mb();
189
3d442233 190 /*
0b13fda1
IM
191 * It's ok to use list_for_each_rcu() here even though we may
192 * delete 'pos', since list_del_rcu() doesn't clear ->next
3d442233 193 */
8969a5ed 194 list_for_each_entry_rcu(data, &call_function.queue, csd.list) {
3d442233
JA
195 int refs;
196
54fdade1 197 if (!cpumask_test_and_clear_cpu(cpu, data->cpumask))
3d442233
JA
198 continue;
199
200 data->csd.func(data->csd.info);
201
54fdade1
XG
202 refs = atomic_dec_return(&data->refs);
203 WARN_ON(refs < 0);
8969a5ed 204 if (!refs) {
9f5a5621 205 raw_spin_lock(&call_function.lock);
8969a5ed 206 list_del_rcu(&data->csd.list);
9f5a5621 207 raw_spin_unlock(&call_function.lock);
8969a5ed 208 }
3d442233
JA
209
210 if (refs)
211 continue;
212
8969a5ed 213 csd_unlock(&data->csd);
3d442233 214 }
3d442233 215
3d442233
JA
216}
217
218/*
0b13fda1
IM
219 * Invoked by arch to handle an IPI for call function single. Must be
220 * called from the arch with interrupts disabled.
3d442233
JA
221 */
222void generic_smp_call_function_single_interrupt(void)
223{
224 struct call_single_queue *q = &__get_cpu_var(call_single_queue);
15d0d3b3 225 unsigned int data_flags;
0b13fda1 226 LIST_HEAD(list);
3d442233 227
269c861b
SS
228 /*
229 * Shouldn't receive this interrupt on a cpu that is not yet online.
230 */
231 WARN_ON_ONCE(!cpu_online(smp_processor_id()));
232
9f5a5621 233 raw_spin_lock(&q->lock);
15d0d3b3 234 list_replace_init(&q->list, &list);
9f5a5621 235 raw_spin_unlock(&q->lock);
3d442233 236
15d0d3b3
NP
237 while (!list_empty(&list)) {
238 struct call_single_data *data;
3d442233 239
0b13fda1 240 data = list_entry(list.next, struct call_single_data, list);
15d0d3b3 241 list_del(&data->list);
3d442233 242
3d442233 243 /*
0b13fda1
IM
244 * 'data' can be invalid after this call if flags == 0
245 * (when called through generic_exec_single()),
246 * so save them away before making the call:
3d442233 247 */
15d0d3b3
NP
248 data_flags = data->flags;
249
250 data->func(data->info);
251
8969a5ed 252 /*
0b13fda1 253 * Unlocked CSDs are valid through generic_exec_single():
8969a5ed
PZ
254 */
255 if (data_flags & CSD_FLAG_LOCK)
256 csd_unlock(data);
3d442233
JA
257 }
258}
259
e03bcb68 260static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_data, csd_data);
d7240b98 261
3d442233
JA
262/*
263 * smp_call_function_single - Run a function on a specific CPU
264 * @func: The function to run. This must be fast and non-blocking.
265 * @info: An arbitrary pointer to pass to the function.
3d442233
JA
266 * @wait: If true, wait until function has completed on other CPUs.
267 *
72f279b2 268 * Returns 0 on success, else a negative status code.
3d442233
JA
269 */
270int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
8691e5a8 271 int wait)
3d442233 272{
8969a5ed
PZ
273 struct call_single_data d = {
274 .flags = 0,
275 };
3d442233 276 unsigned long flags;
0b13fda1 277 int this_cpu;
f73be6de 278 int err = 0;
3d442233 279
0b13fda1
IM
280 /*
281 * prevent preemption and reschedule on another processor,
282 * as well as CPU removal
283 */
284 this_cpu = get_cpu();
285
269c861b
SS
286 /*
287 * Can deadlock when called with interrupts disabled.
288 * We allow cpu's that are not yet online though, as no one else can
289 * send smp call function interrupt to this cpu and as such deadlocks
290 * can't happen.
291 */
292 WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
293 && !oops_in_progress);
3d442233 294
0b13fda1 295 if (cpu == this_cpu) {
3d442233
JA
296 local_irq_save(flags);
297 func(info);
298 local_irq_restore(flags);
0b13fda1
IM
299 } else {
300 if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) {
301 struct call_single_data *data = &d;
3d442233 302
0b13fda1
IM
303 if (!wait)
304 data = &__get_cpu_var(csd_data);
6e275637 305
0b13fda1 306 csd_lock(data);
3d442233 307
0b13fda1
IM
308 data->func = func;
309 data->info = info;
310 generic_exec_single(cpu, data, wait);
311 } else {
312 err = -ENXIO; /* CPU not online */
313 }
3d442233
JA
314 }
315
316 put_cpu();
0b13fda1 317
f73be6de 318 return err;
3d442233
JA
319}
320EXPORT_SYMBOL(smp_call_function_single);
321
2ea6dec4
RR
322/*
323 * smp_call_function_any - Run a function on any of the given cpus
324 * @mask: The mask of cpus it can run on.
325 * @func: The function to run. This must be fast and non-blocking.
326 * @info: An arbitrary pointer to pass to the function.
327 * @wait: If true, wait until function has completed.
328 *
329 * Returns 0 on success, else a negative status code (if no cpus were online).
330 * Note that @wait will be implicitly turned on in case of allocation failures,
331 * since we fall back to on-stack allocation.
332 *
333 * Selection preference:
334 * 1) current cpu if in @mask
335 * 2) any cpu of current node if in @mask
336 * 3) any other online cpu in @mask
337 */
338int smp_call_function_any(const struct cpumask *mask,
339 void (*func)(void *info), void *info, int wait)
340{
341 unsigned int cpu;
342 const struct cpumask *nodemask;
343 int ret;
344
345 /* Try for same CPU (cheapest) */
346 cpu = get_cpu();
347 if (cpumask_test_cpu(cpu, mask))
348 goto call;
349
350 /* Try for same node. */
af2422c4 351 nodemask = cpumask_of_node(cpu_to_node(cpu));
2ea6dec4
RR
352 for (cpu = cpumask_first_and(nodemask, mask); cpu < nr_cpu_ids;
353 cpu = cpumask_next_and(cpu, nodemask, mask)) {
354 if (cpu_online(cpu))
355 goto call;
356 }
357
358 /* Any online will do: smp_call_function_single handles nr_cpu_ids. */
359 cpu = cpumask_any_and(mask, cpu_online_mask);
360call:
361 ret = smp_call_function_single(cpu, func, info, wait);
362 put_cpu();
363 return ret;
364}
365EXPORT_SYMBOL_GPL(smp_call_function_any);
366
3d442233 367/**
27c379f7 368 * __smp_call_function_single(): Run a function on a specific CPU
3d442233
JA
369 * @cpu: The CPU to run on.
370 * @data: Pre-allocated and setup data structure
27c379f7 371 * @wait: If true, wait until function has completed on specified CPU.
3d442233 372 *
0b13fda1
IM
373 * Like smp_call_function_single(), but allow caller to pass in a
374 * pre-allocated data structure. Useful for embedding @data inside
375 * other structures, for instance.
3d442233 376 */
6e275637
PZ
377void __smp_call_function_single(int cpu, struct call_single_data *data,
378 int wait)
3d442233 379{
27c379f7
HC
380 unsigned int this_cpu;
381 unsigned long flags;
6e275637 382
27c379f7 383 this_cpu = get_cpu();
269c861b
SS
384 /*
385 * Can deadlock when called with interrupts disabled.
386 * We allow cpu's that are not yet online though, as no one else can
387 * send smp call function interrupt to this cpu and as such deadlocks
388 * can't happen.
389 */
390 WARN_ON_ONCE(cpu_online(smp_processor_id()) && wait && irqs_disabled()
391 && !oops_in_progress);
3d442233 392
27c379f7
HC
393 if (cpu == this_cpu) {
394 local_irq_save(flags);
395 data->func(data->info);
396 local_irq_restore(flags);
397 } else {
398 csd_lock(data);
399 generic_exec_single(cpu, data, wait);
400 }
401 put_cpu();
3d442233
JA
402}
403
404/**
54b11e6d
RR
405 * smp_call_function_many(): Run a function on a set of other CPUs.
406 * @mask: The set of cpus to run on (only runs on online subset).
3d442233
JA
407 * @func: The function to run. This must be fast and non-blocking.
408 * @info: An arbitrary pointer to pass to the function.
0b13fda1
IM
409 * @wait: If true, wait (atomically) until function has completed
410 * on other CPUs.
3d442233 411 *
72f279b2 412 * If @wait is true, then returns once @func has returned.
3d442233
JA
413 *
414 * You must not call this function with disabled interrupts or from a
415 * hardware interrupt handler or from a bottom half handler. Preemption
416 * must be disabled when calling this function.
417 */
54b11e6d 418void smp_call_function_many(const struct cpumask *mask,
0b13fda1 419 void (*func)(void *), void *info, bool wait)
3d442233 420{
54b11e6d 421 struct call_function_data *data;
3d442233 422 unsigned long flags;
0b13fda1 423 int cpu, next_cpu, this_cpu = smp_processor_id();
3d442233 424
269c861b
SS
425 /*
426 * Can deadlock when called with interrupts disabled.
427 * We allow cpu's that are not yet online though, as no one else can
428 * send smp call function interrupt to this cpu and as such deadlocks
429 * can't happen.
430 */
431 WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
432 && !oops_in_progress);
3d442233 433
0b13fda1 434 /* So, what's a CPU they want? Ignoring this one. */
54b11e6d 435 cpu = cpumask_first_and(mask, cpu_online_mask);
0b13fda1 436 if (cpu == this_cpu)
54b11e6d 437 cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
0b13fda1 438
54b11e6d
RR
439 /* No online cpus? We're done. */
440 if (cpu >= nr_cpu_ids)
441 return;
442
443 /* Do we have another CPU which isn't us? */
444 next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
0b13fda1 445 if (next_cpu == this_cpu)
54b11e6d
RR
446 next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask);
447
448 /* Fastpath: do that cpu by itself. */
449 if (next_cpu >= nr_cpu_ids) {
450 smp_call_function_single(cpu, func, info, wait);
451 return;
3d442233
JA
452 }
453
8969a5ed
PZ
454 data = &__get_cpu_var(cfd_data);
455 csd_lock(&data->csd);
3d442233 456
3d442233
JA
457 data->csd.func = func;
458 data->csd.info = info;
8969a5ed 459 cpumask_and(data->cpumask, mask, cpu_online_mask);
0b13fda1 460 cpumask_clear_cpu(this_cpu, data->cpumask);
54fdade1 461 atomic_set(&data->refs, cpumask_weight(data->cpumask));
3d442233 462
9f5a5621 463 raw_spin_lock_irqsave(&call_function.lock, flags);
8969a5ed
PZ
464 /*
465 * Place entry at the _HEAD_ of the list, so that any cpu still
0b13fda1
IM
466 * observing the entry in generic_smp_call_function_interrupt()
467 * will not miss any other list entries:
8969a5ed
PZ
468 */
469 list_add_rcu(&data->csd.list, &call_function.queue);
9f5a5621 470 raw_spin_unlock_irqrestore(&call_function.lock, flags);
3d442233 471
561920a0
SS
472 /*
473 * Make the list addition visible before sending the ipi.
0b13fda1
IM
474 * (IPIs must obey or appear to obey normal Linux cache
475 * coherency rules -- see comment in generic_exec_single).
561920a0
SS
476 */
477 smp_mb();
478
3d442233 479 /* Send a message to all CPUs in the map */
8969a5ed 480 arch_send_call_function_ipi_mask(data->cpumask);
3d442233 481
0b13fda1 482 /* Optionally wait for the CPUs to complete */
54b11e6d 483 if (wait)
6e275637 484 csd_lock_wait(&data->csd);
3d442233 485}
54b11e6d 486EXPORT_SYMBOL(smp_call_function_many);
3d442233
JA
487
488/**
489 * smp_call_function(): Run a function on all other CPUs.
490 * @func: The function to run. This must be fast and non-blocking.
491 * @info: An arbitrary pointer to pass to the function.
0b13fda1
IM
492 * @wait: If true, wait (atomically) until function has completed
493 * on other CPUs.
3d442233 494 *
54b11e6d 495 * Returns 0.
3d442233
JA
496 *
497 * If @wait is true, then returns once @func has returned; otherwise
72f279b2 498 * it returns just before the target cpu calls @func.
3d442233
JA
499 *
500 * You must not call this function with disabled interrupts or from a
501 * hardware interrupt handler or from a bottom half handler.
502 */
8691e5a8 503int smp_call_function(void (*func)(void *), void *info, int wait)
3d442233 504{
3d442233 505 preempt_disable();
54b11e6d 506 smp_call_function_many(cpu_online_mask, func, info, wait);
3d442233 507 preempt_enable();
0b13fda1 508
54b11e6d 509 return 0;
3d442233
JA
510}
511EXPORT_SYMBOL(smp_call_function);
512
513void ipi_call_lock(void)
514{
9f5a5621 515 raw_spin_lock(&call_function.lock);
3d442233
JA
516}
517
518void ipi_call_unlock(void)
519{
9f5a5621 520 raw_spin_unlock(&call_function.lock);
3d442233
JA
521}
522
523void ipi_call_lock_irq(void)
524{
9f5a5621 525 raw_spin_lock_irq(&call_function.lock);
3d442233
JA
526}
527
528void ipi_call_unlock_irq(void)
529{
9f5a5621 530 raw_spin_unlock_irq(&call_function.lock);
3d442233 531}