]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - kernel/softirq.c
Merge branch 'sh/for-2.6.28' of git://git.kernel.org/pub/scm/linux/kernel/git/lethal...
[mirror_ubuntu-bionic-kernel.git] / kernel / softirq.c
1 /*
2 * linux/kernel/softirq.c
3 *
4 * Copyright (C) 1992 Linus Torvalds
5 *
6 * Distribute under GPLv2.
7 *
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9 *
10 * Remote softirq infrastructure is by Jens Axboe.
11 */
12
13 #include <linux/module.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/interrupt.h>
16 #include <linux/init.h>
17 #include <linux/mm.h>
18 #include <linux/notifier.h>
19 #include <linux/percpu.h>
20 #include <linux/cpu.h>
21 #include <linux/freezer.h>
22 #include <linux/kthread.h>
23 #include <linux/rcupdate.h>
24 #include <linux/smp.h>
25 #include <linux/tick.h>
26
27 #include <asm/irq.h>
28 /*
29 - No shared variables, all the data are CPU local.
30 - If a softirq needs serialization, let it serialize itself
31 by its own spinlocks.
32 - Even if softirq is serialized, only local cpu is marked for
33 execution. Hence, we get something sort of weak cpu binding.
34 Though it is still not clear, will it result in better locality
35 or will not.
36
37 Examples:
38 - NET RX softirq. It is multithreaded and does not require
39 any global serialization.
40 - NET TX softirq. It kicks software netdevice queues, hence
41 it is logically serialized per device, but this serialization
42 is invisible to common code.
43 - Tasklets: serialized wrt itself.
44 */
45
46 #ifndef __ARCH_IRQ_STAT
47 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
48 EXPORT_SYMBOL(irq_stat);
49 #endif
50
51 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
52
53 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
54
55 /*
56 * we cannot loop indefinitely here to avoid userspace starvation,
57 * but we also don't want to introduce a worst case 1/HZ latency
58 * to the pending events, so lets the scheduler to balance
59 * the softirq load for us.
60 */
61 static inline void wakeup_softirqd(void)
62 {
63 /* Interrupts are disabled: no need to stop preemption */
64 struct task_struct *tsk = __get_cpu_var(ksoftirqd);
65
66 if (tsk && tsk->state != TASK_RUNNING)
67 wake_up_process(tsk);
68 }
69
70 /*
71 * This one is for softirq.c-internal use,
72 * where hardirqs are disabled legitimately:
73 */
74 #ifdef CONFIG_TRACE_IRQFLAGS
75 static void __local_bh_disable(unsigned long ip)
76 {
77 unsigned long flags;
78
79 WARN_ON_ONCE(in_irq());
80
81 raw_local_irq_save(flags);
82 add_preempt_count(SOFTIRQ_OFFSET);
83 /*
84 * Were softirqs turned off above:
85 */
86 if (softirq_count() == SOFTIRQ_OFFSET)
87 trace_softirqs_off(ip);
88 raw_local_irq_restore(flags);
89 }
90 #else /* !CONFIG_TRACE_IRQFLAGS */
91 static inline void __local_bh_disable(unsigned long ip)
92 {
93 add_preempt_count(SOFTIRQ_OFFSET);
94 barrier();
95 }
96 #endif /* CONFIG_TRACE_IRQFLAGS */
97
98 void local_bh_disable(void)
99 {
100 __local_bh_disable((unsigned long)__builtin_return_address(0));
101 }
102
103 EXPORT_SYMBOL(local_bh_disable);
104
105 void __local_bh_enable(void)
106 {
107 WARN_ON_ONCE(in_irq());
108
109 /*
110 * softirqs should never be enabled by __local_bh_enable(),
111 * it always nests inside local_bh_enable() sections:
112 */
113 WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
114
115 sub_preempt_count(SOFTIRQ_OFFSET);
116 }
117 EXPORT_SYMBOL_GPL(__local_bh_enable);
118
119 /*
120 * Special-case - softirqs can safely be enabled in
121 * cond_resched_softirq(), or by __do_softirq(),
122 * without processing still-pending softirqs:
123 */
124 void _local_bh_enable(void)
125 {
126 WARN_ON_ONCE(in_irq());
127 WARN_ON_ONCE(!irqs_disabled());
128
129 if (softirq_count() == SOFTIRQ_OFFSET)
130 trace_softirqs_on((unsigned long)__builtin_return_address(0));
131 sub_preempt_count(SOFTIRQ_OFFSET);
132 }
133
134 EXPORT_SYMBOL(_local_bh_enable);
135
136 static inline void _local_bh_enable_ip(unsigned long ip)
137 {
138 WARN_ON_ONCE(in_irq() || irqs_disabled());
139 #ifdef CONFIG_TRACE_IRQFLAGS
140 local_irq_disable();
141 #endif
142 /*
143 * Are softirqs going to be turned on now:
144 */
145 if (softirq_count() == SOFTIRQ_OFFSET)
146 trace_softirqs_on(ip);
147 /*
148 * Keep preemption disabled until we are done with
149 * softirq processing:
150 */
151 sub_preempt_count(SOFTIRQ_OFFSET - 1);
152
153 if (unlikely(!in_interrupt() && local_softirq_pending()))
154 do_softirq();
155
156 dec_preempt_count();
157 #ifdef CONFIG_TRACE_IRQFLAGS
158 local_irq_enable();
159 #endif
160 preempt_check_resched();
161 }
162
163 void local_bh_enable(void)
164 {
165 _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
166 }
167 EXPORT_SYMBOL(local_bh_enable);
168
169 void local_bh_enable_ip(unsigned long ip)
170 {
171 _local_bh_enable_ip(ip);
172 }
173 EXPORT_SYMBOL(local_bh_enable_ip);
174
175 /*
176 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
177 * and we fall back to softirqd after that.
178 *
179 * This number has been established via experimentation.
180 * The two things to balance is latency against fairness -
181 * we want to handle softirqs as soon as possible, but they
182 * should not be able to lock up the box.
183 */
184 #define MAX_SOFTIRQ_RESTART 10
185
186 asmlinkage void __do_softirq(void)
187 {
188 struct softirq_action *h;
189 __u32 pending;
190 int max_restart = MAX_SOFTIRQ_RESTART;
191 int cpu;
192
193 pending = local_softirq_pending();
194 account_system_vtime(current);
195
196 __local_bh_disable((unsigned long)__builtin_return_address(0));
197 trace_softirq_enter();
198
199 cpu = smp_processor_id();
200 restart:
201 /* Reset the pending bitmask before enabling irqs */
202 set_softirq_pending(0);
203
204 local_irq_enable();
205
206 h = softirq_vec;
207
208 do {
209 if (pending & 1) {
210 int prev_count = preempt_count();
211
212 h->action(h);
213
214 if (unlikely(prev_count != preempt_count())) {
215 printk(KERN_ERR "huh, entered softirq %td %p"
216 "with preempt_count %08x,"
217 " exited with %08x?\n", h - softirq_vec,
218 h->action, prev_count, preempt_count());
219 preempt_count() = prev_count;
220 }
221
222 rcu_bh_qsctr_inc(cpu);
223 }
224 h++;
225 pending >>= 1;
226 } while (pending);
227
228 local_irq_disable();
229
230 pending = local_softirq_pending();
231 if (pending && --max_restart)
232 goto restart;
233
234 if (pending)
235 wakeup_softirqd();
236
237 trace_softirq_exit();
238
239 account_system_vtime(current);
240 _local_bh_enable();
241 }
242
243 #ifndef __ARCH_HAS_DO_SOFTIRQ
244
245 asmlinkage void do_softirq(void)
246 {
247 __u32 pending;
248 unsigned long flags;
249
250 if (in_interrupt())
251 return;
252
253 local_irq_save(flags);
254
255 pending = local_softirq_pending();
256
257 if (pending)
258 __do_softirq();
259
260 local_irq_restore(flags);
261 }
262
263 #endif
264
265 /*
266 * Enter an interrupt context.
267 */
268 void irq_enter(void)
269 {
270 int cpu = smp_processor_id();
271
272 if (idle_cpu(cpu) && !in_interrupt()) {
273 __irq_enter();
274 tick_check_idle(cpu);
275 } else
276 __irq_enter();
277 }
278
279 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
280 # define invoke_softirq() __do_softirq()
281 #else
282 # define invoke_softirq() do_softirq()
283 #endif
284
285 /*
286 * Exit an interrupt context. Process softirqs if needed and possible:
287 */
288 void irq_exit(void)
289 {
290 account_system_vtime(current);
291 trace_hardirq_exit();
292 sub_preempt_count(IRQ_EXIT_OFFSET);
293 if (!in_interrupt() && local_softirq_pending())
294 invoke_softirq();
295
296 #ifdef CONFIG_NO_HZ
297 /* Make sure that timer wheel updates are propagated */
298 if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
299 tick_nohz_stop_sched_tick(0);
300 rcu_irq_exit();
301 #endif
302 preempt_enable_no_resched();
303 }
304
305 /*
306 * This function must run with irqs disabled!
307 */
308 inline void raise_softirq_irqoff(unsigned int nr)
309 {
310 __raise_softirq_irqoff(nr);
311
312 /*
313 * If we're in an interrupt or softirq, we're done
314 * (this also catches softirq-disabled code). We will
315 * actually run the softirq once we return from
316 * the irq or softirq.
317 *
318 * Otherwise we wake up ksoftirqd to make sure we
319 * schedule the softirq soon.
320 */
321 if (!in_interrupt())
322 wakeup_softirqd();
323 }
324
325 void raise_softirq(unsigned int nr)
326 {
327 unsigned long flags;
328
329 local_irq_save(flags);
330 raise_softirq_irqoff(nr);
331 local_irq_restore(flags);
332 }
333
334 void open_softirq(int nr, void (*action)(struct softirq_action *))
335 {
336 softirq_vec[nr].action = action;
337 }
338
339 /* Tasklets */
340 struct tasklet_head
341 {
342 struct tasklet_struct *head;
343 struct tasklet_struct **tail;
344 };
345
346 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
347 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
348
349 void __tasklet_schedule(struct tasklet_struct *t)
350 {
351 unsigned long flags;
352
353 local_irq_save(flags);
354 t->next = NULL;
355 *__get_cpu_var(tasklet_vec).tail = t;
356 __get_cpu_var(tasklet_vec).tail = &(t->next);
357 raise_softirq_irqoff(TASKLET_SOFTIRQ);
358 local_irq_restore(flags);
359 }
360
361 EXPORT_SYMBOL(__tasklet_schedule);
362
363 void __tasklet_hi_schedule(struct tasklet_struct *t)
364 {
365 unsigned long flags;
366
367 local_irq_save(flags);
368 t->next = NULL;
369 *__get_cpu_var(tasklet_hi_vec).tail = t;
370 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
371 raise_softirq_irqoff(HI_SOFTIRQ);
372 local_irq_restore(flags);
373 }
374
375 EXPORT_SYMBOL(__tasklet_hi_schedule);
376
377 static void tasklet_action(struct softirq_action *a)
378 {
379 struct tasklet_struct *list;
380
381 local_irq_disable();
382 list = __get_cpu_var(tasklet_vec).head;
383 __get_cpu_var(tasklet_vec).head = NULL;
384 __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
385 local_irq_enable();
386
387 while (list) {
388 struct tasklet_struct *t = list;
389
390 list = list->next;
391
392 if (tasklet_trylock(t)) {
393 if (!atomic_read(&t->count)) {
394 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
395 BUG();
396 t->func(t->data);
397 tasklet_unlock(t);
398 continue;
399 }
400 tasklet_unlock(t);
401 }
402
403 local_irq_disable();
404 t->next = NULL;
405 *__get_cpu_var(tasklet_vec).tail = t;
406 __get_cpu_var(tasklet_vec).tail = &(t->next);
407 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
408 local_irq_enable();
409 }
410 }
411
412 static void tasklet_hi_action(struct softirq_action *a)
413 {
414 struct tasklet_struct *list;
415
416 local_irq_disable();
417 list = __get_cpu_var(tasklet_hi_vec).head;
418 __get_cpu_var(tasklet_hi_vec).head = NULL;
419 __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
420 local_irq_enable();
421
422 while (list) {
423 struct tasklet_struct *t = list;
424
425 list = list->next;
426
427 if (tasklet_trylock(t)) {
428 if (!atomic_read(&t->count)) {
429 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
430 BUG();
431 t->func(t->data);
432 tasklet_unlock(t);
433 continue;
434 }
435 tasklet_unlock(t);
436 }
437
438 local_irq_disable();
439 t->next = NULL;
440 *__get_cpu_var(tasklet_hi_vec).tail = t;
441 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
442 __raise_softirq_irqoff(HI_SOFTIRQ);
443 local_irq_enable();
444 }
445 }
446
447
448 void tasklet_init(struct tasklet_struct *t,
449 void (*func)(unsigned long), unsigned long data)
450 {
451 t->next = NULL;
452 t->state = 0;
453 atomic_set(&t->count, 0);
454 t->func = func;
455 t->data = data;
456 }
457
458 EXPORT_SYMBOL(tasklet_init);
459
460 void tasklet_kill(struct tasklet_struct *t)
461 {
462 if (in_interrupt())
463 printk("Attempt to kill tasklet from interrupt\n");
464
465 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
466 do
467 yield();
468 while (test_bit(TASKLET_STATE_SCHED, &t->state));
469 }
470 tasklet_unlock_wait(t);
471 clear_bit(TASKLET_STATE_SCHED, &t->state);
472 }
473
474 EXPORT_SYMBOL(tasklet_kill);
475
476 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
477 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
478
479 static void __local_trigger(struct call_single_data *cp, int softirq)
480 {
481 struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
482
483 list_add_tail(&cp->list, head);
484
485 /* Trigger the softirq only if the list was previously empty. */
486 if (head->next == &cp->list)
487 raise_softirq_irqoff(softirq);
488 }
489
490 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
491 static void remote_softirq_receive(void *data)
492 {
493 struct call_single_data *cp = data;
494 unsigned long flags;
495 int softirq;
496
497 softirq = cp->priv;
498
499 local_irq_save(flags);
500 __local_trigger(cp, softirq);
501 local_irq_restore(flags);
502 }
503
504 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
505 {
506 if (cpu_online(cpu)) {
507 cp->func = remote_softirq_receive;
508 cp->info = cp;
509 cp->flags = 0;
510 cp->priv = softirq;
511
512 __smp_call_function_single(cpu, cp);
513 return 0;
514 }
515 return 1;
516 }
517 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
518 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
519 {
520 return 1;
521 }
522 #endif
523
524 /**
525 * __send_remote_softirq - try to schedule softirq work on a remote cpu
526 * @cp: private SMP call function data area
527 * @cpu: the remote cpu
528 * @this_cpu: the currently executing cpu
529 * @softirq: the softirq for the work
530 *
531 * Attempt to schedule softirq work on a remote cpu. If this cannot be
532 * done, the work is instead queued up on the local cpu.
533 *
534 * Interrupts must be disabled.
535 */
536 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
537 {
538 if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
539 __local_trigger(cp, softirq);
540 }
541 EXPORT_SYMBOL(__send_remote_softirq);
542
543 /**
544 * send_remote_softirq - try to schedule softirq work on a remote cpu
545 * @cp: private SMP call function data area
546 * @cpu: the remote cpu
547 * @softirq: the softirq for the work
548 *
549 * Like __send_remote_softirq except that disabling interrupts and
550 * computing the current cpu is done for the caller.
551 */
552 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
553 {
554 unsigned long flags;
555 int this_cpu;
556
557 local_irq_save(flags);
558 this_cpu = smp_processor_id();
559 __send_remote_softirq(cp, cpu, this_cpu, softirq);
560 local_irq_restore(flags);
561 }
562 EXPORT_SYMBOL(send_remote_softirq);
563
564 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
565 unsigned long action, void *hcpu)
566 {
567 /*
568 * If a CPU goes away, splice its entries to the current CPU
569 * and trigger a run of the softirq
570 */
571 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
572 int cpu = (unsigned long) hcpu;
573 int i;
574
575 local_irq_disable();
576 for (i = 0; i < NR_SOFTIRQS; i++) {
577 struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
578 struct list_head *local_head;
579
580 if (list_empty(head))
581 continue;
582
583 local_head = &__get_cpu_var(softirq_work_list[i]);
584 list_splice_init(head, local_head);
585 raise_softirq_irqoff(i);
586 }
587 local_irq_enable();
588 }
589
590 return NOTIFY_OK;
591 }
592
593 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
594 .notifier_call = remote_softirq_cpu_notify,
595 };
596
597 void __init softirq_init(void)
598 {
599 int cpu;
600
601 for_each_possible_cpu(cpu) {
602 int i;
603
604 per_cpu(tasklet_vec, cpu).tail =
605 &per_cpu(tasklet_vec, cpu).head;
606 per_cpu(tasklet_hi_vec, cpu).tail =
607 &per_cpu(tasklet_hi_vec, cpu).head;
608 for (i = 0; i < NR_SOFTIRQS; i++)
609 INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
610 }
611
612 register_hotcpu_notifier(&remote_softirq_cpu_notifier);
613
614 open_softirq(TASKLET_SOFTIRQ, tasklet_action);
615 open_softirq(HI_SOFTIRQ, tasklet_hi_action);
616 }
617
618 static int ksoftirqd(void * __bind_cpu)
619 {
620 set_current_state(TASK_INTERRUPTIBLE);
621
622 while (!kthread_should_stop()) {
623 preempt_disable();
624 if (!local_softirq_pending()) {
625 preempt_enable_no_resched();
626 schedule();
627 preempt_disable();
628 }
629
630 __set_current_state(TASK_RUNNING);
631
632 while (local_softirq_pending()) {
633 /* Preempt disable stops cpu going offline.
634 If already offline, we'll be on wrong CPU:
635 don't process */
636 if (cpu_is_offline((long)__bind_cpu))
637 goto wait_to_die;
638 do_softirq();
639 preempt_enable_no_resched();
640 cond_resched();
641 preempt_disable();
642 }
643 preempt_enable();
644 set_current_state(TASK_INTERRUPTIBLE);
645 }
646 __set_current_state(TASK_RUNNING);
647 return 0;
648
649 wait_to_die:
650 preempt_enable();
651 /* Wait for kthread_stop */
652 set_current_state(TASK_INTERRUPTIBLE);
653 while (!kthread_should_stop()) {
654 schedule();
655 set_current_state(TASK_INTERRUPTIBLE);
656 }
657 __set_current_state(TASK_RUNNING);
658 return 0;
659 }
660
661 #ifdef CONFIG_HOTPLUG_CPU
662 /*
663 * tasklet_kill_immediate is called to remove a tasklet which can already be
664 * scheduled for execution on @cpu.
665 *
666 * Unlike tasklet_kill, this function removes the tasklet
667 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
668 *
669 * When this function is called, @cpu must be in the CPU_DEAD state.
670 */
671 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
672 {
673 struct tasklet_struct **i;
674
675 BUG_ON(cpu_online(cpu));
676 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
677
678 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
679 return;
680
681 /* CPU is dead, so no lock needed. */
682 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
683 if (*i == t) {
684 *i = t->next;
685 /* If this was the tail element, move the tail ptr */
686 if (*i == NULL)
687 per_cpu(tasklet_vec, cpu).tail = i;
688 return;
689 }
690 }
691 BUG();
692 }
693
694 static void takeover_tasklets(unsigned int cpu)
695 {
696 /* CPU is dead, so no lock needed. */
697 local_irq_disable();
698
699 /* Find end, append list for that CPU. */
700 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
701 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
702 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
703 per_cpu(tasklet_vec, cpu).head = NULL;
704 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
705 }
706 raise_softirq_irqoff(TASKLET_SOFTIRQ);
707
708 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
709 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
710 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
711 per_cpu(tasklet_hi_vec, cpu).head = NULL;
712 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
713 }
714 raise_softirq_irqoff(HI_SOFTIRQ);
715
716 local_irq_enable();
717 }
718 #endif /* CONFIG_HOTPLUG_CPU */
719
720 static int __cpuinit cpu_callback(struct notifier_block *nfb,
721 unsigned long action,
722 void *hcpu)
723 {
724 int hotcpu = (unsigned long)hcpu;
725 struct task_struct *p;
726
727 switch (action) {
728 case CPU_UP_PREPARE:
729 case CPU_UP_PREPARE_FROZEN:
730 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
731 if (IS_ERR(p)) {
732 printk("ksoftirqd for %i failed\n", hotcpu);
733 return NOTIFY_BAD;
734 }
735 kthread_bind(p, hotcpu);
736 per_cpu(ksoftirqd, hotcpu) = p;
737 break;
738 case CPU_ONLINE:
739 case CPU_ONLINE_FROZEN:
740 wake_up_process(per_cpu(ksoftirqd, hotcpu));
741 break;
742 #ifdef CONFIG_HOTPLUG_CPU
743 case CPU_UP_CANCELED:
744 case CPU_UP_CANCELED_FROZEN:
745 if (!per_cpu(ksoftirqd, hotcpu))
746 break;
747 /* Unbind so it can run. Fall thru. */
748 kthread_bind(per_cpu(ksoftirqd, hotcpu),
749 any_online_cpu(cpu_online_map));
750 case CPU_DEAD:
751 case CPU_DEAD_FROZEN: {
752 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
753
754 p = per_cpu(ksoftirqd, hotcpu);
755 per_cpu(ksoftirqd, hotcpu) = NULL;
756 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
757 kthread_stop(p);
758 takeover_tasklets(hotcpu);
759 break;
760 }
761 #endif /* CONFIG_HOTPLUG_CPU */
762 }
763 return NOTIFY_OK;
764 }
765
766 static struct notifier_block __cpuinitdata cpu_nfb = {
767 .notifier_call = cpu_callback
768 };
769
770 static __init int spawn_ksoftirqd(void)
771 {
772 void *cpu = (void *)(long)smp_processor_id();
773 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
774
775 BUG_ON(err == NOTIFY_BAD);
776 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
777 register_cpu_notifier(&cpu_nfb);
778 return 0;
779 }
780 early_initcall(spawn_ksoftirqd);
781
782 #ifdef CONFIG_SMP
783 /*
784 * Call a function on all processors
785 */
786 int on_each_cpu(void (*func) (void *info), void *info, int wait)
787 {
788 int ret = 0;
789
790 preempt_disable();
791 ret = smp_call_function(func, info, wait);
792 local_irq_disable();
793 func(info);
794 local_irq_enable();
795 preempt_enable();
796 return ret;
797 }
798 EXPORT_SYMBOL(on_each_cpu);
799 #endif