]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - kernel/softirq.c
Merge branch 'next-devicetree' of git://git.secretlab.ca/git/linux-2.6
[mirror_ubuntu-artful-kernel.git] / kernel / softirq.c
1 /*
2 * linux/kernel/softirq.c
3 *
4 * Copyright (C) 1992 Linus Torvalds
5 *
6 * Distribute under GPLv2.
7 *
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9 *
10 * Remote softirq infrastructure is by Jens Axboe.
11 */
12
13 #include <linux/module.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/interrupt.h>
16 #include <linux/init.h>
17 #include <linux/mm.h>
18 #include <linux/notifier.h>
19 #include <linux/percpu.h>
20 #include <linux/cpu.h>
21 #include <linux/freezer.h>
22 #include <linux/kthread.h>
23 #include <linux/rcupdate.h>
24 #include <linux/ftrace.h>
25 #include <linux/smp.h>
26 #include <linux/tick.h>
27
28 #define CREATE_TRACE_POINTS
29 #include <trace/events/irq.h>
30
31 #include <asm/irq.h>
32 /*
33 - No shared variables, all the data are CPU local.
34 - If a softirq needs serialization, let it serialize itself
35 by its own spinlocks.
36 - Even if softirq is serialized, only local cpu is marked for
37 execution. Hence, we get something sort of weak cpu binding.
38 Though it is still not clear, will it result in better locality
39 or will not.
40
41 Examples:
42 - NET RX softirq. It is multithreaded and does not require
43 any global serialization.
44 - NET TX softirq. It kicks software netdevice queues, hence
45 it is logically serialized per device, but this serialization
46 is invisible to common code.
47 - Tasklets: serialized wrt itself.
48 */
49
50 #ifndef __ARCH_IRQ_STAT
51 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
52 EXPORT_SYMBOL(irq_stat);
53 #endif
54
55 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
56
57 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
58
59 char *softirq_to_name[NR_SOFTIRQS] = {
60 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
61 "TASKLET", "SCHED", "HRTIMER", "RCU"
62 };
63
64 /*
65 * we cannot loop indefinitely here to avoid userspace starvation,
66 * but we also don't want to introduce a worst case 1/HZ latency
67 * to the pending events, so lets the scheduler to balance
68 * the softirq load for us.
69 */
70 static void wakeup_softirqd(void)
71 {
72 /* Interrupts are disabled: no need to stop preemption */
73 struct task_struct *tsk = __get_cpu_var(ksoftirqd);
74
75 if (tsk && tsk->state != TASK_RUNNING)
76 wake_up_process(tsk);
77 }
78
79 /*
80 * preempt_count and SOFTIRQ_OFFSET usage:
81 * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
82 * softirq processing.
83 * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
84 * on local_bh_disable or local_bh_enable.
85 * This lets us distinguish between whether we are currently processing
86 * softirq and whether we just have bh disabled.
87 */
88
89 /*
90 * This one is for softirq.c-internal use,
91 * where hardirqs are disabled legitimately:
92 */
93 #ifdef CONFIG_TRACE_IRQFLAGS
94 static void __local_bh_disable(unsigned long ip, unsigned int cnt)
95 {
96 unsigned long flags;
97
98 WARN_ON_ONCE(in_irq());
99
100 raw_local_irq_save(flags);
101 /*
102 * The preempt tracer hooks into add_preempt_count and will break
103 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
104 * is set and before current->softirq_enabled is cleared.
105 * We must manually increment preempt_count here and manually
106 * call the trace_preempt_off later.
107 */
108 preempt_count() += cnt;
109 /*
110 * Were softirqs turned off above:
111 */
112 if (softirq_count() == cnt)
113 trace_softirqs_off(ip);
114 raw_local_irq_restore(flags);
115
116 if (preempt_count() == cnt)
117 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
118 }
119 #else /* !CONFIG_TRACE_IRQFLAGS */
120 static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
121 {
122 add_preempt_count(cnt);
123 barrier();
124 }
125 #endif /* CONFIG_TRACE_IRQFLAGS */
126
127 void local_bh_disable(void)
128 {
129 __local_bh_disable((unsigned long)__builtin_return_address(0),
130 SOFTIRQ_DISABLE_OFFSET);
131 }
132
133 EXPORT_SYMBOL(local_bh_disable);
134
135 static void __local_bh_enable(unsigned int cnt)
136 {
137 WARN_ON_ONCE(in_irq());
138 WARN_ON_ONCE(!irqs_disabled());
139
140 if (softirq_count() == cnt)
141 trace_softirqs_on((unsigned long)__builtin_return_address(0));
142 sub_preempt_count(cnt);
143 }
144
145 /*
146 * Special-case - softirqs can safely be enabled in
147 * cond_resched_softirq(), or by __do_softirq(),
148 * without processing still-pending softirqs:
149 */
150 void _local_bh_enable(void)
151 {
152 __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
153 }
154
155 EXPORT_SYMBOL(_local_bh_enable);
156
157 static inline void _local_bh_enable_ip(unsigned long ip)
158 {
159 WARN_ON_ONCE(in_irq() || irqs_disabled());
160 #ifdef CONFIG_TRACE_IRQFLAGS
161 local_irq_disable();
162 #endif
163 /*
164 * Are softirqs going to be turned on now:
165 */
166 if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
167 trace_softirqs_on(ip);
168 /*
169 * Keep preemption disabled until we are done with
170 * softirq processing:
171 */
172 sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
173
174 if (unlikely(!in_interrupt() && local_softirq_pending()))
175 do_softirq();
176
177 dec_preempt_count();
178 #ifdef CONFIG_TRACE_IRQFLAGS
179 local_irq_enable();
180 #endif
181 preempt_check_resched();
182 }
183
184 void local_bh_enable(void)
185 {
186 _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
187 }
188 EXPORT_SYMBOL(local_bh_enable);
189
190 void local_bh_enable_ip(unsigned long ip)
191 {
192 _local_bh_enable_ip(ip);
193 }
194 EXPORT_SYMBOL(local_bh_enable_ip);
195
196 /*
197 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
198 * and we fall back to softirqd after that.
199 *
200 * This number has been established via experimentation.
201 * The two things to balance is latency against fairness -
202 * we want to handle softirqs as soon as possible, but they
203 * should not be able to lock up the box.
204 */
205 #define MAX_SOFTIRQ_RESTART 10
206
207 asmlinkage void __do_softirq(void)
208 {
209 struct softirq_action *h;
210 __u32 pending;
211 int max_restart = MAX_SOFTIRQ_RESTART;
212 int cpu;
213
214 pending = local_softirq_pending();
215 account_system_vtime(current);
216
217 __local_bh_disable((unsigned long)__builtin_return_address(0),
218 SOFTIRQ_OFFSET);
219 lockdep_softirq_enter();
220
221 cpu = smp_processor_id();
222 restart:
223 /* Reset the pending bitmask before enabling irqs */
224 set_softirq_pending(0);
225
226 local_irq_enable();
227
228 h = softirq_vec;
229
230 do {
231 if (pending & 1) {
232 int prev_count = preempt_count();
233 kstat_incr_softirqs_this_cpu(h - softirq_vec);
234
235 trace_softirq_entry(h, softirq_vec);
236 h->action(h);
237 trace_softirq_exit(h, softirq_vec);
238 if (unlikely(prev_count != preempt_count())) {
239 printk(KERN_ERR "huh, entered softirq %td %s %p"
240 "with preempt_count %08x,"
241 " exited with %08x?\n", h - softirq_vec,
242 softirq_to_name[h - softirq_vec],
243 h->action, prev_count, preempt_count());
244 preempt_count() = prev_count;
245 }
246
247 rcu_bh_qs(cpu);
248 }
249 h++;
250 pending >>= 1;
251 } while (pending);
252
253 local_irq_disable();
254
255 pending = local_softirq_pending();
256 if (pending && --max_restart)
257 goto restart;
258
259 if (pending)
260 wakeup_softirqd();
261
262 lockdep_softirq_exit();
263
264 account_system_vtime(current);
265 __local_bh_enable(SOFTIRQ_OFFSET);
266 }
267
268 #ifndef __ARCH_HAS_DO_SOFTIRQ
269
270 asmlinkage void do_softirq(void)
271 {
272 __u32 pending;
273 unsigned long flags;
274
275 if (in_interrupt())
276 return;
277
278 local_irq_save(flags);
279
280 pending = local_softirq_pending();
281
282 if (pending)
283 __do_softirq();
284
285 local_irq_restore(flags);
286 }
287
288 #endif
289
290 /*
291 * Enter an interrupt context.
292 */
293 void irq_enter(void)
294 {
295 int cpu = smp_processor_id();
296
297 rcu_irq_enter();
298 if (idle_cpu(cpu) && !in_interrupt()) {
299 /*
300 * Prevent raise_softirq from needlessly waking up ksoftirqd
301 * here, as softirq will be serviced on return from interrupt.
302 */
303 local_bh_disable();
304 tick_check_idle(cpu);
305 _local_bh_enable();
306 }
307
308 __irq_enter();
309 }
310
311 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
312 # define invoke_softirq() __do_softirq()
313 #else
314 # define invoke_softirq() do_softirq()
315 #endif
316
317 /*
318 * Exit an interrupt context. Process softirqs if needed and possible:
319 */
320 void irq_exit(void)
321 {
322 account_system_vtime(current);
323 trace_hardirq_exit();
324 sub_preempt_count(IRQ_EXIT_OFFSET);
325 if (!in_interrupt() && local_softirq_pending())
326 invoke_softirq();
327
328 rcu_irq_exit();
329 #ifdef CONFIG_NO_HZ
330 /* Make sure that timer wheel updates are propagated */
331 if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
332 tick_nohz_stop_sched_tick(0);
333 #endif
334 preempt_enable_no_resched();
335 }
336
337 /*
338 * This function must run with irqs disabled!
339 */
340 inline void raise_softirq_irqoff(unsigned int nr)
341 {
342 __raise_softirq_irqoff(nr);
343
344 /*
345 * If we're in an interrupt or softirq, we're done
346 * (this also catches softirq-disabled code). We will
347 * actually run the softirq once we return from
348 * the irq or softirq.
349 *
350 * Otherwise we wake up ksoftirqd to make sure we
351 * schedule the softirq soon.
352 */
353 if (!in_interrupt())
354 wakeup_softirqd();
355 }
356
357 void raise_softirq(unsigned int nr)
358 {
359 unsigned long flags;
360
361 local_irq_save(flags);
362 raise_softirq_irqoff(nr);
363 local_irq_restore(flags);
364 }
365
366 void open_softirq(int nr, void (*action)(struct softirq_action *))
367 {
368 softirq_vec[nr].action = action;
369 }
370
371 /*
372 * Tasklets
373 */
374 struct tasklet_head
375 {
376 struct tasklet_struct *head;
377 struct tasklet_struct **tail;
378 };
379
380 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
381 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
382
383 void __tasklet_schedule(struct tasklet_struct *t)
384 {
385 unsigned long flags;
386
387 local_irq_save(flags);
388 t->next = NULL;
389 *__get_cpu_var(tasklet_vec).tail = t;
390 __get_cpu_var(tasklet_vec).tail = &(t->next);
391 raise_softirq_irqoff(TASKLET_SOFTIRQ);
392 local_irq_restore(flags);
393 }
394
395 EXPORT_SYMBOL(__tasklet_schedule);
396
397 void __tasklet_hi_schedule(struct tasklet_struct *t)
398 {
399 unsigned long flags;
400
401 local_irq_save(flags);
402 t->next = NULL;
403 *__get_cpu_var(tasklet_hi_vec).tail = t;
404 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
405 raise_softirq_irqoff(HI_SOFTIRQ);
406 local_irq_restore(flags);
407 }
408
409 EXPORT_SYMBOL(__tasklet_hi_schedule);
410
411 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
412 {
413 BUG_ON(!irqs_disabled());
414
415 t->next = __get_cpu_var(tasklet_hi_vec).head;
416 __get_cpu_var(tasklet_hi_vec).head = t;
417 __raise_softirq_irqoff(HI_SOFTIRQ);
418 }
419
420 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
421
422 static void tasklet_action(struct softirq_action *a)
423 {
424 struct tasklet_struct *list;
425
426 local_irq_disable();
427 list = __get_cpu_var(tasklet_vec).head;
428 __get_cpu_var(tasklet_vec).head = NULL;
429 __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
430 local_irq_enable();
431
432 while (list) {
433 struct tasklet_struct *t = list;
434
435 list = list->next;
436
437 if (tasklet_trylock(t)) {
438 if (!atomic_read(&t->count)) {
439 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
440 BUG();
441 t->func(t->data);
442 tasklet_unlock(t);
443 continue;
444 }
445 tasklet_unlock(t);
446 }
447
448 local_irq_disable();
449 t->next = NULL;
450 *__get_cpu_var(tasklet_vec).tail = t;
451 __get_cpu_var(tasklet_vec).tail = &(t->next);
452 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
453 local_irq_enable();
454 }
455 }
456
457 static void tasklet_hi_action(struct softirq_action *a)
458 {
459 struct tasklet_struct *list;
460
461 local_irq_disable();
462 list = __get_cpu_var(tasklet_hi_vec).head;
463 __get_cpu_var(tasklet_hi_vec).head = NULL;
464 __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
465 local_irq_enable();
466
467 while (list) {
468 struct tasklet_struct *t = list;
469
470 list = list->next;
471
472 if (tasklet_trylock(t)) {
473 if (!atomic_read(&t->count)) {
474 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
475 BUG();
476 t->func(t->data);
477 tasklet_unlock(t);
478 continue;
479 }
480 tasklet_unlock(t);
481 }
482
483 local_irq_disable();
484 t->next = NULL;
485 *__get_cpu_var(tasklet_hi_vec).tail = t;
486 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
487 __raise_softirq_irqoff(HI_SOFTIRQ);
488 local_irq_enable();
489 }
490 }
491
492
493 void tasklet_init(struct tasklet_struct *t,
494 void (*func)(unsigned long), unsigned long data)
495 {
496 t->next = NULL;
497 t->state = 0;
498 atomic_set(&t->count, 0);
499 t->func = func;
500 t->data = data;
501 }
502
503 EXPORT_SYMBOL(tasklet_init);
504
505 void tasklet_kill(struct tasklet_struct *t)
506 {
507 if (in_interrupt())
508 printk("Attempt to kill tasklet from interrupt\n");
509
510 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
511 do {
512 yield();
513 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
514 }
515 tasklet_unlock_wait(t);
516 clear_bit(TASKLET_STATE_SCHED, &t->state);
517 }
518
519 EXPORT_SYMBOL(tasklet_kill);
520
521 /*
522 * tasklet_hrtimer
523 */
524
525 /*
526 * The trampoline is called when the hrtimer expires. It schedules a tasklet
527 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
528 * hrtimer callback, but from softirq context.
529 */
530 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
531 {
532 struct tasklet_hrtimer *ttimer =
533 container_of(timer, struct tasklet_hrtimer, timer);
534
535 tasklet_hi_schedule(&ttimer->tasklet);
536 return HRTIMER_NORESTART;
537 }
538
539 /*
540 * Helper function which calls the hrtimer callback from
541 * tasklet/softirq context
542 */
543 static void __tasklet_hrtimer_trampoline(unsigned long data)
544 {
545 struct tasklet_hrtimer *ttimer = (void *)data;
546 enum hrtimer_restart restart;
547
548 restart = ttimer->function(&ttimer->timer);
549 if (restart != HRTIMER_NORESTART)
550 hrtimer_restart(&ttimer->timer);
551 }
552
553 /**
554 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
555 * @ttimer: tasklet_hrtimer which is initialized
556 * @function: hrtimer callback funtion which gets called from softirq context
557 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
558 * @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
559 */
560 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
561 enum hrtimer_restart (*function)(struct hrtimer *),
562 clockid_t which_clock, enum hrtimer_mode mode)
563 {
564 hrtimer_init(&ttimer->timer, which_clock, mode);
565 ttimer->timer.function = __hrtimer_tasklet_trampoline;
566 tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
567 (unsigned long)ttimer);
568 ttimer->function = function;
569 }
570 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
571
572 /*
573 * Remote softirq bits
574 */
575
576 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
577 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
578
579 static void __local_trigger(struct call_single_data *cp, int softirq)
580 {
581 struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
582
583 list_add_tail(&cp->list, head);
584
585 /* Trigger the softirq only if the list was previously empty. */
586 if (head->next == &cp->list)
587 raise_softirq_irqoff(softirq);
588 }
589
590 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
591 static void remote_softirq_receive(void *data)
592 {
593 struct call_single_data *cp = data;
594 unsigned long flags;
595 int softirq;
596
597 softirq = cp->priv;
598
599 local_irq_save(flags);
600 __local_trigger(cp, softirq);
601 local_irq_restore(flags);
602 }
603
604 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
605 {
606 if (cpu_online(cpu)) {
607 cp->func = remote_softirq_receive;
608 cp->info = cp;
609 cp->flags = 0;
610 cp->priv = softirq;
611
612 __smp_call_function_single(cpu, cp, 0);
613 return 0;
614 }
615 return 1;
616 }
617 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
618 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
619 {
620 return 1;
621 }
622 #endif
623
624 /**
625 * __send_remote_softirq - try to schedule softirq work on a remote cpu
626 * @cp: private SMP call function data area
627 * @cpu: the remote cpu
628 * @this_cpu: the currently executing cpu
629 * @softirq: the softirq for the work
630 *
631 * Attempt to schedule softirq work on a remote cpu. If this cannot be
632 * done, the work is instead queued up on the local cpu.
633 *
634 * Interrupts must be disabled.
635 */
636 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
637 {
638 if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
639 __local_trigger(cp, softirq);
640 }
641 EXPORT_SYMBOL(__send_remote_softirq);
642
643 /**
644 * send_remote_softirq - try to schedule softirq work on a remote cpu
645 * @cp: private SMP call function data area
646 * @cpu: the remote cpu
647 * @softirq: the softirq for the work
648 *
649 * Like __send_remote_softirq except that disabling interrupts and
650 * computing the current cpu is done for the caller.
651 */
652 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
653 {
654 unsigned long flags;
655 int this_cpu;
656
657 local_irq_save(flags);
658 this_cpu = smp_processor_id();
659 __send_remote_softirq(cp, cpu, this_cpu, softirq);
660 local_irq_restore(flags);
661 }
662 EXPORT_SYMBOL(send_remote_softirq);
663
664 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
665 unsigned long action, void *hcpu)
666 {
667 /*
668 * If a CPU goes away, splice its entries to the current CPU
669 * and trigger a run of the softirq
670 */
671 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
672 int cpu = (unsigned long) hcpu;
673 int i;
674
675 local_irq_disable();
676 for (i = 0; i < NR_SOFTIRQS; i++) {
677 struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
678 struct list_head *local_head;
679
680 if (list_empty(head))
681 continue;
682
683 local_head = &__get_cpu_var(softirq_work_list[i]);
684 list_splice_init(head, local_head);
685 raise_softirq_irqoff(i);
686 }
687 local_irq_enable();
688 }
689
690 return NOTIFY_OK;
691 }
692
693 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
694 .notifier_call = remote_softirq_cpu_notify,
695 };
696
697 void __init softirq_init(void)
698 {
699 int cpu;
700
701 for_each_possible_cpu(cpu) {
702 int i;
703
704 per_cpu(tasklet_vec, cpu).tail =
705 &per_cpu(tasklet_vec, cpu).head;
706 per_cpu(tasklet_hi_vec, cpu).tail =
707 &per_cpu(tasklet_hi_vec, cpu).head;
708 for (i = 0; i < NR_SOFTIRQS; i++)
709 INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
710 }
711
712 register_hotcpu_notifier(&remote_softirq_cpu_notifier);
713
714 open_softirq(TASKLET_SOFTIRQ, tasklet_action);
715 open_softirq(HI_SOFTIRQ, tasklet_hi_action);
716 }
717
718 static int run_ksoftirqd(void * __bind_cpu)
719 {
720 set_current_state(TASK_INTERRUPTIBLE);
721
722 current->flags |= PF_KSOFTIRQD;
723 while (!kthread_should_stop()) {
724 preempt_disable();
725 if (!local_softirq_pending()) {
726 preempt_enable_no_resched();
727 schedule();
728 preempt_disable();
729 }
730
731 __set_current_state(TASK_RUNNING);
732
733 while (local_softirq_pending()) {
734 /* Preempt disable stops cpu going offline.
735 If already offline, we'll be on wrong CPU:
736 don't process */
737 if (cpu_is_offline((long)__bind_cpu))
738 goto wait_to_die;
739 do_softirq();
740 preempt_enable_no_resched();
741 cond_resched();
742 preempt_disable();
743 rcu_note_context_switch((long)__bind_cpu);
744 }
745 preempt_enable();
746 set_current_state(TASK_INTERRUPTIBLE);
747 }
748 __set_current_state(TASK_RUNNING);
749 return 0;
750
751 wait_to_die:
752 preempt_enable();
753 /* Wait for kthread_stop */
754 set_current_state(TASK_INTERRUPTIBLE);
755 while (!kthread_should_stop()) {
756 schedule();
757 set_current_state(TASK_INTERRUPTIBLE);
758 }
759 __set_current_state(TASK_RUNNING);
760 return 0;
761 }
762
763 #ifdef CONFIG_HOTPLUG_CPU
764 /*
765 * tasklet_kill_immediate is called to remove a tasklet which can already be
766 * scheduled for execution on @cpu.
767 *
768 * Unlike tasklet_kill, this function removes the tasklet
769 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
770 *
771 * When this function is called, @cpu must be in the CPU_DEAD state.
772 */
773 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
774 {
775 struct tasklet_struct **i;
776
777 BUG_ON(cpu_online(cpu));
778 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
779
780 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
781 return;
782
783 /* CPU is dead, so no lock needed. */
784 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
785 if (*i == t) {
786 *i = t->next;
787 /* If this was the tail element, move the tail ptr */
788 if (*i == NULL)
789 per_cpu(tasklet_vec, cpu).tail = i;
790 return;
791 }
792 }
793 BUG();
794 }
795
796 static void takeover_tasklets(unsigned int cpu)
797 {
798 /* CPU is dead, so no lock needed. */
799 local_irq_disable();
800
801 /* Find end, append list for that CPU. */
802 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
803 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
804 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
805 per_cpu(tasklet_vec, cpu).head = NULL;
806 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
807 }
808 raise_softirq_irqoff(TASKLET_SOFTIRQ);
809
810 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
811 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
812 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
813 per_cpu(tasklet_hi_vec, cpu).head = NULL;
814 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
815 }
816 raise_softirq_irqoff(HI_SOFTIRQ);
817
818 local_irq_enable();
819 }
820 #endif /* CONFIG_HOTPLUG_CPU */
821
822 static int __cpuinit cpu_callback(struct notifier_block *nfb,
823 unsigned long action,
824 void *hcpu)
825 {
826 int hotcpu = (unsigned long)hcpu;
827 struct task_struct *p;
828
829 switch (action) {
830 case CPU_UP_PREPARE:
831 case CPU_UP_PREPARE_FROZEN:
832 p = kthread_create(run_ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
833 if (IS_ERR(p)) {
834 printk("ksoftirqd for %i failed\n", hotcpu);
835 return notifier_from_errno(PTR_ERR(p));
836 }
837 kthread_bind(p, hotcpu);
838 per_cpu(ksoftirqd, hotcpu) = p;
839 break;
840 case CPU_ONLINE:
841 case CPU_ONLINE_FROZEN:
842 wake_up_process(per_cpu(ksoftirqd, hotcpu));
843 break;
844 #ifdef CONFIG_HOTPLUG_CPU
845 case CPU_UP_CANCELED:
846 case CPU_UP_CANCELED_FROZEN:
847 if (!per_cpu(ksoftirqd, hotcpu))
848 break;
849 /* Unbind so it can run. Fall thru. */
850 kthread_bind(per_cpu(ksoftirqd, hotcpu),
851 cpumask_any(cpu_online_mask));
852 case CPU_DEAD:
853 case CPU_DEAD_FROZEN: {
854 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
855
856 p = per_cpu(ksoftirqd, hotcpu);
857 per_cpu(ksoftirqd, hotcpu) = NULL;
858 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
859 kthread_stop(p);
860 takeover_tasklets(hotcpu);
861 break;
862 }
863 #endif /* CONFIG_HOTPLUG_CPU */
864 }
865 return NOTIFY_OK;
866 }
867
868 static struct notifier_block __cpuinitdata cpu_nfb = {
869 .notifier_call = cpu_callback
870 };
871
872 static __init int spawn_ksoftirqd(void)
873 {
874 void *cpu = (void *)(long)smp_processor_id();
875 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
876
877 BUG_ON(err != NOTIFY_OK);
878 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
879 register_cpu_notifier(&cpu_nfb);
880 return 0;
881 }
882 early_initcall(spawn_ksoftirqd);
883
884 #ifdef CONFIG_SMP
885 /*
886 * Call a function on all processors
887 */
888 int on_each_cpu(void (*func) (void *info), void *info, int wait)
889 {
890 int ret = 0;
891
892 preempt_disable();
893 ret = smp_call_function(func, info, wait);
894 local_irq_disable();
895 func(info);
896 local_irq_enable();
897 preempt_enable();
898 return ret;
899 }
900 EXPORT_SYMBOL(on_each_cpu);
901 #endif
902
903 /*
904 * [ These __weak aliases are kept in a separate compilation unit, so that
905 * GCC does not inline them incorrectly. ]
906 */
907
908 int __init __weak early_irq_init(void)
909 {
910 return 0;
911 }
912
913 #ifdef CONFIG_GENERIC_HARDIRQS
914 int __init __weak arch_probe_nr_irqs(void)
915 {
916 return NR_IRQS_LEGACY;
917 }
918
919 int __init __weak arch_early_irq_init(void)
920 {
921 return 0;
922 }
923 #endif