]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - kernel/softirq.c
tracing: Kill call_filter_disable()
[mirror_ubuntu-zesty-kernel.git] / kernel / softirq.c
CommitLineData
1da177e4
LT
1/*
2 * linux/kernel/softirq.c
3 *
4 * Copyright (C) 1992 Linus Torvalds
5 *
b10db7f0
PM
6 * Distribute under GPLv2.
7 *
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
1da177e4
LT
9 */
10
40322764
JP
11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
9984de1a 13#include <linux/export.h>
1da177e4
LT
14#include <linux/kernel_stat.h>
15#include <linux/interrupt.h>
16#include <linux/init.h>
17#include <linux/mm.h>
18#include <linux/notifier.h>
19#include <linux/percpu.h>
20#include <linux/cpu.h>
83144186 21#include <linux/freezer.h>
1da177e4
LT
22#include <linux/kthread.h>
23#include <linux/rcupdate.h>
7e49fcce 24#include <linux/ftrace.h>
78eef01b 25#include <linux/smp.h>
3e339b5d 26#include <linux/smpboot.h>
79bf2bb3 27#include <linux/tick.h>
d532676c 28#include <linux/irq.h>
a0e39ed3
HC
29
30#define CREATE_TRACE_POINTS
ad8d75ff 31#include <trace/events/irq.h>
1da177e4 32
1da177e4
LT
33/*
34 - No shared variables, all the data are CPU local.
35 - If a softirq needs serialization, let it serialize itself
36 by its own spinlocks.
37 - Even if softirq is serialized, only local cpu is marked for
38 execution. Hence, we get something sort of weak cpu binding.
39 Though it is still not clear, will it result in better locality
40 or will not.
41
42 Examples:
43 - NET RX softirq. It is multithreaded and does not require
44 any global serialization.
45 - NET TX softirq. It kicks software netdevice queues, hence
46 it is logically serialized per device, but this serialization
47 is invisible to common code.
48 - Tasklets: serialized wrt itself.
49 */
50
51#ifndef __ARCH_IRQ_STAT
52irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
53EXPORT_SYMBOL(irq_stat);
54#endif
55
978b0116 56static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
1da177e4 57
4dd53d89 58DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
1da177e4 59
ce85b4f2 60const char * const softirq_to_name[NR_SOFTIRQS] = {
5dd4de58 61 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
09223371 62 "TASKLET", "SCHED", "HRTIMER", "RCU"
5d592b44
JB
63};
64
1da177e4
LT
65/*
66 * we cannot loop indefinitely here to avoid userspace starvation,
67 * but we also don't want to introduce a worst case 1/HZ latency
68 * to the pending events, so lets the scheduler to balance
69 * the softirq load for us.
70 */
676cb02d 71static void wakeup_softirqd(void)
1da177e4
LT
72{
73 /* Interrupts are disabled: no need to stop preemption */
909ea964 74 struct task_struct *tsk = __this_cpu_read(ksoftirqd);
1da177e4
LT
75
76 if (tsk && tsk->state != TASK_RUNNING)
77 wake_up_process(tsk);
78}
79
75e1056f
VP
80/*
81 * preempt_count and SOFTIRQ_OFFSET usage:
82 * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
83 * softirq processing.
84 * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
85 * on local_bh_disable or local_bh_enable.
86 * This lets us distinguish between whether we are currently processing
87 * softirq and whether we just have bh disabled.
88 */
89
de30a2b3
IM
90/*
91 * This one is for softirq.c-internal use,
92 * where hardirqs are disabled legitimately:
93 */
3c829c36 94#ifdef CONFIG_TRACE_IRQFLAGS
0bd3a173 95void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
de30a2b3
IM
96{
97 unsigned long flags;
98
99 WARN_ON_ONCE(in_irq());
100
101 raw_local_irq_save(flags);
7e49fcce 102 /*
bdb43806 103 * The preempt tracer hooks into preempt_count_add and will break
7e49fcce
SR
104 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
105 * is set and before current->softirq_enabled is cleared.
106 * We must manually increment preempt_count here and manually
107 * call the trace_preempt_off later.
108 */
bdb43806 109 __preempt_count_add(cnt);
de30a2b3
IM
110 /*
111 * Were softirqs turned off above:
112 */
9ea4c380 113 if (softirq_count() == (cnt & SOFTIRQ_MASK))
de30a2b3
IM
114 trace_softirqs_off(ip);
115 raw_local_irq_restore(flags);
7e49fcce 116
75e1056f 117 if (preempt_count() == cnt)
7e49fcce 118 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
de30a2b3 119}
0bd3a173 120EXPORT_SYMBOL(__local_bh_disable_ip);
3c829c36 121#endif /* CONFIG_TRACE_IRQFLAGS */
de30a2b3 122
75e1056f
VP
123static void __local_bh_enable(unsigned int cnt)
124{
75e1056f
VP
125 WARN_ON_ONCE(!irqs_disabled());
126
9ea4c380 127 if (softirq_count() == (cnt & SOFTIRQ_MASK))
d2e08473 128 trace_softirqs_on(_RET_IP_);
bdb43806 129 preempt_count_sub(cnt);
75e1056f
VP
130}
131
de30a2b3
IM
132/*
133 * Special-case - softirqs can safely be enabled in
134 * cond_resched_softirq(), or by __do_softirq(),
135 * without processing still-pending softirqs:
136 */
137void _local_bh_enable(void)
138{
5d60d3e7 139 WARN_ON_ONCE(in_irq());
75e1056f 140 __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
de30a2b3 141}
de30a2b3
IM
142EXPORT_SYMBOL(_local_bh_enable);
143
0bd3a173 144void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
de30a2b3 145{
0f476b6d 146 WARN_ON_ONCE(in_irq() || irqs_disabled());
3c829c36 147#ifdef CONFIG_TRACE_IRQFLAGS
0f476b6d 148 local_irq_disable();
3c829c36 149#endif
de30a2b3
IM
150 /*
151 * Are softirqs going to be turned on now:
152 */
75e1056f 153 if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
0f476b6d 154 trace_softirqs_on(ip);
de30a2b3
IM
155 /*
156 * Keep preemption disabled until we are done with
157 * softirq processing:
ce85b4f2 158 */
0bd3a173 159 preempt_count_sub(cnt - 1);
de30a2b3 160
0bed698a
FW
161 if (unlikely(!in_interrupt() && local_softirq_pending())) {
162 /*
163 * Run softirq if any pending. And do it in its own stack
164 * as we may be calling this deep in a task call stack already.
165 */
de30a2b3 166 do_softirq();
0bed698a 167 }
de30a2b3 168
bdb43806 169 preempt_count_dec();
3c829c36 170#ifdef CONFIG_TRACE_IRQFLAGS
0f476b6d 171 local_irq_enable();
3c829c36 172#endif
de30a2b3
IM
173 preempt_check_resched();
174}
0bd3a173 175EXPORT_SYMBOL(__local_bh_enable_ip);
de30a2b3 176
1da177e4 177/*
34376a50
BG
178 * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
179 * but break the loop if need_resched() is set or after 2 ms.
180 * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
181 * certain cases, such as stop_machine(), jiffies may cease to
182 * increment and so we need the MAX_SOFTIRQ_RESTART limit as
183 * well to make sure we eventually return from this method.
1da177e4 184 *
c10d7367 185 * These limits have been established via experimentation.
1da177e4
LT
186 * The two things to balance is latency against fairness -
187 * we want to handle softirqs as soon as possible, but they
188 * should not be able to lock up the box.
189 */
c10d7367 190#define MAX_SOFTIRQ_TIME msecs_to_jiffies(2)
34376a50 191#define MAX_SOFTIRQ_RESTART 10
1da177e4 192
f1a83e65
PZ
193#ifdef CONFIG_TRACE_IRQFLAGS
194/*
f1a83e65
PZ
195 * When we run softirqs from irq_exit() and thus on the hardirq stack we need
196 * to keep the lockdep irq context tracking as tight as possible in order to
197 * not miss-qualify lock contexts and miss possible deadlocks.
198 */
f1a83e65 199
5c4853b6 200static inline bool lockdep_softirq_start(void)
f1a83e65 201{
5c4853b6 202 bool in_hardirq = false;
f1a83e65 203
5c4853b6
FW
204 if (trace_hardirq_context(current)) {
205 in_hardirq = true;
f1a83e65 206 trace_hardirq_exit();
5c4853b6
FW
207 }
208
f1a83e65 209 lockdep_softirq_enter();
5c4853b6
FW
210
211 return in_hardirq;
f1a83e65
PZ
212}
213
5c4853b6 214static inline void lockdep_softirq_end(bool in_hardirq)
f1a83e65
PZ
215{
216 lockdep_softirq_exit();
5c4853b6
FW
217
218 if (in_hardirq)
f1a83e65 219 trace_hardirq_enter();
f1a83e65 220}
f1a83e65 221#else
5c4853b6
FW
222static inline bool lockdep_softirq_start(void) { return false; }
223static inline void lockdep_softirq_end(bool in_hardirq) { }
f1a83e65
PZ
224#endif
225
722a9f92 226asmlinkage __visible void __do_softirq(void)
1da177e4 227{
c10d7367 228 unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
907aed48 229 unsigned long old_flags = current->flags;
34376a50 230 int max_restart = MAX_SOFTIRQ_RESTART;
f1a83e65 231 struct softirq_action *h;
5c4853b6 232 bool in_hardirq;
f1a83e65 233 __u32 pending;
2e702b9f 234 int softirq_bit;
907aed48
MG
235
236 /*
237 * Mask out PF_MEMALLOC s current task context is borrowed for the
238 * softirq. A softirq handled such as network RX might set PF_MEMALLOC
239 * again if the socket is related to swap
240 */
241 current->flags &= ~PF_MEMALLOC;
1da177e4
LT
242
243 pending = local_softirq_pending();
6a61671b 244 account_irq_enter_time(current);
829035fd 245
0bd3a173 246 __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
5c4853b6 247 in_hardirq = lockdep_softirq_start();
1da177e4 248
1da177e4
LT
249restart:
250 /* Reset the pending bitmask before enabling irqs */
3f74478b 251 set_softirq_pending(0);
1da177e4 252
c70f5d66 253 local_irq_enable();
1da177e4
LT
254
255 h = softirq_vec;
256
2e702b9f
JP
257 while ((softirq_bit = ffs(pending))) {
258 unsigned int vec_nr;
259 int prev_count;
260
261 h += softirq_bit - 1;
262
263 vec_nr = h - softirq_vec;
264 prev_count = preempt_count();
265
266 kstat_incr_softirqs_this_cpu(vec_nr);
267
268 trace_softirq_entry(vec_nr);
269 h->action(h);
270 trace_softirq_exit(vec_nr);
271 if (unlikely(prev_count != preempt_count())) {
40322764 272 pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
2e702b9f
JP
273 vec_nr, softirq_to_name[vec_nr], h->action,
274 prev_count, preempt_count());
275 preempt_count_set(prev_count);
1da177e4
LT
276 }
277 h++;
2e702b9f
JP
278 pending >>= softirq_bit;
279 }
1da177e4 280
a5d6d3a1 281 rcu_bh_qs(smp_processor_id());
c70f5d66 282 local_irq_disable();
1da177e4
LT
283
284 pending = local_softirq_pending();
c10d7367 285 if (pending) {
34376a50
BG
286 if (time_before(jiffies, end) && !need_resched() &&
287 --max_restart)
c10d7367 288 goto restart;
1da177e4 289
1da177e4 290 wakeup_softirqd();
c10d7367 291 }
1da177e4 292
5c4853b6 293 lockdep_softirq_end(in_hardirq);
6a61671b 294 account_irq_exit_time(current);
75e1056f 295 __local_bh_enable(SOFTIRQ_OFFSET);
5d60d3e7 296 WARN_ON_ONCE(in_interrupt());
907aed48 297 tsk_restore_flags(current, old_flags, PF_MEMALLOC);
1da177e4
LT
298}
299
722a9f92 300asmlinkage __visible void do_softirq(void)
1da177e4
LT
301{
302 __u32 pending;
303 unsigned long flags;
304
305 if (in_interrupt())
306 return;
307
308 local_irq_save(flags);
309
310 pending = local_softirq_pending();
311
312 if (pending)
7d65f4a6 313 do_softirq_own_stack();
1da177e4
LT
314
315 local_irq_restore(flags);
316}
317
dde4b2b5
IM
318/*
319 * Enter an interrupt context.
320 */
321void irq_enter(void)
322{
64db4cff 323 rcu_irq_enter();
0a8a2e78 324 if (is_idle_task(current) && !in_interrupt()) {
d267f87f
VP
325 /*
326 * Prevent raise_softirq from needlessly waking up ksoftirqd
327 * here, as softirq will be serviced on return from interrupt.
328 */
329 local_bh_disable();
5acac1be 330 tick_irq_enter();
d267f87f
VP
331 _local_bh_enable();
332 }
333
334 __irq_enter();
dde4b2b5
IM
335}
336
8d32a307
TG
337static inline void invoke_softirq(void)
338{
ded79754 339 if (!force_irqthreads) {
cc1f0274 340#ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
ded79754
FW
341 /*
342 * We can safely execute softirq on the current stack if
343 * it is the irq stack, because it should be near empty
cc1f0274
FW
344 * at this stage.
345 */
346 __do_softirq();
347#else
348 /*
349 * Otherwise, irq_exit() is called on the task stack that can
350 * be potentially deep already. So call softirq in its own stack
351 * to prevent from any overrun.
ded79754 352 */
be6e1016 353 do_softirq_own_stack();
cc1f0274 354#endif
ded79754 355 } else {
8d32a307 356 wakeup_softirqd();
ded79754 357 }
8d32a307 358}
1da177e4 359
67826eae
FW
360static inline void tick_irq_exit(void)
361{
362#ifdef CONFIG_NO_HZ_COMMON
363 int cpu = smp_processor_id();
364
365 /* Make sure that timer wheel updates are propagated */
366 if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
367 if (!in_interrupt())
368 tick_nohz_irq_exit();
369 }
370#endif
371}
372
1da177e4
LT
373/*
374 * Exit an interrupt context. Process softirqs if needed and possible:
375 */
376void irq_exit(void)
377{
74eed016 378#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
4cd5d111 379 local_irq_disable();
74eed016
TG
380#else
381 WARN_ON_ONCE(!irqs_disabled());
382#endif
383
6a61671b 384 account_irq_exit_time(current);
bdb43806 385 preempt_count_sub(HARDIRQ_OFFSET);
1da177e4
LT
386 if (!in_interrupt() && local_softirq_pending())
387 invoke_softirq();
79bf2bb3 388
67826eae 389 tick_irq_exit();
416eb33c 390 rcu_irq_exit();
f1a83e65 391 trace_hardirq_exit(); /* must be last! */
1da177e4
LT
392}
393
394/*
395 * This function must run with irqs disabled!
396 */
7ad5b3a5 397inline void raise_softirq_irqoff(unsigned int nr)
1da177e4
LT
398{
399 __raise_softirq_irqoff(nr);
400
401 /*
402 * If we're in an interrupt or softirq, we're done
403 * (this also catches softirq-disabled code). We will
404 * actually run the softirq once we return from
405 * the irq or softirq.
406 *
407 * Otherwise we wake up ksoftirqd to make sure we
408 * schedule the softirq soon.
409 */
410 if (!in_interrupt())
411 wakeup_softirqd();
412}
413
7ad5b3a5 414void raise_softirq(unsigned int nr)
1da177e4
LT
415{
416 unsigned long flags;
417
418 local_irq_save(flags);
419 raise_softirq_irqoff(nr);
420 local_irq_restore(flags);
421}
422
f069686e
SR
423void __raise_softirq_irqoff(unsigned int nr)
424{
425 trace_softirq_raise(nr);
426 or_softirq_pending(1UL << nr);
427}
428
962cf36c 429void open_softirq(int nr, void (*action)(struct softirq_action *))
1da177e4 430{
1da177e4
LT
431 softirq_vec[nr].action = action;
432}
433
9ba5f005
PZ
434/*
435 * Tasklets
436 */
ce85b4f2 437struct tasklet_head {
48f20a9a
OJ
438 struct tasklet_struct *head;
439 struct tasklet_struct **tail;
1da177e4
LT
440};
441
4620b49f
VN
442static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
443static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
1da177e4 444
7ad5b3a5 445void __tasklet_schedule(struct tasklet_struct *t)
1da177e4
LT
446{
447 unsigned long flags;
448
449 local_irq_save(flags);
48f20a9a 450 t->next = NULL;
909ea964
CL
451 *__this_cpu_read(tasklet_vec.tail) = t;
452 __this_cpu_write(tasklet_vec.tail, &(t->next));
1da177e4
LT
453 raise_softirq_irqoff(TASKLET_SOFTIRQ);
454 local_irq_restore(flags);
455}
1da177e4
LT
456EXPORT_SYMBOL(__tasklet_schedule);
457
7ad5b3a5 458void __tasklet_hi_schedule(struct tasklet_struct *t)
1da177e4
LT
459{
460 unsigned long flags;
461
462 local_irq_save(flags);
48f20a9a 463 t->next = NULL;
909ea964
CL
464 *__this_cpu_read(tasklet_hi_vec.tail) = t;
465 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
1da177e4
LT
466 raise_softirq_irqoff(HI_SOFTIRQ);
467 local_irq_restore(flags);
468}
1da177e4
LT
469EXPORT_SYMBOL(__tasklet_hi_schedule);
470
7c692cba
VN
471void __tasklet_hi_schedule_first(struct tasklet_struct *t)
472{
473 BUG_ON(!irqs_disabled());
474
909ea964
CL
475 t->next = __this_cpu_read(tasklet_hi_vec.head);
476 __this_cpu_write(tasklet_hi_vec.head, t);
7c692cba
VN
477 __raise_softirq_irqoff(HI_SOFTIRQ);
478}
7c692cba
VN
479EXPORT_SYMBOL(__tasklet_hi_schedule_first);
480
1da177e4
LT
481static void tasklet_action(struct softirq_action *a)
482{
483 struct tasklet_struct *list;
484
485 local_irq_disable();
909ea964
CL
486 list = __this_cpu_read(tasklet_vec.head);
487 __this_cpu_write(tasklet_vec.head, NULL);
488 __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
1da177e4
LT
489 local_irq_enable();
490
491 while (list) {
492 struct tasklet_struct *t = list;
493
494 list = list->next;
495
496 if (tasklet_trylock(t)) {
497 if (!atomic_read(&t->count)) {
ce85b4f2
JP
498 if (!test_and_clear_bit(TASKLET_STATE_SCHED,
499 &t->state))
1da177e4
LT
500 BUG();
501 t->func(t->data);
502 tasklet_unlock(t);
503 continue;
504 }
505 tasklet_unlock(t);
506 }
507
508 local_irq_disable();
48f20a9a 509 t->next = NULL;
909ea964
CL
510 *__this_cpu_read(tasklet_vec.tail) = t;
511 __this_cpu_write(tasklet_vec.tail, &(t->next));
1da177e4
LT
512 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
513 local_irq_enable();
514 }
515}
516
517static void tasklet_hi_action(struct softirq_action *a)
518{
519 struct tasklet_struct *list;
520
521 local_irq_disable();
909ea964
CL
522 list = __this_cpu_read(tasklet_hi_vec.head);
523 __this_cpu_write(tasklet_hi_vec.head, NULL);
524 __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
1da177e4
LT
525 local_irq_enable();
526
527 while (list) {
528 struct tasklet_struct *t = list;
529
530 list = list->next;
531
532 if (tasklet_trylock(t)) {
533 if (!atomic_read(&t->count)) {
ce85b4f2
JP
534 if (!test_and_clear_bit(TASKLET_STATE_SCHED,
535 &t->state))
1da177e4
LT
536 BUG();
537 t->func(t->data);
538 tasklet_unlock(t);
539 continue;
540 }
541 tasklet_unlock(t);
542 }
543
544 local_irq_disable();
48f20a9a 545 t->next = NULL;
909ea964
CL
546 *__this_cpu_read(tasklet_hi_vec.tail) = t;
547 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
1da177e4
LT
548 __raise_softirq_irqoff(HI_SOFTIRQ);
549 local_irq_enable();
550 }
551}
552
1da177e4
LT
553void tasklet_init(struct tasklet_struct *t,
554 void (*func)(unsigned long), unsigned long data)
555{
556 t->next = NULL;
557 t->state = 0;
558 atomic_set(&t->count, 0);
559 t->func = func;
560 t->data = data;
561}
1da177e4
LT
562EXPORT_SYMBOL(tasklet_init);
563
564void tasklet_kill(struct tasklet_struct *t)
565{
566 if (in_interrupt())
40322764 567 pr_notice("Attempt to kill tasklet from interrupt\n");
1da177e4
LT
568
569 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
79d381c9 570 do {
1da177e4 571 yield();
79d381c9 572 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
1da177e4
LT
573 }
574 tasklet_unlock_wait(t);
575 clear_bit(TASKLET_STATE_SCHED, &t->state);
576}
1da177e4
LT
577EXPORT_SYMBOL(tasklet_kill);
578
9ba5f005
PZ
579/*
580 * tasklet_hrtimer
581 */
582
583/*
b9c30322
PZ
584 * The trampoline is called when the hrtimer expires. It schedules a tasklet
585 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
586 * hrtimer callback, but from softirq context.
9ba5f005
PZ
587 */
588static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
589{
590 struct tasklet_hrtimer *ttimer =
591 container_of(timer, struct tasklet_hrtimer, timer);
592
b9c30322
PZ
593 tasklet_hi_schedule(&ttimer->tasklet);
594 return HRTIMER_NORESTART;
9ba5f005
PZ
595}
596
597/*
598 * Helper function which calls the hrtimer callback from
599 * tasklet/softirq context
600 */
601static void __tasklet_hrtimer_trampoline(unsigned long data)
602{
603 struct tasklet_hrtimer *ttimer = (void *)data;
604 enum hrtimer_restart restart;
605
606 restart = ttimer->function(&ttimer->timer);
607 if (restart != HRTIMER_NORESTART)
608 hrtimer_restart(&ttimer->timer);
609}
610
611/**
612 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
613 * @ttimer: tasklet_hrtimer which is initialized
25985edc 614 * @function: hrtimer callback function which gets called from softirq context
9ba5f005
PZ
615 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
616 * @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
617 */
618void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
619 enum hrtimer_restart (*function)(struct hrtimer *),
620 clockid_t which_clock, enum hrtimer_mode mode)
621{
622 hrtimer_init(&ttimer->timer, which_clock, mode);
623 ttimer->timer.function = __hrtimer_tasklet_trampoline;
624 tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
625 (unsigned long)ttimer);
626 ttimer->function = function;
627}
628EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
629
1da177e4
LT
630void __init softirq_init(void)
631{
48f20a9a
OJ
632 int cpu;
633
634 for_each_possible_cpu(cpu) {
635 per_cpu(tasklet_vec, cpu).tail =
636 &per_cpu(tasklet_vec, cpu).head;
637 per_cpu(tasklet_hi_vec, cpu).tail =
638 &per_cpu(tasklet_hi_vec, cpu).head;
639 }
640
962cf36c
CM
641 open_softirq(TASKLET_SOFTIRQ, tasklet_action);
642 open_softirq(HI_SOFTIRQ, tasklet_hi_action);
1da177e4
LT
643}
644
3e339b5d 645static int ksoftirqd_should_run(unsigned int cpu)
1da177e4 646{
3e339b5d
TG
647 return local_softirq_pending();
648}
1da177e4 649
3e339b5d
TG
650static void run_ksoftirqd(unsigned int cpu)
651{
652 local_irq_disable();
653 if (local_softirq_pending()) {
0bed698a
FW
654 /*
655 * We can safely run softirq on inline stack, as we are not deep
656 * in the task stack here.
657 */
3e339b5d
TG
658 __do_softirq();
659 rcu_note_context_switch(cpu);
660 local_irq_enable();
661 cond_resched();
662 return;
1da177e4 663 }
3e339b5d 664 local_irq_enable();
1da177e4
LT
665}
666
667#ifdef CONFIG_HOTPLUG_CPU
668/*
669 * tasklet_kill_immediate is called to remove a tasklet which can already be
670 * scheduled for execution on @cpu.
671 *
672 * Unlike tasklet_kill, this function removes the tasklet
673 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
674 *
675 * When this function is called, @cpu must be in the CPU_DEAD state.
676 */
677void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
678{
679 struct tasklet_struct **i;
680
681 BUG_ON(cpu_online(cpu));
682 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
683
684 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
685 return;
686
687 /* CPU is dead, so no lock needed. */
48f20a9a 688 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
1da177e4
LT
689 if (*i == t) {
690 *i = t->next;
48f20a9a
OJ
691 /* If this was the tail element, move the tail ptr */
692 if (*i == NULL)
693 per_cpu(tasklet_vec, cpu).tail = i;
1da177e4
LT
694 return;
695 }
696 }
697 BUG();
698}
699
700static void takeover_tasklets(unsigned int cpu)
701{
1da177e4
LT
702 /* CPU is dead, so no lock needed. */
703 local_irq_disable();
704
705 /* Find end, append list for that CPU. */
e5e41723 706 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
909ea964
CL
707 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
708 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
e5e41723
CB
709 per_cpu(tasklet_vec, cpu).head = NULL;
710 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
711 }
1da177e4
LT
712 raise_softirq_irqoff(TASKLET_SOFTIRQ);
713
e5e41723 714 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
909ea964
CL
715 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
716 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
e5e41723
CB
717 per_cpu(tasklet_hi_vec, cpu).head = NULL;
718 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
719 }
1da177e4
LT
720 raise_softirq_irqoff(HI_SOFTIRQ);
721
722 local_irq_enable();
723}
724#endif /* CONFIG_HOTPLUG_CPU */
725
ce85b4f2
JP
726static int cpu_callback(struct notifier_block *nfb, unsigned long action,
727 void *hcpu)
1da177e4 728{
1da177e4 729 switch (action) {
1da177e4 730#ifdef CONFIG_HOTPLUG_CPU
1da177e4 731 case CPU_DEAD:
3e339b5d
TG
732 case CPU_DEAD_FROZEN:
733 takeover_tasklets((unsigned long)hcpu);
1da177e4
LT
734 break;
735#endif /* CONFIG_HOTPLUG_CPU */
3e339b5d 736 }
1da177e4
LT
737 return NOTIFY_OK;
738}
739
0db0628d 740static struct notifier_block cpu_nfb = {
1da177e4
LT
741 .notifier_call = cpu_callback
742};
743
3e339b5d
TG
744static struct smp_hotplug_thread softirq_threads = {
745 .store = &ksoftirqd,
746 .thread_should_run = ksoftirqd_should_run,
747 .thread_fn = run_ksoftirqd,
748 .thread_comm = "ksoftirqd/%u",
749};
750
7babe8db 751static __init int spawn_ksoftirqd(void)
1da177e4 752{
1da177e4 753 register_cpu_notifier(&cpu_nfb);
3e339b5d
TG
754
755 BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
756
1da177e4
LT
757 return 0;
758}
7babe8db 759early_initcall(spawn_ksoftirqd);
78eef01b 760
43a25632
YL
761/*
762 * [ These __weak aliases are kept in a separate compilation unit, so that
763 * GCC does not inline them incorrectly. ]
764 */
765
766int __init __weak early_irq_init(void)
767{
768 return 0;
769}
770
4a046d17
YL
771int __init __weak arch_probe_nr_irqs(void)
772{
b683de2b 773 return NR_IRQS_LEGACY;
4a046d17
YL
774}
775
43a25632
YL
776int __init __weak arch_early_irq_init(void)
777{
778 return 0;
779}
62a08ae2
TG
780
781unsigned int __weak arch_dynirq_lower_bound(unsigned int from)
782{
783 return from;
784}