1 // SPDX-License-Identifier: GPL-2.0-only
3 * linux/kernel/softirq.c
5 * Copyright (C) 1992 Linus Torvalds
7 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12 #include <linux/export.h>
13 #include <linux/kernel_stat.h>
14 #include <linux/interrupt.h>
15 #include <linux/init.h>
17 #include <linux/notifier.h>
18 #include <linux/percpu.h>
19 #include <linux/cpu.h>
20 #include <linux/freezer.h>
21 #include <linux/kthread.h>
22 #include <linux/rcupdate.h>
23 #include <linux/ftrace.h>
24 #include <linux/smp.h>
25 #include <linux/smpboot.h>
26 #include <linux/tick.h>
27 #include <linux/irq.h>
29 #include <asm/softirq_stack.h>
31 #define CREATE_TRACE_POINTS
32 #include <trace/events/irq.h>
35 - No shared variables, all the data are CPU local.
36 - If a softirq needs serialization, let it serialize itself
38 - Even if softirq is serialized, only local cpu is marked for
39 execution. Hence, we get something sort of weak cpu binding.
40 Though it is still not clear, will it result in better locality
44 - NET RX softirq. It is multithreaded and does not require
45 any global serialization.
46 - NET TX softirq. It kicks software netdevice queues, hence
47 it is logically serialized per device, but this serialization
48 is invisible to common code.
49 - Tasklets: serialized wrt itself.
52 #ifndef __ARCH_IRQ_STAT
53 DEFINE_PER_CPU_ALIGNED(irq_cpustat_t
, irq_stat
);
54 EXPORT_PER_CPU_SYMBOL(irq_stat
);
57 static struct softirq_action softirq_vec
[NR_SOFTIRQS
] __cacheline_aligned_in_smp
;
59 DEFINE_PER_CPU(struct task_struct
*, ksoftirqd
);
61 const char * const softirq_to_name
[NR_SOFTIRQS
] = {
62 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL",
63 "TASKLET", "SCHED", "HRTIMER", "RCU"
67 * we cannot loop indefinitely here to avoid userspace starvation,
68 * but we also don't want to introduce a worst case 1/HZ latency
69 * to the pending events, so lets the scheduler to balance
70 * the softirq load for us.
72 static void wakeup_softirqd(void)
74 /* Interrupts are disabled: no need to stop preemption */
75 struct task_struct
*tsk
= __this_cpu_read(ksoftirqd
);
77 if (tsk
&& tsk
->state
!= TASK_RUNNING
)
82 * If ksoftirqd is scheduled, we do not want to process pending softirqs
83 * right now. Let ksoftirqd handle this at its own rate, to get fairness,
84 * unless we're doing some of the synchronous softirqs.
86 #define SOFTIRQ_NOW_MASK ((1 << HI_SOFTIRQ) | (1 << TASKLET_SOFTIRQ))
87 static bool ksoftirqd_running(unsigned long pending
)
89 struct task_struct
*tsk
= __this_cpu_read(ksoftirqd
);
91 if (pending
& SOFTIRQ_NOW_MASK
)
93 return tsk
&& (tsk
->state
== TASK_RUNNING
) &&
94 !__kthread_should_park(tsk
);
97 #ifdef CONFIG_TRACE_IRQFLAGS
98 DEFINE_PER_CPU(int, hardirqs_enabled
);
99 DEFINE_PER_CPU(int, hardirq_context
);
100 EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled
);
101 EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context
);
105 * preempt_count and SOFTIRQ_OFFSET usage:
106 * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
107 * softirq processing.
108 * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
109 * on local_bh_disable or local_bh_enable.
110 * This lets us distinguish between whether we are currently processing
111 * softirq and whether we just have bh disabled.
114 #ifdef CONFIG_TRACE_IRQFLAGS
116 * This is for softirq.c-internal use, where hardirqs are disabled
119 void __local_bh_disable_ip(unsigned long ip
, unsigned int cnt
)
123 WARN_ON_ONCE(in_irq());
125 raw_local_irq_save(flags
);
127 * The preempt tracer hooks into preempt_count_add and will break
128 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
129 * is set and before current->softirq_enabled is cleared.
130 * We must manually increment preempt_count here and manually
131 * call the trace_preempt_off later.
133 __preempt_count_add(cnt
);
135 * Were softirqs turned off above:
137 if (softirq_count() == (cnt
& SOFTIRQ_MASK
))
138 lockdep_softirqs_off(ip
);
139 raw_local_irq_restore(flags
);
141 if (preempt_count() == cnt
) {
142 #ifdef CONFIG_DEBUG_PREEMPT
143 current
->preempt_disable_ip
= get_lock_parent_ip();
145 trace_preempt_off(CALLER_ADDR0
, get_lock_parent_ip());
148 EXPORT_SYMBOL(__local_bh_disable_ip
);
149 #endif /* CONFIG_TRACE_IRQFLAGS */
151 static void __local_bh_enable(unsigned int cnt
)
153 lockdep_assert_irqs_disabled();
155 if (preempt_count() == cnt
)
156 trace_preempt_on(CALLER_ADDR0
, get_lock_parent_ip());
158 if (softirq_count() == (cnt
& SOFTIRQ_MASK
))
159 lockdep_softirqs_on(_RET_IP_
);
161 __preempt_count_sub(cnt
);
165 * Special-case - softirqs can safely be enabled by __do_softirq(),
166 * without processing still-pending softirqs:
168 void _local_bh_enable(void)
170 WARN_ON_ONCE(in_irq());
171 __local_bh_enable(SOFTIRQ_DISABLE_OFFSET
);
173 EXPORT_SYMBOL(_local_bh_enable
);
175 void __local_bh_enable_ip(unsigned long ip
, unsigned int cnt
)
177 WARN_ON_ONCE(in_irq());
178 lockdep_assert_irqs_enabled();
179 #ifdef CONFIG_TRACE_IRQFLAGS
183 * Are softirqs going to be turned on now:
185 if (softirq_count() == SOFTIRQ_DISABLE_OFFSET
)
186 lockdep_softirqs_on(ip
);
188 * Keep preemption disabled until we are done with
189 * softirq processing:
191 __preempt_count_sub(cnt
- 1);
193 if (unlikely(!in_interrupt() && local_softirq_pending())) {
195 * Run softirq if any pending. And do it in its own stack
196 * as we may be calling this deep in a task call stack already.
202 #ifdef CONFIG_TRACE_IRQFLAGS
205 preempt_check_resched();
207 EXPORT_SYMBOL(__local_bh_enable_ip
);
209 static inline void invoke_softirq(void)
211 if (ksoftirqd_running(local_softirq_pending()))
214 if (!force_irqthreads
) {
215 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
217 * We can safely execute softirq on the current stack if
218 * it is the irq stack, because it should be near empty
224 * Otherwise, irq_exit() is called on the task stack that can
225 * be potentially deep already. So call softirq in its own stack
226 * to prevent from any overrun.
228 do_softirq_own_stack();
235 asmlinkage __visible
void do_softirq(void)
243 local_irq_save(flags
);
245 pending
= local_softirq_pending();
247 if (pending
&& !ksoftirqd_running(pending
))
248 do_softirq_own_stack();
250 local_irq_restore(flags
);
254 * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
255 * but break the loop if need_resched() is set or after 2 ms.
256 * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
257 * certain cases, such as stop_machine(), jiffies may cease to
258 * increment and so we need the MAX_SOFTIRQ_RESTART limit as
259 * well to make sure we eventually return from this method.
261 * These limits have been established via experimentation.
262 * The two things to balance is latency against fairness -
263 * we want to handle softirqs as soon as possible, but they
264 * should not be able to lock up the box.
266 #define MAX_SOFTIRQ_TIME msecs_to_jiffies(2)
267 #define MAX_SOFTIRQ_RESTART 10
269 #ifdef CONFIG_TRACE_IRQFLAGS
271 * When we run softirqs from irq_exit() and thus on the hardirq stack we need
272 * to keep the lockdep irq context tracking as tight as possible in order to
273 * not miss-qualify lock contexts and miss possible deadlocks.
276 static inline bool lockdep_softirq_start(void)
278 bool in_hardirq
= false;
280 if (lockdep_hardirq_context()) {
282 lockdep_hardirq_exit();
285 lockdep_softirq_enter();
290 static inline void lockdep_softirq_end(bool in_hardirq
)
292 lockdep_softirq_exit();
295 lockdep_hardirq_enter();
298 static inline bool lockdep_softirq_start(void) { return false; }
299 static inline void lockdep_softirq_end(bool in_hardirq
) { }
302 asmlinkage __visible
void __softirq_entry
__do_softirq(void)
304 unsigned long end
= jiffies
+ MAX_SOFTIRQ_TIME
;
305 unsigned long old_flags
= current
->flags
;
306 int max_restart
= MAX_SOFTIRQ_RESTART
;
307 struct softirq_action
*h
;
313 * Mask out PF_MEMALLOC as the current task context is borrowed for the
314 * softirq. A softirq handled, such as network RX, might set PF_MEMALLOC
315 * again if the socket is related to swapping.
317 current
->flags
&= ~PF_MEMALLOC
;
319 pending
= local_softirq_pending();
321 __local_bh_disable_ip(_RET_IP_
, SOFTIRQ_OFFSET
);
322 in_hardirq
= lockdep_softirq_start();
323 account_softirq_enter(current
);
326 /* Reset the pending bitmask before enabling irqs */
327 set_softirq_pending(0);
333 while ((softirq_bit
= ffs(pending
))) {
337 h
+= softirq_bit
- 1;
339 vec_nr
= h
- softirq_vec
;
340 prev_count
= preempt_count();
342 kstat_incr_softirqs_this_cpu(vec_nr
);
344 trace_softirq_entry(vec_nr
);
346 trace_softirq_exit(vec_nr
);
347 if (unlikely(prev_count
!= preempt_count())) {
348 pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
349 vec_nr
, softirq_to_name
[vec_nr
], h
->action
,
350 prev_count
, preempt_count());
351 preempt_count_set(prev_count
);
354 pending
>>= softirq_bit
;
357 if (__this_cpu_read(ksoftirqd
) == current
)
361 pending
= local_softirq_pending();
363 if (time_before(jiffies
, end
) && !need_resched() &&
370 account_softirq_exit(current
);
371 lockdep_softirq_end(in_hardirq
);
372 __local_bh_enable(SOFTIRQ_OFFSET
);
373 WARN_ON_ONCE(in_interrupt());
374 current_restore_flags(old_flags
, PF_MEMALLOC
);
378 * irq_enter_rcu - Enter an interrupt context with RCU watching
380 void irq_enter_rcu(void)
384 if (is_idle_task(current
) && (irq_count() == HARDIRQ_OFFSET
))
387 account_hardirq_enter(current
);
391 * irq_enter - Enter an interrupt context including RCU update
399 static inline void tick_irq_exit(void)
401 #ifdef CONFIG_NO_HZ_COMMON
402 int cpu
= smp_processor_id();
404 /* Make sure that timer wheel updates are propagated */
405 if ((idle_cpu(cpu
) && !need_resched()) || tick_nohz_full_cpu(cpu
)) {
407 tick_nohz_irq_exit();
412 static inline void __irq_exit_rcu(void)
414 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
417 lockdep_assert_irqs_disabled();
419 account_hardirq_exit(current
);
420 preempt_count_sub(HARDIRQ_OFFSET
);
421 if (!in_interrupt() && local_softirq_pending())
428 * irq_exit_rcu() - Exit an interrupt context without updating RCU
430 * Also processes softirqs if needed and possible.
432 void irq_exit_rcu(void)
436 lockdep_hardirq_exit();
440 * irq_exit - Exit an interrupt context, update RCU and lockdep
442 * Also processes softirqs if needed and possible.
449 lockdep_hardirq_exit();
453 * This function must run with irqs disabled!
455 inline void raise_softirq_irqoff(unsigned int nr
)
457 __raise_softirq_irqoff(nr
);
460 * If we're in an interrupt or softirq, we're done
461 * (this also catches softirq-disabled code). We will
462 * actually run the softirq once we return from
463 * the irq or softirq.
465 * Otherwise we wake up ksoftirqd to make sure we
466 * schedule the softirq soon.
472 void raise_softirq(unsigned int nr
)
476 local_irq_save(flags
);
477 raise_softirq_irqoff(nr
);
478 local_irq_restore(flags
);
481 void __raise_softirq_irqoff(unsigned int nr
)
483 lockdep_assert_irqs_disabled();
484 trace_softirq_raise(nr
);
485 or_softirq_pending(1UL << nr
);
488 void open_softirq(int nr
, void (*action
)(struct softirq_action
*))
490 softirq_vec
[nr
].action
= action
;
496 struct tasklet_head
{
497 struct tasklet_struct
*head
;
498 struct tasklet_struct
**tail
;
501 static DEFINE_PER_CPU(struct tasklet_head
, tasklet_vec
);
502 static DEFINE_PER_CPU(struct tasklet_head
, tasklet_hi_vec
);
504 static void __tasklet_schedule_common(struct tasklet_struct
*t
,
505 struct tasklet_head __percpu
*headp
,
506 unsigned int softirq_nr
)
508 struct tasklet_head
*head
;
511 local_irq_save(flags
);
512 head
= this_cpu_ptr(headp
);
515 head
->tail
= &(t
->next
);
516 raise_softirq_irqoff(softirq_nr
);
517 local_irq_restore(flags
);
520 void __tasklet_schedule(struct tasklet_struct
*t
)
522 __tasklet_schedule_common(t
, &tasklet_vec
,
525 EXPORT_SYMBOL(__tasklet_schedule
);
527 void __tasklet_hi_schedule(struct tasklet_struct
*t
)
529 __tasklet_schedule_common(t
, &tasklet_hi_vec
,
532 EXPORT_SYMBOL(__tasklet_hi_schedule
);
534 static void tasklet_action_common(struct softirq_action
*a
,
535 struct tasklet_head
*tl_head
,
536 unsigned int softirq_nr
)
538 struct tasklet_struct
*list
;
541 list
= tl_head
->head
;
542 tl_head
->head
= NULL
;
543 tl_head
->tail
= &tl_head
->head
;
547 struct tasklet_struct
*t
= list
;
551 if (tasklet_trylock(t
)) {
552 if (!atomic_read(&t
->count
)) {
553 if (!test_and_clear_bit(TASKLET_STATE_SCHED
,
569 tl_head
->tail
= &t
->next
;
570 __raise_softirq_irqoff(softirq_nr
);
575 static __latent_entropy
void tasklet_action(struct softirq_action
*a
)
577 tasklet_action_common(a
, this_cpu_ptr(&tasklet_vec
), TASKLET_SOFTIRQ
);
580 static __latent_entropy
void tasklet_hi_action(struct softirq_action
*a
)
582 tasklet_action_common(a
, this_cpu_ptr(&tasklet_hi_vec
), HI_SOFTIRQ
);
585 void tasklet_setup(struct tasklet_struct
*t
,
586 void (*callback
)(struct tasklet_struct
*))
590 atomic_set(&t
->count
, 0);
591 t
->callback
= callback
;
592 t
->use_callback
= true;
595 EXPORT_SYMBOL(tasklet_setup
);
597 void tasklet_init(struct tasklet_struct
*t
,
598 void (*func
)(unsigned long), unsigned long data
)
602 atomic_set(&t
->count
, 0);
604 t
->use_callback
= false;
607 EXPORT_SYMBOL(tasklet_init
);
609 void tasklet_kill(struct tasklet_struct
*t
)
612 pr_notice("Attempt to kill tasklet from interrupt\n");
614 while (test_and_set_bit(TASKLET_STATE_SCHED
, &t
->state
)) {
617 } while (test_bit(TASKLET_STATE_SCHED
, &t
->state
));
619 tasklet_unlock_wait(t
);
620 clear_bit(TASKLET_STATE_SCHED
, &t
->state
);
622 EXPORT_SYMBOL(tasklet_kill
);
624 void __init
softirq_init(void)
628 for_each_possible_cpu(cpu
) {
629 per_cpu(tasklet_vec
, cpu
).tail
=
630 &per_cpu(tasklet_vec
, cpu
).head
;
631 per_cpu(tasklet_hi_vec
, cpu
).tail
=
632 &per_cpu(tasklet_hi_vec
, cpu
).head
;
635 open_softirq(TASKLET_SOFTIRQ
, tasklet_action
);
636 open_softirq(HI_SOFTIRQ
, tasklet_hi_action
);
639 static int ksoftirqd_should_run(unsigned int cpu
)
641 return local_softirq_pending();
644 static void run_ksoftirqd(unsigned int cpu
)
647 if (local_softirq_pending()) {
649 * We can safely run softirq on inline stack, as we are not deep
650 * in the task stack here.
660 #ifdef CONFIG_HOTPLUG_CPU
662 * tasklet_kill_immediate is called to remove a tasklet which can already be
663 * scheduled for execution on @cpu.
665 * Unlike tasklet_kill, this function removes the tasklet
666 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
668 * When this function is called, @cpu must be in the CPU_DEAD state.
670 void tasklet_kill_immediate(struct tasklet_struct
*t
, unsigned int cpu
)
672 struct tasklet_struct
**i
;
674 BUG_ON(cpu_online(cpu
));
675 BUG_ON(test_bit(TASKLET_STATE_RUN
, &t
->state
));
677 if (!test_bit(TASKLET_STATE_SCHED
, &t
->state
))
680 /* CPU is dead, so no lock needed. */
681 for (i
= &per_cpu(tasklet_vec
, cpu
).head
; *i
; i
= &(*i
)->next
) {
684 /* If this was the tail element, move the tail ptr */
686 per_cpu(tasklet_vec
, cpu
).tail
= i
;
693 static int takeover_tasklets(unsigned int cpu
)
695 /* CPU is dead, so no lock needed. */
698 /* Find end, append list for that CPU. */
699 if (&per_cpu(tasklet_vec
, cpu
).head
!= per_cpu(tasklet_vec
, cpu
).tail
) {
700 *__this_cpu_read(tasklet_vec
.tail
) = per_cpu(tasklet_vec
, cpu
).head
;
701 __this_cpu_write(tasklet_vec
.tail
, per_cpu(tasklet_vec
, cpu
).tail
);
702 per_cpu(tasklet_vec
, cpu
).head
= NULL
;
703 per_cpu(tasklet_vec
, cpu
).tail
= &per_cpu(tasklet_vec
, cpu
).head
;
705 raise_softirq_irqoff(TASKLET_SOFTIRQ
);
707 if (&per_cpu(tasklet_hi_vec
, cpu
).head
!= per_cpu(tasklet_hi_vec
, cpu
).tail
) {
708 *__this_cpu_read(tasklet_hi_vec
.tail
) = per_cpu(tasklet_hi_vec
, cpu
).head
;
709 __this_cpu_write(tasklet_hi_vec
.tail
, per_cpu(tasklet_hi_vec
, cpu
).tail
);
710 per_cpu(tasklet_hi_vec
, cpu
).head
= NULL
;
711 per_cpu(tasklet_hi_vec
, cpu
).tail
= &per_cpu(tasklet_hi_vec
, cpu
).head
;
713 raise_softirq_irqoff(HI_SOFTIRQ
);
719 #define takeover_tasklets NULL
720 #endif /* CONFIG_HOTPLUG_CPU */
722 static struct smp_hotplug_thread softirq_threads
= {
724 .thread_should_run
= ksoftirqd_should_run
,
725 .thread_fn
= run_ksoftirqd
,
726 .thread_comm
= "ksoftirqd/%u",
729 static __init
int spawn_ksoftirqd(void)
731 cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD
, "softirq:dead", NULL
,
733 BUG_ON(smpboot_register_percpu_thread(&softirq_threads
));
737 early_initcall(spawn_ksoftirqd
);
740 * [ These __weak aliases are kept in a separate compilation unit, so that
741 * GCC does not inline them incorrectly. ]
744 int __init __weak
early_irq_init(void)
749 int __init __weak
arch_probe_nr_irqs(void)
751 return NR_IRQS_LEGACY
;
754 int __init __weak
arch_early_irq_init(void)
759 unsigned int __weak
arch_dynirq_lower_bound(unsigned int from
)