]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
1da177e4 LT |
2 | #ifndef LINUX_HARDIRQ_H |
3 | #define LINUX_HARDIRQ_H | |
4 | ||
aaf2bc50 | 5 | #include <linux/context_tracking_state.h> |
92cf2118 | 6 | #include <linux/preempt.h> |
fbb9ce95 | 7 | #include <linux/lockdep.h> |
6a60dd12 | 8 | #include <linux/ftrace_irq.h> |
dcbf832e | 9 | #include <linux/vtime.h> |
0bd3a173 | 10 | #include <asm/hardirq.h> |
1da177e4 | 11 | |
1da177e4 | 12 | extern void synchronize_irq(unsigned int irq); |
02cea395 | 13 | extern bool synchronize_hardirq(unsigned int irq); |
1da177e4 | 14 | |
aaf2bc50 PM |
15 | #ifdef CONFIG_NO_HZ_FULL |
16 | void __rcu_irq_enter_check_tick(void); | |
17 | #else | |
18 | static inline void __rcu_irq_enter_check_tick(void) { } | |
19 | #endif | |
9b1d82fa | 20 | |
aaf2bc50 | 21 | static __always_inline void rcu_irq_enter_check_tick(void) |
9b1d82fa | 22 | { |
aaf2bc50 PM |
23 | if (context_tracking_enabled()) |
24 | __rcu_irq_enter_check_tick(); | |
9b1d82fa PM |
25 | } |
26 | ||
de30a2b3 IM |
27 | /* |
28 | * It is safe to do non-atomic ops on ->hardirq_context, | |
29 | * because NMI handlers may not preempt and the ops are | |
30 | * always balanced, so the interrupted value of ->hardirq_context | |
31 | * will always be restored. | |
32 | */ | |
79bf2bb3 TG |
33 | #define __irq_enter() \ |
34 | do { \ | |
bdb43806 | 35 | preempt_count_add(HARDIRQ_OFFSET); \ |
2502ec37 | 36 | lockdep_hardirq_enter(); \ |
d3759e71 | 37 | account_hardirq_enter(current); \ |
79bf2bb3 TG |
38 | } while (0) |
39 | ||
98a3bf19 TG |
40 | /* |
41 | * Like __irq_enter() without time accounting for fast | |
42 | * interrupts, e.g. reschedule IPI where time accounting | |
43 | * is more expensive than the actual interrupt. | |
44 | */ | |
45 | #define __irq_enter_raw() \ | |
46 | do { \ | |
47 | preempt_count_add(HARDIRQ_OFFSET); \ | |
48 | lockdep_hardirq_enter(); \ | |
49 | } while (0) | |
50 | ||
79bf2bb3 TG |
51 | /* |
52 | * Enter irq context (on NO_HZ, update jiffies): | |
53 | */ | |
8a6bc478 TG |
54 | void irq_enter(void); |
55 | /* | |
56 | * Like irq_enter(), but RCU is already watching. | |
57 | */ | |
58 | void irq_enter_rcu(void); | |
de30a2b3 IM |
59 | |
60 | /* | |
61 | * Exit irq context without processing softirqs: | |
62 | */ | |
63 | #define __irq_exit() \ | |
64 | do { \ | |
d3759e71 | 65 | account_hardirq_exit(current); \ |
2502ec37 | 66 | lockdep_hardirq_exit(); \ |
bdb43806 | 67 | preempt_count_sub(HARDIRQ_OFFSET); \ |
1da177e4 LT |
68 | } while (0) |
69 | ||
98a3bf19 TG |
70 | /* |
71 | * Like __irq_exit() without time accounting | |
72 | */ | |
73 | #define __irq_exit_raw() \ | |
74 | do { \ | |
75 | lockdep_hardirq_exit(); \ | |
76 | preempt_count_sub(HARDIRQ_OFFSET); \ | |
77 | } while (0) | |
78 | ||
de30a2b3 IM |
79 | /* |
80 | * Exit irq context and process softirqs if needed: | |
81 | */ | |
8a6bc478 TG |
82 | void irq_exit(void); |
83 | ||
84 | /* | |
85 | * Like irq_exit(), but return with RCU watching. | |
86 | */ | |
87 | void irq_exit_rcu(void); | |
1da177e4 | 88 | |
5870970b JT |
89 | #ifndef arch_nmi_enter |
90 | #define arch_nmi_enter() do { } while (0) | |
91 | #define arch_nmi_exit() do { } while (0) | |
92 | #endif | |
93 | ||
aaf2bc50 PM |
94 | #ifdef CONFIG_TINY_RCU |
95 | static inline void rcu_nmi_enter(void) { } | |
96 | static inline void rcu_nmi_exit(void) { } | |
97 | #else | |
98 | extern void rcu_nmi_enter(void); | |
99 | extern void rcu_nmi_exit(void); | |
100 | #endif | |
101 | ||
f93524eb PZ |
102 | /* |
103 | * NMI vs Tracing | |
104 | * -------------- | |
105 | * | |
106 | * We must not land in a tracer until (or after) we've changed preempt_count | |
107 | * such that in_nmi() becomes true. To that effect all NMI C entry points must | |
108 | * be marked 'notrace' and call nmi_enter() as soon as possible. | |
109 | */ | |
110 | ||
69ea03b5 PZ |
111 | /* |
112 | * nmi_enter() can nest up to 15 times; see NMI_BITS. | |
113 | */ | |
ba1f2b2e | 114 | #define __nmi_enter() \ |
2a7b8df0 | 115 | do { \ |
ba1f2b2e | 116 | lockdep_off(); \ |
5870970b | 117 | arch_nmi_enter(); \ |
42a0bb3f | 118 | printk_nmi_enter(); \ |
69ea03b5 | 119 | BUG_ON(in_nmi() == NMI_MASK); \ |
f93524eb | 120 | __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \ |
ba1f2b2e PZ |
121 | } while (0) |
122 | ||
123 | #define nmi_enter() \ | |
124 | do { \ | |
125 | __nmi_enter(); \ | |
2502ec37 | 126 | lockdep_hardirq_enter(); \ |
ba1f2b2e | 127 | rcu_nmi_enter(); \ |
2ab70319 TG |
128 | instrumentation_begin(); \ |
129 | ftrace_nmi_enter(); \ | |
130 | instrumentation_end(); \ | |
17666f02 | 131 | } while (0) |
5f34fe1c | 132 | |
ba1f2b2e PZ |
133 | #define __nmi_exit() \ |
134 | do { \ | |
135 | BUG_ON(!in_nmi()); \ | |
136 | __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \ | |
137 | printk_nmi_exit(); \ | |
138 | arch_nmi_exit(); \ | |
139 | lockdep_on(); \ | |
140 | } while (0) | |
141 | ||
2a7b8df0 SR |
142 | #define nmi_exit() \ |
143 | do { \ | |
2ab70319 TG |
144 | instrumentation_begin(); \ |
145 | ftrace_nmi_exit(); \ | |
146 | instrumentation_end(); \ | |
2a7b8df0 | 147 | rcu_nmi_exit(); \ |
ba1f2b2e PZ |
148 | lockdep_hardirq_exit(); \ |
149 | __nmi_exit(); \ | |
17666f02 | 150 | } while (0) |
de30a2b3 | 151 | |
1da177e4 | 152 | #endif /* LINUX_HARDIRQ_H */ |