]>
Commit | Line | Data |
---|---|---|
931b9414 TG |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _ASM_X86_IRQ_STACK_H | |
3 | #define _ASM_X86_IRQ_STACK_H | |
4 | ||
5 | #include <linux/ptrace.h> | |
6 | ||
7 | #include <asm/processor.h> | |
8 | ||
9 | #ifdef CONFIG_X86_64 | |
10 | static __always_inline bool irqstack_active(void) | |
11 | { | |
e7f89001 | 12 | return __this_cpu_read(hardirq_stack_inuse); |
931b9414 TG |
13 | } |
14 | ||
a7b3474c TG |
15 | void asm_call_on_stack(void *sp, void (*func)(void), void *arg); |
16 | void asm_call_sysvec_on_stack(void *sp, void (*func)(struct pt_regs *regs), | |
17 | struct pt_regs *regs); | |
18 | void asm_call_irq_on_stack(void *sp, void (*func)(struct irq_desc *desc), | |
19 | struct irq_desc *desc); | |
931b9414 | 20 | |
a7b3474c | 21 | static __always_inline void __run_on_irqstack(void (*func)(void)) |
931b9414 TG |
22 | { |
23 | void *tos = __this_cpu_read(hardirq_stack_ptr); | |
24 | ||
e7f89001 | 25 | __this_cpu_write(hardirq_stack_inuse, true); |
951c2a51 | 26 | asm_call_on_stack(tos, func, NULL); |
e7f89001 | 27 | __this_cpu_write(hardirq_stack_inuse, false); |
a7b3474c TG |
28 | } |
29 | ||
30 | static __always_inline void | |
31 | __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs), | |
32 | struct pt_regs *regs) | |
33 | { | |
34 | void *tos = __this_cpu_read(hardirq_stack_ptr); | |
35 | ||
e7f89001 | 36 | __this_cpu_write(hardirq_stack_inuse, true); |
951c2a51 | 37 | asm_call_sysvec_on_stack(tos, func, regs); |
e7f89001 | 38 | __this_cpu_write(hardirq_stack_inuse, false); |
a7b3474c TG |
39 | } |
40 | ||
41 | static __always_inline void | |
42 | __run_irq_on_irqstack(void (*func)(struct irq_desc *desc), | |
43 | struct irq_desc *desc) | |
44 | { | |
45 | void *tos = __this_cpu_read(hardirq_stack_ptr); | |
46 | ||
e7f89001 | 47 | __this_cpu_write(hardirq_stack_inuse, true); |
951c2a51 | 48 | asm_call_irq_on_stack(tos, func, desc); |
e7f89001 | 49 | __this_cpu_write(hardirq_stack_inuse, false); |
931b9414 TG |
50 | } |
51 | ||
52 | #else /* CONFIG_X86_64 */ | |
53 | static inline bool irqstack_active(void) { return false; } | |
a7b3474c TG |
54 | static inline void __run_on_irqstack(void (*func)(void)) { } |
55 | static inline void __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs), | |
56 | struct pt_regs *regs) { } | |
57 | static inline void __run_irq_on_irqstack(void (*func)(struct irq_desc *desc), | |
58 | struct irq_desc *desc) { } | |
931b9414 TG |
59 | #endif /* !CONFIG_X86_64 */ |
60 | ||
61 | static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs) | |
62 | { | |
63 | if (IS_ENABLED(CONFIG_X86_32)) | |
64 | return false; | |
65 | if (!regs) | |
66 | return !irqstack_active(); | |
67 | return !user_mode(regs) && !irqstack_active(); | |
68 | } | |
69 | ||
a7b3474c TG |
70 | |
71 | static __always_inline void run_on_irqstack_cond(void (*func)(void), | |
931b9414 TG |
72 | struct pt_regs *regs) |
73 | { | |
a7b3474c TG |
74 | lockdep_assert_irqs_disabled(); |
75 | ||
76 | if (irq_needs_irq_stack(regs)) | |
77 | __run_on_irqstack(func); | |
78 | else | |
79 | func(); | |
80 | } | |
81 | ||
82 | static __always_inline void | |
83 | run_sysvec_on_irqstack_cond(void (*func)(struct pt_regs *regs), | |
84 | struct pt_regs *regs) | |
85 | { | |
86 | lockdep_assert_irqs_disabled(); | |
931b9414 | 87 | |
a7b3474c TG |
88 | if (irq_needs_irq_stack(regs)) |
89 | __run_sysvec_on_irqstack(func, regs); | |
90 | else | |
91 | func(regs); | |
92 | } | |
93 | ||
94 | static __always_inline void | |
95 | run_irq_on_irqstack_cond(void (*func)(struct irq_desc *desc), struct irq_desc *desc, | |
96 | struct pt_regs *regs) | |
97 | { | |
931b9414 TG |
98 | lockdep_assert_irqs_disabled(); |
99 | ||
100 | if (irq_needs_irq_stack(regs)) | |
a7b3474c | 101 | __run_irq_on_irqstack(func, desc); |
931b9414 | 102 | else |
a7b3474c | 103 | func(desc); |
931b9414 TG |
104 | } |
105 | ||
106 | #endif |