]>
Commit | Line | Data |
---|---|---|
1 | /* SPDX-License-Identifier: GPL-2.0 */ | |
2 | #ifndef _ASM_X86_IRQ_STACK_H | |
3 | #define _ASM_X86_IRQ_STACK_H | |
4 | ||
5 | #include <linux/ptrace.h> | |
6 | ||
7 | #include <asm/processor.h> | |
8 | ||
9 | #ifdef CONFIG_X86_64 | |
10 | static __always_inline bool irqstack_active(void) | |
11 | { | |
12 | return __this_cpu_read(hardirq_stack_inuse); | |
13 | } | |
14 | ||
15 | void asm_call_on_stack(void *sp, void (*func)(void), void *arg); | |
16 | void asm_call_sysvec_on_stack(void *sp, void (*func)(struct pt_regs *regs), | |
17 | struct pt_regs *regs); | |
18 | void asm_call_irq_on_stack(void *sp, void (*func)(struct irq_desc *desc), | |
19 | struct irq_desc *desc); | |
20 | ||
21 | static __always_inline void __run_on_irqstack(void (*func)(void)) | |
22 | { | |
23 | void *tos = __this_cpu_read(hardirq_stack_ptr); | |
24 | ||
25 | __this_cpu_write(hardirq_stack_inuse, true); | |
26 | asm_call_on_stack(tos, func, NULL); | |
27 | __this_cpu_write(hardirq_stack_inuse, false); | |
28 | } | |
29 | ||
30 | static __always_inline void | |
31 | __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs), | |
32 | struct pt_regs *regs) | |
33 | { | |
34 | void *tos = __this_cpu_read(hardirq_stack_ptr); | |
35 | ||
36 | __this_cpu_write(hardirq_stack_inuse, true); | |
37 | asm_call_sysvec_on_stack(tos, func, regs); | |
38 | __this_cpu_write(hardirq_stack_inuse, false); | |
39 | } | |
40 | ||
41 | static __always_inline void | |
42 | __run_irq_on_irqstack(void (*func)(struct irq_desc *desc), | |
43 | struct irq_desc *desc) | |
44 | { | |
45 | void *tos = __this_cpu_read(hardirq_stack_ptr); | |
46 | ||
47 | __this_cpu_write(hardirq_stack_inuse, true); | |
48 | asm_call_irq_on_stack(tos, func, desc); | |
49 | __this_cpu_write(hardirq_stack_inuse, false); | |
50 | } | |
51 | ||
52 | #else /* CONFIG_X86_64 */ | |
53 | static inline bool irqstack_active(void) { return false; } | |
54 | static inline void __run_on_irqstack(void (*func)(void)) { } | |
55 | static inline void __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs), | |
56 | struct pt_regs *regs) { } | |
57 | static inline void __run_irq_on_irqstack(void (*func)(struct irq_desc *desc), | |
58 | struct irq_desc *desc) { } | |
59 | #endif /* !CONFIG_X86_64 */ | |
60 | ||
61 | static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs) | |
62 | { | |
63 | if (IS_ENABLED(CONFIG_X86_32)) | |
64 | return false; | |
65 | if (!regs) | |
66 | return !irqstack_active(); | |
67 | return !user_mode(regs) && !irqstack_active(); | |
68 | } | |
69 | ||
70 | ||
71 | static __always_inline void run_on_irqstack_cond(void (*func)(void), | |
72 | struct pt_regs *regs) | |
73 | { | |
74 | lockdep_assert_irqs_disabled(); | |
75 | ||
76 | if (irq_needs_irq_stack(regs)) | |
77 | __run_on_irqstack(func); | |
78 | else | |
79 | func(); | |
80 | } | |
81 | ||
82 | static __always_inline void | |
83 | run_sysvec_on_irqstack_cond(void (*func)(struct pt_regs *regs), | |
84 | struct pt_regs *regs) | |
85 | { | |
86 | lockdep_assert_irqs_disabled(); | |
87 | ||
88 | if (irq_needs_irq_stack(regs)) | |
89 | __run_sysvec_on_irqstack(func, regs); | |
90 | else | |
91 | func(regs); | |
92 | } | |
93 | ||
94 | static __always_inline void | |
95 | run_irq_on_irqstack_cond(void (*func)(struct irq_desc *desc), struct irq_desc *desc, | |
96 | struct pt_regs *regs) | |
97 | { | |
98 | lockdep_assert_irqs_disabled(); | |
99 | ||
100 | if (irq_needs_irq_stack(regs)) | |
101 | __run_irq_on_irqstack(func, desc); | |
102 | else | |
103 | func(desc); | |
104 | } | |
105 | ||
106 | #endif |