x86/irq/64: Adjust the per CPU irq stack pointer by 8

The per CPU hardirq_stack_ptr contains the pointer to the irq stack in the
form that it is ready to be assigned to [ER]SP so that the first push ends
up on the top entry of the stack.

But the stack switching on 64 bit has the following rules:

    1) Store the current stack pointer (RSP) in the top most stack entry
       to allow the unwinder to link back to the previous stack

    2) Set RSP to the top most stack entry

    3) Invoke functions on the irq stack

    4) Pop RSP from the top most stack entry (stored in #1) so it's back
       to the original stack.

That requires all stack switching code to decrement the stored pointer by 8
in order to be able to store the current RSP and then set RSP to that
location. That's a pointless exercise.

Do the -8 adjustment right when storing the pointer and make the data type
a void pointer to avoid confusion vs. the struct irq_stack data type which
is on 64bit only used to declare the backing store. Move the definition
next to the inuse flag so they likely end up in the same cache
line. Sticking them into a struct to enforce it is a seperate change.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Kees Cook <keescook@chromium.org>
Link: https://lore.kernel.org/r/20210210002512.354260928@linutronix.de

diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h
index 4487bb989cea9c7ec456ceee7106e379e2f44c6d..554f84bca1c3226318097e9e2917fd458f826e3b 100644 (file)
--- a/arch/x86/include/asm/irq_stack.h
+++ b/arch/x86/include/asm/irq_stack.h
@@ -23,7 +23,7 @@ static __always_inline void __run_on_irqstack(void (*func)(void))
        void *tos = __this_cpu_read(hardirq_stack_ptr);
 
        __this_cpu_write(hardirq_stack_inuse, true);
-       asm_call_on_stack(tos - 8, func, NULL);
+       asm_call_on_stack(tos, func, NULL);
        __this_cpu_write(hardirq_stack_inuse, false);
 }
 
@@ -34,7 +34,7 @@ __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
        void *tos = __this_cpu_read(hardirq_stack_ptr);
 
        __this_cpu_write(hardirq_stack_inuse, true);
-       asm_call_sysvec_on_stack(tos - 8, func, regs);
+       asm_call_sysvec_on_stack(tos, func, regs);
        __this_cpu_write(hardirq_stack_inuse, false);
 }
 
@@ -45,7 +45,7 @@ __run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
        void *tos = __this_cpu_read(hardirq_stack_ptr);
 
        __this_cpu_write(hardirq_stack_inuse, true);
-       asm_call_irq_on_stack(tos - 8, func, desc);
+       asm_call_irq_on_stack(tos, func, desc);
        __this_cpu_write(hardirq_stack_inuse, false);
 }
 

diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 11d10f4f3059cf3deb6c3b8e4a4321512d5449c9..dc6d149bf851fa911c7c7915fbe68e39f1016b4c 100644 (file)
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -426,8 +426,6 @@ struct irq_stack {
        char            stack[IRQ_STACK_SIZE];
 } __aligned(IRQ_STACK_SIZE);
 
-DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);
-
 #ifdef CONFIG_X86_32
 DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);
 #else
@@ -454,6 +452,7 @@ static inline unsigned long cpu_kernelmode_gs_base(int cpu)
        return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu);
 }
 
+DECLARE_PER_CPU(void *, hardirq_stack_ptr);
 DECLARE_PER_CPU(bool, hardirq_stack_inuse);
 extern asmlinkage void ignore_sysret(void);
 
@@ -473,9 +472,9 @@ struct stack_canary {
 };
 DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
 #endif
-/* Per CPU softirq stack pointer */
+DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);
 DECLARE_PER_CPU(struct irq_stack *, softirq_stack_ptr);
-#endif /* X86_64 */
+#endif /* !X86_64 */
 
 extern unsigned int fpu_kernel_xstate_size;
 extern unsigned int fpu_user_xstate_size;

diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 845c8a46cf7b291c138ce1fdf060afe7e4e6fd33..c5e23f351f676d7134c9798b3ca00b883a11313b 100644 (file)
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1739,7 +1739,7 @@ DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
        &init_task;
 EXPORT_PER_CPU_SYMBOL(current_task);
 
-DEFINE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);
+DEFINE_PER_CPU(void *, hardirq_stack_ptr);
 DEFINE_PER_CPU(bool, hardirq_stack_inuse);
 
 DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;

diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c
index 1dd851397bd909a915447d75975d2ab6db52413b..5601b95944faedef320f6fb980df21f4fbe58cd6 100644 (file)
--- a/arch/x86/kernel/dumpstack_64.c
+++ b/arch/x86/kernel/dumpstack_64.c
@@ -128,12 +128,21 @@ static __always_inline bool in_exception_stack(unsigned long *stack, struct stac
 
 static __always_inline bool in_irq_stack(unsigned long *stack, struct stack_info *info)
 {
-       unsigned long *end   = (unsigned long *)this_cpu_read(hardirq_stack_ptr);
-       unsigned long *begin = end - (IRQ_STACK_SIZE / sizeof(long));
+       unsigned long *end = (unsigned long *)this_cpu_read(hardirq_stack_ptr);
+       unsigned long *begin;
 
        /*
-        * This is a software stack, so 'end' can be a valid stack pointer.
-        * It just means the stack is empty.
+        * @end points directly to the top most stack entry to avoid a -8
+        * adjustment in the stack switch hotpath. Adjust it back before
+        * calculating @begin.
+        */
+       end++;
+       begin = end - (IRQ_STACK_SIZE / sizeof(long));
+
+       /*
+        * Due to the switching logic RSP can never be == @end because the
+        * final operation is 'popq %rsp' which means after that RSP points
+        * to the original stack and not to @end.
         */
        if (stack < begin || stack >= end)
                return false;
@@ -143,8 +152,9 @@ static __always_inline bool in_irq_stack(unsigned long *stack, struct stack_info
        info->end       = end;
 
        /*
-        * The next stack pointer is the first thing pushed by the entry code
-        * after switching to the irq stack.
+        * The next stack pointer is stored at the top of the irq stack
+        * before switching to the irq stack. Actual stack entries are all
+        * below that.
         */
        info->next_sp = (unsigned long *)*(end - 1);
 

diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c
index 440eed558558d9891547ee1bd09a4570bc398156..7103f98899304bc61e8f08838f1ec02aab5612a6 100644 (file)
--- a/arch/x86/kernel/irq_64.c
+++ b/arch/x86/kernel/irq_64.c
@@ -48,7 +48,8 @@ static int map_irq_stack(unsigned int cpu)
        if (!va)
                return -ENOMEM;
 
-       per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE;
+       /* Store actual TOS to avoid adjustment in the hotpath */
+       per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
        return 0;
 }
 #else
@@ -60,7 +61,8 @@ static int map_irq_stack(unsigned int cpu)
 {
        void *va = per_cpu_ptr(&irq_stack_backing_store, cpu);
 
-       per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE;
+       /* Store actual TOS to avoid adjustment in the hotpath */
+       per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
        return 0;
 }
 #endif