arm64: move non-entry code out of .entry.text

Currently, cpu_switch_to and ret_from_fork both live in .entry.text,
though neither form the critical path for an exception entry.

In subsequent patches, we will require that code in .entry.text is part
of the critical path for exception entry, for which we can assume
certain properties (e.g. the presence of exception regs on the stack).

Neither cpu_switch_to nor ret_from_fork will meet these requirements, so
we must move them out of .entry.text. To ensure that neither are kprobed
after being moved out of .entry.text, we must explicitly blacklist them,
requiring a new NOKPROBE() asm helper.

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Will Deacon <will.deacon@arm.com>

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 1b67c3782d005b5d1872e0ceedcf4250b3b3d8c9..610a420182419554cf069ce094e31025948d87d4 100644 (file)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -403,6 +403,17 @@ alternative_endif
        .size   __pi_##x, . - x;        \
        ENDPROC(x)
 
+/*
+ * Annotate a function as being unsuitable for kprobes.
+ */
+#ifdef CONFIG_KPROBES
+#define NOKPROBE(x)                            \
+       .pushsection "_kprobe_blacklist", "aw"; \
+       .quad   x;                              \
+       .popsection;
+#else
+#define NOKPROBE(x)
+#endif
        /*
         * Emit a 64-bit absolute little endian symbol reference in a way that
         * ensures that it will be resolved at build time, even when building a

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 660612a07ec59dfcadba97b0b78d5aa95ac38f94..9e126d3d8b531ee5fb439a3d3b8ecba175bf572d 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -710,38 +710,6 @@ el0_irq_naked:
        b       ret_to_user
 ENDPROC(el0_irq)
 
-/*
- * Register switch for AArch64. The callee-saved registers need to be saved
- * and restored. On entry:
- *   x0 = previous task_struct (must be preserved across the switch)
- *   x1 = next task_struct
- * Previous and next are guaranteed not to be the same.
- *
- */
-ENTRY(cpu_switch_to)
-       mov     x10, #THREAD_CPU_CONTEXT
-       add     x8, x0, x10
-       mov     x9, sp
-       stp     x19, x20, [x8], #16             // store callee-saved registers
-       stp     x21, x22, [x8], #16
-       stp     x23, x24, [x8], #16
-       stp     x25, x26, [x8], #16
-       stp     x27, x28, [x8], #16
-       stp     x29, x9, [x8], #16
-       str     lr, [x8]
-       add     x8, x1, x10
-       ldp     x19, x20, [x8], #16             // restore callee-saved registers
-       ldp     x21, x22, [x8], #16
-       ldp     x23, x24, [x8], #16
-       ldp     x25, x26, [x8], #16
-       ldp     x27, x28, [x8], #16
-       ldp     x29, x9, [x8], #16
-       ldr     lr, [x8]
-       mov     sp, x9
-       msr     sp_el0, x1
-       ret
-ENDPROC(cpu_switch_to)
-
 /*
  * This is the fast syscall return path.  We do as little as possible here,
  * and this includes saving x0 back into the kernel stack.
@@ -784,18 +752,6 @@ finish_ret_to_user:
        kernel_exit 0
 ENDPROC(ret_to_user)
 
-/*
- * This is how we return from a fork.
- */
-ENTRY(ret_from_fork)
-       bl      schedule_tail
-       cbz     x19, 1f                         // not a kernel thread
-       mov     x0, x20
-       blr     x19
-1:     get_thread_info tsk
-       b       ret_to_user
-ENDPROC(ret_from_fork)
-
 /*
  * SVC handler.
  */
@@ -869,3 +825,49 @@ ENTRY(sys_rt_sigreturn_wrapper)
        mov     x0, sp
        b       sys_rt_sigreturn
 ENDPROC(sys_rt_sigreturn_wrapper)
+
+/*
+ * Register switch for AArch64. The callee-saved registers need to be saved
+ * and restored. On entry:
+ *   x0 = previous task_struct (must be preserved across the switch)
+ *   x1 = next task_struct
+ * Previous and next are guaranteed not to be the same.
+ *
+ */
+ENTRY(cpu_switch_to)
+       mov     x10, #THREAD_CPU_CONTEXT
+       add     x8, x0, x10
+       mov     x9, sp
+       stp     x19, x20, [x8], #16             // store callee-saved registers
+       stp     x21, x22, [x8], #16
+       stp     x23, x24, [x8], #16
+       stp     x25, x26, [x8], #16
+       stp     x27, x28, [x8], #16
+       stp     x29, x9, [x8], #16
+       str     lr, [x8]
+       add     x8, x1, x10
+       ldp     x19, x20, [x8], #16             // restore callee-saved registers
+       ldp     x21, x22, [x8], #16
+       ldp     x23, x24, [x8], #16
+       ldp     x25, x26, [x8], #16
+       ldp     x27, x28, [x8], #16
+       ldp     x29, x9, [x8], #16
+       ldr     lr, [x8]
+       mov     sp, x9
+       msr     sp_el0, x1
+       ret
+ENDPROC(cpu_switch_to)
+NOKPROBE(cpu_switch_to)
+
+/*
+ * This is how we return from a fork.
+ */
+ENTRY(ret_from_fork)
+       bl      schedule_tail
+       cbz     x19, 1f                         // not a kernel thread
+       mov     x0, x20
+       blr     x19
+1:     get_thread_info tsk
+       b       ret_to_user
+ENDPROC(ret_from_fork)
+NOKPROBE(ret_from_fork)