vmlinux.lds.h: Create section for protection against instrumentation

Some code pathes, especially the low level entry code, must be protected
against instrumentation for various reasons:

 - Low level entry code can be a fragile beast, especially on x86.

 - With NO_HZ_FULL RCU state needs to be established before using it.

Having a dedicated section for such code allows to validate with tooling
that no unsafe functions are invoked.

Add the .noinstr.text section and the noinstr attribute to mark
functions. noinstr implies notrace. Kprobes will gain a section check
later.

Provide also a set of markers: instrumentation_begin()/end()

These are used to mark code inside a noinstr function which calls
into regular instrumentable text section as safe.

The instrumentation markers are only active when CONFIG_DEBUG_ENTRY is
enabled as the end marker emits a NOP to prevent the compiler from merging
the annotation points. This means the objtool verification requires a
kernel compiled with this option.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Acked-by: Peter Zijlstra <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20200505134100.075416272@linutronix.de

diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S
index 31a0f201fb6f43a315b6d6338eeb380d9a9bde1f..a1706b63b82de2ecaef7d212b469482683dfaa04 100644 (file)
--- a/arch/powerpc/kernel/vmlinux.lds.S
+++ b/arch/powerpc/kernel/vmlinux.lds.S
@@ -90,6 +90,7 @@ SECTIONS
 #ifdef CONFIG_PPC64
                *(.tramp.ftrace.text);
 #endif
+               NOINSTR_TEXT
                SCHED_TEXT
                CPUIDLE_TEXT
                LOCK_TEXT

diff --git a/include/asm-generic/sections.h b/include/asm-generic/sections.h
index d1779d442aa51a4d223ae8fd855c8fd93aa34077..66397ed10acb787d19ad524269096d3d28ff5a33 100644 (file)
--- a/include/asm-generic/sections.h
+++ b/include/asm-generic/sections.h
@@ -53,6 +53,9 @@ extern char __ctors_start[], __ctors_end[];
 /* Start and end of .opd section - used for function descriptors. */
 extern char __start_opd[], __end_opd[];
 
+/* Start and end of instrumentation protected text section */
+extern char __noinstr_text_start[], __noinstr_text_end[];
+
 extern __visible const void __nosave_begin, __nosave_end;
 
 /* Function descriptor handling (if any).  Override in asm/sections.h */

diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 71e387a5fe90e3fbe34835ee3edf08a26454439c..db600ef218d7d2e56b6fdf895b1958f80fa05462 100644 (file)
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -540,6 +540,15 @@
        . = ALIGN((align));                                             \
        __end_rodata = .;
 
+/*
+ * Non-instrumentable text section
+ */
+#define NOINSTR_TEXT                                                   \
+               ALIGN_FUNCTION();                                       \
+               __noinstr_text_start = .;                               \
+               *(.noinstr.text)                                        \
+               __noinstr_text_end = .;
+
 /*
  * .text section. Map to function alignment to avoid address changes
  * during second ld run in second ld pass when generating System.map
@@ -551,6 +560,7 @@
 #define TEXT_TEXT                                                      \
                ALIGN_FUNCTION();                                       \
                *(.text.hot TEXT_MAIN .text.fixup .text.unlikely)       \
+               NOINSTR_TEXT                                            \
                *(.text..refcount)                                      \
                *(.ref.text)                                            \
        MEM_KEEP(init.text*)                                            \

diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index 034b0a644efcc49ececa0fcc9b6c95ecfd4daee2..e9ead0505671572e6511a70ac3ca4e2498780ad7 100644 (file)
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -120,12 +120,65 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
 /* Annotate a C jump table to allow objtool to follow the code flow */
 #define __annotate_jump_table __section(.rodata..c_jump_table)
 
+#ifdef CONFIG_DEBUG_ENTRY
+/* Begin/end of an instrumentation safe region */
+#define instrumentation_begin() ({                                     \
+       asm volatile("%c0:\n\t"                                         \
+                    ".pushsection .discard.instr_begin\n\t"            \
+                    ".long %c0b - .\n\t"                               \
+                    ".popsection\n\t" : : "i" (__COUNTER__));          \
+})
+
+/*
+ * Because instrumentation_{begin,end}() can nest, objtool validation considers
+ * _begin() a +1 and _end() a -1 and computes a sum over the instructions.
+ * When the value is greater than 0, we consider instrumentation allowed.
+ *
+ * There is a problem with code like:
+ *
+ * noinstr void foo()
+ * {
+ *     instrumentation_begin();
+ *     ...
+ *     if (cond) {
+ *             instrumentation_begin();
+ *             ...
+ *             instrumentation_end();
+ *     }
+ *     bar();
+ *     instrumentation_end();
+ * }
+ *
+ * If instrumentation_end() would be an empty label, like all the other
+ * annotations, the inner _end(), which is at the end of a conditional block,
+ * would land on the instruction after the block.
+ *
+ * If we then consider the sum of the !cond path, we'll see that the call to
+ * bar() is with a 0-value, even though, we meant it to happen with a positive
+ * value.
+ *
+ * To avoid this, have _end() be a NOP instruction, this ensures it will be
+ * part of the condition block and does not escape.
+ */
+#define instrumentation_end() ({                                       \
+       asm volatile("%c0: nop\n\t"                                     \
+                    ".pushsection .discard.instr_end\n\t"              \
+                    ".long %c0b - .\n\t"                               \
+                    ".popsection\n\t" : : "i" (__COUNTER__));          \
+})
+#endif /* CONFIG_DEBUG_ENTRY */
+
 #else
 #define annotate_reachable()
 #define annotate_unreachable()
 #define __annotate_jump_table
 #endif
 
+#ifndef instrumentation_begin
+#define instrumentation_begin()                do { } while(0)
+#define instrumentation_end()          do { } while(0)
+#endif
+
 #ifndef ASM_UNREACHABLE
 # define ASM_UNREACHABLE
 #endif

diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h
index e970f97a7fcb1c60f993f6e4eb13e6d947d85e60..5da257cbebf1c6d24a358a1d6c90009cba1b4cc1 100644 (file)
--- a/include/linux/compiler_types.h
+++ b/include/linux/compiler_types.h
@@ -118,6 +118,10 @@ struct ftrace_likely_data {
 #define notrace                        __attribute__((__no_instrument_function__))
 #endif
 
+/* Section for code which can't be instrumented at all */
+#define noinstr                                                                \
+       noinline notrace __attribute((__section__(".noinstr.text")))
+
 /*
  * it doesn't make sense on ARM (currently the only user of __naked)
  * to trace naked functions because then mcount is called without

diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c
index 5c3c50c5ec524aaf0e5cbadca2b0d7e8e285b59b..0053d4fea847d6a114e1c2c35daf14e6e818b23b 100644 (file)
--- a/scripts/mod/modpost.c
+++ b/scripts/mod/modpost.c
@@ -948,7 +948,7 @@ static void check_section(const char *modname, struct elf_info *elf,
 
 #define DATA_SECTIONS ".data", ".data.rel"
 #define TEXT_SECTIONS ".text", ".text.unlikely", ".sched.text", \
-               ".kprobes.text", ".cpuidle.text"
+               ".kprobes.text", ".cpuidle.text", ".noinstr.text"
 #define OTHER_TEXT_SECTIONS ".ref.text", ".head.text", ".spinlock.text", \
                ".fixup", ".entry.text", ".exception.text", ".text.*", \
                ".coldtext"