arch/x86/include/asm/nospec-branch.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2
   3 #ifndef __NOSPEC_BRANCH_H__
   4 #define __NOSPEC_BRANCH_H__
   5
   6 #include <asm/alternative.h>
   7 #include <asm/alternative-asm.h>
   8 #include <asm/cpufeatures.h>
   9
  10 #ifdef __ASSEMBLY__
  11
  12 /*
  13  * This should be used immediately before a retpoline alternative.  It tells
  14  * objtool where the retpolines are so that it can make sense of the control
  15  * flow by just reading the original instruction(s) and ignoring the
  16  * alternatives.
  17  */
  18 .macro ANNOTATE_NOSPEC_ALTERNATIVE
  19         .Lannotate_\@:
  20         .pushsection .discard.nospec
  21         .long .Lannotate_\@ - .
  22         .popsection
  23 .endm
  24
  25 /*
  26  * These are the bare retpoline primitives for indirect jmp and call.
  27  * Do not use these directly; they only exist to make the ALTERNATIVE
  28  * invocation below less ugly.
  29  */
  30 .macro RETPOLINE_JMP reg:req
  31         call    .Ldo_rop_\@
  32 .Lspec_trap_\@:
  33         pause
  34         jmp     .Lspec_trap_\@
  35 .Ldo_rop_\@:
  36         mov     \reg, (%_ASM_SP)
  37         ret
  38 .endm
  39
  40 /*
  41  * This is a wrapper around RETPOLINE_JMP so the called function in reg
  42  * returns to the instruction after the macro.
  43  */
  44 .macro RETPOLINE_CALL reg:req
  45         jmp     .Ldo_call_\@
  46 .Ldo_retpoline_jmp_\@:
  47         RETPOLINE_JMP \reg
  48 .Ldo_call_\@:
  49         call    .Ldo_retpoline_jmp_\@
  50 .endm
  51
  52 /*
  53  * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
  54  * indirect jmp/call which may be susceptible to the Spectre variant 2
  55  * attack.
  56  */
  57 .macro JMP_NOSPEC reg:req
  58 #ifdef CONFIG_RETPOLINE
  59         ANNOTATE_NOSPEC_ALTERNATIVE
  60         ALTERNATIVE_2 __stringify(jmp *\reg),                           \
  61                 __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \
  62                 __stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
  63 #else
  64         jmp     *\reg
  65 #endif
  66 .endm
  67
  68 .macro CALL_NOSPEC reg:req
  69 #ifdef CONFIG_RETPOLINE
  70         ANNOTATE_NOSPEC_ALTERNATIVE
  71         ALTERNATIVE_2 __stringify(call *\reg),                          \
  72                 __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
  73                 __stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
  74 #else
  75         call    *\reg
  76 #endif
  77 .endm
  78
  79 #else /* __ASSEMBLY__ */
  80
  81 #define ANNOTATE_NOSPEC_ALTERNATIVE                             \
  82         "999:\n\t"                                              \
  83         ".pushsection .discard.nospec\n\t"                      \
  84         ".long 999b - .\n\t"                                    \
  85         ".popsection\n\t"
  86
  87 #if defined(CONFIG_X86_64) && defined(RETPOLINE)
  88
  89 /*
  90  * Since the inline asm uses the %V modifier which is only in newer GCC,
  91  * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
  92  */
  93 # define CALL_NOSPEC                                            \
  94         ANNOTATE_NOSPEC_ALTERNATIVE                             \
  95         ALTERNATIVE(                                            \
  96         "call *%[thunk_target]\n",                              \
  97         "call __x86_indirect_thunk_%V[thunk_target]\n",         \
  98         X86_FEATURE_RETPOLINE)
  99 # define THUNK_TARGET(addr) [thunk_target] "r" (addr)
 100
 101 #elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
 102 /*
 103  * For i386 we use the original ret-equivalent retpoline, because
 104  * otherwise we'll run out of registers. We don't care about CET
 105  * here, anyway.
 106  */
 107 # define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n",     \
 108         "       jmp    904f;\n"                                 \
 109         "       .align 16\n"                                    \
 110         "901:   call   903f;\n"                                 \
 111         "902:   pause;\n"                                       \
 112         "       jmp    902b;\n"                                 \
 113         "       .align 16\n"                                    \
 114         "903:   addl   $4, %%esp;\n"                            \
 115         "       pushl  %[thunk_target];\n"                      \
 116         "       ret;\n"                                         \
 117         "       .align 16\n"                                    \
 118         "904:   call   901b;\n",                                \
 119         X86_FEATURE_RETPOLINE)
 120
 121 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 122 #else /* No retpoline */
 123 # define CALL_NOSPEC "call *%[thunk_target]\n"
 124 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 125 #endif
 126
 127 #endif /* __ASSEMBLY__ */
 128 #endif /* __NOSPEC_BRANCH_H__ */