arch/x86/kernel/static_call.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/static_call.h>
   3 #include <linux/memory.h>
   4 #include <linux/bug.h>
   5 #include <asm/text-patching.h>
   6
   7 enum insn_type {
   8         CALL = 0, /* site call */
   9         NOP = 1,  /* site cond-call */
  10         JMP = 2,  /* tramp / site tail-call */
  11         RET = 3,  /* tramp / site cond-tail-call */
  12 };
  13
  14 /*
  15  * cs cs cs xorl %eax, %eax - a single 5 byte instruction that clears %[er]ax
  16  */
  17 static const u8 xor5rax[] = { 0x2e, 0x2e, 0x2e, 0x31, 0xc0 };
  18
  19 static void __ref __static_call_transform(void *insn, enum insn_type type, void *func)
  20 {
  21         const void *emulate = NULL;
  22         int size = CALL_INSN_SIZE;
  23         const void *code;
  24
  25         switch (type) {
  26         case CALL:
  27                 code = text_gen_insn(CALL_INSN_OPCODE, insn, func);
  28                 if (func == &__static_call_return0) {
  29                         emulate = code;
  30                         code = &xor5rax;
  31                 }
  32
  33                 break;
  34
  35         case NOP:
  36                 code = x86_nops[5];
  37                 break;
  38
  39         case JMP:
  40                 code = text_gen_insn(JMP32_INSN_OPCODE, insn, func);
  41                 break;
  42
  43         case RET:
  44                 code = text_gen_insn(RET_INSN_OPCODE, insn, func);
  45                 size = RET_INSN_SIZE;
  46                 break;
  47         }
  48
  49         if (memcmp(insn, code, size) == 0)
  50                 return;
  51
  52         if (unlikely(system_state == SYSTEM_BOOTING))
  53                 return text_poke_early(insn, code, size);
  54
  55         text_poke_bp(insn, code, size, emulate);
  56 }
  57
  58 static void __static_call_validate(void *insn, bool tail)
  59 {
  60         u8 opcode = *(u8 *)insn;
  61
  62         if (tail) {
  63                 if (opcode == JMP32_INSN_OPCODE ||
  64                     opcode == RET_INSN_OPCODE)
  65                         return;
  66         } else {
  67                 if (opcode == CALL_INSN_OPCODE ||
  68                     !memcmp(insn, x86_nops[5], 5) ||
  69                     !memcmp(insn, xor5rax, 5))
  70                         return;
  71         }
  72
  73         /*
  74          * If we ever trigger this, our text is corrupt, we'll probably not live long.
  75          */
  76         WARN_ONCE(1, "unexpected static_call insn opcode 0x%x at %pS\n", opcode, insn);
  77 }
  78
  79 static inline enum insn_type __sc_insn(bool null, bool tail)
  80 {
  81         /*
  82          * Encode the following table without branches:
  83          *
  84          *      tail    null    insn
  85          *      -----+-------+------
  86          *        0  |   0   |  CALL
  87          *        0  |   1   |  NOP
  88          *        1  |   0   |  JMP
  89          *        1  |   1   |  RET
  90          */
  91         return 2*tail + null;
  92 }
  93
  94 void arch_static_call_transform(void *site, void *tramp, void *func, bool tail)
  95 {
  96         mutex_lock(&text_mutex);
  97
  98         if (tramp) {
  99                 __static_call_validate(tramp, true);
 100                 __static_call_transform(tramp, __sc_insn(!func, true), func);
 101         }
 102
 103         if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE) && site) {
 104                 __static_call_validate(site, tail);
 105                 __static_call_transform(site, __sc_insn(!func, tail), func);
 106         }
 107
 108         mutex_unlock(&text_mutex);
 109 }
 110 EXPORT_SYMBOL_GPL(arch_static_call_transform);