1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/static_call.h>
3 #include <linux/memory.h>
5 #include <asm/text-patching.h>
8 CALL
= 0, /* site call */
9 NOP
= 1, /* site cond-call */
10 JMP
= 2, /* tramp / site tail-call */
11 RET
= 3, /* tramp / site cond-tail-call */
15 * ud1 %esp, %ecx - a 3 byte #UD that is unique to trampolines, chosen such
16 * that there is no false-positive trampoline identification while also being a
19 static const u8 tramp_ud
[] = { 0x0f, 0xb9, 0xcc };
22 * cs cs cs xorl %eax, %eax - a single 5 byte instruction that clears %[er]ax
24 static const u8 xor5rax
[] = { 0x2e, 0x2e, 0x2e, 0x31, 0xc0 };
26 static const u8 retinsn
[] = { RET_INSN_OPCODE
, 0xcc, 0xcc, 0xcc, 0xcc };
28 static void __ref
__static_call_transform(void *insn
, enum insn_type type
, void *func
)
30 const void *emulate
= NULL
;
31 int size
= CALL_INSN_SIZE
;
36 code
= text_gen_insn(CALL_INSN_OPCODE
, insn
, func
);
37 if (func
== &__static_call_return0
) {
49 code
= text_gen_insn(JMP32_INSN_OPCODE
, insn
, func
);
53 if (cpu_feature_enabled(X86_FEATURE_RETHUNK
))
54 code
= text_gen_insn(JMP32_INSN_OPCODE
, insn
, &__x86_return_thunk
);
60 if (memcmp(insn
, code
, size
) == 0)
63 if (unlikely(system_state
== SYSTEM_BOOTING
))
64 return text_poke_early(insn
, code
, size
);
66 text_poke_bp(insn
, code
, size
, emulate
);
69 static void __static_call_validate(void *insn
, bool tail
)
71 u8 opcode
= *(u8
*)insn
;
74 if (opcode
== JMP32_INSN_OPCODE
||
75 opcode
== RET_INSN_OPCODE
)
78 if (opcode
== CALL_INSN_OPCODE
||
79 !memcmp(insn
, x86_nops
[5], 5) ||
80 !memcmp(insn
, xor5rax
, 5))
85 * If we ever trigger this, our text is corrupt, we'll probably not live long.
87 WARN_ONCE(1, "unexpected static_call insn opcode 0x%x at %pS\n", opcode
, insn
);
90 static inline enum insn_type
__sc_insn(bool null
, bool tail
)
93 * Encode the following table without branches:
96 * -----+-------+------
102 return 2*tail
+ null
;
105 void arch_static_call_transform(void *site
, void *tramp
, void *func
, bool tail
)
107 mutex_lock(&text_mutex
);
110 __static_call_validate(tramp
, true);
111 __static_call_transform(tramp
, __sc_insn(!func
, true), func
);
114 if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE
) && site
) {
115 __static_call_validate(site
, tail
);
116 __static_call_transform(site
, __sc_insn(!func
, tail
), func
);
119 mutex_unlock(&text_mutex
);
121 EXPORT_SYMBOL_GPL(arch_static_call_transform
);
123 #ifdef CONFIG_RETPOLINE
125 * This is called by apply_returns() to fix up static call trampolines,
126 * specifically ARCH_DEFINE_STATIC_CALL_NULL_TRAMP which is recorded as
127 * having a return trampoline.
129 * The problem is that static_call() is available before determining
130 * X86_FEATURE_RETHUNK and, by implication, running alternatives.
132 * This means that __static_call_transform() above can have overwritten the
133 * return trampoline and we now need to fix things up to be consistent.
135 bool __static_call_fixup(void *tramp
, u8 op
, void *dest
)
137 if (memcmp(tramp
+5, tramp_ud
, 3)) {
138 /* Not a trampoline site, not our problem. */
142 if (op
== RET_INSN_OPCODE
|| dest
== &__x86_return_thunk
)
143 __static_call_transform(tramp
, RET
, NULL
);