]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
d9f5ab7b JB |
2 | /* |
3 | * jump label x86 support | |
4 | * | |
5 | * Copyright (C) 2009 Jason Baron <jbaron@redhat.com> | |
6 | * | |
7 | */ | |
8 | #include <linux/jump_label.h> | |
9 | #include <linux/memory.h> | |
10 | #include <linux/uaccess.h> | |
11 | #include <linux/module.h> | |
12 | #include <linux/list.h> | |
13 | #include <linux/jhash.h> | |
14 | #include <linux/cpu.h> | |
15 | #include <asm/kprobes.h> | |
16 | #include <asm/alternative.h> | |
35de5b06 | 17 | #include <asm/text-patching.h> |
d9f5ab7b | 18 | |
d9f5ab7b JB |
19 | union jump_code_union { |
20 | char code[JUMP_LABEL_NOP_SIZE]; | |
21 | struct { | |
22 | char jump; | |
23 | int offset; | |
24 | } __attribute__((packed)); | |
25 | }; | |
26 | ||
fb40d7a8 SR |
27 | static void bug_at(unsigned char *ip, int line) |
28 | { | |
29 | /* | |
30 | * The location is not an op that we were expecting. | |
31 | * Something went wrong. Crash the box, as something could be | |
32 | * corrupting the kernel. | |
33 | */ | |
6e03f66c | 34 | pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph) %d\n", ip, ip, ip, line); |
fb40d7a8 SR |
35 | BUG(); |
36 | } | |
37 | ||
6fffacb3 PT |
38 | static void __ref __jump_label_transform(struct jump_entry *entry, |
39 | enum jump_label_type type, | |
40 | void *(*poker)(void *, const void *, size_t), | |
41 | int init) | |
d9f5ab7b | 42 | { |
9fc0f798 | 43 | union jump_code_union jmp; |
a8fab074 | 44 | const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP }; |
9c85f3bd | 45 | const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5]; |
9fc0f798 AB |
46 | const void *expect, *code; |
47 | int line; | |
48 | ||
49 | jmp.jump = 0xe9; | |
50 | jmp.offset = jump_entry_target(entry) - | |
51 | (jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE); | |
d9f5ab7b | 52 | |
6fffacb3 PT |
53 | if (early_boot_irqs_disabled) |
54 | poker = text_poke_early; | |
55 | ||
76b235c6 | 56 | if (type == JUMP_LABEL_JMP) { |
a8fab074 | 57 | if (init) { |
9fc0f798 | 58 | expect = default_nop; line = __LINE__; |
a8fab074 | 59 | } else { |
9fc0f798 | 60 | expect = ideal_nop; line = __LINE__; |
a8fab074 | 61 | } |
9c85f3bd | 62 | |
9fc0f798 | 63 | code = &jmp.code; |
9c85f3bd | 64 | } else { |
9c85f3bd | 65 | if (init) { |
9fc0f798 | 66 | expect = default_nop; line = __LINE__; |
9c85f3bd | 67 | } else { |
9fc0f798 | 68 | expect = &jmp.code; line = __LINE__; |
9c85f3bd | 69 | } |
9fc0f798 AB |
70 | |
71 | code = ideal_nop; | |
9c85f3bd | 72 | } |
e71a5be1 | 73 | |
9fc0f798 AB |
74 | if (memcmp((void *)jump_entry_code(entry), expect, JUMP_LABEL_NOP_SIZE)) |
75 | bug_at((void *)jump_entry_code(entry), line); | |
76 | ||
51b2c07b JK |
77 | /* |
78 | * Make text_poke_bp() a default fallback poker. | |
79 | * | |
80 | * At the time the change is being done, just ignore whether we | |
81 | * are doing nop -> jump or jump -> nop transition, and assume | |
82 | * always nop being the 'currently valid' instruction | |
83 | * | |
84 | */ | |
9fc0f798 AB |
85 | if (poker) { |
86 | (*poker)((void *)jump_entry_code(entry), code, | |
87 | JUMP_LABEL_NOP_SIZE); | |
88 | return; | |
89 | } | |
90 | ||
91 | text_poke_bp((void *)jump_entry_code(entry), code, JUMP_LABEL_NOP_SIZE, | |
92 | (void *)jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE); | |
e71a5be1 JF |
93 | } |
94 | ||
95 | void arch_jump_label_transform(struct jump_entry *entry, | |
96 | enum jump_label_type type) | |
97 | { | |
d9f5ab7b | 98 | mutex_lock(&text_mutex); |
442e0973 | 99 | __jump_label_transform(entry, type, NULL, 0); |
d9f5ab7b | 100 | mutex_unlock(&text_mutex); |
d9f5ab7b JB |
101 | } |
102 | ||
11570da1 SR |
103 | static enum { |
104 | JL_STATE_START, | |
105 | JL_STATE_NO_UPDATE, | |
106 | JL_STATE_UPDATE, | |
107 | } jlstate __initdata_or_module = JL_STATE_START; | |
108 | ||
9cdbe1cb | 109 | __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry, |
e71a5be1 JF |
110 | enum jump_label_type type) |
111 | { | |
11570da1 SR |
112 | /* |
113 | * This function is called at boot up and when modules are | |
114 | * first loaded. Check if the default nop, the one that is | |
115 | * inserted at compile time, is the ideal nop. If it is, then | |
116 | * we do not need to update the nop, and we can leave it as is. | |
117 | * If it is not, then we need to update the nop to the ideal nop. | |
118 | */ | |
119 | if (jlstate == JL_STATE_START) { | |
120 | const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP }; | |
121 | const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5]; | |
122 | ||
123 | if (memcmp(ideal_nop, default_nop, 5) != 0) | |
124 | jlstate = JL_STATE_UPDATE; | |
125 | else | |
126 | jlstate = JL_STATE_NO_UPDATE; | |
127 | } | |
128 | if (jlstate == JL_STATE_UPDATE) | |
9c85f3bd | 129 | __jump_label_transform(entry, type, text_poke_early, 1); |
e71a5be1 | 130 | } |