]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
d9f5ab7b JB |
2 | /* |
3 | * jump label x86 support | |
4 | * | |
5 | * Copyright (C) 2009 Jason Baron <jbaron@redhat.com> | |
6 | * | |
7 | */ | |
8 | #include <linux/jump_label.h> | |
9 | #include <linux/memory.h> | |
10 | #include <linux/uaccess.h> | |
11 | #include <linux/module.h> | |
12 | #include <linux/list.h> | |
13 | #include <linux/jhash.h> | |
14 | #include <linux/cpu.h> | |
15 | #include <asm/kprobes.h> | |
16 | #include <asm/alternative.h> | |
35de5b06 | 17 | #include <asm/text-patching.h> |
d9f5ab7b JB |
18 | |
19 | #ifdef HAVE_JUMP_LABEL | |
20 | ||
21 | union jump_code_union { | |
22 | char code[JUMP_LABEL_NOP_SIZE]; | |
23 | struct { | |
24 | char jump; | |
25 | int offset; | |
26 | } __attribute__((packed)); | |
27 | }; | |
28 | ||
fb40d7a8 SR |
29 | static void bug_at(unsigned char *ip, int line) |
30 | { | |
31 | /* | |
32 | * The location is not an op that we were expecting. | |
33 | * Something went wrong. Crash the box, as something could be | |
34 | * corrupting the kernel. | |
35 | */ | |
6e03f66c | 36 | pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph) %d\n", ip, ip, ip, line); |
fb40d7a8 SR |
37 | BUG(); |
38 | } | |
39 | ||
e71a5be1 JF |
40 | static void __jump_label_transform(struct jump_entry *entry, |
41 | enum jump_label_type type, | |
9c85f3bd SR |
42 | void *(*poker)(void *, const void *, size_t), |
43 | int init) | |
d9f5ab7b JB |
44 | { |
45 | union jump_code_union code; | |
a8fab074 | 46 | const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP }; |
9c85f3bd | 47 | const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5]; |
d9f5ab7b | 48 | |
76b235c6 | 49 | if (type == JUMP_LABEL_JMP) { |
a8fab074 HFS |
50 | if (init) { |
51 | /* | |
52 | * Jump label is enabled for the first time. | |
53 | * So we expect a default_nop... | |
54 | */ | |
55 | if (unlikely(memcmp((void *)entry->code, default_nop, 5) | |
56 | != 0)) | |
57 | bug_at((void *)entry->code, __LINE__); | |
58 | } else { | |
59 | /* | |
60 | * ...otherwise expect an ideal_nop. Otherwise | |
61 | * something went horribly wrong. | |
62 | */ | |
63 | if (unlikely(memcmp((void *)entry->code, ideal_nop, 5) | |
64 | != 0)) | |
65 | bug_at((void *)entry->code, __LINE__); | |
66 | } | |
9c85f3bd | 67 | |
d9f5ab7b JB |
68 | code.jump = 0xe9; |
69 | code.offset = entry->target - | |
70 | (entry->code + JUMP_LABEL_NOP_SIZE); | |
9c85f3bd SR |
71 | } else { |
72 | /* | |
73 | * We are disabling this jump label. If it is not what | |
74 | * we think it is, then something must have gone wrong. | |
75 | * If this is the first initialization call, then we | |
76 | * are converting the default nop to the ideal nop. | |
77 | */ | |
78 | if (init) { | |
fb40d7a8 SR |
79 | if (unlikely(memcmp((void *)entry->code, default_nop, 5) != 0)) |
80 | bug_at((void *)entry->code, __LINE__); | |
9c85f3bd SR |
81 | } else { |
82 | code.jump = 0xe9; | |
83 | code.offset = entry->target - | |
84 | (entry->code + JUMP_LABEL_NOP_SIZE); | |
fb40d7a8 SR |
85 | if (unlikely(memcmp((void *)entry->code, &code, 5) != 0)) |
86 | bug_at((void *)entry->code, __LINE__); | |
9c85f3bd | 87 | } |
dc326fca | 88 | memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE); |
9c85f3bd | 89 | } |
e71a5be1 | 90 | |
51b2c07b JK |
91 | /* |
92 | * Make text_poke_bp() a default fallback poker. | |
93 | * | |
94 | * At the time the change is being done, just ignore whether we | |
95 | * are doing nop -> jump or jump -> nop transition, and assume | |
96 | * always nop being the 'currently valid' instruction | |
97 | * | |
98 | */ | |
99 | if (poker) | |
100 | (*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE); | |
101 | else | |
102 | text_poke_bp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE, | |
103 | (void *)entry->code + JUMP_LABEL_NOP_SIZE); | |
e71a5be1 JF |
104 | } |
105 | ||
106 | void arch_jump_label_transform(struct jump_entry *entry, | |
107 | enum jump_label_type type) | |
108 | { | |
d9f5ab7b | 109 | mutex_lock(&text_mutex); |
442e0973 | 110 | __jump_label_transform(entry, type, NULL, 0); |
d9f5ab7b | 111 | mutex_unlock(&text_mutex); |
d9f5ab7b JB |
112 | } |
113 | ||
11570da1 SR |
114 | static enum { |
115 | JL_STATE_START, | |
116 | JL_STATE_NO_UPDATE, | |
117 | JL_STATE_UPDATE, | |
118 | } jlstate __initdata_or_module = JL_STATE_START; | |
119 | ||
9cdbe1cb | 120 | __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry, |
e71a5be1 JF |
121 | enum jump_label_type type) |
122 | { | |
11570da1 SR |
123 | /* |
124 | * This function is called at boot up and when modules are | |
125 | * first loaded. Check if the default nop, the one that is | |
126 | * inserted at compile time, is the ideal nop. If it is, then | |
127 | * we do not need to update the nop, and we can leave it as is. | |
128 | * If it is not, then we need to update the nop to the ideal nop. | |
129 | */ | |
130 | if (jlstate == JL_STATE_START) { | |
131 | const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP }; | |
132 | const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5]; | |
133 | ||
134 | if (memcmp(ideal_nop, default_nop, 5) != 0) | |
135 | jlstate = JL_STATE_UPDATE; | |
136 | else | |
137 | jlstate = JL_STATE_NO_UPDATE; | |
138 | } | |
139 | if (jlstate == JL_STATE_UPDATE) | |
9c85f3bd | 140 | __jump_label_transform(entry, type, text_poke_early, 1); |
e71a5be1 JF |
141 | } |
142 | ||
d9f5ab7b | 143 | #endif |