]>
Commit | Line | Data |
---|---|---|
1 | // SPDX-License-Identifier: GPL-2.0 | |
2 | /* | |
3 | * jump label x86 support | |
4 | * | |
5 | * Copyright (C) 2009 Jason Baron <jbaron@redhat.com> | |
6 | * | |
7 | */ | |
8 | #include <linux/jump_label.h> | |
9 | #include <linux/memory.h> | |
10 | #include <linux/uaccess.h> | |
11 | #include <linux/module.h> | |
12 | #include <linux/list.h> | |
13 | #include <linux/jhash.h> | |
14 | #include <linux/cpu.h> | |
15 | #include <asm/kprobes.h> | |
16 | #include <asm/alternative.h> | |
17 | #include <asm/text-patching.h> | |
18 | ||
19 | union jump_code_union { | |
20 | char code[JUMP_LABEL_NOP_SIZE]; | |
21 | struct { | |
22 | char jump; | |
23 | int offset; | |
24 | } __attribute__((packed)); | |
25 | }; | |
26 | ||
27 | static void bug_at(unsigned char *ip, int line) | |
28 | { | |
29 | /* | |
30 | * The location is not an op that we were expecting. | |
31 | * Something went wrong. Crash the box, as something could be | |
32 | * corrupting the kernel. | |
33 | */ | |
34 | pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph) %d\n", ip, ip, ip, line); | |
35 | BUG(); | |
36 | } | |
37 | ||
38 | static void __ref __jump_label_transform(struct jump_entry *entry, | |
39 | enum jump_label_type type, | |
40 | void *(*poker)(void *, const void *, size_t), | |
41 | int init) | |
42 | { | |
43 | union jump_code_union jmp; | |
44 | const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP }; | |
45 | const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5]; | |
46 | const void *expect, *code; | |
47 | int line; | |
48 | ||
49 | jmp.jump = 0xe9; | |
50 | jmp.offset = jump_entry_target(entry) - | |
51 | (jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE); | |
52 | ||
53 | if (early_boot_irqs_disabled) | |
54 | poker = text_poke_early; | |
55 | ||
56 | if (type == JUMP_LABEL_JMP) { | |
57 | if (init) { | |
58 | expect = default_nop; line = __LINE__; | |
59 | } else { | |
60 | expect = ideal_nop; line = __LINE__; | |
61 | } | |
62 | ||
63 | code = &jmp.code; | |
64 | } else { | |
65 | if (init) { | |
66 | expect = default_nop; line = __LINE__; | |
67 | } else { | |
68 | expect = &jmp.code; line = __LINE__; | |
69 | } | |
70 | ||
71 | code = ideal_nop; | |
72 | } | |
73 | ||
74 | if (memcmp((void *)jump_entry_code(entry), expect, JUMP_LABEL_NOP_SIZE)) | |
75 | bug_at((void *)jump_entry_code(entry), line); | |
76 | ||
77 | /* | |
78 | * Make text_poke_bp() a default fallback poker. | |
79 | * | |
80 | * At the time the change is being done, just ignore whether we | |
81 | * are doing nop -> jump or jump -> nop transition, and assume | |
82 | * always nop being the 'currently valid' instruction | |
83 | * | |
84 | */ | |
85 | if (poker) { | |
86 | (*poker)((void *)jump_entry_code(entry), code, | |
87 | JUMP_LABEL_NOP_SIZE); | |
88 | return; | |
89 | } | |
90 | ||
91 | text_poke_bp((void *)jump_entry_code(entry), code, JUMP_LABEL_NOP_SIZE, | |
92 | (void *)jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE); | |
93 | } | |
94 | ||
95 | void arch_jump_label_transform(struct jump_entry *entry, | |
96 | enum jump_label_type type) | |
97 | { | |
98 | mutex_lock(&text_mutex); | |
99 | __jump_label_transform(entry, type, NULL, 0); | |
100 | mutex_unlock(&text_mutex); | |
101 | } | |
102 | ||
103 | static enum { | |
104 | JL_STATE_START, | |
105 | JL_STATE_NO_UPDATE, | |
106 | JL_STATE_UPDATE, | |
107 | } jlstate __initdata_or_module = JL_STATE_START; | |
108 | ||
109 | __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry, | |
110 | enum jump_label_type type) | |
111 | { | |
112 | /* | |
113 | * This function is called at boot up and when modules are | |
114 | * first loaded. Check if the default nop, the one that is | |
115 | * inserted at compile time, is the ideal nop. If it is, then | |
116 | * we do not need to update the nop, and we can leave it as is. | |
117 | * If it is not, then we need to update the nop to the ideal nop. | |
118 | */ | |
119 | if (jlstate == JL_STATE_START) { | |
120 | const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP }; | |
121 | const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5]; | |
122 | ||
123 | if (memcmp(ideal_nop, default_nop, 5) != 0) | |
124 | jlstate = JL_STATE_UPDATE; | |
125 | else | |
126 | jlstate = JL_STATE_NO_UPDATE; | |
127 | } | |
128 | if (jlstate == JL_STATE_UPDATE) | |
129 | __jump_label_transform(entry, type, text_poke_early, 1); | |
130 | } |