]>
Commit | Line | Data |
---|---|---|
c349cdca JP |
1 | /* |
2 | * patch.c - livepatch patching functions | |
3 | * | |
4 | * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com> | |
5 | * Copyright (C) 2014 SUSE | |
6 | * Copyright (C) 2015 Josh Poimboeuf <jpoimboe@redhat.com> | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU General Public License | |
10 | * as published by the Free Software Foundation; either version 2 | |
11 | * of the License, or (at your option) any later version. | |
12 | * | |
13 | * This program is distributed in the hope that it will be useful, | |
14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
16 | * GNU General Public License for more details. | |
17 | * | |
18 | * You should have received a copy of the GNU General Public License | |
19 | * along with this program; if not, see <http://www.gnu.org/licenses/>. | |
20 | */ | |
21 | ||
22 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
23 | ||
24 | #include <linux/livepatch.h> | |
25 | #include <linux/list.h> | |
26 | #include <linux/ftrace.h> | |
27 | #include <linux/rculist.h> | |
28 | #include <linux/slab.h> | |
29 | #include <linux/bug.h> | |
30 | #include <linux/printk.h> | |
31 | #include "patch.h" | |
d83a7cb3 | 32 | #include "transition.h" |
c349cdca JP |
33 | |
34 | static LIST_HEAD(klp_ops); | |
35 | ||
36 | struct klp_ops *klp_find_ops(unsigned long old_addr) | |
37 | { | |
38 | struct klp_ops *ops; | |
39 | struct klp_func *func; | |
40 | ||
41 | list_for_each_entry(ops, &klp_ops, node) { | |
42 | func = list_first_entry(&ops->func_stack, struct klp_func, | |
43 | stack_node); | |
44 | if (func->old_addr == old_addr) | |
45 | return ops; | |
46 | } | |
47 | ||
48 | return NULL; | |
49 | } | |
50 | ||
51 | static void notrace klp_ftrace_handler(unsigned long ip, | |
52 | unsigned long parent_ip, | |
53 | struct ftrace_ops *fops, | |
54 | struct pt_regs *regs) | |
55 | { | |
56 | struct klp_ops *ops; | |
57 | struct klp_func *func; | |
d83a7cb3 | 58 | int patch_state; |
c349cdca JP |
59 | |
60 | ops = container_of(fops, struct klp_ops, fops); | |
61 | ||
842c0884 PM |
62 | /* |
63 | * A variant of synchronize_sched() is used to allow patching functions | |
64 | * where RCU is not watching, see klp_synchronize_transition(). | |
65 | */ | |
66 | preempt_disable_notrace(); | |
d83a7cb3 | 67 | |
c349cdca JP |
68 | func = list_first_or_null_rcu(&ops->func_stack, struct klp_func, |
69 | stack_node); | |
d83a7cb3 JP |
70 | |
71 | /* | |
72 | * func should never be NULL because preemption should be disabled here | |
73 | * and unregister_ftrace_function() does the equivalent of a | |
74 | * synchronize_sched() before the func_stack removal. | |
75 | */ | |
c349cdca JP |
76 | if (WARN_ON_ONCE(!func)) |
77 | goto unlock; | |
78 | ||
d83a7cb3 JP |
79 | /* |
80 | * In the enable path, enforce the order of the ops->func_stack and | |
81 | * func->transition reads. The corresponding write barrier is in | |
82 | * __klp_enable_patch(). | |
83 | * | |
84 | * (Note that this barrier technically isn't needed in the disable | |
85 | * path. In the rare case where klp_update_patch_state() runs before | |
86 | * this handler, its TIF_PATCH_PENDING read and this func->transition | |
87 | * read need to be ordered. But klp_update_patch_state() already | |
88 | * enforces that.) | |
89 | */ | |
90 | smp_rmb(); | |
91 | ||
92 | if (unlikely(func->transition)) { | |
93 | ||
94 | /* | |
95 | * Enforce the order of the func->transition and | |
96 | * current->patch_state reads. Otherwise we could read an | |
97 | * out-of-date task state and pick the wrong function. The | |
98 | * corresponding write barrier is in klp_init_transition(). | |
99 | */ | |
100 | smp_rmb(); | |
101 | ||
102 | patch_state = current->patch_state; | |
103 | ||
104 | WARN_ON_ONCE(patch_state == KLP_UNDEFINED); | |
105 | ||
106 | if (patch_state == KLP_UNPATCHED) { | |
107 | /* | |
108 | * Use the previously patched version of the function. | |
109 | * If no previous patches exist, continue with the | |
110 | * original function. | |
111 | */ | |
112 | func = list_entry_rcu(func->stack_node.next, | |
113 | struct klp_func, stack_node); | |
114 | ||
115 | if (&func->stack_node == &ops->func_stack) | |
116 | goto unlock; | |
117 | } | |
118 | } | |
119 | ||
c349cdca JP |
120 | klp_arch_set_pc(regs, (unsigned long)func->new_func); |
121 | unlock: | |
842c0884 | 122 | preempt_enable_notrace(); |
c349cdca JP |
123 | } |
124 | ||
125 | /* | |
126 | * Convert a function address into the appropriate ftrace location. | |
127 | * | |
128 | * Usually this is just the address of the function, but on some architectures | |
129 | * it's more complicated so allow them to provide a custom behaviour. | |
130 | */ | |
131 | #ifndef klp_get_ftrace_location | |
132 | static unsigned long klp_get_ftrace_location(unsigned long faddr) | |
133 | { | |
134 | return faddr; | |
135 | } | |
136 | #endif | |
137 | ||
138 | static void klp_unpatch_func(struct klp_func *func) | |
139 | { | |
140 | struct klp_ops *ops; | |
141 | ||
142 | if (WARN_ON(!func->patched)) | |
143 | return; | |
144 | if (WARN_ON(!func->old_addr)) | |
145 | return; | |
146 | ||
147 | ops = klp_find_ops(func->old_addr); | |
148 | if (WARN_ON(!ops)) | |
149 | return; | |
150 | ||
151 | if (list_is_singular(&ops->func_stack)) { | |
152 | unsigned long ftrace_loc; | |
153 | ||
154 | ftrace_loc = klp_get_ftrace_location(func->old_addr); | |
155 | if (WARN_ON(!ftrace_loc)) | |
156 | return; | |
157 | ||
158 | WARN_ON(unregister_ftrace_function(&ops->fops)); | |
159 | WARN_ON(ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0)); | |
160 | ||
161 | list_del_rcu(&func->stack_node); | |
162 | list_del(&ops->node); | |
163 | kfree(ops); | |
164 | } else { | |
165 | list_del_rcu(&func->stack_node); | |
166 | } | |
167 | ||
168 | func->patched = false; | |
169 | } | |
170 | ||
171 | static int klp_patch_func(struct klp_func *func) | |
172 | { | |
173 | struct klp_ops *ops; | |
174 | int ret; | |
175 | ||
176 | if (WARN_ON(!func->old_addr)) | |
177 | return -EINVAL; | |
178 | ||
179 | if (WARN_ON(func->patched)) | |
180 | return -EINVAL; | |
181 | ||
182 | ops = klp_find_ops(func->old_addr); | |
183 | if (!ops) { | |
184 | unsigned long ftrace_loc; | |
185 | ||
186 | ftrace_loc = klp_get_ftrace_location(func->old_addr); | |
187 | if (!ftrace_loc) { | |
188 | pr_err("failed to find location for function '%s'\n", | |
189 | func->old_name); | |
190 | return -EINVAL; | |
191 | } | |
192 | ||
193 | ops = kzalloc(sizeof(*ops), GFP_KERNEL); | |
194 | if (!ops) | |
195 | return -ENOMEM; | |
196 | ||
197 | ops->fops.func = klp_ftrace_handler; | |
198 | ops->fops.flags = FTRACE_OPS_FL_SAVE_REGS | | |
199 | FTRACE_OPS_FL_DYNAMIC | | |
200 | FTRACE_OPS_FL_IPMODIFY; | |
201 | ||
202 | list_add(&ops->node, &klp_ops); | |
203 | ||
204 | INIT_LIST_HEAD(&ops->func_stack); | |
205 | list_add_rcu(&func->stack_node, &ops->func_stack); | |
206 | ||
207 | ret = ftrace_set_filter_ip(&ops->fops, ftrace_loc, 0, 0); | |
208 | if (ret) { | |
209 | pr_err("failed to set ftrace filter for function '%s' (%d)\n", | |
210 | func->old_name, ret); | |
211 | goto err; | |
212 | } | |
213 | ||
214 | ret = register_ftrace_function(&ops->fops); | |
215 | if (ret) { | |
216 | pr_err("failed to register ftrace handler for function '%s' (%d)\n", | |
217 | func->old_name, ret); | |
218 | ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0); | |
219 | goto err; | |
220 | } | |
221 | ||
222 | ||
223 | } else { | |
224 | list_add_rcu(&func->stack_node, &ops->func_stack); | |
225 | } | |
226 | ||
227 | func->patched = true; | |
228 | ||
229 | return 0; | |
230 | ||
231 | err: | |
232 | list_del_rcu(&func->stack_node); | |
233 | list_del(&ops->node); | |
234 | kfree(ops); | |
235 | return ret; | |
236 | } | |
237 | ||
238 | void klp_unpatch_object(struct klp_object *obj) | |
239 | { | |
240 | struct klp_func *func; | |
241 | ||
242 | klp_for_each_func(obj, func) | |
243 | if (func->patched) | |
244 | klp_unpatch_func(func); | |
245 | ||
246 | obj->patched = false; | |
247 | } | |
248 | ||
249 | int klp_patch_object(struct klp_object *obj) | |
250 | { | |
251 | struct klp_func *func; | |
252 | int ret; | |
253 | ||
254 | if (WARN_ON(obj->patched)) | |
255 | return -EINVAL; | |
256 | ||
257 | klp_for_each_func(obj, func) { | |
258 | ret = klp_patch_func(func); | |
259 | if (ret) { | |
260 | klp_unpatch_object(obj); | |
261 | return ret; | |
262 | } | |
263 | } | |
264 | obj->patched = true; | |
265 | ||
266 | return 0; | |
267 | } | |
d83a7cb3 JP |
268 | |
269 | void klp_unpatch_objects(struct klp_patch *patch) | |
270 | { | |
271 | struct klp_object *obj; | |
272 | ||
273 | klp_for_each_object(patch, obj) | |
274 | if (obj->patched) | |
275 | klp_unpatch_object(obj); | |
276 | } |