]>
Commit | Line | Data |
---|---|---|
a17ae4c3 | 1 | // SPDX-License-Identifier: GPL-2.0+ |
4ba069b8 MG |
2 | /* |
3 | * Kernel Probes (KProbes) | |
4 | * | |
a53c8fab | 5 | * Copyright IBM Corp. 2002, 2006 |
4ba069b8 MG |
6 | * |
7 | * s390 port, used ppc64 as template. Mike Grundy <grundym@us.ibm.com> | |
8 | */ | |
9 | ||
4ba069b8 MG |
10 | #include <linux/kprobes.h> |
11 | #include <linux/ptrace.h> | |
12 | #include <linux/preempt.h> | |
13 | #include <linux/stop_machine.h> | |
1eeb66a1 | 14 | #include <linux/kdebug.h> |
a2b53673 | 15 | #include <linux/uaccess.h> |
dcc096c5 | 16 | #include <linux/extable.h> |
4ba069b8 | 17 | #include <linux/module.h> |
5a0e3ad6 | 18 | #include <linux/slab.h> |
adb45839 | 19 | #include <linux/hardirq.h> |
c933146a | 20 | #include <linux/ftrace.h> |
e6c7c630 | 21 | #include <asm/set_memory.h> |
a882b3b0 | 22 | #include <asm/sections.h> |
7c0f6ba6 | 23 | #include <linux/uaccess.h> |
a882b3b0 | 24 | #include <asm/dis.h> |
4ba069b8 | 25 | |
4a188635 | 26 | DEFINE_PER_CPU(struct kprobe *, current_kprobe); |
4ba069b8 MG |
27 | DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); |
28 | ||
4a188635 | 29 | struct kretprobe_blackpoint kretprobe_blacklist[] = { }; |
f438d914 | 30 | |
63c40436 HC |
31 | DEFINE_INSN_CACHE_OPS(dmainsn); |
32 | ||
33 | static void *alloc_dmainsn_page(void) | |
34 | { | |
57d7f939 MS |
35 | void *page; |
36 | ||
37 | page = (void *) __get_free_page(GFP_KERNEL | GFP_DMA); | |
38 | if (page) | |
39 | set_memory_x((unsigned long) page, 1); | |
40 | return page; | |
63c40436 HC |
41 | } |
42 | ||
43 | static void free_dmainsn_page(void *page) | |
44 | { | |
57d7f939 | 45 | set_memory_nx((unsigned long) page, 1); |
63c40436 HC |
46 | free_page((unsigned long)page); |
47 | } | |
48 | ||
49 | struct kprobe_insn_cache kprobe_dmainsn_slots = { | |
50 | .mutex = __MUTEX_INITIALIZER(kprobe_dmainsn_slots.mutex), | |
51 | .alloc = alloc_dmainsn_page, | |
52 | .free = free_dmainsn_page, | |
53 | .pages = LIST_HEAD_INIT(kprobe_dmainsn_slots.pages), | |
54 | .insn_size = MAX_INSN_SIZE, | |
55 | }; | |
56 | ||
7a5388de | 57 | static void copy_instruction(struct kprobe *p) |
63c40436 | 58 | { |
c933146a | 59 | unsigned long ip = (unsigned long) p->addr; |
63c40436 HC |
60 | s64 disp, new_disp; |
61 | u64 addr, new_addr; | |
62 | ||
c933146a HC |
63 | if (ftrace_location(ip) == ip) { |
64 | /* | |
65 | * If kprobes patches the instruction that is morphed by | |
66 | * ftrace make sure that kprobes always sees the branch | |
e6d60b36 HC |
67 | * "jg .+24" that skips the mcount block or the "brcl 0,0" |
68 | * in case of hotpatch. | |
c933146a HC |
69 | */ |
70 | ftrace_generate_nop_insn((struct ftrace_insn *)p->ainsn.insn); | |
71 | p->ainsn.is_ftrace_insn = 1; | |
72 | } else | |
ed7d56e1 | 73 | memcpy(p->ainsn.insn, p->addr, insn_length(*p->addr >> 8)); |
c933146a | 74 | p->opcode = p->ainsn.insn[0]; |
975fab17 | 75 | if (!probe_is_insn_relative_long(p->ainsn.insn)) |
63c40436 HC |
76 | return; |
77 | /* | |
78 | * For pc-relative instructions in RIL-b or RIL-c format patch the | |
79 | * RI2 displacement field. We have already made sure that the insn | |
80 | * slot for the patched instruction is within the same 2GB area | |
81 | * as the original instruction (either kernel image or module area). | |
82 | * Therefore the new displacement will always fit. | |
83 | */ | |
84 | disp = *(s32 *)&p->ainsn.insn[1]; | |
85 | addr = (u64)(unsigned long)p->addr; | |
86 | new_addr = (u64)(unsigned long)p->ainsn.insn; | |
87 | new_disp = ((addr + (disp * 2)) - new_addr) / 2; | |
88 | *(s32 *)&p->ainsn.insn[1] = new_disp; | |
89 | } | |
7a5388de | 90 | NOKPROBE_SYMBOL(copy_instruction); |
63c40436 HC |
91 | |
92 | static inline int is_kernel_addr(void *addr) | |
93 | { | |
94 | return addr < (void *)_end; | |
95 | } | |
96 | ||
7a5388de | 97 | static int s390_get_insn_slot(struct kprobe *p) |
63c40436 HC |
98 | { |
99 | /* | |
100 | * Get an insn slot that is within the same 2GB area like the original | |
101 | * instruction. That way instructions with a 32bit signed displacement | |
102 | * field can be patched and executed within the insn slot. | |
103 | */ | |
104 | p->ainsn.insn = NULL; | |
105 | if (is_kernel_addr(p->addr)) | |
106 | p->ainsn.insn = get_dmainsn_slot(); | |
fcd05b50 | 107 | else if (is_module_addr(p->addr)) |
63c40436 HC |
108 | p->ainsn.insn = get_insn_slot(); |
109 | return p->ainsn.insn ? 0 : -ENOMEM; | |
110 | } | |
7a5388de | 111 | NOKPROBE_SYMBOL(s390_get_insn_slot); |
63c40436 | 112 | |
7a5388de | 113 | static void s390_free_insn_slot(struct kprobe *p) |
63c40436 HC |
114 | { |
115 | if (!p->ainsn.insn) | |
116 | return; | |
117 | if (is_kernel_addr(p->addr)) | |
118 | free_dmainsn_slot(p->ainsn.insn, 0); | |
119 | else | |
120 | free_insn_slot(p->ainsn.insn, 0); | |
121 | p->ainsn.insn = NULL; | |
122 | } | |
7a5388de | 123 | NOKPROBE_SYMBOL(s390_free_insn_slot); |
63c40436 | 124 | |
7a5388de | 125 | int arch_prepare_kprobe(struct kprobe *p) |
ba640a59 MS |
126 | { |
127 | if ((unsigned long) p->addr & 0x01) | |
128 | return -EINVAL; | |
ba640a59 | 129 | /* Make sure the probe isn't going on a difficult instruction */ |
975fab17 | 130 | if (probe_is_prohibited_opcode(p->addr)) |
ba640a59 | 131 | return -EINVAL; |
63c40436 HC |
132 | if (s390_get_insn_slot(p)) |
133 | return -ENOMEM; | |
63c40436 | 134 | copy_instruction(p); |
ba640a59 | 135 | return 0; |
4ba069b8 | 136 | } |
7a5388de | 137 | NOKPROBE_SYMBOL(arch_prepare_kprobe); |
4ba069b8 | 138 | |
c933146a HC |
139 | int arch_check_ftrace_location(struct kprobe *p) |
140 | { | |
141 | return 0; | |
142 | } | |
143 | ||
144 | struct swap_insn_args { | |
145 | struct kprobe *p; | |
146 | unsigned int arm_kprobe : 1; | |
5a8b589f MS |
147 | }; |
148 | ||
7a5388de | 149 | static int swap_instruction(void *data) |
4ba069b8 | 150 | { |
c933146a HC |
151 | struct swap_insn_args *args = data; |
152 | struct ftrace_insn new_insn, *insn; | |
153 | struct kprobe *p = args->p; | |
154 | size_t len; | |
155 | ||
156 | new_insn.opc = args->arm_kprobe ? BREAKPOINT_INSTRUCTION : p->opcode; | |
157 | len = sizeof(new_insn.opc); | |
158 | if (!p->ainsn.is_ftrace_insn) | |
159 | goto skip_ftrace; | |
160 | len = sizeof(new_insn); | |
161 | insn = (struct ftrace_insn *) p->addr; | |
162 | if (args->arm_kprobe) { | |
163 | if (is_ftrace_nop(insn)) | |
164 | new_insn.disp = KPROBE_ON_FTRACE_NOP; | |
165 | else | |
166 | new_insn.disp = KPROBE_ON_FTRACE_CALL; | |
167 | } else { | |
168 | ftrace_generate_call_insn(&new_insn, (unsigned long)p->addr); | |
169 | if (insn->disp == KPROBE_ON_FTRACE_NOP) | |
170 | ftrace_generate_nop_insn(&new_insn); | |
171 | } | |
172 | skip_ftrace: | |
8a5d8473 | 173 | s390_kernel_write(p->addr, &new_insn, len); |
5a8b589f | 174 | return 0; |
4ba069b8 | 175 | } |
7a5388de | 176 | NOKPROBE_SYMBOL(swap_instruction); |
4ba069b8 | 177 | |
7a5388de | 178 | void arch_arm_kprobe(struct kprobe *p) |
4ba069b8 | 179 | { |
c933146a | 180 | struct swap_insn_args args = {.p = p, .arm_kprobe = 1}; |
4ba069b8 | 181 | |
5d5dbc4e | 182 | stop_machine_cpuslocked(swap_instruction, &args, NULL); |
4ba069b8 | 183 | } |
7a5388de | 184 | NOKPROBE_SYMBOL(arch_arm_kprobe); |
4ba069b8 | 185 | |
7a5388de | 186 | void arch_disarm_kprobe(struct kprobe *p) |
4ba069b8 | 187 | { |
c933146a | 188 | struct swap_insn_args args = {.p = p, .arm_kprobe = 0}; |
4ba069b8 | 189 | |
5d5dbc4e | 190 | stop_machine_cpuslocked(swap_instruction, &args, NULL); |
4ba069b8 | 191 | } |
7a5388de | 192 | NOKPROBE_SYMBOL(arch_disarm_kprobe); |
4ba069b8 | 193 | |
7a5388de | 194 | void arch_remove_kprobe(struct kprobe *p) |
4ba069b8 | 195 | { |
63c40436 | 196 | s390_free_insn_slot(p); |
4ba069b8 | 197 | } |
7a5388de | 198 | NOKPROBE_SYMBOL(arch_remove_kprobe); |
4ba069b8 | 199 | |
7a5388de HC |
200 | static void enable_singlestep(struct kprobe_ctlblk *kcb, |
201 | struct pt_regs *regs, | |
202 | unsigned long ip) | |
4ba069b8 | 203 | { |
5e9a2692 | 204 | struct per_regs per_kprobe; |
4ba069b8 | 205 | |
5e9a2692 MS |
206 | /* Set up the PER control registers %cr9-%cr11 */ |
207 | per_kprobe.control = PER_EVENT_IFETCH; | |
208 | per_kprobe.start = ip; | |
209 | per_kprobe.end = ip; | |
4ba069b8 | 210 | |
fc0a1fea MS |
211 | /* Save control regs and psw mask */ |
212 | __ctl_store(kcb->kprobe_saved_ctl, 9, 11); | |
213 | kcb->kprobe_saved_imask = regs->psw.mask & | |
214 | (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT); | |
215 | ||
216 | /* Set PER control regs, turns on single step for the given address */ | |
5e9a2692 | 217 | __ctl_load(per_kprobe, 9, 11); |
4ba069b8 | 218 | regs->psw.mask |= PSW_MASK_PER; |
adb45839 | 219 | regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT); |
fecc868a | 220 | regs->psw.addr = ip; |
4ba069b8 | 221 | } |
7a5388de | 222 | NOKPROBE_SYMBOL(enable_singlestep); |
4ba069b8 | 223 | |
7a5388de HC |
224 | static void disable_singlestep(struct kprobe_ctlblk *kcb, |
225 | struct pt_regs *regs, | |
226 | unsigned long ip) | |
fc0a1fea MS |
227 | { |
228 | /* Restore control regs and psw mask, set new psw address */ | |
229 | __ctl_load(kcb->kprobe_saved_ctl, 9, 11); | |
230 | regs->psw.mask &= ~PSW_MASK_PER; | |
231 | regs->psw.mask |= kcb->kprobe_saved_imask; | |
fecc868a | 232 | regs->psw.addr = ip; |
fc0a1fea | 233 | } |
7a5388de | 234 | NOKPROBE_SYMBOL(disable_singlestep); |
fc0a1fea | 235 | |
b9599798 MS |
236 | /* |
237 | * Activate a kprobe by storing its pointer to current_kprobe. The | |
238 | * previous kprobe is stored in kcb->prev_kprobe. A stack of up to | |
239 | * two kprobes can be active, see KPROBE_REENTER. | |
240 | */ | |
7a5388de | 241 | static void push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p) |
4ba069b8 | 242 | { |
eb7e7d76 | 243 | kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe); |
4ba069b8 | 244 | kcb->prev_kprobe.status = kcb->kprobe_status; |
eb7e7d76 | 245 | __this_cpu_write(current_kprobe, p); |
4ba069b8 | 246 | } |
7a5388de | 247 | NOKPROBE_SYMBOL(push_kprobe); |
4ba069b8 | 248 | |
b9599798 MS |
249 | /* |
250 | * Deactivate a kprobe by backing up to the previous state. If the | |
251 | * current state is KPROBE_REENTER prev_kprobe.kp will be non-NULL, | |
252 | * for any other state prev_kprobe.kp will be NULL. | |
253 | */ | |
7a5388de | 254 | static void pop_kprobe(struct kprobe_ctlblk *kcb) |
4ba069b8 | 255 | { |
eb7e7d76 | 256 | __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); |
4ba069b8 | 257 | kcb->kprobe_status = kcb->prev_kprobe.status; |
4ba069b8 | 258 | } |
7a5388de | 259 | NOKPROBE_SYMBOL(pop_kprobe); |
4ba069b8 | 260 | |
7a5388de | 261 | void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) |
4ba069b8 | 262 | { |
4c4308cb | 263 | ri->ret_addr = (kprobe_opcode_t *) regs->gprs[14]; |
4ba069b8 | 264 | |
4c4308cb | 265 | /* Replace the return addr with trampoline addr */ |
4a188635 | 266 | regs->gprs[14] = (unsigned long) &kretprobe_trampoline; |
4ba069b8 | 267 | } |
7a5388de | 268 | NOKPROBE_SYMBOL(arch_prepare_kretprobe); |
4ba069b8 | 269 | |
7a5388de | 270 | static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p) |
0e917cc3 MS |
271 | { |
272 | switch (kcb->kprobe_status) { | |
273 | case KPROBE_HIT_SSDONE: | |
274 | case KPROBE_HIT_ACTIVE: | |
275 | kprobes_inc_nmissed_count(p); | |
276 | break; | |
277 | case KPROBE_HIT_SS: | |
278 | case KPROBE_REENTER: | |
279 | default: | |
280 | /* | |
281 | * A kprobe on the code path to single step an instruction | |
282 | * is a BUG. The code path resides in the .kprobes.text | |
283 | * section and is executed with interrupts disabled. | |
284 | */ | |
285 | printk(KERN_EMERG "Invalid kprobe detected at %p.\n", p->addr); | |
286 | dump_kprobe(p); | |
287 | BUG(); | |
288 | } | |
289 | } | |
7a5388de | 290 | NOKPROBE_SYMBOL(kprobe_reenter_check); |
0e917cc3 | 291 | |
7a5388de | 292 | static int kprobe_handler(struct pt_regs *regs) |
4ba069b8 | 293 | { |
4ba069b8 | 294 | struct kprobe_ctlblk *kcb; |
0e917cc3 | 295 | struct kprobe *p; |
4ba069b8 MG |
296 | |
297 | /* | |
0e917cc3 MS |
298 | * We want to disable preemption for the entire duration of kprobe |
299 | * processing. That includes the calls to the pre/post handlers | |
300 | * and single stepping the kprobe instruction. | |
4ba069b8 MG |
301 | */ |
302 | preempt_disable(); | |
303 | kcb = get_kprobe_ctlblk(); | |
9cb1ccec | 304 | p = get_kprobe((void *)(regs->psw.addr - 2)); |
4ba069b8 | 305 | |
0e917cc3 MS |
306 | if (p) { |
307 | if (kprobe_running()) { | |
b9599798 MS |
308 | /* |
309 | * We have hit a kprobe while another is still | |
310 | * active. This can happen in the pre and post | |
311 | * handler. Single step the instruction of the | |
312 | * new probe but do not call any handler function | |
313 | * of this secondary kprobe. | |
314 | * push_kprobe and pop_kprobe saves and restores | |
315 | * the currently active kprobe. | |
4ba069b8 | 316 | */ |
0e917cc3 | 317 | kprobe_reenter_check(kcb, p); |
b9599798 | 318 | push_kprobe(kcb, p); |
4ba069b8 | 319 | kcb->kprobe_status = KPROBE_REENTER; |
4ba069b8 | 320 | } else { |
0e917cc3 MS |
321 | /* |
322 | * If we have no pre-handler or it returned 0, we | |
323 | * continue with single stepping. If we have a | |
324 | * pre-handler and it returned non-zero, it prepped | |
325 | * for calling the break_handler below on re-entry | |
326 | * for jprobe processing, so get out doing nothing | |
327 | * more here. | |
328 | */ | |
329 | push_kprobe(kcb, p); | |
330 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; | |
331 | if (p->pre_handler && p->pre_handler(p, regs)) | |
332 | return 1; | |
333 | kcb->kprobe_status = KPROBE_HIT_SS; | |
4ba069b8 | 334 | } |
0e917cc3 | 335 | enable_singlestep(kcb, regs, (unsigned long) p->ainsn.insn); |
4ba069b8 | 336 | return 1; |
0e917cc3 | 337 | } else if (kprobe_running()) { |
eb7e7d76 | 338 | p = __this_cpu_read(current_kprobe); |
0e917cc3 MS |
339 | if (p->break_handler && p->break_handler(p, regs)) { |
340 | /* | |
341 | * Continuation after the jprobe completed and | |
342 | * caused the jprobe_return trap. The jprobe | |
343 | * break_handler "returns" to the original | |
344 | * function that still has the kprobe breakpoint | |
345 | * installed. We continue with single stepping. | |
346 | */ | |
347 | kcb->kprobe_status = KPROBE_HIT_SS; | |
348 | enable_singlestep(kcb, regs, | |
349 | (unsigned long) p->ainsn.insn); | |
350 | return 1; | |
351 | } /* else: | |
352 | * No kprobe at this address and the current kprobe | |
353 | * has no break handler (no jprobe!). The kernel just | |
354 | * exploded, let the standard trap handler pick up the | |
355 | * pieces. | |
356 | */ | |
357 | } /* else: | |
358 | * No kprobe at this address and no active kprobe. The trap has | |
359 | * not been caused by a kprobe breakpoint. The race of breakpoint | |
360 | * vs. kprobe remove does not exist because on s390 as we use | |
361 | * stop_machine to arm/disarm the breakpoints. | |
362 | */ | |
4ba069b8 | 363 | preempt_enable_no_resched(); |
0e917cc3 | 364 | return 0; |
4ba069b8 | 365 | } |
7a5388de | 366 | NOKPROBE_SYMBOL(kprobe_handler); |
4ba069b8 MG |
367 | |
368 | /* | |
369 | * Function return probe trampoline: | |
370 | * - init_kprobes() establishes a probepoint here | |
371 | * - When the probed function returns, this probe | |
372 | * causes the handlers to fire | |
373 | */ | |
a806170e | 374 | static void __used kretprobe_trampoline_holder(void) |
4ba069b8 MG |
375 | { |
376 | asm volatile(".global kretprobe_trampoline\n" | |
377 | "kretprobe_trampoline: bcr 0,0\n"); | |
378 | } | |
379 | ||
380 | /* | |
381 | * Called when the probe at kretprobe trampoline is hit | |
382 | */ | |
7a5388de | 383 | static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) |
4ba069b8 | 384 | { |
4a188635 | 385 | struct kretprobe_instance *ri; |
99219a3f | 386 | struct hlist_head *head, empty_rp; |
b67bfe0d | 387 | struct hlist_node *tmp; |
4a188635 MS |
388 | unsigned long flags, orig_ret_address; |
389 | unsigned long trampoline_address; | |
390 | kprobe_opcode_t *correct_ret_addr; | |
4ba069b8 | 391 | |
99219a3f | 392 | INIT_HLIST_HEAD(&empty_rp); |
ef53d9c5 | 393 | kretprobe_hash_lock(current, &head, &flags); |
4ba069b8 MG |
394 | |
395 | /* | |
396 | * It is possible to have multiple instances associated with a given | |
397 | * task either because an multiple functions in the call path | |
025dfdaf | 398 | * have a return probe installed on them, and/or more than one return |
4ba069b8 MG |
399 | * return probe was registered for a target function. |
400 | * | |
401 | * We can handle this because: | |
402 | * - instances are always inserted at the head of the list | |
403 | * - when multiple return probes are registered for the same | |
404 | * function, the first instance's ret_addr will point to the | |
405 | * real return address, and all the rest will point to | |
406 | * kretprobe_trampoline | |
407 | */ | |
4a188635 MS |
408 | ri = NULL; |
409 | orig_ret_address = 0; | |
410 | correct_ret_addr = NULL; | |
411 | trampoline_address = (unsigned long) &kretprobe_trampoline; | |
b67bfe0d | 412 | hlist_for_each_entry_safe(ri, tmp, head, hlist) { |
4ba069b8 MG |
413 | if (ri->task != current) |
414 | /* another task is sharing our hash bucket */ | |
415 | continue; | |
416 | ||
4a188635 | 417 | orig_ret_address = (unsigned long) ri->ret_addr; |
89480801 MS |
418 | |
419 | if (orig_ret_address != trampoline_address) | |
420 | /* | |
421 | * This is the real return address. Any other | |
422 | * instances associated with this task are for | |
423 | * other calls deeper on the call stack | |
424 | */ | |
425 | break; | |
426 | } | |
427 | ||
428 | kretprobe_assert(ri, orig_ret_address, trampoline_address); | |
429 | ||
430 | correct_ret_addr = ri->ret_addr; | |
b67bfe0d | 431 | hlist_for_each_entry_safe(ri, tmp, head, hlist) { |
89480801 MS |
432 | if (ri->task != current) |
433 | /* another task is sharing our hash bucket */ | |
434 | continue; | |
4ba069b8 | 435 | |
4a188635 | 436 | orig_ret_address = (unsigned long) ri->ret_addr; |
89480801 MS |
437 | |
438 | if (ri->rp && ri->rp->handler) { | |
439 | ri->ret_addr = correct_ret_addr; | |
440 | ri->rp->handler(ri, regs); | |
441 | } | |
442 | ||
99219a3f | 443 | recycle_rp_inst(ri, &empty_rp); |
4ba069b8 | 444 | |
4a188635 | 445 | if (orig_ret_address != trampoline_address) |
4ba069b8 MG |
446 | /* |
447 | * This is the real return address. Any other | |
448 | * instances associated with this task are for | |
449 | * other calls deeper on the call stack | |
450 | */ | |
451 | break; | |
4ba069b8 | 452 | } |
89480801 | 453 | |
fecc868a | 454 | regs->psw.addr = orig_ret_address; |
4ba069b8 | 455 | |
b9599798 | 456 | pop_kprobe(get_kprobe_ctlblk()); |
ef53d9c5 | 457 | kretprobe_hash_unlock(current, &flags); |
4ba069b8 MG |
458 | preempt_enable_no_resched(); |
459 | ||
b67bfe0d | 460 | hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { |
99219a3f | 461 | hlist_del(&ri->hlist); |
462 | kfree(ri); | |
463 | } | |
4ba069b8 MG |
464 | /* |
465 | * By returning a non-zero value, we are telling | |
466 | * kprobe_handler() that we don't want the post_handler | |
467 | * to run (and have re-enabled preemption) | |
468 | */ | |
469 | return 1; | |
470 | } | |
7a5388de | 471 | NOKPROBE_SYMBOL(trampoline_probe_handler); |
4ba069b8 MG |
472 | |
473 | /* | |
474 | * Called after single-stepping. p->addr is the address of the | |
475 | * instruction whose first byte has been replaced by the "breakpoint" | |
476 | * instruction. To avoid the SMP problems that can occur when we | |
477 | * temporarily put back the original opcode to single-step, we | |
478 | * single-stepped a copy of the instruction. The address of this | |
479 | * copy is p->ainsn.insn. | |
480 | */ | |
7a5388de | 481 | static void resume_execution(struct kprobe *p, struct pt_regs *regs) |
4ba069b8 MG |
482 | { |
483 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); | |
9cb1ccec | 484 | unsigned long ip = regs->psw.addr; |
975fab17 | 485 | int fixup = probe_get_fixup_type(p->ainsn.insn); |
4ba069b8 | 486 | |
c933146a HC |
487 | /* Check if the kprobes location is an enabled ftrace caller */ |
488 | if (p->ainsn.is_ftrace_insn) { | |
489 | struct ftrace_insn *insn = (struct ftrace_insn *) p->addr; | |
490 | struct ftrace_insn call_insn; | |
491 | ||
492 | ftrace_generate_call_insn(&call_insn, (unsigned long) p->addr); | |
493 | /* | |
494 | * A kprobe on an enabled ftrace call site actually single | |
495 | * stepped an unconditional branch (ftrace nop equivalent). | |
496 | * Now we need to fixup things and pretend that a brasl r0,... | |
497 | * was executed instead. | |
498 | */ | |
499 | if (insn->disp == KPROBE_ON_FTRACE_CALL) { | |
500 | ip += call_insn.disp * 2 - MCOUNT_INSN_SIZE; | |
501 | regs->gprs[0] = (unsigned long)p->addr + sizeof(*insn); | |
502 | } | |
503 | } | |
504 | ||
ba640a59 | 505 | if (fixup & FIXUP_PSW_NORMAL) |
fc0a1fea | 506 | ip += (unsigned long) p->addr - (unsigned long) p->ainsn.insn; |
4ba069b8 | 507 | |
ba640a59 | 508 | if (fixup & FIXUP_BRANCH_NOT_TAKEN) { |
a882b3b0 | 509 | int ilen = insn_length(p->ainsn.insn[0] >> 8); |
ba640a59 MS |
510 | if (ip - (unsigned long) p->ainsn.insn == ilen) |
511 | ip = (unsigned long) p->addr + ilen; | |
512 | } | |
4ba069b8 | 513 | |
ba640a59 MS |
514 | if (fixup & FIXUP_RETURN_REGISTER) { |
515 | int reg = (p->ainsn.insn[0] & 0xf0) >> 4; | |
516 | regs->gprs[reg] += (unsigned long) p->addr - | |
517 | (unsigned long) p->ainsn.insn; | |
518 | } | |
4ba069b8 | 519 | |
fc0a1fea | 520 | disable_singlestep(kcb, regs, ip); |
4ba069b8 | 521 | } |
7a5388de | 522 | NOKPROBE_SYMBOL(resume_execution); |
4ba069b8 | 523 | |
7a5388de | 524 | static int post_kprobe_handler(struct pt_regs *regs) |
4ba069b8 | 525 | { |
4ba069b8 | 526 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
4a188635 | 527 | struct kprobe *p = kprobe_running(); |
4ba069b8 | 528 | |
4a188635 | 529 | if (!p) |
4ba069b8 MG |
530 | return 0; |
531 | ||
4a188635 | 532 | if (kcb->kprobe_status != KPROBE_REENTER && p->post_handler) { |
4ba069b8 | 533 | kcb->kprobe_status = KPROBE_HIT_SSDONE; |
4a188635 | 534 | p->post_handler(p, regs, 0); |
4ba069b8 MG |
535 | } |
536 | ||
4a188635 | 537 | resume_execution(p, regs); |
b9599798 | 538 | pop_kprobe(kcb); |
4ba069b8 MG |
539 | preempt_enable_no_resched(); |
540 | ||
541 | /* | |
542 | * if somebody else is singlestepping across a probe point, psw mask | |
543 | * will have PER set, in which case, continue the remaining processing | |
544 | * of do_single_step, as if this is not a probe hit. | |
545 | */ | |
4a188635 | 546 | if (regs->psw.mask & PSW_MASK_PER) |
4ba069b8 | 547 | return 0; |
4ba069b8 MG |
548 | |
549 | return 1; | |
550 | } | |
7a5388de | 551 | NOKPROBE_SYMBOL(post_kprobe_handler); |
4ba069b8 | 552 | |
7a5388de | 553 | static int kprobe_trap_handler(struct pt_regs *regs, int trapnr) |
4ba069b8 | 554 | { |
4ba069b8 | 555 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
4a188635 | 556 | struct kprobe *p = kprobe_running(); |
4ba069b8 MG |
557 | const struct exception_table_entry *entry; |
558 | ||
559 | switch(kcb->kprobe_status) { | |
4ba069b8 MG |
560 | case KPROBE_HIT_SS: |
561 | case KPROBE_REENTER: | |
562 | /* | |
563 | * We are here because the instruction being single | |
564 | * stepped caused a page fault. We reset the current | |
565 | * kprobe and the nip points back to the probe address | |
566 | * and allow the page fault handler to continue as a | |
567 | * normal page fault. | |
568 | */ | |
4a188635 | 569 | disable_singlestep(kcb, regs, (unsigned long) p->addr); |
b9599798 | 570 | pop_kprobe(kcb); |
4ba069b8 MG |
571 | preempt_enable_no_resched(); |
572 | break; | |
573 | case KPROBE_HIT_ACTIVE: | |
574 | case KPROBE_HIT_SSDONE: | |
575 | /* | |
576 | * We increment the nmissed count for accounting, | |
23d6d3db | 577 | * we can also use npre/npostfault count for accounting |
4ba069b8 MG |
578 | * these specific fault cases. |
579 | */ | |
4a188635 | 580 | kprobes_inc_nmissed_count(p); |
4ba069b8 MG |
581 | |
582 | /* | |
583 | * We come here because instructions in the pre/post | |
584 | * handler caused the page_fault, this could happen | |
585 | * if handler tries to access user space by | |
586 | * copy_from_user(), get_user() etc. Let the | |
587 | * user-specified handler try to fix it first. | |
588 | */ | |
4a188635 | 589 | if (p->fault_handler && p->fault_handler(p, regs, trapnr)) |
4ba069b8 MG |
590 | return 1; |
591 | ||
592 | /* | |
593 | * In case the user-specified fault handler returned | |
594 | * zero, try to fix up. | |
595 | */ | |
9cb1ccec | 596 | entry = search_exception_tables(regs->psw.addr); |
4ba069b8 | 597 | if (entry) { |
fecc868a | 598 | regs->psw.addr = extable_fixup(entry); |
4ba069b8 MG |
599 | return 1; |
600 | } | |
601 | ||
602 | /* | |
603 | * fixup_exception() could not handle it, | |
604 | * Let do_page_fault() fix it. | |
605 | */ | |
606 | break; | |
607 | default: | |
608 | break; | |
609 | } | |
610 | return 0; | |
611 | } | |
7a5388de | 612 | NOKPROBE_SYMBOL(kprobe_trap_handler); |
4ba069b8 | 613 | |
7a5388de | 614 | int kprobe_fault_handler(struct pt_regs *regs, int trapnr) |
adb45839 MS |
615 | { |
616 | int ret; | |
617 | ||
618 | if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT)) | |
619 | local_irq_disable(); | |
620 | ret = kprobe_trap_handler(regs, trapnr); | |
621 | if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT)) | |
622 | local_irq_restore(regs->psw.mask & ~PSW_MASK_PER); | |
623 | return ret; | |
624 | } | |
7a5388de | 625 | NOKPROBE_SYMBOL(kprobe_fault_handler); |
adb45839 | 626 | |
4ba069b8 MG |
627 | /* |
628 | * Wrapper routine to for handling exceptions. | |
629 | */ | |
7a5388de HC |
630 | int kprobe_exceptions_notify(struct notifier_block *self, |
631 | unsigned long val, void *data) | |
4ba069b8 | 632 | { |
4a188635 | 633 | struct die_args *args = (struct die_args *) data; |
adb45839 | 634 | struct pt_regs *regs = args->regs; |
4ba069b8 MG |
635 | int ret = NOTIFY_DONE; |
636 | ||
adb45839 MS |
637 | if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT)) |
638 | local_irq_disable(); | |
639 | ||
4ba069b8 MG |
640 | switch (val) { |
641 | case DIE_BPT: | |
4a188635 | 642 | if (kprobe_handler(regs)) |
4ba069b8 MG |
643 | ret = NOTIFY_STOP; |
644 | break; | |
645 | case DIE_SSTEP: | |
4a188635 | 646 | if (post_kprobe_handler(regs)) |
4ba069b8 MG |
647 | ret = NOTIFY_STOP; |
648 | break; | |
649 | case DIE_TRAP: | |
adb45839 | 650 | if (!preemptible() && kprobe_running() && |
4a188635 | 651 | kprobe_trap_handler(regs, args->trapnr)) |
4ba069b8 | 652 | ret = NOTIFY_STOP; |
4ba069b8 MG |
653 | break; |
654 | default: | |
655 | break; | |
656 | } | |
adb45839 MS |
657 | |
658 | if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT)) | |
659 | local_irq_restore(regs->psw.mask & ~PSW_MASK_PER); | |
660 | ||
4ba069b8 MG |
661 | return ret; |
662 | } | |
7a5388de | 663 | NOKPROBE_SYMBOL(kprobe_exceptions_notify); |
4ba069b8 | 664 | |
7a5388de | 665 | int setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) |
4ba069b8 MG |
666 | { |
667 | struct jprobe *jp = container_of(p, struct jprobe, kp); | |
4ba069b8 | 668 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
92b8cbf1 | 669 | unsigned long stack; |
4ba069b8 MG |
670 | |
671 | memcpy(&kcb->jprobe_saved_regs, regs, sizeof(struct pt_regs)); | |
672 | ||
673 | /* setup return addr to the jprobe handler routine */ | |
fecc868a | 674 | regs->psw.addr = (unsigned long) jp->entry; |
adb45839 | 675 | regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT); |
4ba069b8 | 676 | |
4ba069b8 | 677 | /* r15 is the stack pointer */ |
92b8cbf1 | 678 | stack = (unsigned long) regs->gprs[15]; |
4ba069b8 | 679 | |
92b8cbf1 | 680 | memcpy(kcb->jprobes_stack, (void *) stack, MIN_STACK_SIZE(stack)); |
e64a5470 JO |
681 | |
682 | /* | |
683 | * jprobes use jprobe_return() which skips the normal return | |
684 | * path of the function, and this messes up the accounting of the | |
685 | * function graph tracer to get messed up. | |
686 | * | |
687 | * Pause function graph tracing while performing the jprobe function. | |
688 | */ | |
689 | pause_graph_tracing(); | |
4ba069b8 MG |
690 | return 1; |
691 | } | |
7a5388de | 692 | NOKPROBE_SYMBOL(setjmp_pre_handler); |
4ba069b8 | 693 | |
7a5388de | 694 | void jprobe_return(void) |
4ba069b8 MG |
695 | { |
696 | asm volatile(".word 0x0002"); | |
697 | } | |
7a5388de | 698 | NOKPROBE_SYMBOL(jprobe_return); |
4ba069b8 | 699 | |
7a5388de | 700 | int longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) |
4ba069b8 MG |
701 | { |
702 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); | |
92b8cbf1 MS |
703 | unsigned long stack; |
704 | ||
e64a5470 JO |
705 | /* It's OK to start function graph tracing again */ |
706 | unpause_graph_tracing(); | |
707 | ||
92b8cbf1 | 708 | stack = (unsigned long) kcb->jprobe_saved_regs.gprs[15]; |
4ba069b8 MG |
709 | |
710 | /* Put the regs back */ | |
711 | memcpy(regs, &kcb->jprobe_saved_regs, sizeof(struct pt_regs)); | |
712 | /* put the stack back */ | |
92b8cbf1 | 713 | memcpy((void *) stack, kcb->jprobes_stack, MIN_STACK_SIZE(stack)); |
4ba069b8 MG |
714 | preempt_enable_no_resched(); |
715 | return 1; | |
716 | } | |
7a5388de | 717 | NOKPROBE_SYMBOL(longjmp_break_handler); |
4ba069b8 | 718 | |
4a188635 MS |
719 | static struct kprobe trampoline = { |
720 | .addr = (kprobe_opcode_t *) &kretprobe_trampoline, | |
4ba069b8 MG |
721 | .pre_handler = trampoline_probe_handler |
722 | }; | |
723 | ||
724 | int __init arch_init_kprobes(void) | |
725 | { | |
4a188635 | 726 | return register_kprobe(&trampoline); |
4ba069b8 | 727 | } |
bf8f6e5b | 728 | |
7a5388de | 729 | int arch_trampoline_kprobe(struct kprobe *p) |
bf8f6e5b | 730 | { |
4a188635 | 731 | return p->addr == (kprobe_opcode_t *) &kretprobe_trampoline; |
bf8f6e5b | 732 | } |
7a5388de | 733 | NOKPROBE_SYMBOL(arch_trampoline_kprobe); |