]>
Commit | Line | Data |
---|---|---|
a17ae4c3 | 1 | // SPDX-License-Identifier: GPL-2.0+ |
4ba069b8 MG |
2 | /* |
3 | * Kernel Probes (KProbes) | |
4 | * | |
a53c8fab | 5 | * Copyright IBM Corp. 2002, 2006 |
4ba069b8 MG |
6 | * |
7 | * s390 port, used ppc64 as template. Mike Grundy <grundym@us.ibm.com> | |
8 | */ | |
9 | ||
4ba069b8 MG |
10 | #include <linux/kprobes.h> |
11 | #include <linux/ptrace.h> | |
12 | #include <linux/preempt.h> | |
13 | #include <linux/stop_machine.h> | |
1eeb66a1 | 14 | #include <linux/kdebug.h> |
a2b53673 | 15 | #include <linux/uaccess.h> |
dcc096c5 | 16 | #include <linux/extable.h> |
4ba069b8 | 17 | #include <linux/module.h> |
5a0e3ad6 | 18 | #include <linux/slab.h> |
adb45839 | 19 | #include <linux/hardirq.h> |
c933146a | 20 | #include <linux/ftrace.h> |
e6c7c630 | 21 | #include <asm/set_memory.h> |
a882b3b0 HC |
22 | #include <asm/sections.h> |
23 | #include <asm/dis.h> | |
4ba069b8 | 24 | |
4a188635 | 25 | DEFINE_PER_CPU(struct kprobe *, current_kprobe); |
4ba069b8 MG |
26 | DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); |
27 | ||
4a188635 | 28 | struct kretprobe_blackpoint kretprobe_blacklist[] = { }; |
f438d914 | 29 | |
fd3d2742 | 30 | DEFINE_INSN_CACHE_OPS(s390_insn); |
63c40436 | 31 | |
fd3d2742 GS |
32 | static int insn_page_in_use; |
33 | static char insn_page[PAGE_SIZE] __aligned(PAGE_SIZE); | |
57d7f939 | 34 | |
fd3d2742 GS |
35 | static void *alloc_s390_insn_page(void) |
36 | { | |
37 | if (xchg(&insn_page_in_use, 1) == 1) | |
38 | return NULL; | |
39 | set_memory_x((unsigned long) &insn_page, 1); | |
40 | return &insn_page; | |
63c40436 HC |
41 | } |
42 | ||
fd3d2742 | 43 | static void free_s390_insn_page(void *page) |
63c40436 | 44 | { |
57d7f939 | 45 | set_memory_nx((unsigned long) page, 1); |
fd3d2742 | 46 | xchg(&insn_page_in_use, 0); |
63c40436 HC |
47 | } |
48 | ||
fd3d2742 GS |
49 | struct kprobe_insn_cache kprobe_s390_insn_slots = { |
50 | .mutex = __MUTEX_INITIALIZER(kprobe_s390_insn_slots.mutex), | |
51 | .alloc = alloc_s390_insn_page, | |
52 | .free = free_s390_insn_page, | |
53 | .pages = LIST_HEAD_INIT(kprobe_s390_insn_slots.pages), | |
63c40436 HC |
54 | .insn_size = MAX_INSN_SIZE, |
55 | }; | |
56 | ||
7a5388de | 57 | static void copy_instruction(struct kprobe *p) |
63c40436 HC |
58 | { |
59 | s64 disp, new_disp; | |
60 | u64 addr, new_addr; | |
61 | ||
e8fab751 | 62 | memcpy(p->ainsn.insn, p->addr, insn_length(*p->addr >> 8)); |
c933146a | 63 | p->opcode = p->ainsn.insn[0]; |
975fab17 | 64 | if (!probe_is_insn_relative_long(p->ainsn.insn)) |
63c40436 HC |
65 | return; |
66 | /* | |
67 | * For pc-relative instructions in RIL-b or RIL-c format patch the | |
68 | * RI2 displacement field. We have already made sure that the insn | |
69 | * slot for the patched instruction is within the same 2GB area | |
70 | * as the original instruction (either kernel image or module area). | |
71 | * Therefore the new displacement will always fit. | |
72 | */ | |
73 | disp = *(s32 *)&p->ainsn.insn[1]; | |
74 | addr = (u64)(unsigned long)p->addr; | |
75 | new_addr = (u64)(unsigned long)p->ainsn.insn; | |
76 | new_disp = ((addr + (disp * 2)) - new_addr) / 2; | |
77 | *(s32 *)&p->ainsn.insn[1] = new_disp; | |
78 | } | |
7a5388de | 79 | NOKPROBE_SYMBOL(copy_instruction); |
63c40436 HC |
80 | |
81 | static inline int is_kernel_addr(void *addr) | |
82 | { | |
83 | return addr < (void *)_end; | |
84 | } | |
85 | ||
7a5388de | 86 | static int s390_get_insn_slot(struct kprobe *p) |
63c40436 HC |
87 | { |
88 | /* | |
89 | * Get an insn slot that is within the same 2GB area like the original | |
90 | * instruction. That way instructions with a 32bit signed displacement | |
91 | * field can be patched and executed within the insn slot. | |
92 | */ | |
93 | p->ainsn.insn = NULL; | |
94 | if (is_kernel_addr(p->addr)) | |
fd3d2742 | 95 | p->ainsn.insn = get_s390_insn_slot(); |
fcd05b50 | 96 | else if (is_module_addr(p->addr)) |
63c40436 HC |
97 | p->ainsn.insn = get_insn_slot(); |
98 | return p->ainsn.insn ? 0 : -ENOMEM; | |
99 | } | |
7a5388de | 100 | NOKPROBE_SYMBOL(s390_get_insn_slot); |
63c40436 | 101 | |
7a5388de | 102 | static void s390_free_insn_slot(struct kprobe *p) |
63c40436 HC |
103 | { |
104 | if (!p->ainsn.insn) | |
105 | return; | |
106 | if (is_kernel_addr(p->addr)) | |
fd3d2742 | 107 | free_s390_insn_slot(p->ainsn.insn, 0); |
63c40436 HC |
108 | else |
109 | free_insn_slot(p->ainsn.insn, 0); | |
110 | p->ainsn.insn = NULL; | |
111 | } | |
7a5388de | 112 | NOKPROBE_SYMBOL(s390_free_insn_slot); |
63c40436 | 113 | |
7a5388de | 114 | int arch_prepare_kprobe(struct kprobe *p) |
ba640a59 MS |
115 | { |
116 | if ((unsigned long) p->addr & 0x01) | |
117 | return -EINVAL; | |
ba640a59 | 118 | /* Make sure the probe isn't going on a difficult instruction */ |
975fab17 | 119 | if (probe_is_prohibited_opcode(p->addr)) |
ba640a59 | 120 | return -EINVAL; |
63c40436 HC |
121 | if (s390_get_insn_slot(p)) |
122 | return -ENOMEM; | |
63c40436 | 123 | copy_instruction(p); |
ba640a59 | 124 | return 0; |
4ba069b8 | 125 | } |
7a5388de | 126 | NOKPROBE_SYMBOL(arch_prepare_kprobe); |
4ba069b8 | 127 | |
c933146a HC |
128 | struct swap_insn_args { |
129 | struct kprobe *p; | |
130 | unsigned int arm_kprobe : 1; | |
5a8b589f MS |
131 | }; |
132 | ||
7a5388de | 133 | static int swap_instruction(void *data) |
4ba069b8 | 134 | { |
c933146a | 135 | struct swap_insn_args *args = data; |
c933146a | 136 | struct kprobe *p = args->p; |
e8fab751 SS |
137 | u16 opc; |
138 | ||
139 | opc = args->arm_kprobe ? BREAKPOINT_INSTRUCTION : p->opcode; | |
140 | s390_kernel_write(p->addr, &opc, sizeof(opc)); | |
5a8b589f | 141 | return 0; |
4ba069b8 | 142 | } |
7a5388de | 143 | NOKPROBE_SYMBOL(swap_instruction); |
4ba069b8 | 144 | |
7a5388de | 145 | void arch_arm_kprobe(struct kprobe *p) |
4ba069b8 | 146 | { |
c933146a | 147 | struct swap_insn_args args = {.p = p, .arm_kprobe = 1}; |
4ba069b8 | 148 | |
5d5dbc4e | 149 | stop_machine_cpuslocked(swap_instruction, &args, NULL); |
4ba069b8 | 150 | } |
7a5388de | 151 | NOKPROBE_SYMBOL(arch_arm_kprobe); |
4ba069b8 | 152 | |
7a5388de | 153 | void arch_disarm_kprobe(struct kprobe *p) |
4ba069b8 | 154 | { |
c933146a | 155 | struct swap_insn_args args = {.p = p, .arm_kprobe = 0}; |
4ba069b8 | 156 | |
5d5dbc4e | 157 | stop_machine_cpuslocked(swap_instruction, &args, NULL); |
4ba069b8 | 158 | } |
7a5388de | 159 | NOKPROBE_SYMBOL(arch_disarm_kprobe); |
4ba069b8 | 160 | |
7a5388de | 161 | void arch_remove_kprobe(struct kprobe *p) |
4ba069b8 | 162 | { |
63c40436 | 163 | s390_free_insn_slot(p); |
4ba069b8 | 164 | } |
7a5388de | 165 | NOKPROBE_SYMBOL(arch_remove_kprobe); |
4ba069b8 | 166 | |
7a5388de HC |
167 | static void enable_singlestep(struct kprobe_ctlblk *kcb, |
168 | struct pt_regs *regs, | |
169 | unsigned long ip) | |
4ba069b8 | 170 | { |
5e9a2692 | 171 | struct per_regs per_kprobe; |
4ba069b8 | 172 | |
5e9a2692 MS |
173 | /* Set up the PER control registers %cr9-%cr11 */ |
174 | per_kprobe.control = PER_EVENT_IFETCH; | |
175 | per_kprobe.start = ip; | |
176 | per_kprobe.end = ip; | |
4ba069b8 | 177 | |
fc0a1fea MS |
178 | /* Save control regs and psw mask */ |
179 | __ctl_store(kcb->kprobe_saved_ctl, 9, 11); | |
180 | kcb->kprobe_saved_imask = regs->psw.mask & | |
181 | (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT); | |
182 | ||
183 | /* Set PER control regs, turns on single step for the given address */ | |
5e9a2692 | 184 | __ctl_load(per_kprobe, 9, 11); |
4ba069b8 | 185 | regs->psw.mask |= PSW_MASK_PER; |
adb45839 | 186 | regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT); |
fecc868a | 187 | regs->psw.addr = ip; |
4ba069b8 | 188 | } |
7a5388de | 189 | NOKPROBE_SYMBOL(enable_singlestep); |
4ba069b8 | 190 | |
7a5388de HC |
191 | static void disable_singlestep(struct kprobe_ctlblk *kcb, |
192 | struct pt_regs *regs, | |
193 | unsigned long ip) | |
fc0a1fea MS |
194 | { |
195 | /* Restore control regs and psw mask, set new psw address */ | |
196 | __ctl_load(kcb->kprobe_saved_ctl, 9, 11); | |
197 | regs->psw.mask &= ~PSW_MASK_PER; | |
198 | regs->psw.mask |= kcb->kprobe_saved_imask; | |
fecc868a | 199 | regs->psw.addr = ip; |
fc0a1fea | 200 | } |
7a5388de | 201 | NOKPROBE_SYMBOL(disable_singlestep); |
fc0a1fea | 202 | |
b9599798 MS |
203 | /* |
204 | * Activate a kprobe by storing its pointer to current_kprobe. The | |
205 | * previous kprobe is stored in kcb->prev_kprobe. A stack of up to | |
206 | * two kprobes can be active, see KPROBE_REENTER. | |
207 | */ | |
7a5388de | 208 | static void push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p) |
4ba069b8 | 209 | { |
eb7e7d76 | 210 | kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe); |
4ba069b8 | 211 | kcb->prev_kprobe.status = kcb->kprobe_status; |
eb7e7d76 | 212 | __this_cpu_write(current_kprobe, p); |
4ba069b8 | 213 | } |
7a5388de | 214 | NOKPROBE_SYMBOL(push_kprobe); |
4ba069b8 | 215 | |
b9599798 MS |
216 | /* |
217 | * Deactivate a kprobe by backing up to the previous state. If the | |
218 | * current state is KPROBE_REENTER prev_kprobe.kp will be non-NULL, | |
219 | * for any other state prev_kprobe.kp will be NULL. | |
220 | */ | |
7a5388de | 221 | static void pop_kprobe(struct kprobe_ctlblk *kcb) |
4ba069b8 | 222 | { |
eb7e7d76 | 223 | __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); |
4ba069b8 | 224 | kcb->kprobe_status = kcb->prev_kprobe.status; |
4ba069b8 | 225 | } |
7a5388de | 226 | NOKPROBE_SYMBOL(pop_kprobe); |
4ba069b8 | 227 | |
7a5388de | 228 | void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) |
4ba069b8 | 229 | { |
4c4308cb | 230 | ri->ret_addr = (kprobe_opcode_t *) regs->gprs[14]; |
4ba069b8 | 231 | |
4c4308cb | 232 | /* Replace the return addr with trampoline addr */ |
4a188635 | 233 | regs->gprs[14] = (unsigned long) &kretprobe_trampoline; |
4ba069b8 | 234 | } |
7a5388de | 235 | NOKPROBE_SYMBOL(arch_prepare_kretprobe); |
4ba069b8 | 236 | |
7a5388de | 237 | static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p) |
0e917cc3 MS |
238 | { |
239 | switch (kcb->kprobe_status) { | |
240 | case KPROBE_HIT_SSDONE: | |
241 | case KPROBE_HIT_ACTIVE: | |
242 | kprobes_inc_nmissed_count(p); | |
243 | break; | |
244 | case KPROBE_HIT_SS: | |
245 | case KPROBE_REENTER: | |
246 | default: | |
247 | /* | |
248 | * A kprobe on the code path to single step an instruction | |
249 | * is a BUG. The code path resides in the .kprobes.text | |
250 | * section and is executed with interrupts disabled. | |
251 | */ | |
0e1647b3 | 252 | pr_err("Invalid kprobe detected.\n"); |
0e917cc3 MS |
253 | dump_kprobe(p); |
254 | BUG(); | |
255 | } | |
256 | } | |
7a5388de | 257 | NOKPROBE_SYMBOL(kprobe_reenter_check); |
0e917cc3 | 258 | |
7a5388de | 259 | static int kprobe_handler(struct pt_regs *regs) |
4ba069b8 | 260 | { |
4ba069b8 | 261 | struct kprobe_ctlblk *kcb; |
0e917cc3 | 262 | struct kprobe *p; |
4ba069b8 MG |
263 | |
264 | /* | |
0e917cc3 MS |
265 | * We want to disable preemption for the entire duration of kprobe |
266 | * processing. That includes the calls to the pre/post handlers | |
267 | * and single stepping the kprobe instruction. | |
4ba069b8 MG |
268 | */ |
269 | preempt_disable(); | |
270 | kcb = get_kprobe_ctlblk(); | |
9cb1ccec | 271 | p = get_kprobe((void *)(regs->psw.addr - 2)); |
4ba069b8 | 272 | |
0e917cc3 MS |
273 | if (p) { |
274 | if (kprobe_running()) { | |
b9599798 MS |
275 | /* |
276 | * We have hit a kprobe while another is still | |
277 | * active. This can happen in the pre and post | |
278 | * handler. Single step the instruction of the | |
279 | * new probe but do not call any handler function | |
280 | * of this secondary kprobe. | |
281 | * push_kprobe and pop_kprobe saves and restores | |
282 | * the currently active kprobe. | |
4ba069b8 | 283 | */ |
0e917cc3 | 284 | kprobe_reenter_check(kcb, p); |
b9599798 | 285 | push_kprobe(kcb, p); |
4ba069b8 | 286 | kcb->kprobe_status = KPROBE_REENTER; |
4ba069b8 | 287 | } else { |
0e917cc3 MS |
288 | /* |
289 | * If we have no pre-handler or it returned 0, we | |
290 | * continue with single stepping. If we have a | |
291 | * pre-handler and it returned non-zero, it prepped | |
fc682f7b MH |
292 | * for changing execution path, so get out doing |
293 | * nothing more here. | |
0e917cc3 MS |
294 | */ |
295 | push_kprobe(kcb, p); | |
296 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; | |
cce188bd MH |
297 | if (p->pre_handler && p->pre_handler(p, regs)) { |
298 | pop_kprobe(kcb); | |
299 | preempt_enable_no_resched(); | |
0e917cc3 | 300 | return 1; |
cce188bd | 301 | } |
0e917cc3 | 302 | kcb->kprobe_status = KPROBE_HIT_SS; |
4ba069b8 | 303 | } |
0e917cc3 | 304 | enable_singlestep(kcb, regs, (unsigned long) p->ainsn.insn); |
4ba069b8 | 305 | return 1; |
0e917cc3 MS |
306 | } /* else: |
307 | * No kprobe at this address and no active kprobe. The trap has | |
308 | * not been caused by a kprobe breakpoint. The race of breakpoint | |
309 | * vs. kprobe remove does not exist because on s390 as we use | |
310 | * stop_machine to arm/disarm the breakpoints. | |
311 | */ | |
4ba069b8 | 312 | preempt_enable_no_resched(); |
0e917cc3 | 313 | return 0; |
4ba069b8 | 314 | } |
7a5388de | 315 | NOKPROBE_SYMBOL(kprobe_handler); |
4ba069b8 MG |
316 | |
317 | /* | |
318 | * Function return probe trampoline: | |
319 | * - init_kprobes() establishes a probepoint here | |
320 | * - When the probed function returns, this probe | |
321 | * causes the handlers to fire | |
322 | */ | |
a806170e | 323 | static void __used kretprobe_trampoline_holder(void) |
4ba069b8 MG |
324 | { |
325 | asm volatile(".global kretprobe_trampoline\n" | |
326 | "kretprobe_trampoline: bcr 0,0\n"); | |
327 | } | |
328 | ||
329 | /* | |
330 | * Called when the probe at kretprobe trampoline is hit | |
331 | */ | |
7a5388de | 332 | static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) |
4ba069b8 | 333 | { |
4a188635 | 334 | struct kretprobe_instance *ri; |
99219a3f | 335 | struct hlist_head *head, empty_rp; |
b67bfe0d | 336 | struct hlist_node *tmp; |
4a188635 MS |
337 | unsigned long flags, orig_ret_address; |
338 | unsigned long trampoline_address; | |
339 | kprobe_opcode_t *correct_ret_addr; | |
4ba069b8 | 340 | |
99219a3f | 341 | INIT_HLIST_HEAD(&empty_rp); |
ef53d9c5 | 342 | kretprobe_hash_lock(current, &head, &flags); |
4ba069b8 MG |
343 | |
344 | /* | |
345 | * It is possible to have multiple instances associated with a given | |
346 | * task either because an multiple functions in the call path | |
025dfdaf | 347 | * have a return probe installed on them, and/or more than one return |
4ba069b8 MG |
348 | * return probe was registered for a target function. |
349 | * | |
350 | * We can handle this because: | |
351 | * - instances are always inserted at the head of the list | |
352 | * - when multiple return probes are registered for the same | |
353 | * function, the first instance's ret_addr will point to the | |
354 | * real return address, and all the rest will point to | |
355 | * kretprobe_trampoline | |
356 | */ | |
4a188635 MS |
357 | ri = NULL; |
358 | orig_ret_address = 0; | |
359 | correct_ret_addr = NULL; | |
360 | trampoline_address = (unsigned long) &kretprobe_trampoline; | |
b67bfe0d | 361 | hlist_for_each_entry_safe(ri, tmp, head, hlist) { |
4ba069b8 MG |
362 | if (ri->task != current) |
363 | /* another task is sharing our hash bucket */ | |
364 | continue; | |
365 | ||
4a188635 | 366 | orig_ret_address = (unsigned long) ri->ret_addr; |
89480801 MS |
367 | |
368 | if (orig_ret_address != trampoline_address) | |
369 | /* | |
370 | * This is the real return address. Any other | |
371 | * instances associated with this task are for | |
372 | * other calls deeper on the call stack | |
373 | */ | |
374 | break; | |
375 | } | |
376 | ||
377 | kretprobe_assert(ri, orig_ret_address, trampoline_address); | |
378 | ||
379 | correct_ret_addr = ri->ret_addr; | |
b67bfe0d | 380 | hlist_for_each_entry_safe(ri, tmp, head, hlist) { |
89480801 MS |
381 | if (ri->task != current) |
382 | /* another task is sharing our hash bucket */ | |
383 | continue; | |
4ba069b8 | 384 | |
4a188635 | 385 | orig_ret_address = (unsigned long) ri->ret_addr; |
89480801 MS |
386 | |
387 | if (ri->rp && ri->rp->handler) { | |
388 | ri->ret_addr = correct_ret_addr; | |
389 | ri->rp->handler(ri, regs); | |
390 | } | |
391 | ||
99219a3f | 392 | recycle_rp_inst(ri, &empty_rp); |
4ba069b8 | 393 | |
4a188635 | 394 | if (orig_ret_address != trampoline_address) |
4ba069b8 MG |
395 | /* |
396 | * This is the real return address. Any other | |
397 | * instances associated with this task are for | |
398 | * other calls deeper on the call stack | |
399 | */ | |
400 | break; | |
4ba069b8 | 401 | } |
89480801 | 402 | |
fecc868a | 403 | regs->psw.addr = orig_ret_address; |
4ba069b8 | 404 | |
ef53d9c5 | 405 | kretprobe_hash_unlock(current, &flags); |
4ba069b8 | 406 | |
b67bfe0d | 407 | hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { |
99219a3f | 408 | hlist_del(&ri->hlist); |
409 | kfree(ri); | |
410 | } | |
4ba069b8 MG |
411 | /* |
412 | * By returning a non-zero value, we are telling | |
413 | * kprobe_handler() that we don't want the post_handler | |
414 | * to run (and have re-enabled preemption) | |
415 | */ | |
416 | return 1; | |
417 | } | |
7a5388de | 418 | NOKPROBE_SYMBOL(trampoline_probe_handler); |
4ba069b8 MG |
419 | |
420 | /* | |
421 | * Called after single-stepping. p->addr is the address of the | |
422 | * instruction whose first byte has been replaced by the "breakpoint" | |
423 | * instruction. To avoid the SMP problems that can occur when we | |
424 | * temporarily put back the original opcode to single-step, we | |
425 | * single-stepped a copy of the instruction. The address of this | |
426 | * copy is p->ainsn.insn. | |
427 | */ | |
7a5388de | 428 | static void resume_execution(struct kprobe *p, struct pt_regs *regs) |
4ba069b8 MG |
429 | { |
430 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); | |
9cb1ccec | 431 | unsigned long ip = regs->psw.addr; |
975fab17 | 432 | int fixup = probe_get_fixup_type(p->ainsn.insn); |
4ba069b8 | 433 | |
ba640a59 | 434 | if (fixup & FIXUP_PSW_NORMAL) |
fc0a1fea | 435 | ip += (unsigned long) p->addr - (unsigned long) p->ainsn.insn; |
4ba069b8 | 436 | |
ba640a59 | 437 | if (fixup & FIXUP_BRANCH_NOT_TAKEN) { |
a882b3b0 | 438 | int ilen = insn_length(p->ainsn.insn[0] >> 8); |
ba640a59 MS |
439 | if (ip - (unsigned long) p->ainsn.insn == ilen) |
440 | ip = (unsigned long) p->addr + ilen; | |
441 | } | |
4ba069b8 | 442 | |
ba640a59 MS |
443 | if (fixup & FIXUP_RETURN_REGISTER) { |
444 | int reg = (p->ainsn.insn[0] & 0xf0) >> 4; | |
445 | regs->gprs[reg] += (unsigned long) p->addr - | |
446 | (unsigned long) p->ainsn.insn; | |
447 | } | |
4ba069b8 | 448 | |
fc0a1fea | 449 | disable_singlestep(kcb, regs, ip); |
4ba069b8 | 450 | } |
7a5388de | 451 | NOKPROBE_SYMBOL(resume_execution); |
4ba069b8 | 452 | |
7a5388de | 453 | static int post_kprobe_handler(struct pt_regs *regs) |
4ba069b8 | 454 | { |
4ba069b8 | 455 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
4a188635 | 456 | struct kprobe *p = kprobe_running(); |
4ba069b8 | 457 | |
4a188635 | 458 | if (!p) |
4ba069b8 MG |
459 | return 0; |
460 | ||
4a188635 | 461 | if (kcb->kprobe_status != KPROBE_REENTER && p->post_handler) { |
4ba069b8 | 462 | kcb->kprobe_status = KPROBE_HIT_SSDONE; |
4a188635 | 463 | p->post_handler(p, regs, 0); |
4ba069b8 MG |
464 | } |
465 | ||
4a188635 | 466 | resume_execution(p, regs); |
b9599798 | 467 | pop_kprobe(kcb); |
4ba069b8 MG |
468 | preempt_enable_no_resched(); |
469 | ||
470 | /* | |
471 | * if somebody else is singlestepping across a probe point, psw mask | |
472 | * will have PER set, in which case, continue the remaining processing | |
473 | * of do_single_step, as if this is not a probe hit. | |
474 | */ | |
4a188635 | 475 | if (regs->psw.mask & PSW_MASK_PER) |
4ba069b8 | 476 | return 0; |
4ba069b8 MG |
477 | |
478 | return 1; | |
479 | } | |
7a5388de | 480 | NOKPROBE_SYMBOL(post_kprobe_handler); |
4ba069b8 | 481 | |
7a5388de | 482 | static int kprobe_trap_handler(struct pt_regs *regs, int trapnr) |
4ba069b8 | 483 | { |
4ba069b8 | 484 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
4a188635 | 485 | struct kprobe *p = kprobe_running(); |
4ba069b8 MG |
486 | const struct exception_table_entry *entry; |
487 | ||
488 | switch(kcb->kprobe_status) { | |
4ba069b8 MG |
489 | case KPROBE_HIT_SS: |
490 | case KPROBE_REENTER: | |
491 | /* | |
492 | * We are here because the instruction being single | |
493 | * stepped caused a page fault. We reset the current | |
494 | * kprobe and the nip points back to the probe address | |
495 | * and allow the page fault handler to continue as a | |
496 | * normal page fault. | |
497 | */ | |
4a188635 | 498 | disable_singlestep(kcb, regs, (unsigned long) p->addr); |
b9599798 | 499 | pop_kprobe(kcb); |
4ba069b8 MG |
500 | preempt_enable_no_resched(); |
501 | break; | |
502 | case KPROBE_HIT_ACTIVE: | |
503 | case KPROBE_HIT_SSDONE: | |
504 | /* | |
505 | * We increment the nmissed count for accounting, | |
23d6d3db | 506 | * we can also use npre/npostfault count for accounting |
4ba069b8 MG |
507 | * these specific fault cases. |
508 | */ | |
4a188635 | 509 | kprobes_inc_nmissed_count(p); |
4ba069b8 MG |
510 | |
511 | /* | |
512 | * We come here because instructions in the pre/post | |
513 | * handler caused the page_fault, this could happen | |
514 | * if handler tries to access user space by | |
515 | * copy_from_user(), get_user() etc. Let the | |
516 | * user-specified handler try to fix it first. | |
517 | */ | |
4a188635 | 518 | if (p->fault_handler && p->fault_handler(p, regs, trapnr)) |
4ba069b8 MG |
519 | return 1; |
520 | ||
521 | /* | |
522 | * In case the user-specified fault handler returned | |
523 | * zero, try to fix up. | |
524 | */ | |
a80313ff | 525 | entry = s390_search_extables(regs->psw.addr); |
4ba069b8 | 526 | if (entry) { |
fecc868a | 527 | regs->psw.addr = extable_fixup(entry); |
4ba069b8 MG |
528 | return 1; |
529 | } | |
530 | ||
531 | /* | |
532 | * fixup_exception() could not handle it, | |
533 | * Let do_page_fault() fix it. | |
534 | */ | |
535 | break; | |
536 | default: | |
537 | break; | |
538 | } | |
539 | return 0; | |
540 | } | |
7a5388de | 541 | NOKPROBE_SYMBOL(kprobe_trap_handler); |
4ba069b8 | 542 | |
7a5388de | 543 | int kprobe_fault_handler(struct pt_regs *regs, int trapnr) |
adb45839 MS |
544 | { |
545 | int ret; | |
546 | ||
547 | if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT)) | |
548 | local_irq_disable(); | |
549 | ret = kprobe_trap_handler(regs, trapnr); | |
550 | if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT)) | |
551 | local_irq_restore(regs->psw.mask & ~PSW_MASK_PER); | |
552 | return ret; | |
553 | } | |
7a5388de | 554 | NOKPROBE_SYMBOL(kprobe_fault_handler); |
adb45839 | 555 | |
4ba069b8 MG |
556 | /* |
557 | * Wrapper routine to for handling exceptions. | |
558 | */ | |
7a5388de HC |
559 | int kprobe_exceptions_notify(struct notifier_block *self, |
560 | unsigned long val, void *data) | |
4ba069b8 | 561 | { |
4a188635 | 562 | struct die_args *args = (struct die_args *) data; |
adb45839 | 563 | struct pt_regs *regs = args->regs; |
4ba069b8 MG |
564 | int ret = NOTIFY_DONE; |
565 | ||
adb45839 MS |
566 | if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT)) |
567 | local_irq_disable(); | |
568 | ||
4ba069b8 MG |
569 | switch (val) { |
570 | case DIE_BPT: | |
4a188635 | 571 | if (kprobe_handler(regs)) |
4ba069b8 MG |
572 | ret = NOTIFY_STOP; |
573 | break; | |
574 | case DIE_SSTEP: | |
4a188635 | 575 | if (post_kprobe_handler(regs)) |
4ba069b8 MG |
576 | ret = NOTIFY_STOP; |
577 | break; | |
578 | case DIE_TRAP: | |
adb45839 | 579 | if (!preemptible() && kprobe_running() && |
4a188635 | 580 | kprobe_trap_handler(regs, args->trapnr)) |
4ba069b8 | 581 | ret = NOTIFY_STOP; |
4ba069b8 MG |
582 | break; |
583 | default: | |
584 | break; | |
585 | } | |
adb45839 MS |
586 | |
587 | if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT)) | |
588 | local_irq_restore(regs->psw.mask & ~PSW_MASK_PER); | |
589 | ||
4ba069b8 MG |
590 | return ret; |
591 | } | |
7a5388de | 592 | NOKPROBE_SYMBOL(kprobe_exceptions_notify); |
4ba069b8 | 593 | |
4a188635 MS |
594 | static struct kprobe trampoline = { |
595 | .addr = (kprobe_opcode_t *) &kretprobe_trampoline, | |
4ba069b8 MG |
596 | .pre_handler = trampoline_probe_handler |
597 | }; | |
598 | ||
599 | int __init arch_init_kprobes(void) | |
600 | { | |
4a188635 | 601 | return register_kprobe(&trampoline); |
4ba069b8 | 602 | } |
bf8f6e5b | 603 | |
7a5388de | 604 | int arch_trampoline_kprobe(struct kprobe *p) |
bf8f6e5b | 605 | { |
4a188635 | 606 | return p->addr == (kprobe_opcode_t *) &kretprobe_trampoline; |
bf8f6e5b | 607 | } |
7a5388de | 608 | NOKPROBE_SYMBOL(arch_trampoline_kprobe); |