]>
Commit | Line | Data |
---|---|---|
873e65bc | 1 | // SPDX-License-Identifier: GPL-2.0-only |
c1bf207d DD |
2 | /* |
3 | * Kernel Probes (KProbes) | |
4 | * arch/mips/kernel/kprobes.c | |
5 | * | |
6 | * Copyright 2006 Sony Corp. | |
7 | * Copyright 2010 Cavium Networks | |
8 | * | |
9 | * Some portions copied from the powerpc version. | |
10 | * | |
11 | * Copyright (C) IBM Corporation, 2002, 2004 | |
c1bf207d DD |
12 | */ |
13 | ||
14 | #include <linux/kprobes.h> | |
15 | #include <linux/preempt.h> | |
41dde781 | 16 | #include <linux/uaccess.h> |
c1bf207d DD |
17 | #include <linux/kdebug.h> |
18 | #include <linux/slab.h> | |
19 | ||
20 | #include <asm/ptrace.h> | |
6457a396 | 21 | #include <asm/branch.h> |
c1bf207d | 22 | #include <asm/break.h> |
e3031b32 MN |
23 | |
24 | #include "probes-common.h" | |
c1bf207d DD |
25 | |
26 | static const union mips_instruction breakpoint_insn = { | |
27 | .b_format = { | |
28 | .opcode = spec_op, | |
29 | .code = BRK_KPROBE_BP, | |
30 | .func = break_op | |
31 | } | |
32 | }; | |
33 | ||
34 | static const union mips_instruction breakpoint2_insn = { | |
35 | .b_format = { | |
36 | .opcode = spec_op, | |
37 | .code = BRK_KPROBE_SSTEPBP, | |
38 | .func = break_op | |
39 | } | |
40 | }; | |
41 | ||
42 | DEFINE_PER_CPU(struct kprobe *, current_kprobe); | |
43 | DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); | |
44 | ||
45 | static int __kprobes insn_has_delayslot(union mips_instruction insn) | |
46 | { | |
e3031b32 | 47 | return __insn_has_delay_slot(insn); |
c1bf207d DD |
48 | } |
49 | ||
9233c1ee MS |
50 | /* |
51 | * insn_has_ll_or_sc function checks whether instruction is ll or sc | |
52 | * one; putting breakpoint on top of atomic ll/sc pair is bad idea; | |
53 | * so we need to prevent it and refuse kprobes insertion for such | |
54 | * instructions; cannot do much about breakpoint in the middle of | |
55 | * ll/sc pair; it is upto user to avoid those places | |
56 | */ | |
57 | static int __kprobes insn_has_ll_or_sc(union mips_instruction insn) | |
58 | { | |
59 | int ret = 0; | |
60 | ||
61 | switch (insn.i_format.opcode) { | |
62 | case ll_op: | |
63 | case lld_op: | |
64 | case sc_op: | |
65 | case scd_op: | |
66 | ret = 1; | |
67 | break; | |
68 | default: | |
69 | break; | |
70 | } | |
71 | return ret; | |
72 | } | |
73 | ||
c1bf207d DD |
74 | int __kprobes arch_prepare_kprobe(struct kprobe *p) |
75 | { | |
76 | union mips_instruction insn; | |
77 | union mips_instruction prev_insn; | |
78 | int ret = 0; | |
79 | ||
c1bf207d DD |
80 | insn = p->addr[0]; |
81 | ||
9233c1ee MS |
82 | if (insn_has_ll_or_sc(insn)) { |
83 | pr_notice("Kprobes for ll and sc instructions are not" | |
84 | "supported\n"); | |
85 | ret = -EINVAL; | |
86 | goto out; | |
87 | } | |
88 | ||
41dde781 MS |
89 | if ((probe_kernel_read(&prev_insn, p->addr - 1, |
90 | sizeof(mips_instruction)) == 0) && | |
91 | insn_has_delayslot(prev_insn)) { | |
92 | pr_notice("Kprobes for branch delayslot are not supported\n"); | |
c1bf207d DD |
93 | ret = -EINVAL; |
94 | goto out; | |
95 | } | |
96 | ||
d05c5130 MN |
97 | if (__insn_is_compact_branch(insn)) { |
98 | pr_notice("Kprobes for compact branches are not supported\n"); | |
99 | ret = -EINVAL; | |
100 | goto out; | |
101 | } | |
102 | ||
c1bf207d DD |
103 | /* insn: must be on special executable page on mips. */ |
104 | p->ainsn.insn = get_insn_slot(); | |
105 | if (!p->ainsn.insn) { | |
106 | ret = -ENOMEM; | |
107 | goto out; | |
108 | } | |
109 | ||
110 | /* | |
111 | * In the kprobe->ainsn.insn[] array we store the original | |
112 | * instruction at index zero and a break trap instruction at | |
113 | * index one. | |
6457a396 MS |
114 | * |
115 | * On MIPS arch if the instruction at probed address is a | |
116 | * branch instruction, we need to execute the instruction at | |
117 | * Branch Delayslot (BD) at the time of probe hit. As MIPS also | |
118 | * doesn't have single stepping support, the BD instruction can | |
119 | * not be executed in-line and it would be executed on SSOL slot | |
120 | * using a normal breakpoint instruction in the next slot. | |
121 | * So, read the instruction and save it for later execution. | |
c1bf207d | 122 | */ |
6457a396 MS |
123 | if (insn_has_delayslot(insn)) |
124 | memcpy(&p->ainsn.insn[0], p->addr + 1, sizeof(kprobe_opcode_t)); | |
125 | else | |
126 | memcpy(&p->ainsn.insn[0], p->addr, sizeof(kprobe_opcode_t)); | |
c1bf207d | 127 | |
c1bf207d DD |
128 | p->ainsn.insn[1] = breakpoint2_insn; |
129 | p->opcode = *p->addr; | |
130 | ||
131 | out: | |
132 | return ret; | |
133 | } | |
134 | ||
135 | void __kprobes arch_arm_kprobe(struct kprobe *p) | |
136 | { | |
137 | *p->addr = breakpoint_insn; | |
138 | flush_insn_slot(p); | |
139 | } | |
140 | ||
141 | void __kprobes arch_disarm_kprobe(struct kprobe *p) | |
142 | { | |
143 | *p->addr = p->opcode; | |
144 | flush_insn_slot(p); | |
145 | } | |
146 | ||
147 | void __kprobes arch_remove_kprobe(struct kprobe *p) | |
148 | { | |
22047b85 MH |
149 | if (p->ainsn.insn) { |
150 | free_insn_slot(p->ainsn.insn, 0); | |
151 | p->ainsn.insn = NULL; | |
152 | } | |
c1bf207d DD |
153 | } |
154 | ||
155 | static void save_previous_kprobe(struct kprobe_ctlblk *kcb) | |
156 | { | |
157 | kcb->prev_kprobe.kp = kprobe_running(); | |
158 | kcb->prev_kprobe.status = kcb->kprobe_status; | |
159 | kcb->prev_kprobe.old_SR = kcb->kprobe_old_SR; | |
160 | kcb->prev_kprobe.saved_SR = kcb->kprobe_saved_SR; | |
161 | kcb->prev_kprobe.saved_epc = kcb->kprobe_saved_epc; | |
162 | } | |
163 | ||
164 | static void restore_previous_kprobe(struct kprobe_ctlblk *kcb) | |
165 | { | |
35898716 | 166 | __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); |
c1bf207d DD |
167 | kcb->kprobe_status = kcb->prev_kprobe.status; |
168 | kcb->kprobe_old_SR = kcb->prev_kprobe.old_SR; | |
169 | kcb->kprobe_saved_SR = kcb->prev_kprobe.saved_SR; | |
170 | kcb->kprobe_saved_epc = kcb->prev_kprobe.saved_epc; | |
171 | } | |
172 | ||
173 | static void set_current_kprobe(struct kprobe *p, struct pt_regs *regs, | |
174 | struct kprobe_ctlblk *kcb) | |
175 | { | |
35898716 | 176 | __this_cpu_write(current_kprobe, p); |
c1bf207d DD |
177 | kcb->kprobe_saved_SR = kcb->kprobe_old_SR = (regs->cp0_status & ST0_IE); |
178 | kcb->kprobe_saved_epc = regs->cp0_epc; | |
179 | } | |
180 | ||
6457a396 MS |
181 | /** |
182 | * evaluate_branch_instrucion - | |
183 | * | |
184 | * Evaluate the branch instruction at probed address during probe hit. The | |
185 | * result of evaluation would be the updated epc. The insturction in delayslot | |
186 | * would actually be single stepped using a normal breakpoint) on SSOL slot. | |
187 | * | |
188 | * The result is also saved in the kprobe control block for later use, | |
189 | * in case we need to execute the delayslot instruction. The latter will be | |
190 | * false for NOP instruction in dealyslot and the branch-likely instructions | |
191 | * when the branch is taken. And for those cases we set a flag as | |
192 | * SKIP_DELAYSLOT in the kprobe control block | |
193 | */ | |
194 | static int evaluate_branch_instruction(struct kprobe *p, struct pt_regs *regs, | |
195 | struct kprobe_ctlblk *kcb) | |
196 | { | |
197 | union mips_instruction insn = p->opcode; | |
198 | long epc; | |
199 | int ret = 0; | |
200 | ||
201 | epc = regs->cp0_epc; | |
202 | if (epc & 3) | |
203 | goto unaligned; | |
204 | ||
205 | if (p->ainsn.insn->word == 0) | |
206 | kcb->flags |= SKIP_DELAYSLOT; | |
207 | else | |
208 | kcb->flags &= ~SKIP_DELAYSLOT; | |
209 | ||
210 | ret = __compute_return_epc_for_insn(regs, insn); | |
211 | if (ret < 0) | |
212 | return ret; | |
213 | ||
214 | if (ret == BRANCH_LIKELY_TAKEN) | |
215 | kcb->flags |= SKIP_DELAYSLOT; | |
216 | ||
217 | kcb->target_epc = regs->cp0_epc; | |
218 | ||
219 | return 0; | |
220 | ||
221 | unaligned: | |
222 | pr_notice("%s: unaligned epc - sending SIGBUS.\n", current->comm); | |
3cf5d076 | 223 | force_sig(SIGBUS); |
6457a396 MS |
224 | return -EFAULT; |
225 | ||
226 | } | |
227 | ||
228 | static void prepare_singlestep(struct kprobe *p, struct pt_regs *regs, | |
229 | struct kprobe_ctlblk *kcb) | |
c1bf207d | 230 | { |
6457a396 MS |
231 | int ret = 0; |
232 | ||
c1bf207d DD |
233 | regs->cp0_status &= ~ST0_IE; |
234 | ||
235 | /* single step inline if the instruction is a break */ | |
236 | if (p->opcode.word == breakpoint_insn.word || | |
237 | p->opcode.word == breakpoint2_insn.word) | |
238 | regs->cp0_epc = (unsigned long)p->addr; | |
6457a396 MS |
239 | else if (insn_has_delayslot(p->opcode)) { |
240 | ret = evaluate_branch_instruction(p, regs, kcb); | |
241 | if (ret < 0) { | |
242 | pr_notice("Kprobes: Error in evaluating branch\n"); | |
243 | return; | |
244 | } | |
245 | } | |
246 | regs->cp0_epc = (unsigned long)&p->ainsn.insn[0]; | |
247 | } | |
248 | ||
249 | /* | |
250 | * Called after single-stepping. p->addr is the address of the | |
251 | * instruction whose first byte has been replaced by the "break 0" | |
70342287 | 252 | * instruction. To avoid the SMP problems that can occur when we |
6457a396 MS |
253 | * temporarily put back the original opcode to single-step, we |
254 | * single-stepped a copy of the instruction. The address of this | |
255 | * copy is p->ainsn.insn. | |
256 | * | |
257 | * This function prepares to return from the post-single-step | |
258 | * breakpoint trap. In case of branch instructions, the target | |
259 | * epc to be restored. | |
260 | */ | |
261 | static void __kprobes resume_execution(struct kprobe *p, | |
262 | struct pt_regs *regs, | |
263 | struct kprobe_ctlblk *kcb) | |
264 | { | |
265 | if (insn_has_delayslot(p->opcode)) | |
266 | regs->cp0_epc = kcb->target_epc; | |
267 | else { | |
268 | unsigned long orig_epc = kcb->kprobe_saved_epc; | |
269 | regs->cp0_epc = orig_epc + 4; | |
270 | } | |
c1bf207d DD |
271 | } |
272 | ||
273 | static int __kprobes kprobe_handler(struct pt_regs *regs) | |
274 | { | |
275 | struct kprobe *p; | |
276 | int ret = 0; | |
277 | kprobe_opcode_t *addr; | |
278 | struct kprobe_ctlblk *kcb; | |
279 | ||
280 | addr = (kprobe_opcode_t *) regs->cp0_epc; | |
281 | ||
282 | /* | |
283 | * We don't want to be preempted for the entire | |
284 | * duration of kprobe processing | |
285 | */ | |
286 | preempt_disable(); | |
287 | kcb = get_kprobe_ctlblk(); | |
288 | ||
289 | /* Check we're not actually recursing */ | |
290 | if (kprobe_running()) { | |
291 | p = get_kprobe(addr); | |
292 | if (p) { | |
293 | if (kcb->kprobe_status == KPROBE_HIT_SS && | |
294 | p->ainsn.insn->word == breakpoint_insn.word) { | |
295 | regs->cp0_status &= ~ST0_IE; | |
296 | regs->cp0_status |= kcb->kprobe_saved_SR; | |
297 | goto no_kprobe; | |
298 | } | |
299 | /* | |
300 | * We have reentered the kprobe_handler(), since | |
301 | * another probe was hit while within the handler. | |
302 | * We here save the original kprobes variables and | |
303 | * just single step on the instruction of the new probe | |
304 | * without calling any user handlers. | |
305 | */ | |
306 | save_previous_kprobe(kcb); | |
307 | set_current_kprobe(p, regs, kcb); | |
308 | kprobes_inc_nmissed_count(p); | |
6457a396 | 309 | prepare_singlestep(p, regs, kcb); |
c1bf207d | 310 | kcb->kprobe_status = KPROBE_REENTER; |
6457a396 MS |
311 | if (kcb->flags & SKIP_DELAYSLOT) { |
312 | resume_execution(p, regs, kcb); | |
313 | restore_previous_kprobe(kcb); | |
314 | preempt_enable_no_resched(); | |
315 | } | |
c1bf207d | 316 | return 1; |
9b85753d MH |
317 | } else if (addr->word != breakpoint_insn.word) { |
318 | /* | |
319 | * The breakpoint instruction was removed by | |
320 | * another cpu right after we hit, no further | |
321 | * handling of this interrupt is appropriate | |
322 | */ | |
323 | ret = 1; | |
c1bf207d DD |
324 | } |
325 | goto no_kprobe; | |
326 | } | |
327 | ||
328 | p = get_kprobe(addr); | |
329 | if (!p) { | |
330 | if (addr->word != breakpoint_insn.word) { | |
331 | /* | |
332 | * The breakpoint instruction was removed right | |
333 | * after we hit it. Another cpu has removed | |
334 | * either a probepoint or a debugger breakpoint | |
335 | * at this address. In either case, no further | |
336 | * handling of this interrupt is appropriate. | |
337 | */ | |
338 | ret = 1; | |
339 | } | |
340 | /* Not one of ours: let kernel handle it */ | |
341 | goto no_kprobe; | |
342 | } | |
343 | ||
344 | set_current_kprobe(p, regs, kcb); | |
345 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; | |
346 | ||
347 | if (p->pre_handler && p->pre_handler(p, regs)) { | |
348 | /* handler has already set things up, so skip ss setup */ | |
cce188bd MH |
349 | reset_current_kprobe(); |
350 | preempt_enable_no_resched(); | |
c1bf207d DD |
351 | return 1; |
352 | } | |
353 | ||
6457a396 MS |
354 | prepare_singlestep(p, regs, kcb); |
355 | if (kcb->flags & SKIP_DELAYSLOT) { | |
356 | kcb->kprobe_status = KPROBE_HIT_SSDONE; | |
357 | if (p->post_handler) | |
358 | p->post_handler(p, regs, 0); | |
359 | resume_execution(p, regs, kcb); | |
360 | preempt_enable_no_resched(); | |
361 | } else | |
362 | kcb->kprobe_status = KPROBE_HIT_SS; | |
363 | ||
c1bf207d DD |
364 | return 1; |
365 | ||
366 | no_kprobe: | |
367 | preempt_enable_no_resched(); | |
368 | return ret; | |
369 | ||
370 | } | |
371 | ||
c1bf207d DD |
372 | static inline int post_kprobe_handler(struct pt_regs *regs) |
373 | { | |
374 | struct kprobe *cur = kprobe_running(); | |
375 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); | |
376 | ||
377 | if (!cur) | |
378 | return 0; | |
379 | ||
380 | if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { | |
381 | kcb->kprobe_status = KPROBE_HIT_SSDONE; | |
382 | cur->post_handler(cur, regs, 0); | |
383 | } | |
384 | ||
385 | resume_execution(cur, regs, kcb); | |
386 | ||
387 | regs->cp0_status |= kcb->kprobe_saved_SR; | |
388 | ||
389 | /* Restore back the original saved kprobes variables and continue. */ | |
390 | if (kcb->kprobe_status == KPROBE_REENTER) { | |
391 | restore_previous_kprobe(kcb); | |
392 | goto out; | |
393 | } | |
394 | reset_current_kprobe(); | |
395 | out: | |
396 | preempt_enable_no_resched(); | |
397 | ||
398 | return 1; | |
399 | } | |
400 | ||
b98cca44 | 401 | int kprobe_fault_handler(struct pt_regs *regs, int trapnr) |
c1bf207d DD |
402 | { |
403 | struct kprobe *cur = kprobe_running(); | |
404 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); | |
405 | ||
406 | if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) | |
407 | return 1; | |
408 | ||
409 | if (kcb->kprobe_status & KPROBE_HIT_SS) { | |
410 | resume_execution(cur, regs, kcb); | |
411 | regs->cp0_status |= kcb->kprobe_old_SR; | |
412 | ||
413 | reset_current_kprobe(); | |
414 | preempt_enable_no_resched(); | |
415 | } | |
416 | return 0; | |
417 | } | |
418 | ||
419 | /* | |
420 | * Wrapper routine for handling exceptions. | |
421 | */ | |
422 | int __kprobes kprobe_exceptions_notify(struct notifier_block *self, | |
423 | unsigned long val, void *data) | |
424 | { | |
425 | ||
426 | struct die_args *args = (struct die_args *)data; | |
427 | int ret = NOTIFY_DONE; | |
428 | ||
429 | switch (val) { | |
430 | case DIE_BREAK: | |
431 | if (kprobe_handler(args->regs)) | |
432 | ret = NOTIFY_STOP; | |
433 | break; | |
434 | case DIE_SSTEPBP: | |
435 | if (post_kprobe_handler(args->regs)) | |
436 | ret = NOTIFY_STOP; | |
437 | break; | |
438 | ||
439 | case DIE_PAGE_FAULT: | |
440 | /* kprobe_running() needs smp_processor_id() */ | |
441 | preempt_disable(); | |
442 | ||
443 | if (kprobe_running() | |
444 | && kprobe_fault_handler(args->regs, args->trapnr)) | |
445 | ret = NOTIFY_STOP; | |
446 | preempt_enable(); | |
447 | break; | |
448 | default: | |
449 | break; | |
450 | } | |
451 | return ret; | |
452 | } | |
453 | ||
c1bf207d DD |
454 | /* |
455 | * Function return probe trampoline: | |
456 | * - init_kprobes() establishes a probepoint here | |
457 | * - When the probed function returns, this probe causes the | |
458 | * handlers to fire | |
459 | */ | |
460 | static void __used kretprobe_trampoline_holder(void) | |
461 | { | |
462 | asm volatile( | |
463 | ".set push\n\t" | |
464 | /* Keep the assembler from reordering and placing JR here. */ | |
465 | ".set noreorder\n\t" | |
466 | "nop\n\t" | |
467 | ".global kretprobe_trampoline\n" | |
468 | "kretprobe_trampoline:\n\t" | |
469 | "nop\n\t" | |
470 | ".set pop" | |
471 | : : : "memory"); | |
472 | } | |
473 | ||
474 | void kretprobe_trampoline(void); | |
475 | ||
476 | void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, | |
477 | struct pt_regs *regs) | |
478 | { | |
479 | ri->ret_addr = (kprobe_opcode_t *) regs->regs[31]; | |
480 | ||
481 | /* Replace the return addr with trampoline addr */ | |
482 | regs->regs[31] = (unsigned long)kretprobe_trampoline; | |
483 | } | |
484 | ||
485 | /* | |
486 | * Called when the probe at kretprobe trampoline is hit | |
487 | */ | |
488 | static int __kprobes trampoline_probe_handler(struct kprobe *p, | |
489 | struct pt_regs *regs) | |
490 | { | |
491 | struct kretprobe_instance *ri = NULL; | |
492 | struct hlist_head *head, empty_rp; | |
b67bfe0d | 493 | struct hlist_node *tmp; |
c1bf207d DD |
494 | unsigned long flags, orig_ret_address = 0; |
495 | unsigned long trampoline_address = (unsigned long)kretprobe_trampoline; | |
496 | ||
497 | INIT_HLIST_HEAD(&empty_rp); | |
498 | kretprobe_hash_lock(current, &head, &flags); | |
499 | ||
500 | /* | |
501 | * It is possible to have multiple instances associated with a given | |
502 | * task either because an multiple functions in the call path | |
503 | * have a return probe installed on them, and/or more than one return | |
504 | * return probe was registered for a target function. | |
505 | * | |
506 | * We can handle this because: | |
507 | * - instances are always inserted at the head of the list | |
508 | * - when multiple return probes are registered for the same | |
70342287 RB |
509 | * function, the first instance's ret_addr will point to the |
510 | * real return address, and all the rest will point to | |
511 | * kretprobe_trampoline | |
c1bf207d | 512 | */ |
b67bfe0d | 513 | hlist_for_each_entry_safe(ri, tmp, head, hlist) { |
c1bf207d DD |
514 | if (ri->task != current) |
515 | /* another task is sharing our hash bucket */ | |
516 | continue; | |
517 | ||
518 | if (ri->rp && ri->rp->handler) | |
519 | ri->rp->handler(ri, regs); | |
520 | ||
521 | orig_ret_address = (unsigned long)ri->ret_addr; | |
522 | recycle_rp_inst(ri, &empty_rp); | |
523 | ||
524 | if (orig_ret_address != trampoline_address) | |
525 | /* | |
526 | * This is the real return address. Any other | |
527 | * instances associated with this task are for | |
528 | * other calls deeper on the call stack | |
529 | */ | |
530 | break; | |
531 | } | |
532 | ||
533 | kretprobe_assert(ri, orig_ret_address, trampoline_address); | |
534 | instruction_pointer(regs) = orig_ret_address; | |
535 | ||
c1bf207d | 536 | kretprobe_hash_unlock(current, &flags); |
c1bf207d | 537 | |
b67bfe0d | 538 | hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { |
c1bf207d DD |
539 | hlist_del(&ri->hlist); |
540 | kfree(ri); | |
541 | } | |
542 | /* | |
543 | * By returning a non-zero value, we are telling | |
544 | * kprobe_handler() that we don't want the post_handler | |
545 | * to run (and have re-enabled preemption) | |
546 | */ | |
547 | return 1; | |
548 | } | |
549 | ||
550 | int __kprobes arch_trampoline_kprobe(struct kprobe *p) | |
551 | { | |
552 | if (p->addr == (kprobe_opcode_t *)kretprobe_trampoline) | |
553 | return 1; | |
554 | ||
555 | return 0; | |
556 | } | |
557 | ||
558 | static struct kprobe trampoline_p = { | |
559 | .addr = (kprobe_opcode_t *)kretprobe_trampoline, | |
560 | .pre_handler = trampoline_probe_handler | |
561 | }; | |
562 | ||
563 | int __init arch_init_kprobes(void) | |
564 | { | |
565 | return register_kprobe(&trampoline_p); | |
566 | } |