1 // SPDX-License-Identifier: GPL-2.0
2 /* arch/sparc64/kernel/kprobes.c
4 * Copyright (C) 2004 David S. Miller <davem@davemloft.net>
7 #include <linux/kernel.h>
8 #include <linux/kprobes.h>
9 #include <linux/extable.h>
10 #include <linux/kdebug.h>
11 #include <linux/slab.h>
12 #include <linux/context_tracking.h>
13 #include <asm/signal.h>
14 #include <asm/cacheflush.h>
15 #include <linux/uaccess.h>
17 /* We do not have hardware single-stepping on sparc64.
18 * So we implement software single-stepping with breakpoint
19 * traps. The top-level scheme is similar to that used
20 * in the x86 kprobes implementation.
22 * In the kprobe->ainsn.insn[] array we store the original
23 * instruction at index zero and a break instruction at
26 * When we hit a kprobe we:
27 * - Run the pre-handler
28 * - Remember "regs->tnpc" and interrupt level stored in
29 * "regs->tstate" so we can restore them later
30 * - Disable PIL interrupts
31 * - Set regs->tpc to point to kprobe->ainsn.insn[0]
32 * - Set regs->tnpc to point to kprobe->ainsn.insn[1]
33 * - Mark that we are actively in a kprobe
35 * At this point we wait for the second breakpoint at
36 * kprobe->ainsn.insn[1] to hit. When it does we:
37 * - Run the post-handler
38 * - Set regs->tpc to "remembered" regs->tnpc stored above,
39 * restore the PIL interrupt level in "regs->tstate" as well
40 * - Make any adjustments necessary to regs->tnpc in order
41 * to handle relative branches correctly. See below.
42 * - Mark that we are no longer actively in a kprobe.
45 DEFINE_PER_CPU(struct kprobe
*, current_kprobe
) = NULL
;
46 DEFINE_PER_CPU(struct kprobe_ctlblk
, kprobe_ctlblk
);
48 struct kretprobe_blackpoint kretprobe_blacklist
[] = {{NULL
, NULL
}};
50 int __kprobes
arch_prepare_kprobe(struct kprobe
*p
)
52 if ((unsigned long) p
->addr
& 0x3UL
)
55 p
->ainsn
.insn
[0] = *p
->addr
;
56 flushi(&p
->ainsn
.insn
[0]);
58 p
->ainsn
.insn
[1] = BREAKPOINT_INSTRUCTION_2
;
59 flushi(&p
->ainsn
.insn
[1]);
65 void __kprobes
arch_arm_kprobe(struct kprobe
*p
)
67 *p
->addr
= BREAKPOINT_INSTRUCTION
;
71 void __kprobes
arch_disarm_kprobe(struct kprobe
*p
)
77 static void __kprobes
save_previous_kprobe(struct kprobe_ctlblk
*kcb
)
79 kcb
->prev_kprobe
.kp
= kprobe_running();
80 kcb
->prev_kprobe
.status
= kcb
->kprobe_status
;
81 kcb
->prev_kprobe
.orig_tnpc
= kcb
->kprobe_orig_tnpc
;
82 kcb
->prev_kprobe
.orig_tstate_pil
= kcb
->kprobe_orig_tstate_pil
;
85 static void __kprobes
restore_previous_kprobe(struct kprobe_ctlblk
*kcb
)
87 __this_cpu_write(current_kprobe
, kcb
->prev_kprobe
.kp
);
88 kcb
->kprobe_status
= kcb
->prev_kprobe
.status
;
89 kcb
->kprobe_orig_tnpc
= kcb
->prev_kprobe
.orig_tnpc
;
90 kcb
->kprobe_orig_tstate_pil
= kcb
->prev_kprobe
.orig_tstate_pil
;
93 static void __kprobes
set_current_kprobe(struct kprobe
*p
, struct pt_regs
*regs
,
94 struct kprobe_ctlblk
*kcb
)
96 __this_cpu_write(current_kprobe
, p
);
97 kcb
->kprobe_orig_tnpc
= regs
->tnpc
;
98 kcb
->kprobe_orig_tstate_pil
= (regs
->tstate
& TSTATE_PIL
);
101 static void __kprobes
prepare_singlestep(struct kprobe
*p
, struct pt_regs
*regs
,
102 struct kprobe_ctlblk
*kcb
)
104 regs
->tstate
|= TSTATE_PIL
;
106 /*single step inline, if it a breakpoint instruction*/
107 if (p
->opcode
== BREAKPOINT_INSTRUCTION
) {
108 regs
->tpc
= (unsigned long) p
->addr
;
109 regs
->tnpc
= kcb
->kprobe_orig_tnpc
;
111 regs
->tpc
= (unsigned long) &p
->ainsn
.insn
[0];
112 regs
->tnpc
= (unsigned long) &p
->ainsn
.insn
[1];
116 static int __kprobes
kprobe_handler(struct pt_regs
*regs
)
119 void *addr
= (void *) regs
->tpc
;
121 struct kprobe_ctlblk
*kcb
;
124 * We don't want to be preempted for the entire
125 * duration of kprobe processing
128 kcb
= get_kprobe_ctlblk();
130 if (kprobe_running()) {
131 p
= get_kprobe(addr
);
133 if (kcb
->kprobe_status
== KPROBE_HIT_SS
) {
134 regs
->tstate
= ((regs
->tstate
& ~TSTATE_PIL
) |
135 kcb
->kprobe_orig_tstate_pil
);
138 /* We have reentered the kprobe_handler(), since
139 * another probe was hit while within the handler.
140 * We here save the original kprobes variables and
141 * just single step on the instruction of the new probe
142 * without calling any user handlers.
144 save_previous_kprobe(kcb
);
145 set_current_kprobe(p
, regs
, kcb
);
146 kprobes_inc_nmissed_count(p
);
147 kcb
->kprobe_status
= KPROBE_REENTER
;
148 prepare_singlestep(p
, regs
, kcb
);
150 } else if (*(u32
*)addr
!= BREAKPOINT_INSTRUCTION
) {
151 /* The breakpoint instruction was removed by
152 * another cpu right after we hit, no further
153 * handling of this interrupt is appropriate
160 p
= get_kprobe(addr
);
162 if (*(u32
*)addr
!= BREAKPOINT_INSTRUCTION
) {
164 * The breakpoint instruction was removed right
165 * after we hit it. Another cpu has removed
166 * either a probepoint or a debugger breakpoint
167 * at this address. In either case, no further
168 * handling of this interrupt is appropriate.
172 /* Not one of ours: let kernel handle it */
176 set_current_kprobe(p
, regs
, kcb
);
177 kcb
->kprobe_status
= KPROBE_HIT_ACTIVE
;
178 if (p
->pre_handler
&& p
->pre_handler(p
, regs
))
181 prepare_singlestep(p
, regs
, kcb
);
182 kcb
->kprobe_status
= KPROBE_HIT_SS
;
186 preempt_enable_no_resched();
190 /* If INSN is a relative control transfer instruction,
191 * return the corrected branch destination value.
193 * regs->tpc and regs->tnpc still hold the values of the
194 * program counters at the time of trap due to the execution
195 * of the BREAKPOINT_INSTRUCTION_2 at p->ainsn.insn[1]
198 static unsigned long __kprobes
relbranch_fixup(u32 insn
, struct kprobe
*p
,
199 struct pt_regs
*regs
)
201 unsigned long real_pc
= (unsigned long) p
->addr
;
203 /* Branch not taken, no mods necessary. */
204 if (regs
->tnpc
== regs
->tpc
+ 0x4UL
)
205 return real_pc
+ 0x8UL
;
207 /* The three cases are call, branch w/prediction,
208 * and traditional branch.
210 if ((insn
& 0xc0000000) == 0x40000000 ||
211 (insn
& 0xc1c00000) == 0x00400000 ||
212 (insn
& 0xc1c00000) == 0x00800000) {
213 unsigned long ainsn_addr
;
215 ainsn_addr
= (unsigned long) &p
->ainsn
.insn
[0];
217 /* The instruction did all the work for us
218 * already, just apply the offset to the correct
219 * instruction location.
221 return (real_pc
+ (regs
->tnpc
- ainsn_addr
));
224 /* It is jmpl or some other absolute PC modification instruction,
230 /* If INSN is an instruction which writes it's PC location
231 * into a destination register, fix that up.
233 static void __kprobes
retpc_fixup(struct pt_regs
*regs
, u32 insn
,
234 unsigned long real_pc
)
236 unsigned long *slot
= NULL
;
238 /* Simplest case is 'call', which always uses %o7 */
239 if ((insn
& 0xc0000000) == 0x40000000) {
240 slot
= ®s
->u_regs
[UREG_I7
];
243 /* 'jmpl' encodes the register inside of the opcode */
244 if ((insn
& 0xc1f80000) == 0x81c00000) {
245 unsigned long rd
= ((insn
>> 25) & 0x1f);
248 slot
= ®s
->u_regs
[rd
];
250 /* Hard case, it goes onto the stack. */
254 slot
= (unsigned long *)
255 (regs
->u_regs
[UREG_FP
] + STACK_BIAS
);
264 * Called after single-stepping. p->addr is the address of the
265 * instruction which has been replaced by the breakpoint
266 * instruction. To avoid the SMP problems that can occur when we
267 * temporarily put back the original opcode to single-step, we
268 * single-stepped a copy of the instruction. The address of this
269 * copy is &p->ainsn.insn[0].
271 * This function prepares to return from the post-single-step
274 static void __kprobes
resume_execution(struct kprobe
*p
,
275 struct pt_regs
*regs
, struct kprobe_ctlblk
*kcb
)
277 u32 insn
= p
->ainsn
.insn
[0];
279 regs
->tnpc
= relbranch_fixup(insn
, p
, regs
);
281 /* This assignment must occur after relbranch_fixup() */
282 regs
->tpc
= kcb
->kprobe_orig_tnpc
;
284 retpc_fixup(regs
, insn
, (unsigned long) p
->addr
);
286 regs
->tstate
= ((regs
->tstate
& ~TSTATE_PIL
) |
287 kcb
->kprobe_orig_tstate_pil
);
290 static int __kprobes
post_kprobe_handler(struct pt_regs
*regs
)
292 struct kprobe
*cur
= kprobe_running();
293 struct kprobe_ctlblk
*kcb
= get_kprobe_ctlblk();
298 if ((kcb
->kprobe_status
!= KPROBE_REENTER
) && cur
->post_handler
) {
299 kcb
->kprobe_status
= KPROBE_HIT_SSDONE
;
300 cur
->post_handler(cur
, regs
, 0);
303 resume_execution(cur
, regs
, kcb
);
305 /*Restore back the original saved kprobes variables and continue. */
306 if (kcb
->kprobe_status
== KPROBE_REENTER
) {
307 restore_previous_kprobe(kcb
);
310 reset_current_kprobe();
312 preempt_enable_no_resched();
317 int __kprobes
kprobe_fault_handler(struct pt_regs
*regs
, int trapnr
)
319 struct kprobe
*cur
= kprobe_running();
320 struct kprobe_ctlblk
*kcb
= get_kprobe_ctlblk();
321 const struct exception_table_entry
*entry
;
323 switch(kcb
->kprobe_status
) {
327 * We are here because the instruction being single
328 * stepped caused a page fault. We reset the current
329 * kprobe and the tpc points back to the probe address
330 * and allow the page fault handler to continue as a
333 regs
->tpc
= (unsigned long)cur
->addr
;
334 regs
->tnpc
= kcb
->kprobe_orig_tnpc
;
335 regs
->tstate
= ((regs
->tstate
& ~TSTATE_PIL
) |
336 kcb
->kprobe_orig_tstate_pil
);
337 if (kcb
->kprobe_status
== KPROBE_REENTER
)
338 restore_previous_kprobe(kcb
);
340 reset_current_kprobe();
341 preempt_enable_no_resched();
343 case KPROBE_HIT_ACTIVE
:
344 case KPROBE_HIT_SSDONE
:
346 * We increment the nmissed count for accounting,
347 * we can also use npre/npostfault count for accounting
348 * these specific fault cases.
350 kprobes_inc_nmissed_count(cur
);
353 * We come here because instructions in the pre/post
354 * handler caused the page_fault, this could happen
355 * if handler tries to access user space by
356 * copy_from_user(), get_user() etc. Let the
357 * user-specified handler try to fix it first.
359 if (cur
->fault_handler
&& cur
->fault_handler(cur
, regs
, trapnr
))
363 * In case the user-specified fault handler returned
364 * zero, try to fix up.
367 entry
= search_exception_tables(regs
->tpc
);
369 regs
->tpc
= entry
->fixup
;
370 regs
->tnpc
= regs
->tpc
+ 4;
375 * fixup_exception() could not handle it,
376 * Let do_page_fault() fix it.
387 * Wrapper routine to for handling exceptions.
389 int __kprobes
kprobe_exceptions_notify(struct notifier_block
*self
,
390 unsigned long val
, void *data
)
392 struct die_args
*args
= (struct die_args
*)data
;
393 int ret
= NOTIFY_DONE
;
395 if (args
->regs
&& user_mode(args
->regs
))
400 if (kprobe_handler(args
->regs
))
404 if (post_kprobe_handler(args
->regs
))
413 asmlinkage
void __kprobes
kprobe_trap(unsigned long trap_level
,
414 struct pt_regs
*regs
)
416 enum ctx_state prev_state
= exception_enter();
418 BUG_ON(trap_level
!= 0x170 && trap_level
!= 0x171);
420 if (user_mode(regs
)) {
422 bad_trap(regs
, trap_level
);
426 /* trap_level == 0x170 --> ta 0x70
427 * trap_level == 0x171 --> ta 0x71
429 if (notify_die((trap_level
== 0x170) ? DIE_DEBUG
: DIE_DEBUG_2
,
430 (trap_level
== 0x170) ? "debug" : "debug_2",
431 regs
, 0, trap_level
, SIGTRAP
) != NOTIFY_STOP
)
432 bad_trap(regs
, trap_level
);
434 exception_exit(prev_state
);
437 /* The value stored in the return address register is actually 2
438 * instructions before where the callee will return to.
439 * Sequences usually look something like this
441 * call some_function <--- return register points here
442 * nop <--- call delay slot
443 * whatever <--- where callee returns to
445 * To keep trampoline_probe_handler logic simpler, we normalize the
446 * value kept in ri->ret_addr so we don't need to keep adjusting it
449 void __kprobes
arch_prepare_kretprobe(struct kretprobe_instance
*ri
,
450 struct pt_regs
*regs
)
452 ri
->ret_addr
= (kprobe_opcode_t
*)(regs
->u_regs
[UREG_RETPC
] + 8);
454 /* Replace the return addr with trampoline addr */
455 regs
->u_regs
[UREG_RETPC
] =
456 ((unsigned long)kretprobe_trampoline
) - 8;
460 * Called when the probe at kretprobe trampoline is hit
462 static int __kprobes
trampoline_probe_handler(struct kprobe
*p
,
463 struct pt_regs
*regs
)
465 struct kretprobe_instance
*ri
= NULL
;
466 struct hlist_head
*head
, empty_rp
;
467 struct hlist_node
*tmp
;
468 unsigned long flags
, orig_ret_address
= 0;
469 unsigned long trampoline_address
=(unsigned long)&kretprobe_trampoline
;
471 INIT_HLIST_HEAD(&empty_rp
);
472 kretprobe_hash_lock(current
, &head
, &flags
);
475 * It is possible to have multiple instances associated with a given
476 * task either because an multiple functions in the call path
477 * have a return probe installed on them, and/or more than one return
478 * return probe was registered for a target function.
480 * We can handle this because:
481 * - instances are always inserted at the head of the list
482 * - when multiple return probes are registered for the same
483 * function, the first instance's ret_addr will point to the
484 * real return address, and all the rest will point to
485 * kretprobe_trampoline
487 hlist_for_each_entry_safe(ri
, tmp
, head
, hlist
) {
488 if (ri
->task
!= current
)
489 /* another task is sharing our hash bucket */
492 if (ri
->rp
&& ri
->rp
->handler
)
493 ri
->rp
->handler(ri
, regs
);
495 orig_ret_address
= (unsigned long)ri
->ret_addr
;
496 recycle_rp_inst(ri
, &empty_rp
);
498 if (orig_ret_address
!= trampoline_address
)
500 * This is the real return address. Any other
501 * instances associated with this task are for
502 * other calls deeper on the call stack
507 kretprobe_assert(ri
, orig_ret_address
, trampoline_address
);
508 regs
->tpc
= orig_ret_address
;
509 regs
->tnpc
= orig_ret_address
+ 4;
511 reset_current_kprobe();
512 kretprobe_hash_unlock(current
, &flags
);
513 preempt_enable_no_resched();
515 hlist_for_each_entry_safe(ri
, tmp
, &empty_rp
, hlist
) {
516 hlist_del(&ri
->hlist
);
520 * By returning a non-zero value, we are telling
521 * kprobe_handler() that we don't want the post_handler
522 * to run (and have re-enabled preemption)
527 static void __used
kretprobe_trampoline_holder(void)
529 asm volatile(".global kretprobe_trampoline\n"
530 "kretprobe_trampoline:\n"
534 static struct kprobe trampoline_p
= {
535 .addr
= (kprobe_opcode_t
*) &kretprobe_trampoline
,
536 .pre_handler
= trampoline_probe_handler
539 int __init
arch_init_kprobes(void)
541 return register_kprobe(&trampoline_p
);
544 int __kprobes
arch_trampoline_kprobe(struct kprobe
*p
)
546 if (p
->addr
== (kprobe_opcode_t
*)&kretprobe_trampoline
)