]>
git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - arch/arm64/kernel/probes/kprobes.c
2 * arch/arm64/kernel/probes/kprobes.c
4 * Kprobes support for ARM64
6 * Copyright (C) 2013 Linaro Limited.
7 * Author: Sandeepa Prabhu <sandeepa.prabhu@linaro.org>
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
19 #include <linux/kasan.h>
20 #include <linux/kernel.h>
21 #include <linux/kprobes.h>
22 #include <linux/module.h>
23 #include <linux/slab.h>
24 #include <linux/stop_machine.h>
25 #include <linux/stringify.h>
26 #include <asm/traps.h>
27 #include <asm/ptrace.h>
28 #include <asm/cacheflush.h>
29 #include <asm/debug-monitors.h>
30 #include <asm/system_misc.h>
32 #include <asm/uaccess.h>
34 #include <asm-generic/sections.h>
36 #include "decode-insn.h"
38 DEFINE_PER_CPU(struct kprobe
*, current_kprobe
) = NULL
;
39 DEFINE_PER_CPU(struct kprobe_ctlblk
, kprobe_ctlblk
);
42 post_kprobe_handler(struct kprobe_ctlblk
*, struct pt_regs
*);
44 static inline unsigned long min_stack_size(unsigned long addr
)
48 if (on_irq_stack(addr
, raw_smp_processor_id()))
49 size
= IRQ_STACK_PTR(raw_smp_processor_id()) - addr
;
51 size
= (unsigned long)current_thread_info() + THREAD_START_SP
- addr
;
53 return min(size
, FIELD_SIZEOF(struct kprobe_ctlblk
, jprobes_stack
));
56 static void __kprobes
arch_prepare_ss_slot(struct kprobe
*p
)
58 /* prepare insn slot */
59 p
->ainsn
.insn
[0] = cpu_to_le32(p
->opcode
);
61 flush_icache_range((uintptr_t) (p
->ainsn
.insn
),
62 (uintptr_t) (p
->ainsn
.insn
) +
63 MAX_INSN_SIZE
* sizeof(kprobe_opcode_t
));
66 * Needs restoring of return address after stepping xol.
68 p
->ainsn
.restore
= (unsigned long) p
->addr
+
69 sizeof(kprobe_opcode_t
);
72 static void __kprobes
arch_prepare_simulate(struct kprobe
*p
)
74 /* This instructions is not executed xol. No need to adjust the PC */
78 static void __kprobes
arch_simulate_insn(struct kprobe
*p
, struct pt_regs
*regs
)
80 struct kprobe_ctlblk
*kcb
= get_kprobe_ctlblk();
83 p
->ainsn
.handler((u32
)p
->opcode
, (long)p
->addr
, regs
);
85 /* single step simulated, now go for post processing */
86 post_kprobe_handler(kcb
, regs
);
89 int __kprobes
arch_prepare_kprobe(struct kprobe
*p
)
91 unsigned long probe_addr
= (unsigned long)p
->addr
;
92 extern char __start_rodata
[];
93 extern char __end_rodata
[];
98 /* copy instruction */
99 p
->opcode
= le32_to_cpu(*p
->addr
);
101 if (in_exception_text(probe_addr
))
103 if (probe_addr
>= (unsigned long) __start_rodata
&&
104 probe_addr
<= (unsigned long) __end_rodata
)
107 /* decode instruction */
108 switch (arm_kprobe_decode_insn(p
->addr
, &p
->ainsn
)) {
109 case INSN_REJECTED
: /* insn not supported */
112 case INSN_GOOD_NO_SLOT
: /* insn need simulation */
113 p
->ainsn
.insn
= NULL
;
116 case INSN_GOOD
: /* instruction uses slot */
117 p
->ainsn
.insn
= get_insn_slot();
123 /* prepare the instruction */
125 arch_prepare_ss_slot(p
);
127 arch_prepare_simulate(p
);
132 static int __kprobes
patch_text(kprobe_opcode_t
*addr
, u32 opcode
)
137 addrs
[0] = (void *)addr
;
138 insns
[0] = (u32
)opcode
;
140 return aarch64_insn_patch_text(addrs
, insns
, 1);
143 /* arm kprobe: install breakpoint in text */
144 void __kprobes
arch_arm_kprobe(struct kprobe
*p
)
146 patch_text(p
->addr
, BRK64_OPCODE_KPROBES
);
149 /* disarm kprobe: remove breakpoint from text */
150 void __kprobes
arch_disarm_kprobe(struct kprobe
*p
)
152 patch_text(p
->addr
, p
->opcode
);
155 void __kprobes
arch_remove_kprobe(struct kprobe
*p
)
158 free_insn_slot(p
->ainsn
.insn
, 0);
159 p
->ainsn
.insn
= NULL
;
163 static void __kprobes
save_previous_kprobe(struct kprobe_ctlblk
*kcb
)
165 kcb
->prev_kprobe
.kp
= kprobe_running();
166 kcb
->prev_kprobe
.status
= kcb
->kprobe_status
;
169 static void __kprobes
restore_previous_kprobe(struct kprobe_ctlblk
*kcb
)
171 __this_cpu_write(current_kprobe
, kcb
->prev_kprobe
.kp
);
172 kcb
->kprobe_status
= kcb
->prev_kprobe
.status
;
175 static void __kprobes
set_current_kprobe(struct kprobe
*p
)
177 __this_cpu_write(current_kprobe
, p
);
181 * The D-flag (Debug mask) is set (masked) upon debug exception entry.
182 * Kprobes needs to clear (unmask) D-flag -ONLY- in case of recursive
183 * probe i.e. when probe hit from kprobe handler context upon
184 * executing the pre/post handlers. In this case we return with
185 * D-flag clear so that single-stepping can be carried-out.
187 * Leave D-flag set in all other cases.
189 static void __kprobes
190 spsr_set_debug_flag(struct pt_regs
*regs
, int mask
)
192 unsigned long spsr
= regs
->pstate
;
203 * Interrupts need to be disabled before single-step mode is set, and not
204 * reenabled until after single-step mode ends.
205 * Without disabling interrupt on local CPU, there is a chance of
206 * interrupt occurrence in the period of exception return and start of
207 * out-of-line single-step, that result in wrongly single stepping
208 * into the interrupt handler.
210 static void __kprobes
kprobes_save_local_irqflag(struct kprobe_ctlblk
*kcb
,
211 struct pt_regs
*regs
)
213 kcb
->saved_irqflag
= regs
->pstate
;
214 regs
->pstate
|= PSR_I_BIT
;
217 static void __kprobes
kprobes_restore_local_irqflag(struct kprobe_ctlblk
*kcb
,
218 struct pt_regs
*regs
)
220 if (kcb
->saved_irqflag
& PSR_I_BIT
)
221 regs
->pstate
|= PSR_I_BIT
;
223 regs
->pstate
&= ~PSR_I_BIT
;
226 static void __kprobes
227 set_ss_context(struct kprobe_ctlblk
*kcb
, unsigned long addr
)
229 kcb
->ss_ctx
.ss_pending
= true;
230 kcb
->ss_ctx
.match_addr
= addr
+ sizeof(kprobe_opcode_t
);
233 static void __kprobes
clear_ss_context(struct kprobe_ctlblk
*kcb
)
235 kcb
->ss_ctx
.ss_pending
= false;
236 kcb
->ss_ctx
.match_addr
= 0;
239 static void __kprobes
setup_singlestep(struct kprobe
*p
,
240 struct pt_regs
*regs
,
241 struct kprobe_ctlblk
*kcb
, int reenter
)
246 save_previous_kprobe(kcb
);
247 set_current_kprobe(p
);
248 kcb
->kprobe_status
= KPROBE_REENTER
;
250 kcb
->kprobe_status
= KPROBE_HIT_SS
;
255 /* prepare for single stepping */
256 slot
= (unsigned long)p
->ainsn
.insn
;
258 set_ss_context(kcb
, slot
); /* mark pending ss */
260 if (kcb
->kprobe_status
== KPROBE_REENTER
)
261 spsr_set_debug_flag(regs
, 0);
263 WARN_ON(regs
->pstate
& PSR_D_BIT
);
265 /* IRQs and single stepping do not mix well. */
266 kprobes_save_local_irqflag(kcb
, regs
);
267 kernel_enable_single_step(regs
);
268 instruction_pointer_set(regs
, slot
);
270 /* insn simulation */
271 arch_simulate_insn(p
, regs
);
275 static int __kprobes
reenter_kprobe(struct kprobe
*p
,
276 struct pt_regs
*regs
,
277 struct kprobe_ctlblk
*kcb
)
279 switch (kcb
->kprobe_status
) {
280 case KPROBE_HIT_SSDONE
:
281 case KPROBE_HIT_ACTIVE
:
282 kprobes_inc_nmissed_count(p
);
283 setup_singlestep(p
, regs
, kcb
, 1);
287 pr_warn("Unrecoverable kprobe detected at %p.\n", p
->addr
);
299 static void __kprobes
300 post_kprobe_handler(struct kprobe_ctlblk
*kcb
, struct pt_regs
*regs
)
302 struct kprobe
*cur
= kprobe_running();
307 /* return addr restore if non-branching insn */
308 if (cur
->ainsn
.restore
!= 0)
309 instruction_pointer_set(regs
, cur
->ainsn
.restore
);
311 /* restore back original saved kprobe variables and continue */
312 if (kcb
->kprobe_status
== KPROBE_REENTER
) {
313 restore_previous_kprobe(kcb
);
316 /* call post handler */
317 kcb
->kprobe_status
= KPROBE_HIT_SSDONE
;
318 if (cur
->post_handler
) {
319 /* post_handler can hit breakpoint and single step
320 * again, so we enable D-flag for recursive exception.
322 cur
->post_handler(cur
, regs
, 0);
325 reset_current_kprobe();
328 int __kprobes
kprobe_fault_handler(struct pt_regs
*regs
, unsigned int fsr
)
330 struct kprobe
*cur
= kprobe_running();
331 struct kprobe_ctlblk
*kcb
= get_kprobe_ctlblk();
333 switch (kcb
->kprobe_status
) {
337 * We are here because the instruction being single
338 * stepped caused a page fault. We reset the current
339 * kprobe and the ip points back to the probe address
340 * and allow the page fault handler to continue as a
343 instruction_pointer_set(regs
, (unsigned long) cur
->addr
);
344 if (!instruction_pointer(regs
))
347 kernel_disable_single_step();
348 if (kcb
->kprobe_status
== KPROBE_REENTER
)
349 spsr_set_debug_flag(regs
, 1);
351 if (kcb
->kprobe_status
== KPROBE_REENTER
)
352 restore_previous_kprobe(kcb
);
354 reset_current_kprobe();
357 case KPROBE_HIT_ACTIVE
:
358 case KPROBE_HIT_SSDONE
:
360 * We increment the nmissed count for accounting,
361 * we can also use npre/npostfault count for accounting
362 * these specific fault cases.
364 kprobes_inc_nmissed_count(cur
);
367 * We come here because instructions in the pre/post
368 * handler caused the page_fault, this could happen
369 * if handler tries to access user space by
370 * copy_from_user(), get_user() etc. Let the
371 * user-specified handler try to fix it first.
373 if (cur
->fault_handler
&& cur
->fault_handler(cur
, regs
, fsr
))
377 * In case the user-specified fault handler returned
378 * zero, try to fix up.
380 if (fixup_exception(regs
))
386 int __kprobes
kprobe_exceptions_notify(struct notifier_block
*self
,
387 unsigned long val
, void *data
)
392 static void __kprobes
kprobe_handler(struct pt_regs
*regs
)
394 struct kprobe
*p
, *cur_kprobe
;
395 struct kprobe_ctlblk
*kcb
;
396 unsigned long addr
= instruction_pointer(regs
);
398 kcb
= get_kprobe_ctlblk();
399 cur_kprobe
= kprobe_running();
401 p
= get_kprobe((kprobe_opcode_t
*) addr
);
405 if (reenter_kprobe(p
, regs
, kcb
))
409 set_current_kprobe(p
);
410 kcb
->kprobe_status
= KPROBE_HIT_ACTIVE
;
413 * If we have no pre-handler or it returned 0, we
414 * continue with normal processing. If we have a
415 * pre-handler and it returned non-zero, it prepped
416 * for calling the break_handler below on re-entry,
417 * so get out doing nothing more here.
419 * pre_handler can hit a breakpoint and can step thru
420 * before return, keep PSTATE D-flag enabled until
421 * pre_handler return back.
423 if (!p
->pre_handler
|| !p
->pre_handler(p
, regs
)) {
424 setup_singlestep(p
, regs
, kcb
, 0);
428 } else if ((le32_to_cpu(*(kprobe_opcode_t
*) addr
) ==
429 BRK64_OPCODE_KPROBES
) && cur_kprobe
) {
430 /* We probably hit a jprobe. Call its break handler. */
431 if (cur_kprobe
->break_handler
&&
432 cur_kprobe
->break_handler(cur_kprobe
, regs
)) {
433 setup_singlestep(cur_kprobe
, regs
, kcb
, 0);
438 * The breakpoint instruction was removed right
439 * after we hit it. Another cpu has removed
440 * either a probepoint or a debugger breakpoint
441 * at this address. In either case, no further
442 * handling of this interrupt is appropriate.
443 * Return back to original instruction, and continue.
448 kprobe_ss_hit(struct kprobe_ctlblk
*kcb
, unsigned long addr
)
450 if ((kcb
->ss_ctx
.ss_pending
)
451 && (kcb
->ss_ctx
.match_addr
== addr
)) {
452 clear_ss_context(kcb
); /* clear pending ss */
453 return DBG_HOOK_HANDLED
;
455 /* not ours, kprobes should ignore it */
456 return DBG_HOOK_ERROR
;
460 kprobe_single_step_handler(struct pt_regs
*regs
, unsigned int esr
)
462 struct kprobe_ctlblk
*kcb
= get_kprobe_ctlblk();
465 /* return error if this is not our step */
466 retval
= kprobe_ss_hit(kcb
, instruction_pointer(regs
));
468 if (retval
== DBG_HOOK_HANDLED
) {
469 kprobes_restore_local_irqflag(kcb
, regs
);
470 kernel_disable_single_step();
472 if (kcb
->kprobe_status
== KPROBE_REENTER
)
473 spsr_set_debug_flag(regs
, 1);
475 post_kprobe_handler(kcb
, regs
);
482 kprobe_breakpoint_handler(struct pt_regs
*regs
, unsigned int esr
)
484 kprobe_handler(regs
);
485 return DBG_HOOK_HANDLED
;
488 int __kprobes
setjmp_pre_handler(struct kprobe
*p
, struct pt_regs
*regs
)
490 struct jprobe
*jp
= container_of(p
, struct jprobe
, kp
);
491 struct kprobe_ctlblk
*kcb
= get_kprobe_ctlblk();
492 long stack_ptr
= kernel_stack_pointer(regs
);
494 kcb
->jprobe_saved_regs
= *regs
;
496 * As Linus pointed out, gcc assumes that the callee
497 * owns the argument space and could overwrite it, e.g.
498 * tailcall optimization. So, to be absolutely safe
499 * we also save and restore enough stack bytes to cover
502 kasan_disable_current();
503 memcpy(kcb
->jprobes_stack
, (void *)stack_ptr
,
504 min_stack_size(stack_ptr
));
505 kasan_enable_current();
507 instruction_pointer_set(regs
, (unsigned long) jp
->entry
);
509 pause_graph_tracing();
513 void __kprobes
jprobe_return(void)
515 struct kprobe_ctlblk
*kcb
= get_kprobe_ctlblk();
518 * Jprobe handler return by entering break exception,
519 * encoded same as kprobe, but with following conditions
520 * -a special PC to identify it from the other kprobes.
521 * -restore stack addr to original saved pt_regs
523 asm volatile(" mov sp, %0 \n"
524 "jprobe_return_break: brk %1 \n"
526 : "r" (kcb
->jprobe_saved_regs
.sp
),
527 "I" (BRK64_ESR_KPROBES
)
533 int __kprobes
longjmp_break_handler(struct kprobe
*p
, struct pt_regs
*regs
)
535 struct kprobe_ctlblk
*kcb
= get_kprobe_ctlblk();
536 long stack_addr
= kcb
->jprobe_saved_regs
.sp
;
537 long orig_sp
= kernel_stack_pointer(regs
);
538 struct jprobe
*jp
= container_of(p
, struct jprobe
, kp
);
539 extern const char jprobe_return_break
[];
541 if (instruction_pointer(regs
) != (u64
) jprobe_return_break
)
544 if (orig_sp
!= stack_addr
) {
545 struct pt_regs
*saved_regs
=
546 (struct pt_regs
*)kcb
->jprobe_saved_regs
.sp
;
547 pr_err("current sp %lx does not match saved sp %lx\n",
548 orig_sp
, stack_addr
);
549 pr_err("Saved registers for jprobe %p\n", jp
);
550 show_regs(saved_regs
);
551 pr_err("Current registers\n");
555 unpause_graph_tracing();
556 *regs
= kcb
->jprobe_saved_regs
;
557 kasan_disable_current();
558 memcpy((void *)stack_addr
, kcb
->jprobes_stack
,
559 min_stack_size(stack_addr
));
560 kasan_enable_current();
561 preempt_enable_no_resched();
565 bool arch_within_kprobe_blacklist(unsigned long addr
)
567 extern char __idmap_text_start
[], __idmap_text_end
[];
568 extern char __hyp_idmap_text_start
[], __hyp_idmap_text_end
[];
570 if ((addr
>= (unsigned long)__kprobes_text_start
&&
571 addr
< (unsigned long)__kprobes_text_end
) ||
572 (addr
>= (unsigned long)__entry_text_start
&&
573 addr
< (unsigned long)__entry_text_end
) ||
574 (addr
>= (unsigned long)__idmap_text_start
&&
575 addr
< (unsigned long)__idmap_text_end
) ||
576 !!search_exception_tables(addr
))
579 if (!is_kernel_in_hyp_mode()) {
580 if ((addr
>= (unsigned long)__hyp_text_start
&&
581 addr
< (unsigned long)__hyp_text_end
) ||
582 (addr
>= (unsigned long)__hyp_idmap_text_start
&&
583 addr
< (unsigned long)__hyp_idmap_text_end
))
590 void __kprobes __used
*trampoline_probe_handler(struct pt_regs
*regs
)
592 struct kretprobe_instance
*ri
= NULL
;
593 struct hlist_head
*head
, empty_rp
;
594 struct hlist_node
*tmp
;
595 unsigned long flags
, orig_ret_address
= 0;
596 unsigned long trampoline_address
=
597 (unsigned long)&kretprobe_trampoline
;
598 kprobe_opcode_t
*correct_ret_addr
= NULL
;
600 INIT_HLIST_HEAD(&empty_rp
);
601 kretprobe_hash_lock(current
, &head
, &flags
);
604 * It is possible to have multiple instances associated with a given
605 * task either because multiple functions in the call path have
606 * return probes installed on them, and/or more than one
607 * return probe was registered for a target function.
609 * We can handle this because:
610 * - instances are always pushed into the head of the list
611 * - when multiple return probes are registered for the same
612 * function, the (chronologically) first instance's ret_addr
613 * will be the real return address, and all the rest will
614 * point to kretprobe_trampoline.
616 hlist_for_each_entry_safe(ri
, tmp
, head
, hlist
) {
617 if (ri
->task
!= current
)
618 /* another task is sharing our hash bucket */
621 orig_ret_address
= (unsigned long)ri
->ret_addr
;
623 if (orig_ret_address
!= trampoline_address
)
625 * This is the real return address. Any other
626 * instances associated with this task are for
627 * other calls deeper on the call stack
632 kretprobe_assert(ri
, orig_ret_address
, trampoline_address
);
634 correct_ret_addr
= ri
->ret_addr
;
635 hlist_for_each_entry_safe(ri
, tmp
, head
, hlist
) {
636 if (ri
->task
!= current
)
637 /* another task is sharing our hash bucket */
640 orig_ret_address
= (unsigned long)ri
->ret_addr
;
641 if (ri
->rp
&& ri
->rp
->handler
) {
642 __this_cpu_write(current_kprobe
, &ri
->rp
->kp
);
643 get_kprobe_ctlblk()->kprobe_status
= KPROBE_HIT_ACTIVE
;
644 ri
->ret_addr
= correct_ret_addr
;
645 ri
->rp
->handler(ri
, regs
);
646 __this_cpu_write(current_kprobe
, NULL
);
649 recycle_rp_inst(ri
, &empty_rp
);
651 if (orig_ret_address
!= trampoline_address
)
653 * This is the real return address. Any other
654 * instances associated with this task are for
655 * other calls deeper on the call stack
660 kretprobe_hash_unlock(current
, &flags
);
662 hlist_for_each_entry_safe(ri
, tmp
, &empty_rp
, hlist
) {
663 hlist_del(&ri
->hlist
);
666 return (void *)orig_ret_address
;
669 void __kprobes
arch_prepare_kretprobe(struct kretprobe_instance
*ri
,
670 struct pt_regs
*regs
)
672 ri
->ret_addr
= (kprobe_opcode_t
*)regs
->regs
[30];
674 /* replace return addr (x30) with trampoline */
675 regs
->regs
[30] = (long)&kretprobe_trampoline
;
678 int __kprobes
arch_trampoline_kprobe(struct kprobe
*p
)
683 int __init
arch_init_kprobes(void)