1 // SPDX-License-Identifier: GPL-2.0-only
3 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
4 * using the CPU's debug registers.
6 * Copyright (C) 2012 ARM Limited
7 * Author: Will Deacon <will.deacon@arm.com>
10 #define pr_fmt(fmt) "hw-breakpoint: " fmt
12 #include <linux/compat.h>
13 #include <linux/cpu_pm.h>
14 #include <linux/errno.h>
15 #include <linux/hw_breakpoint.h>
16 #include <linux/kprobes.h>
17 #include <linux/perf_event.h>
18 #include <linux/ptrace.h>
19 #include <linux/smp.h>
20 #include <linux/uaccess.h>
22 #include <asm/current.h>
23 #include <asm/debug-monitors.h>
24 #include <asm/hw_breakpoint.h>
25 #include <asm/traps.h>
26 #include <asm/cputype.h>
27 #include <asm/system_misc.h>
29 /* Breakpoint currently in use for each BRP. */
30 static DEFINE_PER_CPU(struct perf_event
*, bp_on_reg
[ARM_MAX_BRP
]);
32 /* Watchpoint currently in use for each WRP. */
33 static DEFINE_PER_CPU(struct perf_event
*, wp_on_reg
[ARM_MAX_WRP
]);
35 /* Currently stepping a per-CPU kernel breakpoint. */
36 static DEFINE_PER_CPU(int, stepping_kernel_bp
);
38 /* Number of BRP/WRP registers on this CPU. */
39 static int core_num_brps
;
40 static int core_num_wrps
;
42 int hw_breakpoint_slots(int type
)
45 * We can be called early, so don't rely on
46 * our static variables being initialised.
50 return get_num_brps();
52 return get_num_wrps();
54 pr_warning("unknown slot type: %d\n", type
);
59 #define READ_WB_REG_CASE(OFF, N, REG, VAL) \
61 AARCH64_DBG_READ(N, REG, VAL); \
64 #define WRITE_WB_REG_CASE(OFF, N, REG, VAL) \
66 AARCH64_DBG_WRITE(N, REG, VAL); \
69 #define GEN_READ_WB_REG_CASES(OFF, REG, VAL) \
70 READ_WB_REG_CASE(OFF, 0, REG, VAL); \
71 READ_WB_REG_CASE(OFF, 1, REG, VAL); \
72 READ_WB_REG_CASE(OFF, 2, REG, VAL); \
73 READ_WB_REG_CASE(OFF, 3, REG, VAL); \
74 READ_WB_REG_CASE(OFF, 4, REG, VAL); \
75 READ_WB_REG_CASE(OFF, 5, REG, VAL); \
76 READ_WB_REG_CASE(OFF, 6, REG, VAL); \
77 READ_WB_REG_CASE(OFF, 7, REG, VAL); \
78 READ_WB_REG_CASE(OFF, 8, REG, VAL); \
79 READ_WB_REG_CASE(OFF, 9, REG, VAL); \
80 READ_WB_REG_CASE(OFF, 10, REG, VAL); \
81 READ_WB_REG_CASE(OFF, 11, REG, VAL); \
82 READ_WB_REG_CASE(OFF, 12, REG, VAL); \
83 READ_WB_REG_CASE(OFF, 13, REG, VAL); \
84 READ_WB_REG_CASE(OFF, 14, REG, VAL); \
85 READ_WB_REG_CASE(OFF, 15, REG, VAL)
87 #define GEN_WRITE_WB_REG_CASES(OFF, REG, VAL) \
88 WRITE_WB_REG_CASE(OFF, 0, REG, VAL); \
89 WRITE_WB_REG_CASE(OFF, 1, REG, VAL); \
90 WRITE_WB_REG_CASE(OFF, 2, REG, VAL); \
91 WRITE_WB_REG_CASE(OFF, 3, REG, VAL); \
92 WRITE_WB_REG_CASE(OFF, 4, REG, VAL); \
93 WRITE_WB_REG_CASE(OFF, 5, REG, VAL); \
94 WRITE_WB_REG_CASE(OFF, 6, REG, VAL); \
95 WRITE_WB_REG_CASE(OFF, 7, REG, VAL); \
96 WRITE_WB_REG_CASE(OFF, 8, REG, VAL); \
97 WRITE_WB_REG_CASE(OFF, 9, REG, VAL); \
98 WRITE_WB_REG_CASE(OFF, 10, REG, VAL); \
99 WRITE_WB_REG_CASE(OFF, 11, REG, VAL); \
100 WRITE_WB_REG_CASE(OFF, 12, REG, VAL); \
101 WRITE_WB_REG_CASE(OFF, 13, REG, VAL); \
102 WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \
103 WRITE_WB_REG_CASE(OFF, 15, REG, VAL)
105 static u64
read_wb_reg(int reg
, int n
)
110 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BVR
, AARCH64_DBG_REG_NAME_BVR
, val
);
111 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BCR
, AARCH64_DBG_REG_NAME_BCR
, val
);
112 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WVR
, AARCH64_DBG_REG_NAME_WVR
, val
);
113 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WCR
, AARCH64_DBG_REG_NAME_WCR
, val
);
115 pr_warning("attempt to read from unknown breakpoint register %d\n", n
);
120 NOKPROBE_SYMBOL(read_wb_reg
);
122 static void write_wb_reg(int reg
, int n
, u64 val
)
125 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BVR
, AARCH64_DBG_REG_NAME_BVR
, val
);
126 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BCR
, AARCH64_DBG_REG_NAME_BCR
, val
);
127 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WVR
, AARCH64_DBG_REG_NAME_WVR
, val
);
128 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WCR
, AARCH64_DBG_REG_NAME_WCR
, val
);
130 pr_warning("attempt to write to unknown breakpoint register %d\n", n
);
134 NOKPROBE_SYMBOL(write_wb_reg
);
137 * Convert a breakpoint privilege level to the corresponding exception
140 static enum dbg_active_el
debug_exception_level(int privilege
)
143 case AARCH64_BREAKPOINT_EL0
:
144 return DBG_ACTIVE_EL0
;
145 case AARCH64_BREAKPOINT_EL1
:
146 return DBG_ACTIVE_EL1
;
148 pr_warning("invalid breakpoint privilege level %d\n", privilege
);
152 NOKPROBE_SYMBOL(debug_exception_level
);
154 enum hw_breakpoint_ops
{
155 HW_BREAKPOINT_INSTALL
,
156 HW_BREAKPOINT_UNINSTALL
,
157 HW_BREAKPOINT_RESTORE
160 static int is_compat_bp(struct perf_event
*bp
)
162 struct task_struct
*tsk
= bp
->hw
.target
;
165 * tsk can be NULL for per-cpu (non-ptrace) breakpoints.
166 * In this case, use the native interface, since we don't have
167 * the notion of a "compat CPU" and could end up relying on
168 * deprecated behaviour if we use unaligned watchpoints in
171 return tsk
&& is_compat_thread(task_thread_info(tsk
));
175 * hw_breakpoint_slot_setup - Find and setup a perf slot according to
178 * @slots: pointer to array of slots
179 * @max_slots: max number of slots
180 * @bp: perf_event to setup
181 * @ops: operation to be carried out on the slot
184 * slot index on success
185 * -ENOSPC if no slot is available/matches
186 * -EINVAL on wrong operations parameter
188 static int hw_breakpoint_slot_setup(struct perf_event
**slots
, int max_slots
,
189 struct perf_event
*bp
,
190 enum hw_breakpoint_ops ops
)
193 struct perf_event
**slot
;
195 for (i
= 0; i
< max_slots
; ++i
) {
198 case HW_BREAKPOINT_INSTALL
:
204 case HW_BREAKPOINT_UNINSTALL
:
210 case HW_BREAKPOINT_RESTORE
:
215 pr_warn_once("Unhandled hw breakpoint ops %d\n", ops
);
222 static int hw_breakpoint_control(struct perf_event
*bp
,
223 enum hw_breakpoint_ops ops
)
225 struct arch_hw_breakpoint
*info
= counter_arch_bp(bp
);
226 struct perf_event
**slots
;
227 struct debug_info
*debug_info
= ¤t
->thread
.debug
;
228 int i
, max_slots
, ctrl_reg
, val_reg
, reg_enable
;
229 enum dbg_active_el dbg_el
= debug_exception_level(info
->ctrl
.privilege
);
232 if (info
->ctrl
.type
== ARM_BREAKPOINT_EXECUTE
) {
234 ctrl_reg
= AARCH64_DBG_REG_BCR
;
235 val_reg
= AARCH64_DBG_REG_BVR
;
236 slots
= this_cpu_ptr(bp_on_reg
);
237 max_slots
= core_num_brps
;
238 reg_enable
= !debug_info
->bps_disabled
;
241 ctrl_reg
= AARCH64_DBG_REG_WCR
;
242 val_reg
= AARCH64_DBG_REG_WVR
;
243 slots
= this_cpu_ptr(wp_on_reg
);
244 max_slots
= core_num_wrps
;
245 reg_enable
= !debug_info
->wps_disabled
;
248 i
= hw_breakpoint_slot_setup(slots
, max_slots
, bp
, ops
);
250 if (WARN_ONCE(i
< 0, "Can't find any breakpoint slot"))
254 case HW_BREAKPOINT_INSTALL
:
256 * Ensure debug monitors are enabled at the correct exception
259 enable_debug_monitors(dbg_el
);
261 case HW_BREAKPOINT_RESTORE
:
262 /* Setup the address register. */
263 write_wb_reg(val_reg
, i
, info
->address
);
265 /* Setup the control register. */
266 ctrl
= encode_ctrl_reg(info
->ctrl
);
267 write_wb_reg(ctrl_reg
, i
,
268 reg_enable
? ctrl
| 0x1 : ctrl
& ~0x1);
270 case HW_BREAKPOINT_UNINSTALL
:
271 /* Reset the control register. */
272 write_wb_reg(ctrl_reg
, i
, 0);
275 * Release the debug monitors for the correct exception
278 disable_debug_monitors(dbg_el
);
286 * Install a perf counter breakpoint.
288 int arch_install_hw_breakpoint(struct perf_event
*bp
)
290 return hw_breakpoint_control(bp
, HW_BREAKPOINT_INSTALL
);
293 void arch_uninstall_hw_breakpoint(struct perf_event
*bp
)
295 hw_breakpoint_control(bp
, HW_BREAKPOINT_UNINSTALL
);
298 static int get_hbp_len(u8 hbp_len
)
300 unsigned int len_in_bytes
= 0;
303 case ARM_BREAKPOINT_LEN_1
:
306 case ARM_BREAKPOINT_LEN_2
:
309 case ARM_BREAKPOINT_LEN_3
:
312 case ARM_BREAKPOINT_LEN_4
:
315 case ARM_BREAKPOINT_LEN_5
:
318 case ARM_BREAKPOINT_LEN_6
:
321 case ARM_BREAKPOINT_LEN_7
:
324 case ARM_BREAKPOINT_LEN_8
:
333 * Check whether bp virtual address is in kernel space.
335 int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint
*hw
)
341 len
= get_hbp_len(hw
->ctrl
.len
);
343 return (va
>= TASK_SIZE
) && ((va
+ len
- 1) >= TASK_SIZE
);
347 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
348 * Hopefully this will disappear when ptrace can bypass the conversion
349 * to generic breakpoint descriptions.
351 int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl
,
352 int *gen_len
, int *gen_type
, int *offset
)
356 case ARM_BREAKPOINT_EXECUTE
:
357 *gen_type
= HW_BREAKPOINT_X
;
359 case ARM_BREAKPOINT_LOAD
:
360 *gen_type
= HW_BREAKPOINT_R
;
362 case ARM_BREAKPOINT_STORE
:
363 *gen_type
= HW_BREAKPOINT_W
;
365 case ARM_BREAKPOINT_LOAD
| ARM_BREAKPOINT_STORE
:
366 *gen_type
= HW_BREAKPOINT_RW
;
374 *offset
= __ffs(ctrl
.len
);
377 switch (ctrl
.len
>> *offset
) {
378 case ARM_BREAKPOINT_LEN_1
:
379 *gen_len
= HW_BREAKPOINT_LEN_1
;
381 case ARM_BREAKPOINT_LEN_2
:
382 *gen_len
= HW_BREAKPOINT_LEN_2
;
384 case ARM_BREAKPOINT_LEN_3
:
385 *gen_len
= HW_BREAKPOINT_LEN_3
;
387 case ARM_BREAKPOINT_LEN_4
:
388 *gen_len
= HW_BREAKPOINT_LEN_4
;
390 case ARM_BREAKPOINT_LEN_5
:
391 *gen_len
= HW_BREAKPOINT_LEN_5
;
393 case ARM_BREAKPOINT_LEN_6
:
394 *gen_len
= HW_BREAKPOINT_LEN_6
;
396 case ARM_BREAKPOINT_LEN_7
:
397 *gen_len
= HW_BREAKPOINT_LEN_7
;
399 case ARM_BREAKPOINT_LEN_8
:
400 *gen_len
= HW_BREAKPOINT_LEN_8
;
410 * Construct an arch_hw_breakpoint from a perf_event.
412 static int arch_build_bp_info(struct perf_event
*bp
,
413 const struct perf_event_attr
*attr
,
414 struct arch_hw_breakpoint
*hw
)
417 switch (attr
->bp_type
) {
418 case HW_BREAKPOINT_X
:
419 hw
->ctrl
.type
= ARM_BREAKPOINT_EXECUTE
;
421 case HW_BREAKPOINT_R
:
422 hw
->ctrl
.type
= ARM_BREAKPOINT_LOAD
;
424 case HW_BREAKPOINT_W
:
425 hw
->ctrl
.type
= ARM_BREAKPOINT_STORE
;
427 case HW_BREAKPOINT_RW
:
428 hw
->ctrl
.type
= ARM_BREAKPOINT_LOAD
| ARM_BREAKPOINT_STORE
;
435 switch (attr
->bp_len
) {
436 case HW_BREAKPOINT_LEN_1
:
437 hw
->ctrl
.len
= ARM_BREAKPOINT_LEN_1
;
439 case HW_BREAKPOINT_LEN_2
:
440 hw
->ctrl
.len
= ARM_BREAKPOINT_LEN_2
;
442 case HW_BREAKPOINT_LEN_3
:
443 hw
->ctrl
.len
= ARM_BREAKPOINT_LEN_3
;
445 case HW_BREAKPOINT_LEN_4
:
446 hw
->ctrl
.len
= ARM_BREAKPOINT_LEN_4
;
448 case HW_BREAKPOINT_LEN_5
:
449 hw
->ctrl
.len
= ARM_BREAKPOINT_LEN_5
;
451 case HW_BREAKPOINT_LEN_6
:
452 hw
->ctrl
.len
= ARM_BREAKPOINT_LEN_6
;
454 case HW_BREAKPOINT_LEN_7
:
455 hw
->ctrl
.len
= ARM_BREAKPOINT_LEN_7
;
457 case HW_BREAKPOINT_LEN_8
:
458 hw
->ctrl
.len
= ARM_BREAKPOINT_LEN_8
;
465 * On AArch64, we only permit breakpoints of length 4, whereas
466 * AArch32 also requires breakpoints of length 2 for Thumb.
467 * Watchpoints can be of length 1, 2, 4 or 8 bytes.
469 if (hw
->ctrl
.type
== ARM_BREAKPOINT_EXECUTE
) {
470 if (is_compat_bp(bp
)) {
471 if (hw
->ctrl
.len
!= ARM_BREAKPOINT_LEN_2
&&
472 hw
->ctrl
.len
!= ARM_BREAKPOINT_LEN_4
)
474 } else if (hw
->ctrl
.len
!= ARM_BREAKPOINT_LEN_4
) {
476 * FIXME: Some tools (I'm looking at you perf) assume
477 * that breakpoints should be sizeof(long). This
478 * is nonsense. For now, we fix up the parameter
479 * but we should probably return -EINVAL instead.
481 hw
->ctrl
.len
= ARM_BREAKPOINT_LEN_4
;
486 hw
->address
= attr
->bp_addr
;
490 * Note that we disallow combined EL0/EL1 breakpoints because
491 * that would complicate the stepping code.
493 if (arch_check_bp_in_kernelspace(hw
))
494 hw
->ctrl
.privilege
= AARCH64_BREAKPOINT_EL1
;
496 hw
->ctrl
.privilege
= AARCH64_BREAKPOINT_EL0
;
499 hw
->ctrl
.enabled
= !attr
->disabled
;
505 * Validate the arch-specific HW Breakpoint register settings.
507 int hw_breakpoint_arch_parse(struct perf_event
*bp
,
508 const struct perf_event_attr
*attr
,
509 struct arch_hw_breakpoint
*hw
)
512 u64 alignment_mask
, offset
;
514 /* Build the arch_hw_breakpoint. */
515 ret
= arch_build_bp_info(bp
, attr
, hw
);
520 * Check address alignment.
521 * We don't do any clever alignment correction for watchpoints
522 * because using 64-bit unaligned addresses is deprecated for
525 * AArch32 tasks expect some simple alignment fixups, so emulate
528 if (is_compat_bp(bp
)) {
529 if (hw
->ctrl
.len
== ARM_BREAKPOINT_LEN_8
)
530 alignment_mask
= 0x7;
532 alignment_mask
= 0x3;
533 offset
= hw
->address
& alignment_mask
;
539 /* Allow single byte watchpoint. */
540 if (hw
->ctrl
.len
== ARM_BREAKPOINT_LEN_1
)
543 /* Allow halfword watchpoints and breakpoints. */
544 if (hw
->ctrl
.len
== ARM_BREAKPOINT_LEN_2
)
550 if (hw
->ctrl
.type
== ARM_BREAKPOINT_EXECUTE
)
551 alignment_mask
= 0x3;
553 alignment_mask
= 0x7;
554 offset
= hw
->address
& alignment_mask
;
557 hw
->address
&= ~alignment_mask
;
558 hw
->ctrl
.len
<<= offset
;
561 * Disallow per-task kernel breakpoints since these would
562 * complicate the stepping code.
564 if (hw
->ctrl
.privilege
== AARCH64_BREAKPOINT_EL1
&& bp
->hw
.target
)
571 * Enable/disable all of the breakpoints active at the specified
572 * exception level at the register level.
573 * This is used when single-stepping after a breakpoint exception.
575 static void toggle_bp_registers(int reg
, enum dbg_active_el el
, int enable
)
577 int i
, max_slots
, privilege
;
579 struct perf_event
**slots
;
582 case AARCH64_DBG_REG_BCR
:
583 slots
= this_cpu_ptr(bp_on_reg
);
584 max_slots
= core_num_brps
;
586 case AARCH64_DBG_REG_WCR
:
587 slots
= this_cpu_ptr(wp_on_reg
);
588 max_slots
= core_num_wrps
;
594 for (i
= 0; i
< max_slots
; ++i
) {
598 privilege
= counter_arch_bp(slots
[i
])->ctrl
.privilege
;
599 if (debug_exception_level(privilege
) != el
)
602 ctrl
= read_wb_reg(reg
, i
);
607 write_wb_reg(reg
, i
, ctrl
);
610 NOKPROBE_SYMBOL(toggle_bp_registers
);
613 * Debug exception handlers.
615 static int breakpoint_handler(unsigned long unused
, unsigned int esr
,
616 struct pt_regs
*regs
)
618 int i
, step
= 0, *kernel_step
;
621 struct perf_event
*bp
, **slots
;
622 struct debug_info
*debug_info
;
623 struct arch_hw_breakpoint_ctrl ctrl
;
625 slots
= this_cpu_ptr(bp_on_reg
);
626 addr
= instruction_pointer(regs
);
627 debug_info
= ¤t
->thread
.debug
;
629 for (i
= 0; i
< core_num_brps
; ++i
) {
637 /* Check if the breakpoint value matches. */
638 val
= read_wb_reg(AARCH64_DBG_REG_BVR
, i
);
639 if (val
!= (addr
& ~0x3))
642 /* Possible match, check the byte address select to confirm. */
643 ctrl_reg
= read_wb_reg(AARCH64_DBG_REG_BCR
, i
);
644 decode_ctrl_reg(ctrl_reg
, &ctrl
);
645 if (!((1 << (addr
& 0x3)) & ctrl
.len
))
648 counter_arch_bp(bp
)->trigger
= addr
;
649 perf_bp_event(bp
, regs
);
651 /* Do we need to handle the stepping? */
652 if (is_default_overflow_handler(bp
))
661 if (user_mode(regs
)) {
662 debug_info
->bps_disabled
= 1;
663 toggle_bp_registers(AARCH64_DBG_REG_BCR
, DBG_ACTIVE_EL0
, 0);
665 /* If we're already stepping a watchpoint, just return. */
666 if (debug_info
->wps_disabled
)
669 if (test_thread_flag(TIF_SINGLESTEP
))
670 debug_info
->suspended_step
= 1;
672 user_enable_single_step(current
);
674 toggle_bp_registers(AARCH64_DBG_REG_BCR
, DBG_ACTIVE_EL1
, 0);
675 kernel_step
= this_cpu_ptr(&stepping_kernel_bp
);
677 if (*kernel_step
!= ARM_KERNEL_STEP_NONE
)
680 if (kernel_active_single_step()) {
681 *kernel_step
= ARM_KERNEL_STEP_SUSPEND
;
683 *kernel_step
= ARM_KERNEL_STEP_ACTIVE
;
684 kernel_enable_single_step(regs
);
690 NOKPROBE_SYMBOL(breakpoint_handler
);
693 * Arm64 hardware does not always report a watchpoint hit address that matches
694 * one of the watchpoints set. It can also report an address "near" the
695 * watchpoint if a single instruction access both watched and unwatched
696 * addresses. There is no straight-forward way, short of disassembling the
697 * offending instruction, to map that address back to the watchpoint. This
698 * function computes the distance of the memory access from the watchpoint as a
699 * heuristic for the likelyhood that a given access triggered the watchpoint.
701 * See Section D2.10.5 "Determining the memory location that caused a Watchpoint
702 * exception" of ARMv8 Architecture Reference Manual for details.
704 * The function returns the distance of the address from the bytes watched by
705 * the watchpoint. In case of an exact match, it returns 0.
707 static u64
get_distance_from_watchpoint(unsigned long addr
, u64 val
,
708 struct arch_hw_breakpoint_ctrl
*ctrl
)
713 addr
= untagged_addr(addr
);
715 lens
= __ffs(ctrl
->len
);
716 lene
= __fls(ctrl
->len
);
719 wp_high
= val
+ lene
;
721 return wp_low
- addr
;
722 else if (addr
> wp_high
)
723 return addr
- wp_high
;
728 static int watchpoint_handler(unsigned long addr
, unsigned int esr
,
729 struct pt_regs
*regs
)
731 int i
, step
= 0, *kernel_step
, access
, closest_match
= 0;
732 u64 min_dist
= -1, dist
;
735 struct perf_event
*wp
, **slots
;
736 struct debug_info
*debug_info
;
737 struct arch_hw_breakpoint
*info
;
738 struct arch_hw_breakpoint_ctrl ctrl
;
740 slots
= this_cpu_ptr(wp_on_reg
);
741 debug_info
= ¤t
->thread
.debug
;
744 * Find all watchpoints that match the reported address. If no exact
745 * match is found. Attribute the hit to the closest watchpoint.
748 for (i
= 0; i
< core_num_wrps
; ++i
) {
754 * Check that the access type matches.
755 * 0 => load, otherwise => store
757 access
= (esr
& AARCH64_ESR_ACCESS_MASK
) ? HW_BREAKPOINT_W
:
759 if (!(access
& hw_breakpoint_type(wp
)))
762 /* Check if the watchpoint value and byte select match. */
763 val
= read_wb_reg(AARCH64_DBG_REG_WVR
, i
);
764 ctrl_reg
= read_wb_reg(AARCH64_DBG_REG_WCR
, i
);
765 decode_ctrl_reg(ctrl_reg
, &ctrl
);
766 dist
= get_distance_from_watchpoint(addr
, val
, &ctrl
);
767 if (dist
< min_dist
) {
771 /* Is this an exact match? */
775 info
= counter_arch_bp(wp
);
776 info
->trigger
= addr
;
777 perf_bp_event(wp
, regs
);
779 /* Do we need to handle the stepping? */
780 if (is_default_overflow_handler(wp
))
783 if (min_dist
> 0 && min_dist
!= -1) {
784 /* No exact match found. */
785 wp
= slots
[closest_match
];
786 info
= counter_arch_bp(wp
);
787 info
->trigger
= addr
;
788 perf_bp_event(wp
, regs
);
790 /* Do we need to handle the stepping? */
791 if (is_default_overflow_handler(wp
))
800 * We always disable EL0 watchpoints because the kernel can
801 * cause these to fire via an unprivileged access.
803 toggle_bp_registers(AARCH64_DBG_REG_WCR
, DBG_ACTIVE_EL0
, 0);
805 if (user_mode(regs
)) {
806 debug_info
->wps_disabled
= 1;
808 /* If we're already stepping a breakpoint, just return. */
809 if (debug_info
->bps_disabled
)
812 if (test_thread_flag(TIF_SINGLESTEP
))
813 debug_info
->suspended_step
= 1;
815 user_enable_single_step(current
);
817 toggle_bp_registers(AARCH64_DBG_REG_WCR
, DBG_ACTIVE_EL1
, 0);
818 kernel_step
= this_cpu_ptr(&stepping_kernel_bp
);
820 if (*kernel_step
!= ARM_KERNEL_STEP_NONE
)
823 if (kernel_active_single_step()) {
824 *kernel_step
= ARM_KERNEL_STEP_SUSPEND
;
826 *kernel_step
= ARM_KERNEL_STEP_ACTIVE
;
827 kernel_enable_single_step(regs
);
833 NOKPROBE_SYMBOL(watchpoint_handler
);
836 * Handle single-step exception.
838 int reinstall_suspended_bps(struct pt_regs
*regs
)
840 struct debug_info
*debug_info
= ¤t
->thread
.debug
;
841 int handled_exception
= 0, *kernel_step
;
843 kernel_step
= this_cpu_ptr(&stepping_kernel_bp
);
846 * Called from single-step exception handler.
847 * Return 0 if execution can resume, 1 if a SIGTRAP should be
850 if (user_mode(regs
)) {
851 if (debug_info
->bps_disabled
) {
852 debug_info
->bps_disabled
= 0;
853 toggle_bp_registers(AARCH64_DBG_REG_BCR
, DBG_ACTIVE_EL0
, 1);
854 handled_exception
= 1;
857 if (debug_info
->wps_disabled
) {
858 debug_info
->wps_disabled
= 0;
859 toggle_bp_registers(AARCH64_DBG_REG_WCR
, DBG_ACTIVE_EL0
, 1);
860 handled_exception
= 1;
863 if (handled_exception
) {
864 if (debug_info
->suspended_step
) {
865 debug_info
->suspended_step
= 0;
866 /* Allow exception handling to fall-through. */
867 handled_exception
= 0;
869 user_disable_single_step(current
);
872 } else if (*kernel_step
!= ARM_KERNEL_STEP_NONE
) {
873 toggle_bp_registers(AARCH64_DBG_REG_BCR
, DBG_ACTIVE_EL1
, 1);
874 toggle_bp_registers(AARCH64_DBG_REG_WCR
, DBG_ACTIVE_EL1
, 1);
876 if (!debug_info
->wps_disabled
)
877 toggle_bp_registers(AARCH64_DBG_REG_WCR
, DBG_ACTIVE_EL0
, 1);
879 if (*kernel_step
!= ARM_KERNEL_STEP_SUSPEND
) {
880 kernel_disable_single_step();
881 handled_exception
= 1;
883 handled_exception
= 0;
886 *kernel_step
= ARM_KERNEL_STEP_NONE
;
889 return !handled_exception
;
891 NOKPROBE_SYMBOL(reinstall_suspended_bps
);
894 * Context-switcher for restoring suspended breakpoints.
896 void hw_breakpoint_thread_switch(struct task_struct
*next
)
900 * disabled: 0 0 => The usual case, NOTIFY_DONE
901 * 0 1 => Disable the registers
902 * 1 0 => Enable the registers
903 * 1 1 => NOTIFY_DONE. per-task bps will
904 * get taken care of by perf.
907 struct debug_info
*current_debug_info
, *next_debug_info
;
909 current_debug_info
= ¤t
->thread
.debug
;
910 next_debug_info
= &next
->thread
.debug
;
912 /* Update breakpoints. */
913 if (current_debug_info
->bps_disabled
!= next_debug_info
->bps_disabled
)
914 toggle_bp_registers(AARCH64_DBG_REG_BCR
,
916 !next_debug_info
->bps_disabled
);
918 /* Update watchpoints. */
919 if (current_debug_info
->wps_disabled
!= next_debug_info
->wps_disabled
)
920 toggle_bp_registers(AARCH64_DBG_REG_WCR
,
922 !next_debug_info
->wps_disabled
);
926 * CPU initialisation.
928 static int hw_breakpoint_reset(unsigned int cpu
)
931 struct perf_event
**slots
;
933 * When a CPU goes through cold-boot, it does not have any installed
934 * slot, so it is safe to share the same function for restoring and
935 * resetting breakpoints; when a CPU is hotplugged in, it goes
936 * through the slots, which are all empty, hence it just resets control
937 * and value for debug registers.
938 * When this function is triggered on warm-boot through a CPU PM
939 * notifier some slots might be initialized; if so they are
940 * reprogrammed according to the debug slots content.
942 for (slots
= this_cpu_ptr(bp_on_reg
), i
= 0; i
< core_num_brps
; ++i
) {
944 hw_breakpoint_control(slots
[i
], HW_BREAKPOINT_RESTORE
);
946 write_wb_reg(AARCH64_DBG_REG_BCR
, i
, 0UL);
947 write_wb_reg(AARCH64_DBG_REG_BVR
, i
, 0UL);
951 for (slots
= this_cpu_ptr(wp_on_reg
), i
= 0; i
< core_num_wrps
; ++i
) {
953 hw_breakpoint_control(slots
[i
], HW_BREAKPOINT_RESTORE
);
955 write_wb_reg(AARCH64_DBG_REG_WCR
, i
, 0UL);
956 write_wb_reg(AARCH64_DBG_REG_WVR
, i
, 0UL);
964 extern void cpu_suspend_set_dbg_restorer(int (*hw_bp_restore
)(unsigned int));
966 static inline void cpu_suspend_set_dbg_restorer(int (*hw_bp_restore
)(unsigned int))
972 * One-time initialisation.
974 static int __init
arch_hw_breakpoint_init(void)
978 core_num_brps
= get_num_brps();
979 core_num_wrps
= get_num_wrps();
981 pr_info("found %d breakpoint and %d watchpoint registers.\n",
982 core_num_brps
, core_num_wrps
);
984 /* Register debug fault handlers. */
985 hook_debug_fault_code(DBG_ESR_EVT_HWBP
, breakpoint_handler
, SIGTRAP
,
986 TRAP_HWBKPT
, "hw-breakpoint handler");
987 hook_debug_fault_code(DBG_ESR_EVT_HWWP
, watchpoint_handler
, SIGTRAP
,
988 TRAP_HWBKPT
, "hw-watchpoint handler");
991 * Reset the breakpoint resources. We assume that a halting
992 * debugger will leave the world in a nice state for us.
994 ret
= cpuhp_setup_state(CPUHP_AP_PERF_ARM_HW_BREAKPOINT_STARTING
,
995 "perf/arm64/hw_breakpoint:starting",
996 hw_breakpoint_reset
, NULL
);
998 pr_err("failed to register CPU hotplug notifier: %d\n", ret
);
1000 /* Register cpu_suspend hw breakpoint restore hook */
1001 cpu_suspend_set_dbg_restorer(hw_breakpoint_reset
);
1005 arch_initcall(arch_hw_breakpoint_init
);
1007 void hw_breakpoint_pmu_read(struct perf_event
*bp
)
1012 * Dummy function to register with die_notifier.
1014 int hw_breakpoint_exceptions_notify(struct notifier_block
*unused
,
1015 unsigned long val
, void *data
)