1 // SPDX-License-Identifier: GPL-2.0-only
3 * Based on arch/arm/mm/fault.c
5 * Copyright (C) 1995 Linus Torvalds
6 * Copyright (C) 1995-2004 Russell King
7 * Copyright (C) 2012 ARM Ltd.
10 #include <linux/acpi.h>
11 #include <linux/bitfield.h>
12 #include <linux/extable.h>
13 #include <linux/signal.h>
15 #include <linux/hardirq.h>
16 #include <linux/init.h>
17 #include <linux/kprobes.h>
18 #include <linux/uaccess.h>
19 #include <linux/page-flags.h>
20 #include <linux/sched/signal.h>
21 #include <linux/sched/debug.h>
22 #include <linux/highmem.h>
23 #include <linux/perf_event.h>
24 #include <linux/preempt.h>
25 #include <linux/hugetlb.h>
29 #include <asm/cmpxchg.h>
30 #include <asm/cpufeature.h>
31 #include <asm/exception.h>
32 #include <asm/daifflags.h>
33 #include <asm/debug-monitors.h>
35 #include <asm/kprobes.h>
36 #include <asm/processor.h>
37 #include <asm/sysreg.h>
38 #include <asm/system_misc.h>
39 #include <asm/tlbflush.h>
40 #include <asm/traps.h>
43 int (*fn
)(unsigned long addr
, unsigned int esr
,
44 struct pt_regs
*regs
);
50 static const struct fault_info fault_info
[];
51 static struct fault_info debug_fault_info
[];
53 static inline const struct fault_info
*esr_to_fault_info(unsigned int esr
)
55 return fault_info
+ (esr
& ESR_ELx_FSC
);
58 static inline const struct fault_info
*esr_to_debug_fault_info(unsigned int esr
)
60 return debug_fault_info
+ DBG_ESR_EVT(esr
);
63 static void data_abort_decode(unsigned int esr
)
65 pr_alert("Data abort info:\n");
67 if (esr
& ESR_ELx_ISV
) {
68 pr_alert(" Access size = %u byte(s)\n",
69 1U << ((esr
& ESR_ELx_SAS
) >> ESR_ELx_SAS_SHIFT
));
70 pr_alert(" SSE = %lu, SRT = %lu\n",
71 (esr
& ESR_ELx_SSE
) >> ESR_ELx_SSE_SHIFT
,
72 (esr
& ESR_ELx_SRT_MASK
) >> ESR_ELx_SRT_SHIFT
);
73 pr_alert(" SF = %lu, AR = %lu\n",
74 (esr
& ESR_ELx_SF
) >> ESR_ELx_SF_SHIFT
,
75 (esr
& ESR_ELx_AR
) >> ESR_ELx_AR_SHIFT
);
77 pr_alert(" ISV = 0, ISS = 0x%08lx\n", esr
& ESR_ELx_ISS_MASK
);
80 pr_alert(" CM = %lu, WnR = %lu\n",
81 (esr
& ESR_ELx_CM
) >> ESR_ELx_CM_SHIFT
,
82 (esr
& ESR_ELx_WNR
) >> ESR_ELx_WNR_SHIFT
);
85 static void mem_abort_decode(unsigned int esr
)
87 pr_alert("Mem abort info:\n");
89 pr_alert(" ESR = 0x%08x\n", esr
);
90 pr_alert(" EC = 0x%02lx: %s, IL = %u bits\n",
91 ESR_ELx_EC(esr
), esr_get_class_string(esr
),
92 (esr
& ESR_ELx_IL
) ? 32 : 16);
93 pr_alert(" SET = %lu, FnV = %lu\n",
94 (esr
& ESR_ELx_SET_MASK
) >> ESR_ELx_SET_SHIFT
,
95 (esr
& ESR_ELx_FnV
) >> ESR_ELx_FnV_SHIFT
);
96 pr_alert(" EA = %lu, S1PTW = %lu\n",
97 (esr
& ESR_ELx_EA
) >> ESR_ELx_EA_SHIFT
,
98 (esr
& ESR_ELx_S1PTW
) >> ESR_ELx_S1PTW_SHIFT
);
100 if (esr_is_data_abort(esr
))
101 data_abort_decode(esr
);
104 static inline unsigned long mm_to_pgd_phys(struct mm_struct
*mm
)
106 /* Either init_pg_dir or swapper_pg_dir */
108 return __pa_symbol(mm
->pgd
);
110 return (unsigned long)virt_to_phys(mm
->pgd
);
114 * Dump out the page tables associated with 'addr' in the currently active mm.
116 static void show_pte(unsigned long addr
)
118 struct mm_struct
*mm
;
122 if (is_ttbr0_addr(addr
)) {
124 mm
= current
->active_mm
;
125 if (mm
== &init_mm
) {
126 pr_alert("[%016lx] user address but active_mm is swapper\n",
130 } else if (is_ttbr1_addr(addr
)) {
134 pr_alert("[%016lx] address between user and kernel address ranges\n",
139 pr_alert("%s pgtable: %luk pages, %llu-bit VAs, pgdp=%016lx\n",
140 mm
== &init_mm
? "swapper" : "user", PAGE_SIZE
/ SZ_1K
,
141 vabits_actual
, mm_to_pgd_phys(mm
));
142 pgdp
= pgd_offset(mm
, addr
);
143 pgd
= READ_ONCE(*pgdp
);
144 pr_alert("[%016lx] pgd=%016llx", addr
, pgd_val(pgd
));
152 if (pgd_none(pgd
) || pgd_bad(pgd
))
155 p4dp
= p4d_offset(pgdp
, addr
);
156 p4d
= READ_ONCE(*p4dp
);
157 pr_cont(", p4d=%016llx", p4d_val(p4d
));
158 if (p4d_none(p4d
) || p4d_bad(p4d
))
161 pudp
= pud_offset(p4dp
, addr
);
162 pud
= READ_ONCE(*pudp
);
163 pr_cont(", pud=%016llx", pud_val(pud
));
164 if (pud_none(pud
) || pud_bad(pud
))
167 pmdp
= pmd_offset(pudp
, addr
);
168 pmd
= READ_ONCE(*pmdp
);
169 pr_cont(", pmd=%016llx", pmd_val(pmd
));
170 if (pmd_none(pmd
) || pmd_bad(pmd
))
173 ptep
= pte_offset_map(pmdp
, addr
);
174 pte
= READ_ONCE(*ptep
);
175 pr_cont(", pte=%016llx", pte_val(pte
));
183 * This function sets the access flags (dirty, accessed), as well as write
184 * permission, and only to a more permissive setting.
186 * It needs to cope with hardware update of the accessed/dirty state by other
187 * agents in the system and can safely skip the __sync_icache_dcache() call as,
188 * like set_pte_at(), the PTE is never changed from no-exec to exec here.
190 * Returns whether or not the PTE actually changed.
192 int ptep_set_access_flags(struct vm_area_struct
*vma
,
193 unsigned long address
, pte_t
*ptep
,
194 pte_t entry
, int dirty
)
196 pteval_t old_pteval
, pteval
;
197 pte_t pte
= READ_ONCE(*ptep
);
199 if (pte_same(pte
, entry
))
202 /* only preserve the access flags and write permission */
203 pte_val(entry
) &= PTE_RDONLY
| PTE_AF
| PTE_WRITE
| PTE_DIRTY
;
206 * Setting the flags must be done atomically to avoid racing with the
207 * hardware update of the access/dirty state. The PTE_RDONLY bit must
208 * be set to the most permissive (lowest value) of *ptep and entry
209 * (calculated as: a & b == ~(~a | ~b)).
211 pte_val(entry
) ^= PTE_RDONLY
;
212 pteval
= pte_val(pte
);
215 pteval
^= PTE_RDONLY
;
216 pteval
|= pte_val(entry
);
217 pteval
^= PTE_RDONLY
;
218 pteval
= cmpxchg_relaxed(&pte_val(*ptep
), old_pteval
, pteval
);
219 } while (pteval
!= old_pteval
);
221 /* Invalidate a stale read-only entry */
223 flush_tlb_page(vma
, address
);
227 static bool is_el1_instruction_abort(unsigned int esr
)
229 return ESR_ELx_EC(esr
) == ESR_ELx_EC_IABT_CUR
;
232 static inline bool is_el1_permission_fault(unsigned long addr
, unsigned int esr
,
233 struct pt_regs
*regs
)
235 unsigned int ec
= ESR_ELx_EC(esr
);
236 unsigned int fsc_type
= esr
& ESR_ELx_FSC_TYPE
;
238 if (ec
!= ESR_ELx_EC_DABT_CUR
&& ec
!= ESR_ELx_EC_IABT_CUR
)
241 if (fsc_type
== ESR_ELx_FSC_PERM
)
244 if (is_ttbr0_addr(addr
) && system_uses_ttbr0_pan())
245 return fsc_type
== ESR_ELx_FSC_FAULT
&&
246 (regs
->pstate
& PSR_PAN_BIT
);
251 static bool __kprobes
is_spurious_el1_translation_fault(unsigned long addr
,
253 struct pt_regs
*regs
)
258 if (ESR_ELx_EC(esr
) != ESR_ELx_EC_DABT_CUR
||
259 (esr
& ESR_ELx_FSC_TYPE
) != ESR_ELx_FSC_FAULT
)
262 local_irq_save(flags
);
263 asm volatile("at s1e1r, %0" :: "r" (addr
));
265 par
= read_sysreg(par_el1
);
266 local_irq_restore(flags
);
269 * If we now have a valid translation, treat the translation fault as
272 if (!(par
& SYS_PAR_EL1_F
))
276 * If we got a different type of fault from the AT instruction,
277 * treat the translation fault as spurious.
279 dfsc
= FIELD_GET(SYS_PAR_EL1_FST
, par
);
280 return (dfsc
& ESR_ELx_FSC_TYPE
) != ESR_ELx_FSC_FAULT
;
283 static void die_kernel_fault(const char *msg
, unsigned long addr
,
284 unsigned int esr
, struct pt_regs
*regs
)
288 pr_alert("Unable to handle kernel %s at virtual address %016lx\n", msg
,
291 mem_abort_decode(esr
);
294 die("Oops", regs
, esr
);
299 static void __do_kernel_fault(unsigned long addr
, unsigned int esr
,
300 struct pt_regs
*regs
)
305 * Are we prepared to handle this kernel fault?
306 * We are almost certainly not prepared to handle instruction faults.
308 if (!is_el1_instruction_abort(esr
) && fixup_exception(regs
))
311 if (WARN_RATELIMIT(is_spurious_el1_translation_fault(addr
, esr
, regs
),
312 "Ignoring spurious kernel translation fault at virtual address %016lx\n", addr
))
315 if (is_el1_permission_fault(addr
, esr
, regs
)) {
316 if (esr
& ESR_ELx_WNR
)
317 msg
= "write to read-only memory";
318 else if (is_el1_instruction_abort(esr
))
319 msg
= "execute from non-executable memory";
321 msg
= "read from unreadable memory";
322 } else if (addr
< PAGE_SIZE
) {
323 msg
= "NULL pointer dereference";
325 msg
= "paging request";
328 die_kernel_fault(msg
, addr
, esr
, regs
);
331 static void set_thread_esr(unsigned long address
, unsigned int esr
)
333 current
->thread
.fault_address
= address
;
336 * If the faulting address is in the kernel, we must sanitize the ESR.
337 * From userspace's point of view, kernel-only mappings don't exist
338 * at all, so we report them as level 0 translation faults.
339 * (This is not quite the way that "no mapping there at all" behaves:
340 * an alignment fault not caused by the memory type would take
341 * precedence over translation fault for a real access to empty
342 * space. Unfortunately we can't easily distinguish "alignment fault
343 * not caused by memory type" from "alignment fault caused by memory
344 * type", so we ignore this wrinkle and just return the translation
347 if (!is_ttbr0_addr(current
->thread
.fault_address
)) {
348 switch (ESR_ELx_EC(esr
)) {
349 case ESR_ELx_EC_DABT_LOW
:
351 * These bits provide only information about the
352 * faulting instruction, which userspace knows already.
353 * We explicitly clear bits which are architecturally
354 * RES0 in case they are given meanings in future.
355 * We always report the ESR as if the fault was taken
356 * to EL1 and so ISV and the bits in ISS[23:14] are
357 * clear. (In fact it always will be a fault to EL1.)
359 esr
&= ESR_ELx_EC_MASK
| ESR_ELx_IL
|
360 ESR_ELx_CM
| ESR_ELx_WNR
;
361 esr
|= ESR_ELx_FSC_FAULT
;
363 case ESR_ELx_EC_IABT_LOW
:
365 * Claim a level 0 translation fault.
366 * All other bits are architecturally RES0 for faults
367 * reported with that DFSC value, so we clear them.
369 esr
&= ESR_ELx_EC_MASK
| ESR_ELx_IL
;
370 esr
|= ESR_ELx_FSC_FAULT
;
374 * This should never happen (entry.S only brings us
375 * into this code for insn and data aborts from a lower
376 * exception level). Fail safe by not providing an ESR
377 * context record at all.
379 WARN(1, "ESR 0x%x is not DABT or IABT from EL0\n", esr
);
385 current
->thread
.fault_code
= esr
;
388 static void do_bad_area(unsigned long addr
, unsigned int esr
, struct pt_regs
*regs
)
391 * If we are in kernel mode at this point, we have no context to
392 * handle this fault with.
394 if (user_mode(regs
)) {
395 const struct fault_info
*inf
= esr_to_fault_info(esr
);
397 set_thread_esr(addr
, esr
);
398 arm64_force_sig_fault(inf
->sig
, inf
->code
, (void __user
*)addr
,
401 __do_kernel_fault(addr
, esr
, regs
);
405 #define VM_FAULT_BADMAP 0x010000
406 #define VM_FAULT_BADACCESS 0x020000
408 static vm_fault_t
__do_page_fault(struct mm_struct
*mm
, unsigned long addr
,
409 unsigned int mm_flags
, unsigned long vm_flags
,
410 struct pt_regs
*regs
)
412 struct vm_area_struct
*vma
= find_vma(mm
, addr
);
415 return VM_FAULT_BADMAP
;
418 * Ok, we have a good vm_area for this memory access, so we can handle
421 if (unlikely(vma
->vm_start
> addr
)) {
422 if (!(vma
->vm_flags
& VM_GROWSDOWN
))
423 return VM_FAULT_BADMAP
;
424 if (expand_stack(vma
, addr
))
425 return VM_FAULT_BADMAP
;
429 * Check that the permissions on the VMA allow for the fault which
432 if (!(vma
->vm_flags
& vm_flags
))
433 return VM_FAULT_BADACCESS
;
434 return handle_mm_fault(vma
, addr
& PAGE_MASK
, mm_flags
, regs
);
437 static bool is_el0_instruction_abort(unsigned int esr
)
439 return ESR_ELx_EC(esr
) == ESR_ELx_EC_IABT_LOW
;
443 * Note: not valid for EL1 DC IVAC, but we never use that such that it
444 * should fault. EL0 cannot issue DC IVAC (undef).
446 static bool is_write_abort(unsigned int esr
)
448 return (esr
& ESR_ELx_WNR
) && !(esr
& ESR_ELx_CM
);
451 static int __kprobes
do_page_fault(unsigned long addr
, unsigned int esr
,
452 struct pt_regs
*regs
)
454 const struct fault_info
*inf
;
455 struct mm_struct
*mm
= current
->mm
;
457 unsigned long vm_flags
= VM_ACCESS_FLAGS
;
458 unsigned int mm_flags
= FAULT_FLAG_DEFAULT
;
460 if (kprobe_page_fault(regs
, esr
))
464 * If we're in an interrupt or have no user context, we must not take
467 if (faulthandler_disabled() || !mm
)
471 mm_flags
|= FAULT_FLAG_USER
;
473 if (is_el0_instruction_abort(esr
)) {
475 mm_flags
|= FAULT_FLAG_INSTRUCTION
;
476 } else if (is_write_abort(esr
)) {
478 mm_flags
|= FAULT_FLAG_WRITE
;
481 if (is_ttbr0_addr(addr
) && is_el1_permission_fault(addr
, esr
, regs
)) {
482 /* regs->orig_addr_limit may be 0 if we entered from EL0 */
483 if (regs
->orig_addr_limit
== KERNEL_DS
)
484 die_kernel_fault("access to user memory with fs=KERNEL_DS",
487 if (is_el1_instruction_abort(esr
))
488 die_kernel_fault("execution of user memory",
491 if (!search_exception_tables(regs
->pc
))
492 die_kernel_fault("access to user memory outside uaccess routines",
496 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS
, 1, regs
, addr
);
499 * As per x86, we may deadlock here. However, since the kernel only
500 * validly references user space from well defined areas of the code,
501 * we can bug out early if this is from code which shouldn't.
503 if (!mmap_read_trylock(mm
)) {
504 if (!user_mode(regs
) && !search_exception_tables(regs
->pc
))
510 * The above down_read_trylock() might have succeeded in which
511 * case, we'll have missed the might_sleep() from down_read().
514 #ifdef CONFIG_DEBUG_VM
515 if (!user_mode(regs
) && !search_exception_tables(regs
->pc
)) {
516 mmap_read_unlock(mm
);
522 fault
= __do_page_fault(mm
, addr
, mm_flags
, vm_flags
, regs
);
524 /* Quick path to respond to signals */
525 if (fault_signal_pending(fault
, regs
)) {
526 if (!user_mode(regs
))
531 if (fault
& VM_FAULT_RETRY
) {
532 if (mm_flags
& FAULT_FLAG_ALLOW_RETRY
) {
533 mm_flags
|= FAULT_FLAG_TRIED
;
537 mmap_read_unlock(mm
);
540 * Handle the "normal" (no error) case first.
542 if (likely(!(fault
& (VM_FAULT_ERROR
| VM_FAULT_BADMAP
|
543 VM_FAULT_BADACCESS
))))
547 * If we are in kernel mode at this point, we have no context to
548 * handle this fault with.
550 if (!user_mode(regs
))
553 if (fault
& VM_FAULT_OOM
) {
555 * We ran out of memory, call the OOM killer, and return to
556 * userspace (which will retry the fault, or kill us if we got
559 pagefault_out_of_memory();
563 inf
= esr_to_fault_info(esr
);
564 set_thread_esr(addr
, esr
);
565 if (fault
& VM_FAULT_SIGBUS
) {
567 * We had some memory, but were unable to successfully fix up
570 arm64_force_sig_fault(SIGBUS
, BUS_ADRERR
, (void __user
*)addr
,
572 } else if (fault
& (VM_FAULT_HWPOISON_LARGE
| VM_FAULT_HWPOISON
)) {
576 if (fault
& VM_FAULT_HWPOISON_LARGE
)
577 lsb
= hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault
));
579 arm64_force_sig_mceerr(BUS_MCEERR_AR
, (void __user
*)addr
, lsb
,
583 * Something tried to access memory that isn't in our memory
586 arm64_force_sig_fault(SIGSEGV
,
587 fault
== VM_FAULT_BADACCESS
? SEGV_ACCERR
: SEGV_MAPERR
,
595 __do_kernel_fault(addr
, esr
, regs
);
599 static int __kprobes
do_translation_fault(unsigned long addr
,
601 struct pt_regs
*regs
)
603 if (is_ttbr0_addr(addr
))
604 return do_page_fault(addr
, esr
, regs
);
606 do_bad_area(addr
, esr
, regs
);
610 static int do_alignment_fault(unsigned long addr
, unsigned int esr
,
611 struct pt_regs
*regs
)
613 do_bad_area(addr
, esr
, regs
);
617 static int do_bad(unsigned long addr
, unsigned int esr
, struct pt_regs
*regs
)
619 return 1; /* "fault" */
622 static int do_sea(unsigned long addr
, unsigned int esr
, struct pt_regs
*regs
)
624 const struct fault_info
*inf
;
627 inf
= esr_to_fault_info(esr
);
629 if (user_mode(regs
) && apei_claim_sea(regs
) == 0) {
631 * APEI claimed this as a firmware-first notification.
632 * Some processing deferred to task_work before ret_to_user().
637 if (esr
& ESR_ELx_FnV
)
640 siaddr
= (void __user
*)addr
;
641 arm64_notify_die(inf
->name
, regs
, inf
->sig
, inf
->code
, siaddr
, esr
);
646 static const struct fault_info fault_info
[] = {
647 { do_bad
, SIGKILL
, SI_KERNEL
, "ttbr address size fault" },
648 { do_bad
, SIGKILL
, SI_KERNEL
, "level 1 address size fault" },
649 { do_bad
, SIGKILL
, SI_KERNEL
, "level 2 address size fault" },
650 { do_bad
, SIGKILL
, SI_KERNEL
, "level 3 address size fault" },
651 { do_translation_fault
, SIGSEGV
, SEGV_MAPERR
, "level 0 translation fault" },
652 { do_translation_fault
, SIGSEGV
, SEGV_MAPERR
, "level 1 translation fault" },
653 { do_translation_fault
, SIGSEGV
, SEGV_MAPERR
, "level 2 translation fault" },
654 { do_translation_fault
, SIGSEGV
, SEGV_MAPERR
, "level 3 translation fault" },
655 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 8" },
656 { do_page_fault
, SIGSEGV
, SEGV_ACCERR
, "level 1 access flag fault" },
657 { do_page_fault
, SIGSEGV
, SEGV_ACCERR
, "level 2 access flag fault" },
658 { do_page_fault
, SIGSEGV
, SEGV_ACCERR
, "level 3 access flag fault" },
659 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 12" },
660 { do_page_fault
, SIGSEGV
, SEGV_ACCERR
, "level 1 permission fault" },
661 { do_page_fault
, SIGSEGV
, SEGV_ACCERR
, "level 2 permission fault" },
662 { do_page_fault
, SIGSEGV
, SEGV_ACCERR
, "level 3 permission fault" },
663 { do_sea
, SIGBUS
, BUS_OBJERR
, "synchronous external abort" },
664 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 17" },
665 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 18" },
666 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 19" },
667 { do_sea
, SIGKILL
, SI_KERNEL
, "level 0 (translation table walk)" },
668 { do_sea
, SIGKILL
, SI_KERNEL
, "level 1 (translation table walk)" },
669 { do_sea
, SIGKILL
, SI_KERNEL
, "level 2 (translation table walk)" },
670 { do_sea
, SIGKILL
, SI_KERNEL
, "level 3 (translation table walk)" },
671 { do_sea
, SIGBUS
, BUS_OBJERR
, "synchronous parity or ECC error" }, // Reserved when RAS is implemented
672 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 25" },
673 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 26" },
674 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 27" },
675 { do_sea
, SIGKILL
, SI_KERNEL
, "level 0 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
676 { do_sea
, SIGKILL
, SI_KERNEL
, "level 1 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
677 { do_sea
, SIGKILL
, SI_KERNEL
, "level 2 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
678 { do_sea
, SIGKILL
, SI_KERNEL
, "level 3 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented
679 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 32" },
680 { do_alignment_fault
, SIGBUS
, BUS_ADRALN
, "alignment fault" },
681 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 34" },
682 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 35" },
683 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 36" },
684 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 37" },
685 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 38" },
686 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 39" },
687 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 40" },
688 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 41" },
689 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 42" },
690 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 43" },
691 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 44" },
692 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 45" },
693 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 46" },
694 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 47" },
695 { do_bad
, SIGKILL
, SI_KERNEL
, "TLB conflict abort" },
696 { do_bad
, SIGKILL
, SI_KERNEL
, "Unsupported atomic hardware update fault" },
697 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 50" },
698 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 51" },
699 { do_bad
, SIGKILL
, SI_KERNEL
, "implementation fault (lockdown abort)" },
700 { do_bad
, SIGBUS
, BUS_OBJERR
, "implementation fault (unsupported exclusive)" },
701 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 54" },
702 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 55" },
703 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 56" },
704 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 57" },
705 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 58" },
706 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 59" },
707 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 60" },
708 { do_bad
, SIGKILL
, SI_KERNEL
, "section domain fault" },
709 { do_bad
, SIGKILL
, SI_KERNEL
, "page domain fault" },
710 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 63" },
713 void do_mem_abort(unsigned long addr
, unsigned int esr
, struct pt_regs
*regs
)
715 const struct fault_info
*inf
= esr_to_fault_info(esr
);
717 if (!inf
->fn(addr
, esr
, regs
))
720 if (!user_mode(regs
)) {
721 pr_alert("Unhandled fault at 0x%016lx\n", addr
);
722 mem_abort_decode(esr
);
726 arm64_notify_die(inf
->name
, regs
,
727 inf
->sig
, inf
->code
, (void __user
*)addr
, esr
);
729 NOKPROBE_SYMBOL(do_mem_abort
);
731 void do_el0_irq_bp_hardening(void)
733 /* PC has already been checked in entry.S */
734 arm64_apply_bp_hardening();
736 NOKPROBE_SYMBOL(do_el0_irq_bp_hardening
);
738 void do_sp_pc_abort(unsigned long addr
, unsigned int esr
, struct pt_regs
*regs
)
740 arm64_notify_die("SP/PC alignment exception", regs
,
741 SIGBUS
, BUS_ADRALN
, (void __user
*)addr
, esr
);
743 NOKPROBE_SYMBOL(do_sp_pc_abort
);
745 int __init
early_brk64(unsigned long addr
, unsigned int esr
,
746 struct pt_regs
*regs
);
749 * __refdata because early_brk64 is __init, but the reference to it is
750 * clobbered at arch_initcall time.
751 * See traps.c and debug-monitors.c:debug_traps_init().
753 static struct fault_info __refdata debug_fault_info
[] = {
754 { do_bad
, SIGTRAP
, TRAP_HWBKPT
, "hardware breakpoint" },
755 { do_bad
, SIGTRAP
, TRAP_HWBKPT
, "hardware single-step" },
756 { do_bad
, SIGTRAP
, TRAP_HWBKPT
, "hardware watchpoint" },
757 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 3" },
758 { do_bad
, SIGTRAP
, TRAP_BRKPT
, "aarch32 BKPT" },
759 { do_bad
, SIGKILL
, SI_KERNEL
, "aarch32 vector catch" },
760 { early_brk64
, SIGTRAP
, TRAP_BRKPT
, "aarch64 BRK" },
761 { do_bad
, SIGKILL
, SI_KERNEL
, "unknown 7" },
764 void __init
hook_debug_fault_code(int nr
,
765 int (*fn
)(unsigned long, unsigned int, struct pt_regs
*),
766 int sig
, int code
, const char *name
)
768 BUG_ON(nr
< 0 || nr
>= ARRAY_SIZE(debug_fault_info
));
770 debug_fault_info
[nr
].fn
= fn
;
771 debug_fault_info
[nr
].sig
= sig
;
772 debug_fault_info
[nr
].code
= code
;
773 debug_fault_info
[nr
].name
= name
;
777 * In debug exception context, we explicitly disable preemption despite
778 * having interrupts disabled.
779 * This serves two purposes: it makes it much less likely that we would
780 * accidentally schedule in exception context and it will force a warning
781 * if we somehow manage to schedule by accident.
783 static void debug_exception_enter(struct pt_regs
*regs
)
786 * Tell lockdep we disabled irqs in entry.S. Do nothing if they were
787 * already disabled to preserve the last enabled/disabled addresses.
789 if (interrupts_enabled(regs
))
790 trace_hardirqs_off();
792 if (user_mode(regs
)) {
793 RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
796 * We might have interrupted pretty much anything. In
797 * fact, if we're a debug exception, we can even interrupt
798 * NMI processing. We don't want this code makes in_nmi()
799 * to return true, but we need to notify RCU.
806 /* This code is a bit fragile. Test it. */
807 RCU_LOCKDEP_WARN(!rcu_is_watching(), "exception_enter didn't work");
809 NOKPROBE_SYMBOL(debug_exception_enter
);
811 static void debug_exception_exit(struct pt_regs
*regs
)
813 preempt_enable_no_resched();
815 if (!user_mode(regs
))
818 if (interrupts_enabled(regs
))
821 NOKPROBE_SYMBOL(debug_exception_exit
);
823 #ifdef CONFIG_ARM64_ERRATUM_1463225
824 DECLARE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa
);
826 static int cortex_a76_erratum_1463225_debug_handler(struct pt_regs
*regs
)
831 if (!__this_cpu_read(__in_cortex_a76_erratum_1463225_wa
))
835 * We've taken a dummy step exception from the kernel to ensure
836 * that interrupts are re-enabled on the syscall path. Return back
837 * to cortex_a76_erratum_1463225_svc_handler() with debug exceptions
838 * masked so that we can safely restore the mdscr and get on with
839 * handling the syscall.
841 regs
->pstate
|= PSR_D_BIT
;
845 static int cortex_a76_erratum_1463225_debug_handler(struct pt_regs
*regs
)
849 #endif /* CONFIG_ARM64_ERRATUM_1463225 */
850 NOKPROBE_SYMBOL(cortex_a76_erratum_1463225_debug_handler
);
852 void do_debug_exception(unsigned long addr_if_watchpoint
, unsigned int esr
,
853 struct pt_regs
*regs
)
855 const struct fault_info
*inf
= esr_to_debug_fault_info(esr
);
856 unsigned long pc
= instruction_pointer(regs
);
858 if (cortex_a76_erratum_1463225_debug_handler(regs
))
861 debug_exception_enter(regs
);
863 if (user_mode(regs
) && !is_ttbr0_addr(pc
))
864 arm64_apply_bp_hardening();
866 if (inf
->fn(addr_if_watchpoint
, esr
, regs
)) {
867 arm64_notify_die(inf
->name
, regs
,
868 inf
->sig
, inf
->code
, (void __user
*)pc
, esr
);
871 debug_exception_exit(regs
);
873 NOKPROBE_SYMBOL(do_debug_exception
);