]>
Commit | Line | Data |
---|---|---|
60ffc30d CM |
1 | /* |
2 | * Based on arch/arm/kernel/traps.c | |
3 | * | |
4 | * Copyright (C) 1995-2009 Russell King | |
5 | * Copyright (C) 2012 ARM Ltd. | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License version 2 as | |
9 | * published by the Free Software Foundation. | |
10 | * | |
11 | * This program is distributed in the hope that it will be useful, | |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | * GNU General Public License for more details. | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License | |
17 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
18 | */ | |
19 | ||
9fb7410f | 20 | #include <linux/bug.h> |
60ffc30d CM |
21 | #include <linux/signal.h> |
22 | #include <linux/personality.h> | |
23 | #include <linux/kallsyms.h> | |
24 | #include <linux/spinlock.h> | |
25 | #include <linux/uaccess.h> | |
26 | #include <linux/hardirq.h> | |
27 | #include <linux/kdebug.h> | |
28 | #include <linux/module.h> | |
29 | #include <linux/kexec.h> | |
30 | #include <linux/delay.h> | |
31 | #include <linux/init.h> | |
3f07c014 | 32 | #include <linux/sched/signal.h> |
b17b0153 | 33 | #include <linux/sched/debug.h> |
68db0cf1 | 34 | #include <linux/sched/task_stack.h> |
872d8327 | 35 | #include <linux/sizes.h> |
60ffc30d | 36 | #include <linux/syscalls.h> |
589ee628 | 37 | #include <linux/mm_types.h> |
41eea9cd | 38 | #include <linux/kasan.h> |
60ffc30d CM |
39 | |
40 | #include <asm/atomic.h> | |
9fb7410f | 41 | #include <asm/bug.h> |
c0cda3b8 | 42 | #include <asm/cpufeature.h> |
0fbeb318 | 43 | #include <asm/daifflags.h> |
1442b6ed | 44 | #include <asm/debug-monitors.h> |
60a1f02c | 45 | #include <asm/esr.h> |
9fb7410f | 46 | #include <asm/insn.h> |
60ffc30d | 47 | #include <asm/traps.h> |
872d8327 | 48 | #include <asm/smp.h> |
a9ea0017 | 49 | #include <asm/stack_pointer.h> |
60ffc30d CM |
50 | #include <asm/stacktrace.h> |
51 | #include <asm/exception.h> | |
52 | #include <asm/system_misc.h> | |
7dd01aef | 53 | #include <asm/sysreg.h> |
60ffc30d CM |
54 | |
55 | static const char *handler[]= { | |
56 | "Synchronous Abort", | |
57 | "IRQ", | |
58 | "FIQ", | |
59 | "Error" | |
60 | }; | |
61 | ||
5ee39a71 | 62 | int show_unhandled_signals = 0; |
60ffc30d | 63 | |
9f93f3e9 | 64 | static void dump_backtrace_entry(unsigned long where) |
60ffc30d | 65 | { |
a25ffd3a | 66 | printk(" %pS\n", (void *)where); |
60ffc30d CM |
67 | } |
68 | ||
c5cea06b | 69 | static void __dump_instr(const char *lvl, struct pt_regs *regs) |
60ffc30d CM |
70 | { |
71 | unsigned long addr = instruction_pointer(regs); | |
60ffc30d CM |
72 | char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str; |
73 | int i; | |
74 | ||
60ffc30d CM |
75 | for (i = -4; i < 1; i++) { |
76 | unsigned int val, bad; | |
77 | ||
7a7003b1 | 78 | bad = get_user(val, &((u32 *)addr)[i]); |
60ffc30d CM |
79 | |
80 | if (!bad) | |
81 | p += sprintf(p, i == 0 ? "(%08x) " : "%08x ", val); | |
82 | else { | |
83 | p += sprintf(p, "bad PC value"); | |
84 | break; | |
85 | } | |
86 | } | |
87 | printk("%sCode: %s\n", lvl, str); | |
c5cea06b | 88 | } |
60ffc30d | 89 | |
c5cea06b MR |
90 | static void dump_instr(const char *lvl, struct pt_regs *regs) |
91 | { | |
92 | if (!user_mode(regs)) { | |
93 | mm_segment_t fs = get_fs(); | |
94 | set_fs(KERNEL_DS); | |
95 | __dump_instr(lvl, regs); | |
96 | set_fs(fs); | |
97 | } else { | |
98 | __dump_instr(lvl, regs); | |
99 | } | |
60ffc30d CM |
100 | } |
101 | ||
1149aad1 | 102 | void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) |
60ffc30d CM |
103 | { |
104 | struct stackframe frame; | |
1e6f5440 | 105 | int skip = 0; |
60ffc30d | 106 | |
b5e7307d MR |
107 | pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk); |
108 | ||
1e6f5440 WD |
109 | if (regs) { |
110 | if (user_mode(regs)) | |
111 | return; | |
112 | skip = 1; | |
113 | } | |
114 | ||
b5e7307d MR |
115 | if (!tsk) |
116 | tsk = current; | |
117 | ||
9bbd4c56 MR |
118 | if (!try_get_task_stack(tsk)) |
119 | return; | |
120 | ||
20380bb3 | 121 | if (tsk == current) { |
60ffc30d | 122 | frame.fp = (unsigned long)__builtin_frame_address(0); |
60ffc30d CM |
123 | frame.pc = (unsigned long)dump_backtrace; |
124 | } else { | |
125 | /* | |
126 | * task blocked in __switch_to | |
127 | */ | |
128 | frame.fp = thread_saved_fp(tsk); | |
60ffc30d CM |
129 | frame.pc = thread_saved_pc(tsk); |
130 | } | |
20380bb3 | 131 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
a448276c | 132 | frame.graph = 0; |
20380bb3 | 133 | #endif |
60ffc30d | 134 | |
c9cd0ed9 | 135 | printk("Call trace:\n"); |
a25ffd3a | 136 | do { |
20380bb3 AT |
137 | /* skip until specified stack frame */ |
138 | if (!skip) { | |
73267498 | 139 | dump_backtrace_entry(frame.pc); |
20380bb3 AT |
140 | } else if (frame.fp == regs->regs[29]) { |
141 | skip = 0; | |
142 | /* | |
143 | * Mostly, this is the case where this function is | |
144 | * called in panic/abort. As exception handler's | |
145 | * stack frame does not contain the corresponding pc | |
146 | * at which an exception has taken place, use regs->pc | |
147 | * instead. | |
148 | */ | |
149 | dump_backtrace_entry(regs->pc); | |
150 | } | |
a25ffd3a | 151 | } while (!unwind_frame(tsk, &frame)); |
9bbd4c56 MR |
152 | |
153 | put_task_stack(tsk); | |
60ffc30d CM |
154 | } |
155 | ||
60ffc30d CM |
156 | void show_stack(struct task_struct *tsk, unsigned long *sp) |
157 | { | |
158 | dump_backtrace(NULL, tsk); | |
159 | barrier(); | |
160 | } | |
161 | ||
162 | #ifdef CONFIG_PREEMPT | |
163 | #define S_PREEMPT " PREEMPT" | |
164 | #else | |
165 | #define S_PREEMPT "" | |
166 | #endif | |
60ffc30d | 167 | #define S_SMP " SMP" |
60ffc30d | 168 | |
876e7a38 | 169 | static int __die(const char *str, int err, struct pt_regs *regs) |
60ffc30d | 170 | { |
876e7a38 | 171 | struct task_struct *tsk = current; |
60ffc30d CM |
172 | static int die_counter; |
173 | int ret; | |
174 | ||
175 | pr_emerg("Internal error: %s: %x [#%d]" S_PREEMPT S_SMP "\n", | |
176 | str, err, ++die_counter); | |
177 | ||
178 | /* trap and error numbers are mostly meaningless on ARM */ | |
179 | ret = notify_die(DIE_OOPS, str, regs, err, 0, SIGSEGV); | |
180 | if (ret == NOTIFY_STOP) | |
181 | return ret; | |
182 | ||
183 | print_modules(); | |
60ffc30d | 184 | pr_emerg("Process %.*s (pid: %d, stack limit = 0x%p)\n", |
876e7a38 MR |
185 | TASK_COMM_LEN, tsk->comm, task_pid_nr(tsk), |
186 | end_of_stack(tsk)); | |
1e6f5440 | 187 | show_regs(regs); |
60ffc30d | 188 | |
1e6f5440 | 189 | if (!user_mode(regs)) |
60ffc30d | 190 | dump_instr(KERN_EMERG, regs); |
60ffc30d CM |
191 | |
192 | return ret; | |
193 | } | |
194 | ||
195 | static DEFINE_RAW_SPINLOCK(die_lock); | |
196 | ||
197 | /* | |
198 | * This function is protected against re-entrancy. | |
199 | */ | |
200 | void die(const char *str, struct pt_regs *regs, int err) | |
201 | { | |
60ffc30d | 202 | int ret; |
6f44a0ba QZ |
203 | unsigned long flags; |
204 | ||
205 | raw_spin_lock_irqsave(&die_lock, flags); | |
60ffc30d CM |
206 | |
207 | oops_enter(); | |
208 | ||
60ffc30d CM |
209 | console_verbose(); |
210 | bust_spinlocks(1); | |
876e7a38 | 211 | ret = __die(str, err, regs); |
60ffc30d | 212 | |
876e7a38 | 213 | if (regs && kexec_should_crash(current)) |
60ffc30d CM |
214 | crash_kexec(regs); |
215 | ||
216 | bust_spinlocks(0); | |
373d4d09 | 217 | add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); |
60ffc30d CM |
218 | oops_exit(); |
219 | ||
220 | if (in_interrupt()) | |
221 | panic("Fatal exception in interrupt"); | |
222 | if (panic_on_oops) | |
223 | panic("Fatal exception"); | |
6f44a0ba QZ |
224 | |
225 | raw_spin_unlock_irqrestore(&die_lock, flags); | |
226 | ||
60ffc30d CM |
227 | if (ret != NOTIFY_STOP) |
228 | do_exit(SIGSEGV); | |
229 | } | |
230 | ||
1628a7cc | 231 | static void arm64_show_signal(int signo, const char *str) |
a26731d9 WD |
232 | { |
233 | static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, | |
234 | DEFAULT_RATELIMIT_BURST); | |
24b8f79d | 235 | struct task_struct *tsk = current; |
a1ece821 WD |
236 | unsigned int esr = tsk->thread.fault_code; |
237 | struct pt_regs *regs = task_pt_regs(tsk); | |
238 | ||
1628a7cc EB |
239 | /* Leave if the signal won't be shown */ |
240 | if (!show_unhandled_signals || | |
241 | !unhandled_signal(tsk, signo) || | |
242 | !__ratelimit(&rs)) | |
243 | return; | |
a1ece821 WD |
244 | |
245 | pr_info("%s[%d]: unhandled exception: ", tsk->comm, task_pid_nr(tsk)); | |
246 | if (esr) | |
247 | pr_cont("%s, ESR 0x%08x, ", esr_get_class_string(esr), esr); | |
248 | ||
249 | pr_cont("%s", str); | |
250 | print_vma_addr(KERN_CONT " in ", regs->pc); | |
251 | pr_cont("\n"); | |
252 | __show_regs(regs); | |
1628a7cc | 253 | } |
a1ece821 | 254 | |
feca355b EB |
255 | void arm64_force_sig_fault(int signo, int code, void __user *addr, |
256 | const char *str) | |
257 | { | |
258 | arm64_show_signal(signo, str); | |
259 | force_sig_fault(signo, code, addr, current); | |
260 | } | |
a1ece821 | 261 | |
b4d5557c EB |
262 | void arm64_force_sig_mceerr(int code, void __user *addr, short lsb, |
263 | const char *str) | |
264 | { | |
265 | arm64_show_signal(SIGBUS, str); | |
266 | force_sig_mceerr(code, addr, lsb, current); | |
267 | } | |
268 | ||
f3a900b3 EB |
269 | void arm64_force_sig_ptrace_errno_trap(int errno, void __user *addr, |
270 | const char *str) | |
271 | { | |
272 | arm64_show_signal(SIGTRAP, str); | |
273 | force_sig_ptrace_errno_trap(errno, addr); | |
a1ece821 WD |
274 | } |
275 | ||
60ffc30d | 276 | void arm64_notify_die(const char *str, struct pt_regs *regs, |
6fa998e8 EB |
277 | int signo, int sicode, void __user *addr, |
278 | int err) | |
60ffc30d | 279 | { |
9141300a | 280 | if (user_mode(regs)) { |
a1ece821 | 281 | WARN_ON(regs != current_pt_regs()); |
9141300a CM |
282 | current->thread.fault_address = 0; |
283 | current->thread.fault_code = err; | |
6fa998e8 | 284 | |
feca355b | 285 | arm64_force_sig_fault(signo, sicode, addr, str); |
9141300a | 286 | } else { |
60ffc30d | 287 | die(str, regs, err); |
9141300a | 288 | } |
60ffc30d CM |
289 | } |
290 | ||
6436beee JT |
291 | void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size) |
292 | { | |
293 | regs->pc += size; | |
294 | ||
295 | /* | |
296 | * If we were single stepping, we want to get the step exception after | |
297 | * we return from the trap. | |
298 | */ | |
9478f192 MR |
299 | if (user_mode(regs)) |
300 | user_fastforward_single_step(current); | |
6436beee JT |
301 | } |
302 | ||
9b79f52d PA |
303 | static LIST_HEAD(undef_hook); |
304 | static DEFINE_RAW_SPINLOCK(undef_lock); | |
305 | ||
306 | void register_undef_hook(struct undef_hook *hook) | |
307 | { | |
308 | unsigned long flags; | |
309 | ||
310 | raw_spin_lock_irqsave(&undef_lock, flags); | |
311 | list_add(&hook->node, &undef_hook); | |
312 | raw_spin_unlock_irqrestore(&undef_lock, flags); | |
313 | } | |
314 | ||
315 | void unregister_undef_hook(struct undef_hook *hook) | |
316 | { | |
317 | unsigned long flags; | |
318 | ||
319 | raw_spin_lock_irqsave(&undef_lock, flags); | |
320 | list_del(&hook->node); | |
321 | raw_spin_unlock_irqrestore(&undef_lock, flags); | |
322 | } | |
323 | ||
324 | static int call_undef_hook(struct pt_regs *regs) | |
325 | { | |
326 | struct undef_hook *hook; | |
327 | unsigned long flags; | |
328 | u32 instr; | |
329 | int (*fn)(struct pt_regs *regs, u32 instr) = NULL; | |
330 | void __user *pc = (void __user *)instruction_pointer(regs); | |
331 | ||
0bf0f444 WD |
332 | if (!user_mode(regs)) { |
333 | __le32 instr_le; | |
334 | if (probe_kernel_address((__force __le32 *)pc, instr_le)) | |
335 | goto exit; | |
336 | instr = le32_to_cpu(instr_le); | |
337 | } else if (compat_thumb_mode(regs)) { | |
9b79f52d | 338 | /* 16-bit Thumb instruction */ |
6cf5d4af LVO |
339 | __le16 instr_le; |
340 | if (get_user(instr_le, (__le16 __user *)pc)) | |
9b79f52d | 341 | goto exit; |
6cf5d4af | 342 | instr = le16_to_cpu(instr_le); |
9b79f52d PA |
343 | if (aarch32_insn_is_wide(instr)) { |
344 | u32 instr2; | |
345 | ||
6cf5d4af | 346 | if (get_user(instr_le, (__le16 __user *)(pc + 2))) |
9b79f52d | 347 | goto exit; |
6cf5d4af | 348 | instr2 = le16_to_cpu(instr_le); |
9b79f52d PA |
349 | instr = (instr << 16) | instr2; |
350 | } | |
351 | } else { | |
352 | /* 32-bit ARM instruction */ | |
6cf5d4af LVO |
353 | __le32 instr_le; |
354 | if (get_user(instr_le, (__le32 __user *)pc)) | |
9b79f52d | 355 | goto exit; |
6cf5d4af | 356 | instr = le32_to_cpu(instr_le); |
9b79f52d PA |
357 | } |
358 | ||
359 | raw_spin_lock_irqsave(&undef_lock, flags); | |
360 | list_for_each_entry(hook, &undef_hook, node) | |
361 | if ((instr & hook->instr_mask) == hook->instr_val && | |
362 | (regs->pstate & hook->pstate_mask) == hook->pstate_val) | |
363 | fn = hook->fn; | |
364 | ||
365 | raw_spin_unlock_irqrestore(&undef_lock, flags); | |
366 | exit: | |
367 | return fn ? fn(regs, instr) : 1; | |
368 | } | |
369 | ||
2c9120f3 | 370 | void force_signal_inject(int signal, int code, unsigned long address) |
60ffc30d | 371 | { |
390bf177 | 372 | const char *desc; |
2c9120f3 WD |
373 | struct pt_regs *regs = current_pt_regs(); |
374 | ||
8a60419d WD |
375 | if (WARN_ON(!user_mode(regs))) |
376 | return; | |
377 | ||
390bf177 AP |
378 | switch (signal) { |
379 | case SIGILL: | |
380 | desc = "undefined instruction"; | |
381 | break; | |
382 | case SIGSEGV: | |
383 | desc = "illegal memory access"; | |
384 | break; | |
385 | default: | |
bc0ee476 | 386 | desc = "unknown or unrecoverable error"; |
390bf177 AP |
387 | break; |
388 | } | |
389 | ||
a7e6f1ca | 390 | /* Force signals we don't understand to SIGKILL */ |
b2d71b3c | 391 | if (WARN_ON(signal != SIGKILL && |
a7e6f1ca WD |
392 | siginfo_layout(signal, code) != SIL_FAULT)) { |
393 | signal = SIGKILL; | |
394 | } | |
395 | ||
6fa998e8 | 396 | arm64_notify_die(desc, regs, signal, code, (void __user *)address, 0); |
390bf177 AP |
397 | } |
398 | ||
399 | /* | |
400 | * Set up process info to signal segmentation fault - called on access error. | |
401 | */ | |
2c9120f3 | 402 | void arm64_notify_segfault(unsigned long addr) |
390bf177 AP |
403 | { |
404 | int code; | |
405 | ||
406 | down_read(¤t->mm->mmap_sem); | |
407 | if (find_vma(current->mm, addr) == NULL) | |
408 | code = SEGV_MAPERR; | |
409 | else | |
410 | code = SEGV_ACCERR; | |
411 | up_read(¤t->mm->mmap_sem); | |
60ffc30d | 412 | |
2c9120f3 | 413 | force_signal_inject(SIGSEGV, code, addr); |
390bf177 AP |
414 | } |
415 | ||
416 | asmlinkage void __exception do_undefinstr(struct pt_regs *regs) | |
417 | { | |
60ffc30d | 418 | /* check for AArch32 breakpoint instructions */ |
1442b6ed | 419 | if (!aarch32_break_handler(regs)) |
60ffc30d | 420 | return; |
60ffc30d | 421 | |
9b79f52d PA |
422 | if (call_undef_hook(regs) == 0) |
423 | return; | |
424 | ||
0bf0f444 | 425 | BUG_ON(!user_mode(regs)); |
8a60419d | 426 | force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc); |
60ffc30d CM |
427 | } |
428 | ||
7dd01aef | 429 | #define __user_cache_maint(insn, address, res) \ |
81cddd65 | 430 | if (address >= user_addr_max()) { \ |
87261d19 | 431 | res = -EFAULT; \ |
39bc88e5 CM |
432 | } else { \ |
433 | uaccess_ttbr0_enable(); \ | |
87261d19 AP |
434 | asm volatile ( \ |
435 | "1: " insn ", %1\n" \ | |
436 | " mov %w0, #0\n" \ | |
437 | "2:\n" \ | |
438 | " .pushsection .fixup,\"ax\"\n" \ | |
439 | " .align 2\n" \ | |
440 | "3: mov %w0, %w2\n" \ | |
441 | " b 2b\n" \ | |
442 | " .popsection\n" \ | |
443 | _ASM_EXTABLE(1b, 3b) \ | |
444 | : "=r" (res) \ | |
39bc88e5 CM |
445 | : "r" (address), "i" (-EFAULT)); \ |
446 | uaccess_ttbr0_disable(); \ | |
447 | } | |
7dd01aef | 448 | |
9dbd5bb2 | 449 | static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs) |
7dd01aef AP |
450 | { |
451 | unsigned long address; | |
1c839141 | 452 | int rt = ESR_ELx_SYS64_ISS_RT(esr); |
9dbd5bb2 SP |
453 | int crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT; |
454 | int ret = 0; | |
7dd01aef | 455 | |
81cddd65 | 456 | address = untagged_addr(pt_regs_read_reg(regs, rt)); |
7dd01aef | 457 | |
9dbd5bb2 SP |
458 | switch (crm) { |
459 | case ESR_ELx_SYS64_ISS_CRM_DC_CVAU: /* DC CVAU, gets promoted */ | |
460 | __user_cache_maint("dc civac", address, ret); | |
461 | break; | |
462 | case ESR_ELx_SYS64_ISS_CRM_DC_CVAC: /* DC CVAC, gets promoted */ | |
463 | __user_cache_maint("dc civac", address, ret); | |
464 | break; | |
d16ed410 AM |
465 | case ESR_ELx_SYS64_ISS_CRM_DC_CVADP: /* DC CVADP */ |
466 | __user_cache_maint("sys 3, c7, c13, 1", address, ret); | |
467 | break; | |
e1bc5d1b RM |
468 | case ESR_ELx_SYS64_ISS_CRM_DC_CVAP: /* DC CVAP */ |
469 | __user_cache_maint("sys 3, c7, c12, 1", address, ret); | |
470 | break; | |
9dbd5bb2 SP |
471 | case ESR_ELx_SYS64_ISS_CRM_DC_CIVAC: /* DC CIVAC */ |
472 | __user_cache_maint("dc civac", address, ret); | |
473 | break; | |
474 | case ESR_ELx_SYS64_ISS_CRM_IC_IVAU: /* IC IVAU */ | |
475 | __user_cache_maint("ic ivau", address, ret); | |
476 | break; | |
477 | default: | |
2c9120f3 | 478 | force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc); |
7dd01aef AP |
479 | return; |
480 | } | |
481 | ||
482 | if (ret) | |
2c9120f3 | 483 | arm64_notify_segfault(address); |
7dd01aef | 484 | else |
6436beee | 485 | arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); |
7dd01aef AP |
486 | } |
487 | ||
116c81f4 SP |
488 | static void ctr_read_handler(unsigned int esr, struct pt_regs *regs) |
489 | { | |
1c839141 | 490 | int rt = ESR_ELx_SYS64_ISS_RT(esr); |
8b6e70fc MR |
491 | unsigned long val = arm64_ftr_reg_user_value(&arm64_ftr_reg_ctrel0); |
492 | ||
493 | pt_regs_write_reg(regs, rt, val); | |
116c81f4 | 494 | |
6436beee | 495 | arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); |
116c81f4 SP |
496 | } |
497 | ||
6126ce05 MZ |
498 | static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs) |
499 | { | |
1c839141 | 500 | int rt = ESR_ELx_SYS64_ISS_RT(esr); |
6126ce05 | 501 | |
dea86a80 | 502 | pt_regs_write_reg(regs, rt, arch_timer_read_counter()); |
6436beee | 503 | arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); |
6126ce05 MZ |
504 | } |
505 | ||
9842119a MZ |
506 | static void cntfrq_read_handler(unsigned int esr, struct pt_regs *regs) |
507 | { | |
1c839141 | 508 | int rt = ESR_ELx_SYS64_ISS_RT(esr); |
9842119a | 509 | |
c6f97add | 510 | pt_regs_write_reg(regs, rt, arch_timer_get_rate()); |
6436beee | 511 | arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); |
9842119a MZ |
512 | } |
513 | ||
21f84796 AK |
514 | static void mrs_handler(unsigned int esr, struct pt_regs *regs) |
515 | { | |
516 | u32 sysreg, rt; | |
517 | ||
518 | rt = ESR_ELx_SYS64_ISS_RT(esr); | |
519 | sysreg = esr_sys64_to_sysreg(esr); | |
520 | ||
521 | if (do_emulate_mrs(regs, sysreg, rt) != 0) | |
522 | force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc); | |
523 | } | |
524 | ||
c219bc4e MZ |
525 | static void wfi_handler(unsigned int esr, struct pt_regs *regs) |
526 | { | |
527 | arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); | |
528 | } | |
529 | ||
9dbd5bb2 SP |
530 | struct sys64_hook { |
531 | unsigned int esr_mask; | |
532 | unsigned int esr_val; | |
533 | void (*handler)(unsigned int esr, struct pt_regs *regs); | |
534 | }; | |
535 | ||
536 | static struct sys64_hook sys64_hooks[] = { | |
537 | { | |
538 | .esr_mask = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_MASK, | |
539 | .esr_val = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_VAL, | |
540 | .handler = user_cache_maint_handler, | |
541 | }, | |
116c81f4 SP |
542 | { |
543 | /* Trap read access to CTR_EL0 */ | |
544 | .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, | |
545 | .esr_val = ESR_ELx_SYS64_ISS_SYS_CTR_READ, | |
546 | .handler = ctr_read_handler, | |
547 | }, | |
6126ce05 MZ |
548 | { |
549 | /* Trap read access to CNTVCT_EL0 */ | |
550 | .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, | |
551 | .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTVCT, | |
552 | .handler = cntvct_read_handler, | |
553 | }, | |
9842119a MZ |
554 | { |
555 | /* Trap read access to CNTFRQ_EL0 */ | |
556 | .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK, | |
557 | .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTFRQ, | |
558 | .handler = cntfrq_read_handler, | |
559 | }, | |
21f84796 AK |
560 | { |
561 | /* Trap read access to CPUID registers */ | |
562 | .esr_mask = ESR_ELx_SYS64_ISS_SYS_MRS_OP_MASK, | |
563 | .esr_val = ESR_ELx_SYS64_ISS_SYS_MRS_OP_VAL, | |
564 | .handler = mrs_handler, | |
565 | }, | |
c219bc4e MZ |
566 | { |
567 | /* Trap WFI instructions executed in userspace */ | |
568 | .esr_mask = ESR_ELx_WFx_MASK, | |
569 | .esr_val = ESR_ELx_WFx_WFI_VAL, | |
570 | .handler = wfi_handler, | |
571 | }, | |
9dbd5bb2 SP |
572 | {}, |
573 | }; | |
574 | ||
70c63cdf MZ |
575 | |
576 | #ifdef CONFIG_COMPAT | |
1f1c0140 MZ |
577 | #define PSTATE_IT_1_0_SHIFT 25 |
578 | #define PSTATE_IT_1_0_MASK (0x3 << PSTATE_IT_1_0_SHIFT) | |
579 | #define PSTATE_IT_7_2_SHIFT 10 | |
580 | #define PSTATE_IT_7_2_MASK (0x3f << PSTATE_IT_7_2_SHIFT) | |
581 | ||
582 | static u32 compat_get_it_state(struct pt_regs *regs) | |
583 | { | |
584 | u32 it, pstate = regs->pstate; | |
585 | ||
586 | it = (pstate & PSTATE_IT_1_0_MASK) >> PSTATE_IT_1_0_SHIFT; | |
587 | it |= ((pstate & PSTATE_IT_7_2_MASK) >> PSTATE_IT_7_2_SHIFT) << 2; | |
588 | ||
589 | return it; | |
590 | } | |
591 | ||
592 | static void compat_set_it_state(struct pt_regs *regs, u32 it) | |
593 | { | |
594 | u32 pstate_it; | |
595 | ||
596 | pstate_it = (it << PSTATE_IT_1_0_SHIFT) & PSTATE_IT_1_0_MASK; | |
597 | pstate_it |= ((it >> 2) << PSTATE_IT_7_2_SHIFT) & PSTATE_IT_7_2_MASK; | |
598 | ||
599 | regs->pstate &= ~PSR_AA32_IT_MASK; | |
600 | regs->pstate |= pstate_it; | |
601 | } | |
602 | ||
603 | static bool cp15_cond_valid(unsigned int esr, struct pt_regs *regs) | |
604 | { | |
605 | int cond; | |
606 | ||
607 | /* Only a T32 instruction can trap without CV being set */ | |
608 | if (!(esr & ESR_ELx_CV)) { | |
609 | u32 it; | |
610 | ||
611 | it = compat_get_it_state(regs); | |
612 | if (!it) | |
613 | return true; | |
614 | ||
615 | cond = it >> 4; | |
616 | } else { | |
617 | cond = (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT; | |
618 | } | |
619 | ||
620 | return aarch32_opcode_cond_checks[cond](regs->pstate); | |
621 | } | |
622 | ||
623 | static void advance_itstate(struct pt_regs *regs) | |
624 | { | |
625 | u32 it; | |
626 | ||
627 | /* ARM mode */ | |
628 | if (!(regs->pstate & PSR_AA32_T_BIT) || | |
629 | !(regs->pstate & PSR_AA32_IT_MASK)) | |
630 | return; | |
631 | ||
632 | it = compat_get_it_state(regs); | |
633 | ||
634 | /* | |
635 | * If this is the last instruction of the block, wipe the IT | |
636 | * state. Otherwise advance it. | |
637 | */ | |
638 | if (!(it & 7)) | |
639 | it = 0; | |
640 | else | |
641 | it = (it & 0xe0) | ((it << 1) & 0x1f); | |
642 | ||
643 | compat_set_it_state(regs, it); | |
644 | } | |
645 | ||
646 | static void arm64_compat_skip_faulting_instruction(struct pt_regs *regs, | |
647 | unsigned int sz) | |
648 | { | |
649 | advance_itstate(regs); | |
650 | arm64_skip_faulting_instruction(regs, sz); | |
651 | } | |
652 | ||
32a3e635 MZ |
653 | static void compat_cntfrq_read_handler(unsigned int esr, struct pt_regs *regs) |
654 | { | |
655 | int reg = (esr & ESR_ELx_CP15_32_ISS_RT_MASK) >> ESR_ELx_CP15_32_ISS_RT_SHIFT; | |
656 | ||
657 | pt_regs_write_reg(regs, reg, arch_timer_get_rate()); | |
658 | arm64_compat_skip_faulting_instruction(regs, 4); | |
659 | } | |
660 | ||
2a8905e1 | 661 | static struct sys64_hook cp15_32_hooks[] = { |
32a3e635 MZ |
662 | { |
663 | .esr_mask = ESR_ELx_CP15_32_ISS_SYS_MASK, | |
664 | .esr_val = ESR_ELx_CP15_32_ISS_SYS_CNTFRQ, | |
665 | .handler = compat_cntfrq_read_handler, | |
666 | }, | |
2a8905e1 MZ |
667 | {}, |
668 | }; | |
669 | ||
50de013d MZ |
670 | static void compat_cntvct_read_handler(unsigned int esr, struct pt_regs *regs) |
671 | { | |
672 | int rt = (esr & ESR_ELx_CP15_64_ISS_RT_MASK) >> ESR_ELx_CP15_64_ISS_RT_SHIFT; | |
673 | int rt2 = (esr & ESR_ELx_CP15_64_ISS_RT2_MASK) >> ESR_ELx_CP15_64_ISS_RT2_SHIFT; | |
dea86a80 | 674 | u64 val = arch_timer_read_counter(); |
50de013d MZ |
675 | |
676 | pt_regs_write_reg(regs, rt, lower_32_bits(val)); | |
677 | pt_regs_write_reg(regs, rt2, upper_32_bits(val)); | |
678 | arm64_compat_skip_faulting_instruction(regs, 4); | |
679 | } | |
680 | ||
2a8905e1 | 681 | static struct sys64_hook cp15_64_hooks[] = { |
50de013d MZ |
682 | { |
683 | .esr_mask = ESR_ELx_CP15_64_ISS_SYS_MASK, | |
684 | .esr_val = ESR_ELx_CP15_64_ISS_SYS_CNTVCT, | |
685 | .handler = compat_cntvct_read_handler, | |
686 | }, | |
2a8905e1 MZ |
687 | {}, |
688 | }; | |
689 | ||
70c63cdf MZ |
690 | asmlinkage void __exception do_cp15instr(unsigned int esr, struct pt_regs *regs) |
691 | { | |
2a8905e1 MZ |
692 | struct sys64_hook *hook, *hook_base; |
693 | ||
1f1c0140 MZ |
694 | if (!cp15_cond_valid(esr, regs)) { |
695 | /* | |
696 | * There is no T16 variant of a CP access, so we | |
697 | * always advance PC by 4 bytes. | |
698 | */ | |
699 | arm64_compat_skip_faulting_instruction(regs, 4); | |
700 | return; | |
701 | } | |
702 | ||
2a8905e1 MZ |
703 | switch (ESR_ELx_EC(esr)) { |
704 | case ESR_ELx_EC_CP15_32: | |
705 | hook_base = cp15_32_hooks; | |
706 | break; | |
707 | case ESR_ELx_EC_CP15_64: | |
708 | hook_base = cp15_64_hooks; | |
709 | break; | |
710 | default: | |
711 | do_undefinstr(regs); | |
712 | return; | |
713 | } | |
714 | ||
715 | for (hook = hook_base; hook->handler; hook++) | |
716 | if ((hook->esr_mask & esr) == hook->esr_val) { | |
717 | hook->handler(esr, regs); | |
718 | return; | |
719 | } | |
720 | ||
70c63cdf MZ |
721 | /* |
722 | * New cp15 instructions may previously have been undefined at | |
723 | * EL0. Fall back to our usual undefined instruction handler | |
724 | * so that we handle these consistently. | |
725 | */ | |
726 | do_undefinstr(regs); | |
727 | } | |
728 | #endif | |
729 | ||
9dbd5bb2 SP |
730 | asmlinkage void __exception do_sysinstr(unsigned int esr, struct pt_regs *regs) |
731 | { | |
732 | struct sys64_hook *hook; | |
733 | ||
734 | for (hook = sys64_hooks; hook->handler; hook++) | |
735 | if ((hook->esr_mask & esr) == hook->esr_val) { | |
736 | hook->handler(esr, regs); | |
737 | return; | |
738 | } | |
739 | ||
49f6cba6 MR |
740 | /* |
741 | * New SYS instructions may previously have been undefined at EL0. Fall | |
742 | * back to our usual undefined instruction handler so that we handle | |
743 | * these consistently. | |
744 | */ | |
745 | do_undefinstr(regs); | |
9dbd5bb2 SP |
746 | } |
747 | ||
60a1f02c MR |
748 | static const char *esr_class_str[] = { |
749 | [0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC", | |
750 | [ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized", | |
751 | [ESR_ELx_EC_WFx] = "WFI/WFE", | |
752 | [ESR_ELx_EC_CP15_32] = "CP15 MCR/MRC", | |
753 | [ESR_ELx_EC_CP15_64] = "CP15 MCRR/MRRC", | |
754 | [ESR_ELx_EC_CP14_MR] = "CP14 MCR/MRC", | |
755 | [ESR_ELx_EC_CP14_LS] = "CP14 LDC/STC", | |
756 | [ESR_ELx_EC_FP_ASIMD] = "ASIMD", | |
757 | [ESR_ELx_EC_CP10_ID] = "CP10 MRC/VMRS", | |
758 | [ESR_ELx_EC_CP14_64] = "CP14 MCRR/MRRC", | |
759 | [ESR_ELx_EC_ILL] = "PSTATE.IL", | |
760 | [ESR_ELx_EC_SVC32] = "SVC (AArch32)", | |
761 | [ESR_ELx_EC_HVC32] = "HVC (AArch32)", | |
762 | [ESR_ELx_EC_SMC32] = "SMC (AArch32)", | |
763 | [ESR_ELx_EC_SVC64] = "SVC (AArch64)", | |
764 | [ESR_ELx_EC_HVC64] = "HVC (AArch64)", | |
765 | [ESR_ELx_EC_SMC64] = "SMC (AArch64)", | |
766 | [ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)", | |
67236564 | 767 | [ESR_ELx_EC_SVE] = "SVE", |
60a1f02c MR |
768 | [ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF", |
769 | [ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)", | |
770 | [ESR_ELx_EC_IABT_CUR] = "IABT (current EL)", | |
771 | [ESR_ELx_EC_PC_ALIGN] = "PC Alignment", | |
772 | [ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)", | |
773 | [ESR_ELx_EC_DABT_CUR] = "DABT (current EL)", | |
774 | [ESR_ELx_EC_SP_ALIGN] = "SP Alignment", | |
775 | [ESR_ELx_EC_FP_EXC32] = "FP (AArch32)", | |
776 | [ESR_ELx_EC_FP_EXC64] = "FP (AArch64)", | |
777 | [ESR_ELx_EC_SERROR] = "SError", | |
778 | [ESR_ELx_EC_BREAKPT_LOW] = "Breakpoint (lower EL)", | |
779 | [ESR_ELx_EC_BREAKPT_CUR] = "Breakpoint (current EL)", | |
780 | [ESR_ELx_EC_SOFTSTP_LOW] = "Software Step (lower EL)", | |
781 | [ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)", | |
782 | [ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)", | |
783 | [ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)", | |
784 | [ESR_ELx_EC_BKPT32] = "BKPT (AArch32)", | |
785 | [ESR_ELx_EC_VECTOR32] = "Vector catch (AArch32)", | |
786 | [ESR_ELx_EC_BRK64] = "BRK (AArch64)", | |
787 | }; | |
788 | ||
789 | const char *esr_get_class_string(u32 esr) | |
790 | { | |
275f344b | 791 | return esr_class_str[ESR_ELx_EC(esr)]; |
60a1f02c MR |
792 | } |
793 | ||
60ffc30d | 794 | /* |
7d9e8f71 MR |
795 | * bad_mode handles the impossible case in the exception vector. This is always |
796 | * fatal. | |
60ffc30d CM |
797 | */ |
798 | asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr) | |
799 | { | |
800 | console_verbose(); | |
801 | ||
8051f4d1 MR |
802 | pr_crit("Bad mode in %s handler detected on CPU%d, code 0x%08x -- %s\n", |
803 | handler[reason], smp_processor_id(), esr, | |
804 | esr_get_class_string(esr)); | |
7d9e8f71 | 805 | |
0fbeb318 | 806 | local_daif_mask(); |
7d9e8f71 MR |
807 | panic("bad mode"); |
808 | } | |
809 | ||
810 | /* | |
811 | * bad_el0_sync handles unexpected, but potentially recoverable synchronous | |
812 | * exceptions taken from EL0. Unlike bad_mode, this returns. | |
813 | */ | |
814 | asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) | |
815 | { | |
7d9e8f71 | 816 | void __user *pc = (void __user *)instruction_pointer(regs); |
9955ac47 | 817 | |
7d9e8f71 | 818 | current->thread.fault_address = 0; |
4e829b67 | 819 | current->thread.fault_code = esr; |
7d9e8f71 | 820 | |
feca355b EB |
821 | arm64_force_sig_fault(SIGILL, ILL_ILLOPC, pc, |
822 | "Bad EL0 synchronous exception"); | |
60ffc30d CM |
823 | } |
824 | ||
872d8327 MR |
825 | #ifdef CONFIG_VMAP_STACK |
826 | ||
827 | DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack) | |
828 | __aligned(16); | |
829 | ||
830 | asmlinkage void handle_bad_stack(struct pt_regs *regs) | |
831 | { | |
832 | unsigned long tsk_stk = (unsigned long)current->stack; | |
833 | unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); | |
834 | unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack); | |
835 | unsigned int esr = read_sysreg(esr_el1); | |
836 | unsigned long far = read_sysreg(far_el1); | |
837 | ||
838 | console_verbose(); | |
839 | pr_emerg("Insufficient stack space to handle exception!"); | |
840 | ||
841 | pr_emerg("ESR: 0x%08x -- %s\n", esr, esr_get_class_string(esr)); | |
842 | pr_emerg("FAR: 0x%016lx\n", far); | |
843 | ||
844 | pr_emerg("Task stack: [0x%016lx..0x%016lx]\n", | |
845 | tsk_stk, tsk_stk + THREAD_SIZE); | |
846 | pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n", | |
847 | irq_stk, irq_stk + THREAD_SIZE); | |
848 | pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n", | |
849 | ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE); | |
850 | ||
851 | __show_regs(regs); | |
852 | ||
853 | /* | |
854 | * We use nmi_panic to limit the potential for recusive overflows, and | |
855 | * to get a better stack trace. | |
856 | */ | |
857 | nmi_panic(NULL, "kernel stack overflow"); | |
858 | cpu_park_loop(); | |
859 | } | |
860 | #endif | |
861 | ||
6bf0dcfd | 862 | void __noreturn arm64_serror_panic(struct pt_regs *regs, u32 esr) |
a92d4d14 | 863 | { |
a92d4d14 XX |
864 | console_verbose(); |
865 | ||
866 | pr_crit("SError Interrupt on CPU%d, code 0x%08x -- %s\n", | |
867 | smp_processor_id(), esr, esr_get_class_string(esr)); | |
6bf0dcfd JM |
868 | if (regs) |
869 | __show_regs(regs); | |
870 | ||
871 | nmi_panic(regs, "Asynchronous SError Interrupt"); | |
872 | ||
873 | cpu_park_loop(); | |
874 | unreachable(); | |
875 | } | |
876 | ||
877 | bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned int esr) | |
878 | { | |
879 | u32 aet = arm64_ras_serror_get_severity(esr); | |
880 | ||
881 | switch (aet) { | |
882 | case ESR_ELx_AET_CE: /* corrected error */ | |
883 | case ESR_ELx_AET_UEO: /* restartable, not yet consumed */ | |
884 | /* | |
885 | * The CPU can make progress. We may take UEO again as | |
886 | * a more severe error. | |
887 | */ | |
888 | return false; | |
889 | ||
890 | case ESR_ELx_AET_UEU: /* Uncorrected Unrecoverable */ | |
891 | case ESR_ELx_AET_UER: /* Uncorrected Recoverable */ | |
892 | /* | |
893 | * The CPU can't make progress. The exception may have | |
894 | * been imprecise. | |
895 | */ | |
896 | return true; | |
897 | ||
898 | case ESR_ELx_AET_UC: /* Uncontainable or Uncategorized error */ | |
899 | default: | |
900 | /* Error has been silently propagated */ | |
901 | arm64_serror_panic(regs, esr); | |
902 | } | |
903 | } | |
904 | ||
905 | asmlinkage void do_serror(struct pt_regs *regs, unsigned int esr) | |
906 | { | |
7d31464a JT |
907 | const bool was_in_nmi = in_nmi(); |
908 | ||
909 | if (!was_in_nmi) | |
910 | nmi_enter(); | |
6bf0dcfd JM |
911 | |
912 | /* non-RAS errors are not containable */ | |
913 | if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(regs, esr)) | |
914 | arm64_serror_panic(regs, esr); | |
a92d4d14 | 915 | |
7d31464a JT |
916 | if (!was_in_nmi) |
917 | nmi_exit(); | |
a92d4d14 XX |
918 | } |
919 | ||
60ffc30d CM |
920 | void __pte_error(const char *file, int line, unsigned long val) |
921 | { | |
c9cd0ed9 | 922 | pr_err("%s:%d: bad pte %016lx.\n", file, line, val); |
60ffc30d CM |
923 | } |
924 | ||
925 | void __pmd_error(const char *file, int line, unsigned long val) | |
926 | { | |
c9cd0ed9 | 927 | pr_err("%s:%d: bad pmd %016lx.\n", file, line, val); |
60ffc30d CM |
928 | } |
929 | ||
c79b954b JL |
930 | void __pud_error(const char *file, int line, unsigned long val) |
931 | { | |
c9cd0ed9 | 932 | pr_err("%s:%d: bad pud %016lx.\n", file, line, val); |
c79b954b JL |
933 | } |
934 | ||
60ffc30d CM |
935 | void __pgd_error(const char *file, int line, unsigned long val) |
936 | { | |
c9cd0ed9 | 937 | pr_err("%s:%d: bad pgd %016lx.\n", file, line, val); |
60ffc30d CM |
938 | } |
939 | ||
9fb7410f DM |
940 | /* GENERIC_BUG traps */ |
941 | ||
942 | int is_valid_bugaddr(unsigned long addr) | |
943 | { | |
944 | /* | |
945 | * bug_handler() only called for BRK #BUG_BRK_IMM. | |
946 | * So the answer is trivial -- any spurious instances with no | |
947 | * bug table entry will be rejected by report_bug() and passed | |
948 | * back to the debug-monitors code and handled as a fatal | |
949 | * unexpected debug exception. | |
950 | */ | |
951 | return 1; | |
952 | } | |
953 | ||
954 | static int bug_handler(struct pt_regs *regs, unsigned int esr) | |
955 | { | |
9fb7410f DM |
956 | switch (report_bug(regs->pc, regs)) { |
957 | case BUG_TRAP_TYPE_BUG: | |
958 | die("Oops - BUG", regs, 0); | |
959 | break; | |
960 | ||
961 | case BUG_TRAP_TYPE_WARN: | |
962 | break; | |
963 | ||
964 | default: | |
965 | /* unknown/unrecognised bug trap type */ | |
966 | return DBG_HOOK_ERROR; | |
967 | } | |
968 | ||
969 | /* If thread survives, skip over the BUG instruction and continue: */ | |
6436beee | 970 | arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); |
9fb7410f DM |
971 | return DBG_HOOK_HANDLED; |
972 | } | |
973 | ||
974 | static struct break_hook bug_break_hook = { | |
9fb7410f | 975 | .fn = bug_handler, |
26a04d84 | 976 | .imm = BUG_BRK_IMM, |
9fb7410f DM |
977 | }; |
978 | ||
41eea9cd AK |
979 | #ifdef CONFIG_KASAN_SW_TAGS |
980 | ||
981 | #define KASAN_ESR_RECOVER 0x20 | |
982 | #define KASAN_ESR_WRITE 0x10 | |
983 | #define KASAN_ESR_SIZE_MASK 0x0f | |
984 | #define KASAN_ESR_SIZE(esr) (1 << ((esr) & KASAN_ESR_SIZE_MASK)) | |
985 | ||
986 | static int kasan_handler(struct pt_regs *regs, unsigned int esr) | |
987 | { | |
988 | bool recover = esr & KASAN_ESR_RECOVER; | |
989 | bool write = esr & KASAN_ESR_WRITE; | |
990 | size_t size = KASAN_ESR_SIZE(esr); | |
991 | u64 addr = regs->regs[0]; | |
992 | u64 pc = regs->pc; | |
993 | ||
41eea9cd AK |
994 | kasan_report(addr, size, write, pc); |
995 | ||
996 | /* | |
997 | * The instrumentation allows to control whether we can proceed after | |
998 | * a crash was detected. This is done by passing the -recover flag to | |
999 | * the compiler. Disabling recovery allows to generate more compact | |
1000 | * code. | |
1001 | * | |
1002 | * Unfortunately disabling recovery doesn't work for the kernel right | |
1003 | * now. KASAN reporting is disabled in some contexts (for example when | |
1004 | * the allocator accesses slab object metadata; this is controlled by | |
1005 | * current->kasan_depth). All these accesses are detected by the tool, | |
1006 | * even though the reports for them are not printed. | |
1007 | * | |
1008 | * This is something that might be fixed at some point in the future. | |
1009 | */ | |
1010 | if (!recover) | |
1011 | die("Oops - KASAN", regs, 0); | |
1012 | ||
1013 | /* If thread survives, skip over the brk instruction and continue: */ | |
1014 | arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); | |
1015 | return DBG_HOOK_HANDLED; | |
1016 | } | |
1017 | ||
41eea9cd | 1018 | static struct break_hook kasan_break_hook = { |
26a04d84 WD |
1019 | .fn = kasan_handler, |
1020 | .imm = KASAN_BRK_IMM, | |
1021 | .mask = KASAN_BRK_MASK, | |
41eea9cd AK |
1022 | }; |
1023 | #endif | |
1024 | ||
9fb7410f DM |
1025 | /* |
1026 | * Initial handler for AArch64 BRK exceptions | |
1027 | * This handler only used until debug_traps_init(). | |
1028 | */ | |
1029 | int __init early_brk64(unsigned long addr, unsigned int esr, | |
1030 | struct pt_regs *regs) | |
1031 | { | |
41eea9cd | 1032 | #ifdef CONFIG_KASAN_SW_TAGS |
453b7740 | 1033 | unsigned int comment = esr & ESR_ELx_BRK64_ISS_COMMENT_MASK; |
26a04d84 WD |
1034 | |
1035 | if ((comment & ~KASAN_BRK_MASK) == KASAN_BRK_IMM) | |
41eea9cd AK |
1036 | return kasan_handler(regs, esr) != DBG_HOOK_HANDLED; |
1037 | #endif | |
9fb7410f DM |
1038 | return bug_handler(regs, esr) != DBG_HOOK_HANDLED; |
1039 | } | |
1040 | ||
1041 | /* This registration must happen early, before debug_traps_init(). */ | |
60ffc30d CM |
1042 | void __init trap_init(void) |
1043 | { | |
26a04d84 | 1044 | register_kernel_break_hook(&bug_break_hook); |
41eea9cd | 1045 | #ifdef CONFIG_KASAN_SW_TAGS |
26a04d84 | 1046 | register_kernel_break_hook(&kasan_break_hook); |
41eea9cd | 1047 | #endif |
60ffc30d | 1048 | } |