]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - arch/sparc64/kernel/traps.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Remove obsolete #include <linux/config.h>
[mirror_ubuntu-jammy-kernel.git] / arch / sparc64 / kernel / traps.c
CommitLineData
1da177e4
LT		 1/* $Id: traps.c,v 1.85 2002/02/09 19:49:31 davem Exp $
2 * arch/sparc64/kernel/traps.c
3 *
4 * Copyright (C) 1995,1997 David S. Miller (davem@caip.rutgers.edu)
5 * Copyright (C) 1997,1999,2000 Jakub Jelinek (jakub@redhat.com)
6 */
7
8/*
9 * I like traps on v9, :))))
10 */
11
1da177e4
LT		 12#include <linux/module.h>
13#include <linux/sched.h> /* for jiffies */
14#include <linux/kernel.h>
15#include <linux/kallsyms.h>
16#include <linux/signal.h>
17#include <linux/smp.h>
18#include <linux/smp_lock.h>
19#include <linux/mm.h>
20#include <linux/init.h>
21
22#include <asm/delay.h>
23#include <asm/system.h>
24#include <asm/ptrace.h>
25#include <asm/oplib.h>
26#include <asm/page.h>
27#include <asm/pgtable.h>
28#include <asm/unistd.h>
29#include <asm/uaccess.h>
30#include <asm/fpumacro.h>
31#include <asm/lsu.h>
32#include <asm/dcu.h>
33#include <asm/estate.h>
34#include <asm/chafsr.h>
6c52a96e 35#include <asm/sfafsr.h>
1da177e4
LT		 36#include <asm/psrcompat.h>
37#include <asm/processor.h>
38#include <asm/timer.h>
39#include <asm/kdebug.h>
92704a1c 40#include <asm/head.h>
1da177e4
LT		 41#ifdef CONFIG_KMOD
42#include <linux/kmod.h>
43#endif
07f8e5f3 44#include <asm/prom.h>
1da177e4 45
e041c683 46ATOMIC_NOTIFIER_HEAD(sparc64die_chain);
1da177e4
LT		 47
48int register_die_notifier(struct notifier_block *nb)
49{
e041c683 50 return atomic_notifier_chain_register(&sparc64die_chain, nb);
1da177e4 51}
e041c683
AS		 52EXPORT_SYMBOL(register_die_notifier);
53
54int unregister_die_notifier(struct notifier_block *nb)
55{
56 return atomic_notifier_chain_unregister(&sparc64die_chain, nb);
57}
58EXPORT_SYMBOL(unregister_die_notifier);
1da177e4
LT		 59
60/* When an irrecoverable trap occurs at tl > 0, the trap entry
61 * code logs the trap state registers at every level in the trap
62 * stack. It is found at (pt_regs + sizeof(pt_regs)) and the layout
63 * is as follows:
64 */
65struct tl1_traplog {
66 struct {
67 unsigned long tstate;
68 unsigned long tpc;
69 unsigned long tnpc;
70 unsigned long tt;
71 } trapstack[4];
72 unsigned long tl;
73};
74
75static void dump_tl1_traplog(struct tl1_traplog *p)
76{
3d6395cb 77 int i, limit;
1da177e4 78
04d74758
DM		 79 printk(KERN_EMERG "TRAPLOG: Error at trap level 0x%lx, "
80 "dumping track stack.\n", p->tl);
3d6395cb
DM		 81
82 limit = (tlb_type == hypervisor) ? 2 : 4;
39334a4b 83 for (i = 0; i < limit; i++) {
04d74758 84 printk(KERN_EMERG
1da177e4
LT		 85 "TRAPLOG: Trap level %d TSTATE[%016lx] TPC[%016lx] "
86 "TNPC[%016lx] TT[%lx]\n",
87 i + 1,
88 p->trapstack[i].tstate, p->trapstack[i].tpc,
89 p->trapstack[i].tnpc, p->trapstack[i].tt);
90 }
91}
92
93void do_call_debug(struct pt_regs *regs)
94{
95 notify_die(DIE_CALL, "debug call", regs, 0, 255, SIGINT);
96}
97
98void bad_trap(struct pt_regs *regs, long lvl)
99{
100 char buffer[32];
101 siginfo_t info;
102
103 if (notify_die(DIE_TRAP, "bad trap", regs,
104 0, lvl, SIGTRAP) == NOTIFY_STOP)
105 return;
106
107 if (lvl < 0x100) {
108 sprintf(buffer, "Bad hw trap %lx at tl0\n", lvl);
109 die_if_kernel(buffer, regs);
110 }
111
112 lvl -= 0x100;
113 if (regs->tstate & TSTATE_PRIV) {
114 sprintf(buffer, "Kernel bad sw trap %lx", lvl);
115 die_if_kernel(buffer, regs);
116 }
117 if (test_thread_flag(TIF_32BIT)) {
118 regs->tpc &= 0xffffffff;
119 regs->tnpc &= 0xffffffff;
120 }
121 info.si_signo = SIGILL;
122 info.si_errno = 0;
123 info.si_code = ILL_ILLTRP;
124 info.si_addr = (void __user *)regs->tpc;
125 info.si_trapno = lvl;
126 force_sig_info(SIGILL, &info, current);
127}
128
129void bad_trap_tl1(struct pt_regs *regs, long lvl)
130{
131 char buffer[32];
132
133 if (notify_die(DIE_TRAP_TL1, "bad trap tl1", regs,
134 0, lvl, SIGTRAP) == NOTIFY_STOP)
135 return;
136
137 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
138
139 sprintf (buffer, "Bad trap %lx at tl>0", lvl);
140 die_if_kernel (buffer, regs);
141}
142
143#ifdef CONFIG_DEBUG_BUGVERBOSE
144void do_BUG(const char *file, int line)
145{
146 bust_spinlocks(1);
147 printk("kernel BUG at %s:%d!\n", file, line);
148}
149#endif
150
6c52a96e 151void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
1da177e4
LT		 152{
153 siginfo_t info;
154
155 if (notify_die(DIE_TRAP, "instruction access exception", regs,
156 0, 0x8, SIGTRAP) == NOTIFY_STOP)
157 return;
158
159 if (regs->tstate & TSTATE_PRIV) {
6c52a96e
DM		 160 printk("spitfire_insn_access_exception: SFSR[%016lx] "
161 "SFAR[%016lx], going.\n", sfsr, sfar);
1da177e4
LT		 162 die_if_kernel("Iax", regs);
163 }
164 if (test_thread_flag(TIF_32BIT)) {
165 regs->tpc &= 0xffffffff;
166 regs->tnpc &= 0xffffffff;
167 }
168 info.si_signo = SIGSEGV;
169 info.si_errno = 0;
170 info.si_code = SEGV_MAPERR;
171 info.si_addr = (void __user *)regs->tpc;
172 info.si_trapno = 0;
173 force_sig_info(SIGSEGV, &info, current);
174}
175
6c52a96e 176void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
1da177e4
LT		 177{
178 if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
179 0, 0x8, SIGTRAP) == NOTIFY_STOP)
180 return;
181
182 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
6c52a96e 183 spitfire_insn_access_exception(regs, sfsr, sfar);
1da177e4
LT		 184}
185
ed6b0b45
DM		 186void sun4v_insn_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
187{
188 unsigned short type = (type_ctx >> 16);
189 unsigned short ctx = (type_ctx & 0xffff);
190 siginfo_t info;
191
192 if (notify_die(DIE_TRAP, "instruction access exception", regs,
193 0, 0x8, SIGTRAP) == NOTIFY_STOP)
194 return;
195
196 if (regs->tstate & TSTATE_PRIV) {
197 printk("sun4v_insn_access_exception: ADDR[%016lx] "
198 "CTX[%04x] TYPE[%04x], going.\n",
199 addr, ctx, type);
200 die_if_kernel("Iax", regs);
201 }
202
203 if (test_thread_flag(TIF_32BIT)) {
204 regs->tpc &= 0xffffffff;
205 regs->tnpc &= 0xffffffff;
206 }
207 info.si_signo = SIGSEGV;
208 info.si_errno = 0;
209 info.si_code = SEGV_MAPERR;
210 info.si_addr = (void __user *) addr;
211 info.si_trapno = 0;
212 force_sig_info(SIGSEGV, &info, current);
213}
214
215void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
216{
217 if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
218 0, 0x8, SIGTRAP) == NOTIFY_STOP)
219 return;
220
221 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
222 sun4v_insn_access_exception(regs, addr, type_ctx);
223}
224
6c52a96e 225void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
1da177e4
LT		 226{
227 siginfo_t info;
228
229 if (notify_die(DIE_TRAP, "data access exception", regs,
230 0, 0x30, SIGTRAP) == NOTIFY_STOP)
231 return;
232
233 if (regs->tstate & TSTATE_PRIV) {
234 /* Test if this comes from uaccess places. */
8cf14af0 235 const struct exception_table_entry *entry;
1da177e4 236
8cf14af0
DM		 237 entry = search_exception_tables(regs->tpc);
238 if (entry) {
239 /* Ouch, somebody is trying VM hole tricks on us... */
1da177e4
LT		 240#ifdef DEBUG_EXCEPTIONS
241 printk("Exception: PC<%016lx> faddr<UNKNOWN>\n", regs->tpc);
8cf14af0
DM		 242 printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
243 regs->tpc, entry->fixup);
1da177e4 244#endif
8cf14af0 245 regs->tpc = entry->fixup;
1da177e4 246 regs->tnpc = regs->tpc + 4;
1da177e4
LT		 247 return;
248 }
249 /* Shit... */
6c52a96e
DM		 250 printk("spitfire_data_access_exception: SFSR[%016lx] "
251 "SFAR[%016lx], going.\n", sfsr, sfar);
1da177e4
LT		 252 die_if_kernel("Dax", regs);
253 }
254
255 info.si_signo = SIGSEGV;
256 info.si_errno = 0;
257 info.si_code = SEGV_MAPERR;
258 info.si_addr = (void __user *)sfar;
259 info.si_trapno = 0;
260 force_sig_info(SIGSEGV, &info, current);
261}
262
6c52a96e 263void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
bde4e4ee
DM		 264{
265 if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
266 0, 0x30, SIGTRAP) == NOTIFY_STOP)
267 return;
268
269 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
6c52a96e 270 spitfire_data_access_exception(regs, sfsr, sfar);
bde4e4ee
DM		 271}
272
ed6b0b45
DM		 273void sun4v_data_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
274{
275 unsigned short type = (type_ctx >> 16);
276 unsigned short ctx = (type_ctx & 0xffff);
277 siginfo_t info;
278
279 if (notify_die(DIE_TRAP, "data access exception", regs,
280 0, 0x8, SIGTRAP) == NOTIFY_STOP)
281 return;
282
283 if (regs->tstate & TSTATE_PRIV) {
284 printk("sun4v_data_access_exception: ADDR[%016lx] "
285 "CTX[%04x] TYPE[%04x], going.\n",
286 addr, ctx, type);
55555633 287 die_if_kernel("Dax", regs);
ed6b0b45
DM		 288 }
289
290 if (test_thread_flag(TIF_32BIT)) {
291 regs->tpc &= 0xffffffff;
292 regs->tnpc &= 0xffffffff;
293 }
294 info.si_signo = SIGSEGV;
295 info.si_errno = 0;
296 info.si_code = SEGV_MAPERR;
297 info.si_addr = (void __user *) addr;
298 info.si_trapno = 0;
299 force_sig_info(SIGSEGV, &info, current);
300}
301
302void sun4v_data_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
303{
304 if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
305 0, 0x8, SIGTRAP) == NOTIFY_STOP)
306 return;
307
308 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
309 sun4v_data_access_exception(regs, addr, type_ctx);
310}
311
1da177e4
LT		 312#ifdef CONFIG_PCI
313/* This is really pathetic... */
314extern volatile int pci_poke_in_progress;
315extern volatile int pci_poke_cpu;
316extern volatile int pci_poke_faulted;
317#endif
318
319/* When access exceptions happen, we must do this. */
320static void spitfire_clean_and_reenable_l1_caches(void)
321{
322 unsigned long va;
323
324 if (tlb_type != spitfire)
325 BUG();
326
327 /* Clean 'em. */
328 for (va = 0; va < (PAGE_SIZE << 1); va += 32) {
329 spitfire_put_icache_tag(va, 0x0);
330 spitfire_put_dcache_tag(va, 0x0);
331 }
332
333 /* Re-enable in LSU. */
334 __asm__ __volatile__("flush %%g6\n\t"
335 "membar #Sync\n\t"
336 "stxa %0, [%%g0] %1\n\t"
337 "membar #Sync"
338 : /* no outputs */
339 : "r" (LSU_CONTROL_IC | LSU_CONTROL_DC |
340 LSU_CONTROL_IM | LSU_CONTROL_DM),
341 "i" (ASI_LSU_CONTROL)
342 : "memory");
343}
344
6c52a96e 345static void spitfire_enable_estate_errors(void)
1da177e4 346{
6c52a96e
DM		 347 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
348 "membar #Sync"
349 : /* no outputs */
350 : "r" (ESTATE_ERR_ALL),
351 "i" (ASI_ESTATE_ERROR_EN));
1da177e4
LT		 352}
353
354static char ecc_syndrome_table[] = {
355 0x4c, 0x40, 0x41, 0x48, 0x42, 0x48, 0x48, 0x49,
356 0x43, 0x48, 0x48, 0x49, 0x48, 0x49, 0x49, 0x4a,
357 0x44, 0x48, 0x48, 0x20, 0x48, 0x39, 0x4b, 0x48,
358 0x48, 0x25, 0x31, 0x48, 0x28, 0x48, 0x48, 0x2c,
359 0x45, 0x48, 0x48, 0x21, 0x48, 0x3d, 0x04, 0x48,
360 0x48, 0x4b, 0x35, 0x48, 0x2d, 0x48, 0x48, 0x29,
361 0x48, 0x00, 0x01, 0x48, 0x0a, 0x48, 0x48, 0x4b,
362 0x0f, 0x48, 0x48, 0x4b, 0x48, 0x49, 0x49, 0x48,
363 0x46, 0x48, 0x48, 0x2a, 0x48, 0x3b, 0x27, 0x48,
364 0x48, 0x4b, 0x33, 0x48, 0x22, 0x48, 0x48, 0x2e,
365 0x48, 0x19, 0x1d, 0x48, 0x1b, 0x4a, 0x48, 0x4b,
366 0x1f, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
367 0x48, 0x4b, 0x24, 0x48, 0x07, 0x48, 0x48, 0x36,
368 0x4b, 0x48, 0x48, 0x3e, 0x48, 0x30, 0x38, 0x48,
369 0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x16, 0x48,
370 0x48, 0x12, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
371 0x47, 0x48, 0x48, 0x2f, 0x48, 0x3f, 0x4b, 0x48,
372 0x48, 0x06, 0x37, 0x48, 0x23, 0x48, 0x48, 0x2b,
373 0x48, 0x05, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x32,
374 0x26, 0x48, 0x48, 0x3a, 0x48, 0x34, 0x3c, 0x48,
375 0x48, 0x11, 0x15, 0x48, 0x13, 0x4a, 0x48, 0x4b,
376 0x17, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
377 0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x1e, 0x48,
378 0x48, 0x1a, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
379 0x48, 0x08, 0x0d, 0x48, 0x02, 0x48, 0x48, 0x49,
380 0x03, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x4b, 0x48,
381 0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x10, 0x48,
382 0x48, 0x14, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
383 0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x18, 0x48,
384 0x48, 0x1c, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
385 0x4a, 0x0c, 0x09, 0x48, 0x0e, 0x48, 0x48, 0x4b,
386 0x0b, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x4b, 0x4a
387};
388
1da177e4
LT		 389static char *syndrome_unknown = "<Unknown>";
390
6c52a96e 391static void spitfire_log_udb_syndrome(unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long bit)
1da177e4 392{
6c52a96e
DM		 393 unsigned short scode;
394 char memmod_str[64], *p;
1da177e4 395
6c52a96e
DM		 396 if (udbl & bit) {
397 scode = ecc_syndrome_table[udbl & 0xff];
1da177e4
LT		 398 if (prom_getunumber(scode, afar,
399 memmod_str, sizeof(memmod_str)) == -1)
400 p = syndrome_unknown;
401 else
402 p = memmod_str;
403 printk(KERN_WARNING "CPU[%d]: UDBL Syndrome[%x] "
404 "Memory Module \"%s\"\n",
405 smp_processor_id(), scode, p);
406 }
407
6c52a96e
DM		 408 if (udbh & bit) {
409 scode = ecc_syndrome_table[udbh & 0xff];
1da177e4
LT		 410 if (prom_getunumber(scode, afar,
411 memmod_str, sizeof(memmod_str)) == -1)
412 p = syndrome_unknown;
413 else
414 p = memmod_str;
415 printk(KERN_WARNING "CPU[%d]: UDBH Syndrome[%x] "
416 "Memory Module \"%s\"\n",
417 smp_processor_id(), scode, p);
418 }
6c52a96e
DM		 419
420}
421
422static void spitfire_cee_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, int tl1, struct pt_regs *regs)
423{
424
425 printk(KERN_WARNING "CPU[%d]: Correctable ECC Error "
426 "AFSR[%lx] AFAR[%016lx] UDBL[%lx] UDBH[%lx] TL>1[%d]\n",
427 smp_processor_id(), afsr, afar, udbl, udbh, tl1);
428
429 spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_CE);
430
431 /* We always log it, even if someone is listening for this
432 * trap.
433 */
434 notify_die(DIE_TRAP, "Correctable ECC Error", regs,
435 0, TRAP_TYPE_CEE, SIGTRAP);
436
437 /* The Correctable ECC Error trap does not disable I/D caches. So
438 * we only have to restore the ESTATE Error Enable register.
439 */
440 spitfire_enable_estate_errors();
441}
442
443static void spitfire_ue_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long tt, int tl1, struct pt_regs *regs)
444{
445 siginfo_t info;
446
447 printk(KERN_WARNING "CPU[%d]: Uncorrectable Error AFSR[%lx] "
448 "AFAR[%lx] UDBL[%lx] UDBH[%ld] TT[%lx] TL>1[%d]\n",
449 smp_processor_id(), afsr, afar, udbl, udbh, tt, tl1);
450
451 /* XXX add more human friendly logging of the error status
452 * XXX as is implemented for cheetah
453 */
454
455 spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_UE);
456
457 /* We always log it, even if someone is listening for this
458 * trap.
459 */
460 notify_die(DIE_TRAP, "Uncorrectable Error", regs,
461 0, tt, SIGTRAP);
462
463 if (regs->tstate & TSTATE_PRIV) {
464 if (tl1)
465 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
466 die_if_kernel("UE", regs);
467 }
468
469 /* XXX need more intelligent processing here, such as is implemented
470 * XXX for cheetah errors, in fact if the E-cache still holds the
471 * XXX line with bad parity this will loop
472 */
473
474 spitfire_clean_and_reenable_l1_caches();
475 spitfire_enable_estate_errors();
476
477 if (test_thread_flag(TIF_32BIT)) {
478 regs->tpc &= 0xffffffff;
479 regs->tnpc &= 0xffffffff;
480 }
481 info.si_signo = SIGBUS;
482 info.si_errno = 0;
483 info.si_code = BUS_OBJERR;
484 info.si_addr = (void *)0;
485 info.si_trapno = 0;
486 force_sig_info(SIGBUS, &info, current);
487}
488
489void spitfire_access_error(struct pt_regs *regs, unsigned long status_encoded, unsigned long afar)
490{
491 unsigned long afsr, tt, udbh, udbl;
492 int tl1;
493
494 afsr = (status_encoded & SFSTAT_AFSR_MASK) >> SFSTAT_AFSR_SHIFT;
495 tt = (status_encoded & SFSTAT_TRAP_TYPE) >> SFSTAT_TRAP_TYPE_SHIFT;
496 tl1 = (status_encoded & SFSTAT_TL_GT_ONE) ? 1 : 0;
497 udbl = (status_encoded & SFSTAT_UDBL_MASK) >> SFSTAT_UDBL_SHIFT;
498 udbh = (status_encoded & SFSTAT_UDBH_MASK) >> SFSTAT_UDBH_SHIFT;
499
500#ifdef CONFIG_PCI
501 if (tt == TRAP_TYPE_DAE &&
502 pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
503 spitfire_clean_and_reenable_l1_caches();
504 spitfire_enable_estate_errors();
505
506 pci_poke_faulted = 1;
507 regs->tnpc = regs->tpc + 4;
508 return;
509 }
510#endif
511
512 if (afsr & SFAFSR_UE)
513 spitfire_ue_log(afsr, afar, udbh, udbl, tt, tl1, regs);
514
515 if (tt == TRAP_TYPE_CEE) {
516 /* Handle the case where we took a CEE trap, but ACK'd
517 * only the UE state in the UDB error registers.
518 */
519 if (afsr & SFAFSR_UE) {
520 if (udbh & UDBE_CE) {
521 __asm__ __volatile__(
522 "stxa %0, [%1] %2\n\t"
523 "membar #Sync"
524 : /* no outputs */
525 : "r" (udbh & UDBE_CE),
526 "r" (0x0), "i" (ASI_UDB_ERROR_W));
527 }
528 if (udbl & UDBE_CE) {
529 __asm__ __volatile__(
530 "stxa %0, [%1] %2\n\t"
531 "membar #Sync"
532 : /* no outputs */
533 : "r" (udbl & UDBE_CE),
534 "r" (0x18), "i" (ASI_UDB_ERROR_W));
535 }
536 }
537
538 spitfire_cee_log(afsr, afar, udbh, udbl, tl1, regs);
539 }
1da177e4
LT		 540}
541
816242da
DM		 542int cheetah_pcache_forced_on;
543
544void cheetah_enable_pcache(void)
545{
546 unsigned long dcr;
547
548 printk("CHEETAH: Enabling P-Cache on cpu %d.\n",
549 smp_processor_id());
550
551 __asm__ __volatile__("ldxa [%%g0] %1, %0"
552 : "=r" (dcr)
553 : "i" (ASI_DCU_CONTROL_REG));
554 dcr |= (DCU_PE | DCU_HPE | DCU_SPE | DCU_SL);
555 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
556 "membar #Sync"
557 : /* no outputs */
558 : "r" (dcr), "i" (ASI_DCU_CONTROL_REG));
559}
560
1da177e4
LT		 561/* Cheetah error trap handling. */
562static unsigned long ecache_flush_physbase;
563static unsigned long ecache_flush_linesize;
564static unsigned long ecache_flush_size;
565
566/* WARNING: The error trap handlers in assembly know the precise
567 * layout of the following structure.
568 *
569 * C-level handlers below use this information to log the error
570 * and then determine how to recover (if possible).
571 */
572struct cheetah_err_info {
573/*0x00*/u64 afsr;
574/*0x08*/u64 afar;
575
576 /* D-cache state */
577/*0x10*/u64 dcache_data[4]; /* The actual data */
578/*0x30*/u64 dcache_index; /* D-cache index */
579/*0x38*/u64 dcache_tag; /* D-cache tag/valid */
580/*0x40*/u64 dcache_utag; /* D-cache microtag */
581/*0x48*/u64 dcache_stag; /* D-cache snooptag */
582
583 /* I-cache state */
584/*0x50*/u64 icache_data[8]; /* The actual insns + predecode */
585/*0x90*/u64 icache_index; /* I-cache index */
586/*0x98*/u64 icache_tag; /* I-cache phys tag */
587/*0xa0*/u64 icache_utag; /* I-cache microtag */
588/*0xa8*/u64 icache_stag; /* I-cache snooptag */
589/*0xb0*/u64 icache_upper; /* I-cache upper-tag */
590/*0xb8*/u64 icache_lower; /* I-cache lower-tag */
591
592 /* E-cache state */
593/*0xc0*/u64 ecache_data[4]; /* 32 bytes from staging registers */
594/*0xe0*/u64 ecache_index; /* E-cache index */
595/*0xe8*/u64 ecache_tag; /* E-cache tag/state */
596
597/*0xf0*/u64 __pad[32 - 30];
598};
599#define CHAFSR_INVALID ((u64)-1L)
600
601/* This table is ordered in priority of errors and matches the
602 * AFAR overwrite policy as well.
603 */
604
605struct afsr_error_table {
606 unsigned long mask;
607 const char *name;
608};
609
610static const char CHAFSR_PERR_msg[] =
611 "System interface protocol error";
612static const char CHAFSR_IERR_msg[] =
613 "Internal processor error";
614static const char CHAFSR_ISAP_msg[] =
615 "System request parity error on incoming addresss";
616static const char CHAFSR_UCU_msg[] =
617 "Uncorrectable E-cache ECC error for ifetch/data";
618static const char CHAFSR_UCC_msg[] =
619 "SW Correctable E-cache ECC error for ifetch/data";
620static const char CHAFSR_UE_msg[] =
621 "Uncorrectable system bus data ECC error for read";
622static const char CHAFSR_EDU_msg[] =
623 "Uncorrectable E-cache ECC error for stmerge/blkld";
624static const char CHAFSR_EMU_msg[] =
625 "Uncorrectable system bus MTAG error";
626static const char CHAFSR_WDU_msg[] =
627 "Uncorrectable E-cache ECC error for writeback";
628static const char CHAFSR_CPU_msg[] =
629 "Uncorrectable ECC error for copyout";
630static const char CHAFSR_CE_msg[] =
631 "HW corrected system bus data ECC error for read";
632static const char CHAFSR_EDC_msg[] =
633 "HW corrected E-cache ECC error for stmerge/blkld";
634static const char CHAFSR_EMC_msg[] =
635 "HW corrected system bus MTAG ECC error";
636static const char CHAFSR_WDC_msg[] =
637 "HW corrected E-cache ECC error for writeback";
638static const char CHAFSR_CPC_msg[] =
639 "HW corrected ECC error for copyout";
640static const char CHAFSR_TO_msg[] =
641 "Unmapped error from system bus";
642static const char CHAFSR_BERR_msg[] =
643 "Bus error response from system bus";
644static const char CHAFSR_IVC_msg[] =
645 "HW corrected system bus data ECC error for ivec read";
646static const char CHAFSR_IVU_msg[] =
647 "Uncorrectable system bus data ECC error for ivec read";
648static struct afsr_error_table __cheetah_error_table[] = {
649 { CHAFSR_PERR, CHAFSR_PERR_msg },
650 { CHAFSR_IERR, CHAFSR_IERR_msg },
651 { CHAFSR_ISAP, CHAFSR_ISAP_msg },
652 { CHAFSR_UCU, CHAFSR_UCU_msg },
653 { CHAFSR_UCC, CHAFSR_UCC_msg },
654 { CHAFSR_UE, CHAFSR_UE_msg },
655 { CHAFSR_EDU, CHAFSR_EDU_msg },
656 { CHAFSR_EMU, CHAFSR_EMU_msg },
657 { CHAFSR_WDU, CHAFSR_WDU_msg },
658 { CHAFSR_CPU, CHAFSR_CPU_msg },
659 { CHAFSR_CE, CHAFSR_CE_msg },
660 { CHAFSR_EDC, CHAFSR_EDC_msg },
661 { CHAFSR_EMC, CHAFSR_EMC_msg },
662 { CHAFSR_WDC, CHAFSR_WDC_msg },
663 { CHAFSR_CPC, CHAFSR_CPC_msg },
664 { CHAFSR_TO, CHAFSR_TO_msg },
665 { CHAFSR_BERR, CHAFSR_BERR_msg },
666 /* These two do not update the AFAR. */
667 { CHAFSR_IVC, CHAFSR_IVC_msg },
668 { CHAFSR_IVU, CHAFSR_IVU_msg },
669 { 0, NULL },
670};
671static const char CHPAFSR_DTO_msg[] =
672 "System bus unmapped error for prefetch/storequeue-read";
673static const char CHPAFSR_DBERR_msg[] =
674 "System bus error for prefetch/storequeue-read";
675static const char CHPAFSR_THCE_msg[] =
676 "Hardware corrected E-cache Tag ECC error";
677static const char CHPAFSR_TSCE_msg[] =
678 "SW handled correctable E-cache Tag ECC error";
679static const char CHPAFSR_TUE_msg[] =
680 "Uncorrectable E-cache Tag ECC error";
681static const char CHPAFSR_DUE_msg[] =
682 "System bus uncorrectable data ECC error due to prefetch/store-fill";
683static struct afsr_error_table __cheetah_plus_error_table[] = {
684 { CHAFSR_PERR, CHAFSR_PERR_msg },
685 { CHAFSR_IERR, CHAFSR_IERR_msg },
686 { CHAFSR_ISAP, CHAFSR_ISAP_msg },
687 { CHAFSR_UCU, CHAFSR_UCU_msg },
688 { CHAFSR_UCC, CHAFSR_UCC_msg },
689 { CHAFSR_UE, CHAFSR_UE_msg },
690 { CHAFSR_EDU, CHAFSR_EDU_msg },
691 { CHAFSR_EMU, CHAFSR_EMU_msg },
692 { CHAFSR_WDU, CHAFSR_WDU_msg },
693 { CHAFSR_CPU, CHAFSR_CPU_msg },
694 { CHAFSR_CE, CHAFSR_CE_msg },
695 { CHAFSR_EDC, CHAFSR_EDC_msg },
696 { CHAFSR_EMC, CHAFSR_EMC_msg },
697 { CHAFSR_WDC, CHAFSR_WDC_msg },
698 { CHAFSR_CPC, CHAFSR_CPC_msg },
699 { CHAFSR_TO, CHAFSR_TO_msg },
700 { CHAFSR_BERR, CHAFSR_BERR_msg },
701 { CHPAFSR_DTO, CHPAFSR_DTO_msg },
702 { CHPAFSR_DBERR, CHPAFSR_DBERR_msg },
703 { CHPAFSR_THCE, CHPAFSR_THCE_msg },
704 { CHPAFSR_TSCE, CHPAFSR_TSCE_msg },
705 { CHPAFSR_TUE, CHPAFSR_TUE_msg },
706 { CHPAFSR_DUE, CHPAFSR_DUE_msg },
707 /* These two do not update the AFAR. */
708 { CHAFSR_IVC, CHAFSR_IVC_msg },
709 { CHAFSR_IVU, CHAFSR_IVU_msg },
710 { 0, NULL },
711};
712static const char JPAFSR_JETO_msg[] =
713 "System interface protocol error, hw timeout caused";
714static const char JPAFSR_SCE_msg[] =
715 "Parity error on system snoop results";
716static const char JPAFSR_JEIC_msg[] =
717 "System interface protocol error, illegal command detected";
718static const char JPAFSR_JEIT_msg[] =
719 "System interface protocol error, illegal ADTYPE detected";
720static const char JPAFSR_OM_msg[] =
721 "Out of range memory error has occurred";
722static const char JPAFSR_ETP_msg[] =
723 "Parity error on L2 cache tag SRAM";
724static const char JPAFSR_UMS_msg[] =
725 "Error due to unsupported store";
726static const char JPAFSR_RUE_msg[] =
727 "Uncorrectable ECC error from remote cache/memory";
728static const char JPAFSR_RCE_msg[] =
729 "Correctable ECC error from remote cache/memory";
730static const char JPAFSR_BP_msg[] =
731 "JBUS parity error on returned read data";
732static const char JPAFSR_WBP_msg[] =
733 "JBUS parity error on data for writeback or block store";
734static const char JPAFSR_FRC_msg[] =
735 "Foreign read to DRAM incurring correctable ECC error";
736static const char JPAFSR_FRU_msg[] =
737 "Foreign read to DRAM incurring uncorrectable ECC error";
738static struct afsr_error_table __jalapeno_error_table[] = {
739 { JPAFSR_JETO, JPAFSR_JETO_msg },
740 { JPAFSR_SCE, JPAFSR_SCE_msg },
741 { JPAFSR_JEIC, JPAFSR_JEIC_msg },
742 { JPAFSR_JEIT, JPAFSR_JEIT_msg },
743 { CHAFSR_PERR, CHAFSR_PERR_msg },
744 { CHAFSR_IERR, CHAFSR_IERR_msg },
745 { CHAFSR_ISAP, CHAFSR_ISAP_msg },
746 { CHAFSR_UCU, CHAFSR_UCU_msg },
747 { CHAFSR_UCC, CHAFSR_UCC_msg },
748 { CHAFSR_UE, CHAFSR_UE_msg },
749 { CHAFSR_EDU, CHAFSR_EDU_msg },
750 { JPAFSR_OM, JPAFSR_OM_msg },
751 { CHAFSR_WDU, CHAFSR_WDU_msg },
752 { CHAFSR_CPU, CHAFSR_CPU_msg },
753 { CHAFSR_CE, CHAFSR_CE_msg },
754 { CHAFSR_EDC, CHAFSR_EDC_msg },
755 { JPAFSR_ETP, JPAFSR_ETP_msg },
756 { CHAFSR_WDC, CHAFSR_WDC_msg },
757 { CHAFSR_CPC, CHAFSR_CPC_msg },
758 { CHAFSR_TO, CHAFSR_TO_msg },
759 { CHAFSR_BERR, CHAFSR_BERR_msg },
760 { JPAFSR_UMS, JPAFSR_UMS_msg },
761 { JPAFSR_RUE, JPAFSR_RUE_msg },
762 { JPAFSR_RCE, JPAFSR_RCE_msg },
763 { JPAFSR_BP, JPAFSR_BP_msg },
764 { JPAFSR_WBP, JPAFSR_WBP_msg },
765 { JPAFSR_FRC, JPAFSR_FRC_msg },
766 { JPAFSR_FRU, JPAFSR_FRU_msg },
767 /* These two do not update the AFAR. */
768 { CHAFSR_IVU, CHAFSR_IVU_msg },
769 { 0, NULL },
770};
771static struct afsr_error_table *cheetah_error_table;
772static unsigned long cheetah_afsr_errors;
773
774/* This is allocated at boot time based upon the largest hardware
775 * cpu ID in the system. We allocate two entries per cpu, one for
776 * TL==0 logging and one for TL >= 1 logging.
777 */
778struct cheetah_err_info *cheetah_error_log;
779
780static __inline__ struct cheetah_err_info *cheetah_get_error_log(unsigned long afsr)
781{
782 struct cheetah_err_info *p;
783 int cpu = smp_processor_id();
784
785 if (!cheetah_error_log)
786 return NULL;
787
788 p = cheetah_error_log + (cpu * 2);
789 if ((afsr & CHAFSR_TL1) != 0UL)
790 p++;
791
792 return p;
793}
794
795extern unsigned int tl0_icpe[], tl1_icpe[];
796extern unsigned int tl0_dcpe[], tl1_dcpe[];
797extern unsigned int tl0_fecc[], tl1_fecc[];
798extern unsigned int tl0_cee[], tl1_cee[];
799extern unsigned int tl0_iae[], tl1_iae[];
800extern unsigned int tl0_dae[], tl1_dae[];
801extern unsigned int cheetah_plus_icpe_trap_vector[], cheetah_plus_icpe_trap_vector_tl1[];
802extern unsigned int cheetah_plus_dcpe_trap_vector[], cheetah_plus_dcpe_trap_vector_tl1[];
803extern unsigned int cheetah_fecc_trap_vector[], cheetah_fecc_trap_vector_tl1[];
804extern unsigned int cheetah_cee_trap_vector[], cheetah_cee_trap_vector_tl1[];
805extern unsigned int cheetah_deferred_trap_vector[], cheetah_deferred_trap_vector_tl1[];
806
807void __init cheetah_ecache_flush_init(void)
808{
809 unsigned long largest_size, smallest_linesize, order, ver;
07f8e5f3
DM		 810 struct device_node *dp;
811 int i, instance, sz;
1da177e4
LT		 812
813 /* Scan all cpu device tree nodes, note two values:
814 * 1) largest E-cache size
815 * 2) smallest E-cache line size
816 */
817 largest_size = 0UL;
818 smallest_linesize = ~0UL;
819
820 instance = 0;
07f8e5f3 821 while (!cpu_find_by_instance(instance, &dp, NULL)) {
1da177e4
LT		 822 unsigned long val;
823
07f8e5f3
DM		 824 val = of_getintprop_default(dp, "ecache-size",
825 (2 * 1024 * 1024));
1da177e4
LT		 826 if (val > largest_size)
827 largest_size = val;
07f8e5f3 828 val = of_getintprop_default(dp, "ecache-line-size", 64);
1da177e4
LT		 829 if (val < smallest_linesize)
830 smallest_linesize = val;
831 instance++;
832 }
833
834 if (largest_size == 0UL || smallest_linesize == ~0UL) {
835 prom_printf("cheetah_ecache_flush_init: Cannot probe cpu E-cache "
836 "parameters.\n");
837 prom_halt();
838 }
839
840 ecache_flush_size = (2 * largest_size);
841 ecache_flush_linesize = smallest_linesize;
842
10147570 843 ecache_flush_physbase = find_ecache_flush_span(ecache_flush_size);
1da177e4 844
10147570 845 if (ecache_flush_physbase == ~0UL) {
1da177e4 846 prom_printf("cheetah_ecache_flush_init: Cannot find %d byte "
10147570
DM		 847 "contiguous physical memory.\n",
848 ecache_flush_size);
1da177e4
LT		 849 prom_halt();
850 }
851
852 /* Now allocate error trap reporting scoreboard. */
07f8e5f3 853 sz = NR_CPUS * (2 * sizeof(struct cheetah_err_info));
1da177e4 854 for (order = 0; order < MAX_ORDER; order++) {
07f8e5f3 855 if ((PAGE_SIZE << order) >= sz)
1da177e4
LT		 856 break;
857 }
858 cheetah_error_log = (struct cheetah_err_info *)
859 __get_free_pages(GFP_KERNEL, order);
860 if (!cheetah_error_log) {
861 prom_printf("cheetah_ecache_flush_init: Failed to allocate "
07f8e5f3 862 "error logging scoreboard (%d bytes).\n", sz);
1da177e4
LT		 863 prom_halt();
864 }
865 memset(cheetah_error_log, 0, PAGE_SIZE << order);
866
867 /* Mark all AFSRs as invalid so that the trap handler will
868 * log new new information there.
869 */
870 for (i = 0; i < 2 * NR_CPUS; i++)
871 cheetah_error_log[i].afsr = CHAFSR_INVALID;
872
873 __asm__ ("rdpr %%ver, %0" : "=r" (ver));
92704a1c
DM		 874 if ((ver >> 32) == __JALAPENO_ID ||
875 (ver >> 32) == __SERRANO_ID) {
1da177e4
LT		 876 cheetah_error_table = &__jalapeno_error_table[0];
877 cheetah_afsr_errors = JPAFSR_ERRORS;
878 } else if ((ver >> 32) == 0x003e0015) {
879 cheetah_error_table = &__cheetah_plus_error_table[0];
880 cheetah_afsr_errors = CHPAFSR_ERRORS;
881 } else {
882 cheetah_error_table = &__cheetah_error_table[0];
883 cheetah_afsr_errors = CHAFSR_ERRORS;
884 }
885
886 /* Now patch trap tables. */
887 memcpy(tl0_fecc, cheetah_fecc_trap_vector, (8 * 4));
888 memcpy(tl1_fecc, cheetah_fecc_trap_vector_tl1, (8 * 4));
889 memcpy(tl0_cee, cheetah_cee_trap_vector, (8 * 4));
890 memcpy(tl1_cee, cheetah_cee_trap_vector_tl1, (8 * 4));
891 memcpy(tl0_iae, cheetah_deferred_trap_vector, (8 * 4));
892 memcpy(tl1_iae, cheetah_deferred_trap_vector_tl1, (8 * 4));
893 memcpy(tl0_dae, cheetah_deferred_trap_vector, (8 * 4));
894 memcpy(tl1_dae, cheetah_deferred_trap_vector_tl1, (8 * 4));
895 if (tlb_type == cheetah_plus) {
896 memcpy(tl0_dcpe, cheetah_plus_dcpe_trap_vector, (8 * 4));
897 memcpy(tl1_dcpe, cheetah_plus_dcpe_trap_vector_tl1, (8 * 4));
898 memcpy(tl0_icpe, cheetah_plus_icpe_trap_vector, (8 * 4));
899 memcpy(tl1_icpe, cheetah_plus_icpe_trap_vector_tl1, (8 * 4));
900 }
901 flushi(PAGE_OFFSET);
902}
903
904static void cheetah_flush_ecache(void)
905{
906 unsigned long flush_base = ecache_flush_physbase;
907 unsigned long flush_linesize = ecache_flush_linesize;
908 unsigned long flush_size = ecache_flush_size;
909
910 __asm__ __volatile__("1: subcc %0, %4, %0\n\t"
911 " bne,pt %%xcc, 1b\n\t"
912 " ldxa [%2 + %0] %3, %%g0\n\t"
913 : "=&r" (flush_size)
914 : "0" (flush_size), "r" (flush_base),
915 "i" (ASI_PHYS_USE_EC), "r" (flush_linesize));
916}
917
918static void cheetah_flush_ecache_line(unsigned long physaddr)
919{
920 unsigned long alias;
921
922 physaddr &= ~(8UL - 1UL);
923 physaddr = (ecache_flush_physbase +
924 (physaddr & ((ecache_flush_size>>1UL) - 1UL)));
925 alias = physaddr + (ecache_flush_size >> 1UL);
926 __asm__ __volatile__("ldxa [%0] %2, %%g0\n\t"
927 "ldxa [%1] %2, %%g0\n\t"
928 "membar #Sync"
929 : /* no outputs */
930 : "r" (physaddr), "r" (alias),
931 "i" (ASI_PHYS_USE_EC));
932}
933
934/* Unfortunately, the diagnostic access to the I-cache tags we need to
935 * use to clear the thing interferes with I-cache coherency transactions.
936 *
937 * So we must only flush the I-cache when it is disabled.
938 */
939static void __cheetah_flush_icache(void)
940{
80dc0d6b
DM		 941 unsigned int icache_size, icache_line_size;
942 unsigned long addr;
943
944 icache_size = local_cpu_data().icache_size;
945 icache_line_size = local_cpu_data().icache_line_size;
1da177e4
LT		 946
947 /* Clear the valid bits in all the tags. */
80dc0d6b 948 for (addr = 0; addr < icache_size; addr += icache_line_size) {
1da177e4
LT		 949 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
950 "membar #Sync"
951 : /* no outputs */
80dc0d6b
DM		 952 : "r" (addr | (2 << 3)),
953 "i" (ASI_IC_TAG));
1da177e4
LT		 954 }
955}
956
957static void cheetah_flush_icache(void)
958{
959 unsigned long dcu_save;
960
961 /* Save current DCU, disable I-cache. */
962 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
963 "or %0, %2, %%g1\n\t"
964 "stxa %%g1, [%%g0] %1\n\t"
965 "membar #Sync"
966 : "=r" (dcu_save)
967 : "i" (ASI_DCU_CONTROL_REG), "i" (DCU_IC)
968 : "g1");
969
970 __cheetah_flush_icache();
971
972 /* Restore DCU register */
973 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
974 "membar #Sync"
975 : /* no outputs */
976 : "r" (dcu_save), "i" (ASI_DCU_CONTROL_REG));
977}
978
979static void cheetah_flush_dcache(void)
980{
80dc0d6b
DM		 981 unsigned int dcache_size, dcache_line_size;
982 unsigned long addr;
983
984 dcache_size = local_cpu_data().dcache_size;
985 dcache_line_size = local_cpu_data().dcache_line_size;
1da177e4 986
80dc0d6b 987 for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
1da177e4
LT		 988 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
989 "membar #Sync"
990 : /* no outputs */
80dc0d6b 991 : "r" (addr), "i" (ASI_DCACHE_TAG));
1da177e4
LT		 992 }
993}
994
995/* In order to make the even parity correct we must do two things.
996 * First, we clear DC_data_parity and set DC_utag to an appropriate value.
997 * Next, we clear out all 32-bytes of data for that line. Data of
998 * all-zero + tag parity value of zero == correct parity.
999 */
1000static void cheetah_plus_zap_dcache_parity(void)
1001{
80dc0d6b
DM		 1002 unsigned int dcache_size, dcache_line_size;
1003 unsigned long addr;
1004
1005 dcache_size = local_cpu_data().dcache_size;
1006 dcache_line_size = local_cpu_data().dcache_line_size;
1da177e4 1007
80dc0d6b
DM		 1008 for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
1009 unsigned long tag = (addr >> 14);
1010 unsigned long line;
1da177e4
LT		 1011
1012 __asm__ __volatile__("membar #Sync\n\t"
1013 "stxa %0, [%1] %2\n\t"
1014 "membar #Sync"
1015 : /* no outputs */
80dc0d6b 1016 : "r" (tag), "r" (addr),
1da177e4 1017 "i" (ASI_DCACHE_UTAG));
80dc0d6b 1018 for (line = addr; line < addr + dcache_line_size; line += 8)
1da177e4
LT		 1019 __asm__ __volatile__("membar #Sync\n\t"
1020 "stxa %%g0, [%0] %1\n\t"
1021 "membar #Sync"
1022 : /* no outputs */
80dc0d6b
DM		 1023 : "r" (line),
1024 "i" (ASI_DCACHE_DATA));
1da177e4
LT		 1025 }
1026}
1027
1028/* Conversion tables used to frob Cheetah AFSR syndrome values into
1029 * something palatable to the memory controller driver get_unumber
1030 * routine.
1031 */
1032#define MT0 137
1033#define MT1 138
1034#define MT2 139
1035#define NONE 254
1036#define MTC0 140
1037#define MTC1 141
1038#define MTC2 142
1039#define MTC3 143
1040#define C0 128
1041#define C1 129
1042#define C2 130
1043#define C3 131
1044#define C4 132
1045#define C5 133
1046#define C6 134
1047#define C7 135
1048#define C8 136
1049#define M2 144
1050#define M3 145
1051#define M4 146
1052#define M 147
1053static unsigned char cheetah_ecc_syntab[] = {
1054/*00*/NONE, C0, C1, M2, C2, M2, M3, 47, C3, M2, M2, 53, M2, 41, 29, M,
1055/*01*/C4, M, M, 50, M2, 38, 25, M2, M2, 33, 24, M2, 11, M, M2, 16,
1056/*02*/C5, M, M, 46, M2, 37, 19, M2, M, 31, 32, M, 7, M2, M2, 10,
1057/*03*/M2, 40, 13, M2, 59, M, M2, 66, M, M2, M2, 0, M2, 67, 71, M,
1058/*04*/C6, M, M, 43, M, 36, 18, M, M2, 49, 15, M, 63, M2, M2, 6,
1059/*05*/M2, 44, 28, M2, M, M2, M2, 52, 68, M2, M2, 62, M2, M3, M3, M4,
1060/*06*/M2, 26, 106, M2, 64, M, M2, 2, 120, M, M2, M3, M, M3, M3, M4,
1061/*07*/116, M2, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3,
1062/*08*/C7, M2, M, 42, M, 35, 17, M2, M, 45, 14, M2, 21, M2, M2, 5,
1063/*09*/M, 27, M, M, 99, M, M, 3, 114, M2, M2, 20, M2, M3, M3, M,
1064/*0a*/M2, 23, 113, M2, 112, M2, M, 51, 95, M, M2, M3, M2, M3, M3, M2,
1065/*0b*/103, M, M2, M3, M2, M3, M3, M4, M2, 48, M, M, 73, M2, M, M3,
1066/*0c*/M2, 22, 110, M2, 109, M2, M, 9, 108, M2, M, M3, M2, M3, M3, M,
1067/*0d*/102, M2, M, M, M2, M3, M3, M, M2, M3, M3, M2, M, M4, M, M3,
1068/*0e*/98, M, M2, M3, M2, M, M3, M4, M2, M3, M3, M4, M3, M, M, M,
1069/*0f*/M2, M3, M3, M, M3, M, M, M, 56, M4, M, M3, M4, M, M, M,
1070/*10*/C8, M, M2, 39, M, 34, 105, M2, M, 30, 104, M, 101, M, M, 4,
1071/*11*/M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, M2, M, M3, M,
1072/*12*/M2, 97, 82, M2, 78, M2, M2, 1, 96, M, M, M, M, M, M3, M2,
1073/*13*/94, M, M2, M3, M2, M, M3, M, M2, M, 79, M, 69, M, M4, M,
1074/*14*/M2, 93, 92, M, 91, M, M2, 8, 90, M2, M2, M, M, M, M, M4,
1075/*15*/89, M, M, M3, M2, M3, M3, M, M, M, M3, M2, M3, M2, M, M3,
1076/*16*/86, M, M2, M3, M2, M, M3, M, M2, M, M3, M, M3, M, M, M3,
1077/*17*/M, M, M3, M2, M3, M2, M4, M, 60, M, M2, M3, M4, M, M, M2,
1078/*18*/M2, 88, 85, M2, 84, M, M2, 55, 81, M2, M2, M3, M2, M3, M3, M4,
1079/*19*/77, M, M, M, M2, M3, M, M, M2, M3, M3, M4, M3, M2, M, M,
1080/*1a*/74, M, M2, M3, M, M, M3, M, M, M, M3, M, M3, M, M4, M3,
1081/*1b*/M2, 70, 107, M4, 65, M2, M2, M, 127, M, M, M, M2, M3, M3, M,
1082/*1c*/80, M2, M2, 72, M, 119, 118, M, M2, 126, 76, M, 125, M, M4, M3,
1083/*1d*/M2, 115, 124, M, 75, M, M, M3, 61, M, M4, M, M4, M, M, M,
1084/*1e*/M, 123, 122, M4, 121, M4, M, M3, 117, M2, M2, M3, M4, M3, M, M,
1085/*1f*/111, M, M, M, M4, M3, M3, M, M, M, M3, M, M3, M2, M, M
1086};
1087static unsigned char cheetah_mtag_syntab[] = {
1088 NONE, MTC0,
1089 MTC1, NONE,
1090 MTC2, NONE,
1091 NONE, MT0,
1092 MTC3, NONE,
1093 NONE, MT1,
1094 NONE, MT2,
1095 NONE, NONE
1096};
1097
1098/* Return the highest priority error conditon mentioned. */
1099static __inline__ unsigned long cheetah_get_hipri(unsigned long afsr)
1100{
1101 unsigned long tmp = 0;
1102 int i;
1103
1104 for (i = 0; cheetah_error_table[i].mask; i++) {
1105 if ((tmp = (afsr & cheetah_error_table[i].mask)) != 0UL)
1106 return tmp;
1107 }
1108 return tmp;
1109}
1110
1111static const char *cheetah_get_string(unsigned long bit)
1112{
1113 int i;
1114
1115 for (i = 0; cheetah_error_table[i].mask; i++) {
1116 if ((bit & cheetah_error_table[i].mask) != 0UL)
1117 return cheetah_error_table[i].name;
1118 }
1119 return "???";
1120}
1121
1122extern int chmc_getunumber(int, unsigned long, char *, int);
1123
1124static void cheetah_log_errors(struct pt_regs *regs, struct cheetah_err_info *info,
1125 unsigned long afsr, unsigned long afar, int recoverable)
1126{
1127 unsigned long hipri;
1128 char unum[256];
1129
1130 printk("%s" "ERROR(%d): Cheetah error trap taken afsr[%016lx] afar[%016lx] TL1(%d)\n",
1131 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1132 afsr, afar,
1133 (afsr & CHAFSR_TL1) ? 1 : 0);
955c054f 1134 printk("%s" "ERROR(%d): TPC[%lx] TNPC[%lx] O7[%lx] TSTATE[%lx]\n",
1da177e4 1135 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
955c054f 1136 regs->tpc, regs->tnpc, regs->u_regs[UREG_I7], regs->tstate);
1da177e4
LT		 1137 printk("%s" "ERROR(%d): M_SYND(%lx), E_SYND(%lx)%s%s\n",
1138 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1139 (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT,
1140 (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT,
1141 (afsr & CHAFSR_ME) ? ", Multiple Errors" : "",
1142 (afsr & CHAFSR_PRIV) ? ", Privileged" : "");
1143 hipri = cheetah_get_hipri(afsr);
1144 printk("%s" "ERROR(%d): Highest priority error (%016lx) \"%s\"\n",
1145 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1146 hipri, cheetah_get_string(hipri));
1147
1148 /* Try to get unumber if relevant. */
1149#define ESYND_ERRORS (CHAFSR_IVC | CHAFSR_IVU | \
1150 CHAFSR_CPC | CHAFSR_CPU | \
1151 CHAFSR_UE | CHAFSR_CE | \
1152 CHAFSR_EDC | CHAFSR_EDU | \
1153 CHAFSR_UCC | CHAFSR_UCU | \
1154 CHAFSR_WDU | CHAFSR_WDC)
1155#define MSYND_ERRORS (CHAFSR_EMC | CHAFSR_EMU)
1156 if (afsr & ESYND_ERRORS) {
1157 int syndrome;
1158 int ret;
1159
1160 syndrome = (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT;
1161 syndrome = cheetah_ecc_syntab[syndrome];
1162 ret = chmc_getunumber(syndrome, afar, unum, sizeof(unum));
1163 if (ret != -1)
1164 printk("%s" "ERROR(%d): AFAR E-syndrome [%s]\n",
1165 (recoverable ? KERN_WARNING : KERN_CRIT),
1166 smp_processor_id(), unum);
1167 } else if (afsr & MSYND_ERRORS) {
1168 int syndrome;
1169 int ret;
1170
1171 syndrome = (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT;
1172 syndrome = cheetah_mtag_syntab[syndrome];
1173 ret = chmc_getunumber(syndrome, afar, unum, sizeof(unum));
1174 if (ret != -1)
1175 printk("%s" "ERROR(%d): AFAR M-syndrome [%s]\n",
1176 (recoverable ? KERN_WARNING : KERN_CRIT),
1177 smp_processor_id(), unum);
1178 }
1179
1180 /* Now dump the cache snapshots. */
1181 printk("%s" "ERROR(%d): D-cache idx[%x] tag[%016lx] utag[%016lx] stag[%016lx]\n",
1182 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1183 (int) info->dcache_index,
1184 info->dcache_tag,
1185 info->dcache_utag,
1186 info->dcache_stag);
1187 printk("%s" "ERROR(%d): D-cache data0[%016lx] data1[%016lx] data2[%016lx] data3[%016lx]\n",
1188 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1189 info->dcache_data[0],
1190 info->dcache_data[1],
1191 info->dcache_data[2],
1192 info->dcache_data[3]);
1193 printk("%s" "ERROR(%d): I-cache idx[%x] tag[%016lx] utag[%016lx] stag[%016lx] "
1194 "u[%016lx] l[%016lx]\n",
1195 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1196 (int) info->icache_index,
1197 info->icache_tag,
1198 info->icache_utag,
1199 info->icache_stag,
1200 info->icache_upper,
1201 info->icache_lower);
1202 printk("%s" "ERROR(%d): I-cache INSN0[%016lx] INSN1[%016lx] INSN2[%016lx] INSN3[%016lx]\n",
1203 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1204 info->icache_data[0],
1205 info->icache_data[1],
1206 info->icache_data[2],
1207 info->icache_data[3]);
1208 printk("%s" "ERROR(%d): I-cache INSN4[%016lx] INSN5[%016lx] INSN6[%016lx] INSN7[%016lx]\n",
1209 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1210 info->icache_data[4],
1211 info->icache_data[5],
1212 info->icache_data[6],
1213 info->icache_data[7]);
1214 printk("%s" "ERROR(%d): E-cache idx[%x] tag[%016lx]\n",
1215 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1216 (int) info->ecache_index, info->ecache_tag);
1217 printk("%s" "ERROR(%d): E-cache data0[%016lx] data1[%016lx] data2[%016lx] data3[%016lx]\n",
1218 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1219 info->ecache_data[0],
1220 info->ecache_data[1],
1221 info->ecache_data[2],
1222 info->ecache_data[3]);
1223
1224 afsr = (afsr & ~hipri) & cheetah_afsr_errors;
1225 while (afsr != 0UL) {
1226 unsigned long bit = cheetah_get_hipri(afsr);
1227
1228 printk("%s" "ERROR: Multiple-error (%016lx) \"%s\"\n",
1229 (recoverable ? KERN_WARNING : KERN_CRIT),
1230 bit, cheetah_get_string(bit));
1231
1232 afsr &= ~bit;
1233 }
1234
1235 if (!recoverable)
1236 printk(KERN_CRIT "ERROR: This condition is not recoverable.\n");
1237}
1238
1239static int cheetah_recheck_errors(struct cheetah_err_info *logp)
1240{
1241 unsigned long afsr, afar;
1242 int ret = 0;
1243
1244 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1245 : "=r" (afsr)
1246 : "i" (ASI_AFSR));
1247 if ((afsr & cheetah_afsr_errors) != 0) {
1248 if (logp != NULL) {
1249 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1250 : "=r" (afar)
1251 : "i" (ASI_AFAR));
1252 logp->afsr = afsr;
1253 logp->afar = afar;
1254 }
1255 ret = 1;
1256 }
1257 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
1258 "membar #Sync\n\t"
1259 : : "r" (afsr), "i" (ASI_AFSR));
1260
1261 return ret;
1262}
1263
1264void cheetah_fecc_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1265{
1266 struct cheetah_err_info local_snapshot, *p;
1267 int recoverable;
1268
1269 /* Flush E-cache */
1270 cheetah_flush_ecache();
1271
1272 p = cheetah_get_error_log(afsr);
1273 if (!p) {
1274 prom_printf("ERROR: Early Fast-ECC error afsr[%016lx] afar[%016lx]\n",
1275 afsr, afar);
1276 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1277 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1278 prom_halt();
1279 }
1280
1281 /* Grab snapshot of logged error. */
1282 memcpy(&local_snapshot, p, sizeof(local_snapshot));
1283
1284 /* If the current trap snapshot does not match what the
1285 * trap handler passed along into our args, big trouble.
1286 * In such a case, mark the local copy as invalid.
1287 *
1288 * Else, it matches and we mark the afsr in the non-local
1289 * copy as invalid so we may log new error traps there.
1290 */
1291 if (p->afsr != afsr || p->afar != afar)
1292 local_snapshot.afsr = CHAFSR_INVALID;
1293 else
1294 p->afsr = CHAFSR_INVALID;
1295
1296 cheetah_flush_icache();
1297 cheetah_flush_dcache();
1298
1299 /* Re-enable I-cache/D-cache */
1300 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1301 "or %%g1, %1, %%g1\n\t"
1302 "stxa %%g1, [%%g0] %0\n\t"
1303 "membar #Sync"
1304 : /* no outputs */
1305 : "i" (ASI_DCU_CONTROL_REG),
1306 "i" (DCU_DC | DCU_IC)
1307 : "g1");
1308
1309 /* Re-enable error reporting */
1310 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1311 "or %%g1, %1, %%g1\n\t"
1312 "stxa %%g1, [%%g0] %0\n\t"
1313 "membar #Sync"
1314 : /* no outputs */
1315 : "i" (ASI_ESTATE_ERROR_EN),
1316 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1317 : "g1");
1318
1319 /* Decide if we can continue after handling this trap and
1320 * logging the error.
1321 */
1322 recoverable = 1;
1323 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1324 recoverable = 0;
1325
1326 /* Re-check AFSR/AFAR. What we are looking for here is whether a new
1327 * error was logged while we had error reporting traps disabled.
1328 */
1329 if (cheetah_recheck_errors(&local_snapshot)) {
1330 unsigned long new_afsr = local_snapshot.afsr;
1331
1332 /* If we got a new asynchronous error, die... */
1333 if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
1334 CHAFSR_WDU | CHAFSR_CPU |
1335 CHAFSR_IVU | CHAFSR_UE |
1336 CHAFSR_BERR | CHAFSR_TO))
1337 recoverable = 0;
1338 }
1339
1340 /* Log errors. */
1341 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1342
1343 if (!recoverable)
1344 panic("Irrecoverable Fast-ECC error trap.\n");
1345
1346 /* Flush E-cache to kick the error trap handlers out. */
1347 cheetah_flush_ecache();
1348}
1349
1350/* Try to fix a correctable error by pushing the line out from
1351 * the E-cache. Recheck error reporting registers to see if the
1352 * problem is intermittent.
1353 */
1354static int cheetah_fix_ce(unsigned long physaddr)
1355{
1356 unsigned long orig_estate;
1357 unsigned long alias1, alias2;
1358 int ret;
1359
1360 /* Make sure correctable error traps are disabled. */
1361 __asm__ __volatile__("ldxa [%%g0] %2, %0\n\t"
1362 "andn %0, %1, %%g1\n\t"
1363 "stxa %%g1, [%%g0] %2\n\t"
1364 "membar #Sync"
1365 : "=&r" (orig_estate)
1366 : "i" (ESTATE_ERROR_CEEN),
1367 "i" (ASI_ESTATE_ERROR_EN)
1368 : "g1");
1369
1370 /* We calculate alias addresses that will force the
1371 * cache line in question out of the E-cache. Then
1372 * we bring it back in with an atomic instruction so
1373 * that we get it in some modified/exclusive state,
1374 * then we displace it again to try and get proper ECC
1375 * pushed back into the system.
1376 */
1377 physaddr &= ~(8UL - 1UL);
1378 alias1 = (ecache_flush_physbase +
1379 (physaddr & ((ecache_flush_size >> 1) - 1)));
1380 alias2 = alias1 + (ecache_flush_size >> 1);
1381 __asm__ __volatile__("ldxa [%0] %3, %%g0\n\t"
1382 "ldxa [%1] %3, %%g0\n\t"
1383 "casxa [%2] %3, %%g0, %%g0\n\t"
1384 "membar #StoreLoad | #StoreStore\n\t"
1385 "ldxa [%0] %3, %%g0\n\t"
1386 "ldxa [%1] %3, %%g0\n\t"
1387 "membar #Sync"
1388 : /* no outputs */
1389 : "r" (alias1), "r" (alias2),
1390 "r" (physaddr), "i" (ASI_PHYS_USE_EC));
1391
1392 /* Did that trigger another error? */
1393 if (cheetah_recheck_errors(NULL)) {
1394 /* Try one more time. */
1395 __asm__ __volatile__("ldxa [%0] %1, %%g0\n\t"
1396 "membar #Sync"
1397 : : "r" (physaddr), "i" (ASI_PHYS_USE_EC));
1398 if (cheetah_recheck_errors(NULL))
1399 ret = 2;
1400 else
1401 ret = 1;
1402 } else {
1403 /* No new error, intermittent problem. */
1404 ret = 0;
1405 }
1406
1407 /* Restore error enables. */
1408 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
1409 "membar #Sync"
1410 : : "r" (orig_estate), "i" (ASI_ESTATE_ERROR_EN));
1411
1412 return ret;
1413}
1414
1415/* Return non-zero if PADDR is a valid physical memory address. */
1416static int cheetah_check_main_memory(unsigned long paddr)
1417{
10147570 1418 unsigned long vaddr = PAGE_OFFSET + paddr;
1da177e4 1419
13edad7a 1420 if (vaddr > (unsigned long) high_memory)
ed3ffaf7
DM		 1421 return 0;
1422
10147570 1423 return kern_addr_valid(vaddr);
1da177e4
LT		 1424}
1425
1426void cheetah_cee_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1427{
1428 struct cheetah_err_info local_snapshot, *p;
1429 int recoverable, is_memory;
1430
1431 p = cheetah_get_error_log(afsr);
1432 if (!p) {
1433 prom_printf("ERROR: Early CEE error afsr[%016lx] afar[%016lx]\n",
1434 afsr, afar);
1435 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1436 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1437 prom_halt();
1438 }
1439
1440 /* Grab snapshot of logged error. */
1441 memcpy(&local_snapshot, p, sizeof(local_snapshot));
1442
1443 /* If the current trap snapshot does not match what the
1444 * trap handler passed along into our args, big trouble.
1445 * In such a case, mark the local copy as invalid.
1446 *
1447 * Else, it matches and we mark the afsr in the non-local
1448 * copy as invalid so we may log new error traps there.
1449 */
1450 if (p->afsr != afsr || p->afar != afar)
1451 local_snapshot.afsr = CHAFSR_INVALID;
1452 else
1453 p->afsr = CHAFSR_INVALID;
1454
1455 is_memory = cheetah_check_main_memory(afar);
1456
1457 if (is_memory && (afsr & CHAFSR_CE) != 0UL) {
1458 /* XXX Might want to log the results of this operation
1459 * XXX somewhere... -DaveM
1460 */
1461 cheetah_fix_ce(afar);
1462 }
1463
1464 {
1465 int flush_all, flush_line;
1466
1467 flush_all = flush_line = 0;
1468 if ((afsr & CHAFSR_EDC) != 0UL) {
1469 if ((afsr & cheetah_afsr_errors) == CHAFSR_EDC)
1470 flush_line = 1;
1471 else
1472 flush_all = 1;
1473 } else if ((afsr & CHAFSR_CPC) != 0UL) {
1474 if ((afsr & cheetah_afsr_errors) == CHAFSR_CPC)
1475 flush_line = 1;
1476 else
1477 flush_all = 1;
1478 }
1479
1480 /* Trap handler only disabled I-cache, flush it. */
1481 cheetah_flush_icache();
1482
1483 /* Re-enable I-cache */
1484 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1485 "or %%g1, %1, %%g1\n\t"
1486 "stxa %%g1, [%%g0] %0\n\t"
1487 "membar #Sync"
1488 : /* no outputs */
1489 : "i" (ASI_DCU_CONTROL_REG),
1490 "i" (DCU_IC)
1491 : "g1");
1492
1493 if (flush_all)
1494 cheetah_flush_ecache();
1495 else if (flush_line)
1496 cheetah_flush_ecache_line(afar);
1497 }
1498
1499 /* Re-enable error reporting */
1500 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1501 "or %%g1, %1, %%g1\n\t"
1502 "stxa %%g1, [%%g0] %0\n\t"
1503 "membar #Sync"
1504 : /* no outputs */
1505 : "i" (ASI_ESTATE_ERROR_EN),
1506 "i" (ESTATE_ERROR_CEEN)
1507 : "g1");
1508
1509 /* Decide if we can continue after handling this trap and
1510 * logging the error.
1511 */
1512 recoverable = 1;
1513 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1514 recoverable = 0;
1515
1516 /* Re-check AFSR/AFAR */
1517 (void) cheetah_recheck_errors(&local_snapshot);
1518
1519 /* Log errors. */
1520 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1521
1522 if (!recoverable)
1523 panic("Irrecoverable Correctable-ECC error trap.\n");
1524}
1525
1526void cheetah_deferred_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1527{
1528 struct cheetah_err_info local_snapshot, *p;
1529 int recoverable, is_memory;
1530
1531#ifdef CONFIG_PCI
1532 /* Check for the special PCI poke sequence. */
1533 if (pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
1534 cheetah_flush_icache();
1535 cheetah_flush_dcache();
1536
1537 /* Re-enable I-cache/D-cache */
1538 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1539 "or %%g1, %1, %%g1\n\t"
1540 "stxa %%g1, [%%g0] %0\n\t"
1541 "membar #Sync"
1542 : /* no outputs */
1543 : "i" (ASI_DCU_CONTROL_REG),
1544 "i" (DCU_DC | DCU_IC)
1545 : "g1");
1546
1547 /* Re-enable error reporting */
1548 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1549 "or %%g1, %1, %%g1\n\t"
1550 "stxa %%g1, [%%g0] %0\n\t"
1551 "membar #Sync"
1552 : /* no outputs */
1553 : "i" (ASI_ESTATE_ERROR_EN),
1554 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1555 : "g1");
1556
1557 (void) cheetah_recheck_errors(NULL);
1558
1559 pci_poke_faulted = 1;
1560 regs->tpc += 4;
1561 regs->tnpc = regs->tpc + 4;
1562 return;
1563 }
1564#endif
1565
1566 p = cheetah_get_error_log(afsr);
1567 if (!p) {
1568 prom_printf("ERROR: Early deferred error afsr[%016lx] afar[%016lx]\n",
1569 afsr, afar);
1570 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1571 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1572 prom_halt();
1573 }
1574
1575 /* Grab snapshot of logged error. */
1576 memcpy(&local_snapshot, p, sizeof(local_snapshot));
1577
1578 /* If the current trap snapshot does not match what the
1579 * trap handler passed along into our args, big trouble.
1580 * In such a case, mark the local copy as invalid.
1581 *
1582 * Else, it matches and we mark the afsr in the non-local
1583 * copy as invalid so we may log new error traps there.
1584 */
1585 if (p->afsr != afsr || p->afar != afar)
1586 local_snapshot.afsr = CHAFSR_INVALID;
1587 else
1588 p->afsr = CHAFSR_INVALID;
1589
1590 is_memory = cheetah_check_main_memory(afar);
1591
1592 {
1593 int flush_all, flush_line;
1594
1595 flush_all = flush_line = 0;
1596 if ((afsr & CHAFSR_EDU) != 0UL) {
1597 if ((afsr & cheetah_afsr_errors) == CHAFSR_EDU)
1598 flush_line = 1;
1599 else
1600 flush_all = 1;
1601 } else if ((afsr & CHAFSR_BERR) != 0UL) {
1602 if ((afsr & cheetah_afsr_errors) == CHAFSR_BERR)
1603 flush_line = 1;
1604 else
1605 flush_all = 1;
1606 }
1607
1608 cheetah_flush_icache();
1609 cheetah_flush_dcache();
1610
1611 /* Re-enable I/D caches */
1612 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1613 "or %%g1, %1, %%g1\n\t"
1614 "stxa %%g1, [%%g0] %0\n\t"
1615 "membar #Sync"
1616 : /* no outputs */
1617 : "i" (ASI_DCU_CONTROL_REG),
1618 "i" (DCU_IC | DCU_DC)
1619 : "g1");
1620
1621 if (flush_all)
1622 cheetah_flush_ecache();
1623 else if (flush_line)
1624 cheetah_flush_ecache_line(afar);
1625 }
1626
1627 /* Re-enable error reporting */
1628 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1629 "or %%g1, %1, %%g1\n\t"
1630 "stxa %%g1, [%%g0] %0\n\t"
1631 "membar #Sync"
1632 : /* no outputs */
1633 : "i" (ASI_ESTATE_ERROR_EN),
1634 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1635 : "g1");
1636
1637 /* Decide if we can continue after handling this trap and
1638 * logging the error.
1639 */
1640 recoverable = 1;
1641 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1642 recoverable = 0;
1643
1644 /* Re-check AFSR/AFAR. What we are looking for here is whether a new
1645 * error was logged while we had error reporting traps disabled.
1646 */
1647 if (cheetah_recheck_errors(&local_snapshot)) {
1648 unsigned long new_afsr = local_snapshot.afsr;
1649
1650 /* If we got a new asynchronous error, die... */
1651 if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
1652 CHAFSR_WDU | CHAFSR_CPU |
1653 CHAFSR_IVU | CHAFSR_UE |
1654 CHAFSR_BERR | CHAFSR_TO))
1655 recoverable = 0;
1656 }
1657
1658 /* Log errors. */
1659 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1660
1661 /* "Recoverable" here means we try to yank the page from ever
1662 * being newly used again. This depends upon a few things:
1663 * 1) Must be main memory, and AFAR must be valid.
1664 * 2) If we trapped from user, OK.
1665 * 3) Else, if we trapped from kernel we must find exception
1666 * table entry (ie. we have to have been accessing user
1667 * space).
1668 *
1669 * If AFAR is not in main memory, or we trapped from kernel
1670 * and cannot find an exception table entry, it is unacceptable
1671 * to try and continue.
1672 */
1673 if (recoverable && is_memory) {
1674 if ((regs->tstate & TSTATE_PRIV) == 0UL) {
1675 /* OK, usermode access. */
1676 recoverable = 1;
1677 } else {
8cf14af0 1678 const struct exception_table_entry *entry;
1da177e4 1679
8cf14af0
DM		 1680 entry = search_exception_tables(regs->tpc);
1681 if (entry) {
1da177e4
LT		 1682 /* OK, kernel access to userspace. */
1683 recoverable = 1;
1684
1685 } else {
1686 /* BAD, privileged state is corrupted. */
1687 recoverable = 0;
1688 }
1689
1690 if (recoverable) {
1691 if (pfn_valid(afar >> PAGE_SHIFT))
1692 get_page(pfn_to_page(afar >> PAGE_SHIFT));
1693 else
1694 recoverable = 0;
1695
1696 /* Only perform fixup if we still have a
1697 * recoverable condition.
1698 */
1699 if (recoverable) {
8cf14af0 1700 regs->tpc = entry->fixup;
1da177e4 1701 regs->tnpc = regs->tpc + 4;
1da177e4
LT		 1702 }
1703 }
1704 }
1705 } else {
1706 recoverable = 0;
1707 }
1708
1709 if (!recoverable)
1710 panic("Irrecoverable deferred error trap.\n");
1711}
1712
1713/* Handle a D/I cache parity error trap. TYPE is encoded as:
1714 *
1715 * Bit0: 0=dcache,1=icache
1716 * Bit1: 0=recoverable,1=unrecoverable
1717 *
1718 * The hardware has disabled both the I-cache and D-cache in
1719 * the %dcr register.
1720 */
1721void cheetah_plus_parity_error(int type, struct pt_regs *regs)
1722{
1723 if (type & 0x1)
1724 __cheetah_flush_icache();
1725 else
1726 cheetah_plus_zap_dcache_parity();
1727 cheetah_flush_dcache();
1728
1729 /* Re-enable I-cache/D-cache */
1730 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1731 "or %%g1, %1, %%g1\n\t"
1732 "stxa %%g1, [%%g0] %0\n\t"
1733 "membar #Sync"
1734 : /* no outputs */
1735 : "i" (ASI_DCU_CONTROL_REG),
1736 "i" (DCU_DC | DCU_IC)
1737 : "g1");
1738
1739 if (type & 0x2) {
1740 printk(KERN_EMERG "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
1741 smp_processor_id(),
1742 (type & 0x1) ? 'I' : 'D',
1743 regs->tpc);
1744 panic("Irrecoverable Cheetah+ parity error.");
1745 }
1746
1747 printk(KERN_WARNING "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
1748 smp_processor_id(),
1749 (type & 0x1) ? 'I' : 'D',
1750 regs->tpc);
1751}
1752
5b0c0572
DM		 1753struct sun4v_error_entry {
1754 u64 err_handle;
1755 u64 err_stick;
1756
1757 u32 err_type;
1758#define SUN4V_ERR_TYPE_UNDEFINED 0
1759#define SUN4V_ERR_TYPE_UNCORRECTED_RES 1
1760#define SUN4V_ERR_TYPE_PRECISE_NONRES 2
1761#define SUN4V_ERR_TYPE_DEFERRED_NONRES 3
1762#define SUN4V_ERR_TYPE_WARNING_RES 4
1763
1764 u32 err_attrs;
1765#define SUN4V_ERR_ATTRS_PROCESSOR 0x00000001
1766#define SUN4V_ERR_ATTRS_MEMORY 0x00000002
1767#define SUN4V_ERR_ATTRS_PIO 0x00000004
1768#define SUN4V_ERR_ATTRS_INT_REGISTERS 0x00000008
1769#define SUN4V_ERR_ATTRS_FPU_REGISTERS 0x00000010
1770#define SUN4V_ERR_ATTRS_USER_MODE 0x01000000
1771#define SUN4V_ERR_ATTRS_PRIV_MODE 0x02000000
1772#define SUN4V_ERR_ATTRS_RES_QUEUE_FULL 0x80000000
1773
1774 u64 err_raddr;
1775 u32 err_size;
1776 u16 err_cpu;
1777 u16 err_pad;
1778};
1779
1780static atomic_t sun4v_resum_oflow_cnt = ATOMIC_INIT(0);
1781static atomic_t sun4v_nonresum_oflow_cnt = ATOMIC_INIT(0);
1782
1783static const char *sun4v_err_type_to_str(u32 type)
1784{
1785 switch (type) {
1786 case SUN4V_ERR_TYPE_UNDEFINED:
1787 return "undefined";
1788 case SUN4V_ERR_TYPE_UNCORRECTED_RES:
1789 return "uncorrected resumable";
1790 case SUN4V_ERR_TYPE_PRECISE_NONRES:
1791 return "precise nonresumable";
1792 case SUN4V_ERR_TYPE_DEFERRED_NONRES:
1793 return "deferred nonresumable";
1794 case SUN4V_ERR_TYPE_WARNING_RES:
1795 return "warning resumable";
1796 default:
1797 return "unknown";
1798 };
1799}
1800
5224e6cc
DM		 1801extern void __show_regs(struct pt_regs * regs);
1802
1803static void sun4v_log_error(struct pt_regs *regs, struct sun4v_error_entry *ent, int cpu, const char *pfx, atomic_t *ocnt)
5b0c0572
DM		 1804{
1805 int cnt;
1806
1807 printk("%s: Reporting on cpu %d\n", pfx, cpu);
1808 printk("%s: err_handle[%lx] err_stick[%lx] err_type[%08x:%s]\n",
1809 pfx,
1810 ent->err_handle, ent->err_stick,
1811 ent->err_type,
1812 sun4v_err_type_to_str(ent->err_type));
1813 printk("%s: err_attrs[%08x:%s %s %s %s %s %s %s %s]\n",
1814 pfx,
1815 ent->err_attrs,
1816 ((ent->err_attrs & SUN4V_ERR_ATTRS_PROCESSOR) ?
1817 "processor" : ""),
1818 ((ent->err_attrs & SUN4V_ERR_ATTRS_MEMORY) ?
1819 "memory" : ""),
1820 ((ent->err_attrs & SUN4V_ERR_ATTRS_PIO) ?
1821 "pio" : ""),
1822 ((ent->err_attrs & SUN4V_ERR_ATTRS_INT_REGISTERS) ?
1823 "integer-regs" : ""),
1824 ((ent->err_attrs & SUN4V_ERR_ATTRS_FPU_REGISTERS) ?
1825 "fpu-regs" : ""),
1826 ((ent->err_attrs & SUN4V_ERR_ATTRS_USER_MODE) ?
1827 "user" : ""),
1828 ((ent->err_attrs & SUN4V_ERR_ATTRS_PRIV_MODE) ?
1829 "privileged" : ""),
1830 ((ent->err_attrs & SUN4V_ERR_ATTRS_RES_QUEUE_FULL) ?
1831 "queue-full" : ""));
1832 printk("%s: err_raddr[%016lx] err_size[%u] err_cpu[%u]\n",
1833 pfx,
1834 ent->err_raddr, ent->err_size, ent->err_cpu);
1835
5224e6cc
DM		 1836 __show_regs(regs);
1837
5b0c0572
DM		 1838 if ((cnt = atomic_read(ocnt)) != 0) {
1839 atomic_set(ocnt, 0);
1840 wmb();
1841 printk("%s: Queue overflowed %d times.\n",
1842 pfx, cnt);
1843 }
1844}
1845
1846/* We run with %pil set to 15 and PSTATE_IE enabled in %pstate.
1847 * Log the event and clear the first word of the entry.
1848 */
1849void sun4v_resum_error(struct pt_regs *regs, unsigned long offset)
1850{
1851 struct sun4v_error_entry *ent, local_copy;
1852 struct trap_per_cpu *tb;
1853 unsigned long paddr;
1854 int cpu;
1855
1856 cpu = get_cpu();
1857
1858 tb = &trap_block[cpu];
1859 paddr = tb->resum_kernel_buf_pa + offset;
1860 ent = __va(paddr);
1861
1862 memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
1863
1864 /* We have a local copy now, so release the entry. */
1865 ent->err_handle = 0;
1866 wmb();
1867
1868 put_cpu();
1869
5224e6cc 1870 sun4v_log_error(regs, &local_copy, cpu,
5b0c0572
DM		 1871 KERN_ERR "RESUMABLE ERROR",
1872 &sun4v_resum_oflow_cnt);
1873}
1874
1875/* If we try to printk() we'll probably make matters worse, by trying
1876 * to retake locks this cpu already holds or causing more errors. So
1877 * just bump a counter, and we'll report these counter bumps above.
1878 */
1879void sun4v_resum_overflow(struct pt_regs *regs)
1880{
1881 atomic_inc(&sun4v_resum_oflow_cnt);
1882}
1883
1884/* We run with %pil set to 15 and PSTATE_IE enabled in %pstate.
1885 * Log the event, clear the first word of the entry, and die.
1886 */
1887void sun4v_nonresum_error(struct pt_regs *regs, unsigned long offset)
1888{
1889 struct sun4v_error_entry *ent, local_copy;
1890 struct trap_per_cpu *tb;
1891 unsigned long paddr;
1892 int cpu;
1893
1894 cpu = get_cpu();
1895
1896 tb = &trap_block[cpu];
1897 paddr = tb->nonresum_kernel_buf_pa + offset;
1898 ent = __va(paddr);
1899
1900 memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
1901
1902 /* We have a local copy now, so release the entry. */
1903 ent->err_handle = 0;
1904 wmb();
1905
1906 put_cpu();
1907
1908#ifdef CONFIG_PCI
1909 /* Check for the special PCI poke sequence. */
1910 if (pci_poke_in_progress && pci_poke_cpu == cpu) {
1911 pci_poke_faulted = 1;
1912 regs->tpc += 4;
1913 regs->tnpc = regs->tpc + 4;
1914 return;
1915 }
1916#endif
1917
5224e6cc 1918 sun4v_log_error(regs, &local_copy, cpu,
5b0c0572
DM		 1919 KERN_EMERG "NON-RESUMABLE ERROR",
1920 &sun4v_nonresum_oflow_cnt);
1921
1922 panic("Non-resumable error.");
1923}
1924
1925/* If we try to printk() we'll probably make matters worse, by trying
1926 * to retake locks this cpu already holds or causing more errors. So
1927 * just bump a counter, and we'll report these counter bumps above.
1928 */
1929void sun4v_nonresum_overflow(struct pt_regs *regs)
1930{
1931 /* XXX Actually even this can make not that much sense. Perhaps
1932 * XXX we should just pull the plug and panic directly from here?
1933 */
1934 atomic_inc(&sun4v_nonresum_oflow_cnt);
1935}
1936
6c8927c9
DM		 1937unsigned long sun4v_err_itlb_vaddr;
1938unsigned long sun4v_err_itlb_ctx;
1939unsigned long sun4v_err_itlb_pte;
1940unsigned long sun4v_err_itlb_error;
1941
1942void sun4v_itlb_error_report(struct pt_regs *regs, int tl)
1943{
1944 if (tl > 1)
1945 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
1946
04d74758
DM		 1947 printk(KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n",
1948 regs->tpc, tl);
1949 printk(KERN_EMERG "SUN4V-ITLB: vaddr[%lx] ctx[%lx] "
1950 "pte[%lx] error[%lx]\n",
6c8927c9
DM		 1951 sun4v_err_itlb_vaddr, sun4v_err_itlb_ctx,
1952 sun4v_err_itlb_pte, sun4v_err_itlb_error);
04d74758 1953
6c8927c9
DM		 1954 prom_halt();
1955}
1956
1957unsigned long sun4v_err_dtlb_vaddr;
1958unsigned long sun4v_err_dtlb_ctx;
1959unsigned long sun4v_err_dtlb_pte;
1960unsigned long sun4v_err_dtlb_error;
1961
1962void sun4v_dtlb_error_report(struct pt_regs *regs, int tl)
1963{
1964 if (tl > 1)
1965 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
1966
04d74758
DM		 1967 printk(KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n",
1968 regs->tpc, tl);
1969 printk(KERN_EMERG "SUN4V-DTLB: vaddr[%lx] ctx[%lx] "
1970 "pte[%lx] error[%lx]\n",
6c8927c9
DM		 1971 sun4v_err_dtlb_vaddr, sun4v_err_dtlb_ctx,
1972 sun4v_err_dtlb_pte, sun4v_err_dtlb_error);
04d74758 1973
6c8927c9
DM		 1974 prom_halt();
1975}
1976
2a3a5f5d
DM		 1977void hypervisor_tlbop_error(unsigned long err, unsigned long op)
1978{
1979 printk(KERN_CRIT "SUN4V: TLB hv call error %lu for op %lu\n",
1980 err, op);
1981}
1982
1983void hypervisor_tlbop_error_xcall(unsigned long err, unsigned long op)
1984{
1985 printk(KERN_CRIT "SUN4V: XCALL TLB hv call error %lu for op %lu\n",
1986 err, op);
1987}
1988
1da177e4
LT		 1989void do_fpe_common(struct pt_regs *regs)
1990{
1991 if (regs->tstate & TSTATE_PRIV) {
1992 regs->tpc = regs->tnpc;
1993 regs->tnpc += 4;
1994 } else {
1995 unsigned long fsr = current_thread_info()->xfsr[0];
1996 siginfo_t info;
1997
1998 if (test_thread_flag(TIF_32BIT)) {
1999 regs->tpc &= 0xffffffff;
2000 regs->tnpc &= 0xffffffff;
2001 }
2002 info.si_signo = SIGFPE;
2003 info.si_errno = 0;
2004 info.si_addr = (void __user *)regs->tpc;
2005 info.si_trapno = 0;
2006 info.si_code = __SI_FAULT;
2007 if ((fsr & 0x1c000) == (1 << 14)) {
2008 if (fsr & 0x10)
2009 info.si_code = FPE_FLTINV;
2010 else if (fsr & 0x08)
2011 info.si_code = FPE_FLTOVF;
2012 else if (fsr & 0x04)
2013 info.si_code = FPE_FLTUND;
2014 else if (fsr & 0x02)
2015 info.si_code = FPE_FLTDIV;
2016 else if (fsr & 0x01)
2017 info.si_code = FPE_FLTRES;
2018 }
2019 force_sig_info(SIGFPE, &info, current);
2020 }
2021}
2022
2023void do_fpieee(struct pt_regs *regs)
2024{
2025 if (notify_die(DIE_TRAP, "fpu exception ieee", regs,
2026 0, 0x24, SIGFPE) == NOTIFY_STOP)
2027 return;
2028
2029 do_fpe_common(regs);
2030}
2031
2032extern int do_mathemu(struct pt_regs *, struct fpustate *);
2033
2034void do_fpother(struct pt_regs *regs)
2035{
2036 struct fpustate *f = FPUSTATE;
2037 int ret = 0;
2038
2039 if (notify_die(DIE_TRAP, "fpu exception other", regs,
2040 0, 0x25, SIGFPE) == NOTIFY_STOP)
2041 return;
2042
2043 switch ((current_thread_info()->xfsr[0] & 0x1c000)) {
2044 case (2 << 14): /* unfinished_FPop */
2045 case (3 << 14): /* unimplemented_FPop */
2046 ret = do_mathemu(regs, f);
2047 break;
2048 }
2049 if (ret)
2050 return;
2051 do_fpe_common(regs);
2052}
2053
2054void do_tof(struct pt_regs *regs)
2055{
2056 siginfo_t info;
2057
2058 if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs,
2059 0, 0x26, SIGEMT) == NOTIFY_STOP)
2060 return;
2061
2062 if (regs->tstate & TSTATE_PRIV)
2063 die_if_kernel("Penguin overflow trap from kernel mode", regs);
2064 if (test_thread_flag(TIF_32BIT)) {
2065 regs->tpc &= 0xffffffff;
2066 regs->tnpc &= 0xffffffff;
2067 }
2068 info.si_signo = SIGEMT;
2069 info.si_errno = 0;
2070 info.si_code = EMT_TAGOVF;
2071 info.si_addr = (void __user *)regs->tpc;
2072 info.si_trapno = 0;
2073 force_sig_info(SIGEMT, &info, current);
2074}
2075
2076void do_div0(struct pt_regs *regs)
2077{
2078 siginfo_t info;
2079
2080 if (notify_die(DIE_TRAP, "integer division by zero", regs,
2081 0, 0x28, SIGFPE) == NOTIFY_STOP)
2082 return;
2083
2084 if (regs->tstate & TSTATE_PRIV)
2085 die_if_kernel("TL0: Kernel divide by zero.", regs);
2086 if (test_thread_flag(TIF_32BIT)) {
2087 regs->tpc &= 0xffffffff;
2088 regs->tnpc &= 0xffffffff;
2089 }
2090 info.si_signo = SIGFPE;
2091 info.si_errno = 0;
2092 info.si_code = FPE_INTDIV;
2093 info.si_addr = (void __user *)regs->tpc;
2094 info.si_trapno = 0;
2095 force_sig_info(SIGFPE, &info, current);
2096}
2097
2098void instruction_dump (unsigned int *pc)
2099{
2100 int i;
2101
2102 if ((((unsigned long) pc) & 3))
2103 return;
2104
2105 printk("Instruction DUMP:");
2106 for (i = -3; i < 6; i++)
2107 printk("%c%08x%c",i?' ':'<',pc[i],i?' ':'>');
2108 printk("\n");
2109}
2110
2111static void user_instruction_dump (unsigned int __user *pc)
2112{
2113 int i;
2114 unsigned int buf[9];
2115
2116 if ((((unsigned long) pc) & 3))
2117 return;
2118
2119 if (copy_from_user(buf, pc - 3, sizeof(buf)))
2120 return;
2121
2122 printk("Instruction DUMP:");
2123 for (i = 0; i < 9; i++)
2124 printk("%c%08x%c",i==3?' ':'<',buf[i],i==3?' ':'>');
2125 printk("\n");
2126}
2127
2128void show_stack(struct task_struct *tsk, unsigned long *_ksp)
2129{
2130 unsigned long pc, fp, thread_base, ksp;
ee3eea16 2131 void *tp = task_stack_page(tsk);
1da177e4
LT		 2132 struct reg_window *rw;
2133 int count = 0;
2134
2135 ksp = (unsigned long) _ksp;
2136
2137 if (tp == current_thread_info())
2138 flushw_all();
2139
2140 fp = ksp + STACK_BIAS;
2141 thread_base = (unsigned long) tp;
2142
2143 printk("Call Trace:");
2144#ifdef CONFIG_KALLSYMS
2145 printk("\n");
2146#endif
2147 do {
2148 /* Bogus frame pointer? */
2149 if (fp < (thread_base + sizeof(struct thread_info)) ||
2150 fp >= (thread_base + THREAD_SIZE))
2151 break;
2152 rw = (struct reg_window *)fp;
2153 pc = rw->ins[7];
2154 printk(" [%016lx] ", pc);
2155 print_symbol("%s\n", pc);
2156 fp = rw->ins[6] + STACK_BIAS;
2157 } while (++count < 16);
2158#ifndef CONFIG_KALLSYMS
2159 printk("\n");
2160#endif
2161}
2162
2163void dump_stack(void)
2164{
2165 unsigned long *ksp;
2166
2167 __asm__ __volatile__("mov %%fp, %0"
2168 : "=r" (ksp));
2169 show_stack(current, ksp);
2170}
2171
2172EXPORT_SYMBOL(dump_stack);
2173
2174static inline int is_kernel_stack(struct task_struct *task,
2175 struct reg_window *rw)
2176{
2177 unsigned long rw_addr = (unsigned long) rw;
2178 unsigned long thread_base, thread_end;
2179
2180 if (rw_addr < PAGE_OFFSET) {
2181 if (task != &init_task)
2182 return 0;
2183 }
2184
ee3eea16 2185 thread_base = (unsigned long) task_stack_page(task);
1da177e4
LT		 2186 thread_end = thread_base + sizeof(union thread_union);
2187 if (rw_addr >= thread_base &&
2188 rw_addr < thread_end &&
2189 !(rw_addr & 0x7UL))
2190 return 1;
2191
2192 return 0;
2193}
2194
2195static inline struct reg_window *kernel_stack_up(struct reg_window *rw)
2196{
2197 unsigned long fp = rw->ins[6];
2198
2199 if (!fp)
2200 return NULL;
2201
2202 return (struct reg_window *) (fp + STACK_BIAS);
2203}
2204
2205void die_if_kernel(char *str, struct pt_regs *regs)
2206{
2207 static int die_counter;
1da177e4
LT		 2208 extern void smp_report_regs(void);
2209 int count = 0;
2210
2211 /* Amuse the user. */
2212 printk(
2213" \\|/ ____ \\|/\n"
2214" \"@'/ .. \\`@\"\n"
2215" /_| \\__/ |_\\\n"
2216" \\__U_/\n");
2217
2218 printk("%s(%d): %s [#%d]\n", current->comm, current->pid, str, ++die_counter);
2219 notify_die(DIE_OOPS, str, regs, 0, 255, SIGSEGV);
2220 __asm__ __volatile__("flushw");
2221 __show_regs(regs);
2222 if (regs->tstate & TSTATE_PRIV) {
2223 struct reg_window *rw = (struct reg_window *)
2224 (regs->u_regs[UREG_FP] + STACK_BIAS);
2225
2226 /* Stop the back trace when we hit userland or we
2227 * find some badly aligned kernel stack.
2228 */
2229 while (rw &&
2230 count++ < 30&&
2231 is_kernel_stack(current, rw)) {
2232 printk("Caller[%016lx]", rw->ins[7]);
2233 print_symbol(": %s", rw->ins[7]);
2234 printk("\n");
2235
2236 rw = kernel_stack_up(rw);
2237 }
2238 instruction_dump ((unsigned int *) regs->tpc);
2239 } else {
2240 if (test_thread_flag(TIF_32BIT)) {
2241 regs->tpc &= 0xffffffff;
2242 regs->tnpc &= 0xffffffff;
2243 }
2244 user_instruction_dump ((unsigned int __user *) regs->tpc);
2245 }
37133c00 2246#if 0
1da177e4
LT		 2247#ifdef CONFIG_SMP
2248 smp_report_regs();
2249#endif
37133c00 2250#endif
1da177e4
LT		 2251 if (regs->tstate & TSTATE_PRIV)
2252 do_exit(SIGKILL);
2253 do_exit(SIGSEGV);
2254}
2255
2256extern int handle_popc(u32 insn, struct pt_regs *regs);
2257extern int handle_ldf_stq(u32 insn, struct pt_regs *regs);
2258
2259void do_illegal_instruction(struct pt_regs *regs)
2260{
2261 unsigned long pc = regs->tpc;
2262 unsigned long tstate = regs->tstate;
2263 u32 insn;
2264 siginfo_t info;
2265
2266 if (notify_die(DIE_TRAP, "illegal instruction", regs,
2267 0, 0x10, SIGILL) == NOTIFY_STOP)
2268 return;
2269
2270 if (tstate & TSTATE_PRIV)
2271 die_if_kernel("Kernel illegal instruction", regs);
2272 if (test_thread_flag(TIF_32BIT))
2273 pc = (u32)pc;
2274 if (get_user(insn, (u32 __user *) pc) != -EFAULT) {
2275 if ((insn & 0xc1ffc000) == 0x81700000) /* POPC */ {
2276 if (handle_popc(insn, regs))
2277 return;
2278 } else if ((insn & 0xc1580000) == 0xc1100000) /* LDQ/STQ */ {
2279 if (handle_ldf_stq(insn, regs))
2280 return;
0c51ed93
DM		 2281 } else if (tlb_type == hypervisor) {
2282 extern int vis_emul(struct pt_regs *, unsigned int);
2283
2284 if (!vis_emul(regs, insn))
2285 return;
1da177e4
LT		 2286 }
2287 }
2288 info.si_signo = SIGILL;
2289 info.si_errno = 0;
2290 info.si_code = ILL_ILLOPC;
2291 info.si_addr = (void __user *)pc;
2292 info.si_trapno = 0;
2293 force_sig_info(SIGILL, &info, current);
2294}
2295
ed6b0b45
DM		 2296extern void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn);
2297
1da177e4
LT		 2298void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
2299{
2300 siginfo_t info;
2301
2302 if (notify_die(DIE_TRAP, "memory address unaligned", regs,
2303 0, 0x34, SIGSEGV) == NOTIFY_STOP)
2304 return;
2305
2306 if (regs->tstate & TSTATE_PRIV) {
ed6b0b45 2307 kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
1da177e4
LT		 2308 return;
2309 }
2310 info.si_signo = SIGBUS;
2311 info.si_errno = 0;
2312 info.si_code = BUS_ADRALN;
2313 info.si_addr = (void __user *)sfar;
2314 info.si_trapno = 0;
2315 force_sig_info(SIGBUS, &info, current);
2316}
2317
9f8a5b84 2318void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
ed6b0b45
DM		 2319{
2320 siginfo_t info;
2321
2322 if (notify_die(DIE_TRAP, "memory address unaligned", regs,
2323 0, 0x34, SIGSEGV) == NOTIFY_STOP)
2324 return;
2325
2326 if (regs->tstate & TSTATE_PRIV) {
2327 kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
2328 return;
2329 }
2330 info.si_signo = SIGBUS;
2331 info.si_errno = 0;
2332 info.si_code = BUS_ADRALN;
2333 info.si_addr = (void __user *) addr;
2334 info.si_trapno = 0;
2335 force_sig_info(SIGBUS, &info, current);
2336}
2337
1da177e4
LT		 2338void do_privop(struct pt_regs *regs)
2339{
2340 siginfo_t info;
2341
2342 if (notify_die(DIE_TRAP, "privileged operation", regs,
2343 0, 0x11, SIGILL) == NOTIFY_STOP)
2344 return;
2345
2346 if (test_thread_flag(TIF_32BIT)) {
2347 regs->tpc &= 0xffffffff;
2348 regs->tnpc &= 0xffffffff;
2349 }
2350 info.si_signo = SIGILL;
2351 info.si_errno = 0;
2352 info.si_code = ILL_PRVOPC;
2353 info.si_addr = (void __user *)regs->tpc;
2354 info.si_trapno = 0;
2355 force_sig_info(SIGILL, &info, current);
2356}
2357
2358void do_privact(struct pt_regs *regs)
2359{
2360 do_privop(regs);
2361}
2362
2363/* Trap level 1 stuff or other traps we should never see... */
2364void do_cee(struct pt_regs *regs)
2365{
2366 die_if_kernel("TL0: Cache Error Exception", regs);
2367}
2368
2369void do_cee_tl1(struct pt_regs *regs)
2370{
2371 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2372 die_if_kernel("TL1: Cache Error Exception", regs);
2373}
2374
2375void do_dae_tl1(struct pt_regs *regs)
2376{
2377 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2378 die_if_kernel("TL1: Data Access Exception", regs);
2379}
2380
2381void do_iae_tl1(struct pt_regs *regs)
2382{
2383 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2384 die_if_kernel("TL1: Instruction Access Exception", regs);
2385}
2386
2387void do_div0_tl1(struct pt_regs *regs)
2388{
2389 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2390 die_if_kernel("TL1: DIV0 Exception", regs);
2391}
2392
2393void do_fpdis_tl1(struct pt_regs *regs)
2394{
2395 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2396 die_if_kernel("TL1: FPU Disabled", regs);
2397}
2398
2399void do_fpieee_tl1(struct pt_regs *regs)
2400{
2401 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2402 die_if_kernel("TL1: FPU IEEE Exception", regs);
2403}
2404
2405void do_fpother_tl1(struct pt_regs *regs)
2406{
2407 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2408 die_if_kernel("TL1: FPU Other Exception", regs);
2409}
2410
2411void do_ill_tl1(struct pt_regs *regs)
2412{
2413 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2414 die_if_kernel("TL1: Illegal Instruction Exception", regs);
2415}
2416
2417void do_irq_tl1(struct pt_regs *regs)
2418{
2419 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2420 die_if_kernel("TL1: IRQ Exception", regs);
2421}
2422
2423void do_lddfmna_tl1(struct pt_regs *regs)
2424{
2425 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2426 die_if_kernel("TL1: LDDF Exception", regs);
2427}
2428
2429void do_stdfmna_tl1(struct pt_regs *regs)
2430{
2431 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2432 die_if_kernel("TL1: STDF Exception", regs);
2433}
2434
2435void do_paw(struct pt_regs *regs)
2436{
2437 die_if_kernel("TL0: Phys Watchpoint Exception", regs);
2438}
2439
2440void do_paw_tl1(struct pt_regs *regs)
2441{
2442 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2443 die_if_kernel("TL1: Phys Watchpoint Exception", regs);
2444}
2445
2446void do_vaw(struct pt_regs *regs)
2447{
2448 die_if_kernel("TL0: Virt Watchpoint Exception", regs);
2449}
2450
2451void do_vaw_tl1(struct pt_regs *regs)
2452{
2453 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2454 die_if_kernel("TL1: Virt Watchpoint Exception", regs);
2455}
2456
2457void do_tof_tl1(struct pt_regs *regs)
2458{
2459 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2460 die_if_kernel("TL1: Tag Overflow Exception", regs);
2461}
2462
2463void do_getpsr(struct pt_regs *regs)
2464{
2465 regs->u_regs[UREG_I0] = tstate_to_psr(regs->tstate);
2466 regs->tpc = regs->tnpc;
2467 regs->tnpc += 4;
2468 if (test_thread_flag(TIF_32BIT)) {
2469 regs->tpc &= 0xffffffff;
2470 regs->tnpc &= 0xffffffff;
2471 }
2472}
2473
56fb4df6
DM		 2474struct trap_per_cpu trap_block[NR_CPUS];
2475
2476/* This can get invoked before sched_init() so play it super safe
2477 * and use hard_smp_processor_id().
2478 */
72aff53f 2479void init_cur_cpu_trap(struct thread_info *t)
56fb4df6
DM		 2480{
2481 int cpu = hard_smp_processor_id();
2482 struct trap_per_cpu *p = &trap_block[cpu];
2483
72aff53f 2484 p->thread = t;
56fb4df6
DM		 2485 p->pgd_paddr = 0;
2486}
2487
1da177e4 2488extern void thread_info_offsets_are_bolixed_dave(void);
56fb4df6 2489extern void trap_per_cpu_offsets_are_bolixed_dave(void);
dcc1e8dd 2490extern void tsb_config_offsets_are_bolixed_dave(void);
1da177e4
LT		 2491
2492/* Only invoked on boot processor. */
2493void __init trap_init(void)
2494{
2495 /* Compile time sanity check. */
2496 if (TI_TASK != offsetof(struct thread_info, task) ||
2497 TI_FLAGS != offsetof(struct thread_info, flags) ||
2498 TI_CPU != offsetof(struct thread_info, cpu) ||
2499 TI_FPSAVED != offsetof(struct thread_info, fpsaved) ||
2500 TI_KSP != offsetof(struct thread_info, ksp) ||
2501 TI_FAULT_ADDR != offsetof(struct thread_info, fault_address) ||
2502 TI_KREGS != offsetof(struct thread_info, kregs) ||
2503 TI_UTRAPS != offsetof(struct thread_info, utraps) ||
2504 TI_EXEC_DOMAIN != offsetof(struct thread_info, exec_domain) ||
2505 TI_REG_WINDOW != offsetof(struct thread_info, reg_window) ||
2506 TI_RWIN_SPTRS != offsetof(struct thread_info, rwbuf_stkptrs) ||
2507 TI_GSR != offsetof(struct thread_info, gsr) ||
2508 TI_XFSR != offsetof(struct thread_info, xfsr) ||
2509 TI_USER_CNTD0 != offsetof(struct thread_info, user_cntd0) ||
2510 TI_USER_CNTD1 != offsetof(struct thread_info, user_cntd1) ||
2511 TI_KERN_CNTD0 != offsetof(struct thread_info, kernel_cntd0) ||
2512 TI_KERN_CNTD1 != offsetof(struct thread_info, kernel_cntd1) ||
2513 TI_PCR != offsetof(struct thread_info, pcr_reg) ||
1da177e4 2514 TI_PRE_COUNT != offsetof(struct thread_info, preempt_count) ||
db7d9a4e
DM		 2515 TI_NEW_CHILD != offsetof(struct thread_info, new_child) ||
2516 TI_SYS_NOERROR != offsetof(struct thread_info, syscall_noerror) ||
a3f99858
DM		 2517 TI_RESTART_BLOCK != offsetof(struct thread_info, restart_block) ||
2518 TI_KUNA_REGS != offsetof(struct thread_info, kern_una_regs) ||
2519 TI_KUNA_INSN != offsetof(struct thread_info, kern_una_insn) ||
1da177e4
LT		 2520 TI_FPREGS != offsetof(struct thread_info, fpregs) ||
2521 (TI_FPREGS & (64 - 1)))
2522 thread_info_offsets_are_bolixed_dave();
2523
56fb4df6 2524 if (TRAP_PER_CPU_THREAD != offsetof(struct trap_per_cpu, thread) ||
e088ad7c
DM		 2525 (TRAP_PER_CPU_PGD_PADDR !=
2526 offsetof(struct trap_per_cpu, pgd_paddr)) ||
2527 (TRAP_PER_CPU_CPU_MONDO_PA !=
2528 offsetof(struct trap_per_cpu, cpu_mondo_pa)) ||
2529 (TRAP_PER_CPU_DEV_MONDO_PA !=
2530 offsetof(struct trap_per_cpu, dev_mondo_pa)) ||
2531 (TRAP_PER_CPU_RESUM_MONDO_PA !=
2532 offsetof(struct trap_per_cpu, resum_mondo_pa)) ||
5b0c0572
DM		 2533 (TRAP_PER_CPU_RESUM_KBUF_PA !=
2534 offsetof(struct trap_per_cpu, resum_kernel_buf_pa)) ||
e088ad7c
DM		 2535 (TRAP_PER_CPU_NONRESUM_MONDO_PA !=
2536 offsetof(struct trap_per_cpu, nonresum_mondo_pa)) ||
5b0c0572
DM		 2537 (TRAP_PER_CPU_NONRESUM_KBUF_PA !=
2538 offsetof(struct trap_per_cpu, nonresum_kernel_buf_pa)) ||
e088ad7c 2539 (TRAP_PER_CPU_FAULT_INFO !=
1d2f1f90
DM		 2540 offsetof(struct trap_per_cpu, fault_info)) ||
2541 (TRAP_PER_CPU_CPU_MONDO_BLOCK_PA !=
2542 offsetof(struct trap_per_cpu, cpu_mondo_block_pa)) ||
2543 (TRAP_PER_CPU_CPU_LIST_PA !=
dcc1e8dd
DM		 2544 offsetof(struct trap_per_cpu, cpu_list_pa)) ||
2545 (TRAP_PER_CPU_TSB_HUGE !=
2546 offsetof(struct trap_per_cpu, tsb_huge)) ||
2547 (TRAP_PER_CPU_TSB_HUGE_TEMP !=
fd0504c3
DM		 2548 offsetof(struct trap_per_cpu, tsb_huge_temp)) ||
2549 (TRAP_PER_CPU_IRQ_WORKLIST !=
2550 offsetof(struct trap_per_cpu, irq_worklist)))
56fb4df6
DM		 2551 trap_per_cpu_offsets_are_bolixed_dave();
2552
dcc1e8dd
DM		 2553 if ((TSB_CONFIG_TSB !=
2554 offsetof(struct tsb_config, tsb)) ||
2555 (TSB_CONFIG_RSS_LIMIT !=
2556 offsetof(struct tsb_config, tsb_rss_limit)) ||
2557 (TSB_CONFIG_NENTRIES !=
2558 offsetof(struct tsb_config, tsb_nentries)) ||
2559 (TSB_CONFIG_REG_VAL !=
2560 offsetof(struct tsb_config, tsb_reg_val)) ||
2561 (TSB_CONFIG_MAP_VADDR !=
2562 offsetof(struct tsb_config, tsb_map_vaddr)) ||
2563 (TSB_CONFIG_MAP_PTE !=
2564 offsetof(struct tsb_config, tsb_map_pte)))
2565 tsb_config_offsets_are_bolixed_dave();
2566
1da177e4
LT		 2567 /* Attach to the address space of init_task. On SMP we
2568 * do this in smp.c:smp_callin for other cpus.
2569 */
2570 atomic_inc(&init_mm.mm_count);
2571 current->active_mm = &init_mm;
2572}
Ubuntu Jammy Linux kernel mirror