]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - arch/xtensa/kernel/entry.S
projects / mirror_ubuntu-zesty-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'upstream' of git://git.infradead.org/users/pcmoore/audit
[mirror_ubuntu-zesty-kernel.git] / arch / xtensa / kernel / entry.S
1 /*
2 * arch/xtensa/kernel/entry.S
3 *
4 * Low-level exception handling
5 *
6 * This file is subject to the terms and conditions of the GNU General Public
7 * License. See the file "COPYING" in the main directory of this archive
8 * for more details.
9 *
10 * Copyright (C) 2004 - 2008 by Tensilica Inc.
11 *
12 * Chris Zankel <chris@zankel.net>
13 *
14 */
15
16 #include <linux/linkage.h>
17 #include <asm/asm-offsets.h>
18 #include <asm/processor.h>
19 #include <asm/coprocessor.h>
20 #include <asm/thread_info.h>
21 #include <asm/uaccess.h>
22 #include <asm/unistd.h>
23 #include <asm/ptrace.h>
24 #include <asm/current.h>
25 #include <asm/pgtable.h>
26 #include <asm/page.h>
27 #include <asm/signal.h>
28 #include <asm/tlbflush.h>
29 #include <variant/tie-asm.h>
30
31 /* Unimplemented features. */
32
33 #undef KERNEL_STACK_OVERFLOW_CHECK
34
35 /* Not well tested.
36 *
37 * - fast_coprocessor
38 */
39
40 /*
41 * Macro to find first bit set in WINDOWBASE from the left + 1
42 *
43 * 100....0 -> 1
44 * 010....0 -> 2
45 * 000....1 -> WSBITS
46 */
47
48 .macro ffs_ws bit mask
49
50 #if XCHAL_HAVE_NSA
51 nsau \bit, \mask # 32-WSBITS ... 31 (32 iff 0)
52 addi \bit, \bit, WSBITS - 32 + 1 # uppest bit set -> return 1
53 #else
54 movi \bit, WSBITS
55 #if WSBITS > 16
56 _bltui \mask, 0x10000, 99f
57 addi \bit, \bit, -16
58 extui \mask, \mask, 16, 16
59 #endif
60 #if WSBITS > 8
61 99: _bltui \mask, 0x100, 99f
62 addi \bit, \bit, -8
63 srli \mask, \mask, 8
64 #endif
65 99: _bltui \mask, 0x10, 99f
66 addi \bit, \bit, -4
67 srli \mask, \mask, 4
68 99: _bltui \mask, 0x4, 99f
69 addi \bit, \bit, -2
70 srli \mask, \mask, 2
71 99: _bltui \mask, 0x2, 99f
72 addi \bit, \bit, -1
73 99:
74
75 #endif
76 .endm
77
78 /* ----------------- DEFAULT FIRST LEVEL EXCEPTION HANDLERS ----------------- */
79
80 /*
81 * First-level exception handler for user exceptions.
82 * Save some special registers, extra states and all registers in the AR
83 * register file that were in use in the user task, and jump to the common
84 * exception code.
85 * We save SAR (used to calculate WMASK), and WB and WS (we don't have to
86 * save them for kernel exceptions).
87 *
88 * Entry condition for user_exception:
89 *
90 * a0: trashed, original value saved on stack (PT_AREG0)
91 * a1: a1
92 * a2: new stack pointer, original value in depc
93 * a3: a3
94 * depc: a2, original value saved on stack (PT_DEPC)
95 * excsave1: dispatch table
96 *
97 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
98 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
99 *
100 * Entry condition for _user_exception:
101 *
102 * a0-a3 and depc have been saved to PT_AREG0...PT_AREG3 and PT_DEPC
103 * excsave has been restored, and
104 * stack pointer (a1) has been set.
105 *
106 * Note: _user_exception might be at an odd address. Don't use call0..call12
107 */
108
109 ENTRY(user_exception)
110
111 /* Save a1, a2, a3, and set SP. */
112
113 rsr a0, depc
114 s32i a1, a2, PT_AREG1
115 s32i a0, a2, PT_AREG2
116 s32i a3, a2, PT_AREG3
117 mov a1, a2
118
119 .globl _user_exception
120 _user_exception:
121
122 /* Save SAR and turn off single stepping */
123
124 movi a2, 0
125 rsr a3, sar
126 xsr a2, icountlevel
127 s32i a3, a1, PT_SAR
128 s32i a2, a1, PT_ICOUNTLEVEL
129
130 #if XCHAL_HAVE_THREADPTR
131 rur a2, threadptr
132 s32i a2, a1, PT_THREADPTR
133 #endif
134
135 /* Rotate ws so that the current windowbase is at bit0. */
136 /* Assume ws = xxwww1yyyy. Rotate ws right, so that a2 = yyyyxxwww1 */
137
138 rsr a2, windowbase
139 rsr a3, windowstart
140 ssr a2
141 s32i a2, a1, PT_WINDOWBASE
142 s32i a3, a1, PT_WINDOWSTART
143 slli a2, a3, 32-WSBITS
144 src a2, a3, a2
145 srli a2, a2, 32-WSBITS
146 s32i a2, a1, PT_WMASK # needed for restoring registers
147
148 /* Save only live registers. */
149
150 _bbsi.l a2, 1, 1f
151 s32i a4, a1, PT_AREG4
152 s32i a5, a1, PT_AREG5
153 s32i a6, a1, PT_AREG6
154 s32i a7, a1, PT_AREG7
155 _bbsi.l a2, 2, 1f
156 s32i a8, a1, PT_AREG8
157 s32i a9, a1, PT_AREG9
158 s32i a10, a1, PT_AREG10
159 s32i a11, a1, PT_AREG11
160 _bbsi.l a2, 3, 1f
161 s32i a12, a1, PT_AREG12
162 s32i a13, a1, PT_AREG13
163 s32i a14, a1, PT_AREG14
164 s32i a15, a1, PT_AREG15
165 _bnei a2, 1, 1f # only one valid frame?
166
167 /* Only one valid frame, skip saving regs. */
168
169 j 2f
170
171 /* Save the remaining registers.
172 * We have to save all registers up to the first '1' from
173 * the right, except the current frame (bit 0).
174 * Assume a2 is: 001001000110001
175 * All register frames starting from the top field to the marked '1'
176 * must be saved.
177 */
178
179 1: addi a3, a2, -1 # eliminate '1' in bit 0: yyyyxxww0
180 neg a3, a3 # yyyyxxww0 -> YYYYXXWW1+1
181 and a3, a3, a2 # max. only one bit is set
182
183 /* Find number of frames to save */
184
185 ffs_ws a0, a3 # number of frames to the '1' from left
186
187 /* Store information into WMASK:
188 * bits 0..3: xxx1 masked lower 4 bits of the rotated windowstart,
189 * bits 4...: number of valid 4-register frames
190 */
191
192 slli a3, a0, 4 # number of frames to save in bits 8..4
193 extui a2, a2, 0, 4 # mask for the first 16 registers
194 or a2, a3, a2
195 s32i a2, a1, PT_WMASK # needed when we restore the reg-file
196
197 /* Save 4 registers at a time */
198
199 1: rotw -1
200 s32i a0, a5, PT_AREG_END - 16
201 s32i a1, a5, PT_AREG_END - 12
202 s32i a2, a5, PT_AREG_END - 8
203 s32i a3, a5, PT_AREG_END - 4
204 addi a0, a4, -1
205 addi a1, a5, -16
206 _bnez a0, 1b
207
208 /* WINDOWBASE still in SAR! */
209
210 rsr a2, sar # original WINDOWBASE
211 movi a3, 1
212 ssl a2
213 sll a3, a3
214 wsr a3, windowstart # set corresponding WINDOWSTART bit
215 wsr a2, windowbase # and WINDOWSTART
216 rsync
217
218 /* We are back to the original stack pointer (a1) */
219
220 2: /* Now, jump to the common exception handler. */
221
222 j common_exception
223
224 ENDPROC(user_exception)
225
226 /*
227 * First-level exit handler for kernel exceptions
228 * Save special registers and the live window frame.
229 * Note: Even though we changes the stack pointer, we don't have to do a
230 * MOVSP here, as we do that when we return from the exception.
231 * (See comment in the kernel exception exit code)
232 *
233 * Entry condition for kernel_exception:
234 *
235 * a0: trashed, original value saved on stack (PT_AREG0)
236 * a1: a1
237 * a2: new stack pointer, original in DEPC
238 * a3: a3
239 * depc: a2, original value saved on stack (PT_DEPC)
240 * excsave_1: dispatch table
241 *
242 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
243 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
244 *
245 * Entry condition for _kernel_exception:
246 *
247 * a0-a3 and depc have been saved to PT_AREG0...PT_AREG3 and PT_DEPC
248 * excsave has been restored, and
249 * stack pointer (a1) has been set.
250 *
251 * Note: _kernel_exception might be at an odd address. Don't use call0..call12
252 */
253
254 ENTRY(kernel_exception)
255
256 /* Save a1, a2, a3, and set SP. */
257
258 rsr a0, depc # get a2
259 s32i a1, a2, PT_AREG1
260 s32i a0, a2, PT_AREG2
261 s32i a3, a2, PT_AREG3
262 mov a1, a2
263
264 .globl _kernel_exception
265 _kernel_exception:
266
267 /* Save SAR and turn off single stepping */
268
269 movi a2, 0
270 rsr a3, sar
271 xsr a2, icountlevel
272 s32i a3, a1, PT_SAR
273 s32i a2, a1, PT_ICOUNTLEVEL
274
275 /* Rotate ws so that the current windowbase is at bit0. */
276 /* Assume ws = xxwww1yyyy. Rotate ws right, so that a2 = yyyyxxwww1 */
277
278 rsr a2, windowbase # don't need to save these, we only
279 rsr a3, windowstart # need shifted windowstart: windowmask
280 ssr a2
281 slli a2, a3, 32-WSBITS
282 src a2, a3, a2
283 srli a2, a2, 32-WSBITS
284 s32i a2, a1, PT_WMASK # needed for kernel_exception_exit
285
286 /* Save only the live window-frame */
287
288 _bbsi.l a2, 1, 1f
289 s32i a4, a1, PT_AREG4
290 s32i a5, a1, PT_AREG5
291 s32i a6, a1, PT_AREG6
292 s32i a7, a1, PT_AREG7
293 _bbsi.l a2, 2, 1f
294 s32i a8, a1, PT_AREG8
295 s32i a9, a1, PT_AREG9
296 s32i a10, a1, PT_AREG10
297 s32i a11, a1, PT_AREG11
298 _bbsi.l a2, 3, 1f
299 s32i a12, a1, PT_AREG12
300 s32i a13, a1, PT_AREG13
301 s32i a14, a1, PT_AREG14
302 s32i a15, a1, PT_AREG15
303
304 1:
305
306 #ifdef KERNEL_STACK_OVERFLOW_CHECK
307
308 /* Stack overflow check, for debugging */
309 extui a2, a1, TASK_SIZE_BITS,XX
310 movi a3, SIZE??
311 _bge a2, a3, out_of_stack_panic
312
313 #endif
314
315 /*
316 * This is the common exception handler.
317 * We get here from the user exception handler or simply by falling through
318 * from the kernel exception handler.
319 * Save the remaining special registers, switch to kernel mode, and jump
320 * to the second-level exception handler.
321 *
322 */
323
324 common_exception:
325
326 /* Save some registers, disable loops and clear the syscall flag. */
327
328 rsr a2, debugcause
329 rsr a3, epc1
330 s32i a2, a1, PT_DEBUGCAUSE
331 s32i a3, a1, PT_PC
332
333 movi a2, -1
334 rsr a3, excvaddr
335 s32i a2, a1, PT_SYSCALL
336 movi a2, 0
337 s32i a3, a1, PT_EXCVADDR
338 xsr a2, lcount
339 s32i a2, a1, PT_LCOUNT
340
341 /* It is now save to restore the EXC_TABLE_FIXUP variable. */
342
343 rsr a0, exccause
344 movi a3, 0
345 rsr a2, excsave1
346 s32i a0, a1, PT_EXCCAUSE
347 s32i a3, a2, EXC_TABLE_FIXUP
348
349 /* All unrecoverable states are saved on stack, now, and a1 is valid,
350 * so we can allow exceptions and interrupts (*) again.
351 * Set PS(EXCM = 0, UM = 0, RING = 0, OWB = 0, WOE = 1, INTLEVEL = X)
352 *
353 * (*) We only allow interrupts if they were previously enabled and
354 * we're not handling an IRQ
355 */
356
357 rsr a3, ps
358 addi a0, a0, -EXCCAUSE_LEVEL1_INTERRUPT
359 movi a2, LOCKLEVEL
360 extui a3, a3, PS_INTLEVEL_SHIFT, PS_INTLEVEL_WIDTH
361 # a3 = PS.INTLEVEL
362 moveqz a3, a2, a0 # a3 = LOCKLEVEL iff interrupt
363 movi a2, 1 << PS_WOE_BIT
364 or a3, a3, a2
365 rsr a0, exccause
366 xsr a3, ps
367
368 s32i a3, a1, PT_PS # save ps
369
370 /* Save lbeg, lend */
371
372 rsr a2, lbeg
373 rsr a3, lend
374 s32i a2, a1, PT_LBEG
375 s32i a3, a1, PT_LEND
376
377 /* Save SCOMPARE1 */
378
379 #if XCHAL_HAVE_S32C1I
380 rsr a2, scompare1
381 s32i a2, a1, PT_SCOMPARE1
382 #endif
383
384 /* Save optional registers. */
385
386 save_xtregs_opt a1 a2 a4 a5 a6 a7 PT_XTREGS_OPT
387
388 #ifdef CONFIG_TRACE_IRQFLAGS
389 l32i a4, a1, PT_DEPC
390 /* Double exception means we came here with an exception
391 * while PS.EXCM was set, i.e. interrupts disabled.
392 */
393 bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
394 l32i a4, a1, PT_EXCCAUSE
395 bnei a4, EXCCAUSE_LEVEL1_INTERRUPT, 1f
396 /* We came here with an interrupt means interrupts were enabled
397 * and we've just disabled them.
398 */
399 movi a4, trace_hardirqs_off
400 callx4 a4
401 1:
402 #endif
403
404 /* Go to second-level dispatcher. Set up parameters to pass to the
405 * exception handler and call the exception handler.
406 */
407
408 rsr a4, excsave1
409 mov a6, a1 # pass stack frame
410 mov a7, a0 # pass EXCCAUSE
411 addx4 a4, a0, a4
412 l32i a4, a4, EXC_TABLE_DEFAULT # load handler
413
414 /* Call the second-level handler */
415
416 callx4 a4
417
418 /* Jump here for exception exit */
419 .global common_exception_return
420 common_exception_return:
421
422 1:
423 rsil a2, LOCKLEVEL
424
425 /* Jump if we are returning from kernel exceptions. */
426
427 l32i a3, a1, PT_PS
428 GET_THREAD_INFO(a2, a1)
429 l32i a4, a2, TI_FLAGS
430 _bbci.l a3, PS_UM_BIT, 6f
431
432 /* Specific to a user exception exit:
433 * We need to check some flags for signal handling and rescheduling,
434 * and have to restore WB and WS, extra states, and all registers
435 * in the register file that were in use in the user task.
436 * Note that we don't disable interrupts here.
437 */
438
439 _bbsi.l a4, TIF_NEED_RESCHED, 3f
440 _bbsi.l a4, TIF_NOTIFY_RESUME, 2f
441 _bbci.l a4, TIF_SIGPENDING, 5f
442
443 2: l32i a4, a1, PT_DEPC
444 bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 4f
445
446 /* Call do_signal() */
447
448 rsil a2, 0
449 movi a4, do_notify_resume # int do_notify_resume(struct pt_regs*)
450 mov a6, a1
451 callx4 a4
452 j 1b
453
454 3: /* Reschedule */
455
456 rsil a2, 0
457 movi a4, schedule # void schedule (void)
458 callx4 a4
459 j 1b
460
461 #ifdef CONFIG_PREEMPT
462 6:
463 _bbci.l a4, TIF_NEED_RESCHED, 4f
464
465 /* Check current_thread_info->preempt_count */
466
467 l32i a4, a2, TI_PRE_COUNT
468 bnez a4, 4f
469 movi a4, preempt_schedule_irq
470 callx4 a4
471 j 1b
472 #endif
473
474 5:
475 #ifdef CONFIG_DEBUG_TLB_SANITY
476 l32i a4, a1, PT_DEPC
477 bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 4f
478 movi a4, check_tlb_sanity
479 callx4 a4
480 #endif
481 6:
482 4:
483 #ifdef CONFIG_TRACE_IRQFLAGS
484 l32i a4, a1, PT_DEPC
485 /* Double exception means we came here with an exception
486 * while PS.EXCM was set, i.e. interrupts disabled.
487 */
488 bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
489 l32i a4, a1, PT_EXCCAUSE
490 bnei a4, EXCCAUSE_LEVEL1_INTERRUPT, 1f
491 /* We came here with an interrupt means interrupts were enabled
492 * and we'll reenable them on return.
493 */
494 movi a4, trace_hardirqs_on
495 callx4 a4
496 1:
497 #endif
498 /* Restore optional registers. */
499
500 load_xtregs_opt a1 a2 a4 a5 a6 a7 PT_XTREGS_OPT
501
502 /* Restore SCOMPARE1 */
503
504 #if XCHAL_HAVE_S32C1I
505 l32i a2, a1, PT_SCOMPARE1
506 wsr a2, scompare1
507 #endif
508 wsr a3, ps /* disable interrupts */
509
510 _bbci.l a3, PS_UM_BIT, kernel_exception_exit
511
512 user_exception_exit:
513
514 /* Restore the state of the task and return from the exception. */
515
516 /* Switch to the user thread WINDOWBASE. Save SP temporarily in DEPC */
517
518 l32i a2, a1, PT_WINDOWBASE
519 l32i a3, a1, PT_WINDOWSTART
520 wsr a1, depc # use DEPC as temp storage
521 wsr a3, windowstart # restore WINDOWSTART
522 ssr a2 # preserve user's WB in the SAR
523 wsr a2, windowbase # switch to user's saved WB
524 rsync
525 rsr a1, depc # restore stack pointer
526 l32i a2, a1, PT_WMASK # register frames saved (in bits 4...9)
527 rotw -1 # we restore a4..a7
528 _bltui a6, 16, 1f # only have to restore current window?
529
530 /* The working registers are a0 and a3. We are restoring to
531 * a4..a7. Be careful not to destroy what we have just restored.
532 * Note: wmask has the format YYYYM:
533 * Y: number of registers saved in groups of 4
534 * M: 4 bit mask of first 16 registers
535 */
536
537 mov a2, a6
538 mov a3, a5
539
540 2: rotw -1 # a0..a3 become a4..a7
541 addi a3, a7, -4*4 # next iteration
542 addi a2, a6, -16 # decrementing Y in WMASK
543 l32i a4, a3, PT_AREG_END + 0
544 l32i a5, a3, PT_AREG_END + 4
545 l32i a6, a3, PT_AREG_END + 8
546 l32i a7, a3, PT_AREG_END + 12
547 _bgeui a2, 16, 2b
548
549 /* Clear unrestored registers (don't leak anything to user-land */
550
551 1: rsr a0, windowbase
552 rsr a3, sar
553 sub a3, a0, a3
554 beqz a3, 2f
555 extui a3, a3, 0, WBBITS
556
557 1: rotw -1
558 addi a3, a7, -1
559 movi a4, 0
560 movi a5, 0
561 movi a6, 0
562 movi a7, 0
563 bgei a3, 1, 1b
564
565 /* We are back were we were when we started.
566 * Note: a2 still contains WMASK (if we've returned to the original
567 * frame where we had loaded a2), or at least the lower 4 bits
568 * (if we have restored WSBITS-1 frames).
569 */
570
571 #if XCHAL_HAVE_THREADPTR
572 l32i a3, a1, PT_THREADPTR
573 wur a3, threadptr
574 #endif
575
576 2: j common_exception_exit
577
578 /* This is the kernel exception exit.
579 * We avoided to do a MOVSP when we entered the exception, but we
580 * have to do it here.
581 */
582
583 kernel_exception_exit:
584
585 /* Check if we have to do a movsp.
586 *
587 * We only have to do a movsp if the previous window-frame has
588 * been spilled to the *temporary* exception stack instead of the
589 * task's stack. This is the case if the corresponding bit in
590 * WINDOWSTART for the previous window-frame was set before
591 * (not spilled) but is zero now (spilled).
592 * If this bit is zero, all other bits except the one for the
593 * current window frame are also zero. So, we can use a simple test:
594 * 'and' WINDOWSTART and WINDOWSTART-1:
595 *
596 * (XXXXXX1[0]* - 1) AND XXXXXX1[0]* = XXXXXX0[0]*
597 *
598 * The result is zero only if one bit was set.
599 *
600 * (Note: We might have gone through several task switches before
601 * we come back to the current task, so WINDOWBASE might be
602 * different from the time the exception occurred.)
603 */
604
605 /* Test WINDOWSTART before and after the exception.
606 * We actually have WMASK, so we only have to test if it is 1 or not.
607 */
608
609 l32i a2, a1, PT_WMASK
610 _beqi a2, 1, common_exception_exit # Spilled before exception,jump
611
612 /* Test WINDOWSTART now. If spilled, do the movsp */
613
614 rsr a3, windowstart
615 addi a0, a3, -1
616 and a3, a3, a0
617 _bnez a3, common_exception_exit
618
619 /* Do a movsp (we returned from a call4, so we have at least a0..a7) */
620
621 addi a0, a1, -16
622 l32i a3, a0, 0
623 l32i a4, a0, 4
624 s32i a3, a1, PT_SIZE+0
625 s32i a4, a1, PT_SIZE+4
626 l32i a3, a0, 8
627 l32i a4, a0, 12
628 s32i a3, a1, PT_SIZE+8
629 s32i a4, a1, PT_SIZE+12
630
631 /* Common exception exit.
632 * We restore the special register and the current window frame, and
633 * return from the exception.
634 *
635 * Note: We expect a2 to hold PT_WMASK
636 */
637
638 common_exception_exit:
639
640 /* Restore address registers. */
641
642 _bbsi.l a2, 1, 1f
643 l32i a4, a1, PT_AREG4
644 l32i a5, a1, PT_AREG5
645 l32i a6, a1, PT_AREG6
646 l32i a7, a1, PT_AREG7
647 _bbsi.l a2, 2, 1f
648 l32i a8, a1, PT_AREG8
649 l32i a9, a1, PT_AREG9
650 l32i a10, a1, PT_AREG10
651 l32i a11, a1, PT_AREG11
652 _bbsi.l a2, 3, 1f
653 l32i a12, a1, PT_AREG12
654 l32i a13, a1, PT_AREG13
655 l32i a14, a1, PT_AREG14
656 l32i a15, a1, PT_AREG15
657
658 /* Restore PC, SAR */
659
660 1: l32i a2, a1, PT_PC
661 l32i a3, a1, PT_SAR
662 wsr a2, epc1
663 wsr a3, sar
664
665 /* Restore LBEG, LEND, LCOUNT */
666
667 l32i a2, a1, PT_LBEG
668 l32i a3, a1, PT_LEND
669 wsr a2, lbeg
670 l32i a2, a1, PT_LCOUNT
671 wsr a3, lend
672 wsr a2, lcount
673
674 /* We control single stepping through the ICOUNTLEVEL register. */
675
676 l32i a2, a1, PT_ICOUNTLEVEL
677 movi a3, -2
678 wsr a2, icountlevel
679 wsr a3, icount
680
681 /* Check if it was double exception. */
682
683 l32i a0, a1, PT_DEPC
684 l32i a3, a1, PT_AREG3
685 l32i a2, a1, PT_AREG2
686 _bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
687
688 /* Restore a0...a3 and return */
689
690 l32i a0, a1, PT_AREG0
691 l32i a1, a1, PT_AREG1
692 rfe
693
694 1: wsr a0, depc
695 l32i a0, a1, PT_AREG0
696 l32i a1, a1, PT_AREG1
697 rfde
698
699 ENDPROC(kernel_exception)
700
701 /*
702 * Debug exception handler.
703 *
704 * Currently, we don't support KGDB, so only user application can be debugged.
705 *
706 * When we get here, a0 is trashed and saved to excsave[debuglevel]
707 */
708
709 ENTRY(debug_exception)
710
711 rsr a0, SREG_EPS + XCHAL_DEBUGLEVEL
712 bbsi.l a0, PS_EXCM_BIT, 1f # exception mode
713
714 /* Set EPC1 and EXCCAUSE */
715
716 wsr a2, depc # save a2 temporarily
717 rsr a2, SREG_EPC + XCHAL_DEBUGLEVEL
718 wsr a2, epc1
719
720 movi a2, EXCCAUSE_MAPPED_DEBUG
721 wsr a2, exccause
722
723 /* Restore PS to the value before the debug exc but with PS.EXCM set.*/
724
725 movi a2, 1 << PS_EXCM_BIT
726 or a2, a0, a2
727 movi a0, debug_exception # restore a3, debug jump vector
728 wsr a2, ps
729 xsr a0, SREG_EXCSAVE + XCHAL_DEBUGLEVEL
730
731 /* Switch to kernel/user stack, restore jump vector, and save a0 */
732
733 bbsi.l a2, PS_UM_BIT, 2f # jump if user mode
734
735 addi a2, a1, -16-PT_SIZE # assume kernel stack
736 s32i a0, a2, PT_AREG0
737 movi a0, 0
738 s32i a1, a2, PT_AREG1
739 s32i a0, a2, PT_DEPC # mark it as a regular exception
740 xsr a0, depc
741 s32i a3, a2, PT_AREG3
742 s32i a0, a2, PT_AREG2
743 mov a1, a2
744 j _kernel_exception
745
746 2: rsr a2, excsave1
747 l32i a2, a2, EXC_TABLE_KSTK # load kernel stack pointer
748 s32i a0, a2, PT_AREG0
749 movi a0, 0
750 s32i a1, a2, PT_AREG1
751 s32i a0, a2, PT_DEPC
752 xsr a0, depc
753 s32i a3, a2, PT_AREG3
754 s32i a0, a2, PT_AREG2
755 mov a1, a2
756 j _user_exception
757
758 /* Debug exception while in exception mode. */
759 1: j 1b // FIXME!!
760
761 ENDPROC(debug_exception)
762
763 /*
764 * We get here in case of an unrecoverable exception.
765 * The only thing we can do is to be nice and print a panic message.
766 * We only produce a single stack frame for panic, so ???
767 *
768 *
769 * Entry conditions:
770 *
771 * - a0 contains the caller address; original value saved in excsave1.
772 * - the original a0 contains a valid return address (backtrace) or 0.
773 * - a2 contains a valid stackpointer
774 *
775 * Notes:
776 *
777 * - If the stack pointer could be invalid, the caller has to setup a
778 * dummy stack pointer (e.g. the stack of the init_task)
779 *
780 * - If the return address could be invalid, the caller has to set it
781 * to 0, so the backtrace would stop.
782 *
783 */
784 .align 4
785 unrecoverable_text:
786 .ascii "Unrecoverable error in exception handler\0"
787
788 ENTRY(unrecoverable_exception)
789
790 movi a0, 1
791 movi a1, 0
792
793 wsr a0, windowstart
794 wsr a1, windowbase
795 rsync
796
797 movi a1, (1 << PS_WOE_BIT) | LOCKLEVEL
798 wsr a1, ps
799 rsync
800
801 movi a1, init_task
802 movi a0, 0
803 addi a1, a1, PT_REGS_OFFSET
804
805 movi a4, panic
806 movi a6, unrecoverable_text
807
808 callx4 a4
809
810 1: j 1b
811
812 ENDPROC(unrecoverable_exception)
813
814 /* -------------------------- FAST EXCEPTION HANDLERS ----------------------- */
815
816 /*
817 * Fast-handler for alloca exceptions
818 *
819 * The ALLOCA handler is entered when user code executes the MOVSP
820 * instruction and the caller's frame is not in the register file.
821 *
822 * This algorithm was taken from the Ross Morley's RTOS Porting Layer:
823 *
824 * /home/ross/rtos/porting/XtensaRTOS-PortingLayer-20090507/xtensa_vectors.S
825 *
826 * It leverages the existing window spill/fill routines and their support for
827 * double exceptions. The 'movsp' instruction will only cause an exception if
828 * the next window needs to be loaded. In fact this ALLOCA exception may be
829 * replaced at some point by changing the hardware to do a underflow exception
830 * of the proper size instead.
831 *
832 * This algorithm simply backs out the register changes started by the user
833 * excpetion handler, makes it appear that we have started a window underflow
834 * by rotating the window back and then setting the old window base (OWB) in
835 * the 'ps' register with the rolled back window base. The 'movsp' instruction
836 * will be re-executed and this time since the next window frames is in the
837 * active AR registers it won't cause an exception.
838 *
839 * If the WindowUnderflow code gets a TLB miss the page will get mapped
840 * the the partial windeowUnderflow will be handeled in the double exception
841 * handler.
842 *
843 * Entry condition:
844 *
845 * a0: trashed, original value saved on stack (PT_AREG0)
846 * a1: a1
847 * a2: new stack pointer, original in DEPC
848 * a3: a3
849 * depc: a2, original value saved on stack (PT_DEPC)
850 * excsave_1: dispatch table
851 *
852 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
853 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
854 */
855
856 ENTRY(fast_alloca)
857 rsr a0, windowbase
858 rotw -1
859 rsr a2, ps
860 extui a3, a2, PS_OWB_SHIFT, PS_OWB_WIDTH
861 xor a3, a3, a4
862 l32i a4, a6, PT_AREG0
863 l32i a1, a6, PT_DEPC
864 rsr a6, depc
865 wsr a1, depc
866 slli a3, a3, PS_OWB_SHIFT
867 xor a2, a2, a3
868 wsr a2, ps
869 rsync
870
871 _bbci.l a4, 31, 4f
872 rotw -1
873 _bbci.l a8, 30, 8f
874 rotw -1
875 j _WindowUnderflow12
876 8: j _WindowUnderflow8
877 4: j _WindowUnderflow4
878 ENDPROC(fast_alloca)
879
880 /*
881 * fast system calls.
882 *
883 * WARNING: The kernel doesn't save the entire user context before
884 * handling a fast system call. These functions are small and short,
885 * usually offering some functionality not available to user tasks.
886 *
887 * BE CAREFUL TO PRESERVE THE USER'S CONTEXT.
888 *
889 * Entry condition:
890 *
891 * a0: trashed, original value saved on stack (PT_AREG0)
892 * a1: a1
893 * a2: new stack pointer, original in DEPC
894 * a3: a3
895 * depc: a2, original value saved on stack (PT_DEPC)
896 * excsave_1: dispatch table
897 */
898
899 ENTRY(fast_syscall_kernel)
900
901 /* Skip syscall. */
902
903 rsr a0, epc1
904 addi a0, a0, 3
905 wsr a0, epc1
906
907 l32i a0, a2, PT_DEPC
908 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_syscall_unrecoverable
909
910 rsr a0, depc # get syscall-nr
911 _beqz a0, fast_syscall_spill_registers
912 _beqi a0, __NR_xtensa, fast_syscall_xtensa
913
914 j kernel_exception
915
916 ENDPROC(fast_syscall_kernel)
917
918 ENTRY(fast_syscall_user)
919
920 /* Skip syscall. */
921
922 rsr a0, epc1
923 addi a0, a0, 3
924 wsr a0, epc1
925
926 l32i a0, a2, PT_DEPC
927 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_syscall_unrecoverable
928
929 rsr a0, depc # get syscall-nr
930 _beqz a0, fast_syscall_spill_registers
931 _beqi a0, __NR_xtensa, fast_syscall_xtensa
932
933 j user_exception
934
935 ENDPROC(fast_syscall_user)
936
937 ENTRY(fast_syscall_unrecoverable)
938
939 /* Restore all states. */
940
941 l32i a0, a2, PT_AREG0 # restore a0
942 xsr a2, depc # restore a2, depc
943
944 wsr a0, excsave1
945 movi a0, unrecoverable_exception
946 callx0 a0
947
948 ENDPROC(fast_syscall_unrecoverable)
949
950 /*
951 * sysxtensa syscall handler
952 *
953 * int sysxtensa (SYS_XTENSA_ATOMIC_SET, ptr, val, unused);
954 * int sysxtensa (SYS_XTENSA_ATOMIC_ADD, ptr, val, unused);
955 * int sysxtensa (SYS_XTENSA_ATOMIC_EXG_ADD, ptr, val, unused);
956 * int sysxtensa (SYS_XTENSA_ATOMIC_CMP_SWP, ptr, oldval, newval);
957 * a2 a6 a3 a4 a5
958 *
959 * Entry condition:
960 *
961 * a0: a2 (syscall-nr), original value saved on stack (PT_AREG0)
962 * a1: a1
963 * a2: new stack pointer, original in a0 and DEPC
964 * a3: a3
965 * a4..a15: unchanged
966 * depc: a2, original value saved on stack (PT_DEPC)
967 * excsave_1: dispatch table
968 *
969 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
970 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
971 *
972 * Note: we don't have to save a2; a2 holds the return value
973 *
974 * We use the two macros TRY and CATCH:
975 *
976 * TRY adds an entry to the __ex_table fixup table for the immediately
977 * following instruction.
978 *
979 * CATCH catches any exception that occurred at one of the preceding TRY
980 * statements and continues from there
981 *
982 * Usage TRY l32i a0, a1, 0
983 * <other code>
984 * done: rfe
985 * CATCH <set return code>
986 * j done
987 */
988
989 #ifdef CONFIG_FAST_SYSCALL_XTENSA
990
991 #define TRY \
992 .section __ex_table, "a"; \
993 .word 66f, 67f; \
994 .text; \
995 66:
996
997 #define CATCH \
998 67:
999
1000 ENTRY(fast_syscall_xtensa)
1001
1002 s32i a7, a2, PT_AREG7 # we need an additional register
1003 movi a7, 4 # sizeof(unsigned int)
1004 access_ok a3, a7, a0, a2, .Leac # a0: scratch reg, a2: sp
1005
1006 _bgeui a6, SYS_XTENSA_COUNT, .Lill
1007 _bnei a6, SYS_XTENSA_ATOMIC_CMP_SWP, .Lnswp
1008
1009 /* Fall through for ATOMIC_CMP_SWP. */
1010
1011 .Lswp: /* Atomic compare and swap */
1012
1013 TRY l32i a0, a3, 0 # read old value
1014 bne a0, a4, 1f # same as old value? jump
1015 TRY s32i a5, a3, 0 # different, modify value
1016 l32i a7, a2, PT_AREG7 # restore a7
1017 l32i a0, a2, PT_AREG0 # restore a0
1018 movi a2, 1 # and return 1
1019 rfe
1020
1021 1: l32i a7, a2, PT_AREG7 # restore a7
1022 l32i a0, a2, PT_AREG0 # restore a0
1023 movi a2, 0 # return 0 (note that we cannot set
1024 rfe
1025
1026 .Lnswp: /* Atomic set, add, and exg_add. */
1027
1028 TRY l32i a7, a3, 0 # orig
1029 addi a6, a6, -SYS_XTENSA_ATOMIC_SET
1030 add a0, a4, a7 # + arg
1031 moveqz a0, a4, a6 # set
1032 addi a6, a6, SYS_XTENSA_ATOMIC_SET
1033 TRY s32i a0, a3, 0 # write new value
1034
1035 mov a0, a2
1036 mov a2, a7
1037 l32i a7, a0, PT_AREG7 # restore a7
1038 l32i a0, a0, PT_AREG0 # restore a0
1039 rfe
1040
1041 CATCH
1042 .Leac: l32i a7, a2, PT_AREG7 # restore a7
1043 l32i a0, a2, PT_AREG0 # restore a0
1044 movi a2, -EFAULT
1045 rfe
1046
1047 .Lill: l32i a7, a2, PT_AREG7 # restore a7
1048 l32i a0, a2, PT_AREG0 # restore a0
1049 movi a2, -EINVAL
1050 rfe
1051
1052 ENDPROC(fast_syscall_xtensa)
1053
1054 #else /* CONFIG_FAST_SYSCALL_XTENSA */
1055
1056 ENTRY(fast_syscall_xtensa)
1057
1058 l32i a0, a2, PT_AREG0 # restore a0
1059 movi a2, -ENOSYS
1060 rfe
1061
1062 ENDPROC(fast_syscall_xtensa)
1063
1064 #endif /* CONFIG_FAST_SYSCALL_XTENSA */
1065
1066
1067 /* fast_syscall_spill_registers.
1068 *
1069 * Entry condition:
1070 *
1071 * a0: trashed, original value saved on stack (PT_AREG0)
1072 * a1: a1
1073 * a2: new stack pointer, original in DEPC
1074 * a3: a3
1075 * depc: a2, original value saved on stack (PT_DEPC)
1076 * excsave_1: dispatch table
1077 *
1078 * Note: We assume the stack pointer is EXC_TABLE_KSTK in the fixup handler.
1079 */
1080
1081 #ifdef CONFIG_FAST_SYSCALL_SPILL_REGISTERS
1082
1083 ENTRY(fast_syscall_spill_registers)
1084
1085 /* Register a FIXUP handler (pass current wb as a parameter) */
1086
1087 xsr a3, excsave1
1088 movi a0, fast_syscall_spill_registers_fixup
1089 s32i a0, a3, EXC_TABLE_FIXUP
1090 rsr a0, windowbase
1091 s32i a0, a3, EXC_TABLE_PARAM
1092 xsr a3, excsave1 # restore a3 and excsave_1
1093
1094 /* Save a3, a4 and SAR on stack. */
1095
1096 rsr a0, sar
1097 s32i a3, a2, PT_AREG3
1098 s32i a0, a2, PT_SAR
1099
1100 /* The spill routine might clobber a4, a7, a8, a11, a12, and a15. */
1101
1102 s32i a4, a2, PT_AREG4
1103 s32i a7, a2, PT_AREG7
1104 s32i a8, a2, PT_AREG8
1105 s32i a11, a2, PT_AREG11
1106 s32i a12, a2, PT_AREG12
1107 s32i a15, a2, PT_AREG15
1108
1109 /*
1110 * Rotate ws so that the current windowbase is at bit 0.
1111 * Assume ws = xxxwww1yy (www1 current window frame).
1112 * Rotate ws right so that a4 = yyxxxwww1.
1113 */
1114
1115 rsr a0, windowbase
1116 rsr a3, windowstart # a3 = xxxwww1yy
1117 ssr a0 # holds WB
1118 slli a0, a3, WSBITS
1119 or a3, a3, a0 # a3 = xxxwww1yyxxxwww1yy
1120 srl a3, a3 # a3 = 00xxxwww1yyxxxwww1
1121
1122 /* We are done if there are no more than the current register frame. */
1123
1124 extui a3, a3, 1, WSBITS-1 # a3 = 0yyxxxwww
1125 movi a0, (1 << (WSBITS-1))
1126 _beqz a3, .Lnospill # only one active frame? jump
1127
1128 /* We want 1 at the top, so that we return to the current windowbase */
1129
1130 or a3, a3, a0 # 1yyxxxwww
1131
1132 /* Skip empty frames - get 'oldest' WINDOWSTART-bit. */
1133
1134 wsr a3, windowstart # save shifted windowstart
1135 neg a0, a3
1136 and a3, a0, a3 # first bit set from right: 000010000
1137
1138 ffs_ws a0, a3 # a0: shifts to skip empty frames
1139 movi a3, WSBITS
1140 sub a0, a3, a0 # WSBITS-a0:number of 0-bits from right
1141 ssr a0 # save in SAR for later.
1142
1143 rsr a3, windowbase
1144 add a3, a3, a0
1145 wsr a3, windowbase
1146 rsync
1147
1148 rsr a3, windowstart
1149 srl a3, a3 # shift windowstart
1150
1151 /* WB is now just one frame below the oldest frame in the register
1152 window. WS is shifted so the oldest frame is in bit 0, thus, WB
1153 and WS differ by one 4-register frame. */
1154
1155 /* Save frames. Depending what call was used (call4, call8, call12),
1156 * we have to save 4,8. or 12 registers.
1157 */
1158
1159
1160 .Lloop: _bbsi.l a3, 1, .Lc4
1161 _bbci.l a3, 2, .Lc12
1162
1163 .Lc8: s32e a4, a13, -16
1164 l32e a4, a5, -12
1165 s32e a8, a4, -32
1166 s32e a5, a13, -12
1167 s32e a6, a13, -8
1168 s32e a7, a13, -4
1169 s32e a9, a4, -28
1170 s32e a10, a4, -24
1171 s32e a11, a4, -20
1172 srli a11, a3, 2 # shift windowbase by 2
1173 rotw 2
1174 _bnei a3, 1, .Lloop
1175 j .Lexit
1176
1177 .Lc4: s32e a4, a9, -16
1178 s32e a5, a9, -12
1179 s32e a6, a9, -8
1180 s32e a7, a9, -4
1181
1182 srli a7, a3, 1
1183 rotw 1
1184 _bnei a3, 1, .Lloop
1185 j .Lexit
1186
1187 .Lc12: _bbci.l a3, 3, .Linvalid_mask # bit 2 shouldn't be zero!
1188
1189 /* 12-register frame (call12) */
1190
1191 l32e a0, a5, -12
1192 s32e a8, a0, -48
1193 mov a8, a0
1194
1195 s32e a9, a8, -44
1196 s32e a10, a8, -40
1197 s32e a11, a8, -36
1198 s32e a12, a8, -32
1199 s32e a13, a8, -28
1200 s32e a14, a8, -24
1201 s32e a15, a8, -20
1202 srli a15, a3, 3
1203
1204 /* The stack pointer for a4..a7 is out of reach, so we rotate the
1205 * window, grab the stackpointer, and rotate back.
1206 * Alternatively, we could also use the following approach, but that
1207 * makes the fixup routine much more complicated:
1208 * rotw 1
1209 * s32e a0, a13, -16
1210 * ...
1211 * rotw 2
1212 */
1213
1214 rotw 1
1215 mov a4, a13
1216 rotw -1
1217
1218 s32e a4, a8, -16
1219 s32e a5, a8, -12
1220 s32e a6, a8, -8
1221 s32e a7, a8, -4
1222
1223 rotw 3
1224
1225 _beqi a3, 1, .Lexit
1226 j .Lloop
1227
1228 .Lexit:
1229
1230 /* Done. Do the final rotation and set WS */
1231
1232 rotw 1
1233 rsr a3, windowbase
1234 ssl a3
1235 movi a3, 1
1236 sll a3, a3
1237 wsr a3, windowstart
1238 .Lnospill:
1239
1240 /* Advance PC, restore registers and SAR, and return from exception. */
1241
1242 l32i a3, a2, PT_SAR
1243 l32i a0, a2, PT_AREG0
1244 wsr a3, sar
1245 l32i a3, a2, PT_AREG3
1246
1247 /* Restore clobbered registers. */
1248
1249 l32i a4, a2, PT_AREG4
1250 l32i a7, a2, PT_AREG7
1251 l32i a8, a2, PT_AREG8
1252 l32i a11, a2, PT_AREG11
1253 l32i a12, a2, PT_AREG12
1254 l32i a15, a2, PT_AREG15
1255
1256 movi a2, 0
1257 rfe
1258
1259 .Linvalid_mask:
1260
1261 /* We get here because of an unrecoverable error in the window
1262 * registers, so set up a dummy frame and kill the user application.
1263 * Note: We assume EXC_TABLE_KSTK contains a valid stack pointer.
1264 */
1265
1266 movi a0, 1
1267 movi a1, 0
1268
1269 wsr a0, windowstart
1270 wsr a1, windowbase
1271 rsync
1272
1273 movi a0, 0
1274
1275 rsr a3, excsave1
1276 l32i a1, a3, EXC_TABLE_KSTK
1277
1278 movi a4, (1 << PS_WOE_BIT) | LOCKLEVEL
1279 wsr a4, ps
1280 rsync
1281
1282 movi a6, SIGSEGV
1283 movi a4, do_exit
1284 callx4 a4
1285
1286 /* shouldn't return, so panic */
1287
1288 wsr a0, excsave1
1289 movi a0, unrecoverable_exception
1290 callx0 a0 # should not return
1291 1: j 1b
1292
1293
1294 ENDPROC(fast_syscall_spill_registers)
1295
1296 /* Fixup handler.
1297 *
1298 * We get here if the spill routine causes an exception, e.g. tlb miss.
1299 * We basically restore WINDOWBASE and WINDOWSTART to the condition when
1300 * we entered the spill routine and jump to the user exception handler.
1301 *
1302 * Note that we only need to restore the bits in windowstart that have not
1303 * been spilled yet by the _spill_register routine. Luckily, a3 contains a
1304 * rotated windowstart with only those bits set for frames that haven't been
1305 * spilled yet. Because a3 is rotated such that bit 0 represents the register
1306 * frame for the current windowbase - 1, we need to rotate a3 left by the
1307 * value of the current windowbase + 1 and move it to windowstart.
1308 *
1309 * a0: value of depc, original value in depc
1310 * a2: trashed, original value in EXC_TABLE_DOUBLE_SAVE
1311 * a3: exctable, original value in excsave1
1312 */
1313
1314 ENTRY(fast_syscall_spill_registers_fixup)
1315
1316 rsr a2, windowbase # get current windowbase (a2 is saved)
1317 xsr a0, depc # restore depc and a0
1318 ssl a2 # set shift (32 - WB)
1319
1320 /* We need to make sure the current registers (a0-a3) are preserved.
1321 * To do this, we simply set the bit for the current window frame
1322 * in WS, so that the exception handlers save them to the task stack.
1323 *
1324 * Note: we use a3 to set the windowbase, so we take a special care
1325 * of it, saving it in the original _spill_registers frame across
1326 * the exception handler call.
1327 */
1328
1329 xsr a3, excsave1 # get spill-mask
1330 slli a3, a3, 1 # shift left by one
1331 addi a3, a3, 1 # set the bit for the current window frame
1332
1333 slli a2, a3, 32-WSBITS
1334 src a2, a3, a2 # a2 = xxwww1yyxxxwww1yy......
1335 wsr a2, windowstart # set corrected windowstart
1336
1337 srli a3, a3, 1
1338 rsr a2, excsave1
1339 l32i a2, a2, EXC_TABLE_DOUBLE_SAVE # restore a2
1340 xsr a2, excsave1
1341 s32i a3, a2, EXC_TABLE_DOUBLE_SAVE # save a3
1342 l32i a3, a2, EXC_TABLE_PARAM # original WB (in user task)
1343 xsr a2, excsave1
1344
1345 /* Return to the original (user task) WINDOWBASE.
1346 * We leave the following frame behind:
1347 * a0, a1, a2 same
1348 * a3: trashed (saved in EXC_TABLE_DOUBLE_SAVE)
1349 * depc: depc (we have to return to that address)
1350 * excsave_1: exctable
1351 */
1352
1353 wsr a3, windowbase
1354 rsync
1355
1356 /* We are now in the original frame when we entered _spill_registers:
1357 * a0: return address
1358 * a1: used, stack pointer
1359 * a2: kernel stack pointer
1360 * a3: available
1361 * depc: exception address
1362 * excsave: exctable
1363 * Note: This frame might be the same as above.
1364 */
1365
1366 /* Setup stack pointer. */
1367
1368 addi a2, a2, -PT_USER_SIZE
1369 s32i a0, a2, PT_AREG0
1370
1371 /* Make sure we return to this fixup handler. */
1372
1373 movi a3, fast_syscall_spill_registers_fixup_return
1374 s32i a3, a2, PT_DEPC # setup depc
1375
1376 /* Jump to the exception handler. */
1377
1378 rsr a3, excsave1
1379 rsr a0, exccause
1380 addx4 a0, a0, a3 # find entry in table
1381 l32i a0, a0, EXC_TABLE_FAST_USER # load handler
1382 l32i a3, a3, EXC_TABLE_DOUBLE_SAVE
1383 jx a0
1384
1385 ENDPROC(fast_syscall_spill_registers_fixup)
1386
1387 ENTRY(fast_syscall_spill_registers_fixup_return)
1388
1389 /* When we return here, all registers have been restored (a2: DEPC) */
1390
1391 wsr a2, depc # exception address
1392
1393 /* Restore fixup handler. */
1394
1395 rsr a2, excsave1
1396 s32i a3, a2, EXC_TABLE_DOUBLE_SAVE
1397 movi a3, fast_syscall_spill_registers_fixup
1398 s32i a3, a2, EXC_TABLE_FIXUP
1399 rsr a3, windowbase
1400 s32i a3, a2, EXC_TABLE_PARAM
1401 l32i a2, a2, EXC_TABLE_KSTK
1402
1403 /* Load WB at the time the exception occurred. */
1404
1405 rsr a3, sar # WB is still in SAR
1406 neg a3, a3
1407 wsr a3, windowbase
1408 rsync
1409
1410 rsr a3, excsave1
1411 l32i a3, a3, EXC_TABLE_DOUBLE_SAVE
1412
1413 rfde
1414
1415 ENDPROC(fast_syscall_spill_registers_fixup_return)
1416
1417 #else /* CONFIG_FAST_SYSCALL_SPILL_REGISTERS */
1418
1419 ENTRY(fast_syscall_spill_registers)
1420
1421 l32i a0, a2, PT_AREG0 # restore a0
1422 movi a2, -ENOSYS
1423 rfe
1424
1425 ENDPROC(fast_syscall_spill_registers)
1426
1427 #endif /* CONFIG_FAST_SYSCALL_SPILL_REGISTERS */
1428
1429 #ifdef CONFIG_MMU
1430 /*
1431 * We should never get here. Bail out!
1432 */
1433
1434 ENTRY(fast_second_level_miss_double_kernel)
1435
1436 1: movi a0, unrecoverable_exception
1437 callx0 a0 # should not return
1438 1: j 1b
1439
1440 ENDPROC(fast_second_level_miss_double_kernel)
1441
1442 /* First-level entry handler for user, kernel, and double 2nd-level
1443 * TLB miss exceptions. Note that for now, user and kernel miss
1444 * exceptions share the same entry point and are handled identically.
1445 *
1446 * An old, less-efficient C version of this function used to exist.
1447 * We include it below, interleaved as comments, for reference.
1448 *
1449 * Entry condition:
1450 *
1451 * a0: trashed, original value saved on stack (PT_AREG0)
1452 * a1: a1
1453 * a2: new stack pointer, original in DEPC
1454 * a3: a3
1455 * depc: a2, original value saved on stack (PT_DEPC)
1456 * excsave_1: dispatch table
1457 *
1458 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
1459 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
1460 */
1461
1462 ENTRY(fast_second_level_miss)
1463
1464 /* Save a1 and a3. Note: we don't expect a double exception. */
1465
1466 s32i a1, a2, PT_AREG1
1467 s32i a3, a2, PT_AREG3
1468
1469 /* We need to map the page of PTEs for the user task. Find
1470 * the pointer to that page. Also, it's possible for tsk->mm
1471 * to be NULL while tsk->active_mm is nonzero if we faulted on
1472 * a vmalloc address. In that rare case, we must use
1473 * active_mm instead to avoid a fault in this handler. See
1474 *
1475 * http://mail.nl.linux.org/linux-mm/2002-08/msg00258.html
1476 * (or search Internet on "mm vs. active_mm")
1477 *
1478 * if (!mm)
1479 * mm = tsk->active_mm;
1480 * pgd = pgd_offset (mm, regs->excvaddr);
1481 * pmd = pmd_offset (pgd, regs->excvaddr);
1482 * pmdval = *pmd;
1483 */
1484
1485 GET_CURRENT(a1,a2)
1486 l32i a0, a1, TASK_MM # tsk->mm
1487 beqz a0, 9f
1488
1489 8: rsr a3, excvaddr # fault address
1490 _PGD_OFFSET(a0, a3, a1)
1491 l32i a0, a0, 0 # read pmdval
1492 beqz a0, 2f
1493
1494 /* Read ptevaddr and convert to top of page-table page.
1495 *
1496 * vpnval = read_ptevaddr_register() & PAGE_MASK;
1497 * vpnval += DTLB_WAY_PGTABLE;
1498 * pteval = mk_pte (virt_to_page(pmd_val(pmdval)), PAGE_KERNEL);
1499 * write_dtlb_entry (pteval, vpnval);
1500 *
1501 * The messy computation for 'pteval' above really simplifies
1502 * into the following:
1503 *
1504 * pteval = ((pmdval - PAGE_OFFSET) & PAGE_MASK) | PAGE_DIRECTORY
1505 */
1506
1507 movi a1, (-PAGE_OFFSET) & 0xffffffff
1508 add a0, a0, a1 # pmdval - PAGE_OFFSET
1509 extui a1, a0, 0, PAGE_SHIFT # ... & PAGE_MASK
1510 xor a0, a0, a1
1511
1512 movi a1, _PAGE_DIRECTORY
1513 or a0, a0, a1 # ... | PAGE_DIRECTORY
1514
1515 /*
1516 * We utilize all three wired-ways (7-9) to hold pmd translations.
1517 * Memory regions are mapped to the DTLBs according to bits 28 and 29.
1518 * This allows to map the three most common regions to three different
1519 * DTLBs:
1520 * 0,1 -> way 7 program (0040.0000) and virtual (c000.0000)
1521 * 2 -> way 8 shared libaries (2000.0000)
1522 * 3 -> way 0 stack (3000.0000)
1523 */
1524
1525 extui a3, a3, 28, 2 # addr. bit 28 and 29 0,1,2,3
1526 rsr a1, ptevaddr
1527 addx2 a3, a3, a3 # -> 0,3,6,9
1528 srli a1, a1, PAGE_SHIFT
1529 extui a3, a3, 2, 2 # -> 0,0,1,2
1530 slli a1, a1, PAGE_SHIFT # ptevaddr & PAGE_MASK
1531 addi a3, a3, DTLB_WAY_PGD
1532 add a1, a1, a3 # ... + way_number
1533
1534 3: wdtlb a0, a1
1535 dsync
1536
1537 /* Exit critical section. */
1538
1539 4: rsr a3, excsave1
1540 movi a0, 0
1541 s32i a0, a3, EXC_TABLE_FIXUP
1542
1543 /* Restore the working registers, and return. */
1544
1545 l32i a0, a2, PT_AREG0
1546 l32i a1, a2, PT_AREG1
1547 l32i a3, a2, PT_AREG3
1548 l32i a2, a2, PT_DEPC
1549
1550 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
1551
1552 /* Restore excsave1 and return. */
1553
1554 rsr a2, depc
1555 rfe
1556
1557 /* Return from double exception. */
1558
1559 1: xsr a2, depc
1560 esync
1561 rfde
1562
1563 9: l32i a0, a1, TASK_ACTIVE_MM # unlikely case mm == 0
1564 j 8b
1565
1566 #if (DCACHE_WAY_SIZE > PAGE_SIZE)
1567
1568 2: /* Special case for cache aliasing.
1569 * We (should) only get here if a clear_user_page, copy_user_page
1570 * or the aliased cache flush functions got preemptively interrupted
1571 * by another task. Re-establish temporary mapping to the
1572 * TLBTEMP_BASE areas.
1573 */
1574
1575 /* We shouldn't be in a double exception */
1576
1577 l32i a0, a2, PT_DEPC
1578 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, 2f
1579
1580 /* Make sure the exception originated in the special functions */
1581
1582 movi a0, __tlbtemp_mapping_start
1583 rsr a3, epc1
1584 bltu a3, a0, 2f
1585 movi a0, __tlbtemp_mapping_end
1586 bgeu a3, a0, 2f
1587
1588 /* Check if excvaddr was in one of the TLBTEMP_BASE areas. */
1589
1590 movi a3, TLBTEMP_BASE_1
1591 rsr a0, excvaddr
1592 bltu a0, a3, 2f
1593
1594 addi a1, a0, -TLBTEMP_SIZE
1595 bgeu a1, a3, 2f
1596
1597 /* Check if we have to restore an ITLB mapping. */
1598
1599 movi a1, __tlbtemp_mapping_itlb
1600 rsr a3, epc1
1601 sub a3, a3, a1
1602
1603 /* Calculate VPN */
1604
1605 movi a1, PAGE_MASK
1606 and a1, a1, a0
1607
1608 /* Jump for ITLB entry */
1609
1610 bgez a3, 1f
1611
1612 /* We can use up to two TLBTEMP areas, one for src and one for dst. */
1613
1614 extui a3, a0, PAGE_SHIFT + DCACHE_ALIAS_ORDER, 1
1615 add a1, a3, a1
1616
1617 /* PPN is in a6 for the first TLBTEMP area and in a7 for the second. */
1618
1619 mov a0, a6
1620 movnez a0, a7, a3
1621 j 3b
1622
1623 /* ITLB entry. We only use dst in a6. */
1624
1625 1: witlb a6, a1
1626 isync
1627 j 4b
1628
1629
1630 #endif // DCACHE_WAY_SIZE > PAGE_SIZE
1631
1632
1633 2: /* Invalid PGD, default exception handling */
1634
1635 rsr a1, depc
1636 s32i a1, a2, PT_AREG2
1637 mov a1, a2
1638
1639 rsr a2, ps
1640 bbsi.l a2, PS_UM_BIT, 1f
1641 j _kernel_exception
1642 1: j _user_exception
1643
1644 ENDPROC(fast_second_level_miss)
1645
1646 /*
1647 * StoreProhibitedException
1648 *
1649 * Update the pte and invalidate the itlb mapping for this pte.
1650 *
1651 * Entry condition:
1652 *
1653 * a0: trashed, original value saved on stack (PT_AREG0)
1654 * a1: a1
1655 * a2: new stack pointer, original in DEPC
1656 * a3: a3
1657 * depc: a2, original value saved on stack (PT_DEPC)
1658 * excsave_1: dispatch table
1659 *
1660 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
1661 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
1662 */
1663
1664 ENTRY(fast_store_prohibited)
1665
1666 /* Save a1 and a3. */
1667
1668 s32i a1, a2, PT_AREG1
1669 s32i a3, a2, PT_AREG3
1670
1671 GET_CURRENT(a1,a2)
1672 l32i a0, a1, TASK_MM # tsk->mm
1673 beqz a0, 9f
1674
1675 8: rsr a1, excvaddr # fault address
1676 _PGD_OFFSET(a0, a1, a3)
1677 l32i a0, a0, 0
1678 beqz a0, 2f
1679
1680 /*
1681 * Note that we test _PAGE_WRITABLE_BIT only if PTE is present
1682 * and is not PAGE_NONE. See pgtable.h for possible PTE layouts.
1683 */
1684
1685 _PTE_OFFSET(a0, a1, a3)
1686 l32i a3, a0, 0 # read pteval
1687 movi a1, _PAGE_CA_INVALID
1688 ball a3, a1, 2f
1689 bbci.l a3, _PAGE_WRITABLE_BIT, 2f
1690
1691 movi a1, _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_HW_WRITE
1692 or a3, a3, a1
1693 rsr a1, excvaddr
1694 s32i a3, a0, 0
1695
1696 /* We need to flush the cache if we have page coloring. */
1697 #if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK
1698 dhwb a0, 0
1699 #endif
1700 pdtlb a0, a1
1701 wdtlb a3, a0
1702
1703 /* Exit critical section. */
1704
1705 movi a0, 0
1706 rsr a3, excsave1
1707 s32i a0, a3, EXC_TABLE_FIXUP
1708
1709 /* Restore the working registers, and return. */
1710
1711 l32i a3, a2, PT_AREG3
1712 l32i a1, a2, PT_AREG1
1713 l32i a0, a2, PT_AREG0
1714 l32i a2, a2, PT_DEPC
1715
1716 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
1717
1718 rsr a2, depc
1719 rfe
1720
1721 /* Double exception. Restore FIXUP handler and return. */
1722
1723 1: xsr a2, depc
1724 esync
1725 rfde
1726
1727 9: l32i a0, a1, TASK_ACTIVE_MM # unlikely case mm == 0
1728 j 8b
1729
1730 2: /* If there was a problem, handle fault in C */
1731
1732 rsr a3, depc # still holds a2
1733 s32i a3, a2, PT_AREG2
1734 mov a1, a2
1735
1736 rsr a2, ps
1737 bbsi.l a2, PS_UM_BIT, 1f
1738 j _kernel_exception
1739 1: j _user_exception
1740
1741 ENDPROC(fast_store_prohibited)
1742
1743 #endif /* CONFIG_MMU */
1744
1745 /*
1746 * System Calls.
1747 *
1748 * void system_call (struct pt_regs* regs, int exccause)
1749 * a2 a3
1750 */
1751
1752 ENTRY(system_call)
1753
1754 entry a1, 32
1755
1756 /* regs->syscall = regs->areg[2] */
1757
1758 l32i a3, a2, PT_AREG2
1759 mov a6, a2
1760 movi a4, do_syscall_trace_enter
1761 s32i a3, a2, PT_SYSCALL
1762 callx4 a4
1763
1764 /* syscall = sys_call_table[syscall_nr] */
1765
1766 movi a4, sys_call_table;
1767 movi a5, __NR_syscall_count
1768 movi a6, -ENOSYS
1769 bgeu a3, a5, 1f
1770
1771 addx4 a4, a3, a4
1772 l32i a4, a4, 0
1773 movi a5, sys_ni_syscall;
1774 beq a4, a5, 1f
1775
1776 /* Load args: arg0 - arg5 are passed via regs. */
1777
1778 l32i a6, a2, PT_AREG6
1779 l32i a7, a2, PT_AREG3
1780 l32i a8, a2, PT_AREG4
1781 l32i a9, a2, PT_AREG5
1782 l32i a10, a2, PT_AREG8
1783 l32i a11, a2, PT_AREG9
1784
1785 /* Pass one additional argument to the syscall: pt_regs (on stack) */
1786 s32i a2, a1, 0
1787
1788 callx4 a4
1789
1790 1: /* regs->areg[2] = return_value */
1791
1792 s32i a6, a2, PT_AREG2
1793 movi a4, do_syscall_trace_leave
1794 mov a6, a2
1795 callx4 a4
1796 retw
1797
1798 ENDPROC(system_call)
1799
1800 /*
1801 * Spill live registers on the kernel stack macro.
1802 *
1803 * Entry condition: ps.woe is set, ps.excm is cleared
1804 * Exit condition: windowstart has single bit set
1805 * May clobber: a12, a13
1806 */
1807 .macro spill_registers_kernel
1808
1809 #if XCHAL_NUM_AREGS > 16
1810 call12 1f
1811 _j 2f
1812 retw
1813 .align 4
1814 1:
1815 _entry a1, 48
1816 addi a12, a0, 3
1817 #if XCHAL_NUM_AREGS > 32
1818 .rept (XCHAL_NUM_AREGS - 32) / 12
1819 _entry a1, 48
1820 mov a12, a0
1821 .endr
1822 #endif
1823 _entry a1, 48
1824 #if XCHAL_NUM_AREGS % 12 == 0
1825 mov a8, a8
1826 #elif XCHAL_NUM_AREGS % 12 == 4
1827 mov a12, a12
1828 #elif XCHAL_NUM_AREGS % 12 == 8
1829 mov a4, a4
1830 #endif
1831 retw
1832 2:
1833 #else
1834 mov a12, a12
1835 #endif
1836 .endm
1837
1838 /*
1839 * Task switch.
1840 *
1841 * struct task* _switch_to (struct task* prev, struct task* next)
1842 * a2 a2 a3
1843 */
1844
1845 ENTRY(_switch_to)
1846
1847 entry a1, 16
1848
1849 mov a11, a3 # and 'next' (a3)
1850
1851 l32i a4, a2, TASK_THREAD_INFO
1852 l32i a5, a3, TASK_THREAD_INFO
1853
1854 save_xtregs_user a4 a6 a8 a9 a12 a13 THREAD_XTREGS_USER
1855
1856 #if THREAD_RA > 1020 || THREAD_SP > 1020
1857 addi a10, a2, TASK_THREAD
1858 s32i a0, a10, THREAD_RA - TASK_THREAD # save return address
1859 s32i a1, a10, THREAD_SP - TASK_THREAD # save stack pointer
1860 #else
1861 s32i a0, a2, THREAD_RA # save return address
1862 s32i a1, a2, THREAD_SP # save stack pointer
1863 #endif
1864
1865 /* Disable ints while we manipulate the stack pointer. */
1866
1867 rsil a14, LOCKLEVEL
1868 rsr a3, excsave1
1869 rsync
1870 s32i a3, a3, EXC_TABLE_FIXUP /* enter critical section */
1871
1872 /* Switch CPENABLE */
1873
1874 #if (XTENSA_HAVE_COPROCESSORS || XTENSA_HAVE_IO_PORTS)
1875 l32i a3, a5, THREAD_CPENABLE
1876 xsr a3, cpenable
1877 s32i a3, a4, THREAD_CPENABLE
1878 #endif
1879
1880 /* Flush register file. */
1881
1882 spill_registers_kernel
1883
1884 /* Set kernel stack (and leave critical section)
1885 * Note: It's save to set it here. The stack will not be overwritten
1886 * because the kernel stack will only be loaded again after
1887 * we return from kernel space.
1888 */
1889
1890 rsr a3, excsave1 # exc_table
1891 movi a6, 0
1892 addi a7, a5, PT_REGS_OFFSET
1893 s32i a6, a3, EXC_TABLE_FIXUP
1894 s32i a7, a3, EXC_TABLE_KSTK
1895
1896 /* restore context of the task 'next' */
1897
1898 l32i a0, a11, THREAD_RA # restore return address
1899 l32i a1, a11, THREAD_SP # restore stack pointer
1900
1901 load_xtregs_user a5 a6 a8 a9 a12 a13 THREAD_XTREGS_USER
1902
1903 wsr a14, ps
1904 rsync
1905
1906 retw
1907
1908 ENDPROC(_switch_to)
1909
1910 ENTRY(ret_from_fork)
1911
1912 /* void schedule_tail (struct task_struct *prev)
1913 * Note: prev is still in a6 (return value from fake call4 frame)
1914 */
1915 movi a4, schedule_tail
1916 callx4 a4
1917
1918 movi a4, do_syscall_trace_leave
1919 mov a6, a1
1920 callx4 a4
1921
1922 j common_exception_return
1923
1924 ENDPROC(ret_from_fork)
1925
1926 /*
1927 * Kernel thread creation helper
1928 * On entry, set up by copy_thread: a2 = thread_fn, a3 = thread_fn arg
1929 * left from _switch_to: a6 = prev
1930 */
1931 ENTRY(ret_from_kernel_thread)
1932
1933 call4 schedule_tail
1934 mov a6, a3
1935 callx4 a2
1936 j common_exception_return
1937
1938 ENDPROC(ret_from_kernel_thread)
ubuntu-zesty-kernel mirror