]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - arch/xtensa/kernel/entry.S
xtensa: use the new byteorder headers
[mirror_ubuntu-bionic-kernel.git] / arch / xtensa / kernel / entry.S
CommitLineData
5a0015d6
CZ
1/*
2 * arch/xtensa/kernel/entry.S
3 *
4 * Low-level exception handling
5 *
6 * This file is subject to the terms and conditions of the GNU General Public
7 * License. See the file "COPYING" in the main directory of this archive
8 * for more details.
9 *
6656920b 10 * Copyright (C) 2004-2007 by Tensilica Inc.
5a0015d6
CZ
11 *
12 * Chris Zankel <chris@zankel.net>
13 *
14 */
15
16#include <linux/linkage.h>
0013a854 17#include <asm/asm-offsets.h>
5a0015d6
CZ
18#include <asm/processor.h>
19#include <asm/thread_info.h>
20#include <asm/uaccess.h>
21#include <asm/unistd.h>
22#include <asm/ptrace.h>
23#include <asm/current.h>
24#include <asm/pgtable.h>
25#include <asm/page.h>
26#include <asm/signal.h>
173d6681 27#include <asm/tlbflush.h>
c658eac6 28#include <asm/variant/tie-asm.h>
5a0015d6
CZ
29
30/* Unimplemented features. */
31
5a0015d6
CZ
32#undef KERNEL_STACK_OVERFLOW_CHECK
33#undef PREEMPTIBLE_KERNEL
34#undef ALLOCA_EXCEPTION_IN_IRAM
35
36/* Not well tested.
37 *
38 * - fast_coprocessor
39 */
40
41/*
42 * Macro to find first bit set in WINDOWBASE from the left + 1
43 *
44 * 100....0 -> 1
45 * 010....0 -> 2
46 * 000....1 -> WSBITS
47 */
48
49 .macro ffs_ws bit mask
50
51#if XCHAL_HAVE_NSA
52 nsau \bit, \mask # 32-WSBITS ... 31 (32 iff 0)
53 addi \bit, \bit, WSBITS - 32 + 1 # uppest bit set -> return 1
54#else
55 movi \bit, WSBITS
56#if WSBITS > 16
57 _bltui \mask, 0x10000, 99f
58 addi \bit, \bit, -16
59 extui \mask, \mask, 16, 16
60#endif
61#if WSBITS > 8
6299: _bltui \mask, 0x100, 99f
63 addi \bit, \bit, -8
64 srli \mask, \mask, 8
65#endif
6699: _bltui \mask, 0x10, 99f
67 addi \bit, \bit, -4
68 srli \mask, \mask, 4
6999: _bltui \mask, 0x4, 99f
70 addi \bit, \bit, -2
71 srli \mask, \mask, 2
7299: _bltui \mask, 0x2, 99f
73 addi \bit, \bit, -1
7499:
75
76#endif
77 .endm
78
79/* ----------------- DEFAULT FIRST LEVEL EXCEPTION HANDLERS ----------------- */
80
81/*
82 * First-level exception handler for user exceptions.
83 * Save some special registers, extra states and all registers in the AR
84 * register file that were in use in the user task, and jump to the common
85 * exception code.
86 * We save SAR (used to calculate WMASK), and WB and WS (we don't have to
87 * save them for kernel exceptions).
88 *
89 * Entry condition for user_exception:
90 *
91 * a0: trashed, original value saved on stack (PT_AREG0)
92 * a1: a1
93 * a2: new stack pointer, original value in depc
94 * a3: dispatch table
95 * depc: a2, original value saved on stack (PT_DEPC)
96 * excsave1: a3
97 *
98 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
99 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
100 *
101 * Entry condition for _user_exception:
102 *
103 * a0-a3 and depc have been saved to PT_AREG0...PT_AREG3 and PT_DEPC
104 * excsave has been restored, and
105 * stack pointer (a1) has been set.
106 *
107 * Note: _user_exception might be at an odd adress. Don't use call0..call12
108 */
109
110ENTRY(user_exception)
111
112 /* Save a2, a3, and depc, restore excsave_1 and set SP. */
113
114 xsr a3, EXCSAVE_1
115 rsr a0, DEPC
116 s32i a1, a2, PT_AREG1
117 s32i a0, a2, PT_AREG2
118 s32i a3, a2, PT_AREG3
119 mov a1, a2
120
121 .globl _user_exception
122_user_exception:
123
124 /* Save SAR and turn off single stepping */
125
126 movi a2, 0
127 rsr a3, SAR
29c4dfd9 128 xsr a2, ICOUNTLEVEL
5a0015d6 129 s32i a3, a1, PT_SAR
29c4dfd9 130 s32i a2, a1, PT_ICOUNTLEVEL
5a0015d6
CZ
131
132 /* Rotate ws so that the current windowbase is at bit0. */
133 /* Assume ws = xxwww1yyyy. Rotate ws right, so that a2 = yyyyxxwww1 */
134
135 rsr a2, WINDOWBASE
136 rsr a3, WINDOWSTART
137 ssr a2
138 s32i a2, a1, PT_WINDOWBASE
139 s32i a3, a1, PT_WINDOWSTART
140 slli a2, a3, 32-WSBITS
141 src a2, a3, a2
142 srli a2, a2, 32-WSBITS
143 s32i a2, a1, PT_WMASK # needed for restoring registers
144
145 /* Save only live registers. */
146
147 _bbsi.l a2, 1, 1f
148 s32i a4, a1, PT_AREG4
149 s32i a5, a1, PT_AREG5
150 s32i a6, a1, PT_AREG6
151 s32i a7, a1, PT_AREG7
152 _bbsi.l a2, 2, 1f
153 s32i a8, a1, PT_AREG8
154 s32i a9, a1, PT_AREG9
155 s32i a10, a1, PT_AREG10
156 s32i a11, a1, PT_AREG11
157 _bbsi.l a2, 3, 1f
158 s32i a12, a1, PT_AREG12
159 s32i a13, a1, PT_AREG13
160 s32i a14, a1, PT_AREG14
161 s32i a15, a1, PT_AREG15
162 _bnei a2, 1, 1f # only one valid frame?
163
164 /* Only one valid frame, skip saving regs. */
165
166 j 2f
167
168 /* Save the remaining registers.
169 * We have to save all registers up to the first '1' from
170 * the right, except the current frame (bit 0).
171 * Assume a2 is: 001001000110001
6656920b 172 * All register frames starting from the top field to the marked '1'
5a0015d6
CZ
173 * must be saved.
174 */
175
1761: addi a3, a2, -1 # eliminate '1' in bit 0: yyyyxxww0
177 neg a3, a3 # yyyyxxww0 -> YYYYXXWW1+1
178 and a3, a3, a2 # max. only one bit is set
179
180 /* Find number of frames to save */
181
182 ffs_ws a0, a3 # number of frames to the '1' from left
183
184 /* Store information into WMASK:
185 * bits 0..3: xxx1 masked lower 4 bits of the rotated windowstart,
186 * bits 4...: number of valid 4-register frames
187 */
188
189 slli a3, a0, 4 # number of frames to save in bits 8..4
190 extui a2, a2, 0, 4 # mask for the first 16 registers
191 or a2, a3, a2
192 s32i a2, a1, PT_WMASK # needed when we restore the reg-file
193
194 /* Save 4 registers at a time */
195
1961: rotw -1
197 s32i a0, a5, PT_AREG_END - 16
198 s32i a1, a5, PT_AREG_END - 12
199 s32i a2, a5, PT_AREG_END - 8
200 s32i a3, a5, PT_AREG_END - 4
201 addi a0, a4, -1
202 addi a1, a5, -16
203 _bnez a0, 1b
204
205 /* WINDOWBASE still in SAR! */
206
207 rsr a2, SAR # original WINDOWBASE
208 movi a3, 1
209 ssl a2
210 sll a3, a3
211 wsr a3, WINDOWSTART # set corresponding WINDOWSTART bit
212 wsr a2, WINDOWBASE # and WINDOWSTART
213 rsync
214
215 /* We are back to the original stack pointer (a1) */
216
c658eac6 2172: /* Now, jump to the common exception handler. */
5a0015d6
CZ
218
219 j common_exception
220
221
222/*
223 * First-level exit handler for kernel exceptions
224 * Save special registers and the live window frame.
225 * Note: Even though we changes the stack pointer, we don't have to do a
226 * MOVSP here, as we do that when we return from the exception.
227 * (See comment in the kernel exception exit code)
228 *
229 * Entry condition for kernel_exception:
230 *
231 * a0: trashed, original value saved on stack (PT_AREG0)
232 * a1: a1
233 * a2: new stack pointer, original in DEPC
234 * a3: dispatch table
235 * depc: a2, original value saved on stack (PT_DEPC)
236 * excsave_1: a3
237 *
238 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
239 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
240 *
241 * Entry condition for _kernel_exception:
242 *
243 * a0-a3 and depc have been saved to PT_AREG0...PT_AREG3 and PT_DEPC
244 * excsave has been restored, and
245 * stack pointer (a1) has been set.
246 *
247 * Note: _kernel_exception might be at an odd adress. Don't use call0..call12
248 */
249
250ENTRY(kernel_exception)
251
252 /* Save a0, a2, a3, DEPC and set SP. */
253
254 xsr a3, EXCSAVE_1 # restore a3, excsave_1
255 rsr a0, DEPC # get a2
256 s32i a1, a2, PT_AREG1
257 s32i a0, a2, PT_AREG2
258 s32i a3, a2, PT_AREG3
259 mov a1, a2
260
261 .globl _kernel_exception
262_kernel_exception:
263
264 /* Save SAR and turn off single stepping */
265
266 movi a2, 0
267 rsr a3, SAR
29c4dfd9 268 xsr a2, ICOUNTLEVEL
5a0015d6 269 s32i a3, a1, PT_SAR
29c4dfd9 270 s32i a2, a1, PT_ICOUNTLEVEL
5a0015d6
CZ
271
272 /* Rotate ws so that the current windowbase is at bit0. */
273 /* Assume ws = xxwww1yyyy. Rotate ws right, so that a2 = yyyyxxwww1 */
274
275 rsr a2, WINDOWBASE # don't need to save these, we only
276 rsr a3, WINDOWSTART # need shifted windowstart: windowmask
277 ssr a2
278 slli a2, a3, 32-WSBITS
279 src a2, a3, a2
280 srli a2, a2, 32-WSBITS
281 s32i a2, a1, PT_WMASK # needed for kernel_exception_exit
282
283 /* Save only the live window-frame */
284
285 _bbsi.l a2, 1, 1f
286 s32i a4, a1, PT_AREG4
287 s32i a5, a1, PT_AREG5
288 s32i a6, a1, PT_AREG6
289 s32i a7, a1, PT_AREG7
290 _bbsi.l a2, 2, 1f
291 s32i a8, a1, PT_AREG8
292 s32i a9, a1, PT_AREG9
293 s32i a10, a1, PT_AREG10
294 s32i a11, a1, PT_AREG11
295 _bbsi.l a2, 3, 1f
296 s32i a12, a1, PT_AREG12
297 s32i a13, a1, PT_AREG13
298 s32i a14, a1, PT_AREG14
299 s32i a15, a1, PT_AREG15
300
3011:
302
303#ifdef KERNEL_STACK_OVERFLOW_CHECK
304
305 /* Stack overflow check, for debugging */
306 extui a2, a1, TASK_SIZE_BITS,XX
307 movi a3, SIZE??
308 _bge a2, a3, out_of_stack_panic
309
310#endif
311
312/*
313 * This is the common exception handler.
314 * We get here from the user exception handler or simply by falling through
315 * from the kernel exception handler.
316 * Save the remaining special registers, switch to kernel mode, and jump
317 * to the second-level exception handler.
318 *
319 */
320
321common_exception:
322
29c4dfd9 323 /* Save some registers, disable loops and clear the syscall flag. */
5a0015d6
CZ
324
325 rsr a2, DEBUGCAUSE
326 rsr a3, EPC_1
327 s32i a2, a1, PT_DEBUGCAUSE
328 s32i a3, a1, PT_PC
329
29c4dfd9 330 movi a2, -1
5a0015d6 331 rsr a3, EXCVADDR
29c4dfd9 332 s32i a2, a1, PT_SYSCALL
5a0015d6
CZ
333 movi a2, 0
334 s32i a3, a1, PT_EXCVADDR
335 xsr a2, LCOUNT
336 s32i a2, a1, PT_LCOUNT
337
338 /* It is now save to restore the EXC_TABLE_FIXUP variable. */
339
340 rsr a0, EXCCAUSE
341 movi a3, 0
342 rsr a2, EXCSAVE_1
343 s32i a0, a1, PT_EXCCAUSE
344 s32i a3, a2, EXC_TABLE_FIXUP
345
346 /* All unrecoverable states are saved on stack, now, and a1 is valid,
347 * so we can allow exceptions and interrupts (*) again.
348 * Set PS(EXCM = 0, UM = 0, RING = 0, OWB = 0, WOE = 1, INTLEVEL = X)
349 *
350 * (*) We only allow interrupts if PS.INTLEVEL was not set to 1 before
351 * (interrupts disabled) and if this exception is not an interrupt.
352 */
353
354 rsr a3, PS
355 addi a0, a0, -4
356 movi a2, 1
357 extui a3, a3, 0, 1 # a3 = PS.INTLEVEL[0]
358 moveqz a3, a2, a0 # a3 = 1 iff interrupt exception
173d6681 359 movi a2, 1 << PS_WOE_BIT
5a0015d6
CZ
360 or a3, a3, a2
361 rsr a0, EXCCAUSE
362 xsr a3, PS
363
364 s32i a3, a1, PT_PS # save ps
365
366 /* Save LBEG, LEND */
367
368 rsr a2, LBEG
369 rsr a3, LEND
370 s32i a2, a1, PT_LBEG
371 s32i a3, a1, PT_LEND
372
c658eac6
CZ
373 /* Save optional registers. */
374
375 save_xtregs_opt a1 a2 a4 a5 a6 a7 PT_XTREGS_OPT
376
5a0015d6
CZ
377 /* Go to second-level dispatcher. Set up parameters to pass to the
378 * exception handler and call the exception handler.
379 */
380
381 movi a4, exc_table
382 mov a6, a1 # pass stack frame
383 mov a7, a0 # pass EXCCAUSE
384 addx4 a4, a0, a4
385 l32i a4, a4, EXC_TABLE_DEFAULT # load handler
386
387 /* Call the second-level handler */
388
389 callx4 a4
390
391 /* Jump here for exception exit */
392
393common_exception_return:
394
395 /* Jump if we are returning from kernel exceptions. */
396
3971: l32i a3, a1, PT_PS
e1088430 398 _bbci.l a3, PS_UM_BIT, 4f
5a0015d6
CZ
399
400 /* Specific to a user exception exit:
401 * We need to check some flags for signal handling and rescheduling,
402 * and have to restore WB and WS, extra states, and all registers
403 * in the register file that were in use in the user task.
e1088430 404 * Note that we don't disable interrupts here.
5a0015d6
CZ
405 */
406
407 GET_THREAD_INFO(a2,a1)
408 l32i a4, a2, TI_FLAGS
409
5a0015d6
CZ
410 _bbsi.l a4, TIF_NEED_RESCHED, 3f
411 _bbci.l a4, TIF_SIGPENDING, 4f
412
5a0015d6
CZ
413 l32i a4, a1, PT_DEPC
414 bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 4f
5a0015d6 415
e1088430
CZ
416 /* Call do_signal() */
417
5a0015d6
CZ
418 movi a4, do_signal # int do_signal(struct pt_regs*, sigset_t*)
419 mov a6, a1
420 movi a7, 0
421 callx4 a4
422 j 1b
423
e1088430 4243: /* Reschedule */
5a0015d6 425
5a0015d6
CZ
426 movi a4, schedule # void schedule (void)
427 callx4 a4
428 j 1b
429
e1088430
CZ
4304: /* Restore optional registers. */
431
432 load_xtregs_opt a1 a2 a4 a5 a6 a7 PT_XTREGS_OPT
5a0015d6 433
e1088430
CZ
434 wsr a3, PS /* disable interrupts */
435
436 _bbci.l a3, PS_UM_BIT, kernel_exception_exit
437
438user_exception_exit:
439
440 /* Restore the state of the task and return from the exception. */
5a0015d6 441
5a0015d6
CZ
442 /* Switch to the user thread WINDOWBASE. Save SP temporarily in DEPC */
443
444 l32i a2, a1, PT_WINDOWBASE
445 l32i a3, a1, PT_WINDOWSTART
446 wsr a1, DEPC # use DEPC as temp storage
447 wsr a3, WINDOWSTART # restore WINDOWSTART
448 ssr a2 # preserve user's WB in the SAR
449 wsr a2, WINDOWBASE # switch to user's saved WB
450 rsync
451 rsr a1, DEPC # restore stack pointer
452 l32i a2, a1, PT_WMASK # register frames saved (in bits 4...9)
453 rotw -1 # we restore a4..a7
454 _bltui a6, 16, 1f # only have to restore current window?
455
456 /* The working registers are a0 and a3. We are restoring to
457 * a4..a7. Be careful not to destroy what we have just restored.
458 * Note: wmask has the format YYYYM:
459 * Y: number of registers saved in groups of 4
460 * M: 4 bit mask of first 16 registers
461 */
462
463 mov a2, a6
464 mov a3, a5
465
4662: rotw -1 # a0..a3 become a4..a7
467 addi a3, a7, -4*4 # next iteration
468 addi a2, a6, -16 # decrementing Y in WMASK
469 l32i a4, a3, PT_AREG_END + 0
470 l32i a5, a3, PT_AREG_END + 4
471 l32i a6, a3, PT_AREG_END + 8
472 l32i a7, a3, PT_AREG_END + 12
473 _bgeui a2, 16, 2b
474
475 /* Clear unrestored registers (don't leak anything to user-land */
476
4771: rsr a0, WINDOWBASE
478 rsr a3, SAR
479 sub a3, a0, a3
480 beqz a3, 2f
481 extui a3, a3, 0, WBBITS
482
4831: rotw -1
484 addi a3, a7, -1
485 movi a4, 0
486 movi a5, 0
487 movi a6, 0
488 movi a7, 0
489 bgei a3, 1, 1b
490
491 /* We are back were we were when we started.
492 * Note: a2 still contains WMASK (if we've returned to the original
493 * frame where we had loaded a2), or at least the lower 4 bits
494 * (if we have restored WSBITS-1 frames).
495 */
496
4972: j common_exception_exit
498
499 /* This is the kernel exception exit.
500 * We avoided to do a MOVSP when we entered the exception, but we
501 * have to do it here.
502 */
503
504kernel_exception_exit:
505
5a0015d6
CZ
506#ifdef PREEMPTIBLE_KERNEL
507
508#ifdef CONFIG_PREEMPT
509
510 /*
511 * Note: We've just returned from a call4, so we have
512 * at least 4 addt'l regs.
513 */
514
515 /* Check current_thread_info->preempt_count */
516
517 GET_THREAD_INFO(a2)
518 l32i a3, a2, TI_PREEMPT
519 bnez a3, 1f
520
521 l32i a2, a2, TI_FLAGS
522
5231:
524
525#endif
526
527#endif
528
529 /* Check if we have to do a movsp.
530 *
531 * We only have to do a movsp if the previous window-frame has
532 * been spilled to the *temporary* exception stack instead of the
533 * task's stack. This is the case if the corresponding bit in
534 * WINDOWSTART for the previous window-frame was set before
535 * (not spilled) but is zero now (spilled).
536 * If this bit is zero, all other bits except the one for the
537 * current window frame are also zero. So, we can use a simple test:
538 * 'and' WINDOWSTART and WINDOWSTART-1:
539 *
540 * (XXXXXX1[0]* - 1) AND XXXXXX1[0]* = XXXXXX0[0]*
541 *
542 * The result is zero only if one bit was set.
543 *
544 * (Note: We might have gone through several task switches before
545 * we come back to the current task, so WINDOWBASE might be
546 * different from the time the exception occurred.)
547 */
548
549 /* Test WINDOWSTART before and after the exception.
550 * We actually have WMASK, so we only have to test if it is 1 or not.
551 */
552
553 l32i a2, a1, PT_WMASK
554 _beqi a2, 1, common_exception_exit # Spilled before exception,jump
555
556 /* Test WINDOWSTART now. If spilled, do the movsp */
557
558 rsr a3, WINDOWSTART
559 addi a0, a3, -1
560 and a3, a3, a0
561 _bnez a3, common_exception_exit
562
563 /* Do a movsp (we returned from a call4, so we have at least a0..a7) */
564
565 addi a0, a1, -16
566 l32i a3, a0, 0
567 l32i a4, a0, 4
568 s32i a3, a1, PT_SIZE+0
569 s32i a4, a1, PT_SIZE+4
570 l32i a3, a0, 8
571 l32i a4, a0, 12
572 s32i a3, a1, PT_SIZE+8
573 s32i a4, a1, PT_SIZE+12
574
575 /* Common exception exit.
576 * We restore the special register and the current window frame, and
577 * return from the exception.
578 *
579 * Note: We expect a2 to hold PT_WMASK
580 */
581
582common_exception_exit:
583
c658eac6
CZ
584 /* Restore address registers. */
585
5a0015d6
CZ
586 _bbsi.l a2, 1, 1f
587 l32i a4, a1, PT_AREG4
588 l32i a5, a1, PT_AREG5
589 l32i a6, a1, PT_AREG6
590 l32i a7, a1, PT_AREG7
591 _bbsi.l a2, 2, 1f
592 l32i a8, a1, PT_AREG8
593 l32i a9, a1, PT_AREG9
594 l32i a10, a1, PT_AREG10
595 l32i a11, a1, PT_AREG11
596 _bbsi.l a2, 3, 1f
597 l32i a12, a1, PT_AREG12
598 l32i a13, a1, PT_AREG13
599 l32i a14, a1, PT_AREG14
600 l32i a15, a1, PT_AREG15
601
602 /* Restore PC, SAR */
603
6041: l32i a2, a1, PT_PC
605 l32i a3, a1, PT_SAR
606 wsr a2, EPC_1
607 wsr a3, SAR
608
609 /* Restore LBEG, LEND, LCOUNT */
610
611 l32i a2, a1, PT_LBEG
612 l32i a3, a1, PT_LEND
613 wsr a2, LBEG
614 l32i a2, a1, PT_LCOUNT
615 wsr a3, LEND
616 wsr a2, LCOUNT
617
29c4dfd9
CZ
618 /* We control single stepping through the ICOUNTLEVEL register. */
619
620 l32i a2, a1, PT_ICOUNTLEVEL
621 movi a3, -2
622 wsr a2, ICOUNTLEVEL
623 wsr a3, ICOUNT
624
5a0015d6
CZ
625 /* Check if it was double exception. */
626
627 l32i a0, a1, PT_DEPC
628 l32i a3, a1, PT_AREG3
629 l32i a2, a1, PT_AREG2
630 _bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
631
632 /* Restore a0...a3 and return */
633
634 l32i a0, a1, PT_AREG0
635 l32i a1, a1, PT_AREG1
636 rfe
637
6381: wsr a0, DEPC
639 l32i a0, a1, PT_AREG0
640 l32i a1, a1, PT_AREG1
641 rfde
642
643/*
644 * Debug exception handler.
645 *
646 * Currently, we don't support KGDB, so only user application can be debugged.
647 *
648 * When we get here, a0 is trashed and saved to excsave[debuglevel]
649 */
650
651ENTRY(debug_exception)
652
653 rsr a0, EPS + XCHAL_DEBUGLEVEL
173d6681 654 bbsi.l a0, PS_EXCM_BIT, 1f # exception mode
5a0015d6
CZ
655
656 /* Set EPC_1 and EXCCAUSE */
657
658 wsr a2, DEPC # save a2 temporarily
659 rsr a2, EPC + XCHAL_DEBUGLEVEL
660 wsr a2, EPC_1
661
662 movi a2, EXCCAUSE_MAPPED_DEBUG
663 wsr a2, EXCCAUSE
664
665 /* Restore PS to the value before the debug exc but with PS.EXCM set.*/
666
173d6681 667 movi a2, 1 << PS_EXCM_BIT
5a0015d6
CZ
668 or a2, a0, a2
669 movi a0, debug_exception # restore a3, debug jump vector
670 wsr a2, PS
671 xsr a0, EXCSAVE + XCHAL_DEBUGLEVEL
672
673 /* Switch to kernel/user stack, restore jump vector, and save a0 */
674
173d6681 675 bbsi.l a2, PS_UM_BIT, 2f # jump if user mode
5a0015d6
CZ
676
677 addi a2, a1, -16-PT_SIZE # assume kernel stack
678 s32i a0, a2, PT_AREG0
679 movi a0, 0
680 s32i a1, a2, PT_AREG1
681 s32i a0, a2, PT_DEPC # mark it as a regular exception
682 xsr a0, DEPC
683 s32i a3, a2, PT_AREG3
684 s32i a0, a2, PT_AREG2
685 mov a1, a2
686 j _kernel_exception
687
6882: rsr a2, EXCSAVE_1
689 l32i a2, a2, EXC_TABLE_KSTK # load kernel stack pointer
690 s32i a0, a2, PT_AREG0
691 movi a0, 0
692 s32i a1, a2, PT_AREG1
693 s32i a0, a2, PT_DEPC
694 xsr a0, DEPC
695 s32i a3, a2, PT_AREG3
696 s32i a0, a2, PT_AREG2
697 mov a1, a2
698 j _user_exception
699
700 /* Debug exception while in exception mode. */
7011: j 1b // FIXME!!
702
703
704/*
705 * We get here in case of an unrecoverable exception.
706 * The only thing we can do is to be nice and print a panic message.
707 * We only produce a single stack frame for panic, so ???
708 *
709 *
710 * Entry conditions:
711 *
712 * - a0 contains the caller address; original value saved in excsave1.
713 * - the original a0 contains a valid return address (backtrace) or 0.
714 * - a2 contains a valid stackpointer
715 *
716 * Notes:
717 *
718 * - If the stack pointer could be invalid, the caller has to setup a
719 * dummy stack pointer (e.g. the stack of the init_task)
720 *
721 * - If the return address could be invalid, the caller has to set it
722 * to 0, so the backtrace would stop.
723 *
724 */
725 .align 4
726unrecoverable_text:
727 .ascii "Unrecoverable error in exception handler\0"
728
729ENTRY(unrecoverable_exception)
730
731 movi a0, 1
732 movi a1, 0
733
734 wsr a0, WINDOWSTART
735 wsr a1, WINDOWBASE
736 rsync
737
173d6681 738 movi a1, (1 << PS_WOE_BIT) | 1
5a0015d6
CZ
739 wsr a1, PS
740 rsync
741
742 movi a1, init_task
743 movi a0, 0
744 addi a1, a1, PT_REGS_OFFSET
745
746 movi a4, panic
747 movi a6, unrecoverable_text
748
749 callx4 a4
750
7511: j 1b
752
753
754/* -------------------------- FAST EXCEPTION HANDLERS ----------------------- */
755
756/*
757 * Fast-handler for alloca exceptions
758 *
759 * The ALLOCA handler is entered when user code executes the MOVSP
760 * instruction and the caller's frame is not in the register file.
761 * In this case, the caller frame's a0..a3 are on the stack just
762 * below sp (a1), and this handler moves them.
763 *
764 * For "MOVSP <ar>,<as>" without destination register a1, this routine
765 * simply moves the value from <as> to <ar> without moving the save area.
766 *
767 * Entry condition:
768 *
769 * a0: trashed, original value saved on stack (PT_AREG0)
770 * a1: a1
771 * a2: new stack pointer, original in DEPC
772 * a3: dispatch table
773 * depc: a2, original value saved on stack (PT_DEPC)
774 * excsave_1: a3
775 *
776 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
777 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
778 */
779
780#if XCHAL_HAVE_BE
781#define _EXTUI_MOVSP_SRC(ar) extui ar, ar, 4, 4
782#define _EXTUI_MOVSP_DST(ar) extui ar, ar, 0, 4
783#else
784#define _EXTUI_MOVSP_SRC(ar) extui ar, ar, 0, 4
785#define _EXTUI_MOVSP_DST(ar) extui ar, ar, 4, 4
786#endif
787
788ENTRY(fast_alloca)
789
790 /* We shouldn't be in a double exception. */
791
792 l32i a0, a2, PT_DEPC
793 _bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, .Lunhandled_double
794
795 rsr a0, DEPC # get a2
796 s32i a4, a2, PT_AREG4 # save a4 and
797 s32i a0, a2, PT_AREG2 # a2 to stack
798
799 /* Exit critical section. */
800
801 movi a0, 0
802 s32i a0, a3, EXC_TABLE_FIXUP
803
804 /* Restore a3, excsave_1 */
805
806 xsr a3, EXCSAVE_1 # make sure excsave_1 is valid for dbl.
807 rsr a4, EPC_1 # get exception address
808 s32i a3, a2, PT_AREG3 # save a3 to stack
809
810#ifdef ALLOCA_EXCEPTION_IN_IRAM
811#error iram not supported
812#else
813 /* Note: l8ui not allowed in IRAM/IROM!! */
814 l8ui a0, a4, 1 # read as(src) from MOVSP instruction
815#endif
816 movi a3, .Lmovsp_src
817 _EXTUI_MOVSP_SRC(a0) # extract source register number
818 addx8 a3, a0, a3
819 jx a3
820
821.Lunhandled_double:
822 wsr a0, EXCSAVE_1
823 movi a0, unrecoverable_exception
824 callx0 a0
825
826 .align 8
827.Lmovsp_src:
828 l32i a3, a2, PT_AREG0; _j 1f; .align 8
829 mov a3, a1; _j 1f; .align 8
830 l32i a3, a2, PT_AREG2; _j 1f; .align 8
831 l32i a3, a2, PT_AREG3; _j 1f; .align 8
832 l32i a3, a2, PT_AREG4; _j 1f; .align 8
833 mov a3, a5; _j 1f; .align 8
834 mov a3, a6; _j 1f; .align 8
835 mov a3, a7; _j 1f; .align 8
836 mov a3, a8; _j 1f; .align 8
837 mov a3, a9; _j 1f; .align 8
838 mov a3, a10; _j 1f; .align 8
839 mov a3, a11; _j 1f; .align 8
840 mov a3, a12; _j 1f; .align 8
841 mov a3, a13; _j 1f; .align 8
842 mov a3, a14; _j 1f; .align 8
843 mov a3, a15; _j 1f; .align 8
844
8451:
846
847#ifdef ALLOCA_EXCEPTION_IN_IRAM
848#error iram not supported
849#else
850 l8ui a0, a4, 0 # read ar(dst) from MOVSP instruction
851#endif
852 addi a4, a4, 3 # step over movsp
853 _EXTUI_MOVSP_DST(a0) # extract destination register
854 wsr a4, EPC_1 # save new epc_1
855
856 _bnei a0, 1, 1f # no 'movsp a1, ax': jump
857
858 /* Move the save area. This implies the use of the L32E
859 * and S32E instructions, because this move must be done with
860 * the user's PS.RING privilege levels, not with ring 0
861 * (kernel's) privileges currently active with PS.EXCM
862 * set. Note that we have stil registered a fixup routine with the
863 * double exception vector in case a double exception occurs.
864 */
865
866 /* a0,a4:avail a1:old user stack a2:exc. stack a3:new user stack. */
867
868 l32e a0, a1, -16
869 l32e a4, a1, -12
870 s32e a0, a3, -16
871 s32e a4, a3, -12
872 l32e a0, a1, -8
873 l32e a4, a1, -4
874 s32e a0, a3, -8
875 s32e a4, a3, -4
876
877 /* Restore stack-pointer and all the other saved registers. */
878
879 mov a1, a3
880
881 l32i a4, a2, PT_AREG4
882 l32i a3, a2, PT_AREG3
883 l32i a0, a2, PT_AREG0
884 l32i a2, a2, PT_AREG2
885 rfe
886
887 /* MOVSP <at>,<as> was invoked with <at> != a1.
888 * Because the stack pointer is not being modified,
889 * we should be able to just modify the pointer
890 * without moving any save area.
891 * The processor only traps these occurrences if the
892 * caller window isn't live, so unfortunately we can't
893 * use this as an alternate trap mechanism.
894 * So we just do the move. This requires that we
895 * resolve the destination register, not just the source,
896 * so there's some extra work.
897 * (PERHAPS NOT REALLY NEEDED, BUT CLEANER...)
898 */
899
900 /* a0 dst-reg, a1 user-stack, a2 stack, a3 value of src reg. */
901
9021: movi a4, .Lmovsp_dst
903 addx8 a4, a0, a4
904 jx a4
905
906 .align 8
907.Lmovsp_dst:
908 s32i a3, a2, PT_AREG0; _j 1f; .align 8
909 mov a1, a3; _j 1f; .align 8
910 s32i a3, a2, PT_AREG2; _j 1f; .align 8
911 s32i a3, a2, PT_AREG3; _j 1f; .align 8
912 s32i a3, a2, PT_AREG4; _j 1f; .align 8
913 mov a5, a3; _j 1f; .align 8
914 mov a6, a3; _j 1f; .align 8
915 mov a7, a3; _j 1f; .align 8
916 mov a8, a3; _j 1f; .align 8
917 mov a9, a3; _j 1f; .align 8
918 mov a10, a3; _j 1f; .align 8
919 mov a11, a3; _j 1f; .align 8
920 mov a12, a3; _j 1f; .align 8
921 mov a13, a3; _j 1f; .align 8
922 mov a14, a3; _j 1f; .align 8
923 mov a15, a3; _j 1f; .align 8
924
9251: l32i a4, a2, PT_AREG4
926 l32i a3, a2, PT_AREG3
927 l32i a0, a2, PT_AREG0
928 l32i a2, a2, PT_AREG2
929 rfe
930
931
932/*
933 * fast system calls.
934 *
935 * WARNING: The kernel doesn't save the entire user context before
936 * handling a fast system call. These functions are small and short,
937 * usually offering some functionality not available to user tasks.
938 *
939 * BE CAREFUL TO PRESERVE THE USER'S CONTEXT.
940 *
941 * Entry condition:
942 *
943 * a0: trashed, original value saved on stack (PT_AREG0)
944 * a1: a1
945 * a2: new stack pointer, original in DEPC
946 * a3: dispatch table
947 * depc: a2, original value saved on stack (PT_DEPC)
948 * excsave_1: a3
949 */
950
951ENTRY(fast_syscall_kernel)
952
953 /* Skip syscall. */
954
955 rsr a0, EPC_1
956 addi a0, a0, 3
957 wsr a0, EPC_1
958
959 l32i a0, a2, PT_DEPC
960 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_syscall_unrecoverable
961
962 rsr a0, DEPC # get syscall-nr
963 _beqz a0, fast_syscall_spill_registers
fc4fb2ad 964 _beqi a0, __NR_xtensa, fast_syscall_xtensa
5a0015d6
CZ
965
966 j kernel_exception
967
5a0015d6
CZ
968ENTRY(fast_syscall_user)
969
970 /* Skip syscall. */
971
972 rsr a0, EPC_1
973 addi a0, a0, 3
974 wsr a0, EPC_1
975
976 l32i a0, a2, PT_DEPC
977 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_syscall_unrecoverable
978
979 rsr a0, DEPC # get syscall-nr
980 _beqz a0, fast_syscall_spill_registers
fc4fb2ad 981 _beqi a0, __NR_xtensa, fast_syscall_xtensa
5a0015d6
CZ
982
983 j user_exception
984
985ENTRY(fast_syscall_unrecoverable)
986
987 /* Restore all states. */
988
989 l32i a0, a2, PT_AREG0 # restore a0
990 xsr a2, DEPC # restore a2, depc
991 rsr a3, EXCSAVE_1
992
993 wsr a0, EXCSAVE_1
994 movi a0, unrecoverable_exception
995 callx0 a0
996
997
998
999/*
1000 * sysxtensa syscall handler
1001 *
fc4fb2ad
CZ
1002 * int sysxtensa (SYS_XTENSA_ATOMIC_SET, ptr, val, unused);
1003 * int sysxtensa (SYS_XTENSA_ATOMIC_ADD, ptr, val, unused);
1004 * int sysxtensa (SYS_XTENSA_ATOMIC_EXG_ADD, ptr, val, unused);
1005 * int sysxtensa (SYS_XTENSA_ATOMIC_CMP_SWP, ptr, oldval, newval);
1006 * a2 a6 a3 a4 a5
5a0015d6
CZ
1007 *
1008 * Entry condition:
1009 *
fc4fb2ad 1010 * a0: a2 (syscall-nr), original value saved on stack (PT_AREG0)
5a0015d6 1011 * a1: a1
fc4fb2ad
CZ
1012 * a2: new stack pointer, original in a0 and DEPC
1013 * a3: dispatch table, original in excsave_1
1014 * a4..a15: unchanged
5a0015d6
CZ
1015 * depc: a2, original value saved on stack (PT_DEPC)
1016 * excsave_1: a3
1017 *
1018 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
1019 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
1020 *
1021 * Note: we don't have to save a2; a2 holds the return value
1022 *
1023 * We use the two macros TRY and CATCH:
1024 *
1025 * TRY adds an entry to the __ex_table fixup table for the immediately
1026 * following instruction.
1027 *
1028 * CATCH catches any exception that occurred at one of the preceeding TRY
1029 * statements and continues from there
1030 *
1031 * Usage TRY l32i a0, a1, 0
1032 * <other code>
1033 * done: rfe
1034 * CATCH <set return code>
1035 * j done
1036 */
1037
1038#define TRY \
1039 .section __ex_table, "a"; \
1040 .word 66f, 67f; \
1041 .text; \
104266:
1043
1044#define CATCH \
104567:
1046
fc4fb2ad 1047ENTRY(fast_syscall_xtensa)
5a0015d6 1048
fc4fb2ad 1049 xsr a3, EXCSAVE_1 # restore a3, excsave1
5a0015d6 1050
fc4fb2ad 1051 s32i a7, a2, PT_AREG7 # we need an additional register
5a0015d6 1052 movi a7, 4 # sizeof(unsigned int)
fc4fb2ad 1053 access_ok a3, a7, a0, a2, .Leac # a0: scratch reg, a2: sp
5a0015d6 1054
fc4fb2ad
CZ
1055 addi a6, a6, -1 # assuming SYS_XTENSA_ATOMIC_SET = 1
1056 _bgeui a6, SYS_XTENSA_COUNT - 1, .Lill
1057 _bnei a6, SYS_XTENSA_ATOMIC_CMP_SWP - 1, .Lnswp
5a0015d6 1058
fc4fb2ad 1059 /* Fall through for ATOMIC_CMP_SWP. */
5a0015d6
CZ
1060
1061.Lswp: /* Atomic compare and swap */
1062
fc4fb2ad
CZ
1063TRY l32i a0, a3, 0 # read old value
1064 bne a0, a4, 1f # same as old value? jump
1065TRY s32i a5, a3, 0 # different, modify value
1066 l32i a7, a2, PT_AREG7 # restore a7
1067 l32i a0, a2, PT_AREG0 # restore a0
1068 movi a2, 1 # and return 1
1069 addi a6, a6, 1 # restore a6 (really necessary?)
1070 rfe
5a0015d6 1071
fc4fb2ad
CZ
10721: l32i a7, a2, PT_AREG7 # restore a7
1073 l32i a0, a2, PT_AREG0 # restore a0
1074 movi a2, 0 # return 0 (note that we cannot set
1075 addi a6, a6, 1 # restore a6 (really necessary?)
1076 rfe
5a0015d6 1077
fc4fb2ad 1078.Lnswp: /* Atomic set, add, and exg_add. */
5a0015d6 1079
fc4fb2ad
CZ
1080TRY l32i a7, a3, 0 # orig
1081 add a0, a4, a7 # + arg
1082 moveqz a0, a4, a6 # set
1083TRY s32i a0, a3, 0 # write new value
5a0015d6 1084
fc4fb2ad 1085 mov a0, a2
5a0015d6 1086 mov a2, a7
fc4fb2ad
CZ
1087 l32i a7, a0, PT_AREG7 # restore a7
1088 l32i a0, a0, PT_AREG0 # restore a0
1089 addi a6, a6, 1 # restore a6 (really necessary?)
5a0015d6
CZ
1090 rfe
1091
1092CATCH
fc4fb2ad
CZ
1093.Leac: l32i a7, a2, PT_AREG7 # restore a7
1094 l32i a0, a2, PT_AREG0 # restore a0
1095 movi a2, -EFAULT
1096 rfe
1097
1098.Lill: l32i a7, a2, PT_AREG0 # restore a7
1099 l32i a0, a2, PT_AREG0 # restore a0
1100 movi a2, -EINVAL
1101 rfe
1102
5a0015d6
CZ
1103
1104
1105
1106/* fast_syscall_spill_registers.
1107 *
1108 * Entry condition:
1109 *
1110 * a0: trashed, original value saved on stack (PT_AREG0)
1111 * a1: a1
1112 * a2: new stack pointer, original in DEPC
1113 * a3: dispatch table
1114 * depc: a2, original value saved on stack (PT_DEPC)
1115 * excsave_1: a3
1116 *
1117 * Note: We assume the stack pointer is EXC_TABLE_KSTK in the fixup handler.
5a0015d6
CZ
1118 */
1119
1120ENTRY(fast_syscall_spill_registers)
1121
1122 /* Register a FIXUP handler (pass current wb as a parameter) */
1123
1124 movi a0, fast_syscall_spill_registers_fixup
1125 s32i a0, a3, EXC_TABLE_FIXUP
1126 rsr a0, WINDOWBASE
1127 s32i a0, a3, EXC_TABLE_PARAM
1128
1129 /* Save a3 and SAR on stack. */
1130
1131 rsr a0, SAR
1132 xsr a3, EXCSAVE_1 # restore a3 and excsave_1
5a0015d6 1133 s32i a3, a2, PT_AREG3
c658eac6
CZ
1134 s32i a4, a2, PT_AREG4
1135 s32i a0, a2, PT_AREG5 # store SAR to PT_AREG5
5a0015d6
CZ
1136
1137 /* The spill routine might clobber a7, a11, and a15. */
1138
c658eac6
CZ
1139 s32i a7, a2, PT_AREG7
1140 s32i a11, a2, PT_AREG11
1141 s32i a15, a2, PT_AREG15
5a0015d6 1142
c658eac6 1143 call0 _spill_registers # destroys a3, a4, and SAR
5a0015d6
CZ
1144
1145 /* Advance PC, restore registers and SAR, and return from exception. */
1146
c658eac6
CZ
1147 l32i a3, a2, PT_AREG5
1148 l32i a4, a2, PT_AREG4
5a0015d6
CZ
1149 l32i a0, a2, PT_AREG0
1150 wsr a3, SAR
1151 l32i a3, a2, PT_AREG3
1152
1153 /* Restore clobbered registers. */
1154
c658eac6
CZ
1155 l32i a7, a2, PT_AREG7
1156 l32i a11, a2, PT_AREG11
1157 l32i a15, a2, PT_AREG15
5a0015d6
CZ
1158
1159 movi a2, 0
1160 rfe
1161
1162/* Fixup handler.
1163 *
1164 * We get here if the spill routine causes an exception, e.g. tlb miss.
1165 * We basically restore WINDOWBASE and WINDOWSTART to the condition when
1166 * we entered the spill routine and jump to the user exception handler.
1167 *
1168 * a0: value of depc, original value in depc
1169 * a2: trashed, original value in EXC_TABLE_DOUBLE_SAVE
1170 * a3: exctable, original value in excsave1
1171 */
1172
1173fast_syscall_spill_registers_fixup:
1174
1175 rsr a2, WINDOWBASE # get current windowbase (a2 is saved)
1176 xsr a0, DEPC # restore depc and a0
1177 ssl a2 # set shift (32 - WB)
1178
1179 /* We need to make sure the current registers (a0-a3) are preserved.
1180 * To do this, we simply set the bit for the current window frame
1181 * in WS, so that the exception handlers save them to the task stack.
1182 */
1183
1184 rsr a3, EXCSAVE_1 # get spill-mask
1185 slli a2, a3, 1 # shift left by one
1186
1187 slli a3, a2, 32-WSBITS
1188 src a2, a2, a3 # a1 = xxwww1yyxxxwww1yy......
1189 wsr a2, WINDOWSTART # set corrected windowstart
1190
1191 movi a3, exc_table
1192 l32i a2, a3, EXC_TABLE_DOUBLE_SAVE # restore a2
1193 l32i a3, a3, EXC_TABLE_PARAM # original WB (in user task)
1194
1195 /* Return to the original (user task) WINDOWBASE.
1196 * We leave the following frame behind:
1197 * a0, a1, a2 same
1198 * a3: trashed (saved in excsave_1)
1199 * depc: depc (we have to return to that address)
1200 * excsave_1: a3
1201 */
1202
1203 wsr a3, WINDOWBASE
1204 rsync
1205
1206 /* We are now in the original frame when we entered _spill_registers:
1207 * a0: return address
1208 * a1: used, stack pointer
1209 * a2: kernel stack pointer
1210 * a3: available, saved in EXCSAVE_1
1211 * depc: exception address
1212 * excsave: a3
1213 * Note: This frame might be the same as above.
1214 */
1215
5a0015d6
CZ
1216 /* Setup stack pointer. */
1217
1218 addi a2, a2, -PT_USER_SIZE
1219 s32i a0, a2, PT_AREG0
1220
1221 /* Make sure we return to this fixup handler. */
1222
1223 movi a3, fast_syscall_spill_registers_fixup_return
1224 s32i a3, a2, PT_DEPC # setup depc
1225
1226 /* Jump to the exception handler. */
1227
1228 movi a3, exc_table
1229 rsr a0, EXCCAUSE
c658eac6
CZ
1230 addx4 a0, a0, a3 # find entry in table
1231 l32i a0, a0, EXC_TABLE_FAST_USER # load handler
1232 jx a0
5a0015d6
CZ
1233
1234fast_syscall_spill_registers_fixup_return:
1235
1236 /* When we return here, all registers have been restored (a2: DEPC) */
1237
1238 wsr a2, DEPC # exception address
1239
1240 /* Restore fixup handler. */
1241
1242 xsr a3, EXCSAVE_1
1243 movi a2, fast_syscall_spill_registers_fixup
1244 s32i a2, a3, EXC_TABLE_FIXUP
1245 rsr a2, WINDOWBASE
1246 s32i a2, a3, EXC_TABLE_PARAM
1247 l32i a2, a3, EXC_TABLE_KSTK
1248
5a0015d6
CZ
1249 /* Load WB at the time the exception occurred. */
1250
1251 rsr a3, SAR # WB is still in SAR
1252 neg a3, a3
1253 wsr a3, WINDOWBASE
1254 rsync
1255
1256 /* Restore a3 and return. */
1257
1258 movi a3, exc_table
1259 xsr a3, EXCSAVE_1
1260
1261 rfde
1262
1263
1264/*
1265 * spill all registers.
1266 *
1267 * This is not a real function. The following conditions must be met:
1268 *
1269 * - must be called with call0.
c658eac6 1270 * - uses a3, a4 and SAR.
5a0015d6
CZ
1271 * - the last 'valid' register of each frame are clobbered.
1272 * - the caller must have registered a fixup handler
1273 * (or be inside a critical section)
1274 * - PS_EXCM must be set (PS_WOE cleared?)
1275 */
1276
1277ENTRY(_spill_registers)
1278
1279 /*
1280 * Rotate ws so that the current windowbase is at bit 0.
1281 * Assume ws = xxxwww1yy (www1 current window frame).
c658eac6 1282 * Rotate ws right so that a4 = yyxxxwww1.
5a0015d6
CZ
1283 */
1284
c658eac6 1285 rsr a4, WINDOWBASE
ea0b6b06 1286 rsr a3, WINDOWSTART # a3 = xxxwww1yy
c658eac6
CZ
1287 ssr a4 # holds WB
1288 slli a4, a3, WSBITS
1289 or a3, a3, a4 # a3 = xxxwww1yyxxxwww1yy
ea0b6b06 1290 srl a3, a3 # a3 = 00xxxwww1yyxxxwww1
5a0015d6
CZ
1291
1292 /* We are done if there are no more than the current register frame. */
1293
50c0716a 1294 extui a3, a3, 1, WSBITS-1 # a3 = 0yyxxxwww
c658eac6 1295 movi a4, (1 << (WSBITS-1))
5a0015d6
CZ
1296 _beqz a3, .Lnospill # only one active frame? jump
1297
1298 /* We want 1 at the top, so that we return to the current windowbase */
1299
c658eac6 1300 or a3, a3, a4 # 1yyxxxwww
5a0015d6
CZ
1301
1302 /* Skip empty frames - get 'oldest' WINDOWSTART-bit. */
1303
1304 wsr a3, WINDOWSTART # save shifted windowstart
c658eac6
CZ
1305 neg a4, a3
1306 and a3, a4, a3 # first bit set from right: 000010000
5a0015d6 1307
c658eac6 1308 ffs_ws a4, a3 # a4: shifts to skip empty frames
5a0015d6 1309 movi a3, WSBITS
c658eac6
CZ
1310 sub a4, a3, a4 # WSBITS-a4:number of 0-bits from right
1311 ssr a4 # save in SAR for later.
5a0015d6
CZ
1312
1313 rsr a3, WINDOWBASE
c658eac6 1314 add a3, a3, a4
5a0015d6
CZ
1315 wsr a3, WINDOWBASE
1316 rsync
1317
1318 rsr a3, WINDOWSTART
1319 srl a3, a3 # shift windowstart
1320
1321 /* WB is now just one frame below the oldest frame in the register
1322 window. WS is shifted so the oldest frame is in bit 0, thus, WB
1323 and WS differ by one 4-register frame. */
1324
1325 /* Save frames. Depending what call was used (call4, call8, call12),
1326 * we have to save 4,8. or 12 registers.
1327 */
1328
1329 _bbsi.l a3, 1, .Lc4
1330 _bbsi.l a3, 2, .Lc8
1331
1332 /* Special case: we have a call12-frame starting at a4. */
1333
1334 _bbci.l a3, 3, .Lc12 # bit 3 shouldn't be zero! (Jump to Lc12 first)
1335
1336 s32e a4, a1, -16 # a1 is valid with an empty spill area
1337 l32e a4, a5, -12
1338 s32e a8, a4, -48
1339 mov a8, a4
1340 l32e a4, a1, -16
1341 j .Lc12c
1342
50c0716a 1343.Lnospill:
ea0b6b06 1344 ret
50c0716a 1345
5a0015d6
CZ
1346.Lloop: _bbsi.l a3, 1, .Lc4
1347 _bbci.l a3, 2, .Lc12
1348
1349.Lc8: s32e a4, a13, -16
1350 l32e a4, a5, -12
1351 s32e a8, a4, -32
1352 s32e a5, a13, -12
1353 s32e a6, a13, -8
1354 s32e a7, a13, -4
1355 s32e a9, a4, -28
1356 s32e a10, a4, -24
1357 s32e a11, a4, -20
1358
1359 srli a11, a3, 2 # shift windowbase by 2
1360 rotw 2
1361 _bnei a3, 1, .Lloop
1362
1363.Lexit: /* Done. Do the final rotation, set WS, and return. */
1364
1365 rotw 1
1366 rsr a3, WINDOWBASE
1367 ssl a3
1368 movi a3, 1
1369 sll a3, a3
1370 wsr a3, WINDOWSTART
ea0b6b06 1371 ret
5a0015d6
CZ
1372
1373.Lc4: s32e a4, a9, -16
1374 s32e a5, a9, -12
1375 s32e a6, a9, -8
1376 s32e a7, a9, -4
1377
1378 srli a7, a3, 1
1379 rotw 1
1380 _bnei a3, 1, .Lloop
1381 j .Lexit
1382
1383.Lc12: _bbci.l a3, 3, .Linvalid_mask # bit 2 shouldn't be zero!
1384
1385 /* 12-register frame (call12) */
1386
1387 l32e a2, a5, -12
1388 s32e a8, a2, -48
1389 mov a8, a2
1390
1391.Lc12c: s32e a9, a8, -44
1392 s32e a10, a8, -40
1393 s32e a11, a8, -36
1394 s32e a12, a8, -32
1395 s32e a13, a8, -28
1396 s32e a14, a8, -24
1397 s32e a15, a8, -20
1398 srli a15, a3, 3
1399
1400 /* The stack pointer for a4..a7 is out of reach, so we rotate the
1401 * window, grab the stackpointer, and rotate back.
1402 * Alternatively, we could also use the following approach, but that
1403 * makes the fixup routine much more complicated:
1404 * rotw 1
1405 * s32e a0, a13, -16
1406 * ...
1407 * rotw 2
1408 */
1409
1410 rotw 1
1411 mov a5, a13
1412 rotw -1
1413
1414 s32e a4, a9, -16
1415 s32e a5, a9, -12
1416 s32e a6, a9, -8
1417 s32e a7, a9, -4
1418
1419 rotw 3
1420
1421 _beqi a3, 1, .Lexit
1422 j .Lloop
1423
1424.Linvalid_mask:
1425
1426 /* We get here because of an unrecoverable error in the window
1427 * registers. If we are in user space, we kill the application,
1428 * however, this condition is unrecoverable in kernel space.
1429 */
1430
1431 rsr a0, PS
173d6681 1432 _bbci.l a0, PS_UM_BIT, 1f
5a0015d6
CZ
1433
1434 /* User space: Setup a dummy frame and kill application.
1435 * Note: We assume EXC_TABLE_KSTK contains a valid stack pointer.
1436 */
1437
1438 movi a0, 1
1439 movi a1, 0
1440
1441 wsr a0, WINDOWSTART
1442 wsr a1, WINDOWBASE
1443 rsync
1444
1445 movi a0, 0
1446
1447 movi a3, exc_table
1448 l32i a1, a3, EXC_TABLE_KSTK
1449 wsr a3, EXCSAVE_1
1450
173d6681 1451 movi a4, (1 << PS_WOE_BIT) | 1
5a0015d6
CZ
1452 wsr a4, PS
1453 rsync
1454
1455 movi a6, SIGSEGV
1456 movi a4, do_exit
1457 callx4 a4
1458
14591: /* Kernel space: PANIC! */
1460
1461 wsr a0, EXCSAVE_1
1462 movi a0, unrecoverable_exception
1463 callx0 a0 # should not return
14641: j 1b
1465
1466/*
1467 * We should never get here. Bail out!
1468 */
1469
1470ENTRY(fast_second_level_miss_double_kernel)
1471
14721: movi a0, unrecoverable_exception
1473 callx0 a0 # should not return
14741: j 1b
1475
1476/* First-level entry handler for user, kernel, and double 2nd-level
1477 * TLB miss exceptions. Note that for now, user and kernel miss
1478 * exceptions share the same entry point and are handled identically.
1479 *
1480 * An old, less-efficient C version of this function used to exist.
1481 * We include it below, interleaved as comments, for reference.
1482 *
1483 * Entry condition:
1484 *
1485 * a0: trashed, original value saved on stack (PT_AREG0)
1486 * a1: a1
1487 * a2: new stack pointer, original in DEPC
1488 * a3: dispatch table
1489 * depc: a2, original value saved on stack (PT_DEPC)
1490 * excsave_1: a3
1491 *
1492 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
1493 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
1494 */
1495
1496ENTRY(fast_second_level_miss)
1497
1498 /* Save a1. Note: we don't expect a double exception. */
1499
1500 s32i a1, a2, PT_AREG1
1501
1502 /* We need to map the page of PTEs for the user task. Find
1503 * the pointer to that page. Also, it's possible for tsk->mm
1504 * to be NULL while tsk->active_mm is nonzero if we faulted on
1505 * a vmalloc address. In that rare case, we must use
1506 * active_mm instead to avoid a fault in this handler. See
1507 *
1508 * http://mail.nl.linux.org/linux-mm/2002-08/msg00258.html
1509 * (or search Internet on "mm vs. active_mm")
1510 *
1511 * if (!mm)
1512 * mm = tsk->active_mm;
1513 * pgd = pgd_offset (mm, regs->excvaddr);
1514 * pmd = pmd_offset (pgd, regs->excvaddr);
1515 * pmdval = *pmd;
1516 */
1517
1518 GET_CURRENT(a1,a2)
1519 l32i a0, a1, TASK_MM # tsk->mm
1520 beqz a0, 9f
1521
01858d1b
CZ
1522
1523 /* We deliberately destroy a3 that holds the exception table. */
1524
15258: rsr a3, EXCVADDR # fault address
1526 _PGD_OFFSET(a0, a3, a1)
5a0015d6 1527 l32i a0, a0, 0 # read pmdval
5a0015d6
CZ
1528 beqz a0, 2f
1529
1530 /* Read ptevaddr and convert to top of page-table page.
1531 *
1532 * vpnval = read_ptevaddr_register() & PAGE_MASK;
1533 * vpnval += DTLB_WAY_PGTABLE;
1534 * pteval = mk_pte (virt_to_page(pmd_val(pmdval)), PAGE_KERNEL);
1535 * write_dtlb_entry (pteval, vpnval);
1536 *
1537 * The messy computation for 'pteval' above really simplifies
1538 * into the following:
1539 *
6656920b 1540 * pteval = ((pmdval - PAGE_OFFSET) & PAGE_MASK) | PAGE_DIRECTORY
5a0015d6
CZ
1541 */
1542
1543 movi a1, -PAGE_OFFSET
1544 add a0, a0, a1 # pmdval - PAGE_OFFSET
1545 extui a1, a0, 0, PAGE_SHIFT # ... & PAGE_MASK
1546 xor a0, a0, a1
1547
01858d1b 1548 movi a1, _PAGE_DIRECTORY
5a0015d6
CZ
1549 or a0, a0, a1 # ... | PAGE_DIRECTORY
1550
01858d1b 1551 /*
6656920b 1552 * We utilize all three wired-ways (7-9) to hold pmd translations.
01858d1b
CZ
1553 * Memory regions are mapped to the DTLBs according to bits 28 and 29.
1554 * This allows to map the three most common regions to three different
1555 * DTLBs:
1556 * 0,1 -> way 7 program (0040.0000) and virtual (c000.0000)
1557 * 2 -> way 8 shared libaries (2000.0000)
1558 * 3 -> way 0 stack (3000.0000)
1559 */
1560
1561 extui a3, a3, 28, 2 # addr. bit 28 and 29 0,1,2,3
5a0015d6 1562 rsr a1, PTEVADDR
01858d1b 1563 addx2 a3, a3, a3 # -> 0,3,6,9
5a0015d6 1564 srli a1, a1, PAGE_SHIFT
01858d1b 1565 extui a3, a3, 2, 2 # -> 0,0,1,2
5a0015d6 1566 slli a1, a1, PAGE_SHIFT # ptevaddr & PAGE_MASK
01858d1b
CZ
1567 addi a3, a3, DTLB_WAY_PGD
1568 add a1, a1, a3 # ... + way_number
5a0015d6 1569
01858d1b 15703: wdtlb a0, a1
5a0015d6
CZ
1571 dsync
1572
1573 /* Exit critical section. */
1574
01858d1b 15754: movi a3, exc_table # restore a3
5a0015d6
CZ
1576 movi a0, 0
1577 s32i a0, a3, EXC_TABLE_FIXUP
1578
1579 /* Restore the working registers, and return. */
1580
1581 l32i a0, a2, PT_AREG0
1582 l32i a1, a2, PT_AREG1
1583 l32i a2, a2, PT_DEPC
1584 xsr a3, EXCSAVE_1
1585
1586 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
1587
1588 /* Restore excsave1 and return. */
1589
1590 rsr a2, DEPC
1591 rfe
1592
1593 /* Return from double exception. */
1594
15951: xsr a2, DEPC
1596 esync
1597 rfde
1598
15999: l32i a0, a1, TASK_ACTIVE_MM # unlikely case mm == 0
1600 j 8b
1601
6656920b
CZ
1602#if (DCACHE_WAY_SIZE > PAGE_SIZE)
1603
16042: /* Special case for cache aliasing.
1605 * We (should) only get here if a clear_user_page, copy_user_page
1606 * or the aliased cache flush functions got preemptively interrupted
1607 * by another task. Re-establish temporary mapping to the
1608 * TLBTEMP_BASE areas.
1609 */
1610
1611 /* We shouldn't be in a double exception */
1612
1613 l32i a0, a2, PT_DEPC
1614 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, 2f
1615
1616 /* Make sure the exception originated in the special functions */
1617
1618 movi a0, __tlbtemp_mapping_start
1619 rsr a3, EPC_1
1620 bltu a3, a0, 2f
1621 movi a0, __tlbtemp_mapping_end
1622 bgeu a3, a0, 2f
1623
1624 /* Check if excvaddr was in one of the TLBTEMP_BASE areas. */
1625
1626 movi a3, TLBTEMP_BASE_1
1627 rsr a0, EXCVADDR
1628 bltu a0, a3, 2f
1629
1630 addi a1, a0, -(2 << (DCACHE_ALIAS_ORDER + PAGE_SHIFT))
1631 bgeu a1, a3, 2f
1632
1633 /* Check if we have to restore an ITLB mapping. */
1634
1635 movi a1, __tlbtemp_mapping_itlb
1636 rsr a3, EPC_1
1637 sub a3, a3, a1
1638
1639 /* Calculate VPN */
1640
1641 movi a1, PAGE_MASK
1642 and a1, a1, a0
1643
1644 /* Jump for ITLB entry */
1645
1646 bgez a3, 1f
1647
1648 /* We can use up to two TLBTEMP areas, one for src and one for dst. */
1649
1650 extui a3, a0, PAGE_SHIFT + DCACHE_ALIAS_ORDER, 1
1651 add a1, a3, a1
1652
1653 /* PPN is in a6 for the first TLBTEMP area and in a7 for the second. */
1654
1655 mov a0, a6
1656 movnez a0, a7, a3
1657 j 3b
1658
1659 /* ITLB entry. We only use dst in a6. */
1660
16611: witlb a6, a1
1662 isync
1663 j 4b
1664
1665
1666#endif // DCACHE_WAY_SIZE > PAGE_SIZE
1667
1668
5a0015d6
CZ
16692: /* Invalid PGD, default exception handling */
1670
01858d1b 1671 movi a3, exc_table
5a0015d6
CZ
1672 rsr a1, DEPC
1673 xsr a3, EXCSAVE_1
1674 s32i a1, a2, PT_AREG2
1675 s32i a3, a2, PT_AREG3
1676 mov a1, a2
1677
1678 rsr a2, PS
173d6681 1679 bbsi.l a2, PS_UM_BIT, 1f
5a0015d6
CZ
1680 j _kernel_exception
16811: j _user_exception
1682
1683
1684/*
1685 * StoreProhibitedException
1686 *
1687 * Update the pte and invalidate the itlb mapping for this pte.
1688 *
1689 * Entry condition:
1690 *
1691 * a0: trashed, original value saved on stack (PT_AREG0)
1692 * a1: a1
1693 * a2: new stack pointer, original in DEPC
1694 * a3: dispatch table
1695 * depc: a2, original value saved on stack (PT_DEPC)
1696 * excsave_1: a3
1697 *
1698 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
1699 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
1700 */
1701
1702ENTRY(fast_store_prohibited)
1703
1704 /* Save a1 and a4. */
1705
1706 s32i a1, a2, PT_AREG1
1707 s32i a4, a2, PT_AREG4
1708
1709 GET_CURRENT(a1,a2)
1710 l32i a0, a1, TASK_MM # tsk->mm
1711 beqz a0, 9f
1712
17138: rsr a1, EXCVADDR # fault address
1714 _PGD_OFFSET(a0, a1, a4)
1715 l32i a0, a0, 0
5a0015d6
CZ
1716 beqz a0, 2f
1717
01858d1b
CZ
1718 /* Note that we assume _PAGE_WRITABLE_BIT is only set if pte is valid.*/
1719
5a0015d6
CZ
1720 _PTE_OFFSET(a0, a1, a4)
1721 l32i a4, a0, 0 # read pteval
01858d1b 1722 bbci.l a4, _PAGE_WRITABLE_BIT, 2f
5a0015d6 1723
01858d1b 1724 movi a1, _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_HW_WRITE
5a0015d6
CZ
1725 or a4, a4, a1
1726 rsr a1, EXCVADDR
1727 s32i a4, a0, 0
1728
1729 /* We need to flush the cache if we have page coloring. */
1730#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK
1731 dhwb a0, 0
1732#endif
1733 pdtlb a0, a1
5a0015d6 1734 wdtlb a4, a0
5a0015d6
CZ
1735
1736 /* Exit critical section. */
1737
1738 movi a0, 0
1739 s32i a0, a3, EXC_TABLE_FIXUP
1740
1741 /* Restore the working registers, and return. */
1742
1743 l32i a4, a2, PT_AREG4
1744 l32i a1, a2, PT_AREG1
1745 l32i a0, a2, PT_AREG0
1746 l32i a2, a2, PT_DEPC
1747
1748 /* Restore excsave1 and a3. */
1749
1750 xsr a3, EXCSAVE_1
1751 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
1752
1753 rsr a2, DEPC
1754 rfe
1755
1756 /* Double exception. Restore FIXUP handler and return. */
1757
17581: xsr a2, DEPC
1759 esync
1760 rfde
1761
17629: l32i a0, a1, TASK_ACTIVE_MM # unlikely case mm == 0
1763 j 8b
1764
17652: /* If there was a problem, handle fault in C */
1766
1767 rsr a4, DEPC # still holds a2
1768 xsr a3, EXCSAVE_1
1769 s32i a4, a2, PT_AREG2
1770 s32i a3, a2, PT_AREG3
1771 l32i a4, a2, PT_AREG4
1772 mov a1, a2
1773
1774 rsr a2, PS
173d6681 1775 bbsi.l a2, PS_UM_BIT, 1f
5a0015d6
CZ
1776 j _kernel_exception
17771: j _user_exception
1778
1779
fc4fb2ad
CZ
1780/*
1781 * System Calls.
1782 *
1783 * void system_call (struct pt_regs* regs, int exccause)
1784 * a2 a3
1785 */
1786
1787ENTRY(system_call)
1788 entry a1, 32
1789
1790 /* regs->syscall = regs->areg[2] */
1791
1792 l32i a3, a2, PT_AREG2
1793 mov a6, a2
1794 movi a4, do_syscall_trace_enter
1795 s32i a3, a2, PT_SYSCALL
1796 callx4 a4
1797
1798 /* syscall = sys_call_table[syscall_nr] */
1799
1800 movi a4, sys_call_table;
1801 movi a5, __NR_syscall_count
1802 movi a6, -ENOSYS
1803 bgeu a3, a5, 1f
1804
1805 addx4 a4, a3, a4
1806 l32i a4, a4, 0
1807 movi a5, sys_ni_syscall;
1808 beq a4, a5, 1f
1809
1810 /* Load args: arg0 - arg5 are passed via regs. */
1811
1812 l32i a6, a2, PT_AREG6
1813 l32i a7, a2, PT_AREG3
1814 l32i a8, a2, PT_AREG4
1815 l32i a9, a2, PT_AREG5
1816 l32i a10, a2, PT_AREG8
1817 l32i a11, a2, PT_AREG9
1818
1819 /* Pass one additional argument to the syscall: pt_regs (on stack) */
1820 s32i a2, a1, 0
1821
1822 callx4 a4
1823
18241: /* regs->areg[2] = return_value */
1825
1826 s32i a6, a2, PT_AREG2
1827 movi a4, do_syscall_trace_leave
1828 mov a6, a2
1829 callx4 a4
1830 retw
1831
1832
1833/*
1834 * Create a kernel thread
1835 *
1836 * int kernel_thread(int (*fn)(void *), void *arg, unsigned long flags)
1837 * a2 a2 a3 a4
1838 */
1839
1840ENTRY(kernel_thread)
1841 entry a1, 16
1842
1843 mov a5, a2 # preserve fn over syscall
1844 mov a7, a3 # preserve args over syscall
1845
1846 movi a3, _CLONE_VM | _CLONE_UNTRACED
1847 movi a2, __NR_clone
1848 or a6, a4, a3 # arg0: flags
1849 mov a3, a1 # arg1: sp
1850 syscall
1851
1852 beq a3, a1, 1f # branch if parent
1853 mov a6, a7 # args
1854 callx4 a5 # fn(args)
1855
1856 movi a2, __NR_exit
1857 syscall # return value of fn(args) still in a6
1858
18591: retw
1860
1861/*
1862 * Do a system call from kernel instead of calling sys_execve, so we end up
1863 * with proper pt_regs.
1864 *
1865 * int kernel_execve(const char *fname, char *const argv[], charg *const envp[])
1866 * a2 a2 a3 a4
1867 */
1868
1869ENTRY(kernel_execve)
1870 entry a1, 16
1871 mov a6, a2 # arg0 is in a6
1872 movi a2, __NR_execve
1873 syscall
1874
1875 retw
1876
5a0015d6
CZ
1877/*
1878 * Task switch.
1879 *
1880 * struct task* _switch_to (struct task* prev, struct task* next)
1881 * a2 a2 a3
1882 */
1883
1884ENTRY(_switch_to)
1885
1886 entry a1, 16
1887
c658eac6
CZ
1888 mov a12, a2 # preserve 'prev' (a2)
1889 mov a13, a3 # and 'next' (a3)
5a0015d6 1890
c658eac6
CZ
1891 l32i a4, a2, TASK_THREAD_INFO
1892 l32i a5, a3, TASK_THREAD_INFO
5a0015d6 1893
c658eac6 1894 save_xtregs_user a4 a6 a8 a9 a10 a11 THREAD_XTREGS_USER
5a0015d6 1895
c658eac6
CZ
1896 s32i a0, a12, THREAD_RA # save return address
1897 s32i a1, a12, THREAD_SP # save stack pointer
1898
1899 /* Disable ints while we manipulate the stack pointer. */
1900
1901 movi a14, (1 << PS_EXCM_BIT) | LOCKLEVEL
1902 xsr a14, PS
5a0015d6
CZ
1903 rsr a3, EXCSAVE_1
1904 rsync
1905 s32i a3, a3, EXC_TABLE_FIXUP /* enter critical section */
1906
c658eac6
CZ
1907 /* Switch CPENABLE */
1908
1909#if (XTENSA_HAVE_COPROCESSORS || XTENSA_HAVE_IO_PORTS)
1910 l32i a3, a5, THREAD_CPENABLE
1911 xsr a3, CPENABLE
1912 s32i a3, a4, THREAD_CPENABLE
1913#endif
1914
1915 /* Flush register file. */
1916
1917 call0 _spill_registers # destroys a3, a4, and SAR
5a0015d6
CZ
1918
1919 /* Set kernel stack (and leave critical section)
1920 * Note: It's save to set it here. The stack will not be overwritten
1921 * because the kernel stack will only be loaded again after
1922 * we return from kernel space.
1923 */
1924
5a0015d6 1925 rsr a3, EXCSAVE_1 # exc_table
c658eac6
CZ
1926 movi a6, 0
1927 addi a7, a5, PT_REGS_OFFSET
1928 s32i a6, a3, EXC_TABLE_FIXUP
1929 s32i a7, a3, EXC_TABLE_KSTK
5a0015d6
CZ
1930
1931 /* restore context of the task that 'next' addresses */
1932
c658eac6
CZ
1933 l32i a0, a13, THREAD_RA # restore return address
1934 l32i a1, a13, THREAD_SP # restore stack pointer
1935
1936 load_xtregs_user a5 a6 a8 a9 a10 a11 THREAD_XTREGS_USER
5a0015d6 1937
c658eac6
CZ
1938 wsr a14, PS
1939 mov a2, a12 # return 'prev'
5a0015d6
CZ
1940 rsync
1941
1942 retw
1943
1944
1945ENTRY(ret_from_fork)
1946
1947 /* void schedule_tail (struct task_struct *prev)
1948 * Note: prev is still in a6 (return value from fake call4 frame)
1949 */
1950 movi a4, schedule_tail
1951 callx4 a4
1952
fc4fb2ad
CZ
1953 movi a4, do_syscall_trace_leave
1954 mov a6, a1
5a0015d6
CZ
1955 callx4 a4
1956
1957 j common_exception_return
1958