]>
Commit | Line | Data |
---|---|---|
867e359b CM |
1 | /* |
2 | * Copyright 2010 Tilera Corporation. All Rights Reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation, version 2. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or | |
11 | * NON INFRINGEMENT. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
14 | * Linux interrupt vectors. | |
15 | */ | |
16 | ||
17 | #include <linux/linkage.h> | |
18 | #include <linux/errno.h> | |
19 | #include <linux/init.h> | |
9f9c0382 | 20 | #include <linux/unistd.h> |
867e359b CM |
21 | #include <asm/ptrace.h> |
22 | #include <asm/thread_info.h> | |
867e359b CM |
23 | #include <asm/irqflags.h> |
24 | #include <asm/atomic.h> | |
25 | #include <asm/asm-offsets.h> | |
26 | #include <hv/hypervisor.h> | |
27 | #include <arch/abi.h> | |
28 | #include <arch/interrupts.h> | |
29 | #include <arch/spr_def.h> | |
30 | ||
31 | #ifdef CONFIG_PREEMPT | |
32 | # error "No support for kernel preemption currently" | |
33 | #endif | |
34 | ||
9f9c0382 | 35 | #if INT_INTCTRL_1 < 32 || INT_INTCTRL_1 >= 48 |
867e359b CM |
36 | # error INT_INTCTRL_1 coded to set high interrupt mask |
37 | #endif | |
38 | ||
39 | #define PTREGS_PTR(reg, ptreg) addli reg, sp, C_ABI_SAVE_AREA_SIZE + (ptreg) | |
40 | ||
41 | #define PTREGS_OFFSET_SYSCALL PTREGS_OFFSET_REG(TREG_SYSCALL_NR) | |
42 | ||
43 | #if !CHIP_HAS_WH64() | |
44 | /* By making this an empty macro, we can use wh64 in the code. */ | |
45 | .macro wh64 reg | |
46 | .endm | |
47 | #endif | |
48 | ||
49 | .macro push_reg reg, ptr=sp, delta=-4 | |
50 | { | |
51 | sw \ptr, \reg | |
52 | addli \ptr, \ptr, \delta | |
53 | } | |
54 | .endm | |
55 | ||
56 | .macro pop_reg reg, ptr=sp, delta=4 | |
57 | { | |
58 | lw \reg, \ptr | |
59 | addli \ptr, \ptr, \delta | |
60 | } | |
61 | .endm | |
62 | ||
63 | .macro pop_reg_zero reg, zreg, ptr=sp, delta=4 | |
64 | { | |
65 | move \zreg, zero | |
66 | lw \reg, \ptr | |
67 | addi \ptr, \ptr, \delta | |
68 | } | |
69 | .endm | |
70 | ||
71 | .macro push_extra_callee_saves reg | |
72 | PTREGS_PTR(\reg, PTREGS_OFFSET_REG(51)) | |
73 | push_reg r51, \reg | |
74 | push_reg r50, \reg | |
75 | push_reg r49, \reg | |
76 | push_reg r48, \reg | |
77 | push_reg r47, \reg | |
78 | push_reg r46, \reg | |
79 | push_reg r45, \reg | |
80 | push_reg r44, \reg | |
81 | push_reg r43, \reg | |
82 | push_reg r42, \reg | |
83 | push_reg r41, \reg | |
84 | push_reg r40, \reg | |
85 | push_reg r39, \reg | |
86 | push_reg r38, \reg | |
87 | push_reg r37, \reg | |
88 | push_reg r36, \reg | |
89 | push_reg r35, \reg | |
90 | push_reg r34, \reg, PTREGS_OFFSET_BASE - PTREGS_OFFSET_REG(34) | |
91 | .endm | |
92 | ||
93 | .macro panic str | |
94 | .pushsection .rodata, "a" | |
95 | 1: | |
96 | .asciz "\str" | |
97 | .popsection | |
98 | { | |
99 | moveli r0, lo16(1b) | |
100 | } | |
101 | { | |
102 | auli r0, r0, ha16(1b) | |
103 | jal panic | |
104 | } | |
105 | .endm | |
106 | ||
107 | #ifdef __COLLECT_LINKER_FEEDBACK__ | |
108 | .pushsection .text.intvec_feedback,"ax" | |
109 | intvec_feedback: | |
110 | .popsection | |
111 | #endif | |
112 | ||
113 | /* | |
114 | * Default interrupt handler. | |
115 | * | |
116 | * vecnum is where we'll put this code. | |
117 | * c_routine is the C routine we'll call. | |
118 | * | |
119 | * The C routine is passed two arguments: | |
120 | * - A pointer to the pt_regs state. | |
121 | * - The interrupt vector number. | |
122 | * | |
123 | * The "processing" argument specifies the code for processing | |
124 | * the interrupt. Defaults to "handle_interrupt". | |
125 | */ | |
126 | .macro int_hand vecnum, vecname, c_routine, processing=handle_interrupt | |
127 | .org (\vecnum << 8) | |
128 | intvec_\vecname: | |
129 | .ifc \vecnum, INT_SWINT_1 | |
130 | blz TREG_SYSCALL_NR_NAME, sys_cmpxchg | |
131 | .endif | |
132 | ||
133 | /* Temporarily save a register so we have somewhere to work. */ | |
134 | ||
135 | mtspr SYSTEM_SAVE_1_1, r0 | |
136 | mfspr r0, EX_CONTEXT_1_1 | |
137 | ||
138 | /* The cmpxchg code clears sp to force us to reset it here on fault. */ | |
139 | { | |
140 | bz sp, 2f | |
141 | andi r0, r0, SPR_EX_CONTEXT_1_1__PL_MASK /* mask off ICS */ | |
142 | } | |
143 | ||
144 | .ifc \vecnum, INT_DOUBLE_FAULT | |
145 | /* | |
146 | * For double-faults from user-space, fall through to the normal | |
147 | * register save and stack setup path. Otherwise, it's the | |
148 | * hypervisor giving us one last chance to dump diagnostics, and we | |
149 | * branch to the kernel_double_fault routine to do so. | |
150 | */ | |
151 | bz r0, 1f | |
152 | j _kernel_double_fault | |
153 | 1: | |
154 | .else | |
155 | /* | |
156 | * If we're coming from user-space, then set sp to the top of | |
157 | * the kernel stack. Otherwise, assume sp is already valid. | |
158 | */ | |
159 | { | |
160 | bnz r0, 0f | |
161 | move r0, sp | |
162 | } | |
163 | .endif | |
164 | ||
165 | .ifc \c_routine, do_page_fault | |
166 | /* | |
167 | * The page_fault handler may be downcalled directly by the | |
168 | * hypervisor even when Linux is running and has ICS set. | |
169 | * | |
170 | * In this case the contents of EX_CONTEXT_1_1 reflect the | |
171 | * previous fault and can't be relied on to choose whether or | |
172 | * not to reinitialize the stack pointer. So we add a test | |
173 | * to see whether SYSTEM_SAVE_1_2 has the high bit set, | |
174 | * and if so we don't reinitialize sp, since we must be coming | |
175 | * from Linux. (In fact the precise case is !(val & ~1), | |
176 | * but any Linux PC has to have the high bit set.) | |
177 | * | |
178 | * Note that the hypervisor *always* sets SYSTEM_SAVE_1_2 for | |
179 | * any path that turns into a downcall to one of our TLB handlers. | |
180 | */ | |
181 | mfspr r0, SYSTEM_SAVE_1_2 | |
182 | { | |
183 | blz r0, 0f /* high bit in S_S_1_2 is for a PC to use */ | |
184 | move r0, sp | |
185 | } | |
186 | .endif | |
187 | ||
188 | 2: | |
189 | /* | |
190 | * SYSTEM_SAVE_1_0 holds the cpu number in the low bits, and | |
191 | * the current stack top in the higher bits. So we recover | |
192 | * our stack top by just masking off the low bits, then | |
193 | * point sp at the top aligned address on the actual stack page. | |
194 | */ | |
195 | mfspr r0, SYSTEM_SAVE_1_0 | |
196 | mm r0, r0, zero, LOG2_THREAD_SIZE, 31 | |
197 | ||
198 | 0: | |
199 | /* | |
200 | * Align the stack mod 64 so we can properly predict what | |
201 | * cache lines we need to write-hint to reduce memory fetch | |
202 | * latency as we enter the kernel. The layout of memory is | |
203 | * as follows, with cache line 0 at the lowest VA, and cache | |
204 | * line 4 just below the r0 value this "andi" computes. | |
205 | * Note that we never write to cache line 4, and we skip | |
206 | * cache line 1 for syscalls. | |
207 | * | |
208 | * cache line 4: ptregs padding (two words) | |
209 | * cache line 3: r46...lr, pc, ex1, faultnum, orig_r0, flags, pad | |
210 | * cache line 2: r30...r45 | |
211 | * cache line 1: r14...r29 | |
212 | * cache line 0: 2 x frame, r0..r13 | |
213 | */ | |
214 | andi r0, r0, -64 | |
215 | ||
216 | /* | |
217 | * Push the first four registers on the stack, so that we can set | |
218 | * them to vector-unique values before we jump to the common code. | |
219 | * | |
220 | * Registers are pushed on the stack as a struct pt_regs, | |
221 | * with the sp initially just above the struct, and when we're | |
222 | * done, sp points to the base of the struct, minus | |
223 | * C_ABI_SAVE_AREA_SIZE, so we can directly jal to C code. | |
224 | * | |
225 | * This routine saves just the first four registers, plus the | |
226 | * stack context so we can do proper backtracing right away, | |
227 | * and defers to handle_interrupt to save the rest. | |
228 | * The backtracer needs pc, ex1, lr, sp, r52, and faultnum. | |
229 | */ | |
230 | addli r0, r0, PTREGS_OFFSET_LR - (PTREGS_SIZE + KSTK_PTREGS_GAP) | |
231 | wh64 r0 /* cache line 3 */ | |
232 | { | |
233 | sw r0, lr | |
234 | addli r0, r0, PTREGS_OFFSET_SP - PTREGS_OFFSET_LR | |
235 | } | |
236 | { | |
237 | sw r0, sp | |
238 | addli sp, r0, PTREGS_OFFSET_REG(52) - PTREGS_OFFSET_SP | |
239 | } | |
240 | { | |
241 | sw sp, r52 | |
242 | addli sp, sp, PTREGS_OFFSET_REG(1) - PTREGS_OFFSET_REG(52) | |
243 | } | |
244 | wh64 sp /* cache line 0 */ | |
245 | { | |
246 | sw sp, r1 | |
247 | addli sp, sp, PTREGS_OFFSET_REG(2) - PTREGS_OFFSET_REG(1) | |
248 | } | |
249 | { | |
250 | sw sp, r2 | |
251 | addli sp, sp, PTREGS_OFFSET_REG(3) - PTREGS_OFFSET_REG(2) | |
252 | } | |
253 | { | |
254 | sw sp, r3 | |
255 | addli sp, sp, PTREGS_OFFSET_PC - PTREGS_OFFSET_REG(3) | |
256 | } | |
257 | mfspr r0, EX_CONTEXT_1_0 | |
258 | .ifc \processing,handle_syscall | |
259 | /* | |
260 | * Bump the saved PC by one bundle so that when we return, we won't | |
261 | * execute the same swint instruction again. We need to do this while | |
262 | * we're in the critical section. | |
263 | */ | |
264 | addi r0, r0, 8 | |
265 | .endif | |
266 | { | |
267 | sw sp, r0 | |
268 | addli sp, sp, PTREGS_OFFSET_EX1 - PTREGS_OFFSET_PC | |
269 | } | |
270 | mfspr r0, EX_CONTEXT_1_1 | |
271 | { | |
272 | sw sp, r0 | |
273 | addi sp, sp, PTREGS_OFFSET_FAULTNUM - PTREGS_OFFSET_EX1 | |
274 | /* | |
275 | * Use r0 for syscalls so it's a temporary; use r1 for interrupts | |
276 | * so that it gets passed through unchanged to the handler routine. | |
277 | * Note that the .if conditional confusingly spans bundles. | |
278 | */ | |
279 | .ifc \processing,handle_syscall | |
280 | movei r0, \vecnum | |
281 | } | |
282 | { | |
283 | sw sp, r0 | |
284 | .else | |
285 | movei r1, \vecnum | |
286 | } | |
287 | { | |
288 | sw sp, r1 | |
289 | .endif | |
290 | addli sp, sp, PTREGS_OFFSET_REG(0) - PTREGS_OFFSET_FAULTNUM | |
291 | } | |
292 | mfspr r0, SYSTEM_SAVE_1_1 /* Original r0 */ | |
293 | { | |
294 | sw sp, r0 | |
295 | addi sp, sp, -PTREGS_OFFSET_REG(0) - 4 | |
296 | } | |
297 | { | |
298 | sw sp, zero /* write zero into "Next SP" frame pointer */ | |
299 | addi sp, sp, -4 /* leave SP pointing at bottom of frame */ | |
300 | } | |
301 | .ifc \processing,handle_syscall | |
302 | j handle_syscall | |
303 | .else | |
304 | /* | |
305 | * Capture per-interrupt SPR context to registers. | |
306 | * We overload the meaning of r3 on this path such that if its bit 31 | |
307 | * is set, we have to mask all interrupts including NMIs before | |
308 | * clearing the interrupt critical section bit. | |
309 | * See discussion below at "finish_interrupt_save". | |
310 | */ | |
311 | .ifc \c_routine, do_page_fault | |
312 | mfspr r2, SYSTEM_SAVE_1_3 /* address of page fault */ | |
313 | mfspr r3, SYSTEM_SAVE_1_2 /* info about page fault */ | |
314 | .else | |
315 | .ifc \vecnum, INT_DOUBLE_FAULT | |
316 | { | |
317 | mfspr r2, SYSTEM_SAVE_1_2 /* double fault info from HV */ | |
318 | movei r3, 0 | |
319 | } | |
320 | .else | |
321 | .ifc \c_routine, do_trap | |
322 | { | |
323 | mfspr r2, GPV_REASON | |
324 | movei r3, 0 | |
325 | } | |
326 | .else | |
327 | .ifc \c_routine, op_handle_perf_interrupt | |
328 | { | |
329 | mfspr r2, PERF_COUNT_STS | |
330 | movei r3, -1 /* not used, but set for consistency */ | |
331 | } | |
332 | .else | |
333 | #if CHIP_HAS_AUX_PERF_COUNTERS() | |
334 | .ifc \c_routine, op_handle_aux_perf_interrupt | |
335 | { | |
336 | mfspr r2, AUX_PERF_COUNT_STS | |
337 | movei r3, -1 /* not used, but set for consistency */ | |
338 | } | |
339 | .else | |
340 | #endif | |
341 | movei r3, 0 | |
342 | #if CHIP_HAS_AUX_PERF_COUNTERS() | |
343 | .endif | |
344 | #endif | |
345 | .endif | |
346 | .endif | |
347 | .endif | |
348 | .endif | |
349 | /* Put function pointer in r0 */ | |
350 | moveli r0, lo16(\c_routine) | |
351 | { | |
352 | auli r0, r0, ha16(\c_routine) | |
353 | j \processing | |
354 | } | |
355 | .endif | |
356 | ENDPROC(intvec_\vecname) | |
357 | ||
358 | #ifdef __COLLECT_LINKER_FEEDBACK__ | |
359 | .pushsection .text.intvec_feedback,"ax" | |
360 | .org (\vecnum << 5) | |
361 | FEEDBACK_ENTER_EXPLICIT(intvec_\vecname, .intrpt1, 1 << 8) | |
362 | jrp lr | |
363 | .popsection | |
364 | #endif | |
365 | ||
366 | .endm | |
367 | ||
368 | ||
369 | /* | |
370 | * Save the rest of the registers that we didn't save in the actual | |
371 | * vector itself. We can't use r0-r10 inclusive here. | |
372 | */ | |
373 | .macro finish_interrupt_save, function | |
374 | ||
375 | /* If it's a syscall, save a proper orig_r0, otherwise just zero. */ | |
376 | PTREGS_PTR(r52, PTREGS_OFFSET_ORIG_R0) | |
377 | { | |
378 | .ifc \function,handle_syscall | |
379 | sw r52, r0 | |
380 | .else | |
381 | sw r52, zero | |
382 | .endif | |
383 | PTREGS_PTR(r52, PTREGS_OFFSET_TP) | |
384 | } | |
385 | ||
386 | /* | |
387 | * For ordinary syscalls, we save neither caller- nor callee- | |
388 | * save registers, since the syscall invoker doesn't expect the | |
389 | * caller-saves to be saved, and the called kernel functions will | |
390 | * take care of saving the callee-saves for us. | |
391 | * | |
392 | * For interrupts we save just the caller-save registers. Saving | |
393 | * them is required (since the "caller" can't save them). Again, | |
394 | * the called kernel functions will restore the callee-save | |
395 | * registers for us appropriately. | |
396 | * | |
397 | * On return, we normally restore nothing special for syscalls, | |
398 | * and just the caller-save registers for interrupts. | |
399 | * | |
400 | * However, there are some important caveats to all this: | |
401 | * | |
402 | * - We always save a few callee-save registers to give us | |
403 | * some scratchpad registers to carry across function calls. | |
404 | * | |
405 | * - fork/vfork/etc require us to save all the callee-save | |
406 | * registers, which we do in PTREGS_SYSCALL_ALL_REGS, below. | |
407 | * | |
408 | * - We always save r0..r5 and r10 for syscalls, since we need | |
409 | * to reload them a bit later for the actual kernel call, and | |
410 | * since we might need them for -ERESTARTNOINTR, etc. | |
411 | * | |
412 | * - Before invoking a signal handler, we save the unsaved | |
413 | * callee-save registers so they are visible to the | |
414 | * signal handler or any ptracer. | |
415 | * | |
416 | * - If the unsaved callee-save registers are modified, we set | |
417 | * a bit in pt_regs so we know to reload them from pt_regs | |
418 | * and not just rely on the kernel function unwinding. | |
419 | * (Done for ptrace register writes and SA_SIGINFO handler.) | |
420 | */ | |
421 | { | |
422 | sw r52, tp | |
423 | PTREGS_PTR(r52, PTREGS_OFFSET_REG(33)) | |
424 | } | |
425 | wh64 r52 /* cache line 2 */ | |
426 | push_reg r33, r52 | |
427 | push_reg r32, r52 | |
428 | push_reg r31, r52 | |
429 | .ifc \function,handle_syscall | |
430 | push_reg r30, r52, PTREGS_OFFSET_SYSCALL - PTREGS_OFFSET_REG(30) | |
431 | push_reg TREG_SYSCALL_NR_NAME, r52, \ | |
432 | PTREGS_OFFSET_REG(5) - PTREGS_OFFSET_SYSCALL | |
433 | .else | |
434 | ||
435 | push_reg r30, r52, PTREGS_OFFSET_REG(29) - PTREGS_OFFSET_REG(30) | |
436 | wh64 r52 /* cache line 1 */ | |
437 | push_reg r29, r52 | |
438 | push_reg r28, r52 | |
439 | push_reg r27, r52 | |
440 | push_reg r26, r52 | |
441 | push_reg r25, r52 | |
442 | push_reg r24, r52 | |
443 | push_reg r23, r52 | |
444 | push_reg r22, r52 | |
445 | push_reg r21, r52 | |
446 | push_reg r20, r52 | |
447 | push_reg r19, r52 | |
448 | push_reg r18, r52 | |
449 | push_reg r17, r52 | |
450 | push_reg r16, r52 | |
451 | push_reg r15, r52 | |
452 | push_reg r14, r52 | |
453 | push_reg r13, r52 | |
454 | push_reg r12, r52 | |
455 | push_reg r11, r52 | |
456 | push_reg r10, r52 | |
457 | push_reg r9, r52 | |
458 | push_reg r8, r52 | |
459 | push_reg r7, r52 | |
460 | push_reg r6, r52 | |
461 | ||
462 | .endif | |
463 | ||
464 | push_reg r5, r52 | |
465 | sw r52, r4 | |
466 | ||
467 | /* Load tp with our per-cpu offset. */ | |
468 | #ifdef CONFIG_SMP | |
469 | { | |
470 | mfspr r20, SYSTEM_SAVE_1_0 | |
471 | moveli r21, lo16(__per_cpu_offset) | |
472 | } | |
473 | { | |
474 | auli r21, r21, ha16(__per_cpu_offset) | |
475 | mm r20, r20, zero, 0, LOG2_THREAD_SIZE-1 | |
476 | } | |
477 | s2a r20, r20, r21 | |
478 | lw tp, r20 | |
479 | #else | |
480 | move tp, zero | |
481 | #endif | |
482 | ||
483 | /* | |
484 | * If we will be returning to the kernel, we will need to | |
485 | * reset the interrupt masks to the state they had before. | |
486 | * Set DISABLE_IRQ in flags iff we came from PL1 with irqs disabled. | |
487 | * We load flags in r32 here so we can jump to .Lrestore_regs | |
488 | * directly after do_page_fault_ics() if necessary. | |
489 | */ | |
490 | mfspr r32, EX_CONTEXT_1_1 | |
491 | { | |
492 | andi r32, r32, SPR_EX_CONTEXT_1_1__PL_MASK /* mask off ICS */ | |
493 | PTREGS_PTR(r21, PTREGS_OFFSET_FLAGS) | |
494 | } | |
495 | bzt r32, 1f /* zero if from user space */ | |
496 | IRQS_DISABLED(r32) /* zero if irqs enabled */ | |
497 | #if PT_FLAGS_DISABLE_IRQ != 1 | |
498 | # error Value of IRQS_DISABLED used to set PT_FLAGS_DISABLE_IRQ; fix | |
499 | #endif | |
500 | 1: | |
501 | .ifnc \function,handle_syscall | |
502 | /* Record the fact that we saved the caller-save registers above. */ | |
503 | ori r32, r32, PT_FLAGS_CALLER_SAVES | |
504 | .endif | |
505 | sw r21, r32 | |
506 | ||
507 | #ifdef __COLLECT_LINKER_FEEDBACK__ | |
508 | /* | |
509 | * Notify the feedback routines that we were in the | |
510 | * appropriate fixed interrupt vector area. Note that we | |
511 | * still have ICS set at this point, so we can't invoke any | |
512 | * atomic operations or we will panic. The feedback | |
513 | * routines internally preserve r0..r10 and r30 up. | |
514 | */ | |
515 | .ifnc \function,handle_syscall | |
516 | shli r20, r1, 5 | |
517 | .else | |
518 | moveli r20, INT_SWINT_1 << 5 | |
519 | .endif | |
520 | addli r20, r20, lo16(intvec_feedback) | |
521 | auli r20, r20, ha16(intvec_feedback) | |
522 | jalr r20 | |
523 | ||
524 | /* And now notify the feedback routines that we are here. */ | |
525 | FEEDBACK_ENTER(\function) | |
526 | #endif | |
527 | ||
528 | /* | |
529 | * we've captured enough state to the stack (including in | |
530 | * particular our EX_CONTEXT state) that we can now release | |
531 | * the interrupt critical section and replace it with our | |
532 | * standard "interrupts disabled" mask value. This allows | |
533 | * synchronous interrupts (and profile interrupts) to punch | |
534 | * through from this point onwards. | |
535 | * | |
536 | * If bit 31 of r3 is set during a non-NMI interrupt, we know we | |
537 | * are on the path where the hypervisor has punched through our | |
538 | * ICS with a page fault, so we call out to do_page_fault_ics() | |
539 | * to figure out what to do with it. If the fault was in | |
540 | * an atomic op, we unlock the atomic lock, adjust the | |
541 | * saved register state a little, and return "zero" in r4, | |
542 | * falling through into the normal page-fault interrupt code. | |
543 | * If the fault was in a kernel-space atomic operation, then | |
544 | * do_page_fault_ics() resolves it itself, returns "one" in r4, | |
545 | * and as a result goes directly to restoring registers and iret, | |
546 | * without trying to adjust the interrupt masks at all. | |
547 | * The do_page_fault_ics() API involves passing and returning | |
548 | * a five-word struct (in registers) to avoid writing the | |
549 | * save and restore code here. | |
550 | */ | |
551 | .ifc \function,handle_nmi | |
552 | IRQ_DISABLE_ALL(r20) | |
553 | .else | |
554 | .ifnc \function,handle_syscall | |
555 | bgezt r3, 1f | |
556 | { | |
557 | PTREGS_PTR(r0, PTREGS_OFFSET_BASE) | |
558 | jal do_page_fault_ics | |
559 | } | |
560 | FEEDBACK_REENTER(\function) | |
561 | bzt r4, 1f | |
562 | j .Lrestore_regs | |
563 | 1: | |
564 | .endif | |
565 | IRQ_DISABLE(r20, r21) | |
566 | .endif | |
567 | mtspr INTERRUPT_CRITICAL_SECTION, zero | |
568 | ||
569 | #if CHIP_HAS_WH64() | |
570 | /* | |
571 | * Prepare the first 256 stack bytes to be rapidly accessible | |
572 | * without having to fetch the background data. We don't really | |
573 | * know how far to write-hint, but kernel stacks generally | |
574 | * aren't that big, and write-hinting here does take some time. | |
575 | */ | |
576 | addi r52, sp, -64 | |
577 | { | |
578 | wh64 r52 | |
579 | addi r52, r52, -64 | |
580 | } | |
581 | { | |
582 | wh64 r52 | |
583 | addi r52, r52, -64 | |
584 | } | |
585 | { | |
586 | wh64 r52 | |
587 | addi r52, r52, -64 | |
588 | } | |
589 | wh64 r52 | |
590 | #endif | |
591 | ||
592 | #ifdef CONFIG_TRACE_IRQFLAGS | |
593 | .ifnc \function,handle_nmi | |
594 | /* | |
595 | * We finally have enough state set up to notify the irq | |
596 | * tracing code that irqs were disabled on entry to the handler. | |
597 | * The TRACE_IRQS_OFF call clobbers registers r0-r29. | |
598 | * For syscalls, we already have the register state saved away | |
599 | * on the stack, so we don't bother to do any register saves here, | |
600 | * and later we pop the registers back off the kernel stack. | |
601 | * For interrupt handlers, save r0-r3 in callee-saved registers. | |
602 | */ | |
603 | .ifnc \function,handle_syscall | |
604 | { move r30, r0; move r31, r1 } | |
605 | { move r32, r2; move r33, r3 } | |
606 | .endif | |
607 | TRACE_IRQS_OFF | |
608 | .ifnc \function,handle_syscall | |
609 | { move r0, r30; move r1, r31 } | |
610 | { move r2, r32; move r3, r33 } | |
611 | .endif | |
612 | .endif | |
613 | #endif | |
614 | ||
615 | .endm | |
616 | ||
617 | .macro check_single_stepping, kind, not_single_stepping | |
618 | /* | |
619 | * Check for single stepping in user-level priv | |
620 | * kind can be "normal", "ill", or "syscall" | |
621 | * At end, if fall-thru | |
622 | * r29: thread_info->step_state | |
623 | * r28: &pt_regs->pc | |
624 | * r27: pt_regs->pc | |
625 | * r26: thread_info->step_state->buffer | |
626 | */ | |
627 | ||
628 | /* Check for single stepping */ | |
629 | GET_THREAD_INFO(r29) | |
630 | { | |
631 | /* Get pointer to field holding step state */ | |
632 | addi r29, r29, THREAD_INFO_STEP_STATE_OFFSET | |
633 | ||
634 | /* Get pointer to EX1 in register state */ | |
635 | PTREGS_PTR(r27, PTREGS_OFFSET_EX1) | |
636 | } | |
637 | { | |
638 | /* Get pointer to field holding PC */ | |
639 | PTREGS_PTR(r28, PTREGS_OFFSET_PC) | |
640 | ||
641 | /* Load the pointer to the step state */ | |
642 | lw r29, r29 | |
643 | } | |
644 | /* Load EX1 */ | |
645 | lw r27, r27 | |
646 | { | |
647 | /* Points to flags */ | |
648 | addi r23, r29, SINGLESTEP_STATE_FLAGS_OFFSET | |
649 | ||
650 | /* No single stepping if there is no step state structure */ | |
651 | bzt r29, \not_single_stepping | |
652 | } | |
653 | { | |
654 | /* mask off ICS and any other high bits */ | |
655 | andi r27, r27, SPR_EX_CONTEXT_1_1__PL_MASK | |
656 | ||
657 | /* Load pointer to single step instruction buffer */ | |
658 | lw r26, r29 | |
659 | } | |
660 | /* Check priv state */ | |
661 | bnz r27, \not_single_stepping | |
662 | ||
663 | /* Get flags */ | |
664 | lw r22, r23 | |
665 | { | |
666 | /* Branch if single-step mode not enabled */ | |
667 | bbnst r22, \not_single_stepping | |
668 | ||
669 | /* Clear enabled flag */ | |
670 | andi r22, r22, ~SINGLESTEP_STATE_MASK_IS_ENABLED | |
671 | } | |
672 | .ifc \kind,normal | |
673 | { | |
674 | /* Load PC */ | |
675 | lw r27, r28 | |
676 | ||
677 | /* Point to the entry containing the original PC */ | |
678 | addi r24, r29, SINGLESTEP_STATE_ORIG_PC_OFFSET | |
679 | } | |
680 | { | |
681 | /* Disable single stepping flag */ | |
682 | sw r23, r22 | |
683 | } | |
684 | { | |
685 | /* Get the original pc */ | |
686 | lw r24, r24 | |
687 | ||
688 | /* See if the PC is at the start of the single step buffer */ | |
689 | seq r25, r26, r27 | |
690 | } | |
691 | /* | |
692 | * NOTE: it is really expected that the PC be in the single step buffer | |
693 | * at this point | |
694 | */ | |
695 | bzt r25, \not_single_stepping | |
696 | ||
697 | /* Restore the original PC */ | |
698 | sw r28, r24 | |
699 | .else | |
700 | .ifc \kind,syscall | |
701 | { | |
702 | /* Load PC */ | |
703 | lw r27, r28 | |
704 | ||
705 | /* Point to the entry containing the next PC */ | |
706 | addi r24, r29, SINGLESTEP_STATE_NEXT_PC_OFFSET | |
707 | } | |
708 | { | |
709 | /* Increment the stopped PC by the bundle size */ | |
710 | addi r26, r26, 8 | |
711 | ||
712 | /* Disable single stepping flag */ | |
713 | sw r23, r22 | |
714 | } | |
715 | { | |
716 | /* Get the next pc */ | |
717 | lw r24, r24 | |
718 | ||
719 | /* | |
720 | * See if the PC is one bundle past the start of the | |
721 | * single step buffer | |
722 | */ | |
723 | seq r25, r26, r27 | |
724 | } | |
725 | { | |
726 | /* | |
727 | * NOTE: it is really expected that the PC be in the | |
728 | * single step buffer at this point | |
729 | */ | |
730 | bzt r25, \not_single_stepping | |
731 | } | |
732 | /* Set to the next PC */ | |
733 | sw r28, r24 | |
734 | .else | |
735 | { | |
736 | /* Point to 3rd bundle in buffer */ | |
737 | addi r25, r26, 16 | |
738 | ||
739 | /* Load PC */ | |
740 | lw r27, r28 | |
741 | } | |
742 | { | |
743 | /* Disable single stepping flag */ | |
744 | sw r23, r22 | |
745 | ||
746 | /* See if the PC is in the single step buffer */ | |
747 | slte_u r24, r26, r27 | |
748 | } | |
749 | { | |
750 | slte_u r25, r27, r25 | |
751 | ||
752 | /* | |
753 | * NOTE: it is really expected that the PC be in the | |
754 | * single step buffer at this point | |
755 | */ | |
756 | bzt r24, \not_single_stepping | |
757 | } | |
758 | bzt r25, \not_single_stepping | |
759 | .endif | |
760 | .endif | |
761 | .endm | |
762 | ||
763 | /* | |
764 | * Redispatch a downcall. | |
765 | */ | |
766 | .macro dc_dispatch vecnum, vecname | |
767 | .org (\vecnum << 8) | |
768 | intvec_\vecname: | |
769 | j hv_downcall_dispatch | |
770 | ENDPROC(intvec_\vecname) | |
771 | .endm | |
772 | ||
773 | /* | |
774 | * Common code for most interrupts. The C function we're eventually | |
775 | * going to is in r0, and the faultnum is in r1; the original | |
776 | * values for those registers are on the stack. | |
777 | */ | |
778 | .pushsection .text.handle_interrupt,"ax" | |
779 | handle_interrupt: | |
780 | finish_interrupt_save handle_interrupt | |
781 | ||
782 | /* | |
783 | * Check for if we are single stepping in user level. If so, then | |
784 | * we need to restore the PC. | |
785 | */ | |
786 | ||
787 | check_single_stepping normal, .Ldispatch_interrupt | |
788 | .Ldispatch_interrupt: | |
789 | ||
790 | /* Jump to the C routine; it should enable irqs as soon as possible. */ | |
791 | { | |
792 | jalr r0 | |
793 | PTREGS_PTR(r0, PTREGS_OFFSET_BASE) | |
794 | } | |
795 | FEEDBACK_REENTER(handle_interrupt) | |
796 | { | |
797 | movei r30, 0 /* not an NMI */ | |
798 | j interrupt_return | |
799 | } | |
800 | STD_ENDPROC(handle_interrupt) | |
801 | ||
802 | /* | |
803 | * This routine takes a boolean in r30 indicating if this is an NMI. | |
804 | * If so, we also expect a boolean in r31 indicating whether to | |
805 | * re-enable the oprofile interrupts. | |
806 | */ | |
807 | STD_ENTRY(interrupt_return) | |
808 | /* If we're resuming to kernel space, don't check thread flags. */ | |
809 | { | |
810 | bnz r30, .Lrestore_all /* NMIs don't special-case user-space */ | |
811 | PTREGS_PTR(r29, PTREGS_OFFSET_EX1) | |
812 | } | |
813 | lw r29, r29 | |
814 | andi r29, r29, SPR_EX_CONTEXT_1_1__PL_MASK /* mask off ICS */ | |
815 | { | |
816 | bzt r29, .Lresume_userspace | |
817 | PTREGS_PTR(r29, PTREGS_OFFSET_PC) | |
818 | } | |
819 | ||
820 | /* If we're resuming to _cpu_idle_nap, bump PC forward by 8. */ | |
821 | { | |
822 | lw r28, r29 | |
823 | moveli r27, lo16(_cpu_idle_nap) | |
824 | } | |
825 | { | |
826 | auli r27, r27, ha16(_cpu_idle_nap) | |
827 | } | |
828 | { | |
829 | seq r27, r27, r28 | |
830 | } | |
831 | { | |
832 | bbns r27, .Lrestore_all | |
833 | addi r28, r28, 8 | |
834 | } | |
835 | sw r29, r28 | |
836 | j .Lrestore_all | |
837 | ||
838 | .Lresume_userspace: | |
839 | FEEDBACK_REENTER(interrupt_return) | |
840 | ||
841 | /* | |
842 | * Disable interrupts so as to make sure we don't | |
843 | * miss an interrupt that sets any of the thread flags (like | |
844 | * need_resched or sigpending) between sampling and the iret. | |
845 | * Routines like schedule() or do_signal() may re-enable | |
846 | * interrupts before returning. | |
847 | */ | |
848 | IRQ_DISABLE(r20, r21) | |
849 | TRACE_IRQS_OFF /* Note: clobbers registers r0-r29 */ | |
850 | ||
851 | /* Get base of stack in r32; note r30/31 are used as arguments here. */ | |
852 | GET_THREAD_INFO(r32) | |
853 | ||
854 | ||
855 | /* Check to see if there is any work to do before returning to user. */ | |
856 | { | |
857 | addi r29, r32, THREAD_INFO_FLAGS_OFFSET | |
858 | moveli r28, lo16(_TIF_ALLWORK_MASK) | |
859 | } | |
860 | { | |
861 | lw r29, r29 | |
862 | auli r28, r28, ha16(_TIF_ALLWORK_MASK) | |
863 | } | |
864 | and r28, r29, r28 | |
865 | bnz r28, .Lwork_pending | |
866 | ||
867 | /* | |
868 | * In the NMI case we | |
869 | * omit the call to single_process_check_nohz, which normally checks | |
870 | * to see if we should start or stop the scheduler tick, because | |
871 | * we can't call arbitrary Linux code from an NMI context. | |
872 | * We always call the homecache TLB deferral code to re-trigger | |
873 | * the deferral mechanism. | |
874 | * | |
875 | * The other chunk of responsibility this code has is to reset the | |
876 | * interrupt masks appropriately to reset irqs and NMIs. We have | |
877 | * to call TRACE_IRQS_OFF and TRACE_IRQS_ON to support all the | |
878 | * lockdep-type stuff, but we can't set ICS until afterwards, since | |
879 | * ICS can only be used in very tight chunks of code to avoid | |
880 | * tripping over various assertions that it is off. | |
881 | * | |
882 | * (There is what looks like a window of vulnerability here since | |
883 | * we might take a profile interrupt between the two SPR writes | |
884 | * that set the mask, but since we write the low SPR word first, | |
885 | * and our interrupt entry code checks the low SPR word, any | |
886 | * profile interrupt will actually disable interrupts in both SPRs | |
887 | * before returning, which is OK.) | |
888 | */ | |
889 | .Lrestore_all: | |
890 | PTREGS_PTR(r0, PTREGS_OFFSET_EX1) | |
891 | { | |
892 | lw r0, r0 | |
893 | PTREGS_PTR(r32, PTREGS_OFFSET_FLAGS) | |
894 | } | |
895 | { | |
896 | andi r0, r0, SPR_EX_CONTEXT_1_1__PL_MASK | |
897 | lw r32, r32 | |
898 | } | |
899 | bnz r0, 1f | |
900 | j 2f | |
901 | #if PT_FLAGS_DISABLE_IRQ != 1 | |
902 | # error Assuming PT_FLAGS_DISABLE_IRQ == 1 so we can use bbnst below | |
903 | #endif | |
904 | 1: bbnst r32, 2f | |
905 | IRQ_DISABLE(r20,r21) | |
906 | TRACE_IRQS_OFF | |
907 | movei r0, 1 | |
908 | mtspr INTERRUPT_CRITICAL_SECTION, r0 | |
909 | bzt r30, .Lrestore_regs | |
910 | j 3f | |
911 | 2: TRACE_IRQS_ON | |
912 | movei r0, 1 | |
913 | mtspr INTERRUPT_CRITICAL_SECTION, r0 | |
914 | IRQ_ENABLE(r20, r21) | |
915 | bzt r30, .Lrestore_regs | |
916 | 3: | |
917 | ||
918 | ||
919 | /* | |
920 | * We now commit to returning from this interrupt, since we will be | |
921 | * doing things like setting EX_CONTEXT SPRs and unwinding the stack | |
922 | * frame. No calls should be made to any other code after this point. | |
923 | * This code should only be entered with ICS set. | |
924 | * r32 must still be set to ptregs.flags. | |
925 | * We launch loads to each cache line separately first, so we can | |
926 | * get some parallelism out of the memory subsystem. | |
927 | * We start zeroing caller-saved registers throughout, since | |
928 | * that will save some cycles if this turns out to be a syscall. | |
929 | */ | |
930 | .Lrestore_regs: | |
931 | FEEDBACK_REENTER(interrupt_return) /* called from elsewhere */ | |
932 | ||
933 | /* | |
934 | * Rotate so we have one high bit and one low bit to test. | |
935 | * - low bit says whether to restore all the callee-saved registers, | |
936 | * or just r30-r33, and r52 up. | |
937 | * - high bit (i.e. sign bit) says whether to restore all the | |
938 | * caller-saved registers, or just r0. | |
939 | */ | |
940 | #if PT_FLAGS_CALLER_SAVES != 2 || PT_FLAGS_RESTORE_REGS != 4 | |
941 | # error Rotate trick does not work :-) | |
942 | #endif | |
943 | { | |
944 | rli r20, r32, 30 | |
945 | PTREGS_PTR(sp, PTREGS_OFFSET_REG(0)) | |
946 | } | |
947 | ||
948 | /* | |
949 | * Load cache lines 0, 2, and 3 in that order, then use | |
950 | * the last loaded value, which makes it likely that the other | |
951 | * cache lines have also loaded, at which point we should be | |
952 | * able to safely read all the remaining words on those cache | |
953 | * lines without waiting for the memory subsystem. | |
954 | */ | |
ba00376b | 955 | pop_reg_zero r0, r28, sp, PTREGS_OFFSET_REG(30) - PTREGS_OFFSET_REG(0) |
867e359b CM |
956 | pop_reg_zero r30, r2, sp, PTREGS_OFFSET_PC - PTREGS_OFFSET_REG(30) |
957 | pop_reg_zero r21, r3, sp, PTREGS_OFFSET_EX1 - PTREGS_OFFSET_PC | |
958 | pop_reg_zero lr, r4, sp, PTREGS_OFFSET_REG(52) - PTREGS_OFFSET_EX1 | |
959 | { | |
960 | mtspr EX_CONTEXT_1_0, r21 | |
961 | move r5, zero | |
962 | } | |
963 | { | |
964 | mtspr EX_CONTEXT_1_1, lr | |
965 | andi lr, lr, SPR_EX_CONTEXT_1_1__PL_MASK /* mask off ICS */ | |
966 | } | |
967 | ||
968 | /* Restore callee-saveds that we actually use. */ | |
969 | pop_reg_zero r52, r6, sp, PTREGS_OFFSET_REG(31) - PTREGS_OFFSET_REG(52) | |
970 | pop_reg_zero r31, r7 | |
971 | pop_reg_zero r32, r8 | |
972 | pop_reg_zero r33, r9, sp, PTREGS_OFFSET_REG(29) - PTREGS_OFFSET_REG(33) | |
973 | ||
974 | /* | |
975 | * If we modified other callee-saveds, restore them now. | |
976 | * This is rare, but could be via ptrace or signal handler. | |
977 | */ | |
978 | { | |
979 | move r10, zero | |
980 | bbs r20, .Lrestore_callees | |
981 | } | |
982 | .Lcontinue_restore_regs: | |
983 | ||
984 | /* Check if we're returning from a syscall. */ | |
985 | { | |
986 | move r11, zero | |
987 | blzt r20, 1f /* no, so go restore callee-save registers */ | |
988 | } | |
989 | ||
990 | /* | |
991 | * Check if we're returning to userspace. | |
992 | * Note that if we're not, we don't worry about zeroing everything. | |
993 | */ | |
994 | { | |
995 | addli sp, sp, PTREGS_OFFSET_LR - PTREGS_OFFSET_REG(29) | |
996 | bnz lr, .Lkernel_return | |
997 | } | |
998 | ||
999 | /* | |
1000 | * On return from syscall, we've restored r0 from pt_regs, but we | |
1001 | * clear the remainder of the caller-saved registers. We could | |
1002 | * restore the syscall arguments, but there's not much point, | |
1003 | * and it ensures user programs aren't trying to use the | |
1004 | * caller-saves if we clear them, as well as avoiding leaking | |
1005 | * kernel pointers into userspace. | |
1006 | */ | |
1007 | pop_reg_zero lr, r12, sp, PTREGS_OFFSET_TP - PTREGS_OFFSET_LR | |
1008 | pop_reg_zero tp, r13, sp, PTREGS_OFFSET_SP - PTREGS_OFFSET_TP | |
1009 | { | |
1010 | lw sp, sp | |
1011 | move r14, zero | |
1012 | move r15, zero | |
1013 | } | |
1014 | { move r16, zero; move r17, zero } | |
1015 | { move r18, zero; move r19, zero } | |
1016 | { move r20, zero; move r21, zero } | |
1017 | { move r22, zero; move r23, zero } | |
1018 | { move r24, zero; move r25, zero } | |
1019 | { move r26, zero; move r27, zero } | |
ba00376b CM |
1020 | |
1021 | /* Set r1 to errno if we are returning an error, otherwise zero. */ | |
1022 | { | |
1023 | moveli r29, 1024 | |
1024 | sub r1, zero, r0 | |
1025 | } | |
1026 | slt_u r29, r1, r29 | |
1027 | { | |
1028 | mnz r1, r29, r1 | |
1029 | move r29, zero | |
1030 | } | |
867e359b CM |
1031 | iret |
1032 | ||
1033 | /* | |
1034 | * Not a syscall, so restore caller-saved registers. | |
1035 | * First kick off a load for cache line 1, which we're touching | |
1036 | * for the first time here. | |
1037 | */ | |
1038 | .align 64 | |
1039 | 1: pop_reg r29, sp, PTREGS_OFFSET_REG(1) - PTREGS_OFFSET_REG(29) | |
1040 | pop_reg r1 | |
1041 | pop_reg r2 | |
1042 | pop_reg r3 | |
1043 | pop_reg r4 | |
1044 | pop_reg r5 | |
1045 | pop_reg r6 | |
1046 | pop_reg r7 | |
1047 | pop_reg r8 | |
1048 | pop_reg r9 | |
1049 | pop_reg r10 | |
1050 | pop_reg r11 | |
1051 | pop_reg r12 | |
1052 | pop_reg r13 | |
1053 | pop_reg r14 | |
1054 | pop_reg r15 | |
1055 | pop_reg r16 | |
1056 | pop_reg r17 | |
1057 | pop_reg r18 | |
1058 | pop_reg r19 | |
1059 | pop_reg r20 | |
1060 | pop_reg r21 | |
1061 | pop_reg r22 | |
1062 | pop_reg r23 | |
1063 | pop_reg r24 | |
1064 | pop_reg r25 | |
1065 | pop_reg r26 | |
1066 | pop_reg r27 | |
1067 | pop_reg r28, sp, PTREGS_OFFSET_LR - PTREGS_OFFSET_REG(28) | |
1068 | /* r29 already restored above */ | |
1069 | bnz lr, .Lkernel_return | |
1070 | pop_reg lr, sp, PTREGS_OFFSET_TP - PTREGS_OFFSET_LR | |
1071 | pop_reg tp, sp, PTREGS_OFFSET_SP - PTREGS_OFFSET_TP | |
1072 | lw sp, sp | |
1073 | iret | |
1074 | ||
1075 | /* | |
1076 | * We can't restore tp when in kernel mode, since a thread might | |
1077 | * have migrated from another cpu and brought a stale tp value. | |
1078 | */ | |
1079 | .Lkernel_return: | |
1080 | pop_reg lr, sp, PTREGS_OFFSET_SP - PTREGS_OFFSET_LR | |
1081 | lw sp, sp | |
1082 | iret | |
1083 | ||
1084 | /* Restore callee-saved registers from r34 to r51. */ | |
1085 | .Lrestore_callees: | |
1086 | addli sp, sp, PTREGS_OFFSET_REG(34) - PTREGS_OFFSET_REG(29) | |
1087 | pop_reg r34 | |
1088 | pop_reg r35 | |
1089 | pop_reg r36 | |
1090 | pop_reg r37 | |
1091 | pop_reg r38 | |
1092 | pop_reg r39 | |
1093 | pop_reg r40 | |
1094 | pop_reg r41 | |
1095 | pop_reg r42 | |
1096 | pop_reg r43 | |
1097 | pop_reg r44 | |
1098 | pop_reg r45 | |
1099 | pop_reg r46 | |
1100 | pop_reg r47 | |
1101 | pop_reg r48 | |
1102 | pop_reg r49 | |
1103 | pop_reg r50 | |
1104 | pop_reg r51, sp, PTREGS_OFFSET_REG(29) - PTREGS_OFFSET_REG(51) | |
1105 | j .Lcontinue_restore_regs | |
1106 | ||
1107 | .Lwork_pending: | |
1108 | /* Mask the reschedule flag */ | |
1109 | andi r28, r29, _TIF_NEED_RESCHED | |
1110 | ||
1111 | { | |
1112 | /* | |
1113 | * If the NEED_RESCHED flag is called, we call schedule(), which | |
1114 | * may drop this context right here and go do something else. | |
1115 | * On return, jump back to .Lresume_userspace and recheck. | |
1116 | */ | |
1117 | bz r28, .Lasync_tlb | |
1118 | ||
1119 | /* Mask the async-tlb flag */ | |
1120 | andi r28, r29, _TIF_ASYNC_TLB | |
1121 | } | |
1122 | ||
1123 | jal schedule | |
1124 | FEEDBACK_REENTER(interrupt_return) | |
1125 | ||
1126 | /* Reload the flags and check again */ | |
1127 | j .Lresume_userspace | |
1128 | ||
1129 | .Lasync_tlb: | |
1130 | { | |
1131 | bz r28, .Lneed_sigpending | |
1132 | ||
1133 | /* Mask the sigpending flag */ | |
1134 | andi r28, r29, _TIF_SIGPENDING | |
1135 | } | |
1136 | ||
1137 | PTREGS_PTR(r0, PTREGS_OFFSET_BASE) | |
1138 | jal do_async_page_fault | |
1139 | FEEDBACK_REENTER(interrupt_return) | |
1140 | ||
1141 | /* | |
1142 | * Go restart the "resume userspace" process. We may have | |
1143 | * fired a signal, and we need to disable interrupts again. | |
1144 | */ | |
1145 | j .Lresume_userspace | |
1146 | ||
1147 | .Lneed_sigpending: | |
1148 | /* | |
1149 | * At this point we are either doing signal handling or single-step, | |
1150 | * so either way make sure we have all the registers saved. | |
1151 | */ | |
1152 | push_extra_callee_saves r0 | |
1153 | ||
1154 | { | |
1155 | /* If no signal pending, skip to singlestep check */ | |
1156 | bz r28, .Lneed_singlestep | |
1157 | ||
1158 | /* Mask the singlestep flag */ | |
1159 | andi r28, r29, _TIF_SINGLESTEP | |
1160 | } | |
1161 | ||
1162 | jal do_signal | |
1163 | FEEDBACK_REENTER(interrupt_return) | |
1164 | ||
1165 | /* Reload the flags and check again */ | |
1166 | j .Lresume_userspace | |
1167 | ||
1168 | .Lneed_singlestep: | |
1169 | { | |
1170 | /* Get a pointer to the EX1 field */ | |
1171 | PTREGS_PTR(r29, PTREGS_OFFSET_EX1) | |
1172 | ||
1173 | /* If we get here, our bit must be set. */ | |
1174 | bz r28, .Lwork_confusion | |
1175 | } | |
1176 | /* If we are in priv mode, don't single step */ | |
1177 | lw r28, r29 | |
1178 | andi r28, r28, SPR_EX_CONTEXT_1_1__PL_MASK /* mask off ICS */ | |
1179 | bnz r28, .Lrestore_all | |
1180 | ||
1181 | /* Allow interrupts within the single step code */ | |
1182 | TRACE_IRQS_ON /* Note: clobbers registers r0-r29 */ | |
1183 | IRQ_ENABLE(r20, r21) | |
1184 | ||
1185 | /* try to single-step the current instruction */ | |
1186 | PTREGS_PTR(r0, PTREGS_OFFSET_BASE) | |
1187 | jal single_step_once | |
1188 | FEEDBACK_REENTER(interrupt_return) | |
1189 | ||
1190 | /* Re-disable interrupts. TRACE_IRQS_OFF in .Lrestore_all. */ | |
1191 | IRQ_DISABLE(r20,r21) | |
1192 | ||
1193 | j .Lrestore_all | |
1194 | ||
1195 | .Lwork_confusion: | |
1196 | move r0, r28 | |
1197 | panic "thread_info allwork flags unhandled on userspace resume: %#x" | |
1198 | ||
1199 | STD_ENDPROC(interrupt_return) | |
1200 | ||
1201 | /* | |
1202 | * This interrupt variant clears the INT_INTCTRL_1 interrupt mask bit | |
1203 | * before returning, so we can properly get more downcalls. | |
1204 | */ | |
1205 | .pushsection .text.handle_interrupt_downcall,"ax" | |
1206 | handle_interrupt_downcall: | |
1207 | finish_interrupt_save handle_interrupt_downcall | |
1208 | check_single_stepping normal, .Ldispatch_downcall | |
1209 | .Ldispatch_downcall: | |
1210 | ||
1211 | /* Clear INTCTRL_1 from the set of interrupts we ever enable. */ | |
1212 | GET_INTERRUPTS_ENABLED_MASK_PTR(r30) | |
1213 | { | |
1214 | addi r30, r30, 4 | |
1215 | movei r31, INT_MASK(INT_INTCTRL_1) | |
1216 | } | |
1217 | { | |
1218 | lw r20, r30 | |
1219 | nor r21, r31, zero | |
1220 | } | |
1221 | and r20, r20, r21 | |
1222 | sw r30, r20 | |
1223 | ||
1224 | { | |
1225 | jalr r0 | |
1226 | PTREGS_PTR(r0, PTREGS_OFFSET_BASE) | |
1227 | } | |
1228 | FEEDBACK_REENTER(handle_interrupt_downcall) | |
1229 | ||
1230 | /* Allow INTCTRL_1 to be enabled next time we enable interrupts. */ | |
1231 | lw r20, r30 | |
1232 | or r20, r20, r31 | |
1233 | sw r30, r20 | |
1234 | ||
1235 | { | |
1236 | movei r30, 0 /* not an NMI */ | |
1237 | j interrupt_return | |
1238 | } | |
1239 | STD_ENDPROC(handle_interrupt_downcall) | |
1240 | ||
1241 | /* | |
1242 | * Some interrupts don't check for single stepping | |
1243 | */ | |
1244 | .pushsection .text.handle_interrupt_no_single_step,"ax" | |
1245 | handle_interrupt_no_single_step: | |
1246 | finish_interrupt_save handle_interrupt_no_single_step | |
1247 | { | |
1248 | jalr r0 | |
1249 | PTREGS_PTR(r0, PTREGS_OFFSET_BASE) | |
1250 | } | |
1251 | FEEDBACK_REENTER(handle_interrupt_no_single_step) | |
1252 | { | |
1253 | movei r30, 0 /* not an NMI */ | |
1254 | j interrupt_return | |
1255 | } | |
1256 | STD_ENDPROC(handle_interrupt_no_single_step) | |
1257 | ||
1258 | /* | |
1259 | * "NMI" interrupts mask ALL interrupts before calling the | |
1260 | * handler, and don't check thread flags, etc., on the way | |
1261 | * back out. In general, the only things we do here for NMIs | |
1262 | * are the register save/restore, fixing the PC if we were | |
1263 | * doing single step, and the dataplane kernel-TLB management. | |
1264 | * We don't (for example) deal with start/stop of the sched tick. | |
1265 | */ | |
1266 | .pushsection .text.handle_nmi,"ax" | |
1267 | handle_nmi: | |
1268 | finish_interrupt_save handle_nmi | |
1269 | check_single_stepping normal, .Ldispatch_nmi | |
1270 | .Ldispatch_nmi: | |
1271 | { | |
1272 | jalr r0 | |
1273 | PTREGS_PTR(r0, PTREGS_OFFSET_BASE) | |
1274 | } | |
1275 | FEEDBACK_REENTER(handle_nmi) | |
1276 | j interrupt_return | |
1277 | STD_ENDPROC(handle_nmi) | |
1278 | ||
1279 | /* | |
1280 | * Parallel code for syscalls to handle_interrupt. | |
1281 | */ | |
1282 | .pushsection .text.handle_syscall,"ax" | |
1283 | handle_syscall: | |
1284 | finish_interrupt_save handle_syscall | |
1285 | ||
1286 | /* | |
1287 | * Check for if we are single stepping in user level. If so, then | |
1288 | * we need to restore the PC. | |
1289 | */ | |
1290 | check_single_stepping syscall, .Ldispatch_syscall | |
1291 | .Ldispatch_syscall: | |
1292 | ||
1293 | /* Enable irqs. */ | |
1294 | TRACE_IRQS_ON | |
1295 | IRQ_ENABLE(r20, r21) | |
1296 | ||
1297 | /* Bump the counter for syscalls made on this tile. */ | |
1298 | moveli r20, lo16(irq_stat + IRQ_CPUSTAT_SYSCALL_COUNT_OFFSET) | |
1299 | auli r20, r20, ha16(irq_stat + IRQ_CPUSTAT_SYSCALL_COUNT_OFFSET) | |
1300 | add r20, r20, tp | |
1301 | lw r21, r20 | |
1302 | addi r21, r21, 1 | |
1303 | sw r20, r21 | |
1304 | ||
1305 | /* Trace syscalls, if requested. */ | |
1306 | GET_THREAD_INFO(r31) | |
1307 | addi r31, r31, THREAD_INFO_FLAGS_OFFSET | |
1308 | lw r30, r31 | |
1309 | andi r30, r30, _TIF_SYSCALL_TRACE | |
1310 | bzt r30, .Lrestore_syscall_regs | |
1311 | jal do_syscall_trace | |
1312 | FEEDBACK_REENTER(handle_syscall) | |
1313 | ||
1314 | /* | |
1315 | * We always reload our registers from the stack at this | |
1316 | * point. They might be valid, if we didn't build with | |
1317 | * TRACE_IRQFLAGS, and this isn't a dataplane tile, and we're not | |
1318 | * doing syscall tracing, but there are enough cases now that it | |
1319 | * seems simplest just to do the reload unconditionally. | |
1320 | */ | |
1321 | .Lrestore_syscall_regs: | |
1322 | PTREGS_PTR(r11, PTREGS_OFFSET_REG(0)) | |
1323 | pop_reg r0, r11 | |
1324 | pop_reg r1, r11 | |
1325 | pop_reg r2, r11 | |
1326 | pop_reg r3, r11 | |
1327 | pop_reg r4, r11 | |
1328 | pop_reg r5, r11, PTREGS_OFFSET_SYSCALL - PTREGS_OFFSET_REG(5) | |
1329 | pop_reg TREG_SYSCALL_NR_NAME, r11 | |
1330 | ||
1331 | /* Ensure that the syscall number is within the legal range. */ | |
1332 | moveli r21, __NR_syscalls | |
1333 | { | |
1334 | slt_u r21, TREG_SYSCALL_NR_NAME, r21 | |
1335 | moveli r20, lo16(sys_call_table) | |
1336 | } | |
1337 | { | |
1338 | bbns r21, .Linvalid_syscall | |
1339 | auli r20, r20, ha16(sys_call_table) | |
1340 | } | |
1341 | s2a r20, TREG_SYSCALL_NR_NAME, r20 | |
1342 | lw r20, r20 | |
1343 | ||
1344 | /* Jump to syscall handler. */ | |
1345 | jalr r20; .Lhandle_syscall_link: | |
1346 | FEEDBACK_REENTER(handle_syscall) | |
1347 | ||
1348 | /* | |
1349 | * Write our r0 onto the stack so it gets restored instead | |
1350 | * of whatever the user had there before. | |
1351 | */ | |
1352 | PTREGS_PTR(r29, PTREGS_OFFSET_REG(0)) | |
1353 | sw r29, r0 | |
1354 | ||
1355 | /* Do syscall trace again, if requested. */ | |
1356 | lw r30, r31 | |
1357 | andi r30, r30, _TIF_SYSCALL_TRACE | |
1358 | bzt r30, 1f | |
1359 | jal do_syscall_trace | |
1360 | FEEDBACK_REENTER(handle_syscall) | |
1361 | 1: j .Lresume_userspace /* jump into middle of interrupt_return */ | |
1362 | ||
1363 | .Linvalid_syscall: | |
1364 | /* Report an invalid syscall back to the user program */ | |
1365 | { | |
1366 | PTREGS_PTR(r29, PTREGS_OFFSET_REG(0)) | |
1367 | movei r28, -ENOSYS | |
1368 | } | |
1369 | sw r29, r28 | |
1370 | j .Lresume_userspace /* jump into middle of interrupt_return */ | |
1371 | STD_ENDPROC(handle_syscall) | |
1372 | ||
1373 | /* Return the address for oprofile to suppress in backtraces. */ | |
1374 | STD_ENTRY_SECTION(handle_syscall_link_address, .text.handle_syscall) | |
1375 | lnk r0 | |
1376 | { | |
1377 | addli r0, r0, .Lhandle_syscall_link - . | |
1378 | jrp lr | |
1379 | } | |
1380 | STD_ENDPROC(handle_syscall_link_address) | |
1381 | ||
1382 | STD_ENTRY(ret_from_fork) | |
1383 | jal sim_notify_fork | |
1384 | jal schedule_tail | |
1385 | FEEDBACK_REENTER(ret_from_fork) | |
1386 | j .Lresume_userspace /* jump into middle of interrupt_return */ | |
1387 | STD_ENDPROC(ret_from_fork) | |
1388 | ||
1389 | /* | |
1390 | * Code for ill interrupt. | |
1391 | */ | |
1392 | .pushsection .text.handle_ill,"ax" | |
1393 | handle_ill: | |
1394 | finish_interrupt_save handle_ill | |
1395 | ||
1396 | /* | |
1397 | * Check for if we are single stepping in user level. If so, then | |
1398 | * we need to restore the PC. | |
1399 | */ | |
1400 | check_single_stepping ill, .Ldispatch_normal_ill | |
1401 | ||
1402 | { | |
1403 | /* See if the PC is the 1st bundle in the buffer */ | |
1404 | seq r25, r27, r26 | |
1405 | ||
1406 | /* Point to the 2nd bundle in the buffer */ | |
1407 | addi r26, r26, 8 | |
1408 | } | |
1409 | { | |
1410 | /* Point to the original pc */ | |
1411 | addi r24, r29, SINGLESTEP_STATE_ORIG_PC_OFFSET | |
1412 | ||
1413 | /* Branch if the PC is the 1st bundle in the buffer */ | |
1414 | bnz r25, 3f | |
1415 | } | |
1416 | { | |
1417 | /* See if the PC is the 2nd bundle of the buffer */ | |
1418 | seq r25, r27, r26 | |
1419 | ||
1420 | /* Set PC to next instruction */ | |
1421 | addi r24, r29, SINGLESTEP_STATE_NEXT_PC_OFFSET | |
1422 | } | |
1423 | { | |
1424 | /* Point to flags */ | |
1425 | addi r25, r29, SINGLESTEP_STATE_FLAGS_OFFSET | |
1426 | ||
1427 | /* Branch if PC is in the second bundle */ | |
1428 | bz r25, 2f | |
1429 | } | |
1430 | /* Load flags */ | |
1431 | lw r25, r25 | |
1432 | { | |
1433 | /* | |
1434 | * Get the offset for the register to restore | |
1435 | * Note: the lower bound is 2, so we have implicit scaling by 4. | |
1436 | * No multiplication of the register number by the size of a register | |
1437 | * is needed. | |
1438 | */ | |
1439 | mm r27, r25, zero, SINGLESTEP_STATE_TARGET_LB, \ | |
1440 | SINGLESTEP_STATE_TARGET_UB | |
1441 | ||
1442 | /* Mask Rewrite_LR */ | |
1443 | andi r25, r25, SINGLESTEP_STATE_MASK_UPDATE | |
1444 | } | |
1445 | { | |
1446 | addi r29, r29, SINGLESTEP_STATE_UPDATE_VALUE_OFFSET | |
1447 | ||
1448 | /* Don't rewrite temp register */ | |
1449 | bz r25, 3f | |
1450 | } | |
1451 | { | |
1452 | /* Get the temp value */ | |
1453 | lw r29, r29 | |
1454 | ||
1455 | /* Point to where the register is stored */ | |
1456 | add r27, r27, sp | |
1457 | } | |
1458 | ||
1459 | /* Add in the C ABI save area size to the register offset */ | |
1460 | addi r27, r27, C_ABI_SAVE_AREA_SIZE | |
1461 | ||
1462 | /* Restore the user's register with the temp value */ | |
1463 | sw r27, r29 | |
1464 | j 3f | |
1465 | ||
1466 | 2: | |
1467 | /* Must be in the third bundle */ | |
1468 | addi r24, r29, SINGLESTEP_STATE_BRANCH_NEXT_PC_OFFSET | |
1469 | ||
1470 | 3: | |
1471 | /* set PC and continue */ | |
1472 | lw r26, r24 | |
1473 | sw r28, r26 | |
1474 | ||
1475 | /* Clear TIF_SINGLESTEP */ | |
1476 | GET_THREAD_INFO(r0) | |
1477 | ||
1478 | addi r1, r0, THREAD_INFO_FLAGS_OFFSET | |
1479 | { | |
1480 | lw r2, r1 | |
1481 | addi r0, r0, THREAD_INFO_TASK_OFFSET /* currently a no-op */ | |
1482 | } | |
1483 | andi r2, r2, ~_TIF_SINGLESTEP | |
1484 | sw r1, r2 | |
1485 | ||
1486 | /* Issue a sigtrap */ | |
1487 | { | |
1488 | lw r0, r0 /* indirect thru thread_info to get task_info*/ | |
1489 | addi r1, sp, C_ABI_SAVE_AREA_SIZE /* put ptregs pointer into r1 */ | |
1490 | move r2, zero /* load error code into r2 */ | |
1491 | } | |
1492 | ||
1493 | jal send_sigtrap /* issue a SIGTRAP */ | |
1494 | FEEDBACK_REENTER(handle_ill) | |
1495 | j .Lresume_userspace /* jump into middle of interrupt_return */ | |
1496 | ||
1497 | .Ldispatch_normal_ill: | |
1498 | { | |
1499 | jalr r0 | |
1500 | PTREGS_PTR(r0, PTREGS_OFFSET_BASE) | |
1501 | } | |
1502 | FEEDBACK_REENTER(handle_ill) | |
1503 | { | |
1504 | movei r30, 0 /* not an NMI */ | |
1505 | j interrupt_return | |
1506 | } | |
1507 | STD_ENDPROC(handle_ill) | |
1508 | ||
1509 | .pushsection .rodata, "a" | |
1510 | .align 8 | |
1511 | bpt_code: | |
1512 | bpt | |
1513 | ENDPROC(bpt_code) | |
1514 | .popsection | |
1515 | ||
1516 | /* Various stub interrupt handlers and syscall handlers */ | |
1517 | ||
1518 | STD_ENTRY_LOCAL(_kernel_double_fault) | |
1519 | mfspr r1, EX_CONTEXT_1_0 | |
1520 | move r2, lr | |
1521 | move r3, sp | |
1522 | move r4, r52 | |
1523 | addi sp, sp, -C_ABI_SAVE_AREA_SIZE | |
1524 | j kernel_double_fault | |
1525 | STD_ENDPROC(_kernel_double_fault) | |
1526 | ||
1527 | STD_ENTRY_LOCAL(bad_intr) | |
1528 | mfspr r2, EX_CONTEXT_1_0 | |
1529 | panic "Unhandled interrupt %#x: PC %#lx" | |
1530 | STD_ENDPROC(bad_intr) | |
1531 | ||
1532 | /* Put address of pt_regs in reg and jump. */ | |
1533 | #define PTREGS_SYSCALL(x, reg) \ | |
1534 | STD_ENTRY(x); \ | |
1535 | { \ | |
1536 | PTREGS_PTR(reg, PTREGS_OFFSET_BASE); \ | |
1537 | j _##x \ | |
1538 | }; \ | |
1539 | STD_ENDPROC(x) | |
1540 | ||
1541 | PTREGS_SYSCALL(sys_execve, r3) | |
1542 | PTREGS_SYSCALL(sys_sigaltstack, r2) | |
1543 | PTREGS_SYSCALL(sys_rt_sigreturn, r0) | |
1544 | ||
1545 | /* Save additional callee-saves to pt_regs, put address in reg and jump. */ | |
1546 | #define PTREGS_SYSCALL_ALL_REGS(x, reg) \ | |
1547 | STD_ENTRY(x); \ | |
1548 | push_extra_callee_saves reg; \ | |
1549 | j _##x; \ | |
1550 | STD_ENDPROC(x) | |
1551 | ||
1552 | PTREGS_SYSCALL_ALL_REGS(sys_fork, r0) | |
1553 | PTREGS_SYSCALL_ALL_REGS(sys_vfork, r0) | |
1554 | PTREGS_SYSCALL_ALL_REGS(sys_clone, r4) | |
1555 | PTREGS_SYSCALL_ALL_REGS(sys_cmpxchg_badaddr, r1) | |
1556 | ||
1557 | /* | |
1558 | * This entrypoint is taken for the cmpxchg and atomic_update fast | |
1559 | * swints. We may wish to generalize it to other fast swints at some | |
1560 | * point, but for now there are just two very similar ones, which | |
1561 | * makes it faster. | |
1562 | * | |
1563 | * The fast swint code is designed to have a small footprint. It does | |
1564 | * not save or restore any GPRs, counting on the caller-save registers | |
1565 | * to be available to it on entry. It does not modify any callee-save | |
1566 | * registers (including "lr"). It does not check what PL it is being | |
1567 | * called at, so you'd better not call it other than at PL0. | |
1568 | * | |
1569 | * It does not use the stack, but since it might be re-interrupted by | |
1570 | * a page fault which would assume the stack was valid, it does | |
1571 | * save/restore the stack pointer and zero it out to make sure it gets reset. | |
1572 | * Since we always keep interrupts disabled, the hypervisor won't | |
1573 | * clobber our EX_CONTEXT_1_x registers, so we don't save/restore them | |
1574 | * (other than to advance the PC on return). | |
1575 | * | |
1576 | * We have to manually validate the user vs kernel address range | |
1577 | * (since at PL1 we can read/write both), and for performance reasons | |
1578 | * we don't allow cmpxchg on the fc000000 memory region, since we only | |
1579 | * validate that the user address is below PAGE_OFFSET. | |
1580 | * | |
1581 | * We place it in the __HEAD section to ensure it is relatively | |
1582 | * near to the intvec_SWINT_1 code (reachable by a conditional branch). | |
1583 | * | |
1584 | * Must match register usage in do_page_fault(). | |
1585 | */ | |
1586 | __HEAD | |
1587 | .align 64 | |
1588 | /* Align much later jump on the start of a cache line. */ | |
1589 | #if !ATOMIC_LOCKS_FOUND_VIA_TABLE() | |
1590 | nop; nop | |
1591 | #endif | |
1592 | ENTRY(sys_cmpxchg) | |
1593 | ||
1594 | /* | |
1595 | * Save "sp" and set it zero for any possible page fault. | |
1596 | * | |
1597 | * HACK: We want to both zero sp and check r0's alignment, | |
1598 | * so we do both at once. If "sp" becomes nonzero we | |
1599 | * know r0 is unaligned and branch to the error handler that | |
1600 | * restores sp, so this is OK. | |
1601 | * | |
1602 | * ICS is disabled right now so having a garbage but nonzero | |
1603 | * sp is OK, since we won't execute any faulting instructions | |
1604 | * when it is nonzero. | |
1605 | */ | |
1606 | { | |
1607 | move r27, sp | |
1608 | andi sp, r0, 3 | |
1609 | } | |
1610 | ||
1611 | /* | |
1612 | * Get the lock address in ATOMIC_LOCK_REG, and also validate that the | |
1613 | * address is less than PAGE_OFFSET, since that won't trap at PL1. | |
1614 | * We only use bits less than PAGE_SHIFT to avoid having to worry | |
1615 | * about aliasing among multiple mappings of the same physical page, | |
1616 | * and we ignore the low 3 bits so we have one lock that covers | |
1617 | * both a cmpxchg64() and a cmpxchg() on either its low or high word. | |
1618 | * NOTE: this code must match __atomic_hashed_lock() in lib/atomic.c. | |
1619 | */ | |
1620 | ||
1621 | #if ATOMIC_LOCKS_FOUND_VIA_TABLE() | |
1622 | { | |
1623 | /* Check for unaligned input. */ | |
1624 | bnz sp, .Lcmpxchg_badaddr | |
1625 | mm r25, r0, zero, 3, PAGE_SHIFT-1 | |
1626 | } | |
1627 | { | |
1628 | crc32_32 r25, zero, r25 | |
1629 | moveli r21, lo16(atomic_lock_ptr) | |
1630 | } | |
1631 | { | |
1632 | auli r21, r21, ha16(atomic_lock_ptr) | |
1633 | auli r23, zero, hi16(PAGE_OFFSET) /* hugepage-aligned */ | |
1634 | } | |
1635 | { | |
1636 | shri r20, r25, 32 - ATOMIC_HASH_L1_SHIFT | |
1637 | slt_u r23, r0, r23 | |
1638 | ||
1639 | /* | |
1640 | * Ensure that the TLB is loaded before we take out the lock. | |
1641 | * On TILEPro, this will start fetching the value all the way | |
1642 | * into our L1 as well (and if it gets modified before we | |
1643 | * grab the lock, it will be invalidated from our cache | |
1644 | * before we reload it). On tile64, we'll start fetching it | |
1645 | * into our L1 if we're the home, and if we're not, we'll | |
1646 | * still at least start fetching it into the home's L2. | |
1647 | */ | |
1648 | lw r26, r0 | |
1649 | } | |
1650 | { | |
1651 | s2a r21, r20, r21 | |
1652 | bbns r23, .Lcmpxchg_badaddr | |
1653 | } | |
1654 | { | |
1655 | lw r21, r21 | |
1656 | seqi r23, TREG_SYSCALL_NR_NAME, __NR_FAST_cmpxchg64 | |
1657 | andi r25, r25, ATOMIC_HASH_L2_SIZE - 1 | |
1658 | } | |
1659 | { | |
1660 | /* Branch away at this point if we're doing a 64-bit cmpxchg. */ | |
1661 | bbs r23, .Lcmpxchg64 | |
1662 | andi r23, r0, 7 /* Precompute alignment for cmpxchg64. */ | |
1663 | } | |
1664 | ||
1665 | { | |
1666 | /* | |
1667 | * We very carefully align the code that actually runs with | |
1668 | * the lock held (nine bundles) so that we know it is all in | |
1669 | * the icache when we start. This instruction (the jump) is | |
1670 | * at the start of the first cache line, address zero mod 64; | |
1671 | * we jump to somewhere in the second cache line to issue the | |
1672 | * tns, then jump back to finish up. | |
1673 | */ | |
1674 | s2a ATOMIC_LOCK_REG_NAME, r25, r21 | |
1675 | j .Lcmpxchg32_tns | |
1676 | } | |
1677 | ||
1678 | #else /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */ | |
1679 | { | |
1680 | /* Check for unaligned input. */ | |
1681 | bnz sp, .Lcmpxchg_badaddr | |
1682 | auli r23, zero, hi16(PAGE_OFFSET) /* hugepage-aligned */ | |
1683 | } | |
1684 | { | |
1685 | /* | |
1686 | * Slide bits into position for 'mm'. We want to ignore | |
1687 | * the low 3 bits of r0, and consider only the next | |
1688 | * ATOMIC_HASH_SHIFT bits. | |
1689 | * Because of C pointer arithmetic, we want to compute this: | |
1690 | * | |
1691 | * ((char*)atomic_locks + | |
1692 | * (((r0 >> 3) & (1 << (ATOMIC_HASH_SIZE - 1))) << 2)) | |
1693 | * | |
1694 | * Instead of two shifts we just ">> 1", and use 'mm' | |
1695 | * to ignore the low and high bits we don't want. | |
1696 | */ | |
1697 | shri r25, r0, 1 | |
1698 | ||
1699 | slt_u r23, r0, r23 | |
1700 | ||
1701 | /* | |
1702 | * Ensure that the TLB is loaded before we take out the lock. | |
1703 | * On tilepro, this will start fetching the value all the way | |
1704 | * into our L1 as well (and if it gets modified before we | |
1705 | * grab the lock, it will be invalidated from our cache | |
1706 | * before we reload it). On tile64, we'll start fetching it | |
1707 | * into our L1 if we're the home, and if we're not, we'll | |
1708 | * still at least start fetching it into the home's L2. | |
1709 | */ | |
1710 | lw r26, r0 | |
1711 | } | |
1712 | { | |
1713 | /* atomic_locks is page aligned so this suffices to get its addr. */ | |
1714 | auli r21, zero, hi16(atomic_locks) | |
1715 | ||
1716 | bbns r23, .Lcmpxchg_badaddr | |
1717 | } | |
1718 | { | |
1719 | /* | |
1720 | * Insert the hash bits into the page-aligned pointer. | |
1721 | * ATOMIC_HASH_SHIFT is so big that we don't actually hash | |
1722 | * the unmasked address bits, as that may cause unnecessary | |
1723 | * collisions. | |
1724 | */ | |
1725 | mm ATOMIC_LOCK_REG_NAME, r25, r21, 2, (ATOMIC_HASH_SHIFT + 2) - 1 | |
1726 | ||
1727 | seqi r23, TREG_SYSCALL_NR_NAME, __NR_FAST_cmpxchg64 | |
1728 | } | |
1729 | { | |
1730 | /* Branch away at this point if we're doing a 64-bit cmpxchg. */ | |
1731 | bbs r23, .Lcmpxchg64 | |
1732 | andi r23, r0, 7 /* Precompute alignment for cmpxchg64. */ | |
1733 | } | |
1734 | { | |
1735 | /* | |
1736 | * We very carefully align the code that actually runs with | |
1737 | * the lock held (nine bundles) so that we know it is all in | |
1738 | * the icache when we start. This instruction (the jump) is | |
1739 | * at the start of the first cache line, address zero mod 64; | |
1740 | * we jump to somewhere in the second cache line to issue the | |
1741 | * tns, then jump back to finish up. | |
1742 | */ | |
1743 | j .Lcmpxchg32_tns | |
1744 | } | |
1745 | ||
1746 | #endif /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */ | |
1747 | ||
1748 | ENTRY(__sys_cmpxchg_grab_lock) | |
1749 | ||
1750 | /* | |
1751 | * Perform the actual cmpxchg or atomic_update. | |
1752 | * Note that __futex_mark_unlocked() in uClibc relies on | |
1753 | * atomic_update() to always perform an "mf", so don't make | |
1754 | * it optional or conditional without modifying that code. | |
1755 | */ | |
1756 | .Ldo_cmpxchg32: | |
1757 | { | |
1758 | lw r21, r0 | |
1759 | seqi r23, TREG_SYSCALL_NR_NAME, __NR_FAST_atomic_update | |
1760 | move r24, r2 | |
1761 | } | |
1762 | { | |
1763 | seq r22, r21, r1 /* See if cmpxchg matches. */ | |
1764 | and r25, r21, r1 /* If atomic_update, compute (*mem & mask) */ | |
1765 | } | |
1766 | { | |
1767 | or r22, r22, r23 /* Skip compare branch for atomic_update. */ | |
1768 | add r25, r25, r2 /* Compute (*mem & mask) + addend. */ | |
1769 | } | |
1770 | { | |
1771 | mvnz r24, r23, r25 /* Use atomic_update value if appropriate. */ | |
1772 | bbns r22, .Lcmpxchg32_mismatch | |
1773 | } | |
1774 | sw r0, r24 | |
1775 | ||
1776 | /* Do slow mtspr here so the following "mf" waits less. */ | |
1777 | { | |
1778 | move sp, r27 | |
1779 | mtspr EX_CONTEXT_1_0, r28 | |
1780 | } | |
1781 | mf | |
1782 | ||
1783 | /* The following instruction is the start of the second cache line. */ | |
1784 | { | |
1785 | move r0, r21 | |
1786 | sw ATOMIC_LOCK_REG_NAME, zero | |
1787 | } | |
1788 | iret | |
1789 | ||
1790 | /* Duplicated code here in the case where we don't overlap "mf" */ | |
1791 | .Lcmpxchg32_mismatch: | |
1792 | { | |
1793 | move r0, r21 | |
1794 | sw ATOMIC_LOCK_REG_NAME, zero | |
1795 | } | |
1796 | { | |
1797 | move sp, r27 | |
1798 | mtspr EX_CONTEXT_1_0, r28 | |
1799 | } | |
1800 | iret | |
1801 | ||
1802 | /* | |
1803 | * The locking code is the same for 32-bit cmpxchg/atomic_update, | |
1804 | * and for 64-bit cmpxchg. We provide it as a macro and put | |
1805 | * it into both versions. We can't share the code literally | |
1806 | * since it depends on having the right branch-back address. | |
1807 | * Note that the first few instructions should share the cache | |
1808 | * line with the second half of the actual locked code. | |
1809 | */ | |
1810 | .macro cmpxchg_lock, bitwidth | |
1811 | ||
1812 | /* Lock; if we succeed, jump back up to the read-modify-write. */ | |
1813 | #ifdef CONFIG_SMP | |
1814 | tns r21, ATOMIC_LOCK_REG_NAME | |
1815 | #else | |
1816 | /* | |
1817 | * Non-SMP preserves all the lock infrastructure, to keep the | |
1818 | * code simpler for the interesting (SMP) case. However, we do | |
1819 | * one small optimization here and in atomic_asm.S, which is | |
1820 | * to fake out acquiring the actual lock in the atomic_lock table. | |
1821 | */ | |
1822 | movei r21, 0 | |
1823 | #endif | |
1824 | ||
1825 | /* Issue the slow SPR here while the tns result is in flight. */ | |
1826 | mfspr r28, EX_CONTEXT_1_0 | |
1827 | ||
1828 | { | |
1829 | addi r28, r28, 8 /* return to the instruction after the swint1 */ | |
1830 | bzt r21, .Ldo_cmpxchg\bitwidth | |
1831 | } | |
1832 | /* | |
1833 | * The preceding instruction is the last thing that must be | |
1834 | * on the second cache line. | |
1835 | */ | |
1836 | ||
1837 | #ifdef CONFIG_SMP | |
1838 | /* | |
1839 | * We failed to acquire the tns lock on our first try. Now use | |
1840 | * bounded exponential backoff to retry, like __atomic_spinlock(). | |
1841 | */ | |
1842 | { | |
1843 | moveli r23, 2048 /* maximum backoff time in cycles */ | |
1844 | moveli r25, 32 /* starting backoff time in cycles */ | |
1845 | } | |
1846 | 1: mfspr r26, CYCLE_LOW /* get start point for this backoff */ | |
1847 | 2: mfspr r22, CYCLE_LOW /* test to see if we've backed off enough */ | |
1848 | sub r22, r22, r26 | |
1849 | slt r22, r22, r25 | |
1850 | bbst r22, 2b | |
1851 | { | |
1852 | shli r25, r25, 1 /* double the backoff; retry the tns */ | |
1853 | tns r21, ATOMIC_LOCK_REG_NAME | |
1854 | } | |
1855 | slt r26, r23, r25 /* is the proposed backoff too big? */ | |
1856 | { | |
1857 | mvnz r25, r26, r23 | |
1858 | bzt r21, .Ldo_cmpxchg\bitwidth | |
1859 | } | |
1860 | j 1b | |
1861 | #endif /* CONFIG_SMP */ | |
1862 | .endm | |
1863 | ||
1864 | .Lcmpxchg32_tns: | |
1865 | cmpxchg_lock 32 | |
1866 | ||
1867 | /* | |
1868 | * This code is invoked from sys_cmpxchg after most of the | |
1869 | * preconditions have been checked. We still need to check | |
1870 | * that r0 is 8-byte aligned, since if it's not we won't | |
1871 | * actually be atomic. However, ATOMIC_LOCK_REG has the atomic | |
1872 | * lock pointer and r27/r28 have the saved SP/PC. | |
1873 | * r23 is holding "r0 & 7" so we can test for alignment. | |
1874 | * The compare value is in r2/r3; the new value is in r4/r5. | |
1875 | * On return, we must put the old value in r0/r1. | |
1876 | */ | |
1877 | .align 64 | |
1878 | .Lcmpxchg64: | |
1879 | { | |
1880 | #if ATOMIC_LOCKS_FOUND_VIA_TABLE() | |
1881 | s2a ATOMIC_LOCK_REG_NAME, r25, r21 | |
1882 | #endif | |
1883 | bzt r23, .Lcmpxchg64_tns | |
1884 | } | |
1885 | j .Lcmpxchg_badaddr | |
1886 | ||
1887 | .Ldo_cmpxchg64: | |
1888 | { | |
1889 | lw r21, r0 | |
1890 | addi r25, r0, 4 | |
1891 | } | |
1892 | { | |
1893 | lw r1, r25 | |
1894 | } | |
1895 | seq r26, r21, r2 | |
1896 | { | |
1897 | bz r26, .Lcmpxchg64_mismatch | |
1898 | seq r26, r1, r3 | |
1899 | } | |
1900 | { | |
1901 | bz r26, .Lcmpxchg64_mismatch | |
1902 | } | |
1903 | sw r0, r4 | |
1904 | sw r25, r5 | |
1905 | ||
1906 | /* | |
1907 | * The 32-bit path provides optimized "match" and "mismatch" | |
1908 | * iret paths, but we don't have enough bundles in this cache line | |
1909 | * to do that, so we just make even the "mismatch" path do an "mf". | |
1910 | */ | |
1911 | .Lcmpxchg64_mismatch: | |
1912 | { | |
1913 | move sp, r27 | |
1914 | mtspr EX_CONTEXT_1_0, r28 | |
1915 | } | |
1916 | mf | |
1917 | { | |
1918 | move r0, r21 | |
1919 | sw ATOMIC_LOCK_REG_NAME, zero | |
1920 | } | |
1921 | iret | |
1922 | ||
1923 | .Lcmpxchg64_tns: | |
1924 | cmpxchg_lock 64 | |
1925 | ||
1926 | ||
1927 | /* | |
1928 | * Reset sp and revector to sys_cmpxchg_badaddr(), which will | |
1929 | * just raise the appropriate signal and exit. Doing it this | |
1930 | * way means we don't have to duplicate the code in intvec.S's | |
1931 | * int_hand macro that locates the top of the stack. | |
1932 | */ | |
1933 | .Lcmpxchg_badaddr: | |
1934 | { | |
1935 | moveli TREG_SYSCALL_NR_NAME, __NR_cmpxchg_badaddr | |
1936 | move sp, r27 | |
1937 | } | |
1938 | j intvec_SWINT_1 | |
1939 | ENDPROC(sys_cmpxchg) | |
1940 | ENTRY(__sys_cmpxchg_end) | |
1941 | ||
1942 | ||
1943 | /* The single-step support may need to read all the registers. */ | |
1944 | int_unalign: | |
1945 | push_extra_callee_saves r0 | |
1946 | j do_trap | |
1947 | ||
1948 | /* Include .intrpt1 array of interrupt vectors */ | |
1949 | .section ".intrpt1", "ax" | |
1950 | ||
1951 | #define op_handle_perf_interrupt bad_intr | |
1952 | #define op_handle_aux_perf_interrupt bad_intr | |
1953 | ||
9f9c0382 | 1954 | #ifndef CONFIG_HARDWALL |
867e359b | 1955 | #define do_hardwall_trap bad_intr |
9f9c0382 | 1956 | #endif |
867e359b CM |
1957 | |
1958 | int_hand INT_ITLB_MISS, ITLB_MISS, \ | |
1959 | do_page_fault, handle_interrupt_no_single_step | |
1960 | int_hand INT_MEM_ERROR, MEM_ERROR, bad_intr | |
1961 | int_hand INT_ILL, ILL, do_trap, handle_ill | |
1962 | int_hand INT_GPV, GPV, do_trap | |
1963 | int_hand INT_SN_ACCESS, SN_ACCESS, do_trap | |
1964 | int_hand INT_IDN_ACCESS, IDN_ACCESS, do_trap | |
1965 | int_hand INT_UDN_ACCESS, UDN_ACCESS, do_trap | |
1966 | int_hand INT_IDN_REFILL, IDN_REFILL, bad_intr | |
1967 | int_hand INT_UDN_REFILL, UDN_REFILL, bad_intr | |
1968 | int_hand INT_IDN_COMPLETE, IDN_COMPLETE, bad_intr | |
1969 | int_hand INT_UDN_COMPLETE, UDN_COMPLETE, bad_intr | |
1970 | int_hand INT_SWINT_3, SWINT_3, do_trap | |
1971 | int_hand INT_SWINT_2, SWINT_2, do_trap | |
1972 | int_hand INT_SWINT_1, SWINT_1, SYSCALL, handle_syscall | |
1973 | int_hand INT_SWINT_0, SWINT_0, do_trap | |
1974 | int_hand INT_UNALIGN_DATA, UNALIGN_DATA, int_unalign | |
1975 | int_hand INT_DTLB_MISS, DTLB_MISS, do_page_fault | |
1976 | int_hand INT_DTLB_ACCESS, DTLB_ACCESS, do_page_fault | |
1977 | int_hand INT_DMATLB_MISS, DMATLB_MISS, do_page_fault | |
1978 | int_hand INT_DMATLB_ACCESS, DMATLB_ACCESS, do_page_fault | |
1979 | int_hand INT_SNITLB_MISS, SNITLB_MISS, do_page_fault | |
1980 | int_hand INT_SN_NOTIFY, SN_NOTIFY, bad_intr | |
1981 | int_hand INT_SN_FIREWALL, SN_FIREWALL, do_hardwall_trap | |
1982 | int_hand INT_IDN_FIREWALL, IDN_FIREWALL, bad_intr | |
1983 | int_hand INT_UDN_FIREWALL, UDN_FIREWALL, do_hardwall_trap | |
1984 | int_hand INT_TILE_TIMER, TILE_TIMER, do_timer_interrupt | |
1985 | int_hand INT_IDN_TIMER, IDN_TIMER, bad_intr | |
1986 | int_hand INT_UDN_TIMER, UDN_TIMER, bad_intr | |
1987 | int_hand INT_DMA_NOTIFY, DMA_NOTIFY, bad_intr | |
1988 | int_hand INT_IDN_CA, IDN_CA, bad_intr | |
1989 | int_hand INT_UDN_CA, UDN_CA, bad_intr | |
1990 | int_hand INT_IDN_AVAIL, IDN_AVAIL, bad_intr | |
1991 | int_hand INT_UDN_AVAIL, UDN_AVAIL, bad_intr | |
1992 | int_hand INT_PERF_COUNT, PERF_COUNT, \ | |
1993 | op_handle_perf_interrupt, handle_nmi | |
1994 | int_hand INT_INTCTRL_3, INTCTRL_3, bad_intr | |
1995 | int_hand INT_INTCTRL_2, INTCTRL_2, bad_intr | |
1996 | dc_dispatch INT_INTCTRL_1, INTCTRL_1 | |
1997 | int_hand INT_INTCTRL_0, INTCTRL_0, bad_intr | |
1998 | int_hand INT_MESSAGE_RCV_DWNCL, MESSAGE_RCV_DWNCL, \ | |
1999 | hv_message_intr, handle_interrupt_downcall | |
2000 | int_hand INT_DEV_INTR_DWNCL, DEV_INTR_DWNCL, \ | |
2001 | tile_dev_intr, handle_interrupt_downcall | |
2002 | int_hand INT_I_ASID, I_ASID, bad_intr | |
2003 | int_hand INT_D_ASID, D_ASID, bad_intr | |
2004 | int_hand INT_DMATLB_MISS_DWNCL, DMATLB_MISS_DWNCL, \ | |
2005 | do_page_fault, handle_interrupt_downcall | |
2006 | int_hand INT_SNITLB_MISS_DWNCL, SNITLB_MISS_DWNCL, \ | |
2007 | do_page_fault, handle_interrupt_downcall | |
2008 | int_hand INT_DMATLB_ACCESS_DWNCL, DMATLB_ACCESS_DWNCL, \ | |
2009 | do_page_fault, handle_interrupt_downcall | |
2010 | int_hand INT_SN_CPL, SN_CPL, bad_intr | |
2011 | int_hand INT_DOUBLE_FAULT, DOUBLE_FAULT, do_trap | |
2012 | #if CHIP_HAS_AUX_PERF_COUNTERS() | |
2013 | int_hand INT_AUX_PERF_COUNT, AUX_PERF_COUNT, \ | |
2014 | op_handle_aux_perf_interrupt, handle_nmi | |
2015 | #endif | |
2016 | ||
2017 | /* Synthetic interrupt delivered only by the simulator */ | |
2018 | int_hand INT_BREAKPOINT, BREAKPOINT, do_breakpoint |