]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
bce495d8 | 2 | #include <linux/init.h> |
1da177e4 LT |
3 | #include <linux/linkage.h> |
4 | ||
5 | #include <asm/assembler.h> | |
e6ae744d | 6 | #include <asm/asm-offsets.h> |
1da177e4 | 7 | #include <asm/errno.h> |
bce495d8 | 8 | #include <asm/thread_info.h> |
19c4d593 | 9 | #include <asm/v7m.h> |
1da177e4 LT |
10 | |
11 | @ Bad Abort numbers | |
12 | @ ----------------- | |
13 | @ | |
14 | #define BAD_PREFETCH 0 | |
15 | #define BAD_DATA 1 | |
16 | #define BAD_ADDREXCPTN 2 | |
17 | #define BAD_IRQ 3 | |
18 | #define BAD_UNDEFINSTR 4 | |
19 | ||
1da177e4 | 20 | @ |
925c8a1a RK |
21 | @ Most of the stack format comes from struct pt_regs, but with |
22 | @ the addition of 8 bytes for storing syscall args 5 and 6. | |
2dede2d8 | 23 | @ This _must_ remain a multiple of 8 for EABI. |
1da177e4 | 24 | @ |
1da177e4 LT |
25 | #define S_OFF 8 |
26 | ||
925c8a1a RK |
27 | /* |
28 | * The SWI code relies on the fact that R0 is at the bottom of the stack | |
29 | * (due to slow/fast restore user regs). | |
30 | */ | |
31 | #if S_R0 != 0 | |
32 | #error "Please fix" | |
33 | #endif | |
34 | ||
bce495d8 RK |
35 | .macro zero_fp |
36 | #ifdef CONFIG_FRAME_POINTER | |
37 | mov fp, #0 | |
38 | #endif | |
39 | .endm | |
40 | ||
1da177e4 | 41 | #ifdef CONFIG_ALIGNMENT_TRAP |
195b58ad RK |
42 | #define ATRAP(x...) x |
43 | #else | |
44 | #define ATRAP(x...) | |
45 | #endif | |
46 | ||
47 | .macro alignment_trap, rtmp1, rtmp2, label | |
48 | #ifdef CONFIG_ALIGNMENT_TRAP | |
49 | mrc p15, 0, \rtmp2, c1, c0, 0 | |
50 | ldr \rtmp1, \label | |
51 | ldr \rtmp1, [\rtmp1] | |
52 | teq \rtmp1, \rtmp2 | |
53 | mcrne p15, 0, \rtmp1, c1, c0, 0 | |
1da177e4 LT |
54 | #endif |
55 | .endm | |
56 | ||
19c4d593 UKK |
57 | #ifdef CONFIG_CPU_V7M |
58 | /* | |
59 | * ARMv7-M exception entry/exit macros. | |
60 | * | |
61 | * xPSR, ReturnAddress(), LR (R14), R12, R3, R2, R1, and R0 are | |
62 | * automatically saved on the current stack (32 words) before | |
63 | * switching to the exception stack (SP_main). | |
64 | * | |
65 | * If exception is taken while in user mode, SP_main is | |
66 | * empty. Otherwise, SP_main is aligned to 64 bit automatically | |
67 | * (CCR.STKALIGN set). | |
68 | * | |
69 | * Linux assumes that the interrupts are disabled when entering an | |
70 | * exception handler and it may BUG if this is not the case. Interrupts | |
71 | * are disabled during entry and reenabled in the exit macro. | |
72 | * | |
73 | * v7m_exception_slow_exit is used when returning from SVC or PendSV. | |
74 | * When returning to kernel mode, we don't return from exception. | |
75 | */ | |
76 | .macro v7m_exception_entry | |
77 | @ determine the location of the registers saved by the core during | |
78 | @ exception entry. Depending on the mode the cpu was in when the | |
79 | @ exception happend that is either on the main or the process stack. | |
80 | @ Bit 2 of EXC_RETURN stored in the lr register specifies which stack | |
81 | @ was used. | |
82 | tst lr, #EXC_RET_STACK_MASK | |
83 | mrsne r12, psp | |
84 | moveq r12, sp | |
85 | ||
86 | @ we cannot rely on r0-r3 and r12 matching the value saved in the | |
87 | @ exception frame because of tail-chaining. So these have to be | |
88 | @ reloaded. | |
89 | ldmia r12!, {r0-r3} | |
90 | ||
91 | @ Linux expects to have irqs off. Do it here before taking stack space | |
92 | cpsid i | |
93 | ||
5745eef6 | 94 | sub sp, #PT_REGS_SIZE-S_IP |
19c4d593 UKK |
95 | stmdb sp!, {r0-r11} |
96 | ||
97 | @ load saved r12, lr, return address and xPSR. | |
98 | @ r0-r7 are used for signals and never touched from now on. Clobbering | |
99 | @ r8-r12 is OK. | |
100 | mov r9, r12 | |
101 | ldmia r9!, {r8, r10-r12} | |
102 | ||
103 | @ calculate the original stack pointer value. | |
104 | @ r9 currently points to the memory location just above the auto saved | |
105 | @ xPSR. | |
106 | @ The cpu might automatically 8-byte align the stack. Bit 9 | |
107 | @ of the saved xPSR specifies if stack aligning took place. In this case | |
108 | @ another 32-bit value is included in the stack. | |
109 | ||
110 | tst r12, V7M_xPSR_FRAMEPTRALIGN | |
111 | addne r9, r9, #4 | |
112 | ||
113 | @ store saved r12 using str to have a register to hold the base for stm | |
114 | str r8, [sp, #S_IP] | |
115 | add r8, sp, #S_SP | |
116 | @ store r13-r15, xPSR | |
117 | stmia r8!, {r9-r12} | |
118 | @ store old_r0 | |
119 | str r0, [r8] | |
120 | .endm | |
121 | ||
122 | /* | |
123 | * PENDSV and SVCALL are configured to have the same exception | |
124 | * priorities. As a kernel thread runs at SVCALL execution priority it | |
125 | * can never be preempted and so we will never have to return to a | |
126 | * kernel thread here. | |
127 | */ | |
128 | .macro v7m_exception_slow_exit ret_r0 | |
129 | cpsid i | |
130 | ldr lr, =EXC_RET_THREADMODE_PROCESSSTACK | |
131 | ||
132 | @ read original r12, sp, lr, pc and xPSR | |
133 | add r12, sp, #S_IP | |
134 | ldmia r12, {r1-r5} | |
135 | ||
136 | @ an exception frame is always 8-byte aligned. To tell the hardware if | |
137 | @ the sp to be restored is aligned or not set bit 9 of the saved xPSR | |
138 | @ accordingly. | |
139 | tst r2, #4 | |
140 | subne r2, r2, #4 | |
141 | orrne r5, V7M_xPSR_FRAMEPTRALIGN | |
142 | biceq r5, V7M_xPSR_FRAMEPTRALIGN | |
143 | ||
483a6c9d RV |
144 | @ ensure bit 0 is cleared in the PC, otherwise behaviour is |
145 | @ unpredictable | |
146 | bic r4, #1 | |
147 | ||
19c4d593 UKK |
148 | @ write basic exception frame |
149 | stmdb r2!, {r1, r3-r5} | |
150 | ldmia sp, {r1, r3-r5} | |
151 | .if \ret_r0 | |
152 | stmdb r2!, {r0, r3-r5} | |
153 | .else | |
154 | stmdb r2!, {r1, r3-r5} | |
155 | .endif | |
156 | ||
157 | @ restore process sp | |
158 | msr psp, r2 | |
159 | ||
160 | @ restore original r4-r11 | |
161 | ldmia sp!, {r0-r11} | |
162 | ||
163 | @ restore main sp | |
5745eef6 | 164 | add sp, sp, #PT_REGS_SIZE-S_IP |
19c4d593 UKK |
165 | |
166 | cpsie i | |
167 | bx lr | |
168 | .endm | |
169 | #endif /* CONFIG_CPU_V7M */ | |
170 | ||
b86040a5 CM |
171 | @ |
172 | @ Store/load the USER SP and LR registers by switching to the SYS | |
173 | @ mode. Useful in Thumb-2 mode where "stm/ldm rd, {sp, lr}^" is not | |
174 | @ available. Should only be called from SVC mode | |
175 | @ | |
176 | .macro store_user_sp_lr, rd, rtemp, offset = 0 | |
177 | mrs \rtemp, cpsr | |
178 | eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) | |
179 | msr cpsr_c, \rtemp @ switch to the SYS mode | |
180 | ||
181 | str sp, [\rd, #\offset] @ save sp_usr | |
182 | str lr, [\rd, #\offset + 4] @ save lr_usr | |
183 | ||
184 | eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) | |
185 | msr cpsr_c, \rtemp @ switch back to the SVC mode | |
186 | .endm | |
187 | ||
188 | .macro load_user_sp_lr, rd, rtemp, offset = 0 | |
189 | mrs \rtemp, cpsr | |
190 | eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) | |
191 | msr cpsr_c, \rtemp @ switch to the SYS mode | |
192 | ||
193 | ldr sp, [\rd, #\offset] @ load sp_usr | |
194 | ldr lr, [\rd, #\offset + 4] @ load lr_usr | |
195 | ||
196 | eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) | |
197 | msr cpsr_c, \rtemp @ switch back to the SVC mode | |
198 | .endm | |
199 | ||
aa06e5c1 | 200 | |
9b56febe RK |
201 | .macro svc_exit, rpsr, irq = 0 |
202 | .if \irq != 0 | |
f8f02ec2 | 203 | @ IRQs already off |
9b56febe RK |
204 | #ifdef CONFIG_TRACE_IRQFLAGS |
205 | @ The parent context IRQs must have been enabled to get here in | |
206 | @ the first place, so there's no point checking the PSR I bit. | |
207 | bl trace_hardirqs_on | |
208 | #endif | |
209 | .else | |
f8f02ec2 RK |
210 | @ IRQs off again before pulling preserved data off the stack |
211 | disable_irq_notrace | |
9b56febe RK |
212 | #ifdef CONFIG_TRACE_IRQFLAGS |
213 | tst \rpsr, #PSR_I_BIT | |
214 | bleq trace_hardirqs_on | |
215 | tst \rpsr, #PSR_I_BIT | |
216 | blne trace_hardirqs_off | |
217 | #endif | |
218 | .endif | |
e6978e4b | 219 | ldr r1, [sp, #SVC_ADDR_LIMIT] |
2190fed6 | 220 | uaccess_restore |
e6978e4b | 221 | str r1, [tsk, #TI_ADDR_LIMIT] |
aa06e5c1 RK |
222 | |
223 | #ifndef CONFIG_THUMB2_KERNEL | |
224 | @ ARM mode SVC restore | |
b86040a5 | 225 | msr spsr_cxsf, \rpsr |
2c32c65e MR |
226 | #if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K) |
227 | @ We must avoid clrex due to Cortex-A15 erratum #830321 | |
228 | sub r0, sp, #4 @ uninhabited address | |
229 | strex r1, r2, [r0] @ clear the exclusive monitor | |
200b812d | 230 | #endif |
2c32c65e | 231 | ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr |
aa06e5c1 RK |
232 | #else |
233 | @ Thumb mode SVC restore | |
234 | ldr lr, [sp, #S_SP] @ top of the stack | |
235 | ldrd r0, r1, [sp, #S_LR] @ calling lr and pc | |
236 | ||
237 | @ We must avoid clrex due to Cortex-A15 erratum #830321 | |
238 | strex r2, r1, [sp, #S_LR] @ clear the exclusive monitor | |
239 | ||
240 | stmdb lr!, {r0, r1, \rpsr} @ calling lr and rfe context | |
241 | ldmia sp, {r0 - r12} | |
242 | mov sp, lr | |
243 | ldr lr, [sp], #4 | |
244 | rfeia sp! | |
245 | #endif | |
b86040a5 CM |
246 | .endm |
247 | ||
c0e7f7ee DT |
248 | @ |
249 | @ svc_exit_via_fiq - like svc_exit but switches to FIQ mode before exit | |
250 | @ | |
251 | @ This macro acts in a similar manner to svc_exit but switches to FIQ | |
252 | @ mode to restore the final part of the register state. | |
253 | @ | |
254 | @ We cannot use the normal svc_exit procedure because that would | |
255 | @ clobber spsr_svc (FIQ could be delivered during the first few | |
256 | @ instructions of vector_swi meaning its contents have not been | |
257 | @ saved anywhere). | |
258 | @ | |
259 | @ Note that, unlike svc_exit, this macro also does not allow a caller | |
260 | @ supplied rpsr. This is because the FIQ exceptions are not re-entrant | |
261 | @ and the handlers cannot call into the scheduler (meaning the value | |
262 | @ on the stack remains correct). | |
263 | @ | |
264 | .macro svc_exit_via_fiq | |
e6978e4b | 265 | ldr r1, [sp, #SVC_ADDR_LIMIT] |
2190fed6 | 266 | uaccess_restore |
e6978e4b | 267 | str r1, [tsk, #TI_ADDR_LIMIT] |
aa06e5c1 RK |
268 | #ifndef CONFIG_THUMB2_KERNEL |
269 | @ ARM mode restore | |
c0e7f7ee DT |
270 | mov r0, sp |
271 | ldmib r0, {r1 - r14} @ abort is deadly from here onward (it will | |
272 | @ clobber state restored below) | |
273 | msr cpsr_c, #FIQ_MODE | PSR_I_BIT | PSR_F_BIT | |
274 | add r8, r0, #S_PC | |
275 | ldr r9, [r0, #S_PSR] | |
276 | msr spsr_cxsf, r9 | |
277 | ldr r0, [r0, #S_R0] | |
278 | ldmia r8, {pc}^ | |
aa06e5c1 RK |
279 | #else |
280 | @ Thumb mode restore | |
281 | add r0, sp, #S_R2 | |
282 | ldr lr, [sp, #S_LR] | |
283 | ldr sp, [sp, #S_SP] @ abort is deadly from here onward (it will | |
284 | @ clobber state restored below) | |
285 | ldmia r0, {r2 - r12} | |
286 | mov r1, #FIQ_MODE | PSR_I_BIT | PSR_F_BIT | |
287 | msr cpsr_c, r1 | |
288 | sub r0, #S_R2 | |
289 | add r8, r0, #S_PC | |
290 | ldmia r0, {r0 - r1} | |
291 | rfeia r8 | |
292 | #endif | |
c0e7f7ee DT |
293 | .endm |
294 | ||
aa06e5c1 | 295 | |
b86040a5 | 296 | .macro restore_user_regs, fast = 0, offset = 0 |
2190fed6 | 297 | uaccess_enable r1, isb=0 |
aa06e5c1 RK |
298 | #ifndef CONFIG_THUMB2_KERNEL |
299 | @ ARM mode restore | |
a18f3645 DT |
300 | mov r2, sp |
301 | ldr r1, [r2, #\offset + S_PSR] @ get calling cpsr | |
302 | ldr lr, [r2, #\offset + S_PC]! @ get pc | |
3aaf33be | 303 | tst r1, #PSR_I_BIT | 0x0f |
8bafae20 | 304 | bne 1f |
b86040a5 | 305 | msr spsr_cxsf, r1 @ save in spsr_svc |
2c32c65e MR |
306 | #if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K) |
307 | @ We must avoid clrex due to Cortex-A15 erratum #830321 | |
a18f3645 | 308 | strex r1, r2, [r2] @ clear the exclusive monitor |
200b812d | 309 | #endif |
b86040a5 | 310 | .if \fast |
a18f3645 | 311 | ldmdb r2, {r1 - lr}^ @ get calling r1 - lr |
b86040a5 | 312 | .else |
a18f3645 | 313 | ldmdb r2, {r0 - lr}^ @ get calling r0 - lr |
b86040a5 | 314 | .endif |
8e4971f2 AG |
315 | mov r0, r0 @ ARMv5T and earlier require a nop |
316 | @ after ldm {}^ | |
5745eef6 | 317 | add sp, sp, #\offset + PT_REGS_SIZE |
b86040a5 | 318 | movs pc, lr @ return & move spsr_svc into cpsr |
8bafae20 | 319 | 1: bug "Returning to usermode but unexpected PSR bits set?", \@ |
aa06e5c1 RK |
320 | #elif defined(CONFIG_CPU_V7M) |
321 | @ V7M restore. | |
322 | @ Note that we don't need to do clrex here as clearing the local | |
323 | @ monitor is part of the exception entry and exit sequence. | |
19c4d593 UKK |
324 | .if \offset |
325 | add sp, #\offset | |
326 | .endif | |
327 | v7m_exception_slow_exit ret_r0 = \fast | |
aa06e5c1 RK |
328 | #else |
329 | @ Thumb mode restore | |
b86040a5 CM |
330 | mov r2, sp |
331 | load_user_sp_lr r2, r3, \offset + S_SP @ calling sp, lr | |
332 | ldr r1, [sp, #\offset + S_PSR] @ get calling cpsr | |
333 | ldr lr, [sp, #\offset + S_PC] @ get pc | |
334 | add sp, sp, #\offset + S_SP | |
3aaf33be | 335 | tst r1, #PSR_I_BIT | 0x0f |
8bafae20 | 336 | bne 1f |
b86040a5 | 337 | msr spsr_cxsf, r1 @ save in spsr_svc |
2c32c65e MR |
338 | |
339 | @ We must avoid clrex due to Cortex-A15 erratum #830321 | |
340 | strex r1, r2, [sp] @ clear the exclusive monitor | |
341 | ||
b86040a5 CM |
342 | .if \fast |
343 | ldmdb sp, {r1 - r12} @ get calling r1 - r12 | |
344 | .else | |
345 | ldmdb sp, {r0 - r12} @ get calling r0 - r12 | |
346 | .endif | |
5745eef6 | 347 | add sp, sp, #PT_REGS_SIZE - S_SP |
b86040a5 | 348 | movs pc, lr @ return & move spsr_svc into cpsr |
8bafae20 | 349 | 1: bug "Returning to usermode but unexpected PSR bits set?", \@ |
b86040a5 | 350 | #endif /* !CONFIG_THUMB2_KERNEL */ |
aa06e5c1 | 351 | .endm |
1da177e4 | 352 | |
b0088480 KH |
353 | /* |
354 | * Context tracking subsystem. Used to instrument transitions | |
355 | * between user and kernel mode. | |
356 | */ | |
357 | .macro ct_user_exit, save = 1 | |
358 | #ifdef CONFIG_CONTEXT_TRACKING | |
359 | .if \save | |
360 | stmdb sp!, {r0-r3, ip, lr} | |
0c06a5d4 | 361 | bl context_tracking_user_exit |
b0088480 KH |
362 | ldmia sp!, {r0-r3, ip, lr} |
363 | .else | |
0c06a5d4 | 364 | bl context_tracking_user_exit |
b0088480 KH |
365 | .endif |
366 | #endif | |
367 | .endm | |
368 | ||
369 | .macro ct_user_enter, save = 1 | |
370 | #ifdef CONFIG_CONTEXT_TRACKING | |
371 | .if \save | |
372 | stmdb sp!, {r0-r3, ip, lr} | |
0c06a5d4 | 373 | bl context_tracking_user_enter |
b0088480 KH |
374 | ldmia sp!, {r0-r3, ip, lr} |
375 | .else | |
0c06a5d4 | 376 | bl context_tracking_user_enter |
b0088480 KH |
377 | .endif |
378 | #endif | |
379 | .endm | |
380 | ||
1da177e4 LT |
381 | /* |
382 | * These are the registers used in the syscall handler, and allow us to | |
383 | * have in theory up to 7 arguments to a function - r0 to r6. | |
384 | * | |
385 | * r7 is reserved for the system call number for thumb mode. | |
386 | * | |
387 | * Note that tbl == why is intentional. | |
388 | * | |
389 | * We must set at least "tsk" and "why" when calling ret_with_reschedule. | |
390 | */ | |
391 | scno .req r7 @ syscall number | |
392 | tbl .req r8 @ syscall table pointer | |
393 | why .req r8 @ Linux syscall (!= 0) | |
394 | tsk .req r9 @ current thread_info |