[mirror_ubuntu-jammy-kernel.git] / arch / arm / kernel / entry-header.S

/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/init.h>
#include <linux/linkage.h>

#include <asm/assembler.h>
#include <asm/asm-offsets.h>
#include <asm/errno.h>
#include <asm/thread_info.h>
#include <asm/v7m.h>

@ Bad Abort numbers
@ -----------------
@
#define BAD_PREFETCH	0
#define BAD_DATA	1
#define BAD_ADDREXCPTN	2
#define BAD_IRQ		3
#define BAD_UNDEFINSTR	4

@
@ Most of the stack format comes from struct pt_regs, but with
@ the addition of 8 bytes for storing syscall args 5 and 6.
@ This _must_ remain a multiple of 8 for EABI.
@
#define S_OFF		8

/* 
 * The SWI code relies on the fact that R0 is at the bottom of the stack
 * (due to slow/fast restore user regs).
 */
#if S_R0 != 0
#error "Please fix"
#endif

	.macro	zero_fp
#ifdef CONFIG_FRAME_POINTER
	mov	fp, #0
#endif
	.endm

#ifdef CONFIG_ALIGNMENT_TRAP
#define ATRAP(x...) x
#else
#define ATRAP(x...)
#endif

	.macro	alignment_trap, rtmp1, rtmp2, label
#ifdef CONFIG_ALIGNMENT_TRAP
	mrc	p15, 0, \rtmp2, c1, c0, 0
	ldr	\rtmp1, \label
	ldr	\rtmp1, [\rtmp1]
	teq	\rtmp1, \rtmp2
	mcrne	p15, 0, \rtmp1, c1, c0, 0
#endif
	.endm

#ifdef CONFIG_CPU_V7M
/*
 * ARMv7-M exception entry/exit macros.
 *
 * xPSR, ReturnAddress(), LR (R14), R12, R3, R2, R1, and R0 are
 * automatically saved on the current stack (32 words) before
 * switching to the exception stack (SP_main).
 *
 * If exception is taken while in user mode, SP_main is
 * empty. Otherwise, SP_main is aligned to 64 bit automatically
 * (CCR.STKALIGN set).
 *
 * Linux assumes that the interrupts are disabled when entering an
 * exception handler and it may BUG if this is not the case. Interrupts
 * are disabled during entry and reenabled in the exit macro.
 *
 * v7m_exception_slow_exit is used when returning from SVC or PendSV.
 * When returning to kernel mode, we don't return from exception.
 */
	.macro	v7m_exception_entry
	@ determine the location of the registers saved by the core during
	@ exception entry. Depending on the mode the cpu was in when the
	@ exception happend that is either on the main or the process stack.
	@ Bit 2 of EXC_RETURN stored in the lr register specifies which stack
	@ was used.
	tst	lr, #EXC_RET_STACK_MASK
	mrsne	r12, psp
	moveq	r12, sp

	@ we cannot rely on r0-r3 and r12 matching the value saved in the
	@ exception frame because of tail-chaining. So these have to be
	@ reloaded.
	ldmia	r12!, {r0-r3}

	@ Linux expects to have irqs off. Do it here before taking stack space
	cpsid	i

	sub	sp, #PT_REGS_SIZE-S_IP
	stmdb	sp!, {r0-r11}

	@ load saved r12, lr, return address and xPSR.
	@ r0-r7 are used for signals and never touched from now on. Clobbering
	@ r8-r12 is OK.
	mov	r9, r12
	ldmia	r9!, {r8, r10-r12}

	@ calculate the original stack pointer value.
	@ r9 currently points to the memory location just above the auto saved
	@ xPSR.
	@ The cpu might automatically 8-byte align the stack. Bit 9
	@ of the saved xPSR specifies if stack aligning took place. In this case
	@ another 32-bit value is included in the stack.

	tst	r12, V7M_xPSR_FRAMEPTRALIGN
	addne	r9, r9, #4

	@ store saved r12 using str to have a register to hold the base for stm
	str	r8, [sp, #S_IP]
	add	r8, sp, #S_SP
	@ store r13-r15, xPSR
	stmia	r8!, {r9-r12}
	@ store old_r0
	str	r0, [r8]
	.endm

        /*
	 * PENDSV and SVCALL are configured to have the same exception
	 * priorities. As a kernel thread runs at SVCALL execution priority it
	 * can never be preempted and so we will never have to return to a
	 * kernel thread here.
         */
	.macro	v7m_exception_slow_exit ret_r0
	cpsid	i
	ldr	lr, =EXC_RET_THREADMODE_PROCESSSTACK

	@ read original r12, sp, lr, pc and xPSR
	add	r12, sp, #S_IP
	ldmia	r12, {r1-r5}

	@ an exception frame is always 8-byte aligned. To tell the hardware if
	@ the sp to be restored is aligned or not set bit 9 of the saved xPSR
	@ accordingly.
	tst	r2, #4
	subne	r2, r2, #4
	orrne	r5, V7M_xPSR_FRAMEPTRALIGN
	biceq	r5, V7M_xPSR_FRAMEPTRALIGN

	@ ensure bit 0 is cleared in the PC, otherwise behaviour is
	@ unpredictable
	bic	r4, #1

	@ write basic exception frame
	stmdb	r2!, {r1, r3-r5}
	ldmia	sp, {r1, r3-r5}
	.if	\ret_r0
	stmdb	r2!, {r0, r3-r5}
	.else
	stmdb	r2!, {r1, r3-r5}
	.endif

	@ restore process sp
	msr	psp, r2

	@ restore original r4-r11
	ldmia	sp!, {r0-r11}

	@ restore main sp
	add	sp, sp, #PT_REGS_SIZE-S_IP

	cpsie	i
	bx	lr
	.endm
#endif	/* CONFIG_CPU_V7M */

	@
	@ Store/load the USER SP and LR registers by switching to the SYS
	@ mode. Useful in Thumb-2 mode where "stm/ldm rd, {sp, lr}^" is not
	@ available. Should only be called from SVC mode
	@
	.macro	store_user_sp_lr, rd, rtemp, offset = 0
	mrs	\rtemp, cpsr
	eor	\rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE)
	msr	cpsr_c, \rtemp			@ switch to the SYS mode

	str	sp, [\rd, #\offset]		@ save sp_usr
	str	lr, [\rd, #\offset + 4]		@ save lr_usr

	eor	\rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE)
	msr	cpsr_c, \rtemp			@ switch back to the SVC mode
	.endm

	.macro	load_user_sp_lr, rd, rtemp, offset = 0
	mrs	\rtemp, cpsr
	eor	\rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE)
	msr	cpsr_c, \rtemp			@ switch to the SYS mode

	ldr	sp, [\rd, #\offset]		@ load sp_usr
	ldr	lr, [\rd, #\offset + 4]		@ load lr_usr

	eor	\rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE)
	msr	cpsr_c, \rtemp			@ switch back to the SVC mode
	.endm


	.macro	svc_exit, rpsr, irq = 0
	.if	\irq != 0
	@ IRQs already off
#ifdef CONFIG_TRACE_IRQFLAGS
	@ The parent context IRQs must have been enabled to get here in
	@ the first place, so there's no point checking the PSR I bit.
	bl	trace_hardirqs_on
#endif
	.else
	@ IRQs off again before pulling preserved data off the stack
	disable_irq_notrace
#ifdef CONFIG_TRACE_IRQFLAGS
	tst	\rpsr, #PSR_I_BIT
	bleq	trace_hardirqs_on
	tst	\rpsr, #PSR_I_BIT
	blne	trace_hardirqs_off
#endif
	.endif
	ldr	r1, [sp, #SVC_ADDR_LIMIT]
	uaccess_restore
	str	r1, [tsk, #TI_ADDR_LIMIT]

#ifndef CONFIG_THUMB2_KERNEL
	@ ARM mode SVC restore
	msr	spsr_cxsf, \rpsr
#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K)
	@ We must avoid clrex due to Cortex-A15 erratum #830321
	sub	r0, sp, #4			@ uninhabited address
	strex	r1, r2, [r0]			@ clear the exclusive monitor
#endif
	ldmia	sp, {r0 - pc}^			@ load r0 - pc, cpsr
#else
	@ Thumb mode SVC restore
	ldr	lr, [sp, #S_SP]			@ top of the stack
	ldrd	r0, r1, [sp, #S_LR]		@ calling lr and pc

	@ We must avoid clrex due to Cortex-A15 erratum #830321
	strex	r2, r1, [sp, #S_LR]		@ clear the exclusive monitor

	stmdb	lr!, {r0, r1, \rpsr}		@ calling lr and rfe context
	ldmia	sp, {r0 - r12}
	mov	sp, lr
	ldr	lr, [sp], #4
	rfeia	sp!
#endif
	.endm

	@
	@ svc_exit_via_fiq - like svc_exit but switches to FIQ mode before exit
	@
	@ This macro acts in a similar manner to svc_exit but switches to FIQ
	@ mode to restore the final part of the register state.
	@
	@ We cannot use the normal svc_exit procedure because that would
	@ clobber spsr_svc (FIQ could be delivered during the first few
	@ instructions of vector_swi meaning its contents have not been
	@ saved anywhere).
	@
	@ Note that, unlike svc_exit, this macro also does not allow a caller
	@ supplied rpsr. This is because the FIQ exceptions are not re-entrant
	@ and the handlers cannot call into the scheduler (meaning the value
	@ on the stack remains correct).
	@
	.macro  svc_exit_via_fiq
	ldr	r1, [sp, #SVC_ADDR_LIMIT]
	uaccess_restore
	str	r1, [tsk, #TI_ADDR_LIMIT]
#ifndef CONFIG_THUMB2_KERNEL
	@ ARM mode restore
	mov	r0, sp
	ldmib	r0, {r1 - r14}	@ abort is deadly from here onward (it will
				@ clobber state restored below)
	msr	cpsr_c, #FIQ_MODE | PSR_I_BIT | PSR_F_BIT
	add	r8, r0, #S_PC
	ldr	r9, [r0, #S_PSR]
	msr	spsr_cxsf, r9
	ldr	r0, [r0, #S_R0]
	ldmia	r8, {pc}^
#else
	@ Thumb mode restore
	add	r0, sp, #S_R2
	ldr	lr, [sp, #S_LR]
	ldr	sp, [sp, #S_SP] @ abort is deadly from here onward (it will
			        @ clobber state restored below)
	ldmia	r0, {r2 - r12}
	mov	r1, #FIQ_MODE | PSR_I_BIT | PSR_F_BIT
	msr	cpsr_c, r1
	sub	r0, #S_R2
	add	r8, r0, #S_PC
	ldmia	r0, {r0 - r1}
	rfeia	r8
#endif
	.endm


	.macro	restore_user_regs, fast = 0, offset = 0
	uaccess_enable r1, isb=0
#ifndef CONFIG_THUMB2_KERNEL
	@ ARM mode restore
	mov	r2, sp
	ldr	r1, [r2, #\offset + S_PSR]	@ get calling cpsr
	ldr	lr, [r2, #\offset + S_PC]!	@ get pc
	tst	r1, #PSR_I_BIT | 0x0f
	bne	1f
	msr	spsr_cxsf, r1			@ save in spsr_svc
#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K)
	@ We must avoid clrex due to Cortex-A15 erratum #830321
	strex	r1, r2, [r2]			@ clear the exclusive monitor
#endif
	.if	\fast
	ldmdb	r2, {r1 - lr}^			@ get calling r1 - lr
	.else
	ldmdb	r2, {r0 - lr}^			@ get calling r0 - lr
	.endif
	mov	r0, r0				@ ARMv5T and earlier require a nop
						@ after ldm {}^
	add	sp, sp, #\offset + PT_REGS_SIZE
	movs	pc, lr				@ return & move spsr_svc into cpsr
1:	bug	"Returning to usermode but unexpected PSR bits set?", \@
#elif defined(CONFIG_CPU_V7M)
	@ V7M restore.
	@ Note that we don't need to do clrex here as clearing the local
	@ monitor is part of the exception entry and exit sequence.
	.if	\offset
	add	sp, #\offset
	.endif
	v7m_exception_slow_exit ret_r0 = \fast
#else
	@ Thumb mode restore
	mov	r2, sp
	load_user_sp_lr r2, r3, \offset + S_SP	@ calling sp, lr
	ldr	r1, [sp, #\offset + S_PSR]	@ get calling cpsr
	ldr	lr, [sp, #\offset + S_PC]	@ get pc
	add	sp, sp, #\offset + S_SP
	tst	r1, #PSR_I_BIT | 0x0f
	bne	1f
	msr	spsr_cxsf, r1			@ save in spsr_svc

	@ We must avoid clrex due to Cortex-A15 erratum #830321
	strex	r1, r2, [sp]			@ clear the exclusive monitor

	.if	\fast
	ldmdb	sp, {r1 - r12}			@ get calling r1 - r12
	.else
	ldmdb	sp, {r0 - r12}			@ get calling r0 - r12
	.endif
	add	sp, sp, #PT_REGS_SIZE - S_SP
	movs	pc, lr				@ return & move spsr_svc into cpsr
1:	bug	"Returning to usermode but unexpected PSR bits set?", \@
#endif	/* !CONFIG_THUMB2_KERNEL */
	.endm

/*
 * Context tracking subsystem.  Used to instrument transitions
 * between user and kernel mode.
 */
	.macro ct_user_exit, save = 1
#ifdef CONFIG_CONTEXT_TRACKING
	.if	\save
	stmdb   sp!, {r0-r3, ip, lr}
	bl	context_tracking_user_exit
	ldmia	sp!, {r0-r3, ip, lr}
	.else
	bl	context_tracking_user_exit
	.endif
#endif
	.endm

	.macro ct_user_enter, save = 1
#ifdef CONFIG_CONTEXT_TRACKING
	.if	\save
	stmdb   sp!, {r0-r3, ip, lr}
	bl	context_tracking_user_enter
	ldmia	sp!, {r0-r3, ip, lr}
	.else
	bl	context_tracking_user_enter
	.endif
#endif
	.endm

/*
 * These are the registers used in the syscall handler, and allow us to
 * have in theory up to 7 arguments to a function - r0 to r6.
 *
 * r7 is reserved for the system call number for thumb mode.
 *
 * Note that tbl == why is intentional.
 *
 * We must set at least "tsk" and "why" when calling ret_with_reschedule.
 */
scno	.req	r7		@ syscall number
tbl	.req	r8		@ syscall table pointer
why	.req	r8		@ Linux syscall (!= 0)
tsk	.req	r9		@ current thread_info
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
bce495d8	2	#include <linux/init.h>
1da177e4 LT	3	#include <linux/linkage.h>
	4
	5	#include <asm/assembler.h>
e6ae744d	6	#include <asm/asm-offsets.h>
1da177e4	7	#include <asm/errno.h>
bce495d8	8	#include <asm/thread_info.h>
19c4d593	9	#include <asm/v7m.h>
1da177e4 LT	10
	11	@ Bad Abort numbers
	12	@ -----------------
	13	@
	14	#define BAD_PREFETCH 0
	15	#define BAD_DATA 1
	16	#define BAD_ADDREXCPTN 2
	17	#define BAD_IRQ 3
	18	#define BAD_UNDEFINSTR 4
	19
1da177e4	20	@
925c8a1a RK	21	@ Most of the stack format comes from struct pt_regs, but with
925c8a1a RK	22	@ the addition of 8 bytes for storing syscall args 5 and 6.
2dede2d8	23	@ This _must_ remain a multiple of 8 for EABI.
1da177e4	24	@
1da177e4 LT	25	#define S_OFF 8
1da177e4 LT	26
925c8a1a RK	27	/*
	28	* The SWI code relies on the fact that R0 is at the bottom of the stack
	29	* (due to slow/fast restore user regs).
	30	*/
	31	#if S_R0 != 0
	32	#error "Please fix"
	33	#endif
	34
bce495d8 RK	35	.macro zero_fp
	36	#ifdef CONFIG_FRAME_POINTER
	37	mov fp, #0
	38	#endif
	39	.endm
	40
1da177e4	41	#ifdef CONFIG_ALIGNMENT_TRAP
195b58ad RK	42	#define ATRAP(x...) x
	43	#else
	44	#define ATRAP(x...)
	45	#endif
	46
	47	.macro alignment_trap, rtmp1, rtmp2, label
	48	#ifdef CONFIG_ALIGNMENT_TRAP
	49	mrc p15, 0, \rtmp2, c1, c0, 0
	50	ldr \rtmp1, \label
	51	ldr \rtmp1, [\rtmp1]
	52	teq \rtmp1, \rtmp2
	53	mcrne p15, 0, \rtmp1, c1, c0, 0
1da177e4 LT	54	#endif
	55	.endm
	56
19c4d593 UKK	57	#ifdef CONFIG_CPU_V7M
	58	/*
	59	* ARMv7-M exception entry/exit macros.
	60	*
	61	* xPSR, ReturnAddress(), LR (R14), R12, R3, R2, R1, and R0 are
	62	* automatically saved on the current stack (32 words) before
	63	* switching to the exception stack (SP_main).
	64	*
	65	* If exception is taken while in user mode, SP_main is
	66	* empty. Otherwise, SP_main is aligned to 64 bit automatically
	67	* (CCR.STKALIGN set).
	68	*
	69	* Linux assumes that the interrupts are disabled when entering an
	70	* exception handler and it may BUG if this is not the case. Interrupts
	71	* are disabled during entry and reenabled in the exit macro.
	72	*
	73	* v7m_exception_slow_exit is used when returning from SVC or PendSV.
	74	* When returning to kernel mode, we don't return from exception.
	75	*/
	76	.macro v7m_exception_entry
	77	@ determine the location of the registers saved by the core during
	78	@ exception entry. Depending on the mode the cpu was in when the
	79	@ exception happend that is either on the main or the process stack.
	80	@ Bit 2 of EXC_RETURN stored in the lr register specifies which stack
	81	@ was used.
	82	tst lr, #EXC_RET_STACK_MASK
	83	mrsne r12, psp
	84	moveq r12, sp
	85
	86	@ we cannot rely on r0-r3 and r12 matching the value saved in the
	87	@ exception frame because of tail-chaining. So these have to be
	88	@ reloaded.
	89	ldmia r12!, {r0-r3}
	90
	91	@ Linux expects to have irqs off. Do it here before taking stack space
	92	cpsid i
	93
5745eef6	94	sub sp, #PT_REGS_SIZE-S_IP
19c4d593 UKK	95	stmdb sp!, {r0-r11}
	96
	97	@ load saved r12, lr, return address and xPSR.
	98	@ r0-r7 are used for signals and never touched from now on. Clobbering
	99	@ r8-r12 is OK.
	100	mov r9, r12
	101	ldmia r9!, {r8, r10-r12}
	102
	103	@ calculate the original stack pointer value.
	104	@ r9 currently points to the memory location just above the auto saved
	105	@ xPSR.
	106	@ The cpu might automatically 8-byte align the stack. Bit 9
	107	@ of the saved xPSR specifies if stack aligning took place. In this case
	108	@ another 32-bit value is included in the stack.
	109
	110	tst r12, V7M_xPSR_FRAMEPTRALIGN
	111	addne r9, r9, #4
	112
	113	@ store saved r12 using str to have a register to hold the base for stm
	114	str r8, [sp, #S_IP]
	115	add r8, sp, #S_SP
	116	@ store r13-r15, xPSR
	117	stmia r8!, {r9-r12}
	118	@ store old_r0
	119	str r0, [r8]
	120	.endm
	121
	122	/*
	123	* PENDSV and SVCALL are configured to have the same exception
	124	* priorities. As a kernel thread runs at SVCALL execution priority it
	125	* can never be preempted and so we will never have to return to a
	126	* kernel thread here.
	127	*/
	128	.macro v7m_exception_slow_exit ret_r0
	129	cpsid i
	130	ldr lr, =EXC_RET_THREADMODE_PROCESSSTACK
	131
	132	@ read original r12, sp, lr, pc and xPSR
	133	add r12, sp, #S_IP
	134	ldmia r12, {r1-r5}
	135
	136	@ an exception frame is always 8-byte aligned. To tell the hardware if
	137	@ the sp to be restored is aligned or not set bit 9 of the saved xPSR
	138	@ accordingly.
	139	tst r2, #4
	140	subne r2, r2, #4
	141	orrne r5, V7M_xPSR_FRAMEPTRALIGN
	142	biceq r5, V7M_xPSR_FRAMEPTRALIGN
	143
483a6c9d RV	144	@ ensure bit 0 is cleared in the PC, otherwise behaviour is
	145	@ unpredictable
	146	bic r4, #1
	147
19c4d593 UKK	148	@ write basic exception frame
	149	stmdb r2!, {r1, r3-r5}
	150	ldmia sp, {r1, r3-r5}
	151	.if \ret_r0
	152	stmdb r2!, {r0, r3-r5}
	153	.else
	154	stmdb r2!, {r1, r3-r5}
	155	.endif
	156
	157	@ restore process sp
	158	msr psp, r2
	159
	160	@ restore original r4-r11
	161	ldmia sp!, {r0-r11}
	162
	163	@ restore main sp
5745eef6	164	add sp, sp, #PT_REGS_SIZE-S_IP
19c4d593 UKK	165
	166	cpsie i
	167	bx lr
	168	.endm
	169	#endif /* CONFIG_CPU_V7M */
	170
b86040a5 CM	171	@
	172	@ Store/load the USER SP and LR registers by switching to the SYS
	173	@ mode. Useful in Thumb-2 mode where "stm/ldm rd, {sp, lr}^" is not
	174	@ available. Should only be called from SVC mode
	175	@
	176	.macro store_user_sp_lr, rd, rtemp, offset = 0
	177	mrs \rtemp, cpsr
	178	eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE)
	179	msr cpsr_c, \rtemp @ switch to the SYS mode
	180
	181	str sp, [\rd, #\offset] @ save sp_usr
	182	str lr, [\rd, #\offset + 4] @ save lr_usr
	183
	184	eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE)
	185	msr cpsr_c, \rtemp @ switch back to the SVC mode
	186	.endm
	187
	188	.macro load_user_sp_lr, rd, rtemp, offset = 0
	189	mrs \rtemp, cpsr
	190	eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE)
	191	msr cpsr_c, \rtemp @ switch to the SYS mode
	192
	193	ldr sp, [\rd, #\offset] @ load sp_usr
	194	ldr lr, [\rd, #\offset + 4] @ load lr_usr
	195
	196	eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE)
	197	msr cpsr_c, \rtemp @ switch back to the SVC mode
	198	.endm
	199
aa06e5c1	200
9b56febe RK	201	.macro svc_exit, rpsr, irq = 0
9b56febe RK	202	.if \irq != 0
f8f02ec2	203	@ IRQs already off
9b56febe RK	204	#ifdef CONFIG_TRACE_IRQFLAGS
	205	@ The parent context IRQs must have been enabled to get here in
	206	@ the first place, so there's no point checking the PSR I bit.
	207	bl trace_hardirqs_on
	208	#endif
	209	.else
f8f02ec2 RK	210	@ IRQs off again before pulling preserved data off the stack
f8f02ec2 RK	211	disable_irq_notrace
9b56febe RK	212	#ifdef CONFIG_TRACE_IRQFLAGS
	213	tst \rpsr, #PSR_I_BIT
	214	bleq trace_hardirqs_on
	215	tst \rpsr, #PSR_I_BIT
	216	blne trace_hardirqs_off
	217	#endif
	218	.endif
e6978e4b	219	ldr r1, [sp, #SVC_ADDR_LIMIT]
2190fed6	220	uaccess_restore
e6978e4b	221	str r1, [tsk, #TI_ADDR_LIMIT]
aa06e5c1 RK	222
	223	#ifndef CONFIG_THUMB2_KERNEL
	224	@ ARM mode SVC restore
b86040a5	225	msr spsr_cxsf, \rpsr
2c32c65e MR	226	#if defined(CONFIG_CPU_V6) \|\| defined(CONFIG_CPU_32v6K)
	227	@ We must avoid clrex due to Cortex-A15 erratum #830321
	228	sub r0, sp, #4 @ uninhabited address
	229	strex r1, r2, [r0] @ clear the exclusive monitor
200b812d	230	#endif
2c32c65e	231	ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr
aa06e5c1 RK	232	#else
	233	@ Thumb mode SVC restore
	234	ldr lr, [sp, #S_SP] @ top of the stack
	235	ldrd r0, r1, [sp, #S_LR] @ calling lr and pc
	236
	237	@ We must avoid clrex due to Cortex-A15 erratum #830321
	238	strex r2, r1, [sp, #S_LR] @ clear the exclusive monitor
	239
	240	stmdb lr!, {r0, r1, \rpsr} @ calling lr and rfe context
	241	ldmia sp, {r0 - r12}
	242	mov sp, lr
	243	ldr lr, [sp], #4
	244	rfeia sp!
	245	#endif
b86040a5 CM	246	.endm
b86040a5 CM	247
c0e7f7ee DT	248	@
	249	@ svc_exit_via_fiq - like svc_exit but switches to FIQ mode before exit
	250	@
	251	@ This macro acts in a similar manner to svc_exit but switches to FIQ
	252	@ mode to restore the final part of the register state.
	253	@
	254	@ We cannot use the normal svc_exit procedure because that would
	255	@ clobber spsr_svc (FIQ could be delivered during the first few
	256	@ instructions of vector_swi meaning its contents have not been
	257	@ saved anywhere).
	258	@
	259	@ Note that, unlike svc_exit, this macro also does not allow a caller
	260	@ supplied rpsr. This is because the FIQ exceptions are not re-entrant
	261	@ and the handlers cannot call into the scheduler (meaning the value
	262	@ on the stack remains correct).
	263	@
	264	.macro svc_exit_via_fiq
e6978e4b	265	ldr r1, [sp, #SVC_ADDR_LIMIT]
2190fed6	266	uaccess_restore
e6978e4b	267	str r1, [tsk, #TI_ADDR_LIMIT]
aa06e5c1 RK	268	#ifndef CONFIG_THUMB2_KERNEL
aa06e5c1 RK	269	@ ARM mode restore
c0e7f7ee DT	270	mov r0, sp
	271	ldmib r0, {r1 - r14} @ abort is deadly from here onward (it will
	272	@ clobber state restored below)
	273	msr cpsr_c, #FIQ_MODE \| PSR_I_BIT \| PSR_F_BIT
	274	add r8, r0, #S_PC
	275	ldr r9, [r0, #S_PSR]
	276	msr spsr_cxsf, r9
	277	ldr r0, [r0, #S_R0]
	278	ldmia r8, {pc}^
aa06e5c1 RK	279	#else
	280	@ Thumb mode restore
	281	add r0, sp, #S_R2
	282	ldr lr, [sp, #S_LR]
	283	ldr sp, [sp, #S_SP] @ abort is deadly from here onward (it will
	284	@ clobber state restored below)
	285	ldmia r0, {r2 - r12}
	286	mov r1, #FIQ_MODE \| PSR_I_BIT \| PSR_F_BIT
	287	msr cpsr_c, r1
	288	sub r0, #S_R2
	289	add r8, r0, #S_PC
	290	ldmia r0, {r0 - r1}
	291	rfeia r8
	292	#endif
c0e7f7ee DT	293	.endm
c0e7f7ee DT	294
aa06e5c1	295
b86040a5	296	.macro restore_user_regs, fast = 0, offset = 0
2190fed6	297	uaccess_enable r1, isb=0
aa06e5c1 RK	298	#ifndef CONFIG_THUMB2_KERNEL
aa06e5c1 RK	299	@ ARM mode restore
a18f3645 DT	300	mov r2, sp
	301	ldr r1, [r2, #\offset + S_PSR] @ get calling cpsr
	302	ldr lr, [r2, #\offset + S_PC]! @ get pc
3aaf33be	303	tst r1, #PSR_I_BIT \| 0x0f
8bafae20	304	bne 1f
b86040a5	305	msr spsr_cxsf, r1 @ save in spsr_svc
2c32c65e MR	306	#if defined(CONFIG_CPU_V6) \|\| defined(CONFIG_CPU_32v6K)
2c32c65e MR	307	@ We must avoid clrex due to Cortex-A15 erratum #830321
a18f3645	308	strex r1, r2, [r2] @ clear the exclusive monitor
200b812d	309	#endif
b86040a5	310	.if \fast
a18f3645	311	ldmdb r2, {r1 - lr}^ @ get calling r1 - lr
b86040a5	312	.else
a18f3645	313	ldmdb r2, {r0 - lr}^ @ get calling r0 - lr
b86040a5	314	.endif
8e4971f2 AG	315	mov r0, r0 @ ARMv5T and earlier require a nop
8e4971f2 AG	316	@ after ldm {}^
5745eef6	317	add sp, sp, #\offset + PT_REGS_SIZE
b86040a5	318	movs pc, lr @ return & move spsr_svc into cpsr
8bafae20	319	1: bug "Returning to usermode but unexpected PSR bits set?", \@
aa06e5c1 RK	320	#elif defined(CONFIG_CPU_V7M)
	321	@ V7M restore.
	322	@ Note that we don't need to do clrex here as clearing the local
	323	@ monitor is part of the exception entry and exit sequence.
19c4d593 UKK	324	.if \offset
	325	add sp, #\offset
	326	.endif
	327	v7m_exception_slow_exit ret_r0 = \fast
aa06e5c1 RK	328	#else
aa06e5c1 RK	329	@ Thumb mode restore
b86040a5 CM	330	mov r2, sp
	331	load_user_sp_lr r2, r3, \offset + S_SP @ calling sp, lr
	332	ldr r1, [sp, #\offset + S_PSR] @ get calling cpsr
	333	ldr lr, [sp, #\offset + S_PC] @ get pc
	334	add sp, sp, #\offset + S_SP
3aaf33be	335	tst r1, #PSR_I_BIT \| 0x0f
8bafae20	336	bne 1f
b86040a5	337	msr spsr_cxsf, r1 @ save in spsr_svc
2c32c65e MR	338
	339	@ We must avoid clrex due to Cortex-A15 erratum #830321
	340	strex r1, r2, [sp] @ clear the exclusive monitor
	341
b86040a5 CM	342	.if \fast
	343	ldmdb sp, {r1 - r12} @ get calling r1 - r12
	344	.else
	345	ldmdb sp, {r0 - r12} @ get calling r0 - r12
	346	.endif
5745eef6	347	add sp, sp, #PT_REGS_SIZE - S_SP
b86040a5	348	movs pc, lr @ return & move spsr_svc into cpsr
8bafae20	349	1: bug "Returning to usermode but unexpected PSR bits set?", \@
b86040a5	350	#endif /* !CONFIG_THUMB2_KERNEL */
aa06e5c1	351	.endm
1da177e4	352
b0088480 KH	353	/*
	354	* Context tracking subsystem. Used to instrument transitions
	355	* between user and kernel mode.
	356	*/
	357	.macro ct_user_exit, save = 1
	358	#ifdef CONFIG_CONTEXT_TRACKING
	359	.if \save
	360	stmdb sp!, {r0-r3, ip, lr}
0c06a5d4	361	bl context_tracking_user_exit
b0088480 KH	362	ldmia sp!, {r0-r3, ip, lr}
b0088480 KH	363	.else
0c06a5d4	364	bl context_tracking_user_exit
b0088480 KH	365	.endif
	366	#endif
	367	.endm
	368
	369	.macro ct_user_enter, save = 1
	370	#ifdef CONFIG_CONTEXT_TRACKING
	371	.if \save
	372	stmdb sp!, {r0-r3, ip, lr}
0c06a5d4	373	bl context_tracking_user_enter
b0088480 KH	374	ldmia sp!, {r0-r3, ip, lr}
b0088480 KH	375	.else
0c06a5d4	376	bl context_tracking_user_enter
b0088480 KH	377	.endif
	378	#endif
	379	.endm
	380
1da177e4 LT	381	/*
	382	* These are the registers used in the syscall handler, and allow us to
	383	* have in theory up to 7 arguments to a function - r0 to r6.
	384	*
	385	* r7 is reserved for the system call number for thumb mode.
	386	*
	387	* Note that tbl == why is intentional.
	388	*
	389	* We must set at least "tsk" and "why" when calling ret_with_reschedule.
	390	*/
	391	scno .req r7 @ syscall number
	392	tbl .req r8 @ syscall table pointer
	393	why .req r8 @ Linux syscall (!= 0)
	394	tsk .req r9 @ current thread_info