[mirror_ubuntu-focal-kernel.git] / arch / mips / include / asm / stackframe.h

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 1994, 95, 96, 99, 2001 Ralf Baechle
 * Copyright (C) 1994, 1995, 1996 Paul M. Antoine.
 * Copyright (C) 1999 Silicon Graphics, Inc.
 * Copyright (C) 2007  Maciej W. Rozycki
 */
#ifndef _ASM_STACKFRAME_H
#define _ASM_STACKFRAME_H

#include <linux/threads.h>

#include <asm/asm.h>
#include <asm/asmmacro.h>
#include <asm/mipsregs.h>
#include <asm/asm-offsets.h>
#include <asm/thread_info.h>

/* Make the addition of cfi info a little easier. */
	.macro cfi_rel_offset reg offset=0 docfi=0
	.if \docfi
	.cfi_rel_offset \reg, \offset
	.endif
	.endm

	.macro cfi_st reg offset=0 docfi=0
	LONG_S	\reg, \offset(sp)
	cfi_rel_offset \reg, \offset, \docfi
	.endm

	.macro cfi_restore reg offset=0 docfi=0
	.if \docfi
	.cfi_restore \reg
	.endif
	.endm

	.macro cfi_ld reg offset=0 docfi=0
	LONG_L	\reg, \offset(sp)
	cfi_restore \reg \offset \docfi
	.endm

#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX)
#define STATMASK 0x3f
#else
#define STATMASK 0x1f
#endif

		.macro	SAVE_AT docfi=0
		.set	push
		.set	noat
		cfi_st	$1, PT_R1, \docfi
		.set	pop
		.endm

		.macro	SAVE_TEMP docfi=0
#ifdef CONFIG_CPU_HAS_SMARTMIPS
		mflhxu	v1
		LONG_S	v1, PT_LO(sp)
		mflhxu	v1
		LONG_S	v1, PT_HI(sp)
		mflhxu	v1
		LONG_S	v1, PT_ACX(sp)
#elif !defined(CONFIG_CPU_MIPSR6)
		mfhi	v1
#endif
#ifdef CONFIG_32BIT
		cfi_st	$8, PT_R8, \docfi
		cfi_st	$9, PT_R9, \docfi
#endif
		cfi_st	$10, PT_R10, \docfi
		cfi_st	$11, PT_R11, \docfi
		cfi_st	$12, PT_R12, \docfi
#if !defined(CONFIG_CPU_HAS_SMARTMIPS) && !defined(CONFIG_CPU_MIPSR6)
		LONG_S	v1, PT_HI(sp)
		mflo	v1
#endif
		cfi_st	$13, PT_R13, \docfi
		cfi_st	$14, PT_R14, \docfi
		cfi_st	$15, PT_R15, \docfi
		cfi_st	$24, PT_R24, \docfi
#if !defined(CONFIG_CPU_HAS_SMARTMIPS) && !defined(CONFIG_CPU_MIPSR6)
		LONG_S	v1, PT_LO(sp)
#endif
#ifdef CONFIG_CPU_CAVIUM_OCTEON
		/*
		 * The Octeon multiplier state is affected by general
		 * multiply instructions. It must be saved before and
		 * kernel code might corrupt it
		 */
		jal     octeon_mult_save
#endif
		.endm

		.macro	SAVE_STATIC docfi=0
		cfi_st	$16, PT_R16, \docfi
		cfi_st	$17, PT_R17, \docfi
		cfi_st	$18, PT_R18, \docfi
		cfi_st	$19, PT_R19, \docfi
		cfi_st	$20, PT_R20, \docfi
		cfi_st	$21, PT_R21, \docfi
		cfi_st	$22, PT_R22, \docfi
		cfi_st	$23, PT_R23, \docfi
		cfi_st	$30, PT_R30, \docfi
		.endm

/*
 * get_saved_sp returns the SP for the current CPU by looking in the
 * kernelsp array for it.  If tosp is set, it stores the current sp in
 * k0 and loads the new value in sp.  If not, it clobbers k0 and
 * stores the new value in k1, leaving sp unaffected.
 */
#ifdef CONFIG_SMP

		/* SMP variation */
		.macro	get_saved_sp docfi=0 tosp=0
		ASM_CPUID_MFC0	k0, ASM_SMP_CPUID_REG
#if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32)
		lui	k1, %hi(kernelsp)
#else
		lui	k1, %highest(kernelsp)
		daddiu	k1, %higher(kernelsp)
		dsll	k1, 16
		daddiu	k1, %hi(kernelsp)
		dsll	k1, 16
#endif
		LONG_SRL	k0, SMP_CPUID_PTRSHIFT
		LONG_ADDU	k1, k0
		.if \tosp
		move	k0, sp
		.if \docfi
		.cfi_register sp, k0
		.endif
		LONG_L	sp, %lo(kernelsp)(k1)
		.else
		LONG_L	k1, %lo(kernelsp)(k1)
		.endif
		.endm

		.macro	set_saved_sp stackp temp temp2
		ASM_CPUID_MFC0	\temp, ASM_SMP_CPUID_REG
		LONG_SRL	\temp, SMP_CPUID_PTRSHIFT
		LONG_S	\stackp, kernelsp(\temp)
		.endm
#else /* !CONFIG_SMP */
		/* Uniprocessor variation */
		.macro	get_saved_sp docfi=0 tosp=0
#ifdef CONFIG_CPU_JUMP_WORKAROUNDS
		/*
		 * Clear BTB (branch target buffer), forbid RAS (return address
		 * stack) to workaround the Out-of-order Issue in Loongson2F
		 * via its diagnostic register.
		 */
		move	k0, ra
		jal	1f
		 nop
1:		jal	1f
		 nop
1:		jal	1f
		 nop
1:		jal	1f
		 nop
1:		move	ra, k0
		li	k0, 3
		mtc0	k0, $22
#endif /* CONFIG_CPU_JUMP_WORKAROUNDS */
#if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32)
		lui	k1, %hi(kernelsp)
#else
		lui	k1, %highest(kernelsp)
		daddiu	k1, %higher(kernelsp)
		dsll	k1, k1, 16
		daddiu	k1, %hi(kernelsp)
		dsll	k1, k1, 16
#endif
		.if \tosp
		move	k0, sp
		.if \docfi
		.cfi_register sp, k0
		.endif
		LONG_L	sp, %lo(kernelsp)(k1)
		.else
		LONG_L	k1, %lo(kernelsp)(k1)
		.endif
		.endm

		.macro	set_saved_sp stackp temp temp2
		LONG_S	\stackp, kernelsp
		.endm
#endif

		.macro	SAVE_SOME docfi=0
		.set	push
		.set	noat
		.set	reorder
		mfc0	k0, CP0_STATUS
		sll	k0, 3		/* extract cu0 bit */
		.set	noreorder
		bltz	k0, 8f
#ifdef CONFIG_EVA
		/*
		 * Flush interAptiv's Return Prediction Stack (RPS) by writing
		 * EntryHi. Toggling Config7.RPS is slower and less portable.
		 *
		 * The RPS isn't automatically flushed when exceptions are
		 * taken, which can result in kernel mode speculative accesses
		 * to user addresses if the RPS mispredicts. That's harmless
		 * when user and kernel share the same address space, but with
		 * EVA the same user segments may be unmapped to kernel mode,
		 * even containing sensitive MMIO regions or invalid memory.
		 *
		 * This can happen when the kernel sets the return address to
		 * ret_from_* and jr's to the exception handler, which looks
		 * more like a tail call than a function call. If nested calls
		 * don't evict the last user address in the RPS, it will
		 * mispredict the return and fetch from a user controlled
		 * address into the icache.
		 *
		 * More recent EVA-capable cores with MAAR to restrict
		 * speculative accesses aren't affected.
		 */
		MFC0	k0, CP0_ENTRYHI
		MTC0	k0, CP0_ENTRYHI
#endif
		.set	reorder
		 move	k0, sp
		.if \docfi
		.cfi_register sp, k0
		.endif
		/* Called from user mode, new stack. */
		get_saved_sp docfi=\docfi tosp=1
8:
#ifdef CONFIG_CPU_DADDI_WORKAROUNDS
		.set	at=k1
#endif
		PTR_SUBU sp, PT_SIZE
#ifdef CONFIG_CPU_DADDI_WORKAROUNDS
		.set	noat
#endif
		.if \docfi
		.cfi_def_cfa sp,0
		.endif
		cfi_st	k0, PT_R29, \docfi
		cfi_rel_offset  sp, PT_R29, \docfi
		cfi_st	v1, PT_R3, \docfi
		/*
		 * You might think that you don't need to save $0,
		 * but the FPU emulator and gdb remote debug stub
		 * need it to operate correctly
		 */
		LONG_S	$0, PT_R0(sp)
		mfc0	v1, CP0_STATUS
		cfi_st	v0, PT_R2, \docfi
		LONG_S	v1, PT_STATUS(sp)
		cfi_st	$4, PT_R4, \docfi
		mfc0	v1, CP0_CAUSE
		cfi_st	$5, PT_R5, \docfi
		LONG_S	v1, PT_CAUSE(sp)
		cfi_st	$6, PT_R6, \docfi
		cfi_st	ra, PT_R31, \docfi
		MFC0	ra, CP0_EPC
		cfi_st	$7, PT_R7, \docfi
#ifdef CONFIG_64BIT
		cfi_st	$8, PT_R8, \docfi
		cfi_st	$9, PT_R9, \docfi
#endif
		LONG_S	ra, PT_EPC(sp)
		.if \docfi
		.cfi_rel_offset ra, PT_EPC
		.endif
		cfi_st	$25, PT_R25, \docfi
		cfi_st	$28, PT_R28, \docfi

		/* Set thread_info if we're coming from user mode */
		mfc0	k0, CP0_STATUS
		sll	k0, 3		/* extract cu0 bit */
		bltz	k0, 9f

		ori	$28, sp, _THREAD_MASK
		xori	$28, _THREAD_MASK
#ifdef CONFIG_CPU_CAVIUM_OCTEON
		.set    mips64
		pref    0, 0($28)       /* Prefetch the current pointer */
#endif
9:
		.set	pop
		.endm

		.macro	SAVE_ALL docfi=0
		SAVE_SOME \docfi
		SAVE_AT \docfi
		SAVE_TEMP \docfi
		SAVE_STATIC \docfi
		.endm

		.macro	RESTORE_AT docfi=0
		.set	push
		.set	noat
		cfi_ld	$1, PT_R1, \docfi
		.set	pop
		.endm

		.macro	RESTORE_TEMP docfi=0
#ifdef CONFIG_CPU_CAVIUM_OCTEON
		/* Restore the Octeon multiplier state */
		jal	octeon_mult_restore
#endif
#ifdef CONFIG_CPU_HAS_SMARTMIPS
		LONG_L	$24, PT_ACX(sp)
		mtlhx	$24
		LONG_L	$24, PT_HI(sp)
		mtlhx	$24
		LONG_L	$24, PT_LO(sp)
		mtlhx	$24
#elif !defined(CONFIG_CPU_MIPSR6)
		LONG_L	$24, PT_LO(sp)
		mtlo	$24
		LONG_L	$24, PT_HI(sp)
		mthi	$24
#endif
#ifdef CONFIG_32BIT
		cfi_ld	$8, PT_R8, \docfi
		cfi_ld	$9, PT_R9, \docfi
#endif
		cfi_ld	$10, PT_R10, \docfi
		cfi_ld	$11, PT_R11, \docfi
		cfi_ld	$12, PT_R12, \docfi
		cfi_ld	$13, PT_R13, \docfi
		cfi_ld	$14, PT_R14, \docfi
		cfi_ld	$15, PT_R15, \docfi
		cfi_ld	$24, PT_R24, \docfi
		.endm

		.macro	RESTORE_STATIC docfi=0
		cfi_ld	$16, PT_R16, \docfi
		cfi_ld	$17, PT_R17, \docfi
		cfi_ld	$18, PT_R18, \docfi
		cfi_ld	$19, PT_R19, \docfi
		cfi_ld	$20, PT_R20, \docfi
		cfi_ld	$21, PT_R21, \docfi
		cfi_ld	$22, PT_R22, \docfi
		cfi_ld	$23, PT_R23, \docfi
		cfi_ld	$30, PT_R30, \docfi
		.endm

		.macro	RESTORE_SP docfi=0
		cfi_ld	sp, PT_R29, \docfi
		.endm

#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX)

		.macro	RESTORE_SOME docfi=0
		.set	push
		.set	reorder
		.set	noat
		mfc0	a0, CP0_STATUS
		li	v1, ST0_CU1 | ST0_IM
		ori	a0, STATMASK
		xori	a0, STATMASK
		mtc0	a0, CP0_STATUS
		and	a0, v1
		LONG_L	v0, PT_STATUS(sp)
		nor	v1, $0, v1
		and	v0, v1
		or	v0, a0
		mtc0	v0, CP0_STATUS
		cfi_ld	$31, PT_R31, \docfi
		cfi_ld	$28, PT_R28, \docfi
		cfi_ld	$25, PT_R25, \docfi
		cfi_ld	$7,  PT_R7, \docfi
		cfi_ld	$6,  PT_R6, \docfi
		cfi_ld	$5,  PT_R5, \docfi
		cfi_ld	$4,  PT_R4, \docfi
		cfi_ld	$3,  PT_R3, \docfi
		cfi_ld	$2,  PT_R2, \docfi
		.set	pop
		.endm

		.macro	RESTORE_SP_AND_RET docfi=0
		.set	push
		.set	noreorder
		LONG_L	k0, PT_EPC(sp)
		RESTORE_SP \docfi
		jr	k0
		 rfe
		.set	pop
		.endm

#else
		.macro	RESTORE_SOME docfi=0
		.set	push
		.set	reorder
		.set	noat
		mfc0	a0, CP0_STATUS
		ori	a0, STATMASK
		xori	a0, STATMASK
		mtc0	a0, CP0_STATUS
		li	v1, ST0_CU1 | ST0_FR | ST0_IM
		and	a0, v1
		LONG_L	v0, PT_STATUS(sp)
		nor	v1, $0, v1
		and	v0, v1
		or	v0, a0
		mtc0	v0, CP0_STATUS
		LONG_L	v1, PT_EPC(sp)
		MTC0	v1, CP0_EPC
		cfi_ld	$31, PT_R31, \docfi
		cfi_ld	$28, PT_R28, \docfi
		cfi_ld	$25, PT_R25, \docfi
#ifdef CONFIG_64BIT
		cfi_ld	$8, PT_R8, \docfi
		cfi_ld	$9, PT_R9, \docfi
#endif
		cfi_ld	$7,  PT_R7, \docfi
		cfi_ld	$6,  PT_R6, \docfi
		cfi_ld	$5,  PT_R5, \docfi
		cfi_ld	$4,  PT_R4, \docfi
		cfi_ld	$3,  PT_R3, \docfi
		cfi_ld	$2,  PT_R2, \docfi
		.set	pop
		.endm

		.macro	RESTORE_SP_AND_RET docfi=0
		RESTORE_SP \docfi
#ifdef CONFIG_CPU_MIPSR6
		eretnc
#else
		.set	arch=r4000
		eret
		.set	mips0
#endif
		.endm

#endif

		.macro	RESTORE_ALL docfi=0
		RESTORE_TEMP \docfi
		RESTORE_STATIC \docfi
		RESTORE_AT \docfi
		RESTORE_SOME \docfi
		RESTORE_SP \docfi
		.endm

/*
 * Move to kernel mode and disable interrupts.
 * Set cp0 enable bit as sign that we're running on the kernel stack
 */
		.macro	CLI
		mfc0	t0, CP0_STATUS
		li	t1, ST0_CU0 | STATMASK
		or	t0, t1
		xori	t0, STATMASK
		mtc0	t0, CP0_STATUS
		irq_disable_hazard
		.endm

/*
 * Move to kernel mode and enable interrupts.
 * Set cp0 enable bit as sign that we're running on the kernel stack
 */
		.macro	STI
		mfc0	t0, CP0_STATUS
		li	t1, ST0_CU0 | STATMASK
		or	t0, t1
		xori	t0, STATMASK & ~1
		mtc0	t0, CP0_STATUS
		irq_enable_hazard
		.endm

/*
 * Just move to kernel mode and leave interrupts as they are.  Note
 * for the R3000 this means copying the previous enable from IEp.
 * Set cp0 enable bit as sign that we're running on the kernel stack
 */
		.macro	KMODE
		mfc0	t0, CP0_STATUS
		li	t1, ST0_CU0 | (STATMASK & ~1)
#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX)
		andi	t2, t0, ST0_IEP
		srl	t2, 2
		or	t0, t2
#endif
		or	t0, t1
		xori	t0, STATMASK & ~1
		mtc0	t0, CP0_STATUS
		irq_disable_hazard
		.endm

#endif /* _ASM_STACKFRAME_H */
Commit	Line	Data
1da177e4 LT	1	/*
	2	* This file is subject to the terms and conditions of the GNU General Public
	3	* License. See the file "COPYING" in the main directory of this archive
	4	* for more details.
	5	*
	6	* Copyright (C) 1994, 95, 96, 99, 2001 Ralf Baechle
	7	* Copyright (C) 1994, 1995, 1996 Paul M. Antoine.
	8	* Copyright (C) 1999 Silicon Graphics, Inc.
619b6e18	9	* Copyright (C) 2007 Maciej W. Rozycki
1da177e4 LT	10	*/
	11	#ifndef _ASM_STACKFRAME_H
	12	#define _ASM_STACKFRAME_H
	13
1da177e4 LT	14	#include <linux/threads.h>
	15
	16	#include <asm/asm.h>
41c594ab	17	#include <asm/asmmacro.h>
1da177e4	18	#include <asm/mipsregs.h>
048eb582	19	#include <asm/asm-offsets.h>
c2377a42	20	#include <asm/thread_info.h>
1da177e4	21
866b6a89 CM	22	/* Make the addition of cfi info a little easier. */
	23	.macro cfi_rel_offset reg offset=0 docfi=0
	24	.if \docfi
	25	.cfi_rel_offset \reg, \offset
	26	.endif
	27	.endm
	28
	29	.macro cfi_st reg offset=0 docfi=0
	30	LONG_S \reg, \offset(sp)
	31	cfi_rel_offset \reg, \offset, \docfi
	32	.endm
	33
	34	.macro cfi_restore reg offset=0 docfi=0
	35	.if \docfi
	36	.cfi_restore \reg
	37	.endif
	38	.endm
	39
	40	.macro cfi_ld reg offset=0 docfi=0
	41	LONG_L \reg, \offset(sp)
	42	cfi_restore \reg \offset \docfi
	43	.endm
	44
b633648c	45	#if defined(CONFIG_CPU_R3000) \|\| defined(CONFIG_CPU_TX39XX)
fbf6ede2 MR	46	#define STATMASK 0x3f
	47	#else
	48	#define STATMASK 0x1f
	49	#endif
	50
866b6a89	51	.macro SAVE_AT docfi=0
1da177e4 LT	52	.set push
1da177e4 LT	53	.set noat
866b6a89	54	cfi_st $1, PT_R1, \docfi
1da177e4 LT	55	.set pop
	56	.endm
	57
866b6a89	58	.macro SAVE_TEMP docfi=0
9693a853 FBH	59	#ifdef CONFIG_CPU_HAS_SMARTMIPS
	60	mflhxu v1
	61	LONG_S v1, PT_LO(sp)
	62	mflhxu v1
	63	LONG_S v1, PT_HI(sp)
	64	mflhxu v1
	65	LONG_S v1, PT_ACX(sp)
6a0e9865	66	#elif !defined(CONFIG_CPU_MIPSR6)
1da177e4	67	mfhi v1
9693a853	68	#endif
875d43e7	69	#ifdef CONFIG_32BIT
866b6a89 CM	70	cfi_st $8, PT_R8, \docfi
866b6a89 CM	71	cfi_st $9, PT_R9, \docfi
1da177e4	72	#endif
866b6a89 CM	73	cfi_st $10, PT_R10, \docfi
	74	cfi_st $11, PT_R11, \docfi
	75	cfi_st $12, PT_R12, \docfi
6a0e9865	76	#if !defined(CONFIG_CPU_HAS_SMARTMIPS) && !defined(CONFIG_CPU_MIPSR6)
362e6964 DD	77	LONG_S v1, PT_HI(sp)
	78	mflo v1
	79	#endif
866b6a89 CM	80	cfi_st $13, PT_R13, \docfi
	81	cfi_st $14, PT_R14, \docfi
	82	cfi_st $15, PT_R15, \docfi
	83	cfi_st $24, PT_R24, \docfi
6a0e9865	84	#if !defined(CONFIG_CPU_HAS_SMARTMIPS) && !defined(CONFIG_CPU_MIPSR6)
362e6964	85	LONG_S v1, PT_LO(sp)
8dfdd02a DD	86	#endif
	87	#ifdef CONFIG_CPU_CAVIUM_OCTEON
	88	/*
	89	* The Octeon multiplier state is affected by general
	90	* multiply instructions. It must be saved before and
	91	* kernel code might corrupt it
	92	*/
	93	jal octeon_mult_save
362e6964	94	#endif
1da177e4 LT	95	.endm
1da177e4 LT	96
866b6a89 CM	97	.macro SAVE_STATIC docfi=0
	98	cfi_st $16, PT_R16, \docfi
	99	cfi_st $17, PT_R17, \docfi
	100	cfi_st $18, PT_R18, \docfi
	101	cfi_st $19, PT_R19, \docfi
	102	cfi_st $20, PT_R20, \docfi
	103	cfi_st $21, PT_R21, \docfi
	104	cfi_st $22, PT_R22, \docfi
	105	cfi_st $23, PT_R23, \docfi
	106	cfi_st $30, PT_R30, \docfi
1da177e4 LT	107	.endm
1da177e4 LT	108
9fef6868 CM	109	/*
	110	* get_saved_sp returns the SP for the current CPU by looking in the
	111	* kernelsp array for it. If tosp is set, it stores the current sp in
	112	* k0 and loads the new value in sp. If not, it clobbers k0 and
	113	* stores the new value in k1, leaving sp unaffected.
	114	*/
1da177e4	115	#ifdef CONFIG_SMP
9fef6868 CM	116
	117	/* SMP variation */
	118	.macro get_saved_sp docfi=0 tosp=0
c2377a42	119	ASM_CPUID_MFC0 k0, ASM_SMP_CPUID_REG
054c51b4 FBH	120	#if defined(CONFIG_32BIT) \|\| defined(KBUILD_64BIT_SYM32)
	121	lui k1, %hi(kernelsp)
	122	#else
9b95e629 AN	123	lui k1, %highest(kernelsp)
	124	daddiu k1, %higher(kernelsp)
	125	dsll k1, 16
	126	daddiu k1, %hi(kernelsp)
	127	dsll k1, 16
9b95e629	128	#endif
c2377a42	129	LONG_SRL k0, SMP_CPUID_PTRSHIFT
9b95e629	130	LONG_ADDU k1, k0
9fef6868 CM	131	.if \tosp
	132	move k0, sp
	133	.if \docfi
	134	.cfi_register sp, k0
	135	.endif
	136	LONG_L sp, %lo(kernelsp)(k1)
	137	.else
85b6e818	138	LONG_L k1, %lo(kernelsp)(k1)
9fef6868	139	.endif
1da177e4 LT	140	.endm
	141
	142	.macro set_saved_sp stackp temp temp2
c2377a42 J	143	ASM_CPUID_MFC0 \temp, ASM_SMP_CPUID_REG
c2377a42 J	144	LONG_SRL \temp, SMP_CPUID_PTRSHIFT
9556ac2f	145	LONG_S \stackp, kernelsp(\temp)
1da177e4	146	.endm
c2377a42	147	#else /* !CONFIG_SMP */
9fef6868 CM	148	/* Uniprocessor variation */
9fef6868 CM	149	.macro get_saved_sp docfi=0 tosp=0
b197b628	150	#ifdef CONFIG_CPU_JUMP_WORKAROUNDS
f1df3239 WZ	151	/*
	152	* Clear BTB (branch target buffer), forbid RAS (return address
	153	* stack) to workaround the Out-of-order Issue in Loongson2F
	154	* via its diagnostic register.
	155	*/
	156	move k0, ra
	157	jal 1f
	158	nop
	159	1: jal 1f
	160	nop
	161	1: jal 1f
	162	nop
	163	1: jal 1f
	164	nop
	165	1: move ra, k0
	166	li k0, 3
	167	mtc0 k0, $22
2a0b24f5	168	#endif /* CONFIG_CPU_JUMP_WORKAROUNDS */
054c51b4 FBH	169	#if defined(CONFIG_32BIT) \|\| defined(KBUILD_64BIT_SYM32)
	170	lui k1, %hi(kernelsp)
	171	#else
9556ac2f TS	172	lui k1, %highest(kernelsp)
	173	daddiu k1, %higher(kernelsp)
	174	dsll k1, k1, 16
	175	daddiu k1, %hi(kernelsp)
	176	dsll k1, k1, 16
9556ac2f	177	#endif
9fef6868 CM	178	.if \tosp
	179	move k0, sp
	180	.if \docfi
	181	.cfi_register sp, k0
	182	.endif
	183	LONG_L sp, %lo(kernelsp)(k1)
	184	.else
1da177e4	185	LONG_L k1, %lo(kernelsp)(k1)
9fef6868	186	.endif
1da177e4 LT	187	.endm
	188
	189	.macro set_saved_sp stackp temp temp2
	190	LONG_S \stackp, kernelsp
	191	.endm
	192	#endif
	193
866b6a89	194	.macro SAVE_SOME docfi=0
1da177e4 LT	195	.set push
	196	.set noat
	197	.set reorder
	198	mfc0 k0, CP0_STATUS
	199	sll k0, 3 /* extract cu0 bit */
	200	.set noreorder
	201	bltz k0, 8f
3aff47c0 JH	202	#ifdef CONFIG_EVA
	203	/*
	204	* Flush interAptiv's Return Prediction Stack (RPS) by writing
	205	* EntryHi. Toggling Config7.RPS is slower and less portable.
	206	*
	207	* The RPS isn't automatically flushed when exceptions are
	208	* taken, which can result in kernel mode speculative accesses
	209	* to user addresses if the RPS mispredicts. That's harmless
	210	* when user and kernel share the same address space, but with
	211	* EVA the same user segments may be unmapped to kernel mode,
	212	* even containing sensitive MMIO regions or invalid memory.
	213	*
	214	* This can happen when the kernel sets the return address to
	215	* ret_from_* and jr's to the exception handler, which looks
	216	* more like a tail call than a function call. If nested calls
	217	* don't evict the last user address in the RPS, it will
	218	* mispredict the return and fetch from a user controlled
	219	* address into the icache.
	220	*
	221	* More recent EVA-capable cores with MAAR to restrict
	222	* speculative accesses aren't affected.
	223	*/
	224	MFC0 k0, CP0_ENTRYHI
	225	MTC0 k0, CP0_ENTRYHI
	226	#endif
1da177e4	227	.set reorder
9fef6868	228	move k0, sp
866b6a89 CM	229	.if \docfi
	230	.cfi_register sp, k0
	231	.endif
1da177e4	232	/* Called from user mode, new stack. */
866b6a89	233	get_saved_sp docfi=\docfi tosp=1
9fef6868 CM	234	8:
	235	#ifdef CONFIG_CPU_DADDI_WORKAROUNDS
	236	.set at=k1
	237	#endif
	238	PTR_SUBU sp, PT_SIZE
	239	#ifdef CONFIG_CPU_DADDI_WORKAROUNDS
619b6e18	240	.set noat
619b6e18	241	#endif
866b6a89 CM	242	.if \docfi
	243	.cfi_def_cfa sp,0
	244	.endif
	245	cfi_st k0, PT_R29, \docfi
	246	cfi_rel_offset sp, PT_R29, \docfi
	247	cfi_st v1, PT_R3, \docfi
41c594ab RB	248	/*
	249	* You might think that you don't need to save $0,
	250	* but the FPU emulator and gdb remote debug stub
	251	* need it to operate correctly
	252	*/
1da177e4 LT	253	LONG_S $0, PT_R0(sp)
1da177e4 LT	254	mfc0 v1, CP0_STATUS
866b6a89	255	cfi_st v0, PT_R2, \docfi
2a0b24f5	256	LONG_S v1, PT_STATUS(sp)
866b6a89	257	cfi_st $4, PT_R4, \docfi
362e6964	258	mfc0 v1, CP0_CAUSE
866b6a89	259	cfi_st $5, PT_R5, \docfi
362e6964	260	LONG_S v1, PT_CAUSE(sp)
866b6a89 CM	261	cfi_st $6, PT_R6, \docfi
866b6a89 CM	262	cfi_st ra, PT_R31, \docfi
9fef6868	263	MFC0 ra, CP0_EPC
866b6a89	264	cfi_st $7, PT_R7, \docfi
875d43e7	265	#ifdef CONFIG_64BIT
866b6a89 CM	266	cfi_st $8, PT_R8, \docfi
866b6a89 CM	267	cfi_st $9, PT_R9, \docfi
1da177e4	268	#endif
9fef6868	269	LONG_S ra, PT_EPC(sp)
866b6a89 CM	270	.if \docfi
	271	.cfi_rel_offset ra, PT_EPC
	272	.endif
	273	cfi_st $25, PT_R25, \docfi
	274	cfi_st $28, PT_R28, \docfi
510d8636 MR	275
	276	/* Set thread_info if we're coming from user mode */
	277	mfc0 k0, CP0_STATUS
	278	sll k0, 3 /* extract cu0 bit */
	279	bltz k0, 9f
	280
1da177e4 LT	281	ori $28, sp, _THREAD_MASK
1da177e4 LT	282	xori $28, _THREAD_MASK
2a219b0e	283	#ifdef CONFIG_CPU_CAVIUM_OCTEON
8dfdd02a DD	284	.set mips64
8dfdd02a DD	285	pref 0, 0($28) /* Prefetch the current pointer */
2a219b0e	286	#endif
510d8636	287	9:
1da177e4 LT	288	.set pop
	289	.endm
	290
866b6a89 CM	291	.macro SAVE_ALL docfi=0
	292	SAVE_SOME \docfi
	293	SAVE_AT \docfi
	294	SAVE_TEMP \docfi
	295	SAVE_STATIC \docfi
1da177e4 LT	296	.endm
1da177e4 LT	297
866b6a89	298	.macro RESTORE_AT docfi=0
1da177e4 LT	299	.set push
1da177e4 LT	300	.set noat
866b6a89	301	cfi_ld $1, PT_R1, \docfi
1da177e4 LT	302	.set pop
	303	.endm
	304
866b6a89	305	.macro RESTORE_TEMP docfi=0
8dfdd02a DD	306	#ifdef CONFIG_CPU_CAVIUM_OCTEON
	307	/* Restore the Octeon multiplier state */
	308	jal octeon_mult_restore
	309	#endif
9693a853 FBH	310	#ifdef CONFIG_CPU_HAS_SMARTMIPS
	311	LONG_L $24, PT_ACX(sp)
	312	mtlhx $24
	313	LONG_L $24, PT_HI(sp)
	314	mtlhx $24
1da177e4	315	LONG_L $24, PT_LO(sp)
9693a853	316	mtlhx $24
6a0e9865	317	#elif !defined(CONFIG_CPU_MIPSR6)
9693a853 FBH	318	LONG_L $24, PT_LO(sp)
	319	mtlo $24
	320	LONG_L $24, PT_HI(sp)
	321	mthi $24
	322	#endif
875d43e7	323	#ifdef CONFIG_32BIT
866b6a89 CM	324	cfi_ld $8, PT_R8, \docfi
866b6a89 CM	325	cfi_ld $9, PT_R9, \docfi
1da177e4	326	#endif
866b6a89 CM	327	cfi_ld $10, PT_R10, \docfi
	328	cfi_ld $11, PT_R11, \docfi
	329	cfi_ld $12, PT_R12, \docfi
	330	cfi_ld $13, PT_R13, \docfi
	331	cfi_ld $14, PT_R14, \docfi
	332	cfi_ld $15, PT_R15, \docfi
	333	cfi_ld $24, PT_R24, \docfi
1da177e4 LT	334	.endm
1da177e4 LT	335
866b6a89 CM	336	.macro RESTORE_STATIC docfi=0
	337	cfi_ld $16, PT_R16, \docfi
	338	cfi_ld $17, PT_R17, \docfi
	339	cfi_ld $18, PT_R18, \docfi
	340	cfi_ld $19, PT_R19, \docfi
	341	cfi_ld $20, PT_R20, \docfi
	342	cfi_ld $21, PT_R21, \docfi
	343	cfi_ld $22, PT_R22, \docfi
	344	cfi_ld $23, PT_R23, \docfi
	345	cfi_ld $30, PT_R30, \docfi
	346	.endm
	347
	348	.macro RESTORE_SP docfi=0
	349	cfi_ld sp, PT_R29, \docfi
1da177e4 LT	350	.endm
	351
	352	#if defined(CONFIG_CPU_R3000) \|\| defined(CONFIG_CPU_TX39XX)
	353
866b6a89	354	.macro RESTORE_SOME docfi=0
1da177e4 LT	355	.set push
	356	.set reorder
	357	.set noat
	358	mfc0 a0, CP0_STATUS
00fe56dc	359	li v1, ST0_CU1 \| ST0_IM
fbf6ede2 MR	360	ori a0, STATMASK
	361	xori a0, STATMASK
	362	mtc0 a0, CP0_STATUS
1da177e4 LT	363	and a0, v1
	364	LONG_L v0, PT_STATUS(sp)
	365	nor v1, $0, v1
	366	and v0, v1
	367	or v0, a0
	368	mtc0 v0, CP0_STATUS
866b6a89 CM	369	cfi_ld $31, PT_R31, \docfi
	370	cfi_ld $28, PT_R28, \docfi
	371	cfi_ld $25, PT_R25, \docfi
	372	cfi_ld $7, PT_R7, \docfi
	373	cfi_ld $6, PT_R6, \docfi
	374	cfi_ld $5, PT_R5, \docfi
	375	cfi_ld $4, PT_R4, \docfi
	376	cfi_ld $3, PT_R3, \docfi
	377	cfi_ld $2, PT_R2, \docfi
1da177e4 LT	378	.set pop
	379	.endm
	380
866b6a89	381	.macro RESTORE_SP_AND_RET docfi=0
1da177e4 LT	382	.set push
	383	.set noreorder
	384	LONG_L k0, PT_EPC(sp)
866b6a89	385	RESTORE_SP \docfi
1da177e4 LT	386	jr k0
	387	rfe
	388	.set pop
	389	.endm
	390
	391	#else
866b6a89	392	.macro RESTORE_SOME docfi=0
1da177e4 LT	393	.set push
	394	.set reorder
	395	.set noat
	396	mfc0 a0, CP0_STATUS
41c594ab RB	397	ori a0, STATMASK
41c594ab RB	398	xori a0, STATMASK
1da177e4	399	mtc0 a0, CP0_STATUS
00fe56dc	400	li v1, ST0_CU1 \| ST0_FR \| ST0_IM
1da177e4 LT	401	and a0, v1
	402	LONG_L v0, PT_STATUS(sp)
	403	nor v1, $0, v1
	404	and v0, v1
	405	or v0, a0
	406	mtc0 v0, CP0_STATUS
	407	LONG_L v1, PT_EPC(sp)
	408	MTC0 v1, CP0_EPC
866b6a89 CM	409	cfi_ld $31, PT_R31, \docfi
	410	cfi_ld $28, PT_R28, \docfi
	411	cfi_ld $25, PT_R25, \docfi
875d43e7	412	#ifdef CONFIG_64BIT
866b6a89 CM	413	cfi_ld $8, PT_R8, \docfi
866b6a89 CM	414	cfi_ld $9, PT_R9, \docfi
1da177e4	415	#endif
866b6a89 CM	416	cfi_ld $7, PT_R7, \docfi
	417	cfi_ld $6, PT_R6, \docfi
	418	cfi_ld $5, PT_R5, \docfi
	419	cfi_ld $4, PT_R4, \docfi
	420	cfi_ld $3, PT_R3, \docfi
	421	cfi_ld $2, PT_R2, \docfi
1da177e4 LT	422	.set pop
	423	.endm
	424
866b6a89 CM	425	.macro RESTORE_SP_AND_RET docfi=0
866b6a89 CM	426	RESTORE_SP \docfi
e11124d8 PB	427	#ifdef CONFIG_CPU_MIPSR6
	428	eretnc
	429	#else
a809d460	430	.set arch=r4000
1da177e4 LT	431	eret
1da177e4 LT	432	.set mips0
e11124d8	433	#endif
1da177e4 LT	434	.endm
	435
	436	#endif
	437
866b6a89 CM	438	.macro RESTORE_ALL docfi=0
	439	RESTORE_TEMP \docfi
	440	RESTORE_STATIC \docfi
	441	RESTORE_AT \docfi
	442	RESTORE_SOME \docfi
	443	RESTORE_SP \docfi
1da177e4 LT	444	.endm
1da177e4 LT	445
1da177e4 LT	446	/*
	447	* Move to kernel mode and disable interrupts.
	448	* Set cp0 enable bit as sign that we're running on the kernel stack
	449	*/
	450	.macro CLI
	451	mfc0 t0, CP0_STATUS
fbf6ede2	452	li t1, ST0_CU0 \| STATMASK
1da177e4	453	or t0, t1
fbf6ede2	454	xori t0, STATMASK
1da177e4 LT	455	mtc0 t0, CP0_STATUS
	456	irq_disable_hazard
	457	.endm
	458
	459	/*
	460	* Move to kernel mode and enable interrupts.
	461	* Set cp0 enable bit as sign that we're running on the kernel stack
	462	*/
	463	.macro STI
	464	mfc0 t0, CP0_STATUS
fbf6ede2	465	li t1, ST0_CU0 \| STATMASK
1da177e4	466	or t0, t1
fbf6ede2	467	xori t0, STATMASK & ~1
1da177e4 LT	468	mtc0 t0, CP0_STATUS
	469	irq_enable_hazard
	470	.endm
	471
	472	/*
fbf6ede2 MR	473	* Just move to kernel mode and leave interrupts as they are. Note
fbf6ede2 MR	474	* for the R3000 this means copying the previous enable from IEp.
1da177e4 LT	475	* Set cp0 enable bit as sign that we're running on the kernel stack
	476	*/
	477	.macro KMODE
	478	mfc0 t0, CP0_STATUS
fbf6ede2 MR	479	li t1, ST0_CU0 \| (STATMASK & ~1)
	480	#if defined(CONFIG_CPU_R3000) \|\| defined(CONFIG_CPU_TX39XX)
	481	andi t2, t0, ST0_IEP
	482	srl t2, 2
	483	or t0, t2
	484	#endif
1da177e4	485	or t0, t1
fbf6ede2	486	xori t0, STATMASK & ~1
1da177e4 LT	487	mtc0 t0, CP0_STATUS
	488	irq_disable_hazard
	489	.endm
	490
	491	#endif /* _ASM_STACKFRAME_H */