[mirror_ubuntu-zesty-kernel.git] / arch / powerpc / kernel / vector.S

#include <asm/processor.h>
#include <asm/ppc_asm.h>
#include <asm/reg.h>
#include <asm/asm-offsets.h>
#include <asm/cputable.h>
#include <asm/thread_info.h>
#include <asm/page.h>
#include <asm/ptrace.h>
#include <asm/export.h>

/*
 * Load state from memory into VMX registers including VSCR.
 * Assumes the caller has enabled VMX in the MSR.
 */
_GLOBAL(load_vr_state)
	li	r4,VRSTATE_VSCR
	lvx	v0,r4,r3
	mtvscr	v0
	REST_32VRS(0,r4,r3)
	blr
EXPORT_SYMBOL(load_vr_state)

/*
 * Store VMX state into memory, including VSCR.
 * Assumes the caller has enabled VMX in the MSR.
 */
_GLOBAL(store_vr_state)
	SAVE_32VRS(0, r4, r3)
	mfvscr	v0
	li	r4, VRSTATE_VSCR
	stvx	v0, r4, r3
	blr
EXPORT_SYMBOL(store_vr_state)

/*
 * Disable VMX for the task which had it previously,
 * and save its vector registers in its thread_struct.
 * Enables the VMX for use in the kernel on return.
 * On SMP we know the VMX is free, since we give it up every
 * switch (ie, no lazy save of the vector registers).
 *
 * Note that on 32-bit this can only use registers that will be
 * restored by fast_exception_return, i.e. r3 - r6, r10 and r11.
 */
_GLOBAL(load_up_altivec)
	mfmsr	r5			/* grab the current MSR */
	oris	r5,r5,MSR_VEC@h
	MTMSRD(r5)			/* enable use of AltiVec now */
	isync

	/*
	 * While userspace in general ignores VRSAVE, glibc uses it as a boolean
	 * to optimise userspace context save/restore. Whenever we take an
	 * altivec unavailable exception we must set VRSAVE to something non
	 * zero. Set it to all 1s. See also the programming note in the ISA.
	 */
	mfspr	r4,SPRN_VRSAVE
	cmpwi	0,r4,0
	bne+	1f
	li	r4,-1
	mtspr	SPRN_VRSAVE,r4
1:
	/* enable use of VMX after return */
#ifdef CONFIG_PPC32
	mfspr	r5,SPRN_SPRG_THREAD		/* current task's THREAD (phys) */
	oris	r9,r9,MSR_VEC@h
#else
	ld	r4,PACACURRENT(r13)
	addi	r5,r4,THREAD		/* Get THREAD */
	oris	r12,r12,MSR_VEC@h
	std	r12,_MSR(r1)
#endif
	/* Don't care if r4 overflows, this is desired behaviour */
	lbz	r4,THREAD_LOAD_VEC(r5)
	addi	r4,r4,1
	stb	r4,THREAD_LOAD_VEC(r5)
	addi	r6,r5,THREAD_VRSTATE
	li	r4,1
	li	r10,VRSTATE_VSCR
	stw	r4,THREAD_USED_VR(r5)
	lvx	v0,r10,r6
	mtvscr	v0
	REST_32VRS(0,r4,r6)
	/* restore registers and return */
	blr

/*
 * save_altivec(tsk)
 * Save the vector registers to its thread_struct
 */
_GLOBAL(save_altivec)
	addi	r3,r3,THREAD		/* want THREAD of task */
	PPC_LL	r7,THREAD_VRSAVEAREA(r3)
	PPC_LL	r5,PT_REGS(r3)
	PPC_LCMPI	0,r7,0
	bne	2f
	addi	r7,r3,THREAD_VRSTATE
2:	SAVE_32VRS(0,r4,r7)
	mfvscr	v0
	li	r4,VRSTATE_VSCR
	stvx	v0,r4,r7
	blr

#ifdef CONFIG_VSX

#ifdef CONFIG_PPC32
#error This asm code isn't ready for 32-bit kernels
#endif

/*
 * load_up_vsx(unused, unused, tsk)
 * Disable VSX for the task which had it previously,
 * and save its vector registers in its thread_struct.
 * Reuse the fp and vsx saves, but first check to see if they have
 * been saved already.
 */
_GLOBAL(load_up_vsx)
/* Load FP and VSX registers if they haven't been done yet */
	andi.	r5,r12,MSR_FP
	beql+	load_up_fpu		/* skip if already loaded */
	andis.	r5,r12,MSR_VEC@h
	beql+	load_up_altivec		/* skip if already loaded */

	ld	r4,PACACURRENT(r13)
	addi	r4,r4,THREAD		/* Get THREAD */
	li	r6,1
	stw	r6,THREAD_USED_VSR(r4) /* ... also set thread used vsr */
	/* enable use of VSX after return */
	oris	r12,r12,MSR_VSX@h
	std	r12,_MSR(r1)
	b	fast_exception_return

#endif /* CONFIG_VSX */


/*
 * The routines below are in assembler so we can closely control the
 * usage of floating-point registers.  These routines must be called
 * with preempt disabled.
 */
#ifdef CONFIG_PPC32
	.data
fpzero:
	.long	0
fpone:
	.long	0x3f800000	/* 1.0 in single-precision FP */
fphalf:
	.long	0x3f000000	/* 0.5 in single-precision FP */

#define LDCONST(fr, name)	\
	lis	r11,name@ha;	\
	lfs	fr,name@l(r11)
#else

	.section ".toc","aw"
fpzero:
	.tc	FD_0_0[TC],0
fpone:
	.tc	FD_3ff00000_0[TC],0x3ff0000000000000	/* 1.0 */
fphalf:
	.tc	FD_3fe00000_0[TC],0x3fe0000000000000	/* 0.5 */

#define LDCONST(fr, name)	\
	lfd	fr,name@toc(r2)
#endif

	.text
/*
 * Internal routine to enable floating point and set FPSCR to 0.
 * Don't call it from C; it doesn't use the normal calling convention.
 */
fpenable:
#ifdef CONFIG_PPC32
	stwu	r1,-64(r1)
#else
	stdu	r1,-64(r1)
#endif
	mfmsr	r10
	ori	r11,r10,MSR_FP
	mtmsr	r11
	isync
	stfd	fr0,24(r1)
	stfd	fr1,16(r1)
	stfd	fr31,8(r1)
	LDCONST(fr1, fpzero)
	mffs	fr31
	MTFSF_L(fr1)
	blr

fpdisable:
	mtlr	r12
	MTFSF_L(fr31)
	lfd	fr31,8(r1)
	lfd	fr1,16(r1)
	lfd	fr0,24(r1)
	mtmsr	r10
	isync
	addi	r1,r1,64
	blr

/*
 * Vector add, floating point.
 */
_GLOBAL(vaddfp)
	mflr	r12
	bl	fpenable
	li	r0,4
	mtctr	r0
	li	r6,0
1:	lfsx	fr0,r4,r6
	lfsx	fr1,r5,r6
	fadds	fr0,fr0,fr1
	stfsx	fr0,r3,r6
	addi	r6,r6,4
	bdnz	1b
	b	fpdisable

/*
 * Vector subtract, floating point.
 */
_GLOBAL(vsubfp)
	mflr	r12
	bl	fpenable
	li	r0,4
	mtctr	r0
	li	r6,0
1:	lfsx	fr0,r4,r6
	lfsx	fr1,r5,r6
	fsubs	fr0,fr0,fr1
	stfsx	fr0,r3,r6
	addi	r6,r6,4
	bdnz	1b
	b	fpdisable

/*
 * Vector multiply and add, floating point.
 */
_GLOBAL(vmaddfp)
	mflr	r12
	bl	fpenable
	stfd	fr2,32(r1)
	li	r0,4
	mtctr	r0
	li	r7,0
1:	lfsx	fr0,r4,r7
	lfsx	fr1,r5,r7
	lfsx	fr2,r6,r7
	fmadds	fr0,fr0,fr2,fr1
	stfsx	fr0,r3,r7
	addi	r7,r7,4
	bdnz	1b
	lfd	fr2,32(r1)
	b	fpdisable

/*
 * Vector negative multiply and subtract, floating point.
 */
_GLOBAL(vnmsubfp)
	mflr	r12
	bl	fpenable
	stfd	fr2,32(r1)
	li	r0,4
	mtctr	r0
	li	r7,0
1:	lfsx	fr0,r4,r7
	lfsx	fr1,r5,r7
	lfsx	fr2,r6,r7
	fnmsubs	fr0,fr0,fr2,fr1
	stfsx	fr0,r3,r7
	addi	r7,r7,4
	bdnz	1b
	lfd	fr2,32(r1)
	b	fpdisable

/*
 * Vector reciprocal estimate.  We just compute 1.0/x.
 * r3 -> destination, r4 -> source.
 */
_GLOBAL(vrefp)
	mflr	r12
	bl	fpenable
	li	r0,4
	LDCONST(fr1, fpone)
	mtctr	r0
	li	r6,0
1:	lfsx	fr0,r4,r6
	fdivs	fr0,fr1,fr0
	stfsx	fr0,r3,r6
	addi	r6,r6,4
	bdnz	1b
	b	fpdisable

/*
 * Vector reciprocal square-root estimate, floating point.
 * We use the frsqrte instruction for the initial estimate followed
 * by 2 iterations of Newton-Raphson to get sufficient accuracy.
 * r3 -> destination, r4 -> source.
 */
_GLOBAL(vrsqrtefp)
	mflr	r12
	bl	fpenable
	stfd	fr2,32(r1)
	stfd	fr3,40(r1)
	stfd	fr4,48(r1)
	stfd	fr5,56(r1)
	li	r0,4
	LDCONST(fr4, fpone)
	LDCONST(fr5, fphalf)
	mtctr	r0
	li	r6,0
1:	lfsx	fr0,r4,r6
	frsqrte	fr1,fr0		/* r = frsqrte(s) */
	fmuls	fr3,fr1,fr0	/* r * s */
	fmuls	fr2,fr1,fr5	/* r * 0.5 */
	fnmsubs	fr3,fr1,fr3,fr4	/* 1 - s * r * r */
	fmadds	fr1,fr2,fr3,fr1	/* r = r + 0.5 * r * (1 - s * r * r) */
	fmuls	fr3,fr1,fr0	/* r * s */
	fmuls	fr2,fr1,fr5	/* r * 0.5 */
	fnmsubs	fr3,fr1,fr3,fr4	/* 1 - s * r * r */
	fmadds	fr1,fr2,fr3,fr1	/* r = r + 0.5 * r * (1 - s * r * r) */
	stfsx	fr1,r3,r6
	addi	r6,r6,4
	bdnz	1b
	lfd	fr5,56(r1)
	lfd	fr4,48(r1)
	lfd	fr3,40(r1)
	lfd	fr2,32(r1)
	b	fpdisable
Commit	Line	Data
e821ea70	1	#include <asm/processor.h>
14cf11af	2	#include <asm/ppc_asm.h>
b3b8dc6c	3	#include <asm/reg.h>
e821ea70 BH	4	#include <asm/asm-offsets.h>
	5	#include <asm/cputable.h>
	6	#include <asm/thread_info.h>
	7	#include <asm/page.h>
46f52210	8	#include <asm/ptrace.h>
9445aa1a	9	#include <asm/export.h>
e821ea70	10
18461960 PM	11	/*
	12	* Load state from memory into VMX registers including VSCR.
	13	* Assumes the caller has enabled VMX in the MSR.
	14	*/
	15	_GLOBAL(load_vr_state)
	16	li r4,VRSTATE_VSCR
c2ce6f9f AB	17	lvx v0,r4,r3
c2ce6f9f AB	18	mtvscr v0
18461960 PM	19	REST_32VRS(0,r4,r3)
18461960 PM	20	blr
9445aa1a	21	EXPORT_SYMBOL(load_vr_state)
18461960 PM	22
	23	/*
	24	* Store VMX state into memory, including VSCR.
	25	* Assumes the caller has enabled VMX in the MSR.
	26	*/
	27	_GLOBAL(store_vr_state)
	28	SAVE_32VRS(0, r4, r3)
c2ce6f9f	29	mfvscr v0
18461960	30	li r4, VRSTATE_VSCR
c2ce6f9f	31	stvx v0, r4, r3
18461960	32	blr
9445aa1a	33	EXPORT_SYMBOL(store_vr_state)
18461960	34
e821ea70	35	/*
e821ea70 BH	36	* Disable VMX for the task which had it previously,
	37	* and save its vector registers in its thread_struct.
	38	* Enables the VMX for use in the kernel on return.
	39	* On SMP we know the VMX is free, since we give it up every
	40	* switch (ie, no lazy save of the vector registers).
955c1cab PM	41	*
	42	* Note that on 32-bit this can only use registers that will be
	43	* restored by fast_exception_return, i.e. r3 - r6, r10 and r11.
e821ea70 BH	44	*/
	45	_GLOBAL(load_up_altivec)
	46	mfmsr r5 /* grab the current MSR */
	47	oris r5,r5,MSR_VEC@h
	48	MTMSRD(r5) /* enable use of AltiVec now */
	49	isync
	50
dd570237 AB	51	/*
	52	* While userspace in general ignores VRSAVE, glibc uses it as a boolean
	53	* to optimise userspace context save/restore. Whenever we take an
	54	* altivec unavailable exception we must set VRSAVE to something non
	55	* zero. Set it to all 1s. See also the programming note in the ISA.
e821ea70 BH	56	*/
e821ea70 BH	57	mfspr r4,SPRN_VRSAVE
e090aa80	58	cmpwi 0,r4,0
e821ea70 BH	59	bne+ 1f
	60	li r4,-1
	61	mtspr SPRN_VRSAVE,r4
	62	1:
	63	/* enable use of VMX after return */
	64	#ifdef CONFIG_PPC32
ee43eb78	65	mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */
e821ea70 BH	66	oris r9,r9,MSR_VEC@h
	67	#else
	68	ld r4,PACACURRENT(r13)
	69	addi r5,r4,THREAD /* Get THREAD */
	70	oris r12,r12,MSR_VEC@h
	71	std r12,_MSR(r1)
	72	#endif
70fe3d98 CB	73	/* Don't care if r4 overflows, this is desired behaviour */
	74	lbz r4,THREAD_LOAD_VEC(r5)
	75	addi r4,r4,1
	76	stb r4,THREAD_LOAD_VEC(r5)
955c1cab	77	addi r6,r5,THREAD_VRSTATE
e821ea70	78	li r4,1
de79f7b9	79	li r10,VRSTATE_VSCR
e821ea70	80	stw r4,THREAD_USED_VR(r5)
c2ce6f9f AB	81	lvx v0,r10,r6
c2ce6f9f AB	82	mtvscr v0
955c1cab	83	REST_32VRS(0,r4,r6)
e821ea70 BH	84	/* restore registers and return */
	85	blr
	86
	87	/*
6f515d84 CB	88	* save_altivec(tsk)
6f515d84 CB	89	* Save the vector registers to its thread_struct
e821ea70	90	*/
6f515d84	91	_GLOBAL(save_altivec)
e821ea70	92	addi r3,r3,THREAD /* want THREAD of task */
18461960	93	PPC_LL r7,THREAD_VRSAVEAREA(r3)
e821ea70	94	PPC_LL r5,PT_REGS(r3)
18461960 PM	95	PPC_LCMPI 0,r7,0
	96	bne 2f
	97	addi r7,r3,THREAD_VRSTATE
6f515d84	98	2: SAVE_32VRS(0,r4,r7)
c2ce6f9f	99	mfvscr v0
de79f7b9	100	li r4,VRSTATE_VSCR
c2ce6f9f	101	stvx v0,r4,r7
e821ea70 BH	102	blr
	103
	104	#ifdef CONFIG_VSX
	105
	106	#ifdef CONFIG_PPC32
	107	#error This asm code isn't ready for 32-bit kernels
	108	#endif
	109
	110	/*
	111	* load_up_vsx(unused, unused, tsk)
	112	* Disable VSX for the task which had it previously,
	113	* and save its vector registers in its thread_struct.
	114	* Reuse the fp and vsx saves, but first check to see if they have
	115	* been saved already.
	116	*/
	117	_GLOBAL(load_up_vsx)
	118	/* Load FP and VSX registers if they haven't been done yet */
	119	andi. r5,r12,MSR_FP
	120	beql+ load_up_fpu /* skip if already loaded */
	121	andis. r5,r12,MSR_VEC@h
	122	beql+ load_up_altivec /* skip if already loaded */
	123
e821ea70 BH	124	ld r4,PACACURRENT(r13)
	125	addi r4,r4,THREAD /* Get THREAD */
	126	li r6,1
	127	stw r6,THREAD_USED_VSR(r4) /* ... also set thread used vsr */
	128	/* enable use of VSX after return */
	129	oris r12,r12,MSR_VSX@h
	130	std r12,_MSR(r1)
e821ea70 BH	131	b fast_exception_return
e821ea70 BH	132
e821ea70 BH	133	#endif /* CONFIG_VSX */
e821ea70 BH	134
14cf11af PM	135
	136	/*
	137	* The routines below are in assembler so we can closely control the
	138	* usage of floating-point registers. These routines must be called
	139	* with preempt disabled.
	140	*/
	141	#ifdef CONFIG_PPC32
	142	.data
	143	fpzero:
	144	.long 0
	145	fpone:
	146	.long 0x3f800000 /* 1.0 in single-precision FP */
	147	fphalf:
	148	.long 0x3f000000 /* 0.5 in single-precision FP */
	149
	150	#define LDCONST(fr, name) \
	151	lis r11,name@ha; \
	152	lfs fr,name@l(r11)
	153	#else
	154
	155	.section ".toc","aw"
	156	fpzero:
	157	.tc FD_0_0[TC],0
	158	fpone:
	159	.tc FD_3ff00000_0[TC],0x3ff0000000000000 /* 1.0 */
	160	fphalf:
	161	.tc FD_3fe00000_0[TC],0x3fe0000000000000 /* 0.5 */
	162
	163	#define LDCONST(fr, name) \
	164	lfd fr,name@toc(r2)
	165	#endif
	166
	167	.text
	168	/*
	169	* Internal routine to enable floating point and set FPSCR to 0.
	170	* Don't call it from C; it doesn't use the normal calling convention.
	171	*/
	172	fpenable:
	173	#ifdef CONFIG_PPC32
	174	stwu r1,-64(r1)
	175	#else
	176	stdu r1,-64(r1)
	177	#endif
	178	mfmsr r10
	179	ori r11,r10,MSR_FP
	180	mtmsr r11
	181	isync
	182	stfd fr0,24(r1)
	183	stfd fr1,16(r1)
	184	stfd fr31,8(r1)
	185	LDCONST(fr1, fpzero)
	186	mffs fr31
3a2c48cf	187	MTFSF_L(fr1)
14cf11af PM	188	blr
	189
	190	fpdisable:
	191	mtlr r12
3a2c48cf	192	MTFSF_L(fr31)
14cf11af PM	193	lfd fr31,8(r1)
	194	lfd fr1,16(r1)
	195	lfd fr0,24(r1)
	196	mtmsr r10
	197	isync
	198	addi r1,r1,64
	199	blr
	200
	201	/*
	202	* Vector add, floating point.
	203	*/
	204	_GLOBAL(vaddfp)
	205	mflr r12
	206	bl fpenable
	207	li r0,4
	208	mtctr r0
	209	li r6,0
	210	1: lfsx fr0,r4,r6
	211	lfsx fr1,r5,r6
	212	fadds fr0,fr0,fr1
	213	stfsx fr0,r3,r6
	214	addi r6,r6,4
	215	bdnz 1b
	216	b fpdisable
	217
	218	/*
	219	* Vector subtract, floating point.
	220	*/
	221	_GLOBAL(vsubfp)
	222	mflr r12
	223	bl fpenable
	224	li r0,4
	225	mtctr r0
	226	li r6,0
	227	1: lfsx fr0,r4,r6
	228	lfsx fr1,r5,r6
	229	fsubs fr0,fr0,fr1
	230	stfsx fr0,r3,r6
	231	addi r6,r6,4
	232	bdnz 1b
	233	b fpdisable
	234
	235	/*
	236	* Vector multiply and add, floating point.
	237	*/
	238	_GLOBAL(vmaddfp)
	239	mflr r12
	240	bl fpenable
	241	stfd fr2,32(r1)
	242	li r0,4
	243	mtctr r0
	244	li r7,0
	245	1: lfsx fr0,r4,r7
	246	lfsx fr1,r5,r7
	247	lfsx fr2,r6,r7
	248	fmadds fr0,fr0,fr2,fr1
	249	stfsx fr0,r3,r7
	250	addi r7,r7,4
	251	bdnz 1b
	252	lfd fr2,32(r1)
	253	b fpdisable
	254
	255	/*
	256	* Vector negative multiply and subtract, floating point.
257	*/
258	_GLOBAL(vnmsubfp)
259	mflr r12
260	bl fpenable
261	stfd fr2,32(r1)
262	li r0,4
263	mtctr r0
264	li r7,0
265	1: lfsx fr0,r4,r7
266	lfsx fr1,r5,r7
267	lfsx fr2,r6,r7
268	fnmsubs fr0,fr0,fr2,fr1
269	stfsx fr0,r3,r7
270	addi r7,r7,4
271	bdnz 1b
272	lfd fr2,32(r1)
273	b fpdisable
274
275	/*
276	* Vector reciprocal estimate. We just compute 1.0/x.
277	* r3 -> destination, r4 -> source.
278	*/
279	_GLOBAL(vrefp)
280	mflr r12
281	bl fpenable
282	li r0,4
283	LDCONST(fr1, fpone)
284	mtctr r0
285	li r6,0
286	1: lfsx fr0,r4,r6
287	fdivs fr0,fr1,fr0
288	stfsx fr0,r3,r6
289	addi r6,r6,4
290	bdnz 1b
291	b fpdisable
292
293	/*
294	* Vector reciprocal square-root estimate, floating point.
295	* We use the frsqrte instruction for the initial estimate followed
296	* by 2 iterations of Newton-Raphson to get sufficient accuracy.
297	* r3 -> destination, r4 -> source.
298	*/
299	_GLOBAL(vrsqrtefp)
300	mflr r12
301	bl fpenable
302	stfd fr2,32(r1)
303	stfd fr3,40(r1)
304	stfd fr4,48(r1)
305	stfd fr5,56(r1)
306	li r0,4
307	LDCONST(fr4, fpone)
308	LDCONST(fr5, fphalf)
309	mtctr r0
310	li r6,0
311	1: lfsx fr0,r4,r6
312	frsqrte fr1,fr0 /* r = frsqrte(s) */
313	fmuls fr3,fr1,fr0 /* r * s */
314	fmuls fr2,fr1,fr5 /* r * 0.5 */
315	fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */
316	fmadds fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */
317	fmuls fr3,fr1,fr0 /* r * s */
318	fmuls fr2,fr1,fr5 /* r * 0.5 */
319	fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */
320	fmadds fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */
321	stfsx fr1,r3,r6
322	addi r6,r6,4
323	bdnz 1b
324	lfd fr5,56(r1)
325	lfd fr4,48(r1)
326	lfd fr3,40(r1)
327	lfd fr2,32(r1)
328	b fpdisable