[mirror_ubuntu-hirsute-kernel.git] / arch / powerpc / kernel / vector.S

/* SPDX-License-Identifier: GPL-2.0 */
#include <asm/processor.h>
#include <asm/ppc_asm.h>
#include <asm/reg.h>
#include <asm/asm-offsets.h>
#include <asm/cputable.h>
#include <asm/thread_info.h>
#include <asm/page.h>
#include <asm/ptrace.h>
#include <asm/export.h>

/*
 * Load state from memory into VMX registers including VSCR.
 * Assumes the caller has enabled VMX in the MSR.
 */
_GLOBAL(load_vr_state)
	li	r4,VRSTATE_VSCR
	lvx	v0,r4,r3
	mtvscr	v0
	REST_32VRS(0,r4,r3)
	blr
EXPORT_SYMBOL(load_vr_state)

/*
 * Store VMX state into memory, including VSCR.
 * Assumes the caller has enabled VMX in the MSR.
 */
_GLOBAL(store_vr_state)
	SAVE_32VRS(0, r4, r3)
	mfvscr	v0
	li	r4, VRSTATE_VSCR
	stvx	v0, r4, r3
	blr
EXPORT_SYMBOL(store_vr_state)

/*
 * Disable VMX for the task which had it previously,
 * and save its vector registers in its thread_struct.
 * Enables the VMX for use in the kernel on return.
 * On SMP we know the VMX is free, since we give it up every
 * switch (ie, no lazy save of the vector registers).
 *
 * Note that on 32-bit this can only use registers that will be
 * restored by fast_exception_return, i.e. r3 - r6, r10 and r11.
 */
_GLOBAL(load_up_altivec)
	mfmsr	r5			/* grab the current MSR */
	oris	r5,r5,MSR_VEC@h
	MTMSRD(r5)			/* enable use of AltiVec now */
	isync

	/*
	 * While userspace in general ignores VRSAVE, glibc uses it as a boolean
	 * to optimise userspace context save/restore. Whenever we take an
	 * altivec unavailable exception we must set VRSAVE to something non
	 * zero. Set it to all 1s. See also the programming note in the ISA.
	 */
	mfspr	r4,SPRN_VRSAVE
	cmpwi	0,r4,0
	bne+	1f
	li	r4,-1
	mtspr	SPRN_VRSAVE,r4
1:
	/* enable use of VMX after return */
#ifdef CONFIG_PPC32
	mfspr	r5,SPRN_SPRG_THREAD		/* current task's THREAD (phys) */
	oris	r9,r9,MSR_VEC@h
#else
	ld	r4,PACACURRENT(r13)
	addi	r5,r4,THREAD		/* Get THREAD */
	oris	r12,r12,MSR_VEC@h
	std	r12,_MSR(r1)
#endif
	/* Don't care if r4 overflows, this is desired behaviour */
	lbz	r4,THREAD_LOAD_VEC(r5)
	addi	r4,r4,1
	stb	r4,THREAD_LOAD_VEC(r5)
	addi	r6,r5,THREAD_VRSTATE
	li	r4,1
	li	r10,VRSTATE_VSCR
	stw	r4,THREAD_USED_VR(r5)
	lvx	v0,r10,r6
	mtvscr	v0
	REST_32VRS(0,r4,r6)
	/* restore registers and return */
	blr

/*
 * save_altivec(tsk)
 * Save the vector registers to its thread_struct
 */
_GLOBAL(save_altivec)
	addi	r3,r3,THREAD		/* want THREAD of task */
	PPC_LL	r7,THREAD_VRSAVEAREA(r3)
	PPC_LL	r5,PT_REGS(r3)
	PPC_LCMPI	0,r7,0
	bne	2f
	addi	r7,r3,THREAD_VRSTATE
2:	SAVE_32VRS(0,r4,r7)
	mfvscr	v0
	li	r4,VRSTATE_VSCR
	stvx	v0,r4,r7
	blr

#ifdef CONFIG_VSX

#ifdef CONFIG_PPC32
#error This asm code isn't ready for 32-bit kernels
#endif

/*
 * load_up_vsx(unused, unused, tsk)
 * Disable VSX for the task which had it previously,
 * and save its vector registers in its thread_struct.
 * Reuse the fp and vsx saves, but first check to see if they have
 * been saved already.
 */
_GLOBAL(load_up_vsx)
/* Load FP and VSX registers if they haven't been done yet */
	andi.	r5,r12,MSR_FP
	beql+	load_up_fpu		/* skip if already loaded */
	andis.	r5,r12,MSR_VEC@h
	beql+	load_up_altivec		/* skip if already loaded */

	ld	r4,PACACURRENT(r13)
	addi	r4,r4,THREAD		/* Get THREAD */
	li	r6,1
	stw	r6,THREAD_USED_VSR(r4) /* ... also set thread used vsr */
	/* enable use of VSX after return */
	oris	r12,r12,MSR_VSX@h
	std	r12,_MSR(r1)
	b	fast_exception_return

#endif /* CONFIG_VSX */


/*
 * The routines below are in assembler so we can closely control the
 * usage of floating-point registers.  These routines must be called
 * with preempt disabled.
 */
#ifdef CONFIG_PPC32
	.data
fpzero:
	.long	0
fpone:
	.long	0x3f800000	/* 1.0 in single-precision FP */
fphalf:
	.long	0x3f000000	/* 0.5 in single-precision FP */

#define LDCONST(fr, name)	\
	lis	r11,name@ha;	\
	lfs	fr,name@l(r11)
#else

	.section ".toc","aw"
fpzero:
	.tc	FD_0_0[TC],0
fpone:
	.tc	FD_3ff00000_0[TC],0x3ff0000000000000	/* 1.0 */
fphalf:
	.tc	FD_3fe00000_0[TC],0x3fe0000000000000	/* 0.5 */

#define LDCONST(fr, name)	\
	lfd	fr,name@toc(r2)
#endif

	.text
/*
 * Internal routine to enable floating point and set FPSCR to 0.
 * Don't call it from C; it doesn't use the normal calling convention.
 */
fpenable:
#ifdef CONFIG_PPC32
	stwu	r1,-64(r1)
#else
	stdu	r1,-64(r1)
#endif
	mfmsr	r10
	ori	r11,r10,MSR_FP
	mtmsr	r11
	isync
	stfd	fr0,24(r1)
	stfd	fr1,16(r1)
	stfd	fr31,8(r1)
	LDCONST(fr1, fpzero)
	mffs	fr31
	MTFSF_L(fr1)
	blr

fpdisable:
	mtlr	r12
	MTFSF_L(fr31)
	lfd	fr31,8(r1)
	lfd	fr1,16(r1)
	lfd	fr0,24(r1)
	mtmsr	r10
	isync
	addi	r1,r1,64
	blr

/*
 * Vector add, floating point.
 */
_GLOBAL(vaddfp)
	mflr	r12
	bl	fpenable
	li	r0,4
	mtctr	r0
	li	r6,0
1:	lfsx	fr0,r4,r6
	lfsx	fr1,r5,r6
	fadds	fr0,fr0,fr1
	stfsx	fr0,r3,r6
	addi	r6,r6,4
	bdnz	1b
	b	fpdisable

/*
 * Vector subtract, floating point.
 */
_GLOBAL(vsubfp)
	mflr	r12
	bl	fpenable
	li	r0,4
	mtctr	r0
	li	r6,0
1:	lfsx	fr0,r4,r6
	lfsx	fr1,r5,r6
	fsubs	fr0,fr0,fr1
	stfsx	fr0,r3,r6
	addi	r6,r6,4
	bdnz	1b
	b	fpdisable

/*
 * Vector multiply and add, floating point.
 */
_GLOBAL(vmaddfp)
	mflr	r12
	bl	fpenable
	stfd	fr2,32(r1)
	li	r0,4
	mtctr	r0
	li	r7,0
1:	lfsx	fr0,r4,r7
	lfsx	fr1,r5,r7
	lfsx	fr2,r6,r7
	fmadds	fr0,fr0,fr2,fr1
	stfsx	fr0,r3,r7
	addi	r7,r7,4
	bdnz	1b
	lfd	fr2,32(r1)
	b	fpdisable

/*
 * Vector negative multiply and subtract, floating point.
 */
_GLOBAL(vnmsubfp)
	mflr	r12
	bl	fpenable
	stfd	fr2,32(r1)
	li	r0,4
	mtctr	r0
	li	r7,0
1:	lfsx	fr0,r4,r7
	lfsx	fr1,r5,r7
	lfsx	fr2,r6,r7
	fnmsubs	fr0,fr0,fr2,fr1
	stfsx	fr0,r3,r7
	addi	r7,r7,4
	bdnz	1b
	lfd	fr2,32(r1)
	b	fpdisable

/*
 * Vector reciprocal estimate.  We just compute 1.0/x.
 * r3 -> destination, r4 -> source.
 */
_GLOBAL(vrefp)
	mflr	r12
	bl	fpenable
	li	r0,4
	LDCONST(fr1, fpone)
	mtctr	r0
	li	r6,0
1:	lfsx	fr0,r4,r6
	fdivs	fr0,fr1,fr0
	stfsx	fr0,r3,r6
	addi	r6,r6,4
	bdnz	1b
	b	fpdisable

/*
 * Vector reciprocal square-root estimate, floating point.
 * We use the frsqrte instruction for the initial estimate followed
 * by 2 iterations of Newton-Raphson to get sufficient accuracy.
 * r3 -> destination, r4 -> source.
 */
_GLOBAL(vrsqrtefp)
	mflr	r12
	bl	fpenable
	stfd	fr2,32(r1)
	stfd	fr3,40(r1)
	stfd	fr4,48(r1)
	stfd	fr5,56(r1)
	li	r0,4
	LDCONST(fr4, fpone)
	LDCONST(fr5, fphalf)
	mtctr	r0
	li	r6,0
1:	lfsx	fr0,r4,r6
	frsqrte	fr1,fr0		/* r = frsqrte(s) */
	fmuls	fr3,fr1,fr0	/* r * s */
	fmuls	fr2,fr1,fr5	/* r * 0.5 */
	fnmsubs	fr3,fr1,fr3,fr4	/* 1 - s * r * r */
	fmadds	fr1,fr2,fr3,fr1	/* r = r + 0.5 * r * (1 - s * r * r) */
	fmuls	fr3,fr1,fr0	/* r * s */
	fmuls	fr2,fr1,fr5	/* r * 0.5 */
	fnmsubs	fr3,fr1,fr3,fr4	/* 1 - s * r * r */
	fmadds	fr1,fr2,fr3,fr1	/* r = r + 0.5 * r * (1 - s * r * r) */
	stfsx	fr1,r3,r6
	addi	r6,r6,4
	bdnz	1b
	lfd	fr5,56(r1)
	lfd	fr4,48(r1)
	lfd	fr3,40(r1)
	lfd	fr2,32(r1)
	b	fpdisable
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
e821ea70	2	#include <asm/processor.h>
14cf11af	3	#include <asm/ppc_asm.h>
b3b8dc6c	4	#include <asm/reg.h>
e821ea70 BH	5	#include <asm/asm-offsets.h>
	6	#include <asm/cputable.h>
	7	#include <asm/thread_info.h>
	8	#include <asm/page.h>
46f52210	9	#include <asm/ptrace.h>
9445aa1a	10	#include <asm/export.h>
e821ea70	11
18461960 PM	12	/*
	13	* Load state from memory into VMX registers including VSCR.
	14	* Assumes the caller has enabled VMX in the MSR.
	15	*/
	16	_GLOBAL(load_vr_state)
	17	li r4,VRSTATE_VSCR
c2ce6f9f AB	18	lvx v0,r4,r3
c2ce6f9f AB	19	mtvscr v0
18461960 PM	20	REST_32VRS(0,r4,r3)
18461960 PM	21	blr
9445aa1a	22	EXPORT_SYMBOL(load_vr_state)
18461960 PM	23
	24	/*
	25	* Store VMX state into memory, including VSCR.
	26	* Assumes the caller has enabled VMX in the MSR.
	27	*/
	28	_GLOBAL(store_vr_state)
	29	SAVE_32VRS(0, r4, r3)
c2ce6f9f	30	mfvscr v0
18461960	31	li r4, VRSTATE_VSCR
c2ce6f9f	32	stvx v0, r4, r3
18461960	33	blr
9445aa1a	34	EXPORT_SYMBOL(store_vr_state)
18461960	35
e821ea70	36	/*
e821ea70 BH	37	* Disable VMX for the task which had it previously,
	38	* and save its vector registers in its thread_struct.
	39	* Enables the VMX for use in the kernel on return.
	40	* On SMP we know the VMX is free, since we give it up every
	41	* switch (ie, no lazy save of the vector registers).
955c1cab PM	42	*
	43	* Note that on 32-bit this can only use registers that will be
	44	* restored by fast_exception_return, i.e. r3 - r6, r10 and r11.
e821ea70 BH	45	*/
	46	_GLOBAL(load_up_altivec)
	47	mfmsr r5 /* grab the current MSR */
	48	oris r5,r5,MSR_VEC@h
	49	MTMSRD(r5) /* enable use of AltiVec now */
	50	isync
	51
dd570237 AB	52	/*
	53	* While userspace in general ignores VRSAVE, glibc uses it as a boolean
	54	* to optimise userspace context save/restore. Whenever we take an
	55	* altivec unavailable exception we must set VRSAVE to something non
	56	* zero. Set it to all 1s. See also the programming note in the ISA.
e821ea70 BH	57	*/
e821ea70 BH	58	mfspr r4,SPRN_VRSAVE
e090aa80	59	cmpwi 0,r4,0
e821ea70 BH	60	bne+ 1f
	61	li r4,-1
	62	mtspr SPRN_VRSAVE,r4
	63	1:
	64	/* enable use of VMX after return */
	65	#ifdef CONFIG_PPC32
ee43eb78	66	mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */
e821ea70 BH	67	oris r9,r9,MSR_VEC@h
	68	#else
	69	ld r4,PACACURRENT(r13)
	70	addi r5,r4,THREAD /* Get THREAD */
	71	oris r12,r12,MSR_VEC@h
	72	std r12,_MSR(r1)
	73	#endif
70fe3d98 CB	74	/* Don't care if r4 overflows, this is desired behaviour */
	75	lbz r4,THREAD_LOAD_VEC(r5)
	76	addi r4,r4,1
	77	stb r4,THREAD_LOAD_VEC(r5)
955c1cab	78	addi r6,r5,THREAD_VRSTATE
e821ea70	79	li r4,1
de79f7b9	80	li r10,VRSTATE_VSCR
e821ea70	81	stw r4,THREAD_USED_VR(r5)
c2ce6f9f AB	82	lvx v0,r10,r6
c2ce6f9f AB	83	mtvscr v0
955c1cab	84	REST_32VRS(0,r4,r6)
e821ea70 BH	85	/* restore registers and return */
	86	blr
	87
	88	/*
6f515d84 CB	89	* save_altivec(tsk)
6f515d84 CB	90	* Save the vector registers to its thread_struct
e821ea70	91	*/
6f515d84	92	_GLOBAL(save_altivec)
e821ea70	93	addi r3,r3,THREAD /* want THREAD of task */
18461960	94	PPC_LL r7,THREAD_VRSAVEAREA(r3)
e821ea70	95	PPC_LL r5,PT_REGS(r3)
18461960 PM	96	PPC_LCMPI 0,r7,0
	97	bne 2f
	98	addi r7,r3,THREAD_VRSTATE
6f515d84	99	2: SAVE_32VRS(0,r4,r7)
c2ce6f9f	100	mfvscr v0
de79f7b9	101	li r4,VRSTATE_VSCR
c2ce6f9f	102	stvx v0,r4,r7
e821ea70 BH	103	blr
	104
	105	#ifdef CONFIG_VSX
	106
	107	#ifdef CONFIG_PPC32
	108	#error This asm code isn't ready for 32-bit kernels
	109	#endif
	110
	111	/*
	112	* load_up_vsx(unused, unused, tsk)
	113	* Disable VSX for the task which had it previously,
	114	* and save its vector registers in its thread_struct.
	115	* Reuse the fp and vsx saves, but first check to see if they have
	116	* been saved already.
	117	*/
	118	_GLOBAL(load_up_vsx)
	119	/* Load FP and VSX registers if they haven't been done yet */
	120	andi. r5,r12,MSR_FP
	121	beql+ load_up_fpu /* skip if already loaded */
	122	andis. r5,r12,MSR_VEC@h
	123	beql+ load_up_altivec /* skip if already loaded */
	124
e821ea70 BH	125	ld r4,PACACURRENT(r13)
	126	addi r4,r4,THREAD /* Get THREAD */
	127	li r6,1
	128	stw r6,THREAD_USED_VSR(r4) /* ... also set thread used vsr */
	129	/* enable use of VSX after return */
	130	oris r12,r12,MSR_VSX@h
	131	std r12,_MSR(r1)
e821ea70 BH	132	b fast_exception_return
e821ea70 BH	133
e821ea70 BH	134	#endif /* CONFIG_VSX */
e821ea70 BH	135
14cf11af PM	136
	137	/*
	138	* The routines below are in assembler so we can closely control the
	139	* usage of floating-point registers. These routines must be called
	140	* with preempt disabled.
	141	*/
	142	#ifdef CONFIG_PPC32
	143	.data
	144	fpzero:
	145	.long 0
	146	fpone:
	147	.long 0x3f800000 /* 1.0 in single-precision FP */
	148	fphalf:
	149	.long 0x3f000000 /* 0.5 in single-precision FP */
	150
	151	#define LDCONST(fr, name) \
	152	lis r11,name@ha; \
	153	lfs fr,name@l(r11)
	154	#else
	155
	156	.section ".toc","aw"
	157	fpzero:
	158	.tc FD_0_0[TC],0
	159	fpone:
	160	.tc FD_3ff00000_0[TC],0x3ff0000000000000 /* 1.0 */
	161	fphalf:
	162	.tc FD_3fe00000_0[TC],0x3fe0000000000000 /* 0.5 */
	163
	164	#define LDCONST(fr, name) \
	165	lfd fr,name@toc(r2)
	166	#endif
	167
	168	.text
	169	/*
	170	* Internal routine to enable floating point and set FPSCR to 0.
	171	* Don't call it from C; it doesn't use the normal calling convention.
	172	*/
	173	fpenable:
	174	#ifdef CONFIG_PPC32
	175	stwu r1,-64(r1)
	176	#else
	177	stdu r1,-64(r1)
	178	#endif
	179	mfmsr r10
	180	ori r11,r10,MSR_FP
	181	mtmsr r11
	182	isync
	183	stfd fr0,24(r1)
	184	stfd fr1,16(r1)
	185	stfd fr31,8(r1)
	186	LDCONST(fr1, fpzero)
	187	mffs fr31
3a2c48cf	188	MTFSF_L(fr1)
14cf11af PM	189	blr
	190
	191	fpdisable:
	192	mtlr r12
3a2c48cf	193	MTFSF_L(fr31)
14cf11af PM	194	lfd fr31,8(r1)
	195	lfd fr1,16(r1)
	196	lfd fr0,24(r1)
	197	mtmsr r10
	198	isync
	199	addi r1,r1,64
	200	blr
	201
	202	/*
	203	* Vector add, floating point.
	204	*/
	205	_GLOBAL(vaddfp)
	206	mflr r12
	207	bl fpenable
	208	li r0,4
	209	mtctr r0
	210	li r6,0
	211	1: lfsx fr0,r4,r6
	212	lfsx fr1,r5,r6
	213	fadds fr0,fr0,fr1
	214	stfsx fr0,r3,r6
	215	addi r6,r6,4
	216	bdnz 1b
	217	b fpdisable
	218
	219	/*
	220	* Vector subtract, floating point.
	221	*/
	222	_GLOBAL(vsubfp)
	223	mflr r12
	224	bl fpenable
	225	li r0,4
	226	mtctr r0
	227	li r6,0
	228	1: lfsx fr0,r4,r6
	229	lfsx fr1,r5,r6
	230	fsubs fr0,fr0,fr1
	231	stfsx fr0,r3,r6
	232	addi r6,r6,4
	233	bdnz 1b
	234	b fpdisable
	235
	236	/*
	237	* Vector multiply and add, floating point.
	238	*/
	239	_GLOBAL(vmaddfp)
	240	mflr r12
	241	bl fpenable
	242	stfd fr2,32(r1)
	243	li r0,4
	244	mtctr r0
	245	li r7,0
	246	1: lfsx fr0,r4,r7
	247	lfsx fr1,r5,r7
	248	lfsx fr2,r6,r7
	249	fmadds fr0,fr0,fr2,fr1
	250	stfsx fr0,r3,r7
	251	addi r7,r7,4
	252	bdnz 1b
	253	lfd fr2,32(r1)
	254	b fpdisable
	255
	256	/*
	257	* Vector negative multiply and subtract, floating point.
258	*/
259	_GLOBAL(vnmsubfp)
260	mflr r12
261	bl fpenable
262	stfd fr2,32(r1)
263	li r0,4
264	mtctr r0
265	li r7,0
266	1: lfsx fr0,r4,r7
267	lfsx fr1,r5,r7
268	lfsx fr2,r6,r7
269	fnmsubs fr0,fr0,fr2,fr1
270	stfsx fr0,r3,r7
271	addi r7,r7,4
272	bdnz 1b
273	lfd fr2,32(r1)
274	b fpdisable
275
276	/*
277	* Vector reciprocal estimate. We just compute 1.0/x.
278	* r3 -> destination, r4 -> source.
279	*/
280	_GLOBAL(vrefp)
281	mflr r12
282	bl fpenable
283	li r0,4
284	LDCONST(fr1, fpone)
285	mtctr r0
286	li r6,0
287	1: lfsx fr0,r4,r6
288	fdivs fr0,fr1,fr0
289	stfsx fr0,r3,r6
290	addi r6,r6,4
291	bdnz 1b
292	b fpdisable
293
294	/*
295	* Vector reciprocal square-root estimate, floating point.
296	* We use the frsqrte instruction for the initial estimate followed
297	* by 2 iterations of Newton-Raphson to get sufficient accuracy.
298	* r3 -> destination, r4 -> source.
299	*/
300	_GLOBAL(vrsqrtefp)
301	mflr r12
302	bl fpenable
303	stfd fr2,32(r1)
304	stfd fr3,40(r1)
305	stfd fr4,48(r1)
306	stfd fr5,56(r1)
307	li r0,4
308	LDCONST(fr4, fpone)
309	LDCONST(fr5, fphalf)
310	mtctr r0
311	li r6,0
312	1: lfsx fr0,r4,r6
313	frsqrte fr1,fr0 /* r = frsqrte(s) */
314	fmuls fr3,fr1,fr0 /* r * s */
315	fmuls fr2,fr1,fr5 /* r * 0.5 */
316	fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */
317	fmadds fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */
318	fmuls fr3,fr1,fr0 /* r * s */
319	fmuls fr2,fr1,fr5 /* r * 0.5 */
320	fnmsubs fr3,fr1,fr3,fr4 /* 1 - s * r * r */
321	fmadds fr1,fr2,fr3,fr1 /* r = r + 0.5 * r * (1 - s * r * r) */
322	stfsx fr1,r3,r6
323	addi r6,r6,4
324	bdnz 1b
325	lfd fr5,56(r1)
326	lfd fr4,48(r1)
327	lfd fr3,40(r1)
328	lfd fr2,32(r1)
329	b fpdisable