[mirror_ubuntu-bionic-kernel.git] / arch / s390 / kernel / fpu.c

// SPDX-License-Identifier: GPL-2.0
/*
 * In-kernel vector facility support functions
 *
 * Copyright IBM Corp. 2015
 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
 */
#include <linux/kernel.h>
#include <linux/cpu.h>
#include <linux/sched.h>
#include <asm/fpu/types.h>
#include <asm/fpu/api.h>

asm(".include \"asm/vx-insn.h\"\n");

void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags)
{
	/*
	 * Limit the save to the FPU/vector registers already
	 * in use by the previous context
	 */
	flags &= state->mask;

	if (flags & KERNEL_FPC)
		/* Save floating point control */
		asm volatile("stfpc %0" : "=m" (state->fpc));

	if (!MACHINE_HAS_VX) {
		if (flags & KERNEL_VXR_V0V7) {
			/* Save floating-point registers */
			asm volatile("std 0,%0" : "=Q" (state->fprs[0]));
			asm volatile("std 1,%0" : "=Q" (state->fprs[1]));
			asm volatile("std 2,%0" : "=Q" (state->fprs[2]));
			asm volatile("std 3,%0" : "=Q" (state->fprs[3]));
			asm volatile("std 4,%0" : "=Q" (state->fprs[4]));
			asm volatile("std 5,%0" : "=Q" (state->fprs[5]));
			asm volatile("std 6,%0" : "=Q" (state->fprs[6]));
			asm volatile("std 7,%0" : "=Q" (state->fprs[7]));
			asm volatile("std 8,%0" : "=Q" (state->fprs[8]));
			asm volatile("std 9,%0" : "=Q" (state->fprs[9]));
			asm volatile("std 10,%0" : "=Q" (state->fprs[10]));
			asm volatile("std 11,%0" : "=Q" (state->fprs[11]));
			asm volatile("std 12,%0" : "=Q" (state->fprs[12]));
			asm volatile("std 13,%0" : "=Q" (state->fprs[13]));
			asm volatile("std 14,%0" : "=Q" (state->fprs[14]));
			asm volatile("std 15,%0" : "=Q" (state->fprs[15]));
		}
		return;
	}

	/* Test and save vector registers */
	asm volatile (
		/*
		 * Test if any vector register must be saved and, if so,
		 * test if all register can be saved.
		 */
		"	la	1,%[vxrs]\n"	/* load save area */
		"	tmll	%[m],30\n"	/* KERNEL_VXR */
		"	jz	7f\n"		/* no work -> done */
		"	jo	5f\n"		/* -> save V0..V31 */
		/*
		 * Test for special case KERNEL_FPU_MID only. In this
		 * case a vstm V8..V23 is the best instruction
		 */
		"	chi	%[m],12\n"	/* KERNEL_VXR_MID */
		"	jne	0f\n"		/* -> save V8..V23 */
		"	VSTM	8,23,128,1\n"	/* vstm %v8,%v23,128(%r1) */
		"	j	7f\n"
		/* Test and save the first half of 16 vector registers */
		"0:	tmll	%[m],6\n"	/* KERNEL_VXR_LOW */
		"	jz	3f\n"		/* -> KERNEL_VXR_HIGH */
		"	jo	2f\n"		/* 11 -> save V0..V15 */
		"	brc	2,1f\n"		/* 10 -> save V8..V15 */
		"	VSTM	0,7,0,1\n"	/* vstm %v0,%v7,0(%r1) */
		"	j	3f\n"
		"1:	VSTM	8,15,128,1\n"	/* vstm %v8,%v15,128(%r1) */
		"	j	3f\n"
		"2:	VSTM	0,15,0,1\n"	/* vstm %v0,%v15,0(%r1) */
		/* Test and save the second half of 16 vector registers */
		"3:	tmll	%[m],24\n"	/* KERNEL_VXR_HIGH */
		"	jz	7f\n"
		"	jo	6f\n"		/* 11 -> save V16..V31 */
		"	brc	2,4f\n"		/* 10 -> save V24..V31 */
		"	VSTM	16,23,256,1\n"	/* vstm %v16,%v23,256(%r1) */
		"	j	7f\n"
		"4:	VSTM	24,31,384,1\n"	/* vstm %v24,%v31,384(%r1) */
		"	j	7f\n"
		"5:	VSTM	0,15,0,1\n"	/* vstm %v0,%v15,0(%r1) */
		"6:	VSTM	16,31,256,1\n"	/* vstm %v16,%v31,256(%r1) */
		"7:"
		: [vxrs] "=Q" (*(struct vx_array *) &state->vxrs)
		: [m] "d" (flags)
		: "1", "cc");
}
EXPORT_SYMBOL(__kernel_fpu_begin);

void __kernel_fpu_end(struct kernel_fpu *state, u32 flags)
{
	/*
	 * Limit the restore to the FPU/vector registers of the
	 * previous context that have been overwritte by the
	 * current context
	 */
	flags &= state->mask;

	if (flags & KERNEL_FPC)
		/* Restore floating-point controls */
		asm volatile("lfpc %0" : : "Q" (state->fpc));

	if (!MACHINE_HAS_VX) {
		if (flags & KERNEL_VXR_V0V7) {
			/* Restore floating-point registers */
			asm volatile("ld 0,%0" : : "Q" (state->fprs[0]));
			asm volatile("ld 1,%0" : : "Q" (state->fprs[1]));
			asm volatile("ld 2,%0" : : "Q" (state->fprs[2]));
			asm volatile("ld 3,%0" : : "Q" (state->fprs[3]));
			asm volatile("ld 4,%0" : : "Q" (state->fprs[4]));
			asm volatile("ld 5,%0" : : "Q" (state->fprs[5]));
			asm volatile("ld 6,%0" : : "Q" (state->fprs[6]));
			asm volatile("ld 7,%0" : : "Q" (state->fprs[7]));
			asm volatile("ld 8,%0" : : "Q" (state->fprs[8]));
			asm volatile("ld 9,%0" : : "Q" (state->fprs[9]));
			asm volatile("ld 10,%0" : : "Q" (state->fprs[10]));
			asm volatile("ld 11,%0" : : "Q" (state->fprs[11]));
			asm volatile("ld 12,%0" : : "Q" (state->fprs[12]));
			asm volatile("ld 13,%0" : : "Q" (state->fprs[13]));
			asm volatile("ld 14,%0" : : "Q" (state->fprs[14]));
			asm volatile("ld 15,%0" : : "Q" (state->fprs[15]));
		}
		return;
	}

	/* Test and restore (load) vector registers */
	asm volatile (
		/*
		 * Test if any vector register must be loaded and, if so,
		 * test if all registers can be loaded at once.
		 */
		"	la	1,%[vxrs]\n"	/* load restore area */
		"	tmll	%[m],30\n"	/* KERNEL_VXR */
		"	jz	7f\n"		/* no work -> done */
		"	jo	5f\n"		/* -> restore V0..V31 */
		/*
		 * Test for special case KERNEL_FPU_MID only. In this
		 * case a vlm V8..V23 is the best instruction
		 */
		"	chi	%[m],12\n"	/* KERNEL_VXR_MID */
		"	jne	0f\n"		/* -> restore V8..V23 */
		"	VLM	8,23,128,1\n"	/* vlm %v8,%v23,128(%r1) */
		"	j	7f\n"
		/* Test and restore the first half of 16 vector registers */
		"0:	tmll	%[m],6\n"	/* KERNEL_VXR_LOW */
		"	jz	3f\n"		/* -> KERNEL_VXR_HIGH */
		"	jo	2f\n"		/* 11 -> restore V0..V15 */
		"	brc	2,1f\n"		/* 10 -> restore V8..V15 */
		"	VLM	0,7,0,1\n"	/* vlm %v0,%v7,0(%r1) */
		"	j	3f\n"
		"1:	VLM	8,15,128,1\n"	/* vlm %v8,%v15,128(%r1) */
		"	j	3f\n"
		"2:	VLM	0,15,0,1\n"	/* vlm %v0,%v15,0(%r1) */
		/* Test and restore the second half of 16 vector registers */
		"3:	tmll	%[m],24\n"	/* KERNEL_VXR_HIGH */
		"	jz	7f\n"
		"	jo	6f\n"		/* 11 -> restore V16..V31 */
		"	brc	2,4f\n"		/* 10 -> restore V24..V31 */
		"	VLM	16,23,256,1\n"	/* vlm %v16,%v23,256(%r1) */
		"	j	7f\n"
		"4:	VLM	24,31,384,1\n"	/* vlm %v24,%v31,384(%r1) */
		"	j	7f\n"
		"5:	VLM	0,15,0,1\n"	/* vlm %v0,%v15,0(%r1) */
		"6:	VLM	16,31,256,1\n"	/* vlm %v16,%v31,256(%r1) */
		"7:"
		: [vxrs] "=Q" (*(struct vx_array *) &state->vxrs)
		: [m] "d" (flags)
		: "1", "cc");
}
EXPORT_SYMBOL(__kernel_fpu_end);
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
04864808 HB	2	/*
	3	* In-kernel vector facility support functions
	4	*
	5	* Copyright IBM Corp. 2015
	6	* Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
	7	*/
	8	#include <linux/kernel.h>
	9	#include <linux/cpu.h>
	10	#include <linux/sched.h>
	11	#include <asm/fpu/types.h>
	12	#include <asm/fpu/api.h>
	13
7f79695c	14	asm(".include \"asm/vx-insn.h\"\n");
04864808 HB	15
	16	void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags)
	17	{
04864808	18	/*
7f79695c MS	19	* Limit the save to the FPU/vector registers already
7f79695c MS	20	* in use by the previous context
04864808	21	*/
7f79695c	22	flags &= state->mask;
04864808	23
7f79695c MS	24	if (flags & KERNEL_FPC)
	25	/* Save floating point control */
	26	asm volatile("stfpc %0" : "=m" (state->fpc));
	27
	28	if (!MACHINE_HAS_VX) {
	29	if (flags & KERNEL_VXR_V0V7) {
	30	/* Save floating-point registers */
	31	asm volatile("std 0,%0" : "=Q" (state->fprs[0]));
	32	asm volatile("std 1,%0" : "=Q" (state->fprs[1]));
	33	asm volatile("std 2,%0" : "=Q" (state->fprs[2]));
	34	asm volatile("std 3,%0" : "=Q" (state->fprs[3]));
	35	asm volatile("std 4,%0" : "=Q" (state->fprs[4]));
	36	asm volatile("std 5,%0" : "=Q" (state->fprs[5]));
	37	asm volatile("std 6,%0" : "=Q" (state->fprs[6]));
	38	asm volatile("std 7,%0" : "=Q" (state->fprs[7]));
	39	asm volatile("std 8,%0" : "=Q" (state->fprs[8]));
	40	asm volatile("std 9,%0" : "=Q" (state->fprs[9]));
	41	asm volatile("std 10,%0" : "=Q" (state->fprs[10]));
	42	asm volatile("std 11,%0" : "=Q" (state->fprs[11]));
	43	asm volatile("std 12,%0" : "=Q" (state->fprs[12]));
	44	asm volatile("std 13,%0" : "=Q" (state->fprs[13]));
	45	asm volatile("std 14,%0" : "=Q" (state->fprs[14]));
	46	asm volatile("std 15,%0" : "=Q" (state->fprs[15]));
	47	}
04864808 HB	48	return;
	49	}
	50
04864808 HB	51	/* Test and save vector registers */
	52	asm volatile (
	53	/*
	54	* Test if any vector register must be saved and, if so,
	55	* test if all register can be saved.
	56	*/
04864808	57	" la 1,%[vxrs]\n" /* load save area */
7f79695c MS	58	" tmll %[m],30\n" /* KERNEL_VXR */
	59	" jz 7f\n" /* no work -> done */
	60	" jo 5f\n" /* -> save V0..V31 */
04864808	61	/*
7f79695c MS	62	* Test for special case KERNEL_FPU_MID only. In this
7f79695c MS	63	* case a vstm V8..V23 is the best instruction
04864808	64	*/
7f79695c MS	65	" chi %[m],12\n" /* KERNEL_VXR_MID */
	66	" jne 0f\n" /* -> save V8..V23 */
	67	" VSTM 8,23,128,1\n" /* vstm %v8,%v23,128(%r1) */
	68	" j 7f\n"
04864808	69	/* Test and save the first half of 16 vector registers */
7f79695c MS	70	"0: tmll %[m],6\n" /* KERNEL_VXR_LOW */
7f79695c MS	71	" jz 3f\n" /* -> KERNEL_VXR_HIGH */
04864808	72	" jo 2f\n" /* 11 -> save V0..V15 */
7f79695c MS	73	" brc 2,1f\n" /* 10 -> save V8..V15 */
	74	" VSTM 0,7,0,1\n" /* vstm %v0,%v7,0(%r1) */
	75	" j 3f\n"
	76	"1: VSTM 8,15,128,1\n" /* vstm %v8,%v15,128(%r1) */
	77	" j 3f\n"
	78	"2: VSTM 0,15,0,1\n" /* vstm %v0,%v15,0(%r1) */
04864808	79	/* Test and save the second half of 16 vector registers */
7f79695c MS	80	"3: tmll %[m],24\n" /* KERNEL_VXR_HIGH */
	81	" jz 7f\n"
	82	" jo 6f\n" /* 11 -> save V16..V31 */
	83	" brc 2,4f\n" /* 10 -> save V24..V31 */
	84	" VSTM 16,23,256,1\n" /* vstm %v16,%v23,256(%r1) */
	85	" j 7f\n"
	86	"4: VSTM 24,31,384,1\n" /* vstm %v24,%v31,384(%r1) */
	87	" j 7f\n"
	88	"5: VSTM 0,15,0,1\n" /* vstm %v0,%v15,0(%r1) */
	89	"6: VSTM 16,31,256,1\n" /* vstm %v16,%v31,256(%r1) */
	90	"7:"
04864808	91	: [vxrs] "=Q" ((struct vx_array ) &state->vxrs)
7f79695c	92	: [m] "d" (flags)
04864808 HB	93	: "1", "cc");
	94	}
	95	EXPORT_SYMBOL(__kernel_fpu_begin);
	96
7f79695c	97	void __kernel_fpu_end(struct kernel_fpu *state, u32 flags)
04864808	98	{
04864808	99	/*
7f79695c MS	100	* Limit the restore to the FPU/vector registers of the
	101	* previous context that have been overwritte by the
	102	* current context
04864808	103	*/
7f79695c	104	flags &= state->mask;
04864808	105
7f79695c MS	106	if (flags & KERNEL_FPC)
7f79695c MS	107	/* Restore floating-point controls */
04864808 HB	108	asm volatile("lfpc %0" : : "Q" (state->fpc));
04864808 HB	109
7f79695c MS	110	if (!MACHINE_HAS_VX) {
	111	if (flags & KERNEL_VXR_V0V7) {
	112	/* Restore floating-point registers */
	113	asm volatile("ld 0,%0" : : "Q" (state->fprs[0]));
	114	asm volatile("ld 1,%0" : : "Q" (state->fprs[1]));
	115	asm volatile("ld 2,%0" : : "Q" (state->fprs[2]));
	116	asm volatile("ld 3,%0" : : "Q" (state->fprs[3]));
	117	asm volatile("ld 4,%0" : : "Q" (state->fprs[4]));
	118	asm volatile("ld 5,%0" : : "Q" (state->fprs[5]));
	119	asm volatile("ld 6,%0" : : "Q" (state->fprs[6]));
	120	asm volatile("ld 7,%0" : : "Q" (state->fprs[7]));
	121	asm volatile("ld 8,%0" : : "Q" (state->fprs[8]));
	122	asm volatile("ld 9,%0" : : "Q" (state->fprs[9]));
	123	asm volatile("ld 10,%0" : : "Q" (state->fprs[10]));
	124	asm volatile("ld 11,%0" : : "Q" (state->fprs[11]));
	125	asm volatile("ld 12,%0" : : "Q" (state->fprs[12]));
	126	asm volatile("ld 13,%0" : : "Q" (state->fprs[13]));
	127	asm volatile("ld 14,%0" : : "Q" (state->fprs[14]));
	128	asm volatile("ld 15,%0" : : "Q" (state->fprs[15]));
	129	}
	130	return;
	131	}
	132
04864808 HB	133	/* Test and restore (load) vector registers */
	134	asm volatile (
	135	/*
7f79695c	136	* Test if any vector register must be loaded and, if so,
04864808 HB	137	* test if all registers can be loaded at once.
04864808 HB	138	*/
7f79695c MS	139	" la 1,%[vxrs]\n" /* load restore area */
	140	" tmll %[m],30\n" /* KERNEL_VXR */
	141	" jz 7f\n" /* no work -> done */
	142	" jo 5f\n" /* -> restore V0..V31 */
04864808	143	/*
7f79695c MS	144	* Test for special case KERNEL_FPU_MID only. In this
7f79695c MS	145	* case a vlm V8..V23 is the best instruction
04864808	146	*/
7f79695c MS	147	" chi %[m],12\n" /* KERNEL_VXR_MID */
	148	" jne 0f\n" /* -> restore V8..V23 */
	149	" VLM 8,23,128,1\n" /* vlm %v8,%v23,128(%r1) */
	150	" j 7f\n"
	151	/* Test and restore the first half of 16 vector registers */
	152	"0: tmll %[m],6\n" /* KERNEL_VXR_LOW */
	153	" jz 3f\n" /* -> KERNEL_VXR_HIGH */
	154	" jo 2f\n" /* 11 -> restore V0..V15 */
	155	" brc 2,1f\n" /* 10 -> restore V8..V15 */
	156	" VLM 0,7,0,1\n" /* vlm %v0,%v7,0(%r1) */
	157	" j 3f\n"
	158	"1: VLM 8,15,128,1\n" /* vlm %v8,%v15,128(%r1) */
	159	" j 3f\n"
	160	"2: VLM 0,15,0,1\n" /* vlm %v0,%v15,0(%r1) */
	161	/* Test and restore the second half of 16 vector registers */
	162	"3: tmll %[m],24\n" /* KERNEL_VXR_HIGH */
	163	" jz 7f\n"
	164	" jo 6f\n" /* 11 -> restore V16..V31 */
	165	" brc 2,4f\n" /* 10 -> restore V24..V31 */
	166	" VLM 16,23,256,1\n" /* vlm %v16,%v23,256(%r1) */
	167	" j 7f\n"
	168	"4: VLM 24,31,384,1\n" /* vlm %v24,%v31,384(%r1) */
	169	" j 7f\n"
	170	"5: VLM 0,15,0,1\n" /* vlm %v0,%v15,0(%r1) */
	171	"6: VLM 16,31,256,1\n" /* vlm %v16,%v31,256(%r1) */
	172	"7:"
	173	: [vxrs] "=Q" ((struct vx_array ) &state->vxrs)
	174	: [m] "d" (flags)
04864808	175	: "1", "cc");
04864808 HB	176	}
04864808 HB	177	EXPORT_SYMBOL(__kernel_fpu_end);