[mirror_ubuntu-hirsute-kernel.git] / arch / arm / lib / copy_template.S

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/copy_template.s
 *
 *  Code template for optimized memory copy functions
 *
 *  Author:	Nicolas Pitre
 *  Created:	Sep 28, 2005
 *  Copyright:	MontaVista Software, Inc.
 */

/*
 * Theory of operation
 * -------------------
 *
 * This file provides the core code for a forward memory copy used in
 * the implementation of memcopy(), copy_to_user() and copy_from_user().
 *
 * The including file must define the following accessor macros
 * according to the need of the given function:
 *
 * ldr1w ptr reg abort
 *
 *	This loads one word from 'ptr', stores it in 'reg' and increments
 *	'ptr' to the next word. The 'abort' argument is used for fixup tables.
 *
 * ldr4w ptr reg1 reg2 reg3 reg4 abort
 * ldr8w ptr, reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
 *
 *	This loads four or eight words starting from 'ptr', stores them
 *	in provided registers and increments 'ptr' past those words.
 *	The'abort' argument is used for fixup tables.
 *
 * ldr1b ptr reg cond abort
 *
 *	Similar to ldr1w, but it loads a byte and increments 'ptr' one byte.
 *	It also must apply the condition code if provided, otherwise the
 *	"al" condition is assumed by default.
 *
 * str1w ptr reg abort
 * str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
 * str1b ptr reg cond abort
 *
 *	Same as their ldr* counterparts, but data is stored to 'ptr' location
 *	rather than being loaded.
 *
 * enter reg1 reg2
 *
 *	Preserve the provided registers on the stack plus any additional
 *	data as needed by the implementation including this code. Called
 *	upon code entry.
 *
 * usave reg1 reg2
 *
 *	Unwind annotation macro is corresponding for 'enter' macro.
 *	It tell unwinder that preserved some provided registers on the stack
 *	and additional data by a prior 'enter' macro.
 *
 * exit reg1 reg2
 *
 *	Restore registers with the values previously saved with the
 *	'preserv' macro. Called upon code termination.
 *
 * LDR1W_SHIFT
 * STR1W_SHIFT
 *
 *	Correction to be applied to the "ip" register when branching into
 *	the ldr1w or str1w instructions (some of these macros may expand to
 *	than one 32bit instruction in Thumb-2)
 */


	UNWIND(	.fnstart			)
		enter	r4, lr
	UNWIND(	.fnend				)

	UNWIND(	.fnstart			)
		usave	r4, lr			  @ in first stmdb block

		subs	r2, r2, #4
		blt	8f
		ands	ip, r0, #3
	PLD(	pld	[r1, #0]		)
		bne	9f
		ands	ip, r1, #3
		bne	10f

1:		subs	r2, r2, #(28)
		stmfd	sp!, {r5 - r8}
	UNWIND(	.fnend				)

	UNWIND(	.fnstart			)
		usave	r4, lr
	UNWIND(	.save	{r5 - r8}		) @ in second stmfd block
		blt	5f

	CALGN(	ands	ip, r0, #31		)
	CALGN(	rsb	r3, ip, #32		)
	CALGN(	sbcsne	r4, r3, r2		)  @ C is always set here
	CALGN(	bcs	2f			)
	CALGN(	adr	r4, 6f			)
	CALGN(	subs	r2, r2, r3		)  @ C gets set
	CALGN(	add	pc, r4, ip		)

	PLD(	pld	[r1, #0]		)
2:	PLD(	subs	r2, r2, #96		)
	PLD(	pld	[r1, #28]		)
	PLD(	blt	4f			)
	PLD(	pld	[r1, #60]		)
	PLD(	pld	[r1, #92]		)

3:	PLD(	pld	[r1, #124]		)
4:		ldr8w	r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
		subs	r2, r2, #32
		str8w	r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
		bge	3b
	PLD(	cmn	r2, #96			)
	PLD(	bge	4b			)

5:		ands	ip, r2, #28
		rsb	ip, ip, #32
#if LDR1W_SHIFT > 0
		lsl	ip, ip, #LDR1W_SHIFT
#endif
		addne	pc, pc, ip		@ C is always clear here
		b	7f
6:
		.rept	(1 << LDR1W_SHIFT)
		W(nop)
		.endr
		ldr1w	r1, r3, abort=20f
		ldr1w	r1, r4, abort=20f
		ldr1w	r1, r5, abort=20f
		ldr1w	r1, r6, abort=20f
		ldr1w	r1, r7, abort=20f
		ldr1w	r1, r8, abort=20f
		ldr1w	r1, lr, abort=20f

#if LDR1W_SHIFT < STR1W_SHIFT
		lsl	ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
#elif LDR1W_SHIFT > STR1W_SHIFT
		lsr	ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
#endif
		add	pc, pc, ip
		nop
		.rept	(1 << STR1W_SHIFT)
		W(nop)
		.endr
		str1w	r0, r3, abort=20f
		str1w	r0, r4, abort=20f
		str1w	r0, r5, abort=20f
		str1w	r0, r6, abort=20f
		str1w	r0, r7, abort=20f
		str1w	r0, r8, abort=20f
		str1w	r0, lr, abort=20f

	CALGN(	bcs	2b			)

7:		ldmfd	sp!, {r5 - r8}
	UNWIND(	.fnend				) @ end of second stmfd block

	UNWIND(	.fnstart			)
		usave	r4, lr			  @ still in first stmdb block
8:		movs	r2, r2, lsl #31
		ldr1b	r1, r3, ne, abort=21f
		ldr1b	r1, r4, cs, abort=21f
		ldr1b	r1, ip, cs, abort=21f
		str1b	r0, r3, ne, abort=21f
		str1b	r0, r4, cs, abort=21f
		str1b	r0, ip, cs, abort=21f

		exit	r4, pc

9:		rsb	ip, ip, #4
		cmp	ip, #2
		ldr1b	r1, r3, gt, abort=21f
		ldr1b	r1, r4, ge, abort=21f
		ldr1b	r1, lr, abort=21f
		str1b	r0, r3, gt, abort=21f
		str1b	r0, r4, ge, abort=21f
		subs	r2, r2, ip
		str1b	r0, lr, abort=21f
		blt	8b
		ands	ip, r1, #3
		beq	1b

10:		bic	r1, r1, #3
		cmp	ip, #2
		ldr1w	r1, lr, abort=21f
		beq	17f
		bgt	18f
	UNWIND(	.fnend				)


		.macro	forward_copy_shift pull push

	UNWIND(	.fnstart			)
		usave	r4, lr			  @ still in first stmdb block
		subs	r2, r2, #28
		blt	14f

	CALGN(	ands	ip, r0, #31		)
	CALGN(	rsb	ip, ip, #32		)
	CALGN(	sbcsne	r4, ip, r2		)  @ C is always set here
	CALGN(	subcc	r2, r2, ip		)
	CALGN(	bcc	15f			)

11:		stmfd	sp!, {r5 - r9}
	UNWIND(	.fnend				)

	UNWIND(	.fnstart			)
		usave	r4, lr
	UNWIND(	.save	{r5 - r9}		) @ in new second stmfd block
	PLD(	pld	[r1, #0]		)
	PLD(	subs	r2, r2, #96		)
	PLD(	pld	[r1, #28]		)
	PLD(	blt	13f			)
	PLD(	pld	[r1, #60]		)
	PLD(	pld	[r1, #92]		)

12:	PLD(	pld	[r1, #124]		)
13:		ldr4w	r1, r4, r5, r6, r7, abort=19f
		mov	r3, lr, lspull #\pull
		subs	r2, r2, #32
		ldr4w	r1, r8, r9, ip, lr, abort=19f
		orr	r3, r3, r4, lspush #\push
		mov	r4, r4, lspull #\pull
		orr	r4, r4, r5, lspush #\push
		mov	r5, r5, lspull #\pull
		orr	r5, r5, r6, lspush #\push
		mov	r6, r6, lspull #\pull
		orr	r6, r6, r7, lspush #\push
		mov	r7, r7, lspull #\pull
		orr	r7, r7, r8, lspush #\push
		mov	r8, r8, lspull #\pull
		orr	r8, r8, r9, lspush #\push
		mov	r9, r9, lspull #\pull
		orr	r9, r9, ip, lspush #\push
		mov	ip, ip, lspull #\pull
		orr	ip, ip, lr, lspush #\push
		str8w	r0, r3, r4, r5, r6, r7, r8, r9, ip, abort=19f
		bge	12b
	PLD(	cmn	r2, #96			)
	PLD(	bge	13b			)

		ldmfd	sp!, {r5 - r9}
	UNWIND(	.fnend				) @ end of the second stmfd block

	UNWIND(	.fnstart			)
		usave	r4, lr			  @ still in first stmdb block
14:		ands	ip, r2, #28
		beq	16f

15:		mov	r3, lr, lspull #\pull
		ldr1w	r1, lr, abort=21f
		subs	ip, ip, #4
		orr	r3, r3, lr, lspush #\push
		str1w	r0, r3, abort=21f
		bgt	15b
	CALGN(	cmp	r2, #0			)
	CALGN(	bge	11b			)

16:		sub	r1, r1, #(\push / 8)
		b	8b
	UNWIND(	.fnend				)

		.endm


		forward_copy_shift	pull=8	push=24

17:		forward_copy_shift	pull=16	push=16

18:		forward_copy_shift	pull=24	push=8


/*
 * Abort preamble and completion macros.
 * If a fixup handler is required then those macros must surround it.
 * It is assumed that the fixup code will handle the private part of
 * the exit macro.
 */

	.macro	copy_abort_preamble
19:	ldmfd	sp!, {r5 - r9}
	b	21f
20:	ldmfd	sp!, {r5 - r8}
21:
	.endm

	.macro	copy_abort_end
	ldmfd	sp!, {r4, pc}
	.endm
Commit	Line	Data
d2912cb1	1	/* SPDX-License-Identifier: GPL-2.0-only */
75494230 NP	2	/*
	3	* linux/arch/arm/lib/copy_template.s
	4	*
	5	* Code template for optimized memory copy functions
	6	*
	7	* Author: Nicolas Pitre
	8	* Created: Sep 28, 2005
	9	* Copyright: MontaVista Software, Inc.
75494230 NP	10	*/
75494230 NP	11
75494230 NP	12	/*
	13	* Theory of operation
	14	* -------------------
	15	*
	16	* This file provides the core code for a forward memory copy used in
	17	* the implementation of memcopy(), copy_to_user() and copy_from_user().
	18	*
	19	* The including file must define the following accessor macros
	20	* according to the need of the given function:
	21	*
	22	* ldr1w ptr reg abort
	23	*
	24	* This loads one word from 'ptr', stores it in 'reg' and increments
	25	* 'ptr' to the next word. The 'abort' argument is used for fixup tables.
	26	*
	27	* ldr4w ptr reg1 reg2 reg3 reg4 abort
	28	* ldr8w ptr, reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
	29	*
	30	* This loads four or eight words starting from 'ptr', stores them
	31	* in provided registers and increments 'ptr' past those words.
	32	* The'abort' argument is used for fixup tables.
	33	*
	34	* ldr1b ptr reg cond abort
	35	*
	36	* Similar to ldr1w, but it loads a byte and increments 'ptr' one byte.
	37	* It also must apply the condition code if provided, otherwise the
	38	* "al" condition is assumed by default.
	39	*
	40	* str1w ptr reg abort
	41	* str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
	42	* str1b ptr reg cond abort
	43	*
	44	* Same as their ldr* counterparts, but data is stored to 'ptr' location
	45	* rather than being loaded.
	46	*
	47	* enter reg1 reg2
	48	*
	49	* Preserve the provided registers on the stack plus any additional
	50	* data as needed by the implementation including this code. Called
	51	* upon code entry.
	52	*
279f487e LY	53	* usave reg1 reg2
	54	*
	55	* Unwind annotation macro is corresponding for 'enter' macro.
	56	* It tell unwinder that preserved some provided registers on the stack
	57	* and additional data by a prior 'enter' macro.
	58	*
75494230 NP	59	* exit reg1 reg2
	60	*
	61	* Restore registers with the values previously saved with the
	62	* 'preserv' macro. Called upon code termination.
8b592783 CM	63	*
	64	* LDR1W_SHIFT
	65	* STR1W_SHIFT
	66	*
	67	* Correction to be applied to the "ip" register when branching into
	68	* the ldr1w or str1w instructions (some of these macros may expand to
	69	* than one 32bit instruction in Thumb-2)
75494230 NP	70	*/
	71
	72
279f487e	73	UNWIND( .fnstart )
75494230	74	enter r4, lr
279f487e LY	75	UNWIND( .fnend )
	76
	77	UNWIND( .fnstart )
	78	usave r4, lr @ in first stmdb block
75494230 NP	79
	80	subs r2, r2, #4
	81	blt 8f
	82	ands ip, r0, #3
	83	PLD( pld [r1, #0] )
	84	bne 9f
	85	ands ip, r1, #3
	86	bne 10f
	87
	88	1: subs r2, r2, #(28)
	89	stmfd sp!, {r5 - r8}
279f487e LY	90	UNWIND( .fnend )
	91
	92	UNWIND( .fnstart )
	93	usave r4, lr
	94	UNWIND( .save {r5 - r8} ) @ in second stmfd block
75494230 NP	95	blt 5f
75494230 NP	96
2239aff6	97	CALGN( ands ip, r0, #31 )
75494230	98	CALGN( rsb r3, ip, #32 )
e44fc388	99	CALGN( sbcsne r4, r3, r2 ) @ C is always set here
75494230 NP	100	CALGN( bcs 2f )
	101	CALGN( adr r4, 6f )
	102	CALGN( subs r2, r2, r3 ) @ C gets set
	103	CALGN( add pc, r4, ip )
	104
	105	PLD( pld [r1, #0] )
	106	2: PLD( subs r2, r2, #96 )
	107	PLD( pld [r1, #28] )
	108	PLD( blt 4f )
	109	PLD( pld [r1, #60] )
	110	PLD( pld [r1, #92] )
	111
	112	3: PLD( pld [r1, #124] )
	113	4: ldr8w r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
	114	subs r2, r2, #32
	115	str8w r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
	116	bge 3b
	117	PLD( cmn r2, #96 )
	118	PLD( bge 4b )
	119
	120	5: ands ip, r2, #28
	121	rsb ip, ip, #32
8b592783 CM	122	#if LDR1W_SHIFT > 0
	123	lsl ip, ip, #LDR1W_SHIFT
	124	#endif
75494230 NP	125	addne pc, pc, ip @ C is always clear here
75494230 NP	126	b 7f
8b592783 CM	127	6:
	128	.rept (1 << LDR1W_SHIFT)
	129	W(nop)
	130	.endr
75494230 NP	131	ldr1w r1, r3, abort=20f
	132	ldr1w r1, r4, abort=20f
	133	ldr1w r1, r5, abort=20f
	134	ldr1w r1, r6, abort=20f
	135	ldr1w r1, r7, abort=20f
	136	ldr1w r1, r8, abort=20f
	137	ldr1w r1, lr, abort=20f
	138
8b592783 CM	139	#if LDR1W_SHIFT < STR1W_SHIFT
	140	lsl ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
	141	#elif LDR1W_SHIFT > STR1W_SHIFT
	142	lsr ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
	143	#endif
75494230 NP	144	add pc, pc, ip
75494230 NP	145	nop
8b592783 CM	146	.rept (1 << STR1W_SHIFT)
	147	W(nop)
	148	.endr
75494230 NP	149	str1w r0, r3, abort=20f
	150	str1w r0, r4, abort=20f
	151	str1w r0, r5, abort=20f
	152	str1w r0, r6, abort=20f
	153	str1w r0, r7, abort=20f
	154	str1w r0, r8, abort=20f
	155	str1w r0, lr, abort=20f
	156
	157	CALGN( bcs 2b )
	158
	159	7: ldmfd sp!, {r5 - r8}
279f487e	160	UNWIND( .fnend ) @ end of second stmfd block
75494230	161
279f487e LY	162	UNWIND( .fnstart )
279f487e LY	163	usave r4, lr @ still in first stmdb block
75494230 NP	164	8: movs r2, r2, lsl #31
	165	ldr1b r1, r3, ne, abort=21f
	166	ldr1b r1, r4, cs, abort=21f
	167	ldr1b r1, ip, cs, abort=21f
	168	str1b r0, r3, ne, abort=21f
	169	str1b r0, r4, cs, abort=21f
	170	str1b r0, ip, cs, abort=21f
	171
	172	exit r4, pc
	173
	174	9: rsb ip, ip, #4
	175	cmp ip, #2
	176	ldr1b r1, r3, gt, abort=21f
	177	ldr1b r1, r4, ge, abort=21f
	178	ldr1b r1, lr, abort=21f
	179	str1b r0, r3, gt, abort=21f
	180	str1b r0, r4, ge, abort=21f
	181	subs r2, r2, ip
	182	str1b r0, lr, abort=21f
	183	blt 8b
	184	ands ip, r1, #3
	185	beq 1b
	186
	187	10: bic r1, r1, #3
	188	cmp ip, #2
	189	ldr1w r1, lr, abort=21f
	190	beq 17f
	191	bgt 18f
279f487e	192	UNWIND( .fnend )
75494230 NP	193
	194
	195	.macro forward_copy_shift pull push
	196
279f487e LY	197	UNWIND( .fnstart )
279f487e LY	198	usave r4, lr @ still in first stmdb block
75494230 NP	199	subs r2, r2, #28
	200	blt 14f
	201
2239aff6	202	CALGN( ands ip, r0, #31 )
75494230	203	CALGN( rsb ip, ip, #32 )
e44fc388	204	CALGN( sbcsne r4, ip, r2 ) @ C is always set here
75494230 NP	205	CALGN( subcc r2, r2, ip )
	206	CALGN( bcc 15f )
	207
	208	11: stmfd sp!, {r5 - r9}
279f487e	209	UNWIND( .fnend )
75494230	210
279f487e LY	211	UNWIND( .fnstart )
	212	usave r4, lr
	213	UNWIND( .save {r5 - r9} ) @ in new second stmfd block
75494230 NP	214	PLD( pld [r1, #0] )
	215	PLD( subs r2, r2, #96 )
	216	PLD( pld [r1, #28] )
	217	PLD( blt 13f )
	218	PLD( pld [r1, #60] )
	219	PLD( pld [r1, #92] )
	220
	221	12: PLD( pld [r1, #124] )
	222	13: ldr4w r1, r4, r5, r6, r7, abort=19f
d98b90ea	223	mov r3, lr, lspull #\pull
75494230 NP	224	subs r2, r2, #32
75494230 NP	225	ldr4w r1, r8, r9, ip, lr, abort=19f
d98b90ea VK	226	orr r3, r3, r4, lspush #\push
	227	mov r4, r4, lspull #\pull
	228	orr r4, r4, r5, lspush #\push
	229	mov r5, r5, lspull #\pull
	230	orr r5, r5, r6, lspush #\push
	231	mov r6, r6, lspull #\pull
	232	orr r6, r6, r7, lspush #\push
	233	mov r7, r7, lspull #\pull
	234	orr r7, r7, r8, lspush #\push
	235	mov r8, r8, lspull #\pull
	236	orr r8, r8, r9, lspush #\push
	237	mov r9, r9, lspull #\pull
	238	orr r9, r9, ip, lspush #\push
	239	mov ip, ip, lspull #\pull
	240	orr ip, ip, lr, lspush #\push
baf2df8e	241	str8w r0, r3, r4, r5, r6, r7, r8, r9, ip, abort=19f
75494230 NP	242	bge 12b
	243	PLD( cmn r2, #96 )
	244	PLD( bge 13b )
	245
	246	ldmfd sp!, {r5 - r9}
279f487e	247	UNWIND( .fnend ) @ end of the second stmfd block
75494230	248
279f487e LY	249	UNWIND( .fnstart )
279f487e LY	250	usave r4, lr @ still in first stmdb block
75494230 NP	251	14: ands ip, r2, #28
	252	beq 16f
	253
d98b90ea	254	15: mov r3, lr, lspull #\pull
75494230 NP	255	ldr1w r1, lr, abort=21f
75494230 NP	256	subs ip, ip, #4
d98b90ea	257	orr r3, r3, lr, lspush #\push
75494230 NP	258	str1w r0, r3, abort=21f
	259	bgt 15b
	260	CALGN( cmp r2, #0 )
	261	CALGN( bge 11b )
	262
	263	16: sub r1, r1, #(\push / 8)
	264	b 8b
279f487e	265	UNWIND( .fnend )
75494230 NP	266
	267	.endm
	268
	269
	270	forward_copy_shift pull=8 push=24
	271
	272	17: forward_copy_shift pull=16 push=16
	273
	274	18: forward_copy_shift pull=24 push=8
	275
	276
	277	/*
7f927fcc	278	* Abort preamble and completion macros.
75494230 NP	279	* If a fixup handler is required then those macros must surround it.
	280	* It is assumed that the fixup code will handle the private part of
	281	* the exit macro.
	282	*/
	283
	284	.macro copy_abort_preamble
	285	19: ldmfd sp!, {r5 - r9}
	286	b 21f
	287	20: ldmfd sp!, {r5 - r8}
	288	21:
	289	.endm
	290
	291	.macro copy_abort_end
	292	ldmfd sp!, {r4, pc}
	293	.endm
	294