[mirror_ubuntu-eoan-kernel.git] / arch / alpha / lib / ev6-copy_user.S

/*
 * arch/alpha/lib/ev6-copy_user.S
 *
 * 21264 version contributed by Rick Gorton <rick.gorton@alpha-processor.com>
 *
 * Copy to/from user space, handling exceptions as we go..  This
 * isn't exactly pretty.
 *
 * This is essentially the same as "memcpy()", but with a few twists.
 * Notably, we have to make sure that $0 is always up-to-date and
 * contains the right "bytes left to copy" value (and that it is updated
 * only _after_ a successful copy). There is also some rather minor
 * exception setup stuff..
 *
 * Much of the information about 21264 scheduling/coding comes from:
 *	Compiler Writer's Guide for the Alpha 21264
 *	abbreviated as 'CWG' in other comments here
 *	ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
 * Scheduling notation:
 *	E	- either cluster
 *	U	- upper subcluster; U0 - subcluster U0; U1 - subcluster U1
 *	L	- lower subcluster; L0 - subcluster L0; L1 - subcluster L1
 */

#include <asm/export.h>
/* Allow an exception for an insn; exit if we get one.  */
#define EXI(x,y...)			\
	99: x,##y;			\
	.section __ex_table,"a";	\
	.long 99b - .;			\
	lda $31, $exitin-99b($31);	\
	.previous

#define EXO(x,y...)			\
	99: x,##y;			\
	.section __ex_table,"a";	\
	.long 99b - .;			\
	lda $31, $exitout-99b($31);	\
	.previous

	.set noat
	.align 4
	.globl __copy_user
	.ent __copy_user
				# Pipeline info: Slotting & Comments
__copy_user:
	.prologue 0
	mov $18, $0		# .. .. .. E
	subq $18, 32, $1	# .. .. E. ..	: Is this going to be a small copy?
	nop			# .. E  .. ..
	beq $18, $zerolength	# U  .. .. ..	: U L U L

	and $16,7,$3		# .. .. .. E	: is leading dest misalignment
	ble $1, $onebyteloop	# .. .. U  ..	: 1st branch : small amount of data
	beq $3, $destaligned	# .. U  .. ..	: 2nd (one cycle fetcher stall)
	subq $3, 8, $3		# E  .. .. ..	: L U U L : trip counter
/*
 * The fetcher stall also hides the 1 cycle cross-cluster stall for $3 (L --> U)
 * This loop aligns the destination a byte at a time
 * We know we have at least one trip through this loop
 */
$aligndest:
	EXI( ldbu $1,0($17) )	# .. .. .. L	: Keep loads separate from stores
	addq $16,1,$16		# .. .. E  ..	: Section 3.8 in the CWG
	addq $3,1,$3		# .. E  .. ..	:
	nop			# E  .. .. ..	: U L U L

/*
 * the -1 is to compensate for the inc($16) done in a previous quadpack
 * which allows us zero dependencies within either quadpack in the loop
 */
	EXO( stb $1,-1($16) )	# .. .. .. L	:
	addq $17,1,$17		# .. .. E  ..	: Section 3.8 in the CWG
	subq $0,1,$0		# .. E  .. ..	:
	bne $3, $aligndest	# U  .. .. ..	: U L U L

/*
 * If we fell through into here, we have a minimum of 33 - 7 bytes
 * If we arrived via branch, we have a minimum of 32 bytes
 */
$destaligned:
	and $17,7,$1		# .. .. .. E	: Check _current_ source alignment
	bic $0,7,$4		# .. .. E  ..	: number bytes as a quadword loop
	EXI( ldq_u $3,0($17) )	# .. L  .. ..	: Forward fetch for fallthrough code
	beq $1,$quadaligned	# U  .. .. ..	: U L U L

/*
 * In the worst case, we've just executed an ldq_u here from 0($17)
 * and we'll repeat it once if we take the branch
 */

/* Misaligned quadword loop - not unrolled.  Leave it that way. */
$misquad:
	EXI( ldq_u $2,8($17) )	# .. .. .. L	:
	subq $4,8,$4		# .. .. E  ..	:
	extql $3,$17,$3		# .. U  .. ..	:
	extqh $2,$17,$1		# U  .. .. ..	: U U L L

	bis $3,$1,$1		# .. .. .. E	:
	EXO( stq $1,0($16) )	# .. .. L  ..	:
	addq $17,8,$17		# .. E  .. ..	:
	subq $0,8,$0		# E  .. .. ..	: U L L U

	addq $16,8,$16		# .. .. .. E	:
	bis $2,$2,$3		# .. .. E  ..	:
	nop			# .. E  .. ..	:
	bne $4,$misquad		# U  .. .. ..	: U L U L

	nop			# .. .. .. E
	nop			# .. .. E  ..
	nop			# .. E  .. ..
	beq $0,$zerolength	# U  .. .. ..	: U L U L

/* We know we have at least one trip through the byte loop */
	EXI ( ldbu $2,0($17) )	# .. .. .. L	: No loads in the same quad
	addq $16,1,$16		# .. .. E  ..	: as the store (Section 3.8 in CWG)
	nop			# .. E  .. ..	:
	br $31, $dirtyentry	# L0 .. .. ..	: L U U L
/* Do the trailing byte loop load, then hop into the store part of the loop */

/*
 * A minimum of (33 - 7) bytes to do a quad at a time.
 * Based upon the usage context, it's worth the effort to unroll this loop
 * $0 - number of bytes to be moved
 * $4 - number of bytes to move as quadwords
 * $16 is current destination address
 * $17 is current source address
 */
$quadaligned:
	subq	$4, 32, $2	# .. .. .. E	: do not unroll for small stuff
	nop			# .. .. E  ..
	nop			# .. E  .. ..
	blt	$2, $onequad	# U  .. .. ..	: U L U L

/*
 * There is a significant assumption here that the source and destination
 * addresses differ by more than 32 bytes.  In this particular case, a
 * sparsity of registers further bounds this to be a minimum of 8 bytes.
 * But if this isn't met, then the output result will be incorrect.
 * Furthermore, due to a lack of available registers, we really can't
 * unroll this to be an 8x loop (which would enable us to use the wh64
 * instruction memory hint instruction).
 */
$unroll4:
	EXI( ldq $1,0($17) )	# .. .. .. L
	EXI( ldq $2,8($17) )	# .. .. L  ..
	subq	$4,32,$4	# .. E  .. ..
	nop			# E  .. .. ..	: U U L L

	addq	$17,16,$17	# .. .. .. E
	EXO( stq $1,0($16) )	# .. .. L  ..
	EXO( stq $2,8($16) )	# .. L  .. ..
	subq	$0,16,$0	# E  .. .. ..	: U L L U

	addq	$16,16,$16	# .. .. .. E
	EXI( ldq $1,0($17) )	# .. .. L  ..
	EXI( ldq $2,8($17) )	# .. L  .. ..
	subq	$4, 32, $3	# E  .. .. ..	: U U L L : is there enough for another trip?

	EXO( stq $1,0($16) )	# .. .. .. L
	EXO( stq $2,8($16) )	# .. .. L  ..
	subq	$0,16,$0	# .. E  .. ..
	addq	$17,16,$17	# E  .. .. ..	: U L L U

	nop			# .. .. .. E
	nop			# .. .. E  ..
	addq	$16,16,$16	# .. E  .. ..
	bgt	$3,$unroll4	# U  .. .. ..	: U L U L

	nop
	nop
	nop
	beq	$4, $noquads

$onequad:
	EXI( ldq $1,0($17) )
	subq	$4,8,$4
	addq	$17,8,$17
	nop

	EXO( stq $1,0($16) )
	subq	$0,8,$0
	addq	$16,8,$16
	bne	$4,$onequad

$noquads:
	nop
	nop
	nop
	beq $0,$zerolength

/*
 * For small copies (or the tail of a larger copy), do a very simple byte loop.
 * There's no point in doing a lot of complex alignment calculations to try to
 * to quadword stuff for a small amount of data.
 *	$0 - remaining number of bytes left to copy
 *	$16 - current dest addr
 *	$17 - current source addr
 */

$onebyteloop:
	EXI ( ldbu $2,0($17) )	# .. .. .. L	: No loads in the same quad
	addq $16,1,$16		# .. .. E  ..	: as the store (Section 3.8 in CWG)
	nop			# .. E  .. ..	:
	nop			# E  .. .. ..	: U L U L

$dirtyentry:
/*
 * the -1 is to compensate for the inc($16) done in a previous quadpack
 * which allows us zero dependencies within either quadpack in the loop
 */
	EXO ( stb $2,-1($16) )	# .. .. .. L	:
	addq $17,1,$17		# .. .. E  ..	: quadpack as the load
	subq $0,1,$0		# .. E  .. ..	: change count _after_ copy
	bgt $0,$onebyteloop	# U  .. .. ..	: U L U L

$zerolength:
$exitin:
$exitout:			# Destination for exception recovery(?)
	nop			# .. .. .. E
	nop			# .. .. E  ..
	nop			# .. E  .. ..
	ret $31,($26),1		# L0 .. .. ..	: L U L U

	.end __copy_user
	EXPORT_SYMBOL(__copy_user)
Commit	Line	Data
1da177e4 LT	1	/*
	2	* arch/alpha/lib/ev6-copy_user.S
	3	*
	4	* 21264 version contributed by Rick Gorton <rick.gorton@alpha-processor.com>
	5	*
	6	* Copy to/from user space, handling exceptions as we go.. This
	7	* isn't exactly pretty.
	8	*
	9	* This is essentially the same as "memcpy()", but with a few twists.
	10	* Notably, we have to make sure that $0 is always up-to-date and
	11	* contains the right "bytes left to copy" value (and that it is updated
	12	* only _after_ a successful copy). There is also some rather minor
	13	* exception setup stuff..
	14	*
1da177e4 LT	15	* Much of the information about 21264 scheduling/coding comes from:
	16	* Compiler Writer's Guide for the Alpha 21264
	17	* abbreviated as 'CWG' in other comments here
	18	* ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
	19	* Scheduling notation:
	20	* E - either cluster
	21	* U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1
	22	* L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1
	23	*/
	24
00fc0e0d	25	#include <asm/export.h>
1da177e4 LT	26	/* Allow an exception for an insn; exit if we get one. */
	27	#define EXI(x,y...) \
	28	99: x,##y; \
	29	.section __ex_table,"a"; \
	30	.long 99b - .; \
	31	lda $31, $exitin-99b($31); \
	32	.previous
	33
	34	#define EXO(x,y...) \
	35	99: x,##y; \
	36	.section __ex_table,"a"; \
	37	.long 99b - .; \
	38	lda $31, $exitout-99b($31); \
	39	.previous
	40
	41	.set noat
	42	.align 4
	43	.globl __copy_user
	44	.ent __copy_user
	45	# Pipeline info: Slotting & Comments
	46	__copy_user:
	47	.prologue 0
4606f68f RH	48	mov $18, $0 # .. .. .. E
	49	subq $18, 32, $1 # .. .. E. .. : Is this going to be a small copy?
	50	nop # .. E .. ..
	51	beq $18, $zerolength # U .. .. .. : U L U L
1da177e4	52
85250231	53	and $16,7,$3 # .. .. .. E : is leading dest misalignment
1da177e4 LT	54	ble $1, $onebyteloop # .. .. U .. : 1st branch : small amount of data
	55	beq $3, $destaligned # .. U .. .. : 2nd (one cycle fetcher stall)
	56	subq $3, 8, $3 # E .. .. .. : L U U L : trip counter
	57	/*
	58	* The fetcher stall also hides the 1 cycle cross-cluster stall for $3 (L --> U)
	59	* This loop aligns the destination a byte at a time
	60	* We know we have at least one trip through this loop
	61	*/
	62	$aligndest:
85250231 AV	63	EXI( ldbu $1,0($17) ) # .. .. .. L : Keep loads separate from stores
85250231 AV	64	addq $16,1,$16 # .. .. E .. : Section 3.8 in the CWG
1da177e4 LT	65	addq $3,1,$3 # .. E .. .. :
	66	nop # E .. .. .. : U L U L
	67
	68	/*
85250231	69	* the -1 is to compensate for the inc($16) done in a previous quadpack
1da177e4 LT	70	* which allows us zero dependencies within either quadpack in the loop
1da177e4 LT	71	*/
85250231 AV	72	EXO( stb $1,-1($16) ) # .. .. .. L :
85250231 AV	73	addq $17,1,$17 # .. .. E .. : Section 3.8 in the CWG
1da177e4 LT	74	subq $0,1,$0 # .. E .. .. :
	75	bne $3, $aligndest # U .. .. .. : U L U L
	76
	77	/*
	78	* If we fell through into here, we have a minimum of 33 - 7 bytes
	79	* If we arrived via branch, we have a minimum of 32 bytes
	80	*/
	81	$destaligned:
85250231	82	and $17,7,$1 # .. .. .. E : Check _current_ source alignment
1da177e4	83	bic $0,7,$4 # .. .. E .. : number bytes as a quadword loop
85250231	84	EXI( ldq_u $3,0($17) ) # .. L .. .. : Forward fetch for fallthrough code
1da177e4 LT	85	beq $1,$quadaligned # U .. .. .. : U L U L
	86
	87	/*
85250231	88	* In the worst case, we've just executed an ldq_u here from 0($17)
1da177e4 LT	89	* and we'll repeat it once if we take the branch
	90	*/
	91
	92	/* Misaligned quadword loop - not unrolled. Leave it that way. */
	93	$misquad:
85250231	94	EXI( ldq_u $2,8($17) ) # .. .. .. L :
1da177e4	95	subq $4,8,$4 # .. .. E .. :
85250231 AV	96	extql $3,$17,$3 # .. U .. .. :
85250231 AV	97	extqh $2,$17,$1 # U .. .. .. : U U L L
1da177e4 LT	98
1da177e4 LT	99	bis $3,$1,$1 # .. .. .. E :
85250231 AV	100	EXO( stq $1,0($16) ) # .. .. L .. :
85250231 AV	101	addq $17,8,$17 # .. E .. .. :
1da177e4 LT	102	subq $0,8,$0 # E .. .. .. : U L L U
1da177e4 LT	103
85250231	104	addq $16,8,$16 # .. .. .. E :
1da177e4 LT	105	bis $2,$2,$3 # .. .. E .. :
	106	nop # .. E .. .. :
	107	bne $4,$misquad # U .. .. .. : U L U L
	108
	109	nop # .. .. .. E
	110	nop # .. .. E ..
	111	nop # .. E .. ..
	112	beq $0,$zerolength # U .. .. .. : U L U L
	113
	114	/* We know we have at least one trip through the byte loop */
85250231 AV	115	EXI ( ldbu $2,0($17) ) # .. .. .. L : No loads in the same quad
85250231 AV	116	addq $16,1,$16 # .. .. E .. : as the store (Section 3.8 in CWG)
1da177e4 LT	117	nop # .. E .. .. :
	118	br $31, $dirtyentry # L0 .. .. .. : L U U L
	119	/* Do the trailing byte loop load, then hop into the store part of the loop */
	120
	121	/*
	122	* A minimum of (33 - 7) bytes to do a quad at a time.
	123	* Based upon the usage context, it's worth the effort to unroll this loop
	124	* $0 - number of bytes to be moved
	125	* $4 - number of bytes to move as quadwords
85250231 AV	126	* $16 is current destination address
85250231 AV	127	* $17 is current source address
1da177e4 LT	128	*/
	129	$quadaligned:
	130	subq $4, 32, $2 # .. .. .. E : do not unroll for small stuff
	131	nop # .. .. E ..
	132	nop # .. E .. ..
	133	blt $2, $onequad # U .. .. .. : U L U L
	134
	135	/*
	136	* There is a significant assumption here that the source and destination
	137	* addresses differ by more than 32 bytes. In this particular case, a
	138	* sparsity of registers further bounds this to be a minimum of 8 bytes.
	139	* But if this isn't met, then the output result will be incorrect.
	140	* Furthermore, due to a lack of available registers, we really can't
	141	* unroll this to be an 8x loop (which would enable us to use the wh64
	142	* instruction memory hint instruction).
	143	*/
	144	$unroll4:
85250231 AV	145	EXI( ldq $1,0($17) ) # .. .. .. L
85250231 AV	146	EXI( ldq $2,8($17) ) # .. .. L ..
1da177e4 LT	147	subq $4,32,$4 # .. E .. ..
	148	nop # E .. .. .. : U U L L
	149
85250231 AV	150	addq $17,16,$17 # .. .. .. E
	151	EXO( stq $1,0($16) ) # .. .. L ..
	152	EXO( stq $2,8($16) ) # .. L .. ..
1da177e4 LT	153	subq $0,16,$0 # E .. .. .. : U L L U
1da177e4 LT	154
85250231 AV	155	addq $16,16,$16 # .. .. .. E
	156	EXI( ldq $1,0($17) ) # .. .. L ..
	157	EXI( ldq $2,8($17) ) # .. L .. ..
1da177e4 LT	158	subq $4, 32, $3 # E .. .. .. : U U L L : is there enough for another trip?
1da177e4 LT	159
85250231 AV	160	EXO( stq $1,0($16) ) # .. .. .. L
85250231 AV	161	EXO( stq $2,8($16) ) # .. .. L ..
1da177e4	162	subq $0,16,$0 # .. E .. ..
85250231	163	addq $17,16,$17 # E .. .. .. : U L L U
1da177e4 LT	164
	165	nop # .. .. .. E
	166	nop # .. .. E ..
85250231	167	addq $16,16,$16 # .. E .. ..
1da177e4 LT	168	bgt $3,$unroll4 # U .. .. .. : U L U L
	169
	170	nop
	171	nop
	172	nop
	173	beq $4, $noquads
	174
	175	$onequad:
85250231	176	EXI( ldq $1,0($17) )
1da177e4	177	subq $4,8,$4
85250231	178	addq $17,8,$17
1da177e4 LT	179	nop
1da177e4 LT	180
85250231	181	EXO( stq $1,0($16) )
1da177e4	182	subq $0,8,$0
85250231	183	addq $16,8,$16
1da177e4 LT	184	bne $4,$onequad
	185
	186	$noquads:
	187	nop
	188	nop
	189	nop
	190	beq $0,$zerolength
	191
	192	/*
	193	* For small copies (or the tail of a larger copy), do a very simple byte loop.
	194	* There's no point in doing a lot of complex alignment calculations to try to
	195	* to quadword stuff for a small amount of data.
	196	* $0 - remaining number of bytes left to copy
85250231 AV	197	* $16 - current dest addr
85250231 AV	198	* $17 - current source addr
1da177e4 LT	199	*/
	200
	201	$onebyteloop:
85250231 AV	202	EXI ( ldbu $2,0($17) ) # .. .. .. L : No loads in the same quad
85250231 AV	203	addq $16,1,$16 # .. .. E .. : as the store (Section 3.8 in CWG)
1da177e4 LT	204	nop # .. E .. .. :
	205	nop # E .. .. .. : U L U L
	206
	207	$dirtyentry:
	208	/*
85250231	209	* the -1 is to compensate for the inc($16) done in a previous quadpack
1da177e4 LT	210	* which allows us zero dependencies within either quadpack in the loop
1da177e4 LT	211	*/
85250231 AV	212	EXO ( stb $2,-1($16) ) # .. .. .. L :
85250231 AV	213	addq $17,1,$17 # .. .. E .. : quadpack as the load
1da177e4 LT	214	subq $0,1,$0 # .. E .. .. : change count _after_ copy
	215	bgt $0,$onebyteloop # U .. .. .. : U L U L
	216
	217	$zerolength:
085354f9	218	$exitin:
1da177e4 LT	219	$exitout: # Destination for exception recovery(?)
	220	nop # .. .. .. E
	221	nop # .. .. E ..
	222	nop # .. E .. ..
85250231	223	ret $31,($26),1 # L0 .. .. .. : L U L U
1da177e4	224
1da177e4	225	.end __copy_user
00fc0e0d	226	EXPORT_SYMBOL(__copy_user)