[mirror_ubuntu-artful-kernel.git] / arch / sparc / lib / atomic_64.S

/* atomic.S: These things are too big to do inline.
 *
 * Copyright (C) 1999, 2007 2012 David S. Miller (davem@davemloft.net)
 */

#include <linux/linkage.h>
#include <asm/asi.h>
#include <asm/backoff.h>
#include <asm/export.h>

	.text

	/* Three versions of the atomic routines, one that
	 * does not return a value and does not perform
	 * memory barriers, and a two which return
	 * a value, the new and old value resp. and does the
	 * barriers.
	 */

#define ATOMIC_OP(op)							\
ENTRY(atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */		\
	BACKOFF_SETUP(%o2);						\
1:	lduw	[%o1], %g1;						\
	op	%g1, %o0, %g7;						\
	cas	[%o1], %g1, %g7;					\
	cmp	%g1, %g7;						\
	bne,pn	%icc, BACKOFF_LABEL(2f, 1b);				\
	 nop;								\
	retl;								\
	 nop;								\
2:	BACKOFF_SPIN(%o2, %o3, 1b);					\
ENDPROC(atomic_##op);							\
EXPORT_SYMBOL(atomic_##op);

#define ATOMIC_OP_RETURN(op)						\
ENTRY(atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */	\
	BACKOFF_SETUP(%o2);						\
1:	lduw	[%o1], %g1;						\
	op	%g1, %o0, %g7;						\
	cas	[%o1], %g1, %g7;					\
	cmp	%g1, %g7;						\
	bne,pn	%icc, BACKOFF_LABEL(2f, 1b);				\
	 op	%g1, %o0, %g1;						\
	retl;								\
	 sra	%g1, 0, %o0;						\
2:	BACKOFF_SPIN(%o2, %o3, 1b);					\
ENDPROC(atomic_##op##_return);						\
EXPORT_SYMBOL(atomic_##op##_return);

#define ATOMIC_FETCH_OP(op)						\
ENTRY(atomic_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */	\
	BACKOFF_SETUP(%o2);						\
1:	lduw	[%o1], %g1;						\
	op	%g1, %o0, %g7;						\
	cas	[%o1], %g1, %g7;					\
	cmp	%g1, %g7;						\
	bne,pn	%icc, BACKOFF_LABEL(2f, 1b);				\
	 nop;								\
	retl;								\
	 sra	%g1, 0, %o0;						\
2:	BACKOFF_SPIN(%o2, %o3, 1b);					\
ENDPROC(atomic_fetch_##op);						\
EXPORT_SYMBOL(atomic_fetch_##op);

ATOMIC_OP(add)
ATOMIC_OP_RETURN(add)
ATOMIC_FETCH_OP(add)

ATOMIC_OP(sub)
ATOMIC_OP_RETURN(sub)
ATOMIC_FETCH_OP(sub)

ATOMIC_OP(and)
ATOMIC_FETCH_OP(and)

ATOMIC_OP(or)
ATOMIC_FETCH_OP(or)

ATOMIC_OP(xor)
ATOMIC_FETCH_OP(xor)

#undef ATOMIC_FETCH_OP
#undef ATOMIC_OP_RETURN
#undef ATOMIC_OP

#define ATOMIC64_OP(op)							\
ENTRY(atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */		\
	BACKOFF_SETUP(%o2);						\
1:	ldx	[%o1], %g1;						\
	op	%g1, %o0, %g7;						\
	casx	[%o1], %g1, %g7;					\
	cmp	%g1, %g7;						\
	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b);				\
	 nop;								\
	retl;								\
	 nop;								\
2:	BACKOFF_SPIN(%o2, %o3, 1b);					\
ENDPROC(atomic64_##op);							\
EXPORT_SYMBOL(atomic64_##op);

#define ATOMIC64_OP_RETURN(op)						\
ENTRY(atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */	\
	BACKOFF_SETUP(%o2);						\
1:	ldx	[%o1], %g1;						\
	op	%g1, %o0, %g7;						\
	casx	[%o1], %g1, %g7;					\
	cmp	%g1, %g7;						\
	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b);				\
	 nop;								\
	retl;								\
	 op	%g1, %o0, %o0;						\
2:	BACKOFF_SPIN(%o2, %o3, 1b);					\
ENDPROC(atomic64_##op##_return);					\
EXPORT_SYMBOL(atomic64_##op##_return);

#define ATOMIC64_FETCH_OP(op)						\
ENTRY(atomic64_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */	\
	BACKOFF_SETUP(%o2);						\
1:	ldx	[%o1], %g1;						\
	op	%g1, %o0, %g7;						\
	casx	[%o1], %g1, %g7;					\
	cmp	%g1, %g7;						\
	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b);				\
	 nop;								\
	retl;								\
	 mov	%g1, %o0;						\
2:	BACKOFF_SPIN(%o2, %o3, 1b);					\
ENDPROC(atomic64_fetch_##op);						\
EXPORT_SYMBOL(atomic64_fetch_##op);

ATOMIC64_OP(add)
ATOMIC64_OP_RETURN(add)
ATOMIC64_FETCH_OP(add)

ATOMIC64_OP(sub)
ATOMIC64_OP_RETURN(sub)
ATOMIC64_FETCH_OP(sub)

ATOMIC64_OP(and)
ATOMIC64_FETCH_OP(and)

ATOMIC64_OP(or)
ATOMIC64_FETCH_OP(or)

ATOMIC64_OP(xor)
ATOMIC64_FETCH_OP(xor)

#undef ATOMIC64_FETCH_OP
#undef ATOMIC64_OP_RETURN
#undef ATOMIC64_OP

ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */
	BACKOFF_SETUP(%o2)
1:	ldx	[%o0], %g1
	brlez,pn %g1, 3f
	 sub	%g1, 1, %g7
	casx	[%o0], %g1, %g7
	cmp	%g1, %g7
	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b)
	 nop
3:	retl
	 sub	%g1, 1, %o0
2:	BACKOFF_SPIN(%o2, %o3, 1b)
ENDPROC(atomic64_dec_if_positive)
EXPORT_SYMBOL(atomic64_dec_if_positive)
Commit	Line	Data
24f287e4	1	/* atomic.S: These things are too big to do inline.
1da177e4	2	*
193d2aad	3	* Copyright (C) 1999, 2007 2012 David S. Miller (davem@davemloft.net)
1da177e4 LT	4	*/
1da177e4 LT	5
8695c37d	6	#include <linux/linkage.h>
1da177e4	7	#include <asm/asi.h>
24f287e4	8	#include <asm/backoff.h>
d3867f04	9	#include <asm/export.h>
1da177e4	10
1da177e4 LT	11	.text
1da177e4 LT	12
3a1adb23	13	/* Three versions of the atomic routines, one that
1da177e4	14	* does not return a value and does not perform
3a1adb23 PZ	15	* memory barriers, and a two which return
	16	* a value, the new and old value resp. and does the
	17	* barriers.
1da177e4	18	*/
1da177e4	19
4f3316c2 PZ	20	#define ATOMIC_OP(op) \
	21	ENTRY(atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */ \
	22	BACKOFF_SETUP(%o2); \
	23	1: lduw [%o1], %g1; \
	24	op %g1, %o0, %g7; \
	25	cas [%o1], %g1, %g7; \
	26	cmp %g1, %g7; \
	27	bne,pn %icc, BACKOFF_LABEL(2f, 1b); \
	28	nop; \
	29	retl; \
	30	nop; \
	31	2: BACKOFF_SPIN(%o2, %o3, 1b); \
	32	ENDPROC(atomic_##op); \
d3867f04	33	EXPORT_SYMBOL(atomic_##op);
1da177e4	34
4f3316c2 PZ	35	#define ATOMIC_OP_RETURN(op) \
	36	ENTRY(atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \
	37	BACKOFF_SETUP(%o2); \
	38	1: lduw [%o1], %g1; \
	39	op %g1, %o0, %g7; \
	40	cas [%o1], %g1, %g7; \
	41	cmp %g1, %g7; \
	42	bne,pn %icc, BACKOFF_LABEL(2f, 1b); \
caa17d49	43	op %g1, %o0, %g1; \
4f3316c2 PZ	44	retl; \
	45	sra %g1, 0, %o0; \
	46	2: BACKOFF_SPIN(%o2, %o3, 1b); \
d3867f04 AV	47	ENDPROC(atomic_##op##_return); \
d3867f04 AV	48	EXPORT_SYMBOL(atomic_##op##_return);
1da177e4	49
3a1adb23 PZ	50	#define ATOMIC_FETCH_OP(op) \
	51	ENTRY(atomic_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \
	52	BACKOFF_SETUP(%o2); \
	53	1: lduw [%o1], %g1; \
	54	op %g1, %o0, %g7; \
	55	cas [%o1], %g1, %g7; \
	56	cmp %g1, %g7; \
	57	bne,pn %icc, BACKOFF_LABEL(2f, 1b); \
	58	nop; \
	59	retl; \
	60	sra %g1, 0, %o0; \
	61	2: BACKOFF_SPIN(%o2, %o3, 1b); \
d3867f04 AV	62	ENDPROC(atomic_fetch_##op); \
d3867f04 AV	63	EXPORT_SYMBOL(atomic_fetch_##op);
3a1adb23	64
d16c0649 NM	65	ATOMIC_OP(add)
	66	ATOMIC_OP_RETURN(add)
	67	ATOMIC_FETCH_OP(add)
1da177e4	68
d16c0649 NM	69	ATOMIC_OP(sub)
	70	ATOMIC_OP_RETURN(sub)
	71	ATOMIC_FETCH_OP(sub)
1da177e4	72
d16c0649 NM	73	ATOMIC_OP(and)
d16c0649 NM	74	ATOMIC_FETCH_OP(and)
3a1adb23	75
d16c0649 NM	76	ATOMIC_OP(or)
	77	ATOMIC_FETCH_OP(or)
	78
	79	ATOMIC_OP(xor)
	80	ATOMIC_FETCH_OP(xor)
3a1adb23	81
3a1adb23	82	#undef ATOMIC_FETCH_OP
4f3316c2 PZ	83	#undef ATOMIC_OP_RETURN
4f3316c2 PZ	84	#undef ATOMIC_OP
1da177e4	85
4f3316c2 PZ	86	#define ATOMIC64_OP(op) \
	87	ENTRY(atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */ \
	88	BACKOFF_SETUP(%o2); \
	89	1: ldx [%o1], %g1; \
	90	op %g1, %o0, %g7; \
	91	casx [%o1], %g1, %g7; \
	92	cmp %g1, %g7; \
	93	bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \
	94	nop; \
	95	retl; \
	96	nop; \
	97	2: BACKOFF_SPIN(%o2, %o3, 1b); \
	98	ENDPROC(atomic64_##op); \
d3867f04	99	EXPORT_SYMBOL(atomic64_##op);
1da177e4	100
4f3316c2 PZ	101	#define ATOMIC64_OP_RETURN(op) \
	102	ENTRY(atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \
	103	BACKOFF_SETUP(%o2); \
	104	1: ldx [%o1], %g1; \
	105	op %g1, %o0, %g7; \
	106	casx [%o1], %g1, %g7; \
	107	cmp %g1, %g7; \
	108	bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \
	109	nop; \
	110	retl; \
caa17d49	111	op %g1, %o0, %o0; \
4f3316c2	112	2: BACKOFF_SPIN(%o2, %o3, 1b); \
d3867f04 AV	113	ENDPROC(atomic64_##op##_return); \
d3867f04 AV	114	EXPORT_SYMBOL(atomic64_##op##_return);
4f3316c2	115
3a1adb23 PZ	116	#define ATOMIC64_FETCH_OP(op) \
	117	ENTRY(atomic64_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \
	118	BACKOFF_SETUP(%o2); \
	119	1: ldx [%o1], %g1; \
	120	op %g1, %o0, %g7; \
	121	casx [%o1], %g1, %g7; \
	122	cmp %g1, %g7; \
	123	bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \
	124	nop; \
	125	retl; \
	126	mov %g1, %o0; \
	127	2: BACKOFF_SPIN(%o2, %o3, 1b); \
d3867f04 AV	128	ENDPROC(atomic64_fetch_##op); \
d3867f04 AV	129	EXPORT_SYMBOL(atomic64_fetch_##op);
3a1adb23	130
d16c0649 NM	131	ATOMIC64_OP(add)
	132	ATOMIC64_OP_RETURN(add)
	133	ATOMIC64_FETCH_OP(add)
	134
	135	ATOMIC64_OP(sub)
	136	ATOMIC64_OP_RETURN(sub)
	137	ATOMIC64_FETCH_OP(sub)
4f3316c2	138
d16c0649 NM	139	ATOMIC64_OP(and)
d16c0649 NM	140	ATOMIC64_FETCH_OP(and)
4f3316c2	141
d16c0649 NM	142	ATOMIC64_OP(or)
d16c0649 NM	143	ATOMIC64_FETCH_OP(or)
3a1adb23	144
d16c0649 NM	145	ATOMIC64_OP(xor)
d16c0649 NM	146	ATOMIC64_FETCH_OP(xor)
3a1adb23	147
3a1adb23	148	#undef ATOMIC64_FETCH_OP
4f3316c2 PZ	149	#undef ATOMIC64_OP_RETURN
4f3316c2 PZ	150	#undef ATOMIC64_OP
193d2aad DM	151
	152	ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */
	153	BACKOFF_SETUP(%o2)
	154	1: ldx [%o0], %g1
	155	brlez,pn %g1, 3f
	156	sub %g1, 1, %g7
	157	casx [%o0], %g1, %g7
	158	cmp %g1, %g7
	159	bne,pn %xcc, BACKOFF_LABEL(2f, 1b)
	160	nop
	161	3: retl
	162	sub %g1, 1, %o0
	163	2: BACKOFF_SPIN(%o2, %o3, 1b)
	164	ENDPROC(atomic64_dec_if_positive)
d3867f04	165	EXPORT_SYMBOL(atomic64_dec_if_positive)