[mirror_ubuntu-artful-kernel.git] / arch / arm64 / include / asm / atomic_lse.h

/*
 * Based on arch/arm/include/asm/atomic.h
 *
 * Copyright (C) 1996 Russell King.
 * Copyright (C) 2002 Deep Blue Solutions Ltd.
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef __ASM_ATOMIC_LSE_H
#define __ASM_ATOMIC_LSE_H

#ifndef __ARM64_IN_ATOMIC_IMPL
#error "please don't include this file directly"
#endif

#define __LL_SC_ATOMIC(op)	__LL_SC_CALL(atomic_##op)

static inline void atomic_andnot(int i, atomic_t *v)
{
	register int w0 asm ("w0") = i;
	register atomic_t *x1 asm ("x1") = v;

	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(andnot),
	"	stclr	%w[i], %[v]\n")
	: [i] "+r" (w0), [v] "+Q" (v->counter)
	: "r" (x1)
	: "x30");
}

static inline void atomic_or(int i, atomic_t *v)
{
	register int w0 asm ("w0") = i;
	register atomic_t *x1 asm ("x1") = v;

	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(or),
	"	stset	%w[i], %[v]\n")
	: [i] "+r" (w0), [v] "+Q" (v->counter)
	: "r" (x1)
	: "x30");
}

static inline void atomic_xor(int i, atomic_t *v)
{
	register int w0 asm ("w0") = i;
	register atomic_t *x1 asm ("x1") = v;

	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(xor),
	"	steor	%w[i], %[v]\n")
	: [i] "+r" (w0), [v] "+Q" (v->counter)
	: "r" (x1)
	: "x30");
}

static inline void atomic_add(int i, atomic_t *v)
{
	register int w0 asm ("w0") = i;
	register atomic_t *x1 asm ("x1") = v;

	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(add),
	"	stadd	%w[i], %[v]\n")
	: [i] "+r" (w0), [v] "+Q" (v->counter)
	: "r" (x1)
	: "x30");
}

#define ATOMIC_OP_ADD_RETURN(name, mb, cl...)				\
static inline int atomic_add_return##name(int i, atomic_t *v)		\
{									\
	register int w0 asm ("w0") = i;					\
	register atomic_t *x1 asm ("x1") = v;				\
									\
	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
	/* LL/SC */							\
	"	nop\n"							\
	__LL_SC_ATOMIC(add_return##name),				\
	/* LSE atomics */						\
	"	ldadd" #mb "	%w[i], w30, %[v]\n"			\
	"	add	%w[i], %w[i], w30")				\
	: [i] "+r" (w0), [v] "+Q" (v->counter)				\
	: "r" (x1)							\
	: "x30" , ##cl);						\
									\
	return w0;							\
}

ATOMIC_OP_ADD_RETURN(_relaxed,   )
ATOMIC_OP_ADD_RETURN(_acquire,  a, "memory")
ATOMIC_OP_ADD_RETURN(_release,  l, "memory")
ATOMIC_OP_ADD_RETURN(        , al, "memory")

#undef ATOMIC_OP_ADD_RETURN

static inline void atomic_and(int i, atomic_t *v)
{
	register int w0 asm ("w0") = i;
	register atomic_t *x1 asm ("x1") = v;

	asm volatile(ARM64_LSE_ATOMIC_INSN(
	/* LL/SC */
	"	nop\n"
	__LL_SC_ATOMIC(and),
	/* LSE atomics */
	"	mvn	%w[i], %w[i]\n"
	"	stclr	%w[i], %[v]")
	: [i] "+r" (w0), [v] "+Q" (v->counter)
	: "r" (x1)
	: "x30");
}

static inline void atomic_sub(int i, atomic_t *v)
{
	register int w0 asm ("w0") = i;
	register atomic_t *x1 asm ("x1") = v;

	asm volatile(ARM64_LSE_ATOMIC_INSN(
	/* LL/SC */
	"	nop\n"
	__LL_SC_ATOMIC(sub),
	/* LSE atomics */
	"	neg	%w[i], %w[i]\n"
	"	stadd	%w[i], %[v]")
	: [i] "+r" (w0), [v] "+Q" (v->counter)
	: "r" (x1)
	: "x30");
}

#define ATOMIC_OP_SUB_RETURN(name, mb, cl...)				\
static inline int atomic_sub_return##name(int i, atomic_t *v)		\
{									\
	register int w0 asm ("w0") = i;					\
	register atomic_t *x1 asm ("x1") = v;				\
									\
	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
	/* LL/SC */							\
	"	nop\n"							\
	__LL_SC_ATOMIC(sub_return##name)				\
	"	nop",							\
	/* LSE atomics */						\
	"	neg	%w[i], %w[i]\n"					\
	"	ldadd" #mb "	%w[i], w30, %[v]\n"			\
	"	add	%w[i], %w[i], w30")				\
	: [i] "+r" (w0), [v] "+Q" (v->counter)				\
	: "r" (x1)							\
	: "x30" , ##cl);						\
									\
	return w0;							\
}

ATOMIC_OP_SUB_RETURN(_relaxed,   )
ATOMIC_OP_SUB_RETURN(_acquire,  a, "memory")
ATOMIC_OP_SUB_RETURN(_release,  l, "memory")
ATOMIC_OP_SUB_RETURN(        , al, "memory")

#undef ATOMIC_OP_SUB_RETURN
#undef __LL_SC_ATOMIC

#define __LL_SC_ATOMIC64(op)	__LL_SC_CALL(atomic64_##op)

static inline void atomic64_andnot(long i, atomic64_t *v)
{
	register long x0 asm ("x0") = i;
	register atomic64_t *x1 asm ("x1") = v;

	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(andnot),
	"	stclr	%[i], %[v]\n")
	: [i] "+r" (x0), [v] "+Q" (v->counter)
	: "r" (x1)
	: "x30");
}

static inline void atomic64_or(long i, atomic64_t *v)
{
	register long x0 asm ("x0") = i;
	register atomic64_t *x1 asm ("x1") = v;

	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(or),
	"	stset	%[i], %[v]\n")
	: [i] "+r" (x0), [v] "+Q" (v->counter)
	: "r" (x1)
	: "x30");
}

static inline void atomic64_xor(long i, atomic64_t *v)
{
	register long x0 asm ("x0") = i;
	register atomic64_t *x1 asm ("x1") = v;

	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(xor),
	"	steor	%[i], %[v]\n")
	: [i] "+r" (x0), [v] "+Q" (v->counter)
	: "r" (x1)
	: "x30");
}

static inline void atomic64_add(long i, atomic64_t *v)
{
	register long x0 asm ("x0") = i;
	register atomic64_t *x1 asm ("x1") = v;

	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(add),
	"	stadd	%[i], %[v]\n")
	: [i] "+r" (x0), [v] "+Q" (v->counter)
	: "r" (x1)
	: "x30");
}

#define ATOMIC64_OP_ADD_RETURN(name, mb, cl...)				\
static inline long atomic64_add_return##name(long i, atomic64_t *v)	\
{									\
	register long x0 asm ("x0") = i;				\
	register atomic64_t *x1 asm ("x1") = v;				\
									\
	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
	/* LL/SC */							\
	"	nop\n"							\
	__LL_SC_ATOMIC64(add_return##name),				\
	/* LSE atomics */						\
	"	ldadd" #mb "	%[i], x30, %[v]\n"			\
	"	add	%[i], %[i], x30")				\
	: [i] "+r" (x0), [v] "+Q" (v->counter)				\
	: "r" (x1)							\
	: "x30" , ##cl);						\
									\
	return x0;							\
}

ATOMIC64_OP_ADD_RETURN(_relaxed,   )
ATOMIC64_OP_ADD_RETURN(_acquire,  a, "memory")
ATOMIC64_OP_ADD_RETURN(_release,  l, "memory")
ATOMIC64_OP_ADD_RETURN(        , al, "memory")

#undef ATOMIC64_OP_ADD_RETURN

static inline void atomic64_and(long i, atomic64_t *v)
{
	register long x0 asm ("x0") = i;
	register atomic64_t *x1 asm ("x1") = v;

	asm volatile(ARM64_LSE_ATOMIC_INSN(
	/* LL/SC */
	"	nop\n"
	__LL_SC_ATOMIC64(and),
	/* LSE atomics */
	"	mvn	%[i], %[i]\n"
	"	stclr	%[i], %[v]")
	: [i] "+r" (x0), [v] "+Q" (v->counter)
	: "r" (x1)
	: "x30");
}

static inline void atomic64_sub(long i, atomic64_t *v)
{
	register long x0 asm ("x0") = i;
	register atomic64_t *x1 asm ("x1") = v;

	asm volatile(ARM64_LSE_ATOMIC_INSN(
	/* LL/SC */
	"	nop\n"
	__LL_SC_ATOMIC64(sub),
	/* LSE atomics */
	"	neg	%[i], %[i]\n"
	"	stadd	%[i], %[v]")
	: [i] "+r" (x0), [v] "+Q" (v->counter)
	: "r" (x1)
	: "x30");
}

#define ATOMIC64_OP_SUB_RETURN(name, mb, cl...)				\
static inline long atomic64_sub_return##name(long i, atomic64_t *v)	\
{									\
	register long x0 asm ("x0") = i;				\
	register atomic64_t *x1 asm ("x1") = v;				\
									\
	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
	/* LL/SC */							\
	"	nop\n"							\
	__LL_SC_ATOMIC64(sub_return##name)				\
	"	nop",							\
	/* LSE atomics */						\
	"	neg	%[i], %[i]\n"					\
	"	ldadd" #mb "	%[i], x30, %[v]\n"			\
	"	add	%[i], %[i], x30")				\
	: [i] "+r" (x0), [v] "+Q" (v->counter)				\
	: "r" (x1)							\
	: "x30" , ##cl);						\
									\
	return x0;							\
}

ATOMIC64_OP_SUB_RETURN(_relaxed,   )
ATOMIC64_OP_SUB_RETURN(_acquire,  a, "memory")
ATOMIC64_OP_SUB_RETURN(_release,  l, "memory")
ATOMIC64_OP_SUB_RETURN(        , al, "memory")

#undef ATOMIC64_OP_SUB_RETURN

static inline long atomic64_dec_if_positive(atomic64_t *v)
{
	register long x0 asm ("x0") = (long)v;

	asm volatile(ARM64_LSE_ATOMIC_INSN(
	/* LL/SC */
	"	nop\n"
	__LL_SC_ATOMIC64(dec_if_positive)
	"	nop\n"
	"	nop\n"
	"	nop\n"
	"	nop\n"
	"	nop",
	/* LSE atomics */
	"1:	ldr	x30, %[v]\n"
	"	subs	%[ret], x30, #1\n"
	"	b.lt	2f\n"
	"	casal	x30, %[ret], %[v]\n"
	"	sub	x30, x30, #1\n"
	"	sub	x30, x30, %[ret]\n"
	"	cbnz	x30, 1b\n"
	"2:")
	: [ret] "+&r" (x0), [v] "+Q" (v->counter)
	:
	: "x30", "cc", "memory");

	return x0;
}

#undef __LL_SC_ATOMIC64

#define __LL_SC_CMPXCHG(op)	__LL_SC_CALL(__cmpxchg_case_##op)

#define __CMPXCHG_CASE(w, sz, name, mb, cl...)				\
static inline unsigned long __cmpxchg_case_##name(volatile void *ptr,	\
						  unsigned long old,	\
						  unsigned long new)	\
{									\
	register unsigned long x0 asm ("x0") = (unsigned long)ptr;	\
	register unsigned long x1 asm ("x1") = old;			\
	register unsigned long x2 asm ("x2") = new;			\
									\
	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
	/* LL/SC */							\
	"	nop\n"							\
		__LL_SC_CMPXCHG(name)					\
	"	nop",							\
	/* LSE atomics */						\
	"	mov	" #w "30, %" #w "[old]\n"			\
	"	cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n"		\
	"	mov	%" #w "[ret], " #w "30")			\
	: [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr)		\
	: [old] "r" (x1), [new] "r" (x2)				\
	: "x30" , ##cl);						\
									\
	return x0;							\
}

__CMPXCHG_CASE(w, b,     1,   )
__CMPXCHG_CASE(w, h,     2,   )
__CMPXCHG_CASE(w,  ,     4,   )
__CMPXCHG_CASE(x,  ,     8,   )
__CMPXCHG_CASE(w, b, acq_1,  a, "memory")
__CMPXCHG_CASE(w, h, acq_2,  a, "memory")
__CMPXCHG_CASE(w,  , acq_4,  a, "memory")
__CMPXCHG_CASE(x,  , acq_8,  a, "memory")
__CMPXCHG_CASE(w, b, rel_1,  l, "memory")
__CMPXCHG_CASE(w, h, rel_2,  l, "memory")
__CMPXCHG_CASE(w,  , rel_4,  l, "memory")
__CMPXCHG_CASE(x,  , rel_8,  l, "memory")
__CMPXCHG_CASE(w, b,  mb_1, al, "memory")
__CMPXCHG_CASE(w, h,  mb_2, al, "memory")
__CMPXCHG_CASE(w,  ,  mb_4, al, "memory")
__CMPXCHG_CASE(x,  ,  mb_8, al, "memory")

#undef __LL_SC_CMPXCHG
#undef __CMPXCHG_CASE

#define __LL_SC_CMPXCHG_DBL(op)	__LL_SC_CALL(__cmpxchg_double##op)

#define __CMPXCHG_DBL(name, mb, cl...)					\
static inline int __cmpxchg_double##name(unsigned long old1,		\
					 unsigned long old2,		\
					 unsigned long new1,		\
					 unsigned long new2,		\
					 volatile void *ptr)		\
{									\
	unsigned long oldval1 = old1;					\
	unsigned long oldval2 = old2;					\
	register unsigned long x0 asm ("x0") = old1;			\
	register unsigned long x1 asm ("x1") = old2;			\
	register unsigned long x2 asm ("x2") = new1;			\
	register unsigned long x3 asm ("x3") = new2;			\
	register unsigned long x4 asm ("x4") = (unsigned long)ptr;	\
									\
	asm volatile(ARM64_LSE_ATOMIC_INSN(				\
	/* LL/SC */							\
	"	nop\n"							\
	"	nop\n"							\
	"	nop\n"							\
	__LL_SC_CMPXCHG_DBL(name),					\
	/* LSE atomics */						\
	"	casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
	"	eor	%[old1], %[old1], %[oldval1]\n"			\
	"	eor	%[old2], %[old2], %[oldval2]\n"			\
	"	orr	%[old1], %[old1], %[old2]")			\
	: [old1] "+r" (x0), [old2] "+r" (x1),				\
	  [v] "+Q" (*(unsigned long *)ptr)				\
	: [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4),		\
	  [oldval1] "r" (oldval1), [oldval2] "r" (oldval2)		\
	: "x30" , ##cl);						\
									\
	return x0;							\
}

__CMPXCHG_DBL(   ,   )
__CMPXCHG_DBL(_mb, al, "memory")

#undef __LL_SC_CMPXCHG_DBL
#undef __CMPXCHG_DBL

#endif	/* __ASM_ATOMIC_LSE_H */
Commit	Line	Data
c0385b24 WD	1	/*
	2	* Based on arch/arm/include/asm/atomic.h
	3	*
	4	* Copyright (C) 1996 Russell King.
	5	* Copyright (C) 2002 Deep Blue Solutions Ltd.
	6	* Copyright (C) 2012 ARM Ltd.
	7	*
	8	* This program is free software; you can redistribute it and/or modify
	9	* it under the terms of the GNU General Public License version 2 as
	10	* published by the Free Software Foundation.
	11	*
	12	* This program is distributed in the hope that it will be useful,
	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	* GNU General Public License for more details.
	16	*
	17	* You should have received a copy of the GNU General Public License
	18	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	19	*/
	20
	21	#ifndef __ASM_ATOMIC_LSE_H
	22	#define __ASM_ATOMIC_LSE_H
	23
	24	#ifndef __ARM64_IN_ATOMIC_IMPL
	25	#error "please don't include this file directly"
	26	#endif
	27
c09d6a04 WD	28	#define __LL_SC_ATOMIC(op) __LL_SC_CALL(atomic_##op)
	29
	30	static inline void atomic_andnot(int i, atomic_t *v)
	31	{
	32	register int w0 asm ("w0") = i;
	33	register atomic_t *x1 asm ("x1") = v;
	34
	35	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(andnot),
	36	" stclr %w[i], %[v]\n")
	37	: [i] "+r" (w0), [v] "+Q" (v->counter)
	38	: "r" (x1)
	39	: "x30");
	40	}
	41
	42	static inline void atomic_or(int i, atomic_t *v)
	43	{
	44	register int w0 asm ("w0") = i;
	45	register atomic_t *x1 asm ("x1") = v;
	46
	47	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(or),
	48	" stset %w[i], %[v]\n")
	49	: [i] "+r" (w0), [v] "+Q" (v->counter)
	50	: "r" (x1)
	51	: "x30");
	52	}
	53
	54	static inline void atomic_xor(int i, atomic_t *v)
	55	{
	56	register int w0 asm ("w0") = i;
	57	register atomic_t *x1 asm ("x1") = v;
	58
	59	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(xor),
	60	" steor %w[i], %[v]\n")
	61	: [i] "+r" (w0), [v] "+Q" (v->counter)
	62	: "r" (x1)
	63	: "x30");
	64	}
	65
	66	static inline void atomic_add(int i, atomic_t *v)
	67	{
	68	register int w0 asm ("w0") = i;
	69	register atomic_t *x1 asm ("x1") = v;
	70
	71	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(add),
	72	" stadd %w[i], %[v]\n")
	73	: [i] "+r" (w0), [v] "+Q" (v->counter)
	74	: "r" (x1)
	75	: "x30");
	76	}
	77
305d454a WD	78	#define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \
	79	static inline int atomic_add_return##name(int i, atomic_t *v) \
	80	{ \
	81	register int w0 asm ("w0") = i; \
	82	register atomic_t *x1 asm ("x1") = v; \
	83	\
	84	asm volatile(ARM64_LSE_ATOMIC_INSN( \
	85	/* LL/SC */ \
	86	" nop\n" \
	87	__LL_SC_ATOMIC(add_return##name), \
	88	/* LSE atomics */ \
	89	" ldadd" #mb " %w[i], w30, %[v]\n" \
	90	" add %w[i], %w[i], w30") \
	91	: [i] "+r" (w0), [v] "+Q" (v->counter) \
	92	: "r" (x1) \
	93	: "x30" , ##cl); \
	94	\
	95	return w0; \
	96	}
c09d6a04	97
305d454a WD	98	ATOMIC_OP_ADD_RETURN(_relaxed, )
	99	ATOMIC_OP_ADD_RETURN(_acquire, a, "memory")
	100	ATOMIC_OP_ADD_RETURN(_release, l, "memory")
	101	ATOMIC_OP_ADD_RETURN( , al, "memory")
c09d6a04	102
305d454a	103	#undef ATOMIC_OP_ADD_RETURN
c09d6a04 WD	104
	105	static inline void atomic_and(int i, atomic_t *v)
	106	{
	107	register int w0 asm ("w0") = i;
	108	register atomic_t *x1 asm ("x1") = v;
	109
	110	asm volatile(ARM64_LSE_ATOMIC_INSN(
	111	/* LL/SC */
	112	" nop\n"
	113	__LL_SC_ATOMIC(and),
	114	/* LSE atomics */
	115	" mvn %w[i], %w[i]\n"
	116	" stclr %w[i], %[v]")
	117	: [i] "+r" (w0), [v] "+Q" (v->counter)
	118	: "r" (x1)
	119	: "x30");
	120	}
	121
	122	static inline void atomic_sub(int i, atomic_t *v)
	123	{
	124	register int w0 asm ("w0") = i;
	125	register atomic_t *x1 asm ("x1") = v;
	126
	127	asm volatile(ARM64_LSE_ATOMIC_INSN(
	128	/* LL/SC */
	129	" nop\n"
	130	__LL_SC_ATOMIC(sub),
	131	/* LSE atomics */
	132	" neg %w[i], %w[i]\n"
	133	" stadd %w[i], %[v]")
	134	: [i] "+r" (w0), [v] "+Q" (v->counter)
	135	: "r" (x1)
	136	: "x30");
	137	}
	138
305d454a WD	139	#define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \
	140	static inline int atomic_sub_return##name(int i, atomic_t *v) \
	141	{ \
	142	register int w0 asm ("w0") = i; \
	143	register atomic_t *x1 asm ("x1") = v; \
	144	\
	145	asm volatile(ARM64_LSE_ATOMIC_INSN( \
	146	/* LL/SC */ \
	147	" nop\n" \
	148	__LL_SC_ATOMIC(sub_return##name) \
	149	" nop", \
	150	/* LSE atomics */ \
	151	" neg %w[i], %w[i]\n" \
	152	" ldadd" #mb " %w[i], w30, %[v]\n" \
	153	" add %w[i], %w[i], w30") \
	154	: [i] "+r" (w0), [v] "+Q" (v->counter) \
	155	: "r" (x1) \
	156	: "x30" , ##cl); \
	157	\
	158	return w0; \
c09d6a04	159	}
c0385b24	160
305d454a WD	161	ATOMIC_OP_SUB_RETURN(_relaxed, )
	162	ATOMIC_OP_SUB_RETURN(_acquire, a, "memory")
	163	ATOMIC_OP_SUB_RETURN(_release, l, "memory")
	164	ATOMIC_OP_SUB_RETURN( , al, "memory")
	165
	166	#undef ATOMIC_OP_SUB_RETURN
c09d6a04	167	#undef __LL_SC_ATOMIC
c0385b24	168
c09d6a04 WD	169	#define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op)
	170
	171	static inline void atomic64_andnot(long i, atomic64_t *v)
	172	{
	173	register long x0 asm ("x0") = i;
	174	register atomic64_t *x1 asm ("x1") = v;
	175
	176	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(andnot),
	177	" stclr %[i], %[v]\n")
	178	: [i] "+r" (x0), [v] "+Q" (v->counter)
	179	: "r" (x1)
	180	: "x30");
	181	}
	182
	183	static inline void atomic64_or(long i, atomic64_t *v)
	184	{
	185	register long x0 asm ("x0") = i;
	186	register atomic64_t *x1 asm ("x1") = v;
	187
	188	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(or),
	189	" stset %[i], %[v]\n")
	190	: [i] "+r" (x0), [v] "+Q" (v->counter)
	191	: "r" (x1)
	192	: "x30");
	193	}
	194
	195	static inline void atomic64_xor(long i, atomic64_t *v)
	196	{
	197	register long x0 asm ("x0") = i;
	198	register atomic64_t *x1 asm ("x1") = v;
	199
	200	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(xor),
	201	" steor %[i], %[v]\n")
	202	: [i] "+r" (x0), [v] "+Q" (v->counter)
	203	: "r" (x1)
	204	: "x30");
	205	}
	206
	207	static inline void atomic64_add(long i, atomic64_t *v)
	208	{
	209	register long x0 asm ("x0") = i;
	210	register atomic64_t *x1 asm ("x1") = v;
	211
	212	asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(add),
	213	" stadd %[i], %[v]\n")
	214	: [i] "+r" (x0), [v] "+Q" (v->counter)
	215	: "r" (x1)
	216	: "x30");
	217	}
	218
305d454a WD	219	#define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \
	220	static inline long atomic64_add_return##name(long i, atomic64_t *v) \
	221	{ \
	222	register long x0 asm ("x0") = i; \
	223	register atomic64_t *x1 asm ("x1") = v; \
	224	\
	225	asm volatile(ARM64_LSE_ATOMIC_INSN( \
	226	/* LL/SC */ \
	227	" nop\n" \
	228	__LL_SC_ATOMIC64(add_return##name), \
	229	/* LSE atomics */ \
	230	" ldadd" #mb " %[i], x30, %[v]\n" \
	231	" add %[i], %[i], x30") \
	232	: [i] "+r" (x0), [v] "+Q" (v->counter) \
	233	: "r" (x1) \
	234	: "x30" , ##cl); \
	235	\
	236	return x0; \
	237	}
c09d6a04	238
305d454a WD	239	ATOMIC64_OP_ADD_RETURN(_relaxed, )
	240	ATOMIC64_OP_ADD_RETURN(_acquire, a, "memory")
	241	ATOMIC64_OP_ADD_RETURN(_release, l, "memory")
	242	ATOMIC64_OP_ADD_RETURN( , al, "memory")
c09d6a04	243
305d454a	244	#undef ATOMIC64_OP_ADD_RETURN
c09d6a04 WD	245
	246	static inline void atomic64_and(long i, atomic64_t *v)
	247	{
	248	register long x0 asm ("x0") = i;
	249	register atomic64_t *x1 asm ("x1") = v;
	250
	251	asm volatile(ARM64_LSE_ATOMIC_INSN(
	252	/* LL/SC */
	253	" nop\n"
	254	__LL_SC_ATOMIC64(and),
	255	/* LSE atomics */
	256	" mvn %[i], %[i]\n"
	257	" stclr %[i], %[v]")
	258	: [i] "+r" (x0), [v] "+Q" (v->counter)
	259	: "r" (x1)
	260	: "x30");
	261	}
	262
	263	static inline void atomic64_sub(long i, atomic64_t *v)
	264	{
	265	register long x0 asm ("x0") = i;
	266	register atomic64_t *x1 asm ("x1") = v;
	267
	268	asm volatile(ARM64_LSE_ATOMIC_INSN(
	269	/* LL/SC */
	270	" nop\n"
	271	__LL_SC_ATOMIC64(sub),
	272	/* LSE atomics */
	273	" neg %[i], %[i]\n"
	274	" stadd %[i], %[v]")
	275	: [i] "+r" (x0), [v] "+Q" (v->counter)
	276	: "r" (x1)
	277	: "x30");
	278	}
	279
305d454a WD	280	#define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \
	281	static inline long atomic64_sub_return##name(long i, atomic64_t *v) \
	282	{ \
	283	register long x0 asm ("x0") = i; \
	284	register atomic64_t *x1 asm ("x1") = v; \
	285	\
	286	asm volatile(ARM64_LSE_ATOMIC_INSN( \
	287	/* LL/SC */ \
	288	" nop\n" \
	289	__LL_SC_ATOMIC64(sub_return##name) \
	290	" nop", \
	291	/* LSE atomics */ \
	292	" neg %[i], %[i]\n" \
	293	" ldadd" #mb " %[i], x30, %[v]\n" \
	294	" add %[i], %[i], x30") \
	295	: [i] "+r" (x0), [v] "+Q" (v->counter) \
	296	: "r" (x1) \
	297	: "x30" , ##cl); \
	298	\
	299	return x0; \
	300	}
c09d6a04	301
305d454a WD	302	ATOMIC64_OP_SUB_RETURN(_relaxed, )
	303	ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory")
	304	ATOMIC64_OP_SUB_RETURN(_release, l, "memory")
	305	ATOMIC64_OP_SUB_RETURN( , al, "memory")
c09d6a04	306
305d454a	307	#undef ATOMIC64_OP_SUB_RETURN
c0385b24 WD	308
	309	static inline long atomic64_dec_if_positive(atomic64_t *v)
	310	{
c09d6a04	311	register long x0 asm ("x0") = (long)v;
c0385b24	312
c09d6a04 WD	313	asm volatile(ARM64_LSE_ATOMIC_INSN(
	314	/* LL/SC */
	315	" nop\n"
	316	__LL_SC_ATOMIC64(dec_if_positive)
	317	" nop\n"
	318	" nop\n"
	319	" nop\n"
	320	" nop\n"
	321	" nop",
	322	/* LSE atomics */
	323	"1: ldr x30, %[v]\n"
	324	" subs %[ret], x30, #1\n"
db26217e	325	" b.lt 2f\n"
c09d6a04 WD	326	" casal x30, %[ret], %[v]\n"
	327	" sub x30, x30, #1\n"
	328	" sub x30, x30, %[ret]\n"
	329	" cbnz x30, 1b\n"
	330	"2:")
	331	: [ret] "+&r" (x0), [v] "+Q" (v->counter)
c0385b24 WD	332	:
	333	: "x30", "cc", "memory");
	334
	335	return x0;
	336	}
	337
c09d6a04 WD	338	#undef __LL_SC_ATOMIC64
c09d6a04 WD	339
c342f782 WD	340	#define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op)
	341
	342	#define __CMPXCHG_CASE(w, sz, name, mb, cl...) \
	343	static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \
	344	unsigned long old, \
	345	unsigned long new) \
	346	{ \
	347	register unsigned long x0 asm ("x0") = (unsigned long)ptr; \
	348	register unsigned long x1 asm ("x1") = old; \
	349	register unsigned long x2 asm ("x2") = new; \
	350	\
	351	asm volatile(ARM64_LSE_ATOMIC_INSN( \
	352	/* LL/SC */ \
484c96db WD	353	" nop\n" \
	354	__LL_SC_CMPXCHG(name) \
	355	" nop", \
c342f782 WD	356	/* LSE atomics */ \
	357	" mov " #w "30, %" #w "[old]\n" \
	358	" cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n" \
	359	" mov %" #w "[ret], " #w "30") \
	360	: [ret] "+r" (x0), [v] "+Q" ((unsigned long )ptr) \
	361	: [old] "r" (x1), [new] "r" (x2) \
	362	: "x30" , ##cl); \
	363	\
	364	return x0; \
	365	}
	366
305d454a WD	367	__CMPXCHG_CASE(w, b, 1, )
	368	__CMPXCHG_CASE(w, h, 2, )
	369	__CMPXCHG_CASE(w, , 4, )
	370	__CMPXCHG_CASE(x, , 8, )
	371	__CMPXCHG_CASE(w, b, acq_1, a, "memory")
	372	__CMPXCHG_CASE(w, h, acq_2, a, "memory")
	373	__CMPXCHG_CASE(w, , acq_4, a, "memory")
	374	__CMPXCHG_CASE(x, , acq_8, a, "memory")
	375	__CMPXCHG_CASE(w, b, rel_1, l, "memory")
	376	__CMPXCHG_CASE(w, h, rel_2, l, "memory")
	377	__CMPXCHG_CASE(w, , rel_4, l, "memory")
	378	__CMPXCHG_CASE(x, , rel_8, l, "memory")
	379	__CMPXCHG_CASE(w, b, mb_1, al, "memory")
	380	__CMPXCHG_CASE(w, h, mb_2, al, "memory")
	381	__CMPXCHG_CASE(w, , mb_4, al, "memory")
	382	__CMPXCHG_CASE(x, , mb_8, al, "memory")
c342f782 WD	383
	384	#undef __LL_SC_CMPXCHG
	385	#undef __CMPXCHG_CASE
	386
e9a4b795 WD	387	#define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op)
	388
	389	#define __CMPXCHG_DBL(name, mb, cl...) \
	390	static inline int __cmpxchg_double##name(unsigned long old1, \
	391	unsigned long old2, \
	392	unsigned long new1, \
	393	unsigned long new2, \
	394	volatile void *ptr) \
	395	{ \
	396	unsigned long oldval1 = old1; \
	397	unsigned long oldval2 = old2; \
	398	register unsigned long x0 asm ("x0") = old1; \
	399	register unsigned long x1 asm ("x1") = old2; \
	400	register unsigned long x2 asm ("x2") = new1; \
	401	register unsigned long x3 asm ("x3") = new2; \
	402	register unsigned long x4 asm ("x4") = (unsigned long)ptr; \
	403	\
	404	asm volatile(ARM64_LSE_ATOMIC_INSN( \
	405	/* LL/SC */ \
	406	" nop\n" \
	407	" nop\n" \
	408	" nop\n" \
	409	__LL_SC_CMPXCHG_DBL(name), \
	410	/* LSE atomics */ \
	411	" casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
	412	" eor %[old1], %[old1], %[oldval1]\n" \
	413	" eor %[old2], %[old2], %[oldval2]\n" \
	414	" orr %[old1], %[old1], %[old2]") \
	415	: [old1] "+r" (x0), [old2] "+r" (x1), \
	416	[v] "+Q" ((unsigned long )ptr) \
	417	: [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \
	418	[oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \
	419	: "x30" , ##cl); \
	420	\
	421	return x0; \
	422	}
	423
	424	__CMPXCHG_DBL( , )
	425	__CMPXCHG_DBL(_mb, al, "memory")
	426
	427	#undef __LL_SC_CMPXCHG_DBL
	428	#undef __CMPXCHG_DBL
	429
c0385b24	430	#endif /* __ASM_ATOMIC_LSE_H */