[mirror_ubuntu-bionic-kernel.git] / arch / ia64 / include / asm / futex.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_FUTEX_H
#define _ASM_FUTEX_H

#include <linux/futex.h>
#include <linux/uaccess.h>
#include <asm/errno.h>

#define __futex_atomic_op1(insn, ret, oldval, uaddr, oparg) \
do {									\
	register unsigned long r8 __asm ("r8") = 0;			\
	__asm__ __volatile__(						\
		"	mf;;					\n"	\
		"[1:] "	insn ";;				\n"	\
		"	.xdata4 \"__ex_table\", 1b-., 2f-.	\n"	\
		"[2:]"							\
		: "+r" (r8), "=r" (oldval)				\
		: "r" (uaddr), "r" (oparg)				\
		: "memory");						\
	ret = r8;							\
} while (0)

#define __futex_atomic_op2(insn, ret, oldval, uaddr, oparg) \
do {									\
	register unsigned long r8 __asm ("r8") = 0;			\
	int val, newval;						\
	do {								\
		__asm__ __volatile__(					\
			"	mf;;				  \n"	\
			"[1:]	ld4 %3=[%4];;			  \n"	\
			"	mov %2=%3			  \n"	\
				insn	";;			  \n"	\
			"	mov ar.ccv=%2;;			  \n"	\
			"[2:]	cmpxchg4.acq %1=[%4],%3,ar.ccv;;  \n"	\
			"	.xdata4 \"__ex_table\", 1b-., 3f-.\n"	\
			"	.xdata4 \"__ex_table\", 2b-., 3f-.\n"	\
			"[3:]"						\
			: "+r" (r8), "=r" (val), "=&r" (oldval),	\
			   "=&r" (newval)				\
			: "r" (uaddr), "r" (oparg)			\
			: "memory");					\
		if (unlikely (r8))					\
			break;						\
	} while (unlikely (val != oldval));				\
	ret = r8;							\
} while (0)

static inline int
arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
{
	int oldval = 0, ret;

	pagefault_disable();

	switch (op) {
	case FUTEX_OP_SET:
		__futex_atomic_op1("xchg4 %1=[%2],%3", ret, oldval, uaddr,
				   oparg);
		break;
	case FUTEX_OP_ADD:
		__futex_atomic_op2("add %3=%3,%5", ret, oldval, uaddr, oparg);
		break;
	case FUTEX_OP_OR:
		__futex_atomic_op2("or %3=%3,%5", ret, oldval, uaddr, oparg);
		break;
	case FUTEX_OP_ANDN:
		__futex_atomic_op2("and %3=%3,%5", ret, oldval, uaddr,
				   ~oparg);
		break;
	case FUTEX_OP_XOR:
		__futex_atomic_op2("xor %3=%3,%5", ret, oldval, uaddr, oparg);
		break;
	default:
		ret = -ENOSYS;
	}

	pagefault_enable();

	if (!ret)
		*oval = oldval;

	return ret;
}

static inline int
futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
			      u32 oldval, u32 newval)
{
	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
		return -EFAULT;

	{
		register unsigned long r8 __asm ("r8") = 0;
		unsigned long prev;
		__asm__ __volatile__(
			"	mf;;					\n"
			"	mov ar.ccv=%4;;				\n"
			"[1:]	cmpxchg4.acq %1=[%2],%3,ar.ccv		\n"
			"	.xdata4 \"__ex_table\", 1b-., 2f-.	\n"
			"[2:]"
			: "+r" (r8), "=&r" (prev)
			: "r" (uaddr), "r" (newval),
			  "rO" ((long) (unsigned) oldval)
			: "memory");
		*uval = prev;
		return r8;
	}
}

#endif /* _ASM_FUTEX_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
4732efbe JJ	2	#ifndef _ASM_FUTEX_H
	3	#define _ASM_FUTEX_H
	4
a192dc16	5	#include <linux/futex.h>
730f412c	6	#include <linux/uaccess.h>
a192dc16	7	#include <asm/errno.h>
4732efbe	8
a192dc16 JJ	9	#define __futex_atomic_op1(insn, ret, oldval, uaddr, oparg) \
	10	do { \
	11	register unsigned long r8 __asm ("r8") = 0; \
	12	__asm__ __volatile__( \
	13	" mf;; \n" \
	14	"[1:] " insn ";; \n" \
	15	" .xdata4 \"__ex_table\", 1b-., 2f-. \n" \
	16	"[2:]" \
	17	: "+r" (r8), "=r" (oldval) \
	18	: "r" (uaddr), "r" (oparg) \
	19	: "memory"); \
	20	ret = r8; \
	21	} while (0)
	22
	23	#define __futex_atomic_op2(insn, ret, oldval, uaddr, oparg) \
	24	do { \
	25	register unsigned long r8 __asm ("r8") = 0; \
	26	int val, newval; \
	27	do { \
	28	__asm__ __volatile__( \
	29	" mf;; \n" \
	30	"[1:] ld4 %3=[%4];; \n" \
	31	" mov %2=%3 \n" \
	32	insn ";; \n" \
	33	" mov ar.ccv=%2;; \n" \
	34	"[2:] cmpxchg4.acq %1=[%4],%3,ar.ccv;; \n" \
	35	" .xdata4 \"__ex_table\", 1b-., 3f-.\n" \
	36	" .xdata4 \"__ex_table\", 2b-., 3f-.\n" \
	37	"[3:]" \
	38	: "+r" (r8), "=r" (val), "=&r" (oldval), \
	39	"=&r" (newval) \
	40	: "r" (uaddr), "r" (oparg) \
	41	: "memory"); \
	42	if (unlikely (r8)) \
	43	break; \
	44	} while (unlikely (val != oldval)); \
	45	ret = r8; \
	46	} while (0)
	47
	48	static inline int
30d6e0a4	49	arch_futex_atomic_op_inuser(int op, int oparg, int oval, u32 __user uaddr)
a192dc16	50	{
a192dc16	51	int oldval = 0, ret;
a192dc16	52
a866374a	53	pagefault_disable();
a192dc16 JJ	54
	55	switch (op) {
	56	case FUTEX_OP_SET:
	57	__futex_atomic_op1("xchg4 %1=[%2],%3", ret, oldval, uaddr,
	58	oparg);
	59	break;
	60	case FUTEX_OP_ADD:
	61	__futex_atomic_op2("add %3=%3,%5", ret, oldval, uaddr, oparg);
	62	break;
	63	case FUTEX_OP_OR:
	64	__futex_atomic_op2("or %3=%3,%5", ret, oldval, uaddr, oparg);
	65	break;
	66	case FUTEX_OP_ANDN:
	67	__futex_atomic_op2("and %3=%3,%5", ret, oldval, uaddr,
	68	~oparg);
	69	break;
	70	case FUTEX_OP_XOR:
	71	__futex_atomic_op2("xor %3=%3,%5", ret, oldval, uaddr, oparg);
	72	break;
	73	default:
	74	ret = -ENOSYS;
	75	}
	76
a866374a	77	pagefault_enable();
a192dc16	78
30d6e0a4 JS	79	if (!ret)
	80	*oval = oldval;
	81
a192dc16 JJ	82	return ret;
	83	}
	84
	85	static inline int
8d7718aa ML	86	futex_atomic_cmpxchg_inatomic(u32 uval, u32 __user uaddr,
8d7718aa ML	87	u32 oldval, u32 newval)
a192dc16	88	{
8d7718aa	89	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
a192dc16 JJ	90	return -EFAULT;
	91
	92	{
136f39dd	93	register unsigned long r8 __asm ("r8") = 0;
37a9d912	94	unsigned long prev;
a192dc16 JJ	95	__asm__ __volatile__(
a192dc16 JJ	96	" mf;; \n"
c76f39bd TL	97	" mov ar.ccv=%4;; \n"
c76f39bd TL	98	"[1:] cmpxchg4.acq %1=[%2],%3,ar.ccv \n"
a192dc16 JJ	99	" .xdata4 \"__ex_table\", 1b-., 2f-. \n"
a192dc16 JJ	100	"[2:]"
136f39dd	101	: "+r" (r8), "=&r" (prev)
a192dc16 JJ	102	: "r" (uaddr), "r" (newval),
	103	"rO" ((long) (unsigned) oldval)
	104	: "memory");
37a9d912	105	*uval = prev;
a192dc16 JJ	106	return r8;
	107	}
	108	}
	109
	110	#endif /* _ASM_FUTEX_H */