[mirror_ubuntu-jammy-kernel.git] / arch / arm64 / include / asm / futex.h

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (C) 2012 ARM Ltd.
 */
#ifndef __ASM_FUTEX_H
#define __ASM_FUTEX_H

#ifdef __KERNEL__

#include <linux/futex.h>
#include <linux/uaccess.h>

#include <asm/errno.h>

#define FUTEX_MAX_LOOPS	128 /* What's the largest number you can think of? */

#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\
do {									\
	unsigned int loops = FUTEX_MAX_LOOPS;				\
									\
	uaccess_enable();						\
	asm volatile(							\
"	prfm	pstl1strm, %2\n"					\
"1:	ldxr	%w1, %2\n"						\
	insn "\n"							\
"2:	stlxr	%w0, %w3, %2\n"						\
"	cbz	%w0, 3f\n"						\
"	sub	%w4, %w4, %w0\n"					\
"	cbnz	%w4, 1b\n"						\
"	mov	%w0, %w7\n"						\
"3:\n"									\
"	dmb	ish\n"							\
"	.pushsection .fixup,\"ax\"\n"					\
"	.align	2\n"							\
"4:	mov	%w0, %w6\n"						\
"	b	3b\n"							\
"	.popsection\n"							\
	_ASM_EXTABLE(1b, 4b)						\
	_ASM_EXTABLE(2b, 4b)						\
	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp),	\
	  "+r" (loops)							\
	: "r" (oparg), "Ir" (-EFAULT), "Ir" (-EAGAIN)			\
	: "memory");							\
	uaccess_disable();						\
} while (0)

static inline int
arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uaddr)
{
	int oldval = 0, ret, tmp;
	u32 __user *uaddr = __uaccess_mask_ptr(_uaddr);

	pagefault_disable();

	switch (op) {
	case FUTEX_OP_SET:
		__futex_atomic_op("mov	%w3, %w5",
				  ret, oldval, uaddr, tmp, oparg);
		break;
	case FUTEX_OP_ADD:
		__futex_atomic_op("add	%w3, %w1, %w5",
				  ret, oldval, uaddr, tmp, oparg);
		break;
	case FUTEX_OP_OR:
		__futex_atomic_op("orr	%w3, %w1, %w5",
				  ret, oldval, uaddr, tmp, oparg);
		break;
	case FUTEX_OP_ANDN:
		__futex_atomic_op("and	%w3, %w1, %w5",
				  ret, oldval, uaddr, tmp, ~oparg);
		break;
	case FUTEX_OP_XOR:
		__futex_atomic_op("eor	%w3, %w1, %w5",
				  ret, oldval, uaddr, tmp, oparg);
		break;
	default:
		ret = -ENOSYS;
	}

	pagefault_enable();

	if (!ret)
		*oval = oldval;

	return ret;
}

static inline int
futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr,
			      u32 oldval, u32 newval)
{
	int ret = 0;
	unsigned int loops = FUTEX_MAX_LOOPS;
	u32 val, tmp;
	u32 __user *uaddr;

	if (!access_ok(_uaddr, sizeof(u32)))
		return -EFAULT;

	uaddr = __uaccess_mask_ptr(_uaddr);
	uaccess_enable();
	asm volatile("// futex_atomic_cmpxchg_inatomic\n"
"	prfm	pstl1strm, %2\n"
"1:	ldxr	%w1, %2\n"
"	sub	%w3, %w1, %w5\n"
"	cbnz	%w3, 4f\n"
"2:	stlxr	%w3, %w6, %2\n"
"	cbz	%w3, 3f\n"
"	sub	%w4, %w4, %w3\n"
"	cbnz	%w4, 1b\n"
"	mov	%w0, %w8\n"
"3:\n"
"	dmb	ish\n"
"4:\n"
"	.pushsection .fixup,\"ax\"\n"
"5:	mov	%w0, %w7\n"
"	b	4b\n"
"	.popsection\n"
	_ASM_EXTABLE(1b, 5b)
	_ASM_EXTABLE(2b, 5b)
	: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)
	: "r" (oldval), "r" (newval), "Ir" (-EFAULT), "Ir" (-EAGAIN)
	: "memory");
	uaccess_disable();

	if (!ret)
		*uval = val;

	return ret;
}

#endif /* __KERNEL__ */
#endif /* __ASM_FUTEX_H */
Commit	Line	Data
caab277b	1	/* SPDX-License-Identifier: GPL-2.0-only */
6170a974 CM	2	/*
6170a974 CM	3	* Copyright (C) 2012 ARM Ltd.
6170a974 CM	4	*/
	5	#ifndef __ASM_FUTEX_H
	6	#define __ASM_FUTEX_H
	7
	8	#ifdef __KERNEL__
	9
	10	#include <linux/futex.h>
	11	#include <linux/uaccess.h>
338d4f49	12
6170a974 CM	13	#include <asm/errno.h>
6170a974 CM	14
03110a5c WD	15	#define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of? */
03110a5c WD	16
6170a974	17	#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg) \
bd38967d	18	do { \
03110a5c WD	19	unsigned int loops = FUTEX_MAX_LOOPS; \
03110a5c WD	20	\
bd38967d	21	uaccess_enable(); \
6170a974	22	asm volatile( \
0ea366f5	23	" prfm pstl1strm, %2\n" \
8e86f0b4	24	"1: ldxr %w1, %2\n" \
6170a974	25	insn "\n" \
045afc24	26	"2: stlxr %w0, %w3, %2\n" \
03110a5c WD	27	" cbz %w0, 3f\n" \
	28	" sub %w4, %w4, %w0\n" \
	29	" cbnz %w4, 1b\n" \
	30	" mov %w0, %w7\n" \
6170a974	31	"3:\n" \
03110a5c	32	" dmb ish\n" \
6170a974	33	" .pushsection .fixup,\"ax\"\n" \
4da7a56c	34	" .align 2\n" \
03110a5c	35	"4: mov %w0, %w6\n" \
6170a974 CM	36	" b 3b\n" \
6170a974 CM	37	" .popsection\n" \
6c94f27a AB	38	_ASM_EXTABLE(1b, 4b) \
6c94f27a AB	39	_ASM_EXTABLE(2b, 4b) \
03110a5c WD	40	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp), \
	41	"+r" (loops) \
	42	: "r" (oparg), "Ir" (-EFAULT), "Ir" (-EAGAIN) \
bd38967d CM	43	: "memory"); \
	44	uaccess_disable(); \
	45	} while (0)
6170a974 CM	46
6170a974 CM	47	static inline int
91b2d344	48	arch_futex_atomic_op_inuser(int op, int oparg, int oval, u32 __user _uaddr)
6170a974	49	{
ff8acf92	50	int oldval = 0, ret, tmp;
91b2d344	51	u32 __user *uaddr = __uaccess_mask_ptr(_uaddr);
6170a974	52
2f09b227	53	pagefault_disable();
6170a974 CM	54
	55	switch (op) {
	56	case FUTEX_OP_SET:
03110a5c	57	__futex_atomic_op("mov %w3, %w5",
6170a974 CM	58	ret, oldval, uaddr, tmp, oparg);
	59	break;
	60	case FUTEX_OP_ADD:
03110a5c	61	__futex_atomic_op("add %w3, %w1, %w5",
6170a974 CM	62	ret, oldval, uaddr, tmp, oparg);
	63	break;
	64	case FUTEX_OP_OR:
03110a5c	65	__futex_atomic_op("orr %w3, %w1, %w5",
6170a974 CM	66	ret, oldval, uaddr, tmp, oparg);
	67	break;
	68	case FUTEX_OP_ANDN:
03110a5c	69	__futex_atomic_op("and %w3, %w1, %w5",
6170a974 CM	70	ret, oldval, uaddr, tmp, ~oparg);
	71	break;
	72	case FUTEX_OP_XOR:
03110a5c	73	__futex_atomic_op("eor %w3, %w1, %w5",
6170a974 CM	74	ret, oldval, uaddr, tmp, oparg);
	75	break;
	76	default:
	77	ret = -ENOSYS;
	78	}
	79
2f09b227	80	pagefault_enable();
6170a974	81
30d6e0a4 JS	82	if (!ret)
	83	*oval = oldval;
	84
6170a974 CM	85	return ret;
	86	}
	87
	88	static inline int
91b2d344	89	futex_atomic_cmpxchg_inatomic(u32 uval, u32 __user _uaddr,
6170a974 CM	90	u32 oldval, u32 newval)
	91	{
	92	int ret = 0;
03110a5c	93	unsigned int loops = FUTEX_MAX_LOOPS;
6170a974	94	u32 val, tmp;
91b2d344	95	u32 __user *uaddr;
6170a974	96
96d4f267	97	if (!access_ok(_uaddr, sizeof(u32)))
6170a974 CM	98	return -EFAULT;
6170a974 CM	99
91b2d344	100	uaddr = __uaccess_mask_ptr(_uaddr);
bd38967d	101	uaccess_enable();
6170a974	102	asm volatile("// futex_atomic_cmpxchg_inatomic\n"
0ea366f5	103	" prfm pstl1strm, %2\n"
8e86f0b4	104	"1: ldxr %w1, %2\n"
03110a5c WD	105	" sub %w3, %w1, %w5\n"
	106	" cbnz %w3, 4f\n"
	107	"2: stlxr %w3, %w6, %2\n"
	108	" cbz %w3, 3f\n"
	109	" sub %w4, %w4, %w3\n"
	110	" cbnz %w4, 1b\n"
	111	" mov %w0, %w8\n"
6170a974	112	"3:\n"
03110a5c WD	113	" dmb ish\n"
03110a5c WD	114	"4:\n"
6170a974	115	" .pushsection .fixup,\"ax\"\n"
03110a5c WD	116	"5: mov %w0, %w7\n"
03110a5c WD	117	" b 4b\n"
6170a974	118	" .popsection\n"
03110a5c WD	119	_ASM_EXTABLE(1b, 5b)
	120	_ASM_EXTABLE(2b, 5b)
	121	: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)
	122	: "r" (oldval), "r" (newval), "Ir" (-EFAULT), "Ir" (-EAGAIN)
95c41896	123	: "memory");
bd38967d	124	uaccess_disable();
6170a974	125
8e4e0ac0 WD	126	if (!ret)
	127	*uval = val;
	128
6170a974 CM	129	return ret;
	130	}
	131
	132	#endif /* __KERNEL__ */
	133	#endif /* __ASM_FUTEX_H */