[mirror_ubuntu-bionic-kernel.git] / arch / x86 / include / asm / cmpxchg_32.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_CMPXCHG_32_H
#define _ASM_X86_CMPXCHG_32_H

/*
 * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you
 *       you need to test for the feature in boot_cpu_data.
 */

/*
 * CMPXCHG8B only writes to the target if we had the previous
 * value in registers, otherwise it acts as a read and gives us the
 * "new previous" value.  That is why there is a loop.  Preloading
 * EDX:EAX is a performance optimization: in the common case it means
 * we need only one locked operation.
 *
 * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very
 * least an FPU save and/or %cr0.ts manipulation.
 *
 * cmpxchg8b must be used with the lock prefix here to allow the
 * instruction to be executed atomically.  We need to have the reader
 * side to see the coherent 64bit value.
 */
static inline void set_64bit(volatile u64 *ptr, u64 value)
{
	u32 low  = value;
	u32 high = value >> 32;
	u64 prev = *ptr;

	asm volatile("\n1:\t"
		     LOCK_PREFIX "cmpxchg8b %0\n\t"
		     "jnz 1b"
		     : "=m" (*ptr), "+A" (prev)
		     : "b" (low), "c" (high)
		     : "memory");
}

#ifdef CONFIG_X86_CMPXCHG64
#define cmpxchg64(ptr, o, n)						\
	((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
					 (unsigned long long)(n)))
#define cmpxchg64_local(ptr, o, n)					\
	((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
					       (unsigned long long)(n)))
#endif

static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
{
	u64 prev;
	asm volatile(LOCK_PREFIX "cmpxchg8b %1"
		     : "=A" (prev),
		       "+m" (*ptr)
		     : "b" ((u32)new),
		       "c" ((u32)(new >> 32)),
		       "0" (old)
		     : "memory");
	return prev;
}

static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
{
	u64 prev;
	asm volatile("cmpxchg8b %1"
		     : "=A" (prev),
		       "+m" (*ptr)
		     : "b" ((u32)new),
		       "c" ((u32)(new >> 32)),
		       "0" (old)
		     : "memory");
	return prev;
}

#ifndef CONFIG_X86_CMPXCHG64
/*
 * Building a kernel capable running on 80386 and 80486. It may be necessary
 * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
 */

#define cmpxchg64(ptr, o, n)					\
({								\
	__typeof__(*(ptr)) __ret;				\
	__typeof__(*(ptr)) __old = (o);				\
	__typeof__(*(ptr)) __new = (n);				\
	alternative_io(LOCK_PREFIX_HERE				\
			"call cmpxchg8b_emu",			\
			"lock; cmpxchg8b (%%esi)" ,		\
		       X86_FEATURE_CX8,				\
		       "=A" (__ret),				\
		       "S" ((ptr)), "0" (__old),		\
		       "b" ((unsigned int)__new),		\
		       "c" ((unsigned int)(__new>>32))		\
		       : "memory");				\
	__ret; })


#define cmpxchg64_local(ptr, o, n)				\
({								\
	__typeof__(*(ptr)) __ret;				\
	__typeof__(*(ptr)) __old = (o);				\
	__typeof__(*(ptr)) __new = (n);				\
	alternative_io("call cmpxchg8b_emu",			\
		       "cmpxchg8b (%%esi)" ,			\
		       X86_FEATURE_CX8,				\
		       "=A" (__ret),				\
		       "S" ((ptr)), "0" (__old),		\
		       "b" ((unsigned int)__new),		\
		       "c" ((unsigned int)(__new>>32))		\
		       : "memory");				\
	__ret; })

#endif

#define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX8)

#endif /* _ASM_X86_CMPXCHG_32_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
1965aae3 PA	2	#ifndef _ASM_X86_CMPXCHG_32_H
1965aae3 PA	3	#define _ASM_X86_CMPXCHG_32_H
a436ed9c	4
2d9ce177 AK	5	/*
	6	* Note: if you use set64_bit(), __cmpxchg64(), or their variants, you
	7	* you need to test for the feature in boot_cpu_data.
	8	*/
	9
a436ed9c	10	/*
69309a05 PA	11	* CMPXCHG8B only writes to the target if we had the previous
	12	* value in registers, otherwise it acts as a read and gives us the
	13	* "new previous" value. That is why there is a loop. Preloading
	14	* EDX:EAX is a performance optimization: in the common case it means
	15	* we need only one locked operation.
a436ed9c	16	*
69309a05 PA	17	* A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very
	18	* least an FPU save and/or %cr0.ts manipulation.
	19	*
	20	* cmpxchg8b must be used with the lock prefix here to allow the
	21	* instruction to be executed atomically. We need to have the reader
	22	* side to see the coherent 64bit value.
a436ed9c	23	*/
69309a05	24	static inline void set_64bit(volatile u64 *ptr, u64 value)
a436ed9c	25	{
69309a05 PA	26	u32 low = value;
	27	u32 high = value >> 32;
	28	u64 prev = *ptr;
	29
8121019c	30	asm volatile("\n1:\t"
69309a05	31	LOCK_PREFIX "cmpxchg8b %0\n\t"
8121019c	32	"jnz 1b"
69309a05 PA	33	: "=m" (*ptr), "+A" (prev)
	34	: "b" (low), "c" (high)
	35	: "memory");
a436ed9c	36	}
a436ed9c	37
2c0b8a75	38	#ifdef CONFIG_X86_CMPXCHG64
8121019c JP	39	#define cmpxchg64(ptr, o, n) \
	40	((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
	41	(unsigned long long)(n)))
	42	#define cmpxchg64_local(ptr, o, n) \
	43	((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
	44	(unsigned long long)(n)))
a436ed9c JD	45	#endif
a436ed9c JD	46
4532b305	47	static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
2c0b8a75	48	{
4532b305	49	u64 prev;
113fc5a6 PA	50	asm volatile(LOCK_PREFIX "cmpxchg8b %1"
113fc5a6 PA	51	: "=A" (prev),
4532b305 PA	52	"+m" (*ptr)
	53	: "b" ((u32)new),
	54	"c" ((u32)(new >> 32)),
113fc5a6	55	"0" (old)
8121019c	56	: "memory");
2c0b8a75 MD	57	return prev;
	58	}
	59
4532b305	60	static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
2c0b8a75	61	{
4532b305	62	u64 prev;
113fc5a6 PA	63	asm volatile("cmpxchg8b %1"
113fc5a6 PA	64	: "=A" (prev),
4532b305 PA	65	"+m" (*ptr)
	66	: "b" ((u32)new),
	67	"c" ((u32)(new >> 32)),
113fc5a6	68	"0" (old)
8121019c	69	: "memory");
2c0b8a75 MD	70	return prev;
	71	}
	72
2c0b8a75 MD	73	#ifndef CONFIG_X86_CMPXCHG64
	74	/*
	75	* Building a kernel capable running on 80386 and 80486. It may be necessary
	76	* to simulate the cmpxchg8b on the 80386 and 80486 CPU.
	77	*/
a436ed9c	78
79e1dd05 AV	79	#define cmpxchg64(ptr, o, n) \
	80	({ \
	81	__typeof__(*(ptr)) __ret; \
	82	__typeof__(*(ptr)) __old = (o); \
	83	__typeof__(*(ptr)) __new = (n); \
9c76b384 LB	84	alternative_io(LOCK_PREFIX_HERE \
9c76b384 LB	85	"call cmpxchg8b_emu", \
79e1dd05 AV	86	"lock; cmpxchg8b (%%esi)" , \
	87	X86_FEATURE_CX8, \
	88	"=A" (__ret), \
	89	"S" ((ptr)), "0" (__old), \
	90	"b" ((unsigned int)__new), \
	91	"c" ((unsigned int)(__new>>32)) \
	92	: "memory"); \
	93	__ret; })
	94
	95
a378d933 PA	96	#define cmpxchg64_local(ptr, o, n) \
	97	({ \
	98	__typeof__(*(ptr)) __ret; \
	99	__typeof__(*(ptr)) __old = (o); \
	100	__typeof__(*(ptr)) __new = (n); \
	101	alternative_io("call cmpxchg8b_emu", \
	102	"cmpxchg8b (%%esi)" , \
	103	X86_FEATURE_CX8, \
	104	"=A" (__ret), \
	105	"S" ((ptr)), "0" (__old), \
	106	"b" ((unsigned int)__new), \
	107	"c" ((unsigned int)(__new>>32)) \
	108	: "memory"); \
	109	__ret; })
2c0b8a75 MD	110
2c0b8a75 MD	111	#endif
a436ed9c	112
362f924b	113	#define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX8)
3824abd1	114
1965aae3	115	#endif /* _ASM_X86_CMPXCHG_32_H */