[mirror_ubuntu-artful-kernel.git] / arch / x86 / include / asm / cmpxchg.h

#ifndef ASM_X86_CMPXCHG_H
#define ASM_X86_CMPXCHG_H

#include <linux/compiler.h>
#include <asm/cpufeatures.h>
#include <asm/alternative.h> /* Provides LOCK_PREFIX */

/*
 * Non-existant functions to indicate usage errors at link time
 * (or compile-time if the compiler implements __compiletime_error().
 */
extern void __xchg_wrong_size(void)
	__compiletime_error("Bad argument size for xchg");
extern void __cmpxchg_wrong_size(void)
	__compiletime_error("Bad argument size for cmpxchg");
extern void __xadd_wrong_size(void)
	__compiletime_error("Bad argument size for xadd");
extern void __add_wrong_size(void)
	__compiletime_error("Bad argument size for add");

/*
 * Constants for operation sizes. On 32-bit, the 64-bit size it set to
 * -1 because sizeof will never return -1, thereby making those switch
 * case statements guaranteeed dead code which the compiler will
 * eliminate, and allowing the "missing symbol in the default case" to
 * indicate a usage error.
 */
#define __X86_CASE_B	1
#define __X86_CASE_W	2
#define __X86_CASE_L	4
#ifdef CONFIG_64BIT
#define __X86_CASE_Q	8
#else
#define	__X86_CASE_Q	-1		/* sizeof will never return -1 */
#endif

/* 
 * An exchange-type operation, which takes a value and a pointer, and
 * returns the old value.
 */
#define __xchg_op(ptr, arg, op, lock)					\
	({								\
	        __typeof__ (*(ptr)) __ret = (arg);			\
		switch (sizeof(*(ptr))) {				\
		case __X86_CASE_B:					\
			asm volatile (lock #op "b %b0, %1\n"		\
				      : "+q" (__ret), "+m" (*(ptr))	\
				      : : "memory", "cc");		\
			break;						\
		case __X86_CASE_W:					\
			asm volatile (lock #op "w %w0, %1\n"		\
				      : "+r" (__ret), "+m" (*(ptr))	\
				      : : "memory", "cc");		\
			break;						\
		case __X86_CASE_L:					\
			asm volatile (lock #op "l %0, %1\n"		\
				      : "+r" (__ret), "+m" (*(ptr))	\
				      : : "memory", "cc");		\
			break;						\
		case __X86_CASE_Q:					\
			asm volatile (lock #op "q %q0, %1\n"		\
				      : "+r" (__ret), "+m" (*(ptr))	\
				      : : "memory", "cc");		\
			break;						\
		default:						\
			__ ## op ## _wrong_size();			\
		}							\
		__ret;							\
	})

/*
 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
 * Since this is generally used to protect other memory information, we
 * use "asm volatile" and "memory" clobbers to prevent gcc from moving
 * information around.
 */
#define xchg(ptr, v)	__xchg_op((ptr), (v), xchg, "")

/*
 * Atomic compare and exchange.  Compare OLD with MEM, if identical,
 * store NEW in MEM.  Return the initial value in MEM.  Success is
 * indicated by comparing RETURN with OLD.
 */
#define __raw_cmpxchg(ptr, old, new, size, lock)			\
({									\
	__typeof__(*(ptr)) __ret;					\
	__typeof__(*(ptr)) __old = (old);				\
	__typeof__(*(ptr)) __new = (new);				\
	switch (size) {							\
	case __X86_CASE_B:						\
	{								\
		volatile u8 *__ptr = (volatile u8 *)(ptr);		\
		asm volatile(lock "cmpxchgb %2,%1"			\
			     : "=a" (__ret), "+m" (*__ptr)		\
			     : "q" (__new), "0" (__old)			\
			     : "memory");				\
		break;							\
	}								\
	case __X86_CASE_W:						\
	{								\
		volatile u16 *__ptr = (volatile u16 *)(ptr);		\
		asm volatile(lock "cmpxchgw %2,%1"			\
			     : "=a" (__ret), "+m" (*__ptr)		\
			     : "r" (__new), "0" (__old)			\
			     : "memory");				\
		break;							\
	}								\
	case __X86_CASE_L:						\
	{								\
		volatile u32 *__ptr = (volatile u32 *)(ptr);		\
		asm volatile(lock "cmpxchgl %2,%1"			\
			     : "=a" (__ret), "+m" (*__ptr)		\
			     : "r" (__new), "0" (__old)			\
			     : "memory");				\
		break;							\
	}								\
	case __X86_CASE_Q:						\
	{								\
		volatile u64 *__ptr = (volatile u64 *)(ptr);		\
		asm volatile(lock "cmpxchgq %2,%1"			\
			     : "=a" (__ret), "+m" (*__ptr)		\
			     : "r" (__new), "0" (__old)			\
			     : "memory");				\
		break;							\
	}								\
	default:							\
		__cmpxchg_wrong_size();					\
	}								\
	__ret;								\
})

#define __cmpxchg(ptr, old, new, size)					\
	__raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)

#define __sync_cmpxchg(ptr, old, new, size)				\
	__raw_cmpxchg((ptr), (old), (new), (size), "lock; ")

#define __cmpxchg_local(ptr, old, new, size)				\
	__raw_cmpxchg((ptr), (old), (new), (size), "")

#ifdef CONFIG_X86_32
# include <asm/cmpxchg_32.h>
#else
# include <asm/cmpxchg_64.h>
#endif

#define cmpxchg(ptr, old, new)						\
	__cmpxchg(ptr, old, new, sizeof(*(ptr)))

#define sync_cmpxchg(ptr, old, new)					\
	__sync_cmpxchg(ptr, old, new, sizeof(*(ptr)))

#define cmpxchg_local(ptr, old, new)					\
	__cmpxchg_local(ptr, old, new, sizeof(*(ptr)))


#define __raw_try_cmpxchg(_ptr, _pold, _new, size, lock)		\
({									\
	bool success;							\
	__typeof__(_ptr) _old = (_pold);				\
	__typeof__(*(_ptr)) __old = *_old;				\
	__typeof__(*(_ptr)) __new = (_new);				\
	switch (size) {							\
	case __X86_CASE_B:						\
	{								\
		volatile u8 *__ptr = (volatile u8 *)(_ptr);		\
		asm volatile(lock "cmpxchgb %[new], %[ptr]"		\
			     CC_SET(z)					\
			     : CC_OUT(z) (success),			\
			       [ptr] "+m" (*__ptr),			\
			       [old] "+a" (__old)			\
			     : [new] "q" (__new)			\
			     : "memory");				\
		break;							\
	}								\
	case __X86_CASE_W:						\
	{								\
		volatile u16 *__ptr = (volatile u16 *)(_ptr);		\
		asm volatile(lock "cmpxchgw %[new], %[ptr]"		\
			     CC_SET(z)					\
			     : CC_OUT(z) (success),			\
			       [ptr] "+m" (*__ptr),			\
			       [old] "+a" (__old)			\
			     : [new] "r" (__new)			\
			     : "memory");				\
		break;							\
	}								\
	case __X86_CASE_L:						\
	{								\
		volatile u32 *__ptr = (volatile u32 *)(_ptr);		\
		asm volatile(lock "cmpxchgl %[new], %[ptr]"		\
			     CC_SET(z)					\
			     : CC_OUT(z) (success),			\
			       [ptr] "+m" (*__ptr),			\
			       [old] "+a" (__old)			\
			     : [new] "r" (__new)			\
			     : "memory");				\
		break;							\
	}								\
	case __X86_CASE_Q:						\
	{								\
		volatile u64 *__ptr = (volatile u64 *)(_ptr);		\
		asm volatile(lock "cmpxchgq %[new], %[ptr]"		\
			     CC_SET(z)					\
			     : CC_OUT(z) (success),			\
			       [ptr] "+m" (*__ptr),			\
			       [old] "+a" (__old)			\
			     : [new] "r" (__new)			\
			     : "memory");				\
		break;							\
	}								\
	default:							\
		__cmpxchg_wrong_size();					\
	}								\
	if (unlikely(!success))						\
		*_old = __old;						\
	likely(success);						\
})

#define __try_cmpxchg(ptr, pold, new, size)				\
	__raw_try_cmpxchg((ptr), (pold), (new), (size), LOCK_PREFIX)

#define try_cmpxchg(ptr, pold, new)					\
	__try_cmpxchg((ptr), (pold), (new), sizeof(*(ptr)))

/*
 * xadd() adds "inc" to "*ptr" and atomically returns the previous
 * value of "*ptr".
 *
 * xadd() is locked when multiple CPUs are online
 */
#define __xadd(ptr, inc, lock)	__xchg_op((ptr), (inc), xadd, lock)
#define xadd(ptr, inc)		__xadd((ptr), (inc), LOCK_PREFIX)

#define __cmpxchg_double(pfx, p1, p2, o1, o2, n1, n2)			\
({									\
	bool __ret;							\
	__typeof__(*(p1)) __old1 = (o1), __new1 = (n1);			\
	__typeof__(*(p2)) __old2 = (o2), __new2 = (n2);			\
	BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long));			\
	BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long));			\
	VM_BUG_ON((unsigned long)(p1) % (2 * sizeof(long)));		\
	VM_BUG_ON((unsigned long)((p1) + 1) != (unsigned long)(p2));	\
	asm volatile(pfx "cmpxchg%c4b %2; sete %0"			\
		     : "=a" (__ret), "+d" (__old2),			\
		       "+m" (*(p1)), "+m" (*(p2))			\
		     : "i" (2 * sizeof(long)), "a" (__old1),		\
		       "b" (__new1), "c" (__new2));			\
	__ret;								\
})

#define cmpxchg_double(p1, p2, o1, o2, n1, n2) \
	__cmpxchg_double(LOCK_PREFIX, p1, p2, o1, o2, n1, n2)

#define cmpxchg_double_local(p1, p2, o1, o2, n1, n2) \
	__cmpxchg_double(, p1, p2, o1, o2, n1, n2)

#endif	/* ASM_X86_CMPXCHG_H */
Commit	Line	Data
e9826380 JF	1	#ifndef ASM_X86_CMPXCHG_H
	2	#define ASM_X86_CMPXCHG_H
	3
61e2cd0a	4	#include <linux/compiler.h>
cd4d09ec	5	#include <asm/cpufeatures.h>
e9826380 JF	6	#include <asm/alternative.h> /* Provides LOCK_PREFIX */
e9826380 JF	7
61e2cd0a JF	8	/*
	9	* Non-existant functions to indicate usage errors at link time
	10	* (or compile-time if the compiler implements __compiletime_error().
	11	*/
	12	extern void __xchg_wrong_size(void)
	13	__compiletime_error("Bad argument size for xchg");
	14	extern void __cmpxchg_wrong_size(void)
	15	__compiletime_error("Bad argument size for cmpxchg");
	16	extern void __xadd_wrong_size(void)
	17	__compiletime_error("Bad argument size for xadd");
3d94ae0c JF	18	extern void __add_wrong_size(void)
3d94ae0c JF	19	__compiletime_error("Bad argument size for add");
e9826380 JF	20
	21	/*
	22	* Constants for operation sizes. On 32-bit, the 64-bit size it set to
	23	* -1 because sizeof will never return -1, thereby making those switch
	24	* case statements guaranteeed dead code which the compiler will
	25	* eliminate, and allowing the "missing symbol in the default case" to
	26	* indicate a usage error.
	27	*/
	28	#define __X86_CASE_B 1
	29	#define __X86_CASE_W 2
	30	#define __X86_CASE_L 4
	31	#ifdef CONFIG_64BIT
	32	#define __X86_CASE_Q 8
	33	#else
	34	#define __X86_CASE_Q -1 /* sizeof will never return -1 */
	35	#endif
	36
31a8394e JF	37	/*
31a8394e JF	38	* An exchange-type operation, which takes a value and a pointer, and
7f5281ae	39	* returns the old value.
31a8394e JF	40	*/
	41	#define __xchg_op(ptr, arg, op, lock) \
	42	({ \
	43	__typeof__ (*(ptr)) __ret = (arg); \
	44	switch (sizeof(*(ptr))) { \
	45	case __X86_CASE_B: \
	46	asm volatile (lock #op "b %b0, %1\n" \
2ca052a3	47	: "+q" (__ret), "+m" (*(ptr)) \
31a8394e JF	48	: : "memory", "cc"); \
	49	break; \
	50	case __X86_CASE_W: \
	51	asm volatile (lock #op "w %w0, %1\n" \
	52	: "+r" (__ret), "+m" (*(ptr)) \
	53	: : "memory", "cc"); \
	54	break; \
	55	case __X86_CASE_L: \
	56	asm volatile (lock #op "l %0, %1\n" \
	57	: "+r" (__ret), "+m" (*(ptr)) \
	58	: : "memory", "cc"); \
	59	break; \
	60	case __X86_CASE_Q: \
	61	asm volatile (lock #op "q %q0, %1\n" \
	62	: "+r" (__ret), "+m" (*(ptr)) \
	63	: : "memory", "cc"); \
	64	break; \
	65	default: \
	66	__ ## op ## _wrong_size(); \
	67	} \
	68	__ret; \
	69	})
	70
e9826380 JF	71	/*
	72	* Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
	73	* Since this is generally used to protect other memory information, we
	74	* use "asm volatile" and "memory" clobbers to prevent gcc from moving
	75	* information around.
	76	*/
31a8394e	77	#define xchg(ptr, v) __xchg_op((ptr), (v), xchg, "")
e9826380 JF	78
	79	/*
	80	* Atomic compare and exchange. Compare OLD with MEM, if identical,
	81	* store NEW in MEM. Return the initial value in MEM. Success is
	82	* indicated by comparing RETURN with OLD.
	83	*/
	84	#define __raw_cmpxchg(ptr, old, new, size, lock) \
	85	({ \
	86	__typeof__(*(ptr)) __ret; \
	87	__typeof__(*(ptr)) __old = (old); \
	88	__typeof__(*(ptr)) __new = (new); \
	89	switch (size) { \
	90	case __X86_CASE_B: \
	91	{ \
	92	volatile u8 __ptr = (volatile u8 )(ptr); \
	93	asm volatile(lock "cmpxchgb %2,%1" \
	94	: "=a" (__ret), "+m" (*__ptr) \
	95	: "q" (__new), "0" (__old) \
	96	: "memory"); \
	97	break; \
	98	} \
	99	case __X86_CASE_W: \
	100	{ \
	101	volatile u16 __ptr = (volatile u16 )(ptr); \
	102	asm volatile(lock "cmpxchgw %2,%1" \
	103	: "=a" (__ret), "+m" (*__ptr) \
	104	: "r" (__new), "0" (__old) \
	105	: "memory"); \
	106	break; \
	107	} \
	108	case __X86_CASE_L: \
	109	{ \
	110	volatile u32 __ptr = (volatile u32 )(ptr); \
	111	asm volatile(lock "cmpxchgl %2,%1" \
	112	: "=a" (__ret), "+m" (*__ptr) \
	113	: "r" (__new), "0" (__old) \
	114	: "memory"); \
	115	break; \
	116	} \
	117	case __X86_CASE_Q: \
	118	{ \
	119	volatile u64 __ptr = (volatile u64 )(ptr); \
	120	asm volatile(lock "cmpxchgq %2,%1" \
	121	: "=a" (__ret), "+m" (*__ptr) \
	122	: "r" (__new), "0" (__old) \
	123	: "memory"); \
	124	break; \
	125	} \
	126	default: \
	127	__cmpxchg_wrong_size(); \
	128	} \
	129	__ret; \
	130	})
	131
	132	#define __cmpxchg(ptr, old, new, size) \
	133	__raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
	134
	135	#define __sync_cmpxchg(ptr, old, new, size) \
	136	__raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
	137
	138	#define __cmpxchg_local(ptr, old, new, size) \
	139	__raw_cmpxchg((ptr), (old), (new), (size), "")
	140
96a388de	141	#ifdef CONFIG_X86_32
a1ce3928	142	# include <asm/cmpxchg_32.h>
96a388de	143	#else
a1ce3928	144	# include <asm/cmpxchg_64.h>
96a388de	145	#endif
e9826380	146
e9826380	147	#define cmpxchg(ptr, old, new) \
fc395b92	148	__cmpxchg(ptr, old, new, sizeof(*(ptr)))
e9826380 JF	149
e9826380 JF	150	#define sync_cmpxchg(ptr, old, new) \
fc395b92	151	__sync_cmpxchg(ptr, old, new, sizeof(*(ptr)))
e9826380 JF	152
e9826380 JF	153	#define cmpxchg_local(ptr, old, new) \
fc395b92	154	__cmpxchg_local(ptr, old, new, sizeof(*(ptr)))
e9826380	155
a9ebf306 PZ	156
	157	#define __raw_try_cmpxchg(_ptr, _pold, _new, size, lock) \
	158	({ \
	159	bool success; \
	160	__typeof__(_ptr) _old = (_pold); \
	161	__typeof__((_ptr)) __old = _old; \
	162	__typeof__(*(_ptr)) __new = (_new); \
	163	switch (size) { \
	164	case __X86_CASE_B: \
	165	{ \
	166	volatile u8 __ptr = (volatile u8 )(_ptr); \
	167	asm volatile(lock "cmpxchgb %[new], %[ptr]" \
	168	CC_SET(z) \
	169	: CC_OUT(z) (success), \
	170	[ptr] "+m" (*__ptr), \
	171	[old] "+a" (__old) \
	172	: [new] "q" (__new) \
	173	: "memory"); \
	174	break; \
	175	} \
	176	case __X86_CASE_W: \
	177	{ \
	178	volatile u16 __ptr = (volatile u16 )(_ptr); \
	179	asm volatile(lock "cmpxchgw %[new], %[ptr]" \
	180	CC_SET(z) \
	181	: CC_OUT(z) (success), \
	182	[ptr] "+m" (*__ptr), \
	183	[old] "+a" (__old) \
	184	: [new] "r" (__new) \
	185	: "memory"); \
	186	break; \
	187	} \
	188	case __X86_CASE_L: \
	189	{ \
	190	volatile u32 __ptr = (volatile u32 )(_ptr); \
	191	asm volatile(lock "cmpxchgl %[new], %[ptr]" \
	192	CC_SET(z) \
	193	: CC_OUT(z) (success), \
	194	[ptr] "+m" (*__ptr), \
	195	[old] "+a" (__old) \
	196	: [new] "r" (__new) \
	197	: "memory"); \
	198	break; \
	199	} \
	200	case __X86_CASE_Q: \
	201	{ \
	202	volatile u64 __ptr = (volatile u64 )(_ptr); \
	203	asm volatile(lock "cmpxchgq %[new], %[ptr]" \
	204	CC_SET(z) \
	205	: CC_OUT(z) (success), \
	206	[ptr] "+m" (*__ptr), \
	207	[old] "+a" (__old) \
	208	: [new] "r" (__new) \
	209	: "memory"); \
	210	break; \
	211	} \
	212	default: \
	213	__cmpxchg_wrong_size(); \
	214	} \
44fe8445 PZ	215	if (unlikely(!success)) \
	216	*_old = __old; \
	217	likely(success); \
a9ebf306 PZ	218	})
	219
	220	#define __try_cmpxchg(ptr, pold, new, size) \
	221	__raw_try_cmpxchg((ptr), (pold), (new), (size), LOCK_PREFIX)
	222
	223	#define try_cmpxchg(ptr, pold, new) \
	224	__try_cmpxchg((ptr), (pold), (new), sizeof(*(ptr)))
	225
433b3520 JF	226	/*
	227	* xadd() adds "inc" to "*ptr" and atomically returns the previous
	228	* value of "*ptr".
	229	*
	230	* xadd() is locked when multiple CPUs are online
433b3520	231	*/
31a8394e	232	#define __xadd(ptr, inc, lock) __xchg_op((ptr), (inc), xadd, lock)
433b3520	233	#define xadd(ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX)
3d94ae0c	234
cdcd6298 JB	235	#define __cmpxchg_double(pfx, p1, p2, o1, o2, n1, n2) \
	236	({ \
	237	bool __ret; \
	238	__typeof__(*(p1)) __old1 = (o1), __new1 = (n1); \
	239	__typeof__(*(p2)) __old2 = (o2), __new2 = (n2); \
	240	BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \
	241	BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \
	242	VM_BUG_ON((unsigned long)(p1) % (2 * sizeof(long))); \
	243	VM_BUG_ON((unsigned long)((p1) + 1) != (unsigned long)(p2)); \
	244	asm volatile(pfx "cmpxchg%c4b %2; sete %0" \
	245	: "=a" (__ret), "+d" (__old2), \
	246	"+m" ((p1)), "+m" ((p2)) \
	247	: "i" (2 * sizeof(long)), "a" (__old1), \
	248	"b" (__new1), "c" (__new2)); \
	249	__ret; \
	250	})
	251
	252	#define cmpxchg_double(p1, p2, o1, o2, n1, n2) \
	253	__cmpxchg_double(LOCK_PREFIX, p1, p2, o1, o2, n1, n2)
	254
	255	#define cmpxchg_double_local(p1, p2, o1, o2, n1, n2) \
	256	__cmpxchg_double(, p1, p2, o1, o2, n1, n2)
	257
e9826380	258	#endif /* ASM_X86_CMPXCHG_H */