[mirror_ubuntu-bionic-kernel.git] / include / asm-generic / percpu.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_GENERIC_PERCPU_H_
#define _ASM_GENERIC_PERCPU_H_

#include <linux/compiler.h>
#include <linux/threads.h>
#include <linux/percpu-defs.h>

#ifdef CONFIG_SMP

/*
 * per_cpu_offset() is the offset that has to be added to a
 * percpu variable to get to the instance for a certain processor.
 *
 * Most arches use the __per_cpu_offset array for those offsets but
 * some arches have their own ways of determining the offset (x86_64, s390).
 */
#ifndef __per_cpu_offset
extern unsigned long __per_cpu_offset[NR_CPUS];

#define per_cpu_offset(x) (__per_cpu_offset[x])
#endif

/*
 * Determine the offset for the currently active processor.
 * An arch may define __my_cpu_offset to provide a more effective
 * means of obtaining the offset to the per cpu variables of the
 * current processor.
 */
#ifndef __my_cpu_offset
#define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
#endif
#ifdef CONFIG_DEBUG_PREEMPT
#define my_cpu_offset per_cpu_offset(smp_processor_id())
#else
#define my_cpu_offset __my_cpu_offset
#endif

/*
 * Arch may define arch_raw_cpu_ptr() to provide more efficient address
 * translations for raw_cpu_ptr().
 */
#ifndef arch_raw_cpu_ptr
#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
#endif

#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
extern void setup_per_cpu_areas(void);
#endif

#endif	/* SMP */

#ifndef PER_CPU_BASE_SECTION
#ifdef CONFIG_SMP
#define PER_CPU_BASE_SECTION ".data..percpu"
#else
#define PER_CPU_BASE_SECTION ".data"
#endif
#endif

#ifndef PER_CPU_ATTRIBUTES
#define PER_CPU_ATTRIBUTES
#endif

#ifndef PER_CPU_DEF_ATTRIBUTES
#define PER_CPU_DEF_ATTRIBUTES
#endif

#define raw_cpu_generic_read(pcp)					\
({									\
	*raw_cpu_ptr(&(pcp));						\
})

#define raw_cpu_generic_to_op(pcp, val, op)				\
do {									\
	*raw_cpu_ptr(&(pcp)) op val;					\
} while (0)

#define raw_cpu_generic_add_return(pcp, val)				\
({									\
	typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp));			\
									\
	*__p += val;							\
	*__p;								\
})

#define raw_cpu_generic_xchg(pcp, nval)					\
({									\
	typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp));			\
	typeof(pcp) __ret;						\
	__ret = *__p;							\
	*__p = nval;							\
	__ret;								\
})

#define raw_cpu_generic_cmpxchg(pcp, oval, nval)			\
({									\
	typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp));			\
	typeof(pcp) __ret;						\
	__ret = *__p;							\
	if (__ret == (oval))						\
		*__p = nval;						\
	__ret;								\
})

#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
({									\
	typeof(&(pcp1)) __p1 = raw_cpu_ptr(&(pcp1));			\
	typeof(&(pcp2)) __p2 = raw_cpu_ptr(&(pcp2));			\
	int __ret = 0;							\
	if (*__p1 == (oval1) && *__p2  == (oval2)) {			\
		*__p1 = nval1;						\
		*__p2 = nval2;						\
		__ret = 1;						\
	}								\
	(__ret);							\
})

#define __this_cpu_generic_read_nopreempt(pcp)				\
({									\
	typeof(pcp) __ret;						\
	preempt_disable_notrace();					\
	__ret = READ_ONCE(*raw_cpu_ptr(&(pcp)));			\
	preempt_enable_notrace();					\
	__ret;								\
})

#define __this_cpu_generic_read_noirq(pcp)				\
({									\
	typeof(pcp) __ret;						\
	unsigned long __flags;						\
	raw_local_irq_save(__flags);					\
	__ret = raw_cpu_generic_read(pcp);				\
	raw_local_irq_restore(__flags);					\
	__ret;								\
})

#define this_cpu_generic_read(pcp)					\
({									\
	typeof(pcp) __ret;						\
	if (__native_word(pcp))						\
		__ret = __this_cpu_generic_read_nopreempt(pcp);		\
	else								\
		__ret = __this_cpu_generic_read_noirq(pcp);		\
	__ret;								\
})

#define this_cpu_generic_to_op(pcp, val, op)				\
do {									\
	unsigned long __flags;						\
	raw_local_irq_save(__flags);					\
	raw_cpu_generic_to_op(pcp, val, op);				\
	raw_local_irq_restore(__flags);					\
} while (0)


#define this_cpu_generic_add_return(pcp, val)				\
({									\
	typeof(pcp) __ret;						\
	unsigned long __flags;						\
	raw_local_irq_save(__flags);					\
	__ret = raw_cpu_generic_add_return(pcp, val);			\
	raw_local_irq_restore(__flags);					\
	__ret;								\
})

#define this_cpu_generic_xchg(pcp, nval)				\
({									\
	typeof(pcp) __ret;						\
	unsigned long __flags;						\
	raw_local_irq_save(__flags);					\
	__ret = raw_cpu_generic_xchg(pcp, nval);			\
	raw_local_irq_restore(__flags);					\
	__ret;								\
})

#define this_cpu_generic_cmpxchg(pcp, oval, nval)			\
({									\
	typeof(pcp) __ret;						\
	unsigned long __flags;						\
	raw_local_irq_save(__flags);					\
	__ret = raw_cpu_generic_cmpxchg(pcp, oval, nval);		\
	raw_local_irq_restore(__flags);					\
	__ret;								\
})

#define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)	\
({									\
	int __ret;							\
	unsigned long __flags;						\
	raw_local_irq_save(__flags);					\
	__ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2,		\
			oval1, oval2, nval1, nval2);			\
	raw_local_irq_restore(__flags);					\
	__ret;								\
})

#ifndef raw_cpu_read_1
#define raw_cpu_read_1(pcp)		raw_cpu_generic_read(pcp)
#endif
#ifndef raw_cpu_read_2
#define raw_cpu_read_2(pcp)		raw_cpu_generic_read(pcp)
#endif
#ifndef raw_cpu_read_4
#define raw_cpu_read_4(pcp)		raw_cpu_generic_read(pcp)
#endif
#ifndef raw_cpu_read_8
#define raw_cpu_read_8(pcp)		raw_cpu_generic_read(pcp)
#endif

#ifndef raw_cpu_write_1
#define raw_cpu_write_1(pcp, val)	raw_cpu_generic_to_op(pcp, val, =)
#endif
#ifndef raw_cpu_write_2
#define raw_cpu_write_2(pcp, val)	raw_cpu_generic_to_op(pcp, val, =)
#endif
#ifndef raw_cpu_write_4
#define raw_cpu_write_4(pcp, val)	raw_cpu_generic_to_op(pcp, val, =)
#endif
#ifndef raw_cpu_write_8
#define raw_cpu_write_8(pcp, val)	raw_cpu_generic_to_op(pcp, val, =)
#endif

#ifndef raw_cpu_add_1
#define raw_cpu_add_1(pcp, val)		raw_cpu_generic_to_op(pcp, val, +=)
#endif
#ifndef raw_cpu_add_2
#define raw_cpu_add_2(pcp, val)		raw_cpu_generic_to_op(pcp, val, +=)
#endif
#ifndef raw_cpu_add_4
#define raw_cpu_add_4(pcp, val)		raw_cpu_generic_to_op(pcp, val, +=)
#endif
#ifndef raw_cpu_add_8
#define raw_cpu_add_8(pcp, val)		raw_cpu_generic_to_op(pcp, val, +=)
#endif

#ifndef raw_cpu_and_1
#define raw_cpu_and_1(pcp, val)		raw_cpu_generic_to_op(pcp, val, &=)
#endif
#ifndef raw_cpu_and_2
#define raw_cpu_and_2(pcp, val)		raw_cpu_generic_to_op(pcp, val, &=)
#endif
#ifndef raw_cpu_and_4
#define raw_cpu_and_4(pcp, val)		raw_cpu_generic_to_op(pcp, val, &=)
#endif
#ifndef raw_cpu_and_8
#define raw_cpu_and_8(pcp, val)		raw_cpu_generic_to_op(pcp, val, &=)
#endif

#ifndef raw_cpu_or_1
#define raw_cpu_or_1(pcp, val)		raw_cpu_generic_to_op(pcp, val, |=)
#endif
#ifndef raw_cpu_or_2
#define raw_cpu_or_2(pcp, val)		raw_cpu_generic_to_op(pcp, val, |=)
#endif
#ifndef raw_cpu_or_4
#define raw_cpu_or_4(pcp, val)		raw_cpu_generic_to_op(pcp, val, |=)
#endif
#ifndef raw_cpu_or_8
#define raw_cpu_or_8(pcp, val)		raw_cpu_generic_to_op(pcp, val, |=)
#endif

#ifndef raw_cpu_add_return_1
#define raw_cpu_add_return_1(pcp, val)	raw_cpu_generic_add_return(pcp, val)
#endif
#ifndef raw_cpu_add_return_2
#define raw_cpu_add_return_2(pcp, val)	raw_cpu_generic_add_return(pcp, val)
#endif
#ifndef raw_cpu_add_return_4
#define raw_cpu_add_return_4(pcp, val)	raw_cpu_generic_add_return(pcp, val)
#endif
#ifndef raw_cpu_add_return_8
#define raw_cpu_add_return_8(pcp, val)	raw_cpu_generic_add_return(pcp, val)
#endif

#ifndef raw_cpu_xchg_1
#define raw_cpu_xchg_1(pcp, nval)	raw_cpu_generic_xchg(pcp, nval)
#endif
#ifndef raw_cpu_xchg_2
#define raw_cpu_xchg_2(pcp, nval)	raw_cpu_generic_xchg(pcp, nval)
#endif
#ifndef raw_cpu_xchg_4
#define raw_cpu_xchg_4(pcp, nval)	raw_cpu_generic_xchg(pcp, nval)
#endif
#ifndef raw_cpu_xchg_8
#define raw_cpu_xchg_8(pcp, nval)	raw_cpu_generic_xchg(pcp, nval)
#endif

#ifndef raw_cpu_cmpxchg_1
#define raw_cpu_cmpxchg_1(pcp, oval, nval) \
	raw_cpu_generic_cmpxchg(pcp, oval, nval)
#endif
#ifndef raw_cpu_cmpxchg_2
#define raw_cpu_cmpxchg_2(pcp, oval, nval) \
	raw_cpu_generic_cmpxchg(pcp, oval, nval)
#endif
#ifndef raw_cpu_cmpxchg_4
#define raw_cpu_cmpxchg_4(pcp, oval, nval) \
	raw_cpu_generic_cmpxchg(pcp, oval, nval)
#endif
#ifndef raw_cpu_cmpxchg_8
#define raw_cpu_cmpxchg_8(pcp, oval, nval) \
	raw_cpu_generic_cmpxchg(pcp, oval, nval)
#endif

#ifndef raw_cpu_cmpxchg_double_1
#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
#endif
#ifndef raw_cpu_cmpxchg_double_2
#define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
#endif
#ifndef raw_cpu_cmpxchg_double_4
#define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
#endif
#ifndef raw_cpu_cmpxchg_double_8
#define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
#endif

#ifndef this_cpu_read_1
#define this_cpu_read_1(pcp)		this_cpu_generic_read(pcp)
#endif
#ifndef this_cpu_read_2
#define this_cpu_read_2(pcp)		this_cpu_generic_read(pcp)
#endif
#ifndef this_cpu_read_4
#define this_cpu_read_4(pcp)		this_cpu_generic_read(pcp)
#endif
#ifndef this_cpu_read_8
#define this_cpu_read_8(pcp)		this_cpu_generic_read(pcp)
#endif

#ifndef this_cpu_write_1
#define this_cpu_write_1(pcp, val)	this_cpu_generic_to_op(pcp, val, =)
#endif
#ifndef this_cpu_write_2
#define this_cpu_write_2(pcp, val)	this_cpu_generic_to_op(pcp, val, =)
#endif
#ifndef this_cpu_write_4
#define this_cpu_write_4(pcp, val)	this_cpu_generic_to_op(pcp, val, =)
#endif
#ifndef this_cpu_write_8
#define this_cpu_write_8(pcp, val)	this_cpu_generic_to_op(pcp, val, =)
#endif

#ifndef this_cpu_add_1
#define this_cpu_add_1(pcp, val)	this_cpu_generic_to_op(pcp, val, +=)
#endif
#ifndef this_cpu_add_2
#define this_cpu_add_2(pcp, val)	this_cpu_generic_to_op(pcp, val, +=)
#endif
#ifndef this_cpu_add_4
#define this_cpu_add_4(pcp, val)	this_cpu_generic_to_op(pcp, val, +=)
#endif
#ifndef this_cpu_add_8
#define this_cpu_add_8(pcp, val)	this_cpu_generic_to_op(pcp, val, +=)
#endif

#ifndef this_cpu_and_1
#define this_cpu_and_1(pcp, val)	this_cpu_generic_to_op(pcp, val, &=)
#endif
#ifndef this_cpu_and_2
#define this_cpu_and_2(pcp, val)	this_cpu_generic_to_op(pcp, val, &=)
#endif
#ifndef this_cpu_and_4
#define this_cpu_and_4(pcp, val)	this_cpu_generic_to_op(pcp, val, &=)
#endif
#ifndef this_cpu_and_8
#define this_cpu_and_8(pcp, val)	this_cpu_generic_to_op(pcp, val, &=)
#endif

#ifndef this_cpu_or_1
#define this_cpu_or_1(pcp, val)		this_cpu_generic_to_op(pcp, val, |=)
#endif
#ifndef this_cpu_or_2
#define this_cpu_or_2(pcp, val)		this_cpu_generic_to_op(pcp, val, |=)
#endif
#ifndef this_cpu_or_4
#define this_cpu_or_4(pcp, val)		this_cpu_generic_to_op(pcp, val, |=)
#endif
#ifndef this_cpu_or_8
#define this_cpu_or_8(pcp, val)		this_cpu_generic_to_op(pcp, val, |=)
#endif

#ifndef this_cpu_add_return_1
#define this_cpu_add_return_1(pcp, val)	this_cpu_generic_add_return(pcp, val)
#endif
#ifndef this_cpu_add_return_2
#define this_cpu_add_return_2(pcp, val)	this_cpu_generic_add_return(pcp, val)
#endif
#ifndef this_cpu_add_return_4
#define this_cpu_add_return_4(pcp, val)	this_cpu_generic_add_return(pcp, val)
#endif
#ifndef this_cpu_add_return_8
#define this_cpu_add_return_8(pcp, val)	this_cpu_generic_add_return(pcp, val)
#endif

#ifndef this_cpu_xchg_1
#define this_cpu_xchg_1(pcp, nval)	this_cpu_generic_xchg(pcp, nval)
#endif
#ifndef this_cpu_xchg_2
#define this_cpu_xchg_2(pcp, nval)	this_cpu_generic_xchg(pcp, nval)
#endif
#ifndef this_cpu_xchg_4
#define this_cpu_xchg_4(pcp, nval)	this_cpu_generic_xchg(pcp, nval)
#endif
#ifndef this_cpu_xchg_8
#define this_cpu_xchg_8(pcp, nval)	this_cpu_generic_xchg(pcp, nval)
#endif

#ifndef this_cpu_cmpxchg_1
#define this_cpu_cmpxchg_1(pcp, oval, nval) \
	this_cpu_generic_cmpxchg(pcp, oval, nval)
#endif
#ifndef this_cpu_cmpxchg_2
#define this_cpu_cmpxchg_2(pcp, oval, nval) \
	this_cpu_generic_cmpxchg(pcp, oval, nval)
#endif
#ifndef this_cpu_cmpxchg_4
#define this_cpu_cmpxchg_4(pcp, oval, nval) \
	this_cpu_generic_cmpxchg(pcp, oval, nval)
#endif
#ifndef this_cpu_cmpxchg_8
#define this_cpu_cmpxchg_8(pcp, oval, nval) \
	this_cpu_generic_cmpxchg(pcp, oval, nval)
#endif

#ifndef this_cpu_cmpxchg_double_1
#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
#endif
#ifndef this_cpu_cmpxchg_double_2
#define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
#endif
#ifndef this_cpu_cmpxchg_double_4
#define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
#endif
#ifndef this_cpu_cmpxchg_double_8
#define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
	this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
#endif

#endif /* _ASM_GENERIC_PERCPU_H_ */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
1da177e4 LT	2	#ifndef _ASM_GENERIC_PERCPU_H_
1da177e4 LT	3	#define _ASM_GENERIC_PERCPU_H_
5028eaa9	4
1da177e4	5	#include <linux/compiler.h>
ae1ee11b	6	#include <linux/threads.h>
5028eaa9	7	#include <linux/percpu-defs.h>
acdac872	8
1da177e4 LT	9	#ifdef CONFIG_SMP
1da177e4 LT	10
acdac872	11	/*
	12	* per_cpu_offset() is the offset that has to be added to a
	13	* percpu variable to get to the instance for a certain processor.
	14	*
	15	* Most arches use the __per_cpu_offset array for those offsets but
	16	* some arches have their own ways of determining the offset (x86_64, s390).
	17	*/
	18	#ifndef __per_cpu_offset
1da177e4 LT	19	extern unsigned long __per_cpu_offset[NR_CPUS];
1da177e4 LT	20
a875a69f	21	#define per_cpu_offset(x) (__per_cpu_offset[x])
acdac872	22	#endif
	23
	24	/*
	25	* Determine the offset for the currently active processor.
	26	* An arch may define __my_cpu_offset to provide a more effective
	27	* means of obtaining the offset to the per cpu variables of the
	28	* current processor.
	29	*/
	30	#ifndef __my_cpu_offset
	31	#define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
1e835278 HD	32	#endif
1e835278 HD	33	#ifdef CONFIG_DEBUG_PREEMPT
acdac872	34	#define my_cpu_offset per_cpu_offset(smp_processor_id())
	35	#else
	36	#define my_cpu_offset __my_cpu_offset
	37	#endif
	38
bbc344e1 TH	39	/*
	40	* Arch may define arch_raw_cpu_ptr() to provide more efficient address
	41	* translations for raw_cpu_ptr().
	42	*/
	43	#ifndef arch_raw_cpu_ptr
	44	#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
db7829c6	45	#endif
bbc344e1	46
dd5af90a	47	#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
acdac872	48	extern void setup_per_cpu_areas(void);
acdac872	49	#endif
1da177e4	50
1da177e4 LT	51	#endif /* SMP */
1da177e4 LT	52
9b8de747 DH	53	#ifndef PER_CPU_BASE_SECTION
9b8de747 DH	54	#ifdef CONFIG_SMP
3d9a854c	55	#define PER_CPU_BASE_SECTION ".data..percpu"
9b8de747 DH	56	#else
	57	#define PER_CPU_BASE_SECTION ".data"
	58	#endif
	59	#endif
	60
acdac872	61	#ifndef PER_CPU_ATTRIBUTES
	62	#define PER_CPU_ATTRIBUTES
	63	#endif
	64
b01e8dc3 TH	65	#ifndef PER_CPU_DEF_ATTRIBUTES
	66	#define PER_CPU_DEF_ATTRIBUTES
	67	#endif
	68
1b5ca121 NP	69	#define raw_cpu_generic_read(pcp) \
	70	({ \
	71	*raw_cpu_ptr(&(pcp)); \
	72	})
	73
9c28278a TH	74	#define raw_cpu_generic_to_op(pcp, val, op) \
	75	do { \
	76	*raw_cpu_ptr(&(pcp)) op val; \
	77	} while (0)
	78
	79	#define raw_cpu_generic_add_return(pcp, val) \
	80	({ \
1b5ca121 NP	81	typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \
	82	\
	83	*__p += val; \
	84	*__p; \
9c28278a TH	85	})
	86
	87	#define raw_cpu_generic_xchg(pcp, nval) \
eba11788	88	({ \
1b5ca121	89	typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \
eba11788	90	typeof(pcp) __ret; \
1b5ca121 NP	91	__ret = *__p; \
1b5ca121 NP	92	*__p = nval; \
eba11788	93	__ret; \
9c28278a TH	94	})
	95
	96	#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
	97	({ \
1b5ca121	98	typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \
eba11788	99	typeof(pcp) __ret; \
1b5ca121	100	__ret = *__p; \
eba11788	101	if (__ret == (oval)) \
1b5ca121	102	*__p = nval; \
eba11788	103	__ret; \
9c28278a TH	104	})
9c28278a TH	105
eba11788	106	#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
9c28278a	107	({ \
1b5ca121 NP	108	typeof(&(pcp1)) __p1 = raw_cpu_ptr(&(pcp1)); \
1b5ca121 NP	109	typeof(&(pcp2)) __p2 = raw_cpu_ptr(&(pcp2)); \
9c28278a	110	int __ret = 0; \
1b5ca121 NP	111	if (__p1 == (oval1) && __p2 == (oval2)) { \
	112	*__p1 = nval1; \
	113	*__p2 = nval2; \
9c28278a TH	114	__ret = 1; \
	115	} \
	116	(__ret); \
	117	})
	118
e88d62cd	119	#define __this_cpu_generic_read_nopreempt(pcp) \
eba11788 TH	120	({ \
eba11788 TH	121	typeof(pcp) __ret; \
7f8d61f0	122	preempt_disable_notrace(); \
e88d62cd	123	__ret = READ_ONCE(*raw_cpu_ptr(&(pcp))); \
7f8d61f0	124	preempt_enable_notrace(); \
eba11788	125	__ret; \
9c28278a TH	126	})
9c28278a TH	127
e88d62cd MR	128	#define __this_cpu_generic_read_noirq(pcp) \
	129	({ \
	130	typeof(pcp) __ret; \
	131	unsigned long __flags; \
	132	raw_local_irq_save(__flags); \
	133	__ret = raw_cpu_generic_read(pcp); \
	134	raw_local_irq_restore(__flags); \
	135	__ret; \
	136	})
	137
	138	#define this_cpu_generic_read(pcp) \
	139	({ \
	140	typeof(pcp) __ret; \
	141	if (__native_word(pcp)) \
	142	__ret = __this_cpu_generic_read_nopreempt(pcp); \
	143	else \
	144	__ret = __this_cpu_generic_read_noirq(pcp); \
	145	__ret; \
	146	})
	147
eba11788	148	#define this_cpu_generic_to_op(pcp, val, op) \
9c28278a	149	do { \
eba11788 TH	150	unsigned long __flags; \
eba11788 TH	151	raw_local_irq_save(__flags); \
1b5ca121	152	raw_cpu_generic_to_op(pcp, val, op); \
eba11788	153	raw_local_irq_restore(__flags); \
9c28278a TH	154	} while (0)
9c28278a TH	155
1b5ca121	156
eba11788	157	#define this_cpu_generic_add_return(pcp, val) \
9c28278a	158	({ \
eba11788 TH	159	typeof(pcp) __ret; \
	160	unsigned long __flags; \
	161	raw_local_irq_save(__flags); \
1b5ca121	162	__ret = raw_cpu_generic_add_return(pcp, val); \
eba11788 TH	163	raw_local_irq_restore(__flags); \
eba11788 TH	164	__ret; \
9c28278a TH	165	})
9c28278a TH	166
eba11788 TH	167	#define this_cpu_generic_xchg(pcp, nval) \
	168	({ \
	169	typeof(pcp) __ret; \
	170	unsigned long __flags; \
	171	raw_local_irq_save(__flags); \
1b5ca121	172	__ret = raw_cpu_generic_xchg(pcp, nval); \
eba11788 TH	173	raw_local_irq_restore(__flags); \
eba11788 TH	174	__ret; \
9c28278a TH	175	})
9c28278a TH	176
eba11788	177	#define this_cpu_generic_cmpxchg(pcp, oval, nval) \
9c28278a	178	({ \
eba11788 TH	179	typeof(pcp) __ret; \
	180	unsigned long __flags; \
	181	raw_local_irq_save(__flags); \
1b5ca121	182	__ret = raw_cpu_generic_cmpxchg(pcp, oval, nval); \
eba11788 TH	183	raw_local_irq_restore(__flags); \
eba11788 TH	184	__ret; \
9c28278a TH	185	})
9c28278a TH	186
eba11788	187	#define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
9c28278a	188	({ \
eba11788 TH	189	int __ret; \
	190	unsigned long __flags; \
	191	raw_local_irq_save(__flags); \
	192	__ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \
9c28278a	193	oval1, oval2, nval1, nval2); \
eba11788 TH	194	raw_local_irq_restore(__flags); \
eba11788 TH	195	__ret; \
9c28278a TH	196	})
9c28278a TH	197
eba11788	198	#ifndef raw_cpu_read_1
1b5ca121	199	#define raw_cpu_read_1(pcp) raw_cpu_generic_read(pcp)
eba11788 TH	200	#endif
eba11788 TH	201	#ifndef raw_cpu_read_2
1b5ca121	202	#define raw_cpu_read_2(pcp) raw_cpu_generic_read(pcp)
eba11788 TH	203	#endif
eba11788 TH	204	#ifndef raw_cpu_read_4
1b5ca121	205	#define raw_cpu_read_4(pcp) raw_cpu_generic_read(pcp)
eba11788 TH	206	#endif
eba11788 TH	207	#ifndef raw_cpu_read_8
1b5ca121	208	#define raw_cpu_read_8(pcp) raw_cpu_generic_read(pcp)
eba11788 TH	209	#endif
	210
	211	#ifndef raw_cpu_write_1
	212	#define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
	213	#endif
	214	#ifndef raw_cpu_write_2
	215	#define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
	216	#endif
	217	#ifndef raw_cpu_write_4
	218	#define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
	219	#endif
	220	#ifndef raw_cpu_write_8
	221	#define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
	222	#endif
	223
	224	#ifndef raw_cpu_add_1
	225	#define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
	226	#endif
	227	#ifndef raw_cpu_add_2
	228	#define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
	229	#endif
	230	#ifndef raw_cpu_add_4
	231	#define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
	232	#endif
	233	#ifndef raw_cpu_add_8
	234	#define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
	235	#endif
	236
	237	#ifndef raw_cpu_and_1
	238	#define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
	239	#endif
	240	#ifndef raw_cpu_and_2
	241	#define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
	242	#endif
	243	#ifndef raw_cpu_and_4
	244	#define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
	245	#endif
	246	#ifndef raw_cpu_and_8
	247	#define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
	248	#endif
	249
	250	#ifndef raw_cpu_or_1
	251	#define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op(pcp, val, \|=)
	252	#endif
	253	#ifndef raw_cpu_or_2
	254	#define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op(pcp, val, \|=)
	255	#endif
	256	#ifndef raw_cpu_or_4
	257	#define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op(pcp, val, \|=)
	258	#endif
	259	#ifndef raw_cpu_or_8
	260	#define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op(pcp, val, \|=)
	261	#endif
	262
	263	#ifndef raw_cpu_add_return_1
	264	#define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val)
	265	#endif
	266	#ifndef raw_cpu_add_return_2
	267	#define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val)
	268	#endif
	269	#ifndef raw_cpu_add_return_4
	270	#define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val)
	271	#endif
	272	#ifndef raw_cpu_add_return_8
273	#define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val)
274	#endif
275
276	#ifndef raw_cpu_xchg_1
277	#define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
278	#endif
279	#ifndef raw_cpu_xchg_2
280	#define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
281	#endif
282	#ifndef raw_cpu_xchg_4
283	#define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
284	#endif
285	#ifndef raw_cpu_xchg_8
286	#define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
287	#endif
288
289	#ifndef raw_cpu_cmpxchg_1
290	#define raw_cpu_cmpxchg_1(pcp, oval, nval) \
291	raw_cpu_generic_cmpxchg(pcp, oval, nval)
292	#endif
293	#ifndef raw_cpu_cmpxchg_2
294	#define raw_cpu_cmpxchg_2(pcp, oval, nval) \
295	raw_cpu_generic_cmpxchg(pcp, oval, nval)
296	#endif
297	#ifndef raw_cpu_cmpxchg_4
298	#define raw_cpu_cmpxchg_4(pcp, oval, nval) \
299	raw_cpu_generic_cmpxchg(pcp, oval, nval)
300	#endif
301	#ifndef raw_cpu_cmpxchg_8
302	#define raw_cpu_cmpxchg_8(pcp, oval, nval) \
303	raw_cpu_generic_cmpxchg(pcp, oval, nval)
304	#endif
305
306	#ifndef raw_cpu_cmpxchg_double_1
307	#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
47b69ad6	308	raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
eba11788 TH	309	#endif
	310	#ifndef raw_cpu_cmpxchg_double_2
	311	#define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
47b69ad6	312	raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
eba11788 TH	313	#endif
	314	#ifndef raw_cpu_cmpxchg_double_4
	315	#define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
47b69ad6	316	raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
eba11788 TH	317	#endif
	318	#ifndef raw_cpu_cmpxchg_double_8
	319	#define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
47b69ad6	320	raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
eba11788 TH	321	#endif
	322
	323	#ifndef this_cpu_read_1
	324	#define this_cpu_read_1(pcp) this_cpu_generic_read(pcp)
	325	#endif
	326	#ifndef this_cpu_read_2
	327	#define this_cpu_read_2(pcp) this_cpu_generic_read(pcp)
	328	#endif
	329	#ifndef this_cpu_read_4
	330	#define this_cpu_read_4(pcp) this_cpu_generic_read(pcp)
	331	#endif
	332	#ifndef this_cpu_read_8
	333	#define this_cpu_read_8(pcp) this_cpu_generic_read(pcp)
	334	#endif
	335
	336	#ifndef this_cpu_write_1
	337	#define this_cpu_write_1(pcp, val) this_cpu_generic_to_op(pcp, val, =)
	338	#endif
	339	#ifndef this_cpu_write_2
	340	#define this_cpu_write_2(pcp, val) this_cpu_generic_to_op(pcp, val, =)
	341	#endif
	342	#ifndef this_cpu_write_4
	343	#define this_cpu_write_4(pcp, val) this_cpu_generic_to_op(pcp, val, =)
	344	#endif
	345	#ifndef this_cpu_write_8
	346	#define this_cpu_write_8(pcp, val) this_cpu_generic_to_op(pcp, val, =)
	347	#endif
	348
	349	#ifndef this_cpu_add_1
	350	#define this_cpu_add_1(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
	351	#endif
	352	#ifndef this_cpu_add_2
	353	#define this_cpu_add_2(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
	354	#endif
	355	#ifndef this_cpu_add_4
	356	#define this_cpu_add_4(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
	357	#endif
	358	#ifndef this_cpu_add_8
	359	#define this_cpu_add_8(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
	360	#endif
	361
	362	#ifndef this_cpu_and_1
	363	#define this_cpu_and_1(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
	364	#endif
	365	#ifndef this_cpu_and_2
	366	#define this_cpu_and_2(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
	367	#endif
	368	#ifndef this_cpu_and_4
	369	#define this_cpu_and_4(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
	370	#endif
	371	#ifndef this_cpu_and_8
	372	#define this_cpu_and_8(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
	373	#endif
	374
	375	#ifndef this_cpu_or_1
	376	#define this_cpu_or_1(pcp, val) this_cpu_generic_to_op(pcp, val, \|=)
	377	#endif
	378	#ifndef this_cpu_or_2
	379	#define this_cpu_or_2(pcp, val) this_cpu_generic_to_op(pcp, val, \|=)
	380	#endif
	381	#ifndef this_cpu_or_4
	382	#define this_cpu_or_4(pcp, val) this_cpu_generic_to_op(pcp, val, \|=)
	383	#endif
	384	#ifndef this_cpu_or_8
385	#define this_cpu_or_8(pcp, val) this_cpu_generic_to_op(pcp, val, \|=)
386	#endif
387
388	#ifndef this_cpu_add_return_1
389	#define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val)
390	#endif
391	#ifndef this_cpu_add_return_2
392	#define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val)
393	#endif
394	#ifndef this_cpu_add_return_4
395	#define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val)
396	#endif
397	#ifndef this_cpu_add_return_8
398	#define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val)
399	#endif
400
401	#ifndef this_cpu_xchg_1
402	#define this_cpu_xchg_1(pcp, nval) this_cpu_generic_xchg(pcp, nval)
403	#endif
404	#ifndef this_cpu_xchg_2
405	#define this_cpu_xchg_2(pcp, nval) this_cpu_generic_xchg(pcp, nval)
406	#endif
407	#ifndef this_cpu_xchg_4
408	#define this_cpu_xchg_4(pcp, nval) this_cpu_generic_xchg(pcp, nval)
409	#endif
410	#ifndef this_cpu_xchg_8
411	#define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval)
412	#endif
413
414	#ifndef this_cpu_cmpxchg_1
415	#define this_cpu_cmpxchg_1(pcp, oval, nval) \
416	this_cpu_generic_cmpxchg(pcp, oval, nval)
417	#endif
418	#ifndef this_cpu_cmpxchg_2
419	#define this_cpu_cmpxchg_2(pcp, oval, nval) \
420	this_cpu_generic_cmpxchg(pcp, oval, nval)
421	#endif
422	#ifndef this_cpu_cmpxchg_4
423	#define this_cpu_cmpxchg_4(pcp, oval, nval) \
424	this_cpu_generic_cmpxchg(pcp, oval, nval)
425	#endif
426	#ifndef this_cpu_cmpxchg_8
427	#define this_cpu_cmpxchg_8(pcp, oval, nval) \
428	this_cpu_generic_cmpxchg(pcp, oval, nval)
429	#endif
430
431	#ifndef this_cpu_cmpxchg_double_1
432	#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
433	this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
434	#endif
435	#ifndef this_cpu_cmpxchg_double_2
436	#define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
437	this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
438	#endif
439	#ifndef this_cpu_cmpxchg_double_4
440	#define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
441	this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
442	#endif
443	#ifndef this_cpu_cmpxchg_double_8
444	#define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
445	this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
446	#endif
47b69ad6	447
1da177e4	448	#endif /* _ASM_GENERIC_PERCPU_H_ */