[mirror_ubuntu-jammy-kernel.git] / include / asm-i386 / percpu.h

#ifndef __ARCH_I386_PERCPU__
#define __ARCH_I386_PERCPU__

#ifdef __ASSEMBLY__

/*
 * PER_CPU finds an address of a per-cpu variable.
 *
 * Args:
 *    var - variable name
 *    reg - 32bit register
 *
 * The resulting address is stored in the "reg" argument.
 *
 * Example:
 *    PER_CPU(cpu_gdt_descr, %ebx)
 */
#ifdef CONFIG_SMP
#define PER_CPU(var, reg)			\
	movl %fs:per_cpu__this_cpu_off, reg;		\
	addl $per_cpu__##var, reg
#else /* ! SMP */
#define PER_CPU(var, reg) \
	movl $per_cpu__##var, reg;
#endif	/* SMP */

#else /* ...!ASSEMBLY */

/*
 * PER_CPU finds an address of a per-cpu variable.
 *
 * Args:
 *    var - variable name
 *    cpu - 32bit register containing the current CPU number
 *
 * The resulting address is stored in the "cpu" argument.
 *
 * Example:
 *    PER_CPU(cpu_gdt_descr, %ebx)
 */
#ifdef CONFIG_SMP
/* Same as generic implementation except for optimized local access. */
#define __GENERIC_PER_CPU

/* This is used for other cpus to find our section. */
extern unsigned long __per_cpu_offset[];

/* Separate out the type, so (int[3], foo) works. */
#define DECLARE_PER_CPU(type, name) extern __typeof__(type) per_cpu__##name
#define DEFINE_PER_CPU(type, name) \
    __attribute__((__section__(".data.percpu"))) __typeof__(type) per_cpu__##name

/* We can use this directly for local CPU (faster). */
DECLARE_PER_CPU(unsigned long, this_cpu_off);

/* var is in discarded region: offset to particular copy we want */
#define per_cpu(var, cpu) (*({				\
	extern int simple_indentifier_##var(void);	\
	RELOC_HIDE(&per_cpu__##var, __per_cpu_offset[cpu]); }))

#define __raw_get_cpu_var(var) (*({					\
	extern int simple_indentifier_##var(void);			\
	RELOC_HIDE(&per_cpu__##var, x86_read_percpu(this_cpu_off));	\
}))

#define __get_cpu_var(var) __raw_get_cpu_var(var)

/* A macro to avoid #include hell... */
#define percpu_modcopy(pcpudst, src, size)			\
do {								\
	unsigned int __i;					\
	for_each_possible_cpu(__i)				\
		memcpy((pcpudst)+__per_cpu_offset[__i],		\
		       (src), (size));				\
} while (0)

#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)

/* fs segment starts at (positive) offset == __per_cpu_offset[cpu] */
#define __percpu_seg "%%fs:"
#else  /* !SMP */
#include <asm-generic/percpu.h>
#define __percpu_seg ""
#endif	/* SMP */

/* For arch-specific code, we can use direct single-insn ops (they
 * don't give an lvalue though). */
extern void __bad_percpu_size(void);

#define percpu_to_op(op,var,val)				\
	do {							\
		typedef typeof(var) T__;			\
		if (0) { T__ tmp__; tmp__ = (val); }		\
		switch (sizeof(var)) {				\
		case 1:						\
			asm(op "b %1,"__percpu_seg"%0"		\
			    : "+m" (var)			\
			    :"ri" ((T__)val));			\
			break;					\
		case 2:						\
			asm(op "w %1,"__percpu_seg"%0"		\
			    : "+m" (var)			\
			    :"ri" ((T__)val));			\
			break;					\
		case 4:						\
			asm(op "l %1,"__percpu_seg"%0"		\
			    : "+m" (var)			\
			    :"ri" ((T__)val));			\
			break;					\
		default: __bad_percpu_size();			\
		}						\
	} while (0)

#define percpu_from_op(op,var)					\
	({							\
		typeof(var) ret__;				\
		switch (sizeof(var)) {				\
		case 1:						\
			asm(op "b "__percpu_seg"%1,%0"		\
			    : "=r" (ret__)			\
			    : "m" (var));			\
			break;					\
		case 2:						\
			asm(op "w "__percpu_seg"%1,%0"		\
			    : "=r" (ret__)			\
			    : "m" (var));			\
			break;					\
		case 4:						\
			asm(op "l "__percpu_seg"%1,%0"		\
			    : "=r" (ret__)			\
			    : "m" (var));			\
			break;					\
		default: __bad_percpu_size();			\
		}						\
		ret__; })

#define x86_read_percpu(var) percpu_from_op("mov", per_cpu__##var)
#define x86_write_percpu(var,val) percpu_to_op("mov", per_cpu__##var, val)
#define x86_add_percpu(var,val) percpu_to_op("add", per_cpu__##var, val)
#define x86_sub_percpu(var,val) percpu_to_op("sub", per_cpu__##var, val)
#define x86_or_percpu(var,val) percpu_to_op("or", per_cpu__##var, val)
#endif /* !__ASSEMBLY__ */

#endif /* __ARCH_I386_PERCPU__ */
Commit	Line	Data
1da177e4 LT	1	#ifndef __ARCH_I386_PERCPU__
	2	#define __ARCH_I386_PERCPU__
	3
7c3576d2 JF	4	#ifdef __ASSEMBLY__
	5
	6	/*
	7	* PER_CPU finds an address of a per-cpu variable.
	8	*
	9	* Args:
	10	* var - variable name
	11	* reg - 32bit register
	12	*
	13	* The resulting address is stored in the "reg" argument.
	14	*
	15	* Example:
	16	* PER_CPU(cpu_gdt_descr, %ebx)
	17	*/
	18	#ifdef CONFIG_SMP
	19	#define PER_CPU(var, reg) \
	20	movl %fs:per_cpu__this_cpu_off, reg; \
	21	addl $per_cpu__##var, reg
	22	#else /* ! SMP */
	23	#define PER_CPU(var, reg) \
	24	movl $per_cpu__##var, reg;
	25	#endif /* SMP */
	26
	27	#else /* ...!ASSEMBLY */
be44d2aa SS	28
	29	/*
	30	* PER_CPU finds an address of a per-cpu variable.
	31	*
	32	* Args:
	33	* var - variable name
	34	* cpu - 32bit register containing the current CPU number
	35	*
	36	* The resulting address is stored in the "cpu" argument.
	37	*
	38	* Example:
	39	* PER_CPU(cpu_gdt_descr, %ebx)
	40	*/
	41	#ifdef CONFIG_SMP
7c3576d2 JF	42	/* Same as generic implementation except for optimized local access. */
	43	#define __GENERIC_PER_CPU
	44
	45	/* This is used for other cpus to find our section. */
	46	extern unsigned long __per_cpu_offset[];
	47
	48	/* Separate out the type, so (int[3], foo) works. */
	49	#define DECLARE_PER_CPU(type, name) extern __typeof__(type) per_cpu__##name
	50	#define DEFINE_PER_CPU(type, name) \
	51	__attribute__((__section__(".data.percpu"))) __typeof__(type) per_cpu__##name
	52
	53	/* We can use this directly for local CPU (faster). */
	54	DECLARE_PER_CPU(unsigned long, this_cpu_off);
	55
	56	/* var is in discarded region: offset to particular copy we want */
	57	#define per_cpu(var, cpu) (*({ \
	58	extern int simple_indentifier_##var(void); \
	59	RELOC_HIDE(&per_cpu__##var, __per_cpu_offset[cpu]); }))
	60
	61	#define __raw_get_cpu_var(var) (*({ \
	62	extern int simple_indentifier_##var(void); \
	63	RELOC_HIDE(&per_cpu__##var, x86_read_percpu(this_cpu_off)); \
	64	}))
	65
	66	#define __get_cpu_var(var) __raw_get_cpu_var(var)
	67
	68	/* A macro to avoid #include hell... */
	69	#define percpu_modcopy(pcpudst, src, size) \
	70	do { \
	71	unsigned int __i; \
	72	for_each_possible_cpu(__i) \
	73	memcpy((pcpudst)+__per_cpu_offset[__i], \
	74	(src), (size)); \
	75	} while (0)
	76
	77	#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
	78	#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
	79
	80	/* fs segment starts at (positive) offset == __per_cpu_offset[cpu] */
	81	#define __percpu_seg "%%fs:"
	82	#else /* !SMP */
	83	#include <asm-generic/percpu.h>
	84	#define __percpu_seg ""
be44d2aa SS	85	#endif /* SMP */
be44d2aa SS	86
7c3576d2 JF	87	/* For arch-specific code, we can use direct single-insn ops (they
	88	* don't give an lvalue though). */
	89	extern void __bad_percpu_size(void);
	90
	91	#define percpu_to_op(op,var,val) \
	92	do { \
	93	typedef typeof(var) T__; \
	94	if (0) { T__ tmp__; tmp__ = (val); } \
	95	switch (sizeof(var)) { \
	96	case 1: \
	97	asm(op "b %1,"__percpu_seg"%0" \
	98	: "+m" (var) \
	99	:"ri" ((T__)val)); \
	100	break; \
	101	case 2: \
	102	asm(op "w %1,"__percpu_seg"%0" \
	103	: "+m" (var) \
	104	:"ri" ((T__)val)); \
	105	break; \
	106	case 4: \
	107	asm(op "l %1,"__percpu_seg"%0" \
	108	: "+m" (var) \
	109	:"ri" ((T__)val)); \
	110	break; \
	111	default: __bad_percpu_size(); \
	112	} \
	113	} while (0)
	114
	115	#define percpu_from_op(op,var) \
	116	({ \
	117	typeof(var) ret__; \
	118	switch (sizeof(var)) { \
	119	case 1: \
	120	asm(op "b "__percpu_seg"%1,%0" \
	121	: "=r" (ret__) \
	122	: "m" (var)); \
	123	break; \
	124	case 2: \
	125	asm(op "w "__percpu_seg"%1,%0" \
	126	: "=r" (ret__) \
	127	: "m" (var)); \
	128	break; \
	129	case 4: \
	130	asm(op "l "__percpu_seg"%1,%0" \
	131	: "=r" (ret__) \
	132	: "m" (var)); \
	133	break; \
	134	default: __bad_percpu_size(); \
	135	} \
	136	ret__; })
	137
	138	#define x86_read_percpu(var) percpu_from_op("mov", per_cpu__##var)
	139	#define x86_write_percpu(var,val) percpu_to_op("mov", per_cpu__##var, val)
	140	#define x86_add_percpu(var,val) percpu_to_op("add", per_cpu__##var, val)
	141	#define x86_sub_percpu(var,val) percpu_to_op("sub", per_cpu__##var, val)
	142	#define x86_or_percpu(var,val) percpu_to_op("or", per_cpu__##var, val)
be44d2aa	143	#endif /* !__ASSEMBLY__ */
1da177e4 LT	144
1da177e4 LT	145	#endif /* __ARCH_I386_PERCPU__ */