[mirror_ubuntu-artful-kernel.git] / arch / x86 / include / asm / system.h

#ifndef _ASM_X86_SYSTEM_H
#define _ASM_X86_SYSTEM_H

#include <asm/asm.h>
#include <asm/segment.h>
#include <asm/cpufeature.h>
#include <asm/cmpxchg.h>
#include <asm/nops.h>

#include <linux/kernel.h>
#include <linux/irqflags.h>

/* entries in ARCH_DLINFO: */
#ifdef CONFIG_IA32_EMULATION
# define AT_VECTOR_SIZE_ARCH 2
#else
# define AT_VECTOR_SIZE_ARCH 1
#endif

struct task_struct; /* one of the stranger aspects of C forward declarations */
struct task_struct *__switch_to(struct task_struct *prev,
				struct task_struct *next);
struct tss_struct;
void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
		      struct tss_struct *tss);

#ifdef CONFIG_X86_32

#ifdef CONFIG_CC_STACKPROTECTOR
#define __switch_canary							\
	"movl %P[task_canary](%[next]), %%ebx\n\t"			\
	"movl %%ebx, "__percpu_arg([stack_canary])"\n\t"
#define __switch_canary_oparam						\
	, [stack_canary] "=m" (per_cpu_var(stack_canary))
#define __switch_canary_iparam						\
	, [task_canary] "i" (offsetof(struct task_struct, stack_canary))
#else	/* CC_STACKPROTECTOR */
#define __switch_canary
#define __switch_canary_oparam
#define __switch_canary_iparam
#endif	/* CC_STACKPROTECTOR */

/*
 * Saving eflags is important. It switches not only IOPL between tasks,
 * it also protects other tasks from NT leaking through sysenter etc.
 */
#define switch_to(prev, next, last)					\
do {									\
	/*								\
	 * Context-switching clobbers all registers, so we clobber	\
	 * them explicitly, via unused output variables.		\
	 * (EAX and EBP is not listed because EBP is saved/restored	\
	 * explicitly for wchan access and EAX is the return value of	\
	 * __switch_to())						\
	 */								\
	unsigned long ebx, ecx, edx, esi, edi;				\
									\
	asm volatile("pushfl\n\t"		/* save    flags */	\
		     "pushl %%ebp\n\t"		/* save    EBP   */	\
		     "movl %%esp,%[prev_sp]\n\t"	/* save    ESP   */ \
		     "movl %[next_sp],%%esp\n\t"	/* restore ESP   */ \
		     "movl $1f,%[prev_ip]\n\t"	/* save    EIP   */	\
		     "pushl %[next_ip]\n\t"	/* restore EIP   */	\
		     __switch_canary					\
		     "jmp __switch_to\n"	/* regparm call  */	\
		     "1:\t"						\
		     "popl %%ebp\n\t"		/* restore EBP   */	\
		     "popfl\n"			/* restore flags */	\
									\
		     /* output parameters */				\
		     : [prev_sp] "=m" (prev->thread.sp),		\
		       [prev_ip] "=m" (prev->thread.ip),		\
		       "=a" (last),					\
									\
		       /* clobbered output registers: */		\
		       "=b" (ebx), "=c" (ecx), "=d" (edx),		\
		       "=S" (esi), "=D" (edi)				\
		       							\
		       __switch_canary_oparam				\
									\
		       /* input parameters: */				\
		     : [next_sp]  "m" (next->thread.sp),		\
		       [next_ip]  "m" (next->thread.ip),		\
		       							\
		       /* regparm parameters for __switch_to(): */	\
		       [prev]     "a" (prev),				\
		       [next]     "d" (next)				\
									\
		       __switch_canary_iparam				\
									\
		     : /* reloaded segment registers */			\
			"memory");					\
} while (0)

/*
 * disable hlt during certain critical i/o operations
 */
#define HAVE_DISABLE_HLT
#else
#define __SAVE(reg, offset) "movq %%" #reg ",(14-" #offset ")*8(%%rsp)\n\t"
#define __RESTORE(reg, offset) "movq (14-" #offset ")*8(%%rsp),%%" #reg "\n\t"

/* frame pointer must be last for get_wchan */
#define SAVE_CONTEXT    "pushf ; pushq %%rbp ; movq %%rsi,%%rbp\n\t"
#define RESTORE_CONTEXT "movq %%rbp,%%rsi ; popq %%rbp ; popf\t"

#define __EXTRA_CLOBBER  \
	, "rcx", "rbx", "rdx", "r8", "r9", "r10", "r11", \
	  "r12", "r13", "r14", "r15"

#ifdef CONFIG_CC_STACKPROTECTOR
#define __switch_canary							  \
	"movq %P[task_canary](%%rsi),%%r8\n\t"				  \
	"movq %%r8,"__percpu_arg([gs_canary])"\n\t"
#define __switch_canary_oparam						  \
	, [gs_canary] "=m" (per_cpu_var(irq_stack_union.stack_canary))
#define __switch_canary_iparam						  \
	, [task_canary] "i" (offsetof(struct task_struct, stack_canary))
#else	/* CC_STACKPROTECTOR */
#define __switch_canary
#define __switch_canary_oparam
#define __switch_canary_iparam
#endif	/* CC_STACKPROTECTOR */

/* Save restore flags to clear handle leaking NT */
#define switch_to(prev, next, last) \
	asm volatile(SAVE_CONTEXT					  \
	     "movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */	  \
	     "movq %P[threadrsp](%[next]),%%rsp\n\t" /* restore RSP */	  \
	     "call __switch_to\n\t"					  \
	     ".globl thread_return\n"					  \
	     "thread_return:\n\t"					  \
	     "movq "__percpu_arg([current_task])",%%rsi\n\t"		  \
	     __switch_canary						  \
	     "movq %P[thread_info](%%rsi),%%r8\n\t"			  \
	     "movq %%rax,%%rdi\n\t" 					  \
	     "testl  %[_tif_fork],%P[ti_flags](%%r8)\n\t"	  \
	     "jnz   ret_from_fork\n\t"					  \
	     RESTORE_CONTEXT						  \
	     : "=a" (last)					  	  \
	       __switch_canary_oparam					  \
	     : [next] "S" (next), [prev] "D" (prev),			  \
	       [threadrsp] "i" (offsetof(struct task_struct, thread.sp)), \
	       [ti_flags] "i" (offsetof(struct thread_info, flags)),	  \
	       [_tif_fork] "i" (_TIF_FORK),			  	  \
	       [thread_info] "i" (offsetof(struct task_struct, stack)),   \
	       [current_task] "m" (per_cpu_var(current_task))		  \
	       __switch_canary_iparam					  \
	     : "memory", "cc" __EXTRA_CLOBBER)
#endif

#ifdef __KERNEL__
#define _set_base(addr, base) do { unsigned long __pr; \
__asm__ __volatile__ ("movw %%dx,%1\n\t" \
	"rorl $16,%%edx\n\t" \
	"movb %%dl,%2\n\t" \
	"movb %%dh,%3" \
	:"=&d" (__pr) \
	:"m" (*((addr)+2)), \
	 "m" (*((addr)+4)), \
	 "m" (*((addr)+7)), \
	 "0" (base) \
	); } while (0)

#define _set_limit(addr, limit) do { unsigned long __lr; \
__asm__ __volatile__ ("movw %%dx,%1\n\t" \
	"rorl $16,%%edx\n\t" \
	"movb %2,%%dh\n\t" \
	"andb $0xf0,%%dh\n\t" \
	"orb %%dh,%%dl\n\t" \
	"movb %%dl,%2" \
	:"=&d" (__lr) \
	:"m" (*(addr)), \
	 "m" (*((addr)+6)), \
	 "0" (limit) \
	); } while (0)

#define set_base(ldt, base) _set_base(((char *)&(ldt)) , (base))
#define set_limit(ldt, limit) _set_limit(((char *)&(ldt)) , ((limit)-1))

extern void native_load_gs_index(unsigned);

/*
 * Load a segment. Fall back on loading the zero
 * segment if something goes wrong..
 */
#define loadsegment(seg, value)			\
	asm volatile("\n"			\
		     "1:\t"			\
		     "movl %k0,%%" #seg "\n"	\
		     "2:\n"			\
		     ".section .fixup,\"ax\"\n"	\
		     "3:\t"			\
		     "movl %k1, %%" #seg "\n\t"	\
		     "jmp 2b\n"			\
		     ".previous\n"		\
		     _ASM_EXTABLE(1b,3b)	\
		     : :"r" (value), "r" (0) : "memory")


/*
 * Save a segment register away
 */
#define savesegment(seg, value)				\
	asm("mov %%" #seg ",%0":"=r" (value) : : "memory")

/*
 * x86_32 user gs accessors.
 */
#ifdef CONFIG_X86_32
#ifdef CONFIG_X86_32_LAZY_GS
#define get_user_gs(regs)	(u16)({unsigned long v; savesegment(gs, v); v;})
#define set_user_gs(regs, v)	loadsegment(gs, (unsigned long)(v))
#define task_user_gs(tsk)	((tsk)->thread.gs)
#define lazy_save_gs(v)		savesegment(gs, (v))
#define lazy_load_gs(v)		loadsegment(gs, (v))
#else	/* X86_32_LAZY_GS */
#define get_user_gs(regs)	(u16)((regs)->gs)
#define set_user_gs(regs, v)	do { (regs)->gs = (v); } while (0)
#define task_user_gs(tsk)	(task_pt_regs(tsk)->gs)
#define lazy_save_gs(v)		do { } while (0)
#define lazy_load_gs(v)		do { } while (0)
#endif	/* X86_32_LAZY_GS */
#endif	/* X86_32 */

static inline unsigned long get_limit(unsigned long segment)
{
	unsigned long __limit;
	asm("lsll %1,%0" : "=r" (__limit) : "r" (segment));
	return __limit + 1;
}

static inline void native_clts(void)
{
	asm volatile("clts");
}

/*
 * Volatile isn't enough to prevent the compiler from reordering the
 * read/write functions for the control registers and messing everything up.
 * A memory clobber would solve the problem, but would prevent reordering of
 * all loads stores around it, which can hurt performance. Solution is to
 * use a variable and mimic reads and writes to it to enforce serialization
 */
static unsigned long __force_order;

static inline unsigned long native_read_cr0(void)
{
	unsigned long val;
	asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order));
	return val;
}

static inline void native_write_cr0(unsigned long val)
{
	asm volatile("mov %0,%%cr0": : "r" (val), "m" (__force_order));
}

static inline unsigned long native_read_cr2(void)
{
	unsigned long val;
	asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order));
	return val;
}

static inline void native_write_cr2(unsigned long val)
{
	asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order));
}

static inline unsigned long native_read_cr3(void)
{
	unsigned long val;
	asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order));
	return val;
}

static inline void native_write_cr3(unsigned long val)
{
	asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order));
}

static inline unsigned long native_read_cr4(void)
{
	unsigned long val;
	asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order));
	return val;
}

static inline unsigned long native_read_cr4_safe(void)
{
	unsigned long val;
	/* This could fault if %cr4 does not exist. In x86_64, a cr4 always
	 * exists, so it will never fail. */
#ifdef CONFIG_X86_32
	asm volatile("1: mov %%cr4, %0\n"
		     "2:\n"
		     _ASM_EXTABLE(1b, 2b)
		     : "=r" (val), "=m" (__force_order) : "0" (0));
#else
	val = native_read_cr4();
#endif
	return val;
}

static inline void native_write_cr4(unsigned long val)
{
	asm volatile("mov %0,%%cr4": : "r" (val), "m" (__force_order));
}

#ifdef CONFIG_X86_64
static inline unsigned long native_read_cr8(void)
{
	unsigned long cr8;
	asm volatile("movq %%cr8,%0" : "=r" (cr8));
	return cr8;
}

static inline void native_write_cr8(unsigned long val)
{
	asm volatile("movq %0,%%cr8" :: "r" (val) : "memory");
}
#endif

static inline void native_wbinvd(void)
{
	asm volatile("wbinvd": : :"memory");
}

#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#else
#define read_cr0()	(native_read_cr0())
#define write_cr0(x)	(native_write_cr0(x))
#define read_cr2()	(native_read_cr2())
#define write_cr2(x)	(native_write_cr2(x))
#define read_cr3()	(native_read_cr3())
#define write_cr3(x)	(native_write_cr3(x))
#define read_cr4()	(native_read_cr4())
#define read_cr4_safe()	(native_read_cr4_safe())
#define write_cr4(x)	(native_write_cr4(x))
#define wbinvd()	(native_wbinvd())
#ifdef CONFIG_X86_64
#define read_cr8()	(native_read_cr8())
#define write_cr8(x)	(native_write_cr8(x))
#define load_gs_index   native_load_gs_index
#endif

/* Clear the 'TS' bit */
#define clts()		(native_clts())

#endif/* CONFIG_PARAVIRT */

#define stts() write_cr0(read_cr0() | X86_CR0_TS)

#endif /* __KERNEL__ */

static inline void clflush(volatile void *__p)
{
	asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p));
}

#define nop() asm volatile ("nop")

void disable_hlt(void);
void enable_hlt(void);

void cpu_idle_wait(void);

extern unsigned long arch_align_stack(unsigned long sp);
extern void free_init_pages(char *what, unsigned long begin, unsigned long end);

void default_idle(void);

void stop_this_cpu(void *dummy);

/*
 * Force strict CPU ordering.
 * And yes, this is required on UP too when we're talking
 * to devices.
 */
#ifdef CONFIG_X86_32
/*
 * Some non-Intel clones support out of order store. wmb() ceases to be a
 * nop for these.
 */
#define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2)
#define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2)
#define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM)
#else
#define mb() 	asm volatile("mfence":::"memory")
#define rmb()	asm volatile("lfence":::"memory")
#define wmb()	asm volatile("sfence" ::: "memory")
#endif

/**
 * read_barrier_depends - Flush all pending reads that subsequents reads
 * depend on.
 *
 * No data-dependent reads from memory-like regions are ever reordered
 * over this barrier.  All reads preceding this primitive are guaranteed
 * to access memory (but not necessarily other CPUs' caches) before any
 * reads following this primitive that depend on the data return by
 * any of the preceding reads.  This primitive is much lighter weight than
 * rmb() on most CPUs, and is never heavier weight than is
 * rmb().
 *
 * These ordering constraints are respected by both the local CPU
 * and the compiler.
 *
 * Ordering is not guaranteed by anything other than these primitives,
 * not even by data dependencies.  See the documentation for
 * memory_barrier() for examples and URLs to more information.
 *
 * For example, the following code would force ordering (the initial
 * value of "a" is zero, "b" is one, and "p" is "&a"):
 *
 * <programlisting>
 *	CPU 0				CPU 1
 *
 *	b = 2;
 *	memory_barrier();
 *	p = &b;				q = p;
 *					read_barrier_depends();
 *					d = *q;
 * </programlisting>
 *
 * because the read of "*q" depends on the read of "p" and these
 * two reads are separated by a read_barrier_depends().  However,
 * the following code, with the same initial values for "a" and "b":
 *
 * <programlisting>
 *	CPU 0				CPU 1
 *
 *	a = 2;
 *	memory_barrier();
 *	b = 3;				y = b;
 *					read_barrier_depends();
 *					x = a;
 * </programlisting>
 *
 * does not enforce ordering, since there is no data dependency between
 * the read of "a" and the read of "b".  Therefore, on some CPUs, such
 * as Alpha, "y" could be set to 3 and "x" to 0.  Use rmb()
 * in cases like this where there are no data dependencies.
 **/

#define read_barrier_depends()	do { } while (0)

#ifdef CONFIG_SMP
#define smp_mb()	mb()
#ifdef CONFIG_X86_PPRO_FENCE
# define smp_rmb()	rmb()
#else
# define smp_rmb()	barrier()
#endif
#ifdef CONFIG_X86_OOSTORE
# define smp_wmb() 	wmb()
#else
# define smp_wmb()	barrier()
#endif
#define smp_read_barrier_depends()	read_barrier_depends()
#define set_mb(var, value) do { (void)xchg(&var, value); } while (0)
#else
#define smp_mb()	barrier()
#define smp_rmb()	barrier()
#define smp_wmb()	barrier()
#define smp_read_barrier_depends()	do { } while (0)
#define set_mb(var, value) do { var = value; barrier(); } while (0)
#endif

/*
 * Stop RDTSC speculation. This is needed when you need to use RDTSC
 * (or get_cycles or vread that possibly accesses the TSC) in a defined
 * code region.
 *
 * (Could use an alternative three way for this if there was one.)
 */
static inline void rdtsc_barrier(void)
{
	alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC);
	alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC);
}

#endif /* _ASM_X86_SYSTEM_H */
Commit	Line	Data
1965aae3 PA	1	#ifndef _ASM_X86_SYSTEM_H
1965aae3 PA	2	#define _ASM_X86_SYSTEM_H
d8954222 GOC	3
d8954222 GOC	4	#include <asm/asm.h>
d46d7d75 GOC	5	#include <asm/segment.h>
	6	#include <asm/cpufeature.h>
	7	#include <asm/cmpxchg.h>
fde1b3fa	8	#include <asm/nops.h>
d8954222	9
d3ca901f	10	#include <linux/kernel.h>
d46d7d75	11	#include <linux/irqflags.h>
d3ca901f	12
ded9aa0d JB	13	/* entries in ARCH_DLINFO: */
	14	#ifdef CONFIG_IA32_EMULATION
	15	# define AT_VECTOR_SIZE_ARCH 2
	16	#else
	17	# define AT_VECTOR_SIZE_ARCH 1
	18	#endif
	19
0a3b4d15	20	struct task_struct; /* one of the stranger aspects of C forward declarations */
599db4fe HH	21	struct task_struct __switch_to(struct task_struct prev,
599db4fe HH	22	struct task_struct *next);
2fb6b2a0	23	struct tss_struct;
389d1fb1 JF	24	void __switch_to_xtra(struct task_struct prev_p, struct task_struct next_p,
389d1fb1 JF	25	struct tss_struct *tss);
0a3b4d15	26
aab02f0a JS	27	#ifdef CONFIG_X86_32
aab02f0a JS	28
60a5317f TH	29	#ifdef CONFIG_CC_STACKPROTECTOR
60a5317f TH	30	#define __switch_canary \
5c79d2a5 TH	31	"movl %P[task_canary](%[next]), %%ebx\n\t" \
5c79d2a5 TH	32	"movl %%ebx, "__percpu_arg([stack_canary])"\n\t"
60a5317f TH	33	#define __switch_canary_oparam \
	34	, [stack_canary] "=m" (per_cpu_var(stack_canary))
	35	#define __switch_canary_iparam \
60a5317f TH	36	, [task_canary] "i" (offsetof(struct task_struct, stack_canary))
	37	#else /* CC_STACKPROTECTOR */
	38	#define __switch_canary
	39	#define __switch_canary_oparam
	40	#define __switch_canary_iparam
	41	#endif /* CC_STACKPROTECTOR */
	42
0a3b4d15 GOC	43	/*
	44	* Saving eflags is important. It switches not only IOPL between tasks,
	45	* it also protects other tasks from NT leaking through sysenter etc.
	46	*/
23b55bd9 IM	47	#define switch_to(prev, next, last) \
23b55bd9 IM	48	do { \
8b6451fe IM	49	/* \
	50	* Context-switching clobbers all registers, so we clobber \
	51	* them explicitly, via unused output variables. \
	52	* (EAX and EBP is not listed because EBP is saved/restored \
	53	* explicitly for wchan access and EAX is the return value of \
	54	* __switch_to()) \
	55	*/ \
	56	unsigned long ebx, ecx, edx, esi, edi; \
23b55bd9	57	\
c5386c20 JP	58	asm volatile("pushfl\n\t" /* save flags */ \
	59	"pushl %%ebp\n\t" /* save EBP */ \
	60	"movl %%esp,%[prev_sp]\n\t" /* save ESP */ \
	61	"movl %[next_sp],%%esp\n\t" /* restore ESP */ \
	62	"movl $1f,%[prev_ip]\n\t" /* save EIP */ \
	63	"pushl %[next_ip]\n\t" /* restore EIP */ \
5c79d2a5	64	__switch_canary \
c5386c20 JP	65	"jmp __switch_to\n" /* regparm call */ \
	66	"1:\t" \
	67	"popl %%ebp\n\t" /* restore EBP */ \
	68	"popfl\n" /* restore flags */ \
23b55bd9	69	\
c5386c20 JP	70	/* output parameters */ \
	71	: [prev_sp] "=m" (prev->thread.sp), \
	72	[prev_ip] "=m" (prev->thread.ip), \
	73	"=a" (last), \
23b55bd9	74	\
c5386c20 JP	75	/* clobbered output registers: */ \
	76	"=b" (ebx), "=c" (ecx), "=d" (edx), \
	77	"=S" (esi), "=D" (edi) \
	78	\
60a5317f TH	79	__switch_canary_oparam \
60a5317f TH	80	\
c5386c20 JP	81	/* input parameters: */ \
	82	: [next_sp] "m" (next->thread.sp), \
	83	[next_ip] "m" (next->thread.ip), \
	84	\
	85	/* regparm parameters for __switch_to(): */ \
	86	[prev] "a" (prev), \
33f8c40a VN	87	[next] "d" (next) \
33f8c40a VN	88	\
60a5317f TH	89	__switch_canary_iparam \
60a5317f TH	90	\
33f8c40a VN	91	: /* reloaded segment registers */ \
33f8c40a VN	92	"memory"); \
0a3b4d15 GOC	93	} while (0)
0a3b4d15 GOC	94
d46d7d75 GOC	95	/*
	96	* disable hlt during certain critical i/o operations
	97	*/
	98	#define HAVE_DISABLE_HLT
96a388de	99	#else
0a3b4d15 GOC	100	#define __SAVE(reg, offset) "movq %%" #reg ",(14-" #offset ")*8(%%rsp)\n\t"
	101	#define __RESTORE(reg, offset) "movq (14-" #offset ")*8(%%rsp),%%" #reg "\n\t"
	102
	103	/* frame pointer must be last for get_wchan */
	104	#define SAVE_CONTEXT "pushf ; pushq %%rbp ; movq %%rsi,%%rbp\n\t"
	105	#define RESTORE_CONTEXT "movq %%rbp,%%rsi ; popq %%rbp ; popf\t"
	106
	107	#define __EXTRA_CLOBBER \
	108	, "rcx", "rbx", "rdx", "r8", "r9", "r10", "r11", \
	109	"r12", "r13", "r14", "r15"
	110
b4a8f7a2 TH	111	#ifdef CONFIG_CC_STACKPROTECTOR
	112	#define __switch_canary \
	113	"movq %P[task_canary](%%rsi),%%r8\n\t" \
67e68bde TH	114	"movq %%r8,"__percpu_arg([gs_canary])"\n\t"
	115	#define __switch_canary_oparam \
	116	, [gs_canary] "=m" (per_cpu_var(irq_stack_union.stack_canary))
	117	#define __switch_canary_iparam \
	118	, [task_canary] "i" (offsetof(struct task_struct, stack_canary))
b4a8f7a2 TH	119	#else /* CC_STACKPROTECTOR */
b4a8f7a2 TH	120	#define __switch_canary
67e68bde TH	121	#define __switch_canary_oparam
67e68bde TH	122	#define __switch_canary_iparam
b4a8f7a2 TH	123	#endif /* CC_STACKPROTECTOR */
b4a8f7a2 TH	124
0a3b4d15 GOC	125	/* Save restore flags to clear handle leaking NT */
0a3b4d15 GOC	126	#define switch_to(prev, next, last) \
b4a8f7a2	127	asm volatile(SAVE_CONTEXT \
0a3b4d15 GOC	128	"movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */ \
	129	"movq %P[threadrsp](%[next]),%%rsp\n\t" /* restore RSP */ \
	130	"call __switch_to\n\t" \
	131	".globl thread_return\n" \
	132	"thread_return:\n\t" \
87b26406	133	"movq "__percpu_arg([current_task])",%%rsi\n\t" \
b4a8f7a2	134	__switch_canary \
0a3b4d15	135	"movq %P[thread_info](%%rsi),%%r8\n\t" \
0a3b4d15	136	"movq %%rax,%%rdi\n\t" \
7106a5ab BL	137	"testl %[_tif_fork],%P[ti_flags](%%r8)\n\t" \
7106a5ab BL	138	"jnz ret_from_fork\n\t" \
0a3b4d15 GOC	139	RESTORE_CONTEXT \
0a3b4d15 GOC	140	: "=a" (last) \
67e68bde	141	__switch_canary_oparam \
0a3b4d15 GOC	142	: [next] "S" (next), [prev] "D" (prev), \
	143	[threadrsp] "i" (offsetof(struct task_struct, thread.sp)), \
	144	[ti_flags] "i" (offsetof(struct thread_info, flags)), \
7106a5ab	145	[_tif_fork] "i" (_TIF_FORK), \
0a3b4d15	146	[thread_info] "i" (offsetof(struct task_struct, stack)), \
b4a8f7a2	147	[current_task] "m" (per_cpu_var(current_task)) \
67e68bde	148	__switch_canary_iparam \
0a3b4d15	149	: "memory", "cc" __EXTRA_CLOBBER)
96a388de	150	#endif
d8954222 GOC	151
	152	#ifdef __KERNEL__
	153	#define _set_base(addr, base) do { unsigned long __pr; \
	154	__asm__ __volatile__ ("movw %%dx,%1\n\t" \
	155	"rorl $16,%%edx\n\t" \
	156	"movb %%dl,%2\n\t" \
	157	"movb %%dh,%3" \
	158	:"=&d" (__pr) \
	159	:"m" (*((addr)+2)), \
	160	"m" (*((addr)+4)), \
	161	"m" (*((addr)+7)), \
	162	"0" (base) \
	163	); } while (0)
	164
	165	#define _set_limit(addr, limit) do { unsigned long __lr; \
	166	__asm__ __volatile__ ("movw %%dx,%1\n\t" \
	167	"rorl $16,%%edx\n\t" \
	168	"movb %2,%%dh\n\t" \
	169	"andb $0xf0,%%dh\n\t" \
	170	"orb %%dh,%%dl\n\t" \
	171	"movb %%dl,%2" \
	172	:"=&d" (__lr) \
	173	:"m" (*(addr)), \
	174	"m" (*((addr)+6)), \
	175	"0" (limit) \
	176	); } while (0)
	177
	178	#define set_base(ldt, base) _set_base(((char *)&(ldt)) , (base))
	179	#define set_limit(ldt, limit) _set_limit(((char *)&(ldt)) , ((limit)-1))
	180
9f9d489a	181	extern void native_load_gs_index(unsigned);
d3ca901f	182
a6b46552 GOC	183	/*
	184	* Load a segment. Fall back on loading the zero
	185	* segment if something goes wrong..
	186	*/
	187	#define loadsegment(seg, value) \
	188	asm volatile("\n" \
c5386c20 JP	189	"1:\t" \
	190	"movl %k0,%%" #seg "\n" \
	191	"2:\n" \
	192	".section .fixup,\"ax\"\n" \
	193	"3:\t" \
	194	"movl %k1, %%" #seg "\n\t" \
	195	"jmp 2b\n" \
	196	".previous\n" \
	197	_ASM_EXTABLE(1b,3b) \
d338c73c	198	: :"r" (value), "r" (0) : "memory")
a6b46552 GOC	199
a6b46552 GOC	200
d8954222 GOC	201	/*
	202	* Save a segment register away
	203	*/
c5386c20	204	#define savesegment(seg, value) \
d9fc3fd3	205	asm("mov %%" #seg ",%0":"=r" (value) : : "memory")
d8954222	206
d9a89a26 TH	207	/*
	208	* x86_32 user gs accessors.
	209	*/
	210	#ifdef CONFIG_X86_32
ccbeed3a	211	#ifdef CONFIG_X86_32_LAZY_GS
d9a89a26 TH	212	#define get_user_gs(regs) (u16)({unsigned long v; savesegment(gs, v); v;})
	213	#define set_user_gs(regs, v) loadsegment(gs, (unsigned long)(v))
	214	#define task_user_gs(tsk) ((tsk)->thread.gs)
ccbeed3a TH	215	#define lazy_save_gs(v) savesegment(gs, (v))
	216	#define lazy_load_gs(v) loadsegment(gs, (v))
	217	#else /* X86_32_LAZY_GS */
	218	#define get_user_gs(regs) (u16)((regs)->gs)
	219	#define set_user_gs(regs, v) do { (regs)->gs = (v); } while (0)
	220	#define task_user_gs(tsk) (task_pt_regs(tsk)->gs)
	221	#define lazy_save_gs(v) do { } while (0)
	222	#define lazy_load_gs(v) do { } while (0)
	223	#endif /* X86_32_LAZY_GS */
	224	#endif /* X86_32 */
d9a89a26	225
d8954222 GOC	226	static inline unsigned long get_limit(unsigned long segment)
	227	{
	228	unsigned long __limit;
c5386c20 JP	229	asm("lsll %1,%0" : "=r" (__limit) : "r" (segment));
c5386c20 JP	230	return __limit + 1;
d8954222	231	}
d3ca901f GOC	232
	233	static inline void native_clts(void)
	234	{
c5386c20	235	asm volatile("clts");
d3ca901f GOC	236	}
	237
	238	/*
	239	* Volatile isn't enough to prevent the compiler from reordering the
	240	* read/write functions for the control registers and messing everything up.
	241	* A memory clobber would solve the problem, but would prevent reordering of
	242	* all loads stores around it, which can hurt performance. Solution is to
	243	* use a variable and mimic reads and writes to it to enforce serialization
	244	*/
	245	static unsigned long __force_order;
	246
	247	static inline unsigned long native_read_cr0(void)
	248	{
	249	unsigned long val;
c5386c20	250	asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order));
d3ca901f GOC	251	return val;
	252	}
	253
	254	static inline void native_write_cr0(unsigned long val)
	255	{
c5386c20	256	asm volatile("mov %0,%%cr0": : "r" (val), "m" (__force_order));
d3ca901f GOC	257	}
	258
	259	static inline unsigned long native_read_cr2(void)
	260	{
	261	unsigned long val;
c5386c20	262	asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order));
d3ca901f GOC	263	return val;
	264	}
	265
	266	static inline void native_write_cr2(unsigned long val)
	267	{
c5386c20	268	asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order));
d3ca901f GOC	269	}
	270
	271	static inline unsigned long native_read_cr3(void)
	272	{
	273	unsigned long val;
c5386c20	274	asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order));
d3ca901f GOC	275	return val;
	276	}
	277
	278	static inline void native_write_cr3(unsigned long val)
	279	{
c5386c20	280	asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order));
d3ca901f GOC	281	}
	282
	283	static inline unsigned long native_read_cr4(void)
	284	{
	285	unsigned long val;
c5386c20	286	asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order));
d3ca901f GOC	287	return val;
	288	}
	289
	290	static inline unsigned long native_read_cr4_safe(void)
	291	{
	292	unsigned long val;
	293	/* This could fault if %cr4 does not exist. In x86_64, a cr4 always
	294	* exists, so it will never fail. */
	295	#ifdef CONFIG_X86_32
88976ee1 PA	296	asm volatile("1: mov %%cr4, %0\n"
88976ee1 PA	297	"2:\n"
c5386c20	298	_ASM_EXTABLE(1b, 2b)
88976ee1	299	: "=r" (val), "=m" (__force_order) : "0" (0));
d3ca901f GOC	300	#else
	301	val = native_read_cr4();
	302	#endif
	303	return val;
	304	}
	305
	306	static inline void native_write_cr4(unsigned long val)
	307	{
c5386c20	308	asm volatile("mov %0,%%cr4": : "r" (val), "m" (__force_order));
d3ca901f GOC	309	}
d3ca901f GOC	310
94ea03cd GOC	311	#ifdef CONFIG_X86_64
	312	static inline unsigned long native_read_cr8(void)
	313	{
	314	unsigned long cr8;
	315	asm volatile("movq %%cr8,%0" : "=r" (cr8));
	316	return cr8;
	317	}
	318
	319	static inline void native_write_cr8(unsigned long val)
	320	{
	321	asm volatile("movq %0,%%cr8" :: "r" (val) : "memory");
	322	}
	323	#endif
	324
d3ca901f GOC	325	static inline void native_wbinvd(void)
	326	{
	327	asm volatile("wbinvd": : :"memory");
	328	}
c5386c20	329
d3ca901f GOC	330	#ifdef CONFIG_PARAVIRT
	331	#include <asm/paravirt.h>
	332	#else
	333	#define read_cr0() (native_read_cr0())
	334	#define write_cr0(x) (native_write_cr0(x))
	335	#define read_cr2() (native_read_cr2())
	336	#define write_cr2(x) (native_write_cr2(x))
	337	#define read_cr3() (native_read_cr3())
	338	#define write_cr3(x) (native_write_cr3(x))
	339	#define read_cr4() (native_read_cr4())
	340	#define read_cr4_safe() (native_read_cr4_safe())
	341	#define write_cr4(x) (native_write_cr4(x))
	342	#define wbinvd() (native_wbinvd())
d46d7d75	343	#ifdef CONFIG_X86_64
94ea03cd GOC	344	#define read_cr8() (native_read_cr8())
94ea03cd GOC	345	#define write_cr8(x) (native_write_cr8(x))
9f9d489a	346	#define load_gs_index native_load_gs_index
d46d7d75 GOC	347	#endif
d46d7d75 GOC	348
d3ca901f GOC	349	/* Clear the 'TS' bit */
	350	#define clts() (native_clts())
	351
	352	#endif/* CONFIG_PARAVIRT */
	353
4e09e21c	354	#define stts() write_cr0(read_cr0() \| X86_CR0_TS)
d3ca901f	355
d8954222 GOC	356	#endif /* __KERNEL__ */
d8954222 GOC	357
84fb144b	358	static inline void clflush(volatile void *__p)
d8954222	359	{
84fb144b	360	asm volatile("clflush %0" : "+m" ((volatile char __force )__p));
d8954222 GOC	361	}
d8954222 GOC	362
c5386c20	363	#define nop() asm volatile ("nop")
d8954222 GOC	364
	365	void disable_hlt(void);
	366	void enable_hlt(void);
	367
d8954222 GOC	368	void cpu_idle_wait(void);
	369
	370	extern unsigned long arch_align_stack(unsigned long sp);
	371	extern void free_init_pages(char *what, unsigned long begin, unsigned long end);
	372
	373	void default_idle(void);
	374
d3ec5cae IV	375	void stop_this_cpu(void *dummy);
d3ec5cae IV	376
833d8469 GOC	377	/*
	378	* Force strict CPU ordering.
	379	* And yes, this is required on UP too when we're talking
	380	* to devices.
	381	*/
	382	#ifdef CONFIG_X86_32
	383	/*
0d7a1819	384	* Some non-Intel clones support out of order store. wmb() ceases to be a
833d8469 GOC	385	* nop for these.
	386	*/
	387	#define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2)
	388	#define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2)
	389	#define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM)
	390	#else
	391	#define mb() asm volatile("mfence":::"memory")
	392	#define rmb() asm volatile("lfence":::"memory")
	393	#define wmb() asm volatile("sfence" ::: "memory")
	394	#endif
	395
	396	/**
	397	* read_barrier_depends - Flush all pending reads that subsequents reads
	398	* depend on.
	399	*
	400	* No data-dependent reads from memory-like regions are ever reordered
	401	* over this barrier. All reads preceding this primitive are guaranteed
	402	* to access memory (but not necessarily other CPUs' caches) before any
	403	* reads following this primitive that depend on the data return by
	404	* any of the preceding reads. This primitive is much lighter weight than
	405	* rmb() on most CPUs, and is never heavier weight than is
	406	* rmb().
	407	*
	408	* These ordering constraints are respected by both the local CPU
	409	* and the compiler.
	410	*
	411	* Ordering is not guaranteed by anything other than these primitives,
	412	* not even by data dependencies. See the documentation for
	413	* memory_barrier() for examples and URLs to more information.
	414	*
	415	* For example, the following code would force ordering (the initial
	416	* value of "a" is zero, "b" is one, and "p" is "&a"):
	417	*
	418	* <programlisting>
	419	* CPU 0 CPU 1
	420	*
	421	* b = 2;
	422	* memory_barrier();
	423	* p = &b; q = p;
	424	* read_barrier_depends();
	425	* d = *q;
	426	* </programlisting>
	427	*
	428	* because the read of "*q" depends on the read of "p" and these
	429	* two reads are separated by a read_barrier_depends(). However,
	430	* the following code, with the same initial values for "a" and "b":
	431	*
	432	* <programlisting>
	433	* CPU 0 CPU 1
	434	*
	435	* a = 2;
	436	* memory_barrier();
	437	* b = 3; y = b;
	438	* read_barrier_depends();
	439	* x = a;
	440	* </programlisting>
	441	*
	442	* does not enforce ordering, since there is no data dependency between
	443	* the read of "a" and the read of "b". Therefore, on some CPUs, such
	444	* as Alpha, "y" could be set to 3 and "x" to 0. Use rmb()
	445	* in cases like this where there are no data dependencies.
	446	**/
	447
	448	#define read_barrier_depends() do { } while (0)
449
450	#ifdef CONFIG_SMP
451	#define smp_mb() mb()
452	#ifdef CONFIG_X86_PPRO_FENCE
453	# define smp_rmb() rmb()
454	#else
455	# define smp_rmb() barrier()
456	#endif
457	#ifdef CONFIG_X86_OOSTORE
458	# define smp_wmb() wmb()
459	#else
460	# define smp_wmb() barrier()
461	#endif
462	#define smp_read_barrier_depends() read_barrier_depends()
c5386c20	463	#define set_mb(var, value) do { (void)xchg(&var, value); } while (0)
833d8469 GOC	464	#else
	465	#define smp_mb() barrier()
	466	#define smp_rmb() barrier()
	467	#define smp_wmb() barrier()
	468	#define smp_read_barrier_depends() do { } while (0)
	469	#define set_mb(var, value) do { var = value; barrier(); } while (0)
	470	#endif
	471
fde1b3fa AK	472	/*
	473	* Stop RDTSC speculation. This is needed when you need to use RDTSC
	474	* (or get_cycles or vread that possibly accesses the TSC) in a defined
	475	* code region.
	476	*
	477	* (Could use an alternative three way for this if there was one.)
	478	*/
	479	static inline void rdtsc_barrier(void)
	480	{
	481	alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC);
	482	alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC);
	483	}
833d8469	484
1965aae3	485	#endif /* _ASM_X86_SYSTEM_H */