[mirror_ubuntu-artful-kernel.git] / arch / x86 / include / asm / switch_to.h

#ifndef _ASM_X86_SWITCH_TO_H
#define _ASM_X86_SWITCH_TO_H

struct task_struct; /* one of the stranger aspects of C forward declarations */
__visible struct task_struct *__switch_to(struct task_struct *prev,
					   struct task_struct *next);
struct tss_struct;
void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
		      struct tss_struct *tss);

#ifdef CONFIG_X86_32

#ifdef CONFIG_CC_STACKPROTECTOR
#define __switch_canary							\
	"movl %P[task_canary](%[next]), %%ebx\n\t"			\
	"movl %%ebx, "__percpu_arg([stack_canary])"\n\t"
#define __switch_canary_oparam						\
	, [stack_canary] "=m" (stack_canary.canary)
#define __switch_canary_iparam						\
	, [task_canary] "i" (offsetof(struct task_struct, stack_canary))
#else	/* CC_STACKPROTECTOR */
#define __switch_canary
#define __switch_canary_oparam
#define __switch_canary_iparam
#endif	/* CC_STACKPROTECTOR */

/*
 * Saving eflags is important. It switches not only IOPL between tasks,
 * it also protects other tasks from NT leaking through sysenter etc.
 */
#define switch_to(prev, next, last)					\
do {									\
	/*								\
	 * Context-switching clobbers all registers, so we clobber	\
	 * them explicitly, via unused output variables.		\
	 * (EAX and EBP is not listed because EBP is saved/restored	\
	 * explicitly for wchan access and EAX is the return value of	\
	 * __switch_to())						\
	 */								\
	unsigned long ebx, ecx, edx, esi, edi;				\
									\
	asm volatile("pushl %%ebp\n\t"		/* save    EBP   */	\
		     "movl %%esp,%[prev_sp]\n\t"	/* save    ESP   */ \
		     "movl %[next_sp],%%esp\n\t"	/* restore ESP   */ \
		     "movl $1f,%[prev_ip]\n\t"	/* save    EIP   */	\
		     "pushl %[next_ip]\n\t"	/* restore EIP   */	\
		     __switch_canary					\
		     "jmp __switch_to\n"	/* regparm call  */	\
		     "1:\t"						\
		     "popl %%ebp\n\t"		/* restore EBP   */	\
									\
		     /* output parameters */				\
		     : [prev_sp] "=m" (prev->thread.sp),		\
		       [prev_ip] "=m" (prev->thread.ip),		\
		       "=a" (last),					\
									\
		       /* clobbered output registers: */		\
		       "=b" (ebx), "=c" (ecx), "=d" (edx),		\
		       "=S" (esi), "=D" (edi)				\
		       							\
		       __switch_canary_oparam				\
									\
		       /* input parameters: */				\
		     : [next_sp]  "m" (next->thread.sp),		\
		       [next_ip]  "m" (next->thread.ip),		\
		       							\
		       /* regparm parameters for __switch_to(): */	\
		       [prev]     "a" (prev),				\
		       [next]     "d" (next)				\
									\
		       __switch_canary_iparam				\
									\
		     : /* reloaded segment registers */			\
			"memory");					\
} while (0)

#else /* CONFIG_X86_32 */

/* frame pointer must be last for get_wchan */
#define SAVE_CONTEXT    "pushq %%rbp ; movq %%rsi,%%rbp\n\t"
#define RESTORE_CONTEXT "movq %%rbp,%%rsi ; popq %%rbp\t"

#define __EXTRA_CLOBBER  \
	, "rcx", "rbx", "rdx", "r8", "r9", "r10", "r11", \
	  "r12", "r13", "r14", "r15", "flags"

#ifdef CONFIG_CC_STACKPROTECTOR
#define __switch_canary							  \
	"movq %P[task_canary](%%rsi),%%r8\n\t"				  \
	"movq %%r8,"__percpu_arg([gs_canary])"\n\t"
#define __switch_canary_oparam						  \
	, [gs_canary] "=m" (irq_stack_union.stack_canary)
#define __switch_canary_iparam						  \
	, [task_canary] "i" (offsetof(struct task_struct, stack_canary))
#else	/* CC_STACKPROTECTOR */
#define __switch_canary
#define __switch_canary_oparam
#define __switch_canary_iparam
#endif	/* CC_STACKPROTECTOR */

/*
 * There is no need to save or restore flags, because flags are always
 * clean in kernel mode, with the possible exception of IOPL.  Kernel IOPL
 * has no effect.
 */
#define switch_to(prev, next, last) \
	asm volatile(SAVE_CONTEXT					  \
	     "movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */	  \
	     "movq %P[threadrsp](%[next]),%%rsp\n\t" /* restore RSP */	  \
	     "call __switch_to\n\t"					  \
	     "movq "__percpu_arg([current_task])",%%rsi\n\t"		  \
	     __switch_canary						  \
	     "movq %P[thread_info](%%rsi),%%r8\n\t"			  \
	     "movq %%rax,%%rdi\n\t" 					  \
	     "testl  %[_tif_fork],%P[ti_flags](%%r8)\n\t"		  \
	     "jnz   ret_from_fork\n\t"					  \
	     RESTORE_CONTEXT						  \
	     : "=a" (last)					  	  \
	       __switch_canary_oparam					  \
	     : [next] "S" (next), [prev] "D" (prev),			  \
	       [threadrsp] "i" (offsetof(struct task_struct, thread.sp)), \
	       [ti_flags] "i" (offsetof(struct thread_info, flags)),	  \
	       [_tif_fork] "i" (_TIF_FORK),			  	  \
	       [thread_info] "i" (offsetof(struct task_struct, stack)),   \
	       [current_task] "m" (current_task)			  \
	       __switch_canary_iparam					  \
	     : "memory", "cc" __EXTRA_CLOBBER)

#endif /* CONFIG_X86_32 */

#endif /* _ASM_X86_SWITCH_TO_H */
Commit	Line	Data
f05e798a DH	1	#ifndef _ASM_X86_SWITCH_TO_H
	2	#define _ASM_X86_SWITCH_TO_H
	3
	4	struct task_struct; /* one of the stranger aspects of C forward declarations */
35ea7903 AK	5	__visible struct task_struct __switch_to(struct task_struct prev,
35ea7903 AK	6	struct task_struct *next);
f05e798a DH	7	struct tss_struct;
	8	void __switch_to_xtra(struct task_struct prev_p, struct task_struct next_p,
	9	struct tss_struct *tss);
	10
	11	#ifdef CONFIG_X86_32
	12
	13	#ifdef CONFIG_CC_STACKPROTECTOR
	14	#define __switch_canary \
	15	"movl %P[task_canary](%[next]), %%ebx\n\t" \
	16	"movl %%ebx, "__percpu_arg([stack_canary])"\n\t"
	17	#define __switch_canary_oparam \
	18	, [stack_canary] "=m" (stack_canary.canary)
	19	#define __switch_canary_iparam \
	20	, [task_canary] "i" (offsetof(struct task_struct, stack_canary))
	21	#else /* CC_STACKPROTECTOR */
	22	#define __switch_canary
	23	#define __switch_canary_oparam
	24	#define __switch_canary_iparam
	25	#endif /* CC_STACKPROTECTOR */
	26
	27	/*
	28	* Saving eflags is important. It switches not only IOPL between tasks,
	29	* it also protects other tasks from NT leaking through sysenter etc.
	30	*/
	31	#define switch_to(prev, next, last) \
	32	do { \
	33	/* \
	34	* Context-switching clobbers all registers, so we clobber \
	35	* them explicitly, via unused output variables. \
	36	* (EAX and EBP is not listed because EBP is saved/restored \
	37	* explicitly for wchan access and EAX is the return value of \
	38	* __switch_to()) \
	39	*/ \
	40	unsigned long ebx, ecx, edx, esi, edi; \
	41	\
092c74e4	42	asm volatile("pushl %%ebp\n\t" /* save EBP */ \
f05e798a DH	43	"movl %%esp,%[prev_sp]\n\t" /* save ESP */ \
	44	"movl %[next_sp],%%esp\n\t" /* restore ESP */ \
	45	"movl $1f,%[prev_ip]\n\t" /* save EIP */ \
	46	"pushl %[next_ip]\n\t" /* restore EIP */ \
	47	__switch_canary \
	48	"jmp __switch_to\n" /* regparm call */ \
	49	"1:\t" \
	50	"popl %%ebp\n\t" /* restore EBP */ \
f05e798a DH	51	\
	52	/* output parameters */ \
	53	: [prev_sp] "=m" (prev->thread.sp), \
	54	[prev_ip] "=m" (prev->thread.ip), \
	55	"=a" (last), \
	56	\
	57	/* clobbered output registers: */ \
	58	"=b" (ebx), "=c" (ecx), "=d" (edx), \
	59	"=S" (esi), "=D" (edi) \
	60	\
	61	__switch_canary_oparam \
	62	\
	63	/* input parameters: */ \
	64	: [next_sp] "m" (next->thread.sp), \
	65	[next_ip] "m" (next->thread.ip), \
	66	\
	67	/* regparm parameters for __switch_to(): */ \
	68	[prev] "a" (prev), \
	69	[next] "d" (next) \
	70	\
	71	__switch_canary_iparam \
	72	\
	73	: /* reloaded segment registers */ \
	74	"memory"); \
	75	} while (0)
	76
	77	#else /* CONFIG_X86_32 */
	78
	79	/* frame pointer must be last for get_wchan */
3f2c5085 AL	80	#define SAVE_CONTEXT "pushq %%rbp ; movq %%rsi,%%rbp\n\t"
3f2c5085 AL	81	#define RESTORE_CONTEXT "movq %%rbp,%%rsi ; popq %%rbp\t"
f05e798a DH	82
	83	#define __EXTRA_CLOBBER \
	84	, "rcx", "rbx", "rdx", "r8", "r9", "r10", "r11", \
3f2c5085	85	"r12", "r13", "r14", "r15", "flags"
f05e798a DH	86
	87	#ifdef CONFIG_CC_STACKPROTECTOR
	88	#define __switch_canary \
	89	"movq %P[task_canary](%%rsi),%%r8\n\t" \
	90	"movq %%r8,"__percpu_arg([gs_canary])"\n\t"
	91	#define __switch_canary_oparam \
	92	, [gs_canary] "=m" (irq_stack_union.stack_canary)
	93	#define __switch_canary_iparam \
	94	, [task_canary] "i" (offsetof(struct task_struct, stack_canary))
	95	#else /* CC_STACKPROTECTOR */
	96	#define __switch_canary
	97	#define __switch_canary_oparam
	98	#define __switch_canary_iparam
	99	#endif /* CC_STACKPROTECTOR */
	100
3f2c5085 AL	101	/*
	102	* There is no need to save or restore flags, because flags are always
	103	* clean in kernel mode, with the possible exception of IOPL. Kernel IOPL
	104	* has no effect.
	105	*/
f05e798a DH	106	#define switch_to(prev, next, last) \
	107	asm volatile(SAVE_CONTEXT \
	108	"movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */ \
	109	"movq %P[threadrsp](%[next]),%%rsp\n\t" /* restore RSP */ \
	110	"call __switch_to\n\t" \
	111	"movq "__percpu_arg([current_task])",%%rsi\n\t" \
	112	__switch_canary \
	113	"movq %P[thread_info](%%rsi),%%r8\n\t" \
	114	"movq %%rax,%%rdi\n\t" \
	115	"testl %[_tif_fork],%P[ti_flags](%%r8)\n\t" \
	116	"jnz ret_from_fork\n\t" \
	117	RESTORE_CONTEXT \
	118	: "=a" (last) \
	119	__switch_canary_oparam \
	120	: [next] "S" (next), [prev] "D" (prev), \
	121	[threadrsp] "i" (offsetof(struct task_struct, thread.sp)), \
	122	[ti_flags] "i" (offsetof(struct thread_info, flags)), \
	123	[_tif_fork] "i" (_TIF_FORK), \
	124	[thread_info] "i" (offsetof(struct task_struct, stack)), \
	125	[current_task] "m" (current_task) \
	126	__switch_canary_iparam \
	127	: "memory", "cc" __EXTRA_CLOBBER)
	128
	129	#endif /* CONFIG_X86_32 */
	130
	131	#endif /* _ASM_X86_SWITCH_TO_H */