[mirror_ubuntu-artful-kernel.git] / arch / x86 / kernel / process_64.c

/*
 *  Copyright (C) 1995  Linus Torvalds
 *
 *  Pentium III FXSR, SSE support
 *	Gareth Hughes <gareth@valinux.com>, May 2000
 *
 *  X86-64 port
 *	Andi Kleen.
 *
 *	CPU hotplug support - ashok.raj@intel.com
 */

/*
 * This file handles the architecture-dependent parts of process handling..
 */

#include <linux/cpu.h>
#include <linux/errno.h>
#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/elfcore.h>
#include <linux/smp.h>
#include <linux/slab.h>
#include <linux/user.h>
#include <linux/interrupt.h>
#include <linux/delay.h>
#include <linux/module.h>
#include <linux/ptrace.h>
#include <linux/notifier.h>
#include <linux/kprobes.h>
#include <linux/kdebug.h>
#include <linux/prctl.h>
#include <linux/uaccess.h>
#include <linux/io.h>
#include <linux/ftrace.h>

#include <asm/pgtable.h>
#include <asm/processor.h>
#include <asm/i387.h>
#include <asm/fpu-internal.h>
#include <asm/mmu_context.h>
#include <asm/prctl.h>
#include <asm/desc.h>
#include <asm/proto.h>
#include <asm/ia32.h>
#include <asm/idle.h>
#include <asm/syscalls.h>
#include <asm/debugreg.h>
#include <asm/switch_to.h>

asmlinkage extern void ret_from_fork(void);

DEFINE_PER_CPU(unsigned long, old_rsp);

/* Prints also some state that isn't saved in the pt_regs */
void __show_regs(struct pt_regs *regs, int all)
{
	unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
	unsigned long d0, d1, d2, d3, d6, d7;
	unsigned int fsindex, gsindex;
	unsigned int ds, cs, es;

	show_regs_common();
	printk(KERN_DEFAULT "RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->ip);
	printk_address(regs->ip, 1);
	printk(KERN_DEFAULT "RSP: %04lx:%016lx  EFLAGS: %08lx\n", regs->ss,
			regs->sp, regs->flags);
	printk(KERN_DEFAULT "RAX: %016lx RBX: %016lx RCX: %016lx\n",
	       regs->ax, regs->bx, regs->cx);
	printk(KERN_DEFAULT "RDX: %016lx RSI: %016lx RDI: %016lx\n",
	       regs->dx, regs->si, regs->di);
	printk(KERN_DEFAULT "RBP: %016lx R08: %016lx R09: %016lx\n",
	       regs->bp, regs->r8, regs->r9);
	printk(KERN_DEFAULT "R10: %016lx R11: %016lx R12: %016lx\n",
	       regs->r10, regs->r11, regs->r12);
	printk(KERN_DEFAULT "R13: %016lx R14: %016lx R15: %016lx\n",
	       regs->r13, regs->r14, regs->r15);

	asm("movl %%ds,%0" : "=r" (ds));
	asm("movl %%cs,%0" : "=r" (cs));
	asm("movl %%es,%0" : "=r" (es));
	asm("movl %%fs,%0" : "=r" (fsindex));
	asm("movl %%gs,%0" : "=r" (gsindex));

	rdmsrl(MSR_FS_BASE, fs);
	rdmsrl(MSR_GS_BASE, gs);
	rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);

	if (!all)
		return;

	cr0 = read_cr0();
	cr2 = read_cr2();
	cr3 = read_cr3();
	cr4 = read_cr4();

	printk(KERN_DEFAULT "FS:  %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
	       fs, fsindex, gs, gsindex, shadowgs);
	printk(KERN_DEFAULT "CS:  %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds,
			es, cr0);
	printk(KERN_DEFAULT "CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3,
			cr4);

	get_debugreg(d0, 0);
	get_debugreg(d1, 1);
	get_debugreg(d2, 2);
	printk(KERN_DEFAULT "DR0: %016lx DR1: %016lx DR2: %016lx\n", d0, d1, d2);
	get_debugreg(d3, 3);
	get_debugreg(d6, 6);
	get_debugreg(d7, 7);
	printk(KERN_DEFAULT "DR3: %016lx DR6: %016lx DR7: %016lx\n", d3, d6, d7);
}

void release_thread(struct task_struct *dead_task)
{
	if (dead_task->mm) {
		if (dead_task->mm->context.size) {
			pr_warn("WARNING: dead process %8s still has LDT? <%p/%d>\n",
				dead_task->comm,
				dead_task->mm->context.ldt,
				dead_task->mm->context.size);
			BUG();
		}
	}
}

static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
{
	struct user_desc ud = {
		.base_addr = addr,
		.limit = 0xfffff,
		.seg_32bit = 1,
		.limit_in_pages = 1,
		.useable = 1,
	};
	struct desc_struct *desc = t->thread.tls_array;
	desc += tls;
	fill_ldt(desc, &ud);
}

static inline u32 read_32bit_tls(struct task_struct *t, int tls)
{
	return get_desc_base(&t->thread.tls_array[tls]);
}

int copy_thread(unsigned long clone_flags, unsigned long sp,
		unsigned long unused,
	struct task_struct *p, struct pt_regs *regs)
{
	int err;
	struct pt_regs *childregs;
	struct task_struct *me = current;

	childregs = ((struct pt_regs *)
			(THREAD_SIZE + task_stack_page(p))) - 1;
	*childregs = *regs;

	childregs->ax = 0;
	if (user_mode(regs))
		childregs->sp = sp;
	else
		childregs->sp = (unsigned long)childregs;

	p->thread.sp = (unsigned long) childregs;
	p->thread.sp0 = (unsigned long) (childregs+1);
	p->thread.usersp = me->thread.usersp;

	set_tsk_thread_flag(p, TIF_FORK);

	p->fpu_counter = 0;
	p->thread.io_bitmap_ptr = NULL;

	savesegment(gs, p->thread.gsindex);
	p->thread.gs = p->thread.gsindex ? 0 : me->thread.gs;
	savesegment(fs, p->thread.fsindex);
	p->thread.fs = p->thread.fsindex ? 0 : me->thread.fs;
	savesegment(es, p->thread.es);
	savesegment(ds, p->thread.ds);

	err = -ENOMEM;
	memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));

	if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) {
		p->thread.io_bitmap_ptr = kmemdup(me->thread.io_bitmap_ptr,
						  IO_BITMAP_BYTES, GFP_KERNEL);
		if (!p->thread.io_bitmap_ptr) {
			p->thread.io_bitmap_max = 0;
			return -ENOMEM;
		}
		set_tsk_thread_flag(p, TIF_IO_BITMAP);
	}

	/*
	 * Set a new TLS for the child thread?
	 */
	if (clone_flags & CLONE_SETTLS) {
#ifdef CONFIG_IA32_EMULATION
		if (test_thread_flag(TIF_IA32))
			err = do_set_thread_area(p, -1,
				(struct user_desc __user *)childregs->si, 0);
		else
#endif
			err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8);
		if (err)
			goto out;
	}
	err = 0;
out:
	if (err && p->thread.io_bitmap_ptr) {
		kfree(p->thread.io_bitmap_ptr);
		p->thread.io_bitmap_max = 0;
	}

	return err;
}

static void
start_thread_common(struct pt_regs *regs, unsigned long new_ip,
		    unsigned long new_sp,
		    unsigned int _cs, unsigned int _ss, unsigned int _ds)
{
	loadsegment(fs, 0);
	loadsegment(es, _ds);
	loadsegment(ds, _ds);
	load_gs_index(0);
	current->thread.usersp	= new_sp;
	regs->ip		= new_ip;
	regs->sp		= new_sp;
	this_cpu_write(old_rsp, new_sp);
	regs->cs		= _cs;
	regs->ss		= _ss;
	regs->flags		= X86_EFLAGS_IF;
}

void
start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
{
	start_thread_common(regs, new_ip, new_sp,
			    __USER_CS, __USER_DS, 0);
}

#ifdef CONFIG_IA32_EMULATION
void start_thread_ia32(struct pt_regs *regs, u32 new_ip, u32 new_sp)
{
	start_thread_common(regs, new_ip, new_sp,
			    test_thread_flag(TIF_X32)
			    ? __USER_CS : __USER32_CS,
			    __USER_DS, __USER_DS);
}
#endif

/*
 *	switch_to(x,y) should switch tasks from x to y.
 *
 * This could still be optimized:
 * - fold all the options into a flag word and test it with a single test.
 * - could test fs/gs bitsliced
 *
 * Kprobes not supported here. Set the probe on schedule instead.
 * Function graph tracer not supported too.
 */
__notrace_funcgraph struct task_struct *
__switch_to(struct task_struct *prev_p, struct task_struct *next_p)
{
	struct thread_struct *prev = &prev_p->thread;
	struct thread_struct *next = &next_p->thread;
	int cpu = smp_processor_id();
	struct tss_struct *tss = &per_cpu(init_tss, cpu);
	unsigned fsindex, gsindex;
	fpu_switch_t fpu;

	fpu = switch_fpu_prepare(prev_p, next_p, cpu);

	/*
	 * Reload esp0, LDT and the page table pointer:
	 */
	load_sp0(tss, next);

	/*
	 * Switch DS and ES.
	 * This won't pick up thread selector changes, but I guess that is ok.
	 */
	savesegment(es, prev->es);
	if (unlikely(next->es | prev->es))
		loadsegment(es, next->es);

	savesegment(ds, prev->ds);
	if (unlikely(next->ds | prev->ds))
		loadsegment(ds, next->ds);


	/* We must save %fs and %gs before load_TLS() because
	 * %fs and %gs may be cleared by load_TLS().
	 *
	 * (e.g. xen_load_tls())
	 */
	savesegment(fs, fsindex);
	savesegment(gs, gsindex);

	load_TLS(next, cpu);

	/*
	 * Leave lazy mode, flushing any hypercalls made here.
	 * This must be done before restoring TLS segments so
	 * the GDT and LDT are properly updated, and must be
	 * done before math_state_restore, so the TS bit is up
	 * to date.
	 */
	arch_end_context_switch(next_p);

	/*
	 * Switch FS and GS.
	 *
	 * Segment register != 0 always requires a reload.  Also
	 * reload when it has changed.  When prev process used 64bit
	 * base always reload to avoid an information leak.
	 */
	if (unlikely(fsindex | next->fsindex | prev->fs)) {
		loadsegment(fs, next->fsindex);
		/*
		 * Check if the user used a selector != 0; if yes
		 *  clear 64bit base, since overloaded base is always
		 *  mapped to the Null selector
		 */
		if (fsindex)
			prev->fs = 0;
	}
	/* when next process has a 64bit base use it */
	if (next->fs)
		wrmsrl(MSR_FS_BASE, next->fs);
	prev->fsindex = fsindex;

	if (unlikely(gsindex | next->gsindex | prev->gs)) {
		load_gs_index(next->gsindex);
		if (gsindex)
			prev->gs = 0;
	}
	if (next->gs)
		wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
	prev->gsindex = gsindex;

	switch_fpu_finish(next_p, fpu);

	/*
	 * Switch the PDA and FPU contexts.
	 */
	prev->usersp = this_cpu_read(old_rsp);
	this_cpu_write(old_rsp, next->usersp);
	this_cpu_write(current_task, next_p);

	this_cpu_write(kernel_stack,
		  (unsigned long)task_stack_page(next_p) +
		  THREAD_SIZE - KERNEL_STACK_OFFSET);

	/*
	 * Now maybe reload the debug registers and handle I/O bitmaps
	 */
	if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT ||
		     task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV))
		__switch_to_xtra(prev_p, next_p, tss);

	return prev_p;
}

void set_personality_64bit(void)
{
	/* inherit personality from parent */

	/* Make sure to be in 64bit mode */
	clear_thread_flag(TIF_IA32);
	clear_thread_flag(TIF_ADDR32);
	clear_thread_flag(TIF_X32);

	/* Ensure the corresponding mm is not marked. */
	if (current->mm)
		current->mm->context.ia32_compat = 0;

	/* TBD: overwrites user setup. Should have two bits.
	   But 64bit processes have always behaved this way,
	   so it's not too bad. The main problem is just that
	   32bit childs are affected again. */
	current->personality &= ~READ_IMPLIES_EXEC;
}

void set_personality_ia32(bool x32)
{
	/* inherit personality from parent */

	/* Make sure to be in 32bit mode */
	set_thread_flag(TIF_ADDR32);

	/* Mark the associated mm as containing 32-bit tasks. */
	if (current->mm)
		current->mm->context.ia32_compat = 1;

	if (x32) {
		clear_thread_flag(TIF_IA32);
		set_thread_flag(TIF_X32);
		current->personality &= ~READ_IMPLIES_EXEC;
		/* is_compat_task() uses the presence of the x32
		   syscall bit flag to determine compat status */
		current_thread_info()->status &= ~TS_COMPAT;
	} else {
		set_thread_flag(TIF_IA32);
		clear_thread_flag(TIF_X32);
		current->personality |= force_personality32;
		/* Prepare the first "return" to user space */
		current_thread_info()->status |= TS_COMPAT;
	}
}
EXPORT_SYMBOL_GPL(set_personality_ia32);

unsigned long get_wchan(struct task_struct *p)
{
	unsigned long stack;
	u64 fp, ip;
	int count = 0;

	if (!p || p == current || p->state == TASK_RUNNING)
		return 0;
	stack = (unsigned long)task_stack_page(p);
	if (p->thread.sp < stack || p->thread.sp >= stack+THREAD_SIZE)
		return 0;
	fp = *(u64 *)(p->thread.sp);
	do {
		if (fp < (unsigned long)stack ||
		    fp >= (unsigned long)stack+THREAD_SIZE)
			return 0;
		ip = *(u64 *)(fp+8);
		if (!in_sched_functions(ip))
			return ip;
		fp = *(u64 *)fp;
	} while (count++ < 16);
	return 0;
}

long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
{
	int ret = 0;
	int doit = task == current;
	int cpu;

	switch (code) {
	case ARCH_SET_GS:
		if (addr >= TASK_SIZE_OF(task))
			return -EPERM;
		cpu = get_cpu();
		/* handle small bases via the GDT because that's faster to
		   switch. */
		if (addr <= 0xffffffff) {
			set_32bit_tls(task, GS_TLS, addr);
			if (doit) {
				load_TLS(&task->thread, cpu);
				load_gs_index(GS_TLS_SEL);
			}
			task->thread.gsindex = GS_TLS_SEL;
			task->thread.gs = 0;
		} else {
			task->thread.gsindex = 0;
			task->thread.gs = addr;
			if (doit) {
				load_gs_index(0);
				ret = wrmsrl_safe(MSR_KERNEL_GS_BASE, addr);
			}
		}
		put_cpu();
		break;
	case ARCH_SET_FS:
		/* Not strictly needed for fs, but do it for symmetry
		   with gs */
		if (addr >= TASK_SIZE_OF(task))
			return -EPERM;
		cpu = get_cpu();
		/* handle small bases via the GDT because that's faster to
		   switch. */
		if (addr <= 0xffffffff) {
			set_32bit_tls(task, FS_TLS, addr);
			if (doit) {
				load_TLS(&task->thread, cpu);
				loadsegment(fs, FS_TLS_SEL);
			}
			task->thread.fsindex = FS_TLS_SEL;
			task->thread.fs = 0;
		} else {
			task->thread.fsindex = 0;
			task->thread.fs = addr;
			if (doit) {
				/* set the selector to 0 to not confuse
				   __switch_to */
				loadsegment(fs, 0);
				ret = wrmsrl_safe(MSR_FS_BASE, addr);
			}
		}
		put_cpu();
		break;
	case ARCH_GET_FS: {
		unsigned long base;
		if (task->thread.fsindex == FS_TLS_SEL)
			base = read_32bit_tls(task, FS_TLS);
		else if (doit)
			rdmsrl(MSR_FS_BASE, base);
		else
			base = task->thread.fs;
		ret = put_user(base, (unsigned long __user *)addr);
		break;
	}
	case ARCH_GET_GS: {
		unsigned long base;
		unsigned gsindex;
		if (task->thread.gsindex == GS_TLS_SEL)
			base = read_32bit_tls(task, GS_TLS);
		else if (doit) {
			savesegment(gs, gsindex);
			if (gsindex)
				rdmsrl(MSR_KERNEL_GS_BASE, base);
			else
				base = task->thread.gs;
		} else
			base = task->thread.gs;
		ret = put_user(base, (unsigned long __user *)addr);
		break;
	}

	default:
		ret = -EINVAL;
		break;
	}

	return ret;
}

long sys_arch_prctl(int code, unsigned long addr)
{
	return do_arch_prctl(current, code, addr);
}

unsigned long KSTK_ESP(struct task_struct *task)
{
	return (test_tsk_thread_flag(task, TIF_IA32)) ?
			(task_pt_regs(task)->sp) : ((task)->thread.usersp);
}
Commit	Line	Data
1da177e4	1	/*
1da177e4 LT	2	* Copyright (C) 1995 Linus Torvalds
	3	*
	4	* Pentium III FXSR, SSE support
	5	* Gareth Hughes <gareth@valinux.com>, May 2000
6612538c	6	*
1da177e4 LT	7	* X86-64 port
1da177e4 LT	8	* Andi Kleen.
76e4f660 AR	9	*
76e4f660 AR	10	* CPU hotplug support - ashok.raj@intel.com
1da177e4 LT	11	*/
	12
	13	/*
	14	* This file handles the architecture-dependent parts of process handling..
	15	*/
	16
76e4f660	17	#include <linux/cpu.h>
1da177e4 LT	18	#include <linux/errno.h>
1da177e4 LT	19	#include <linux/sched.h>
6612538c	20	#include <linux/fs.h>
1da177e4 LT	21	#include <linux/kernel.h>
	22	#include <linux/mm.h>
	23	#include <linux/elfcore.h>
	24	#include <linux/smp.h>
	25	#include <linux/slab.h>
	26	#include <linux/user.h>
1da177e4 LT	27	#include <linux/interrupt.h>
1da177e4 LT	28	#include <linux/delay.h>
6612538c	29	#include <linux/module.h>
1da177e4	30	#include <linux/ptrace.h>
95833c83	31	#include <linux/notifier.h>
c6fd91f0	32	#include <linux/kprobes.h>
1eeb66a1	33	#include <linux/kdebug.h>
529e25f6	34	#include <linux/prctl.h>
7de08b4e GP	35	#include <linux/uaccess.h>
7de08b4e GP	36	#include <linux/io.h>
8b96f011	37	#include <linux/ftrace.h>
1da177e4	38
1da177e4	39	#include <asm/pgtable.h>
1da177e4 LT	40	#include <asm/processor.h>
1da177e4 LT	41	#include <asm/i387.h>
1361b83a	42	#include <asm/fpu-internal.h>
1da177e4	43	#include <asm/mmu_context.h>
1da177e4	44	#include <asm/prctl.h>
1da177e4 LT	45	#include <asm/desc.h>
	46	#include <asm/proto.h>
	47	#include <asm/ia32.h>
95833c83	48	#include <asm/idle.h>
bbc1f698	49	#include <asm/syscalls.h>
66cb5917	50	#include <asm/debugreg.h>
f05e798a	51	#include <asm/switch_to.h>
1da177e4 LT	52
	53	asmlinkage extern void ret_from_fork(void);
	54
3d1e42a7	55	DEFINE_PER_CPU(unsigned long, old_rsp);
1da177e4	56
6612538c	57	/* Prints also some state that isn't saved in the pt_regs */
e2ce07c8	58	void __show_regs(struct pt_regs *regs, int all)
1da177e4 LT	59	{
1da177e4 LT	60	unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
bb1995d5	61	unsigned long d0, d1, d2, d3, d6, d7;
6612538c HS	62	unsigned int fsindex, gsindex;
6612538c HS	63	unsigned int ds, cs, es;
814e2c84 AI	64
814e2c84 AI	65	show_regs_common();
d015a092	66	printk(KERN_DEFAULT "RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->ip);
aafbd7eb	67	printk_address(regs->ip, 1);
d015a092	68	printk(KERN_DEFAULT "RSP: %04lx:%016lx EFLAGS: %08lx\n", regs->ss,
8092c654	69	regs->sp, regs->flags);
d015a092	70	printk(KERN_DEFAULT "RAX: %016lx RBX: %016lx RCX: %016lx\n",
65ea5b03	71	regs->ax, regs->bx, regs->cx);
d015a092	72	printk(KERN_DEFAULT "RDX: %016lx RSI: %016lx RDI: %016lx\n",
65ea5b03	73	regs->dx, regs->si, regs->di);
d015a092	74	printk(KERN_DEFAULT "RBP: %016lx R08: %016lx R09: %016lx\n",
65ea5b03	75	regs->bp, regs->r8, regs->r9);
d015a092	76	printk(KERN_DEFAULT "R10: %016lx R11: %016lx R12: %016lx\n",
7de08b4e	77	regs->r10, regs->r11, regs->r12);
d015a092	78	printk(KERN_DEFAULT "R13: %016lx R14: %016lx R15: %016lx\n",
7de08b4e	79	regs->r13, regs->r14, regs->r15);
1da177e4	80
7de08b4e GP	81	asm("movl %%ds,%0" : "=r" (ds));
	82	asm("movl %%cs,%0" : "=r" (cs));
	83	asm("movl %%es,%0" : "=r" (es));
1da177e4 LT	84	asm("movl %%fs,%0" : "=r" (fsindex));
	85	asm("movl %%gs,%0" : "=r" (gsindex));
	86
	87	rdmsrl(MSR_FS_BASE, fs);
7de08b4e GP	88	rdmsrl(MSR_GS_BASE, gs);
7de08b4e GP	89	rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
1da177e4	90
e2ce07c8 PE	91	if (!all)
e2ce07c8 PE	92	return;
1da177e4	93
f51c9452 GOC	94	cr0 = read_cr0();
	95	cr2 = read_cr2();
	96	cr3 = read_cr3();
	97	cr4 = read_cr4();
1da177e4	98
d015a092	99	printk(KERN_DEFAULT "FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
7de08b4e	100	fs, fsindex, gs, gsindex, shadowgs);
d015a092	101	printk(KERN_DEFAULT "CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds,
8092c654	102	es, cr0);
d015a092	103	printk(KERN_DEFAULT "CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3,
8092c654	104	cr4);
bb1995d5 AS	105
	106	get_debugreg(d0, 0);
	107	get_debugreg(d1, 1);
	108	get_debugreg(d2, 2);
d015a092	109	printk(KERN_DEFAULT "DR0: %016lx DR1: %016lx DR2: %016lx\n", d0, d1, d2);
bb1995d5 AS	110	get_debugreg(d3, 3);
	111	get_debugreg(d6, 6);
	112	get_debugreg(d7, 7);
d015a092	113	printk(KERN_DEFAULT "DR3: %016lx DR6: %016lx DR7: %016lx\n", d3, d6, d7);
1da177e4 LT	114	}
1da177e4 LT	115
1da177e4 LT	116	void release_thread(struct task_struct *dead_task)
	117	{
	118	if (dead_task->mm) {
	119	if (dead_task->mm->context.size) {
c767a54b JP	120	pr_warn("WARNING: dead process %8s still has LDT? <%p/%d>\n",
	121	dead_task->comm,
	122	dead_task->mm->context.ldt,
	123	dead_task->mm->context.size);
1da177e4 LT	124	BUG();
	125	}
	126	}
	127	}
	128
	129	static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
	130	{
6612538c	131	struct user_desc ud = {
1da177e4 LT	132	.base_addr = addr,
	133	.limit = 0xfffff,
	134	.seg_32bit = 1,
	135	.limit_in_pages = 1,
	136	.useable = 1,
	137	};
ade1af77	138	struct desc_struct *desc = t->thread.tls_array;
1da177e4	139	desc += tls;
80fbb69a	140	fill_ldt(desc, &ud);
1da177e4 LT	141	}
	142
	143	static inline u32 read_32bit_tls(struct task_struct *t, int tls)
	144	{
91394eb0	145	return get_desc_base(&t->thread.tls_array[tls]);
1da177e4 LT	146	}
1da177e4 LT	147
6f2c55b8	148	int copy_thread(unsigned long clone_flags, unsigned long sp,
1da177e4	149	unsigned long unused,
7de08b4e	150	struct task_struct p, struct pt_regs regs)
1da177e4 LT	151	{
1da177e4 LT	152	int err;
7de08b4e	153	struct pt_regs *childregs;
1da177e4 LT	154	struct task_struct *me = current;
1da177e4 LT	155
a88cde13	156	childregs = ((struct pt_regs *)
57eafdc2	157	(THREAD_SIZE + task_stack_page(p))) - 1;
1da177e4 LT	158	childregs = regs;
1da177e4 LT	159
65ea5b03	160	childregs->ax = 0;
fa4b8f84 BG	161	if (user_mode(regs))
	162	childregs->sp = sp;
	163	else
65ea5b03	164	childregs->sp = (unsigned long)childregs;
1da177e4	165
faca6227 PA	166	p->thread.sp = (unsigned long) childregs;
	167	p->thread.sp0 = (unsigned long) (childregs+1);
	168	p->thread.usersp = me->thread.usersp;
1da177e4	169
e4f17c43	170	set_tsk_thread_flag(p, TIF_FORK);
1da177e4	171
cea20ca3	172	p->fpu_counter = 0;
66cb5917	173	p->thread.io_bitmap_ptr = NULL;
1da177e4	174
ada85708	175	savesegment(gs, p->thread.gsindex);
7ce5a2b9	176	p->thread.gs = p->thread.gsindex ? 0 : me->thread.gs;
ada85708	177	savesegment(fs, p->thread.fsindex);
7ce5a2b9	178	p->thread.fs = p->thread.fsindex ? 0 : me->thread.fs;
ada85708 JF	179	savesegment(es, p->thread.es);
ada85708 JF	180	savesegment(ds, p->thread.ds);
1da177e4	181
66cb5917	182	err = -ENOMEM;
24f1e32c	183	memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
66cb5917	184
d3a4f48d	185	if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) {
cced4022 TM	186	p->thread.io_bitmap_ptr = kmemdup(me->thread.io_bitmap_ptr,
cced4022 TM	187	IO_BITMAP_BYTES, GFP_KERNEL);
1da177e4 LT	188	if (!p->thread.io_bitmap_ptr) {
	189	p->thread.io_bitmap_max = 0;
	190	return -ENOMEM;
	191	}
d3a4f48d	192	set_tsk_thread_flag(p, TIF_IO_BITMAP);
6612538c	193	}
1da177e4 LT	194
	195	/*
	196	* Set a new TLS for the child thread?
	197	*/
	198	if (clone_flags & CLONE_SETTLS) {
	199	#ifdef CONFIG_IA32_EMULATION
	200	if (test_thread_flag(TIF_IA32))
efd1ca52	201	err = do_set_thread_area(p, -1,
65ea5b03	202	(struct user_desc __user *)childregs->si, 0);
7de08b4e GP	203	else
	204	#endif
	205	err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8);
	206	if (err)
1da177e4 LT	207	goto out;
	208	}
	209	err = 0;
	210	out:
	211	if (err && p->thread.io_bitmap_ptr) {
	212	kfree(p->thread.io_bitmap_ptr);
	213	p->thread.io_bitmap_max = 0;
	214	}
66cb5917	215
1da177e4 LT	216	return err;
	217	}
	218
e634d8fc PA	219	static void
	220	start_thread_common(struct pt_regs *regs, unsigned long new_ip,
	221	unsigned long new_sp,
	222	unsigned int _cs, unsigned int _ss, unsigned int _ds)
513ad84b	223	{
ada85708	224	loadsegment(fs, 0);
e634d8fc PA	225	loadsegment(es, _ds);
e634d8fc PA	226	loadsegment(ds, _ds);
513ad84b	227	load_gs_index(0);
42dfc43e	228	current->thread.usersp = new_sp;
513ad84b IM	229	regs->ip = new_ip;
513ad84b IM	230	regs->sp = new_sp;
c6ae41e7	231	this_cpu_write(old_rsp, new_sp);
e634d8fc PA	232	regs->cs = _cs;
e634d8fc PA	233	regs->ss = _ss;
a6f05a6a	234	regs->flags = X86_EFLAGS_IF;
513ad84b	235	}
e634d8fc PA	236
	237	void
	238	start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
	239	{
	240	start_thread_common(regs, new_ip, new_sp,
	241	__USER_CS, __USER_DS, 0);
	242	}
513ad84b	243
a6f05a6a PA	244	#ifdef CONFIG_IA32_EMULATION
	245	void start_thread_ia32(struct pt_regs *regs, u32 new_ip, u32 new_sp)
	246	{
e634d8fc	247	start_thread_common(regs, new_ip, new_sp,
d1a797f3 PA	248	test_thread_flag(TIF_X32)
	249	? __USER_CS : __USER32_CS,
	250	__USER_DS, __USER_DS);
a6f05a6a PA	251	}
a6f05a6a PA	252	#endif
513ad84b	253
1da177e4 LT	254	/*
	255	* switch_to(x,y) should switch tasks from x to y.
	256	*
6612538c	257	* This could still be optimized:
1da177e4 LT	258	* - fold all the options into a flag word and test it with a single test.
1da177e4 LT	259	* - could test fs/gs bitsliced
099f318b AK	260	*
099f318b AK	261	* Kprobes not supported here. Set the probe on schedule instead.
8b96f011	262	* Function graph tracer not supported too.
1da177e4	263	*/
8b96f011	264	__notrace_funcgraph struct task_struct *
a88cde13	265	__switch_to(struct task_struct prev_p, struct task_struct next_p)
1da177e4	266	{
87b935a0 JF	267	struct thread_struct *prev = &prev_p->thread;
87b935a0 JF	268	struct thread_struct *next = &next_p->thread;
6612538c	269	int cpu = smp_processor_id();
1da177e4	270	struct tss_struct *tss = &per_cpu(init_tss, cpu);
478de5a9	271	unsigned fsindex, gsindex;
34ddc81a	272	fpu_switch_t fpu;
e07e23e1	273
7e16838d	274	fpu = switch_fpu_prepare(prev_p, next_p, cpu);
4903062b	275
1da177e4 LT	276	/*
	277	* Reload esp0, LDT and the page table pointer:
	278	*/
7818a1e0	279	load_sp0(tss, next);
1da177e4	280
7de08b4e	281	/*
1da177e4 LT	282	* Switch DS and ES.
	283	* This won't pick up thread selector changes, but I guess that is ok.
	284	*/
ada85708	285	savesegment(es, prev->es);
1da177e4	286	if (unlikely(next->es \| prev->es))
7de08b4e	287	loadsegment(es, next->es);
ada85708 JF	288
ada85708 JF	289	savesegment(ds, prev->ds);
1da177e4 LT	290	if (unlikely(next->ds \| prev->ds))
	291	loadsegment(ds, next->ds);
	292
478de5a9 JF	293
	294	/* We must save %fs and %gs before load_TLS() because
	295	* %fs and %gs may be cleared by load_TLS().
	296	*
	297	* (e.g. xen_load_tls())
	298	*/
	299	savesegment(fs, fsindex);
	300	savesegment(gs, gsindex);
	301
1da177e4 LT	302	load_TLS(next, cpu);
1da177e4 LT	303
3fe0a63e JF	304	/*
	305	* Leave lazy mode, flushing any hypercalls made here.
	306	* This must be done before restoring TLS segments so
	307	* the GDT and LDT are properly updated, and must be
	308	* done before math_state_restore, so the TS bit is up
	309	* to date.
	310	*/
224101ed	311	arch_end_context_switch(next_p);
3fe0a63e	312
7de08b4e	313	/*
1da177e4	314	* Switch FS and GS.
87b935a0 JF	315	*
	316	* Segment register != 0 always requires a reload. Also
	317	* reload when it has changed. When prev process used 64bit
	318	* base always reload to avoid an information leak.
1da177e4	319	*/
87b935a0 JF	320	if (unlikely(fsindex \| next->fsindex \| prev->fs)) {
87b935a0 JF	321	loadsegment(fs, next->fsindex);
7de08b4e	322	/*
87b935a0 JF	323	* Check if the user used a selector != 0; if yes
	324	* clear 64bit base, since overloaded base is always
	325	* mapped to the Null selector
	326	*/
	327	if (fsindex)
7de08b4e	328	prev->fs = 0;
1da177e4	329	}
87b935a0 JF	330	/* when next process has a 64bit base use it */
	331	if (next->fs)
	332	wrmsrl(MSR_FS_BASE, next->fs);
	333	prev->fsindex = fsindex;
	334
	335	if (unlikely(gsindex \| next->gsindex \| prev->gs)) {
	336	load_gs_index(next->gsindex);
	337	if (gsindex)
7de08b4e	338	prev->gs = 0;
1da177e4	339	}
87b935a0 JF	340	if (next->gs)
	341	wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
	342	prev->gsindex = gsindex;
1da177e4	343
34ddc81a LT	344	switch_fpu_finish(next_p, fpu);
34ddc81a LT	345
7de08b4e	346	/*
45948d77	347	* Switch the PDA and FPU contexts.
1da177e4	348	*/
c6ae41e7 AS	349	prev->usersp = this_cpu_read(old_rsp);
	350	this_cpu_write(old_rsp, next->usersp);
	351	this_cpu_write(current_task, next_p);
18bd057b	352
c6ae41e7	353	this_cpu_write(kernel_stack,
87b935a0	354	(unsigned long)task_stack_page(next_p) +
9af45651	355	THREAD_SIZE - KERNEL_STACK_OFFSET);
1da177e4 LT	356
1da177e4 LT	357	/*
d3a4f48d	358	* Now maybe reload the debug registers and handle I/O bitmaps
1da177e4	359	*/
eee3af4a MM	360	if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT \|\|
eee3af4a MM	361	task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV))
d3a4f48d	362	__switch_to_xtra(prev_p, next_p, tss);
1da177e4 LT	363
	364	return prev_p;
	365	}
	366
1da177e4 LT	367	void set_personality_64bit(void)
	368	{
	369	/* inherit personality from parent */
	370
	371	/* Make sure to be in 64bit mode */
6612538c	372	clear_thread_flag(TIF_IA32);
6bd33008	373	clear_thread_flag(TIF_ADDR32);
bb212724	374	clear_thread_flag(TIF_X32);
1da177e4	375
375906f8 SW	376	/* Ensure the corresponding mm is not marked. */
	377	if (current->mm)
	378	current->mm->context.ia32_compat = 0;
	379
1da177e4 LT	380	/* TBD: overwrites user setup. Should have two bits.
	381	But 64bit processes have always behaved this way,
	382	so it's not too bad. The main problem is just that
6612538c	383	32bit childs are affected again. */
1da177e4 LT	384	current->personality &= ~READ_IMPLIES_EXEC;
	385	}
	386
d1a797f3	387	void set_personality_ia32(bool x32)
05d43ed8 PA	388	{
	389	/* inherit personality from parent */
	390
	391	/* Make sure to be in 32bit mode */
6bd33008	392	set_thread_flag(TIF_ADDR32);
05d43ed8	393
375906f8 SW	394	/* Mark the associated mm as containing 32-bit tasks. */
	395	if (current->mm)
	396	current->mm->context.ia32_compat = 1;
	397
d1a797f3 PA	398	if (x32) {
	399	clear_thread_flag(TIF_IA32);
	400	set_thread_flag(TIF_X32);
	401	current->personality &= ~READ_IMPLIES_EXEC;
ce5f7a99 BP	402	/* is_compat_task() uses the presence of the x32
	403	syscall bit flag to determine compat status */
	404	current_thread_info()->status &= ~TS_COMPAT;
d1a797f3 PA	405	} else {
	406	set_thread_flag(TIF_IA32);
	407	clear_thread_flag(TIF_X32);
	408	current->personality \|= force_personality32;
	409	/* Prepare the first "return" to user space */
	410	current_thread_info()->status \|= TS_COMPAT;
	411	}
05d43ed8	412	}
febb72a6	413	EXPORT_SYMBOL_GPL(set_personality_ia32);
05d43ed8	414
1da177e4 LT	415	unsigned long get_wchan(struct task_struct *p)
	416	{
	417	unsigned long stack;
7de08b4e	418	u64 fp, ip;
1da177e4 LT	419	int count = 0;
1da177e4 LT	420
7de08b4e GP	421	if (!p \|\| p == current \|\| p->state == TASK_RUNNING)
7de08b4e GP	422	return 0;
57eafdc2	423	stack = (unsigned long)task_stack_page(p);
e1e23bb0	424	if (p->thread.sp < stack \|\| p->thread.sp >= stack+THREAD_SIZE)
1da177e4	425	return 0;
faca6227	426	fp = (u64 )(p->thread.sp);
7de08b4e	427	do {
a88cde13	428	if (fp < (unsigned long)stack \|\|
e1e23bb0	429	fp >= (unsigned long)stack+THREAD_SIZE)
7de08b4e	430	return 0;
65ea5b03 PA	431	ip = (u64 )(fp+8);
	432	if (!in_sched_functions(ip))
	433	return ip;
7de08b4e GP	434	fp = (u64 )fp;
7de08b4e GP	435	} while (count++ < 16);
1da177e4 LT	436	return 0;
	437	}
	438
	439	long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
7de08b4e GP	440	{
7de08b4e GP	441	int ret = 0;
1da177e4 LT	442	int doit = task == current;
	443	int cpu;
	444
7de08b4e	445	switch (code) {
1da177e4	446	case ARCH_SET_GS:
84929801	447	if (addr >= TASK_SIZE_OF(task))
7de08b4e	448	return -EPERM;
1da177e4	449	cpu = get_cpu();
7de08b4e	450	/* handle small bases via the GDT because that's faster to
1da177e4	451	switch. */
7de08b4e GP	452	if (addr <= 0xffffffff) {
	453	set_32bit_tls(task, GS_TLS, addr);
	454	if (doit) {
1da177e4	455	load_TLS(&task->thread, cpu);
7de08b4e	456	load_gs_index(GS_TLS_SEL);
1da177e4	457	}
7de08b4e	458	task->thread.gsindex = GS_TLS_SEL;
1da177e4	459	task->thread.gs = 0;
7de08b4e	460	} else {
1da177e4 LT	461	task->thread.gsindex = 0;
	462	task->thread.gs = addr;
	463	if (doit) {
a88cde13	464	load_gs_index(0);
715c85b1	465	ret = wrmsrl_safe(MSR_KERNEL_GS_BASE, addr);
7de08b4e	466	}
1da177e4 LT	467	}
	468	put_cpu();
	469	break;
	470	case ARCH_SET_FS:
	471	/* Not strictly needed for fs, but do it for symmetry
	472	with gs */
84929801	473	if (addr >= TASK_SIZE_OF(task))
6612538c	474	return -EPERM;
1da177e4	475	cpu = get_cpu();
6612538c	476	/* handle small bases via the GDT because that's faster to
1da177e4	477	switch. */
6612538c	478	if (addr <= 0xffffffff) {
1da177e4	479	set_32bit_tls(task, FS_TLS, addr);
6612538c HS	480	if (doit) {
6612538c HS	481	load_TLS(&task->thread, cpu);
ada85708	482	loadsegment(fs, FS_TLS_SEL);
1da177e4 LT	483	}
	484	task->thread.fsindex = FS_TLS_SEL;
	485	task->thread.fs = 0;
6612538c	486	} else {
1da177e4 LT	487	task->thread.fsindex = 0;
	488	task->thread.fs = addr;
	489	if (doit) {
	490	/* set the selector to 0 to not confuse
	491	__switch_to */
ada85708	492	loadsegment(fs, 0);
715c85b1	493	ret = wrmsrl_safe(MSR_FS_BASE, addr);
1da177e4 LT	494	}
	495	}
	496	put_cpu();
	497	break;
6612538c HS	498	case ARCH_GET_FS: {
6612538c HS	499	unsigned long base;
1da177e4 LT	500	if (task->thread.fsindex == FS_TLS_SEL)
1da177e4 LT	501	base = read_32bit_tls(task, FS_TLS);
a88cde13	502	else if (doit)
1da177e4	503	rdmsrl(MSR_FS_BASE, base);
a88cde13	504	else
1da177e4	505	base = task->thread.fs;
6612538c HS	506	ret = put_user(base, (unsigned long __user *)addr);
6612538c HS	507	break;
1da177e4	508	}
6612538c	509	case ARCH_GET_GS: {
1da177e4	510	unsigned long base;
97c2803c	511	unsigned gsindex;
1da177e4 LT	512	if (task->thread.gsindex == GS_TLS_SEL)
1da177e4 LT	513	base = read_32bit_tls(task, GS_TLS);
97c2803c	514	else if (doit) {
ada85708	515	savesegment(gs, gsindex);
97c2803c JB	516	if (gsindex)
	517	rdmsrl(MSR_KERNEL_GS_BASE, base);
	518	else
	519	base = task->thread.gs;
7de08b4e	520	} else
1da177e4	521	base = task->thread.gs;
6612538c	522	ret = put_user(base, (unsigned long __user *)addr);
1da177e4 LT	523	break;
	524	}
	525
	526	default:
	527	ret = -EINVAL;
	528	break;
6612538c	529	}
1da177e4	530
6612538c HS	531	return ret;
6612538c HS	532	}
1da177e4 LT	533
	534	long sys_arch_prctl(int code, unsigned long addr)
	535	{
	536	return do_arch_prctl(current, code, addr);
1da177e4 LT	537	}
1da177e4 LT	538
89240ba0 SS	539	unsigned long KSTK_ESP(struct task_struct *task)
	540	{
	541	return (test_tsk_thread_flag(task, TIF_IA32)) ?
	542	(task_pt_regs(task)->sp) : ((task)->thread.usersp);
	543	}