[mirror_ubuntu-jammy-kernel.git] / arch / x86 / kernel / process.c

// SPDX-License-Identifier: GPL-2.0
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/prctl.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/sched/idle.h>
#include <linux/sched/debug.h>
#include <linux/sched/task.h>
#include <linux/sched/task_stack.h>
#include <linux/init.h>
#include <linux/export.h>
#include <linux/pm.h>
#include <linux/tick.h>
#include <linux/random.h>
#include <linux/user-return-notifier.h>
#include <linux/dmi.h>
#include <linux/utsname.h>
#include <linux/stackprotector.h>
#include <linux/cpuidle.h>
#include <trace/events/power.h>
#include <linux/hw_breakpoint.h>
#include <asm/cpu.h>
#include <asm/apic.h>
#include <asm/syscalls.h>
#include <linux/uaccess.h>
#include <asm/mwait.h>
#include <asm/fpu/internal.h>
#include <asm/debugreg.h>
#include <asm/nmi.h>
#include <asm/tlbflush.h>
#include <asm/mce.h>
#include <asm/vm86.h>
#include <asm/switch_to.h>
#include <asm/desc.h>
#include <asm/prctl.h>

/*
 * per-CPU TSS segments. Threads are completely 'soft' on Linux,
 * no more per-task TSS's. The TSS size is kept cacheline-aligned
 * so they are allowed to end up in the .data..cacheline_aligned
 * section. Since TSS's are completely CPU-local, we want them
 * on exact cacheline boundaries, to eliminate cacheline ping-pong.
 */
__visible DEFINE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw) = {
	.x86_tss = {
		/*
		 * .sp0 is only used when entering ring 0 from a lower
		 * privilege level.  Since the init task never runs anything
		 * but ring 0 code, there is no need for a valid value here.
		 * Poison it.
		 */
		.sp0 = (1UL << (BITS_PER_LONG-1)) + 1,

#ifdef CONFIG_X86_64
		/*
		 * .sp1 is cpu_current_top_of_stack.  The init task never
		 * runs user code, but cpu_current_top_of_stack should still
		 * be well defined before the first context switch.
		 */
		.sp1 = TOP_OF_INIT_STACK,
#endif

#ifdef CONFIG_X86_32
		.ss0 = __KERNEL_DS,
		.ss1 = __KERNEL_CS,
		.io_bitmap_base	= INVALID_IO_BITMAP_OFFSET,
#endif
	 },
#ifdef CONFIG_X86_32
	 /*
	  * Note that the .io_bitmap member must be extra-big. This is because
	  * the CPU will access an additional byte beyond the end of the IO
	  * permission bitmap. The extra byte must be all 1 bits, and must
	  * be within the limit.
	  */
	.io_bitmap		= { [0 ... IO_BITMAP_LONGS] = ~0 },
#endif
};
EXPORT_PER_CPU_SYMBOL(cpu_tss_rw);

DEFINE_PER_CPU(bool, __tss_limit_invalid);
EXPORT_PER_CPU_SYMBOL_GPL(__tss_limit_invalid);

/*
 * this gets called so that we can store lazy state into memory and copy the
 * current task into the new thread.
 */
int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
{
	memcpy(dst, src, arch_task_struct_size);
#ifdef CONFIG_VM86
	dst->thread.vm86 = NULL;
#endif

	return fpu__copy(&dst->thread.fpu, &src->thread.fpu);
}

/*
 * Free current thread data structures etc..
 */
void exit_thread(struct task_struct *tsk)
{
	struct thread_struct *t = &tsk->thread;
	unsigned long *bp = t->io_bitmap_ptr;
	struct fpu *fpu = &t->fpu;

	if (bp) {
		struct tss_struct *tss = &per_cpu(cpu_tss_rw, get_cpu());

		t->io_bitmap_ptr = NULL;
		clear_thread_flag(TIF_IO_BITMAP);
		/*
		 * Careful, clear this in the TSS too:
		 */
		memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
		t->io_bitmap_max = 0;
		put_cpu();
		kfree(bp);
	}

	free_vm86(t);

	fpu__drop(fpu);
}

void flush_thread(void)
{
	struct task_struct *tsk = current;

	flush_ptrace_hw_breakpoint(tsk);
	memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));

	fpu__clear(&tsk->thread.fpu);
}

void disable_TSC(void)
{
	preempt_disable();
	if (!test_and_set_thread_flag(TIF_NOTSC))
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOTSC in the current running context.
		 */
		cr4_set_bits(X86_CR4_TSD);
	preempt_enable();
}

static void enable_TSC(void)
{
	preempt_disable();
	if (test_and_clear_thread_flag(TIF_NOTSC))
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOTSC in the current running context.
		 */
		cr4_clear_bits(X86_CR4_TSD);
	preempt_enable();
}

int get_tsc_mode(unsigned long adr)
{
	unsigned int val;

	if (test_thread_flag(TIF_NOTSC))
		val = PR_TSC_SIGSEGV;
	else
		val = PR_TSC_ENABLE;

	return put_user(val, (unsigned int __user *)adr);
}

int set_tsc_mode(unsigned int val)
{
	if (val == PR_TSC_SIGSEGV)
		disable_TSC();
	else if (val == PR_TSC_ENABLE)
		enable_TSC();
	else
		return -EINVAL;

	return 0;
}

DEFINE_PER_CPU(u64, msr_misc_features_shadow);

static void set_cpuid_faulting(bool on)
{
	u64 msrval;

	msrval = this_cpu_read(msr_misc_features_shadow);
	msrval &= ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT;
	msrval |= (on << MSR_MISC_FEATURES_ENABLES_CPUID_FAULT_BIT);
	this_cpu_write(msr_misc_features_shadow, msrval);
	wrmsrl(MSR_MISC_FEATURES_ENABLES, msrval);
}

static void disable_cpuid(void)
{
	preempt_disable();
	if (!test_and_set_thread_flag(TIF_NOCPUID)) {
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOCPUID in the current running context.
		 */
		set_cpuid_faulting(true);
	}
	preempt_enable();
}

static void enable_cpuid(void)
{
	preempt_disable();
	if (test_and_clear_thread_flag(TIF_NOCPUID)) {
		/*
		 * Must flip the CPU state synchronously with
		 * TIF_NOCPUID in the current running context.
		 */
		set_cpuid_faulting(false);
	}
	preempt_enable();
}

static int get_cpuid_mode(void)
{
	return !test_thread_flag(TIF_NOCPUID);
}

static int set_cpuid_mode(struct task_struct *task, unsigned long cpuid_enabled)
{
	if (!static_cpu_has(X86_FEATURE_CPUID_FAULT))
		return -ENODEV;

	if (cpuid_enabled)
		enable_cpuid();
	else
		disable_cpuid();

	return 0;
}

/*
 * Called immediately after a successful exec.
 */
void arch_setup_new_exec(void)
{
	/* If cpuid was previously disabled for this task, re-enable it. */
	if (test_thread_flag(TIF_NOCPUID))
		enable_cpuid();
}

static inline void switch_to_bitmap(struct tss_struct *tss,
				    struct thread_struct *prev,
				    struct thread_struct *next,
				    unsigned long tifp, unsigned long tifn)
{
	if (tifn & _TIF_IO_BITMAP) {
		/*
		 * Copy the relevant range of the IO bitmap.
		 * Normally this is 128 bytes or less:
		 */
		memcpy(tss->io_bitmap, next->io_bitmap_ptr,
		       max(prev->io_bitmap_max, next->io_bitmap_max));
		/*
		 * Make sure that the TSS limit is correct for the CPU
		 * to notice the IO bitmap.
		 */
		refresh_tss_limit();
	} else if (tifp & _TIF_IO_BITMAP) {
		/*
		 * Clear any possible leftover bits:
		 */
		memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
	}
}

void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
		      struct tss_struct *tss)
{
	struct thread_struct *prev, *next;
	unsigned long tifp, tifn;

	prev = &prev_p->thread;
	next = &next_p->thread;

	tifn = READ_ONCE(task_thread_info(next_p)->flags);
	tifp = READ_ONCE(task_thread_info(prev_p)->flags);
	switch_to_bitmap(tss, prev, next, tifp, tifn);

	propagate_user_return_notify(prev_p, next_p);

	if ((tifp & _TIF_BLOCKSTEP || tifn & _TIF_BLOCKSTEP) &&
	    arch_has_block_step()) {
		unsigned long debugctl, msk;

		rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
		debugctl &= ~DEBUGCTLMSR_BTF;
		msk = tifn & _TIF_BLOCKSTEP;
		debugctl |= (msk >> TIF_BLOCKSTEP) << DEBUGCTLMSR_BTF_SHIFT;
		wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
	}

	if ((tifp ^ tifn) & _TIF_NOTSC)
		cr4_toggle_bits_irqsoff(X86_CR4_TSD);

	if ((tifp ^ tifn) & _TIF_NOCPUID)
		set_cpuid_faulting(!!(tifn & _TIF_NOCPUID));
}

/*
 * Idle related variables and functions
 */
unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
EXPORT_SYMBOL(boot_option_idle_override);

static void (*x86_idle)(void);

#ifndef CONFIG_SMP
static inline void play_dead(void)
{
	BUG();
}
#endif

void arch_cpu_idle_enter(void)
{
	tsc_verify_tsc_adjust(false);
	local_touch_nmi();
}

void arch_cpu_idle_dead(void)
{
	play_dead();
}

/*
 * Called from the generic idle code.
 */
void arch_cpu_idle(void)
{
	x86_idle();
}

/*
 * We use this if we don't have any better idle routine..
 */
void __cpuidle default_idle(void)
{
	trace_cpu_idle_rcuidle(1, smp_processor_id());
	safe_halt();
	trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
}
#ifdef CONFIG_APM_MODULE
EXPORT_SYMBOL(default_idle);
#endif

#ifdef CONFIG_XEN
bool xen_set_default_idle(void)
{
	bool ret = !!x86_idle;

	x86_idle = default_idle;

	return ret;
}
#endif

void stop_this_cpu(void *dummy)
{
	local_irq_disable();
	/*
	 * Remove this CPU:
	 */
	set_cpu_online(smp_processor_id(), false);
	disable_local_APIC();
	mcheck_cpu_clear(this_cpu_ptr(&cpu_info));

	/*
	 * Use wbinvd on processors that support SME. This provides support
	 * for performing a successful kexec when going from SME inactive
	 * to SME active (or vice-versa). The cache must be cleared so that
	 * if there are entries with the same physical address, both with and
	 * without the encryption bit, they don't race each other when flushed
	 * and potentially end up with the wrong entry being committed to
	 * memory.
	 */
	if (boot_cpu_has(X86_FEATURE_SME))
		native_wbinvd();
	for (;;) {
		/*
		 * Use native_halt() so that memory contents don't change
		 * (stack usage and variables) after possibly issuing the
		 * native_wbinvd() above.
		 */
		native_halt();
	}
}

/*
 * AMD Erratum 400 aware idle routine. We handle it the same way as C3 power
 * states (local apic timer and TSC stop).
 */
static void amd_e400_idle(void)
{
	/*
	 * We cannot use static_cpu_has_bug() here because X86_BUG_AMD_APIC_C1E
	 * gets set after static_cpu_has() places have been converted via
	 * alternatives.
	 */
	if (!boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E)) {
		default_idle();
		return;
	}

	tick_broadcast_enter();

	default_idle();

	/*
	 * The switch back from broadcast mode needs to be called with
	 * interrupts disabled.
	 */
	local_irq_disable();
	tick_broadcast_exit();
	local_irq_enable();
}

/*
 * Intel Core2 and older machines prefer MWAIT over HALT for C1.
 * We can't rely on cpuidle installing MWAIT, because it will not load
 * on systems that support only C1 -- so the boot default must be MWAIT.
 *
 * Some AMD machines are the opposite, they depend on using HALT.
 *
 * So for default C1, which is used during boot until cpuidle loads,
 * use MWAIT-C1 on Intel HW that has it, else use HALT.
 */
static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
{
	if (c->x86_vendor != X86_VENDOR_INTEL)
		return 0;

	if (!cpu_has(c, X86_FEATURE_MWAIT) || static_cpu_has_bug(X86_BUG_MONITOR))
		return 0;

	return 1;
}

/*
 * MONITOR/MWAIT with no hints, used for default C1 state. This invokes MWAIT
 * with interrupts enabled and no flags, which is backwards compatible with the
 * original MWAIT implementation.
 */
static __cpuidle void mwait_idle(void)
{
	if (!current_set_polling_and_test()) {
		trace_cpu_idle_rcuidle(1, smp_processor_id());
		if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) {
			mb(); /* quirk */
			clflush((void *)&current_thread_info()->flags);
			mb(); /* quirk */
		}

		__monitor((void *)&current_thread_info()->flags, 0, 0);
		if (!need_resched())
			__sti_mwait(0, 0);
		else
			local_irq_enable();
		trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
	} else {
		local_irq_enable();
	}
	__current_clr_polling();
}

void select_idle_routine(const struct cpuinfo_x86 *c)
{
#ifdef CONFIG_SMP
	if (boot_option_idle_override == IDLE_POLL && smp_num_siblings > 1)
		pr_warn_once("WARNING: polling idle and HT enabled, performance may degrade\n");
#endif
	if (x86_idle || boot_option_idle_override == IDLE_POLL)
		return;

	if (boot_cpu_has_bug(X86_BUG_AMD_E400)) {
		pr_info("using AMD E400 aware idle routine\n");
		x86_idle = amd_e400_idle;
	} else if (prefer_mwait_c1_over_halt(c)) {
		pr_info("using mwait in idle threads\n");
		x86_idle = mwait_idle;
	} else
		x86_idle = default_idle;
}

void amd_e400_c1e_apic_setup(void)
{
	if (boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E)) {
		pr_info("Switch to broadcast mode on CPU%d\n", smp_processor_id());
		local_irq_disable();
		tick_broadcast_force();
		local_irq_enable();
	}
}

void __init arch_post_acpi_subsys_init(void)
{
	u32 lo, hi;

	if (!boot_cpu_has_bug(X86_BUG_AMD_E400))
		return;

	/*
	 * AMD E400 detection needs to happen after ACPI has been enabled. If
	 * the machine is affected K8_INTP_C1E_ACTIVE_MASK bits are set in
	 * MSR_K8_INT_PENDING_MSG.
	 */
	rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
	if (!(lo & K8_INTP_C1E_ACTIVE_MASK))
		return;

	boot_cpu_set_bug(X86_BUG_AMD_APIC_C1E);

	if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
		mark_tsc_unstable("TSC halt in AMD C1E");
	pr_info("System has AMD C1E enabled\n");
}

static int __init idle_setup(char *str)
{
	if (!str)
		return -EINVAL;

	if (!strcmp(str, "poll")) {
		pr_info("using polling idle threads\n");
		boot_option_idle_override = IDLE_POLL;
		cpu_idle_poll_ctrl(true);
	} else if (!strcmp(str, "halt")) {
		/*
		 * When the boot option of idle=halt is added, halt is
		 * forced to be used for CPU idle. In such case CPU C2/C3
		 * won't be used again.
		 * To continue to load the CPU idle driver, don't touch
		 * the boot_option_idle_override.
		 */
		x86_idle = default_idle;
		boot_option_idle_override = IDLE_HALT;
	} else if (!strcmp(str, "nomwait")) {
		/*
		 * If the boot option of "idle=nomwait" is added,
		 * it means that mwait will be disabled for CPU C2/C3
		 * states. In such case it won't touch the variable
		 * of boot_option_idle_override.
		 */
		boot_option_idle_override = IDLE_NOMWAIT;
	} else
		return -1;

	return 0;
}
early_param("idle", idle_setup);

unsigned long arch_align_stack(unsigned long sp)
{
	if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
		sp -= get_random_int() % 8192;
	return sp & ~0xf;
}

unsigned long arch_randomize_brk(struct mm_struct *mm)
{
	return randomize_page(mm->brk, 0x02000000);
}

/*
 * Called from fs/proc with a reference on @p to find the function
 * which called into schedule(). This needs to be done carefully
 * because the task might wake up and we might look at a stack
 * changing under us.
 */
unsigned long get_wchan(struct task_struct *p)
{
	unsigned long start, bottom, top, sp, fp, ip, ret = 0;
	int count = 0;

	if (!p || p == current || p->state == TASK_RUNNING)
		return 0;

	if (!try_get_task_stack(p))
		return 0;

	start = (unsigned long)task_stack_page(p);
	if (!start)
		goto out;

	/*
	 * Layout of the stack page:
	 *
	 * ----------- topmax = start + THREAD_SIZE - sizeof(unsigned long)
	 * PADDING
	 * ----------- top = topmax - TOP_OF_KERNEL_STACK_PADDING
	 * stack
	 * ----------- bottom = start
	 *
	 * The tasks stack pointer points at the location where the
	 * framepointer is stored. The data on the stack is:
	 * ... IP FP ... IP FP
	 *
	 * We need to read FP and IP, so we need to adjust the upper
	 * bound by another unsigned long.
	 */
	top = start + THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;
	top -= 2 * sizeof(unsigned long);
	bottom = start;

	sp = READ_ONCE(p->thread.sp);
	if (sp < bottom || sp > top)
		goto out;

	fp = READ_ONCE_NOCHECK(((struct inactive_task_frame *)sp)->bp);
	do {
		if (fp < bottom || fp > top)
			goto out;
		ip = READ_ONCE_NOCHECK(*(unsigned long *)(fp + sizeof(unsigned long)));
		if (!in_sched_functions(ip)) {
			ret = ip;
			goto out;
		}
		fp = READ_ONCE_NOCHECK(*(unsigned long *)fp);
	} while (count++ < 16 && p->state != TASK_RUNNING);

out:
	put_task_stack(p);
	return ret;
}

long do_arch_prctl_common(struct task_struct *task, int option,
			  unsigned long cpuid_enabled)
{
	switch (option) {
	case ARCH_GET_CPUID:
		return get_cpuid_mode();
	case ARCH_SET_CPUID:
		return set_cpuid_mode(task, cpuid_enabled);
	}

	return -EINVAL;
}
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
c767a54b JP	2	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
c767a54b JP	3
61c4628b SS	4	#include <linux/errno.h>
	5	#include <linux/kernel.h>
	6	#include <linux/mm.h>
	7	#include <linux/smp.h>
389d1fb1	8	#include <linux/prctl.h>
61c4628b SS	9	#include <linux/slab.h>
61c4628b SS	10	#include <linux/sched.h>
4c822698	11	#include <linux/sched/idle.h>
b17b0153	12	#include <linux/sched/debug.h>
29930025	13	#include <linux/sched/task.h>
68db0cf1	14	#include <linux/sched/task_stack.h>
186f4360 PG	15	#include <linux/init.h>
186f4360 PG	16	#include <linux/export.h>
7f424a8b	17	#include <linux/pm.h>
162a688e	18	#include <linux/tick.h>
9d62dcdf	19	#include <linux/random.h>
7c68af6e	20	#include <linux/user-return-notifier.h>
814e2c84 AI	21	#include <linux/dmi.h>
814e2c84 AI	22	#include <linux/utsname.h>
90e24014	23	#include <linux/stackprotector.h>
90e24014	24	#include <linux/cpuidle.h>
61613521	25	#include <trace/events/power.h>
24f1e32c	26	#include <linux/hw_breakpoint.h>
93789b32	27	#include <asm/cpu.h>
d3ec5cae	28	#include <asm/apic.h>
2c1b284e	29	#include <asm/syscalls.h>
7c0f6ba6	30	#include <linux/uaccess.h>
b253149b	31	#include <asm/mwait.h>
78f7f1e5	32	#include <asm/fpu/internal.h>
66cb5917	33	#include <asm/debugreg.h>
90e24014	34	#include <asm/nmi.h>
375074cc	35	#include <asm/tlbflush.h>
8838eb6c	36	#include <asm/mce.h>
9fda6a06	37	#include <asm/vm86.h>
7b32aead	38	#include <asm/switch_to.h>
b7ffc44d	39	#include <asm/desc.h>
e9ea1e7f	40	#include <asm/prctl.h>
90e24014	41
45046892 TG	42	/*
	43	* per-CPU TSS segments. Threads are completely 'soft' on Linux,
	44	* no more per-task TSS's. The TSS size is kept cacheline-aligned
	45	* so they are allowed to end up in the .data..cacheline_aligned
	46	* section. Since TSS's are completely CPU-local, we want them
	47	* on exact cacheline boundaries, to eliminate cacheline ping-pong.
	48	*/
2fd9c41a	49	__visible DEFINE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw) = {
d0a0de21	50	.x86_tss = {
20bb8344 AL	51	/*
	52	* .sp0 is only used when entering ring 0 from a lower
	53	* privilege level. Since the init task never runs anything
	54	* but ring 0 code, there is no need for a valid value here.
	55	* Poison it.
	56	*/
	57	.sp0 = (1UL << (BITS_PER_LONG-1)) + 1,
9aaefe7b AL	58
	59	#ifdef CONFIG_X86_64
	60	/*
	61	* .sp1 is cpu_current_top_of_stack. The init task never
	62	* runs user code, but cpu_current_top_of_stack should still
	63	* be well defined before the first context switch.
	64	*/
	65	.sp1 = TOP_OF_INIT_STACK,
	66	#endif
	67
d0a0de21 AL	68	#ifdef CONFIG_X86_32
	69	.ss0 = __KERNEL_DS,
	70	.ss1 = __KERNEL_CS,
	71	.io_bitmap_base = INVALID_IO_BITMAP_OFFSET,
	72	#endif
	73	},
	74	#ifdef CONFIG_X86_32
	75	/*
	76	* Note that the .io_bitmap member must be extra-big. This is because
	77	* the CPU will access an additional byte beyond the end of the IO
	78	* permission bitmap. The extra byte must be all 1 bits, and must
	79	* be within the limit.
	80	*/
	81	.io_bitmap = { [0 ... IO_BITMAP_LONGS] = ~0 },
	82	#endif
	83	};
c482feef	84	EXPORT_PER_CPU_SYMBOL(cpu_tss_rw);
45046892	85
b7ceaec1 AL	86	DEFINE_PER_CPU(bool, __tss_limit_invalid);
b7ceaec1 AL	87	EXPORT_PER_CPU_SYMBOL_GPL(__tss_limit_invalid);
b7ffc44d	88
55ccf3fe SS	89	/*
	90	* this gets called so that we can store lazy state into memory and copy the
	91	* current task into the new thread.
	92	*/
61c4628b SS	93	int arch_dup_task_struct(struct task_struct dst, struct task_struct src)
61c4628b SS	94	{
5aaeb5c0	95	memcpy(dst, src, arch_task_struct_size);
2459ee86 AL	96	#ifdef CONFIG_VM86
	97	dst->thread.vm86 = NULL;
	98	#endif
f1853505	99
c69e098b	100	return fpu__copy(&dst->thread.fpu, &src->thread.fpu);
61c4628b	101	}
7f424a8b	102
389d1fb1 JF	103	/*
	104	* Free current thread data structures etc..
	105	*/
e6464694	106	void exit_thread(struct task_struct *tsk)
389d1fb1	107	{
e6464694	108	struct thread_struct *t = &tsk->thread;
250981e6	109	unsigned long *bp = t->io_bitmap_ptr;
ca6787ba	110	struct fpu *fpu = &t->fpu;
389d1fb1	111
250981e6	112	if (bp) {
c482feef	113	struct tss_struct *tss = &per_cpu(cpu_tss_rw, get_cpu());
389d1fb1	114
389d1fb1 JF	115	t->io_bitmap_ptr = NULL;
	116	clear_thread_flag(TIF_IO_BITMAP);
	117	/*
	118	* Careful, clear this in the TSS too:
	119	*/
	120	memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
	121	t->io_bitmap_max = 0;
	122	put_cpu();
250981e6	123	kfree(bp);
389d1fb1	124	}
1dcc8d7b	125
9fda6a06 BG	126	free_vm86(t);
9fda6a06 BG	127
50338615	128	fpu__drop(fpu);
389d1fb1 JF	129	}
	130
	131	void flush_thread(void)
	132	{
	133	struct task_struct *tsk = current;
	134
24f1e32c	135	flush_ptrace_hw_breakpoint(tsk);
389d1fb1	136	memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
110d7f75	137
04c8e01d	138	fpu__clear(&tsk->thread.fpu);
389d1fb1 JF	139	}
389d1fb1 JF	140
389d1fb1 JF	141	void disable_TSC(void)
	142	{
	143	preempt_disable();
	144	if (!test_and_set_thread_flag(TIF_NOTSC))
	145	/*
	146	* Must flip the CPU state synchronously with
	147	* TIF_NOTSC in the current running context.
	148	*/
5a920155	149	cr4_set_bits(X86_CR4_TSD);
389d1fb1 JF	150	preempt_enable();
	151	}
	152
389d1fb1 JF	153	static void enable_TSC(void)
	154	{
	155	preempt_disable();
	156	if (test_and_clear_thread_flag(TIF_NOTSC))
	157	/*
	158	* Must flip the CPU state synchronously with
	159	* TIF_NOTSC in the current running context.
	160	*/
5a920155	161	cr4_clear_bits(X86_CR4_TSD);
389d1fb1 JF	162	preempt_enable();
	163	}
	164
	165	int get_tsc_mode(unsigned long adr)
	166	{
	167	unsigned int val;
	168
	169	if (test_thread_flag(TIF_NOTSC))
	170	val = PR_TSC_SIGSEGV;
	171	else
	172	val = PR_TSC_ENABLE;
	173
	174	return put_user(val, (unsigned int __user *)adr);
	175	}
	176
	177	int set_tsc_mode(unsigned int val)
	178	{
	179	if (val == PR_TSC_SIGSEGV)
	180	disable_TSC();
	181	else if (val == PR_TSC_ENABLE)
	182	enable_TSC();
	183	else
	184	return -EINVAL;
	185
	186	return 0;
	187	}
	188
e9ea1e7f KH	189	DEFINE_PER_CPU(u64, msr_misc_features_shadow);
	190
	191	static void set_cpuid_faulting(bool on)
	192	{
	193	u64 msrval;
	194
	195	msrval = this_cpu_read(msr_misc_features_shadow);
	196	msrval &= ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT;
	197	msrval \|= (on << MSR_MISC_FEATURES_ENABLES_CPUID_FAULT_BIT);
	198	this_cpu_write(msr_misc_features_shadow, msrval);
	199	wrmsrl(MSR_MISC_FEATURES_ENABLES, msrval);
	200	}
	201
	202	static void disable_cpuid(void)
	203	{
	204	preempt_disable();
	205	if (!test_and_set_thread_flag(TIF_NOCPUID)) {
	206	/*
	207	* Must flip the CPU state synchronously with
	208	* TIF_NOCPUID in the current running context.
	209	*/
	210	set_cpuid_faulting(true);
	211	}
	212	preempt_enable();
	213	}
	214
	215	static void enable_cpuid(void)
	216	{
	217	preempt_disable();
	218	if (test_and_clear_thread_flag(TIF_NOCPUID)) {
	219	/*
	220	* Must flip the CPU state synchronously with
	221	* TIF_NOCPUID in the current running context.
	222	*/
	223	set_cpuid_faulting(false);
	224	}
	225	preempt_enable();
	226	}
	227
	228	static int get_cpuid_mode(void)
	229	{
	230	return !test_thread_flag(TIF_NOCPUID);
	231	}
	232
	233	static int set_cpuid_mode(struct task_struct *task, unsigned long cpuid_enabled)
	234	{
	235	if (!static_cpu_has(X86_FEATURE_CPUID_FAULT))
	236	return -ENODEV;
	237
	238	if (cpuid_enabled)
	239	enable_cpuid();
	240	else
	241	disable_cpuid();
	242
	243	return 0;
	244	}
	245
	246	/*
	247	* Called immediately after a successful exec.
	248	*/
	249	void arch_setup_new_exec(void)
	250	{
	251	/* If cpuid was previously disabled for this task, re-enable it. */
	252	if (test_thread_flag(TIF_NOCPUID))
253	enable_cpuid();
254	}
255
af8b3cd3 KH	256	static inline void switch_to_bitmap(struct tss_struct *tss,
	257	struct thread_struct *prev,
	258	struct thread_struct *next,
	259	unsigned long tifp, unsigned long tifn)
	260	{
	261	if (tifn & _TIF_IO_BITMAP) {
	262	/*
	263	* Copy the relevant range of the IO bitmap.
	264	* Normally this is 128 bytes or less:
	265	*/
	266	memcpy(tss->io_bitmap, next->io_bitmap_ptr,
	267	max(prev->io_bitmap_max, next->io_bitmap_max));
	268	/*
	269	* Make sure that the TSS limit is correct for the CPU
	270	* to notice the IO bitmap.
	271	*/
	272	refresh_tss_limit();
	273	} else if (tifp & _TIF_IO_BITMAP) {
	274	/*
	275	* Clear any possible leftover bits:
	276	*/
	277	memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
	278	}
	279	}
	280
389d1fb1 JF	281	void __switch_to_xtra(struct task_struct prev_p, struct task_struct next_p,
	282	struct tss_struct *tss)
	283	{
	284	struct thread_struct prev, next;
af8b3cd3	285	unsigned long tifp, tifn;
389d1fb1 JF	286
	287	prev = &prev_p->thread;
	288	next = &next_p->thread;
	289
af8b3cd3 KH	290	tifn = READ_ONCE(task_thread_info(next_p)->flags);
	291	tifp = READ_ONCE(task_thread_info(prev_p)->flags);
	292	switch_to_bitmap(tss, prev, next, tifp, tifn);
	293
	294	propagate_user_return_notify(prev_p, next_p);
	295
b9894a2f KH	296	if ((tifp & _TIF_BLOCKSTEP \|\| tifn & _TIF_BLOCKSTEP) &&
	297	arch_has_block_step()) {
	298	unsigned long debugctl, msk;
ea8e61b7	299
b9894a2f	300	rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
ea8e61b7	301	debugctl &= ~DEBUGCTLMSR_BTF;
b9894a2f KH	302	msk = tifn & _TIF_BLOCKSTEP;
	303	debugctl \|= (msk >> TIF_BLOCKSTEP) << DEBUGCTLMSR_BTF_SHIFT;
	304	wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
ea8e61b7	305	}
389d1fb1	306
5a920155	307	if ((tifp ^ tifn) & _TIF_NOTSC)
9d0b6232	308	cr4_toggle_bits_irqsoff(X86_CR4_TSD);
e9ea1e7f KH	309
	310	if ((tifp ^ tifn) & _TIF_NOCPUID)
	311	set_cpuid_faulting(!!(tifn & _TIF_NOCPUID));
389d1fb1 JF	312	}
389d1fb1 JF	313
00dba564 TG	314	/*
	315	* Idle related variables and functions
	316	*/
d1896049	317	unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
00dba564 TG	318	EXPORT_SYMBOL(boot_option_idle_override);
00dba564 TG	319
a476bda3	320	static void (*x86_idle)(void);
00dba564	321
90e24014 RW	322	#ifndef CONFIG_SMP
	323	static inline void play_dead(void)
	324	{
	325	BUG();
	326	}
	327	#endif
	328
7d1a9417 TG	329	void arch_cpu_idle_enter(void)
7d1a9417 TG	330	{
6a369583	331	tsc_verify_tsc_adjust(false);
7d1a9417	332	local_touch_nmi();
7d1a9417	333	}
90e24014	334
7d1a9417 TG	335	void arch_cpu_idle_dead(void)
	336	{
	337	play_dead();
	338	}
90e24014	339
7d1a9417 TG	340	/*
	341	* Called from the generic idle code.
	342	*/
	343	void arch_cpu_idle(void)
	344	{
16f8b05a	345	x86_idle();
90e24014 RW	346	}
90e24014 RW	347
00dba564	348	/*
7d1a9417	349	* We use this if we don't have any better idle routine..
00dba564	350	*/
6727ad9e	351	void __cpuidle default_idle(void)
00dba564	352	{
4d0e42cc	353	trace_cpu_idle_rcuidle(1, smp_processor_id());
7d1a9417	354	safe_halt();
4d0e42cc	355	trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
00dba564	356	}
60b8b1de	357	#ifdef CONFIG_APM_MODULE
00dba564 TG	358	EXPORT_SYMBOL(default_idle);
	359	#endif
	360
6a377ddc LB	361	#ifdef CONFIG_XEN
6a377ddc LB	362	bool xen_set_default_idle(void)
e5fd47bf	363	{
a476bda3	364	bool ret = !!x86_idle;
e5fd47bf	365
a476bda3	366	x86_idle = default_idle;
e5fd47bf KRW	367
	368	return ret;
	369	}
6a377ddc	370	#endif
bba4ed01	371
d3ec5cae IV	372	void stop_this_cpu(void *dummy)
	373	{
	374	local_irq_disable();
	375	/*
	376	* Remove this CPU:
	377	*/
4f062896	378	set_cpu_online(smp_processor_id(), false);
d3ec5cae	379	disable_local_APIC();
8838eb6c	380	mcheck_cpu_clear(this_cpu_ptr(&cpu_info));
d3ec5cae	381
f23d74f6 TL	382	/*
	383	* Use wbinvd on processors that support SME. This provides support
	384	* for performing a successful kexec when going from SME inactive
	385	* to SME active (or vice-versa). The cache must be cleared so that
	386	* if there are entries with the same physical address, both with and
	387	* without the encryption bit, they don't race each other when flushed
	388	* and potentially end up with the wrong entry being committed to
	389	* memory.
	390	*/
	391	if (boot_cpu_has(X86_FEATURE_SME))
	392	native_wbinvd();
bba4ed01 TL	393	for (;;) {
bba4ed01 TL	394	/*
f23d74f6 TL	395	* Use native_halt() so that memory contents don't change
	396	* (stack usage and variables) after possibly issuing the
	397	* native_wbinvd() above.
bba4ed01	398	*/
f23d74f6	399	native_halt();
bba4ed01	400	}
7f424a8b PZ	401	}
7f424a8b PZ	402
aa276e1c	403	/*
07c94a38 BP	404	* AMD Erratum 400 aware idle routine. We handle it the same way as C3 power
07c94a38 BP	405	* states (local apic timer and TSC stop).
aa276e1c	406	*/
02c68a02	407	static void amd_e400_idle(void)
aa276e1c	408	{
07c94a38 BP	409	/*
	410	* We cannot use static_cpu_has_bug() here because X86_BUG_AMD_APIC_C1E
	411	* gets set after static_cpu_has() places have been converted via
	412	* alternatives.
	413	*/
	414	if (!boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E)) {
	415	default_idle();
	416	return;
aa276e1c TG	417	}
aa276e1c TG	418
07c94a38	419	tick_broadcast_enter();
aa276e1c	420
07c94a38	421	default_idle();
0beefa20	422
07c94a38 BP	423	/*
	424	* The switch back from broadcast mode needs to be called with
	425	* interrupts disabled.
	426	*/
	427	local_irq_disable();
	428	tick_broadcast_exit();
	429	local_irq_enable();
aa276e1c TG	430	}
aa276e1c TG	431
b253149b LB	432	/*
	433	* Intel Core2 and older machines prefer MWAIT over HALT for C1.
	434	* We can't rely on cpuidle installing MWAIT, because it will not load
	435	* on systems that support only C1 -- so the boot default must be MWAIT.
	436	*
	437	* Some AMD machines are the opposite, they depend on using HALT.
	438	*
	439	* So for default C1, which is used during boot until cpuidle loads,
	440	* use MWAIT-C1 on Intel HW that has it, else use HALT.
	441	*/
	442	static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
	443	{
	444	if (c->x86_vendor != X86_VENDOR_INTEL)
	445	return 0;
	446
08e237fa	447	if (!cpu_has(c, X86_FEATURE_MWAIT) \|\| static_cpu_has_bug(X86_BUG_MONITOR))
b253149b LB	448	return 0;
	449
	450	return 1;
	451	}
	452
	453	/*
0fb0328d HR	454	* MONITOR/MWAIT with no hints, used for default C1 state. This invokes MWAIT
	455	* with interrupts enabled and no flags, which is backwards compatible with the
	456	* original MWAIT implementation.
b253149b	457	*/
6727ad9e	458	static __cpuidle void mwait_idle(void)
b253149b	459	{
f8e617f4	460	if (!current_set_polling_and_test()) {
e43d0189	461	trace_cpu_idle_rcuidle(1, smp_processor_id());
f8e617f4	462	if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) {
ca59809f	463	mb(); /* quirk */
b253149b	464	clflush((void *)&current_thread_info()->flags);
ca59809f	465	mb(); /* quirk */
f8e617f4	466	}
b253149b LB	467
b253149b LB	468	__monitor((void *)&current_thread_info()->flags, 0, 0);
b253149b LB	469	if (!need_resched())
	470	__sti_mwait(0, 0);
	471	else
	472	local_irq_enable();
e43d0189	473	trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
f8e617f4	474	} else {
b253149b	475	local_irq_enable();
f8e617f4 MG	476	}
f8e617f4 MG	477	__current_clr_polling();
b253149b LB	478	}
b253149b LB	479
148f9bb8	480	void select_idle_routine(const struct cpuinfo_x86 *c)
7f424a8b	481	{
3e5095d1	482	#ifdef CONFIG_SMP
7d1a9417	483	if (boot_option_idle_override == IDLE_POLL && smp_num_siblings > 1)
c767a54b	484	pr_warn_once("WARNING: polling idle and HT enabled, performance may degrade\n");
7f424a8b	485	#endif
7d1a9417	486	if (x86_idle \|\| boot_option_idle_override == IDLE_POLL)
6ddd2a27 TG	487	return;
6ddd2a27 TG	488
3344ed30	489	if (boot_cpu_has_bug(X86_BUG_AMD_E400)) {
c767a54b	490	pr_info("using AMD E400 aware idle routine\n");
a476bda3	491	x86_idle = amd_e400_idle;
b253149b LB	492	} else if (prefer_mwait_c1_over_halt(c)) {
	493	pr_info("using mwait in idle threads\n");
	494	x86_idle = mwait_idle;
6ddd2a27	495	} else
a476bda3	496	x86_idle = default_idle;
7f424a8b PZ	497	}
7f424a8b PZ	498
07c94a38	499	void amd_e400_c1e_apic_setup(void)
30e1e6d1	500	{
07c94a38 BP	501	if (boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E)) {
	502	pr_info("Switch to broadcast mode on CPU%d\n", smp_processor_id());
	503	local_irq_disable();
	504	tick_broadcast_force();
	505	local_irq_enable();
	506	}
30e1e6d1 RR	507	}
30e1e6d1 RR	508
e7ff3a47 TG	509	void __init arch_post_acpi_subsys_init(void)
	510	{
	511	u32 lo, hi;
	512
	513	if (!boot_cpu_has_bug(X86_BUG_AMD_E400))
	514	return;
	515
	516	/*
	517	* AMD E400 detection needs to happen after ACPI has been enabled. If
	518	* the machine is affected K8_INTP_C1E_ACTIVE_MASK bits are set in
	519	* MSR_K8_INT_PENDING_MSG.
	520	*/
	521	rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
	522	if (!(lo & K8_INTP_C1E_ACTIVE_MASK))
	523	return;
	524
	525	boot_cpu_set_bug(X86_BUG_AMD_APIC_C1E);
	526
	527	if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
	528	mark_tsc_unstable("TSC halt in AMD C1E");
	529	pr_info("System has AMD C1E enabled\n");
	530	}
	531
7f424a8b PZ	532	static int __init idle_setup(char *str)
7f424a8b PZ	533	{
ab6bc3e3 CG	534	if (!str)
	535	return -EINVAL;
	536
7f424a8b	537	if (!strcmp(str, "poll")) {
c767a54b	538	pr_info("using polling idle threads\n");
d1896049	539	boot_option_idle_override = IDLE_POLL;
7d1a9417	540	cpu_idle_poll_ctrl(true);
d1896049	541	} else if (!strcmp(str, "halt")) {
c1e3b377 ZY	542	/*
	543	* When the boot option of idle=halt is added, halt is
	544	* forced to be used for CPU idle. In such case CPU C2/C3
	545	* won't be used again.
	546	* To continue to load the CPU idle driver, don't touch
	547	* the boot_option_idle_override.
	548	*/
a476bda3	549	x86_idle = default_idle;
d1896049	550	boot_option_idle_override = IDLE_HALT;
da5e09a1 ZY	551	} else if (!strcmp(str, "nomwait")) {
	552	/*
	553	* If the boot option of "idle=nomwait" is added,
	554	* it means that mwait will be disabled for CPU C2/C3
	555	* states. In such case it won't touch the variable
	556	* of boot_option_idle_override.
	557	*/
d1896049	558	boot_option_idle_override = IDLE_NOMWAIT;
c1e3b377	559	} else
7f424a8b PZ	560	return -1;
7f424a8b PZ	561
7f424a8b PZ	562	return 0;
	563	}
	564	early_param("idle", idle_setup);
	565
9d62dcdf AW	566	unsigned long arch_align_stack(unsigned long sp)
	567	{
	568	if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
	569	sp -= get_random_int() % 8192;
	570	return sp & ~0xf;
	571	}
	572
	573	unsigned long arch_randomize_brk(struct mm_struct *mm)
	574	{
9c6f0902	575	return randomize_page(mm->brk, 0x02000000);
9d62dcdf AW	576	}
9d62dcdf AW	577
7ba78053 TG	578	/*
	579	* Called from fs/proc with a reference on @p to find the function
	580	* which called into schedule(). This needs to be done carefully
	581	* because the task might wake up and we might look at a stack
	582	* changing under us.
	583	*/
	584	unsigned long get_wchan(struct task_struct *p)
	585	{
74327a3e	586	unsigned long start, bottom, top, sp, fp, ip, ret = 0;
7ba78053 TG	587	int count = 0;
	588
	589	if (!p \|\| p == current \|\| p->state == TASK_RUNNING)
	590	return 0;
	591
74327a3e AL	592	if (!try_get_task_stack(p))
	593	return 0;
	594
7ba78053 TG	595	start = (unsigned long)task_stack_page(p);
7ba78053 TG	596	if (!start)
74327a3e	597	goto out;
7ba78053 TG	598
	599	/*
	600	* Layout of the stack page:
	601	*
	602	* ----------- topmax = start + THREAD_SIZE - sizeof(unsigned long)
	603	* PADDING
	604	* ----------- top = topmax - TOP_OF_KERNEL_STACK_PADDING
	605	* stack
15f4eae7	606	* ----------- bottom = start
7ba78053 TG	607	*
	608	* The tasks stack pointer points at the location where the
	609	* framepointer is stored. The data on the stack is:
	610	* ... IP FP ... IP FP
	611	*
	612	* We need to read FP and IP, so we need to adjust the upper
	613	* bound by another unsigned long.
	614	*/
	615	top = start + THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;
	616	top -= 2 * sizeof(unsigned long);
15f4eae7	617	bottom = start;
7ba78053 TG	618
	619	sp = READ_ONCE(p->thread.sp);
	620	if (sp < bottom \|\| sp > top)
74327a3e	621	goto out;
7ba78053	622
7b32aead	623	fp = READ_ONCE_NOCHECK(((struct inactive_task_frame *)sp)->bp);
7ba78053 TG	624	do {
7ba78053 TG	625	if (fp < bottom \|\| fp > top)
74327a3e	626	goto out;
f7d27c35	627	ip = READ_ONCE_NOCHECK((unsigned long )(fp + sizeof(unsigned long)));
74327a3e AL	628	if (!in_sched_functions(ip)) {
	629	ret = ip;
	630	goto out;
	631	}
f7d27c35	632	fp = READ_ONCE_NOCHECK((unsigned long )fp);
7ba78053	633	} while (count++ < 16 && p->state != TASK_RUNNING);
74327a3e AL	634
	635	out:
	636	put_task_stack(p);
	637	return ret;
7ba78053	638	}
b0b9b014 KH	639
	640	long do_arch_prctl_common(struct task_struct *task, int option,
	641	unsigned long cpuid_enabled)
	642	{
e9ea1e7f KH	643	switch (option) {
	644	case ARCH_GET_CPUID:
	645	return get_cpuid_mode();
	646	case ARCH_SET_CPUID:
	647	return set_cpuid_mode(task, cpuid_enabled);
	648	}
	649
b0b9b014 KH	650	return -EINVAL;
b0b9b014 KH	651	}