[mirror_ubuntu-eoan-kernel.git] / arch / x86 / kernel / cpu / mcheck / mce_intel.c

/*
 * Intel specific MCE features.
 * Copyright 2004 Zwane Mwaikambo <zwane@linuxpower.ca>
 * Copyright (C) 2008, 2009 Intel Corporation
 * Author: Andi Kleen
 */

#include <linux/gfp.h>
#include <linux/interrupt.h>
#include <linux/percpu.h>
#include <linux/sched.h>
#include <linux/cpumask.h>
#include <asm/apic.h>
#include <asm/cpufeature.h>
#include <asm/intel-family.h>
#include <asm/processor.h>
#include <asm/msr.h>
#include <asm/mce.h>

#include "mce-internal.h"

/*
 * Support for Intel Correct Machine Check Interrupts. This allows
 * the CPU to raise an interrupt when a corrected machine check happened.
 * Normally we pick those up using a regular polling timer.
 * Also supports reliable discovery of shared banks.
 */

/*
 * CMCI can be delivered to multiple cpus that share a machine check bank
 * so we need to designate a single cpu to process errors logged in each bank
 * in the interrupt handler (otherwise we would have many races and potential
 * double reporting of the same error).
 * Note that this can change when a cpu is offlined or brought online since
 * some MCA banks are shared across cpus. When a cpu is offlined, cmci_clear()
 * disables CMCI on all banks owned by the cpu and clears this bitfield. At
 * this point, cmci_rediscover() kicks in and a different cpu may end up
 * taking ownership of some of the shared MCA banks that were previously
 * owned by the offlined cpu.
 */
static DEFINE_PER_CPU(mce_banks_t, mce_banks_owned);

/*
 * CMCI storm detection backoff counter
 *
 * During storm, we reset this counter to INITIAL_CHECK_INTERVAL in case we've
 * encountered an error. If not, we decrement it by one. We signal the end of
 * the CMCI storm when it reaches 0.
 */
static DEFINE_PER_CPU(int, cmci_backoff_cnt);

/*
 * cmci_discover_lock protects against parallel discovery attempts
 * which could race against each other.
 */
static DEFINE_RAW_SPINLOCK(cmci_discover_lock);

#define CMCI_THRESHOLD		1
#define CMCI_POLL_INTERVAL	(30 * HZ)
#define CMCI_STORM_INTERVAL	(HZ)
#define CMCI_STORM_THRESHOLD	15

static DEFINE_PER_CPU(unsigned long, cmci_time_stamp);
static DEFINE_PER_CPU(unsigned int, cmci_storm_cnt);
static DEFINE_PER_CPU(unsigned int, cmci_storm_state);

enum {
	CMCI_STORM_NONE,
	CMCI_STORM_ACTIVE,
	CMCI_STORM_SUBSIDED,
};

static atomic_t cmci_storm_on_cpus;

static int cmci_supported(int *banks)
{
	u64 cap;

	if (mca_cfg.cmci_disabled || mca_cfg.ignore_ce)
		return 0;

	/*
	 * Vendor check is not strictly needed, but the initial
	 * initialization is vendor keyed and this
	 * makes sure none of the backdoors are entered otherwise.
	 */
	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
		return 0;
	if (!boot_cpu_has(X86_FEATURE_APIC) || lapic_get_maxlvt() < 6)
		return 0;
	rdmsrl(MSR_IA32_MCG_CAP, cap);
	*banks = min_t(unsigned, MAX_NR_BANKS, cap & 0xff);
	return !!(cap & MCG_CMCI_P);
}

static bool lmce_supported(void)
{
	u64 tmp;

	if (mca_cfg.lmce_disabled)
		return false;

	rdmsrl(MSR_IA32_MCG_CAP, tmp);

	/*
	 * LMCE depends on recovery support in the processor. Hence both
	 * MCG_SER_P and MCG_LMCE_P should be present in MCG_CAP.
	 */
	if ((tmp & (MCG_SER_P | MCG_LMCE_P)) !=
		   (MCG_SER_P | MCG_LMCE_P))
		return false;

	/*
	 * BIOS should indicate support for LMCE by setting bit 20 in
	 * IA32_FEATURE_CONTROL without which touching MCG_EXT_CTL will
	 * generate a #GP fault.
	 */
	rdmsrl(MSR_IA32_FEATURE_CONTROL, tmp);
	if ((tmp & (FEATURE_CONTROL_LOCKED | FEATURE_CONTROL_LMCE)) ==
		   (FEATURE_CONTROL_LOCKED | FEATURE_CONTROL_LMCE))
		return true;

	return false;
}

bool mce_intel_cmci_poll(void)
{
	if (__this_cpu_read(cmci_storm_state) == CMCI_STORM_NONE)
		return false;

	/*
	 * Reset the counter if we've logged an error in the last poll
	 * during the storm.
	 */
	if (machine_check_poll(0, this_cpu_ptr(&mce_banks_owned)))
		this_cpu_write(cmci_backoff_cnt, INITIAL_CHECK_INTERVAL);
	else
		this_cpu_dec(cmci_backoff_cnt);

	return true;
}

void mce_intel_hcpu_update(unsigned long cpu)
{
	if (per_cpu(cmci_storm_state, cpu) == CMCI_STORM_ACTIVE)
		atomic_dec(&cmci_storm_on_cpus);

	per_cpu(cmci_storm_state, cpu) = CMCI_STORM_NONE;
}

static void cmci_toggle_interrupt_mode(bool on)
{
	unsigned long flags, *owned;
	int bank;
	u64 val;

	raw_spin_lock_irqsave(&cmci_discover_lock, flags);
	owned = this_cpu_ptr(mce_banks_owned);
	for_each_set_bit(bank, owned, MAX_NR_BANKS) {
		rdmsrl(MSR_IA32_MCx_CTL2(bank), val);

		if (on)
			val |= MCI_CTL2_CMCI_EN;
		else
			val &= ~MCI_CTL2_CMCI_EN;

		wrmsrl(MSR_IA32_MCx_CTL2(bank), val);
	}
	raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
}

unsigned long cmci_intel_adjust_timer(unsigned long interval)
{
	if ((this_cpu_read(cmci_backoff_cnt) > 0) &&
	    (__this_cpu_read(cmci_storm_state) == CMCI_STORM_ACTIVE)) {
		mce_notify_irq();
		return CMCI_STORM_INTERVAL;
	}

	switch (__this_cpu_read(cmci_storm_state)) {
	case CMCI_STORM_ACTIVE:

		/*
		 * We switch back to interrupt mode once the poll timer has
		 * silenced itself. That means no events recorded and the timer
		 * interval is back to our poll interval.
		 */
		__this_cpu_write(cmci_storm_state, CMCI_STORM_SUBSIDED);
		if (!atomic_sub_return(1, &cmci_storm_on_cpus))
			pr_notice("CMCI storm subsided: switching to interrupt mode\n");

		/* FALLTHROUGH */

	case CMCI_STORM_SUBSIDED:
		/*
		 * We wait for all CPUs to go back to SUBSIDED state. When that
		 * happens we switch back to interrupt mode.
		 */
		if (!atomic_read(&cmci_storm_on_cpus)) {
			__this_cpu_write(cmci_storm_state, CMCI_STORM_NONE);
			cmci_toggle_interrupt_mode(true);
			cmci_recheck();
		}
		return CMCI_POLL_INTERVAL;
	default:

		/* We have shiny weather. Let the poll do whatever it thinks. */
		return interval;
	}
}

static bool cmci_storm_detect(void)
{
	unsigned int cnt = __this_cpu_read(cmci_storm_cnt);
	unsigned long ts = __this_cpu_read(cmci_time_stamp);
	unsigned long now = jiffies;
	int r;

	if (__this_cpu_read(cmci_storm_state) != CMCI_STORM_NONE)
		return true;

	if (time_before_eq(now, ts + CMCI_STORM_INTERVAL)) {
		cnt++;
	} else {
		cnt = 1;
		__this_cpu_write(cmci_time_stamp, now);
	}
	__this_cpu_write(cmci_storm_cnt, cnt);

	if (cnt <= CMCI_STORM_THRESHOLD)
		return false;

	cmci_toggle_interrupt_mode(false);
	__this_cpu_write(cmci_storm_state, CMCI_STORM_ACTIVE);
	r = atomic_add_return(1, &cmci_storm_on_cpus);
	mce_timer_kick(CMCI_STORM_INTERVAL);
	this_cpu_write(cmci_backoff_cnt, INITIAL_CHECK_INTERVAL);

	if (r == 1)
		pr_notice("CMCI storm detected: switching to poll mode\n");
	return true;
}

/*
 * The interrupt handler. This is called on every event.
 * Just call the poller directly to log any events.
 * This could in theory increase the threshold under high load,
 * but doesn't for now.
 */
static void intel_threshold_interrupt(void)
{
	if (cmci_storm_detect())
		return;

	machine_check_poll(MCP_TIMESTAMP, this_cpu_ptr(&mce_banks_owned));
}

/*
 * Enable CMCI (Corrected Machine Check Interrupt) for available MCE banks
 * on this CPU. Use the algorithm recommended in the SDM to discover shared
 * banks.
 */
static void cmci_discover(int banks)
{
	unsigned long *owned = (void *)this_cpu_ptr(&mce_banks_owned);
	unsigned long flags;
	int i;
	int bios_wrong_thresh = 0;

	raw_spin_lock_irqsave(&cmci_discover_lock, flags);
	for (i = 0; i < banks; i++) {
		u64 val;
		int bios_zero_thresh = 0;

		if (test_bit(i, owned))
			continue;

		/* Skip banks in firmware first mode */
		if (test_bit(i, mce_banks_ce_disabled))
			continue;

		rdmsrl(MSR_IA32_MCx_CTL2(i), val);

		/* Already owned by someone else? */
		if (val & MCI_CTL2_CMCI_EN) {
			clear_bit(i, owned);
			__clear_bit(i, this_cpu_ptr(mce_poll_banks));
			continue;
		}

		if (!mca_cfg.bios_cmci_threshold) {
			val &= ~MCI_CTL2_CMCI_THRESHOLD_MASK;
			val |= CMCI_THRESHOLD;
		} else if (!(val & MCI_CTL2_CMCI_THRESHOLD_MASK)) {
			/*
			 * If bios_cmci_threshold boot option was specified
			 * but the threshold is zero, we'll try to initialize
			 * it to 1.
			 */
			bios_zero_thresh = 1;
			val |= CMCI_THRESHOLD;
		}

		val |= MCI_CTL2_CMCI_EN;
		wrmsrl(MSR_IA32_MCx_CTL2(i), val);
		rdmsrl(MSR_IA32_MCx_CTL2(i), val);

		/* Did the enable bit stick? -- the bank supports CMCI */
		if (val & MCI_CTL2_CMCI_EN) {
			set_bit(i, owned);
			__clear_bit(i, this_cpu_ptr(mce_poll_banks));
			/*
			 * We are able to set thresholds for some banks that
			 * had a threshold of 0. This means the BIOS has not
			 * set the thresholds properly or does not work with
			 * this boot option. Note down now and report later.
			 */
			if (mca_cfg.bios_cmci_threshold && bios_zero_thresh &&
					(val & MCI_CTL2_CMCI_THRESHOLD_MASK))
				bios_wrong_thresh = 1;
		} else {
			WARN_ON(!test_bit(i, this_cpu_ptr(mce_poll_banks)));
		}
	}
	raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
	if (mca_cfg.bios_cmci_threshold && bios_wrong_thresh) {
		pr_info_once(
			"bios_cmci_threshold: Some banks do not have valid thresholds set\n");
		pr_info_once(
			"bios_cmci_threshold: Make sure your BIOS supports this boot option\n");
	}
}

/*
 * Just in case we missed an event during initialization check
 * all the CMCI owned banks.
 */
void cmci_recheck(void)
{
	unsigned long flags;
	int banks;

	if (!mce_available(raw_cpu_ptr(&cpu_info)) || !cmci_supported(&banks))
		return;

	local_irq_save(flags);
	machine_check_poll(0, this_cpu_ptr(&mce_banks_owned));
	local_irq_restore(flags);
}

/* Caller must hold the lock on cmci_discover_lock */
static void __cmci_disable_bank(int bank)
{
	u64 val;

	if (!test_bit(bank, this_cpu_ptr(mce_banks_owned)))
		return;
	rdmsrl(MSR_IA32_MCx_CTL2(bank), val);
	val &= ~MCI_CTL2_CMCI_EN;
	wrmsrl(MSR_IA32_MCx_CTL2(bank), val);
	__clear_bit(bank, this_cpu_ptr(mce_banks_owned));
}

/*
 * Disable CMCI on this CPU for all banks it owns when it goes down.
 * This allows other CPUs to claim the banks on rediscovery.
 */
void cmci_clear(void)
{
	unsigned long flags;
	int i;
	int banks;

	if (!cmci_supported(&banks))
		return;
	raw_spin_lock_irqsave(&cmci_discover_lock, flags);
	for (i = 0; i < banks; i++)
		__cmci_disable_bank(i);
	raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
}

static void cmci_rediscover_work_func(void *arg)
{
	int banks;

	/* Recheck banks in case CPUs don't all have the same */
	if (cmci_supported(&banks))
		cmci_discover(banks);
}

/* After a CPU went down cycle through all the others and rediscover */
void cmci_rediscover(void)
{
	int banks;

	if (!cmci_supported(&banks))
		return;

	on_each_cpu(cmci_rediscover_work_func, NULL, 1);
}

/*
 * Reenable CMCI on this CPU in case a CPU down failed.
 */
void cmci_reenable(void)
{
	int banks;
	if (cmci_supported(&banks))
		cmci_discover(banks);
}

void cmci_disable_bank(int bank)
{
	int banks;
	unsigned long flags;

	if (!cmci_supported(&banks))
		return;

	raw_spin_lock_irqsave(&cmci_discover_lock, flags);
	__cmci_disable_bank(bank);
	raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
}

static void intel_init_cmci(void)
{
	int banks;

	if (!cmci_supported(&banks))
		return;

	mce_threshold_vector = intel_threshold_interrupt;
	cmci_discover(banks);
	/*
	 * For CPU #0 this runs with still disabled APIC, but that's
	 * ok because only the vector is set up. We still do another
	 * check for the banks later for CPU #0 just to make sure
	 * to not miss any events.
	 */
	apic_write(APIC_LVTCMCI, THRESHOLD_APIC_VECTOR|APIC_DM_FIXED);
	cmci_recheck();
}

static void intel_init_lmce(void)
{
	u64 val;

	if (!lmce_supported())
		return;

	rdmsrl(MSR_IA32_MCG_EXT_CTL, val);

	if (!(val & MCG_EXT_CTL_LMCE_EN))
		wrmsrl(MSR_IA32_MCG_EXT_CTL, val | MCG_EXT_CTL_LMCE_EN);
}

static void intel_clear_lmce(void)
{
	u64 val;

	if (!lmce_supported())
		return;

	rdmsrl(MSR_IA32_MCG_EXT_CTL, val);
	val &= ~MCG_EXT_CTL_LMCE_EN;
	wrmsrl(MSR_IA32_MCG_EXT_CTL, val);
}

static void intel_ppin_init(struct cpuinfo_x86 *c)
{
	unsigned long long val;

	/*
	 * Even if testing the presence of the MSR would be enough, we don't
	 * want to risk the situation where other models reuse this MSR for
	 * other purposes.
	 */
	switch (c->x86_model) {
	case INTEL_FAM6_IVYBRIDGE_X:
	case INTEL_FAM6_HASWELL_X:
	case INTEL_FAM6_BROADWELL_XEON_D:
	case INTEL_FAM6_BROADWELL_X:
	case INTEL_FAM6_SKYLAKE_X:
		if (rdmsrl_safe(MSR_PPIN_CTL, &val))
			return;

		if ((val & 3UL) == 1UL) {
			/* PPIN available but disabled: */
			return;
		}

		/* If PPIN is disabled, but not locked, try to enable: */
		if (!(val & 3UL)) {
			wrmsrl_safe(MSR_PPIN_CTL,  val | 2UL);
			rdmsrl_safe(MSR_PPIN_CTL, &val);
		}

		if ((val & 3UL) == 2UL)
			set_cpu_cap(c, X86_FEATURE_INTEL_PPIN);
	}
}

void mce_intel_feature_init(struct cpuinfo_x86 *c)
{
	intel_init_thermal(c);
	intel_init_cmci();
	intel_init_lmce();
	intel_ppin_init(c);
}

void mce_intel_feature_clear(struct cpuinfo_x86 *c)
{
	intel_clear_lmce();
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* Intel specific MCE features.
	3	* Copyright 2004 Zwane Mwaikambo <zwane@linuxpower.ca>
88ccbedd AK	4	* Copyright (C) 2008, 2009 Intel Corporation
88ccbedd AK	5	* Author: Andi Kleen
1da177e4 LT	6	*/
1da177e4 LT	7
5a0e3ad6	8	#include <linux/gfp.h>
1da177e4 LT	9	#include <linux/interrupt.h>
1da177e4 LT	10	#include <linux/percpu.h>
d43c36dc	11	#include <linux/sched.h>
27f6c573	12	#include <linux/cpumask.h>
1bf7b31e	13	#include <asm/apic.h>
3f5a7896 TL	14	#include <asm/cpufeature.h>
3f5a7896 TL	15	#include <asm/intel-family.h>
1da177e4 LT	16	#include <asm/processor.h>
	17	#include <asm/msr.h>
	18	#include <asm/mce.h>
1da177e4	19
55babd8f CG	20	#include "mce-internal.h"
55babd8f CG	21
88ccbedd AK	22	/*
	23	* Support for Intel Correct Machine Check Interrupts. This allows
	24	* the CPU to raise an interrupt when a corrected machine check happened.
	25	* Normally we pick those up using a regular polling timer.
	26	* Also supports reliable discovery of shared banks.
	27	*/
	28
0644414e NR	29	/*
	30	* CMCI can be delivered to multiple cpus that share a machine check bank
	31	* so we need to designate a single cpu to process errors logged in each bank
	32	* in the interrupt handler (otherwise we would have many races and potential
	33	* double reporting of the same error).
	34	* Note that this can change when a cpu is offlined or brought online since
	35	* some MCA banks are shared across cpus. When a cpu is offlined, cmci_clear()
	36	* disables CMCI on all banks owned by the cpu and clears this bitfield. At
	37	* this point, cmci_rediscover() kicks in and a different cpu may end up
	38	* taking ownership of some of the shared MCA banks that were previously
	39	* owned by the offlined cpu.
	40	*/
88ccbedd AK	41	static DEFINE_PER_CPU(mce_banks_t, mce_banks_owned);
88ccbedd AK	42
3f2f0680 BP	43	/*
	44	* CMCI storm detection backoff counter
	45	*
	46	* During storm, we reset this counter to INITIAL_CHECK_INTERVAL in case we've
	47	* encountered an error. If not, we decrement it by one. We signal the end of
	48	* the CMCI storm when it reaches 0.
	49	*/
	50	static DEFINE_PER_CPU(int, cmci_backoff_cnt);
	51
88ccbedd AK	52	/*
	53	* cmci_discover_lock protects against parallel discovery attempts
	54	* which could race against each other.
	55	*/
ed5c41d3	56	static DEFINE_RAW_SPINLOCK(cmci_discover_lock);
88ccbedd	57
55babd8f CG	58	#define CMCI_THRESHOLD 1
55babd8f CG	59	#define CMCI_POLL_INTERVAL (30 * HZ)
3f2f0680	60	#define CMCI_STORM_INTERVAL (HZ)
55babd8f CG	61	#define CMCI_STORM_THRESHOLD 15
	62
	63	static DEFINE_PER_CPU(unsigned long, cmci_time_stamp);
	64	static DEFINE_PER_CPU(unsigned int, cmci_storm_cnt);
	65	static DEFINE_PER_CPU(unsigned int, cmci_storm_state);
	66
	67	enum {
	68	CMCI_STORM_NONE,
	69	CMCI_STORM_ACTIVE,
	70	CMCI_STORM_SUBSIDED,
	71	};
	72
	73	static atomic_t cmci_storm_on_cpus;
88ccbedd	74
df20e2eb	75	static int cmci_supported(int *banks)
88ccbedd AK	76	{
	77	u64 cap;
	78
7af19e4a	79	if (mca_cfg.cmci_disabled \|\| mca_cfg.ignore_ce)
62fdac59 HS	80	return 0;
62fdac59 HS	81
88ccbedd AK	82	/*
	83	* Vendor check is not strictly needed, but the initial
	84	* initialization is vendor keyed and this
	85	* makes sure none of the backdoors are entered otherwise.
	86	*/
	87	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
	88	return 0;
93984fbd	89	if (!boot_cpu_has(X86_FEATURE_APIC) \|\| lapic_get_maxlvt() < 6)
88ccbedd AK	90	return 0;
	91	rdmsrl(MSR_IA32_MCG_CAP, cap);
	92	*banks = min_t(unsigned, MAX_NR_BANKS, cap & 0xff);
	93	return !!(cap & MCG_CMCI_P);
	94	}
	95
88d53867 AR	96	static bool lmce_supported(void)
	97	{
	98	u64 tmp;
	99
	100	if (mca_cfg.lmce_disabled)
	101	return false;
	102
	103	rdmsrl(MSR_IA32_MCG_CAP, tmp);
	104
	105	/*
	106	* LMCE depends on recovery support in the processor. Hence both
	107	* MCG_SER_P and MCG_LMCE_P should be present in MCG_CAP.
	108	*/
	109	if ((tmp & (MCG_SER_P \| MCG_LMCE_P)) !=
	110	(MCG_SER_P \| MCG_LMCE_P))
	111	return false;
	112
	113	/*
	114	* BIOS should indicate support for LMCE by setting bit 20 in
	115	* IA32_FEATURE_CONTROL without which touching MCG_EXT_CTL will
	116	* generate a #GP fault.
	117	*/
	118	rdmsrl(MSR_IA32_FEATURE_CONTROL, tmp);
	119	if ((tmp & (FEATURE_CONTROL_LOCKED \| FEATURE_CONTROL_LMCE)) ==
	120	(FEATURE_CONTROL_LOCKED \| FEATURE_CONTROL_LMCE))
	121	return true;
	122
	123	return false;
	124	}
	125
3f2f0680	126	bool mce_intel_cmci_poll(void)
55babd8f CG	127	{
55babd8f CG	128	if (__this_cpu_read(cmci_storm_state) == CMCI_STORM_NONE)
3f2f0680 BP	129	return false;
	130
	131	/*
	132	* Reset the counter if we've logged an error in the last poll
	133	* during the storm.
	134	*/
54467353	135	if (machine_check_poll(0, this_cpu_ptr(&mce_banks_owned)))
3f2f0680 BP	136	this_cpu_write(cmci_backoff_cnt, INITIAL_CHECK_INTERVAL);
	137	else
	138	this_cpu_dec(cmci_backoff_cnt);
	139
	140	return true;
55babd8f CG	141	}
	142
	143	void mce_intel_hcpu_update(unsigned long cpu)
	144	{
	145	if (per_cpu(cmci_storm_state, cpu) == CMCI_STORM_ACTIVE)
	146	atomic_dec(&cmci_storm_on_cpus);
	147
	148	per_cpu(cmci_storm_state, cpu) = CMCI_STORM_NONE;
	149	}
	150
1b484655 XX	151	static void cmci_toggle_interrupt_mode(bool on)
	152	{
	153	unsigned long flags, *owned;
	154	int bank;
	155	u64 val;
	156
	157	raw_spin_lock_irqsave(&cmci_discover_lock, flags);
	158	owned = this_cpu_ptr(mce_banks_owned);
	159	for_each_set_bit(bank, owned, MAX_NR_BANKS) {
	160	rdmsrl(MSR_IA32_MCx_CTL2(bank), val);
	161
	162	if (on)
	163	val \|= MCI_CTL2_CMCI_EN;
	164	else
	165	val &= ~MCI_CTL2_CMCI_EN;
	166
	167	wrmsrl(MSR_IA32_MCx_CTL2(bank), val);
	168	}
	169	raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
	170	}
	171
3f2f0680	172	unsigned long cmci_intel_adjust_timer(unsigned long interval)
55babd8f	173	{
3f2f0680 BP	174	if ((this_cpu_read(cmci_backoff_cnt) > 0) &&
	175	(__this_cpu_read(cmci_storm_state) == CMCI_STORM_ACTIVE)) {
	176	mce_notify_irq();
	177	return CMCI_STORM_INTERVAL;
	178	}
55babd8f CG	179
	180	switch (__this_cpu_read(cmci_storm_state)) {
	181	case CMCI_STORM_ACTIVE:
3f2f0680	182
55babd8f CG	183	/*
55babd8f CG	184	* We switch back to interrupt mode once the poll timer has
3f2f0680 BP	185	* silenced itself. That means no events recorded and the timer
3f2f0680 BP	186	* interval is back to our poll interval.
55babd8f CG	187	*/
55babd8f CG	188	__this_cpu_write(cmci_storm_state, CMCI_STORM_SUBSIDED);
3f2f0680	189	if (!atomic_sub_return(1, &cmci_storm_on_cpus))
55babd8f	190	pr_notice("CMCI storm subsided: switching to interrupt mode\n");
3f2f0680	191
55babd8f CG	192	/* FALLTHROUGH */
	193
	194	case CMCI_STORM_SUBSIDED:
	195	/*
3f2f0680 BP	196	* We wait for all CPUs to go back to SUBSIDED state. When that
3f2f0680 BP	197	* happens we switch back to interrupt mode.
55babd8f CG	198	*/
	199	if (!atomic_read(&cmci_storm_on_cpus)) {
	200	__this_cpu_write(cmci_storm_state, CMCI_STORM_NONE);
1b484655	201	cmci_toggle_interrupt_mode(true);
55babd8f CG	202	cmci_recheck();
	203	}
	204	return CMCI_POLL_INTERVAL;
	205	default:
3f2f0680 BP	206
3f2f0680 BP	207	/* We have shiny weather. Let the poll do whatever it thinks. */
55babd8f CG	208	return interval;
	209	}
	210	}
	211
	212	static bool cmci_storm_detect(void)
	213	{
	214	unsigned int cnt = __this_cpu_read(cmci_storm_cnt);
	215	unsigned long ts = __this_cpu_read(cmci_time_stamp);
	216	unsigned long now = jiffies;
	217	int r;
	218
	219	if (__this_cpu_read(cmci_storm_state) != CMCI_STORM_NONE)
	220	return true;
	221
	222	if (time_before_eq(now, ts + CMCI_STORM_INTERVAL)) {
	223	cnt++;
	224	} else {
	225	cnt = 1;
	226	__this_cpu_write(cmci_time_stamp, now);
	227	}
	228	__this_cpu_write(cmci_storm_cnt, cnt);
	229
	230	if (cnt <= CMCI_STORM_THRESHOLD)
	231	return false;
	232
1b484655	233	cmci_toggle_interrupt_mode(false);
55babd8f CG	234	__this_cpu_write(cmci_storm_state, CMCI_STORM_ACTIVE);
55babd8f CG	235	r = atomic_add_return(1, &cmci_storm_on_cpus);
3f2f0680 BP	236	mce_timer_kick(CMCI_STORM_INTERVAL);
3f2f0680 BP	237	this_cpu_write(cmci_backoff_cnt, INITIAL_CHECK_INTERVAL);
55babd8f CG	238
	239	if (r == 1)
	240	pr_notice("CMCI storm detected: switching to poll mode\n");
	241	return true;
	242	}
	243
88ccbedd AK	244	/*
	245	* The interrupt handler. This is called on every event.
	246	* Just call the poller directly to log any events.
	247	* This could in theory increase the threshold under high load,
	248	* but doesn't for now.
	249	*/
	250	static void intel_threshold_interrupt(void)
	251	{
55babd8f CG	252	if (cmci_storm_detect())
55babd8f CG	253	return;
3f2f0680	254
89cbc767	255	machine_check_poll(MCP_TIMESTAMP, this_cpu_ptr(&mce_banks_owned));
88ccbedd AK	256	}
88ccbedd AK	257
88ccbedd AK	258	/*
	259	* Enable CMCI (Corrected Machine Check Interrupt) for available MCE banks
	260	* on this CPU. Use the algorithm recommended in the SDM to discover shared
	261	* banks.
	262	*/
4670a300	263	static void cmci_discover(int banks)
88ccbedd	264	{
89cbc767	265	unsigned long owned = (void )this_cpu_ptr(&mce_banks_owned);
e5299926	266	unsigned long flags;
88ccbedd	267	int i;
450cc201	268	int bios_wrong_thresh = 0;
88ccbedd	269
ed5c41d3	270	raw_spin_lock_irqsave(&cmci_discover_lock, flags);
88ccbedd AK	271	for (i = 0; i < banks; i++) {
88ccbedd AK	272	u64 val;
450cc201	273	int bios_zero_thresh = 0;
88ccbedd AK	274
	275	if (test_bit(i, owned))
	276	continue;
	277
c3d1fb56 NR	278	/* Skip banks in firmware first mode */
	279	if (test_bit(i, mce_banks_ce_disabled))
	280	continue;
	281
a2d32bcb	282	rdmsrl(MSR_IA32_MCx_CTL2(i), val);
88ccbedd AK	283
88ccbedd AK	284	/* Already owned by someone else? */
1f9a0bd4	285	if (val & MCI_CTL2_CMCI_EN) {
4670a300	286	clear_bit(i, owned);
89cbc767	287	__clear_bit(i, this_cpu_ptr(mce_poll_banks));
88ccbedd AK	288	continue;
	289	}
	290
1462594b	291	if (!mca_cfg.bios_cmci_threshold) {
450cc201 NR	292	val &= ~MCI_CTL2_CMCI_THRESHOLD_MASK;
	293	val \|= CMCI_THRESHOLD;
	294	} else if (!(val & MCI_CTL2_CMCI_THRESHOLD_MASK)) {
	295	/*
	296	* If bios_cmci_threshold boot option was specified
	297	* but the threshold is zero, we'll try to initialize
	298	* it to 1.
	299	*/
	300	bios_zero_thresh = 1;
	301	val \|= CMCI_THRESHOLD;
	302	}
	303
	304	val \|= MCI_CTL2_CMCI_EN;
a2d32bcb AK	305	wrmsrl(MSR_IA32_MCx_CTL2(i), val);
a2d32bcb AK	306	rdmsrl(MSR_IA32_MCx_CTL2(i), val);
88ccbedd AK	307
88ccbedd AK	308	/* Did the enable bit stick? -- the bank supports CMCI */
1f9a0bd4	309	if (val & MCI_CTL2_CMCI_EN) {
4670a300	310	set_bit(i, owned);
89cbc767	311	__clear_bit(i, this_cpu_ptr(mce_poll_banks));
450cc201 NR	312	/*
	313	* We are able to set thresholds for some banks that
	314	* had a threshold of 0. This means the BIOS has not
	315	* set the thresholds properly or does not work with
	316	* this boot option. Note down now and report later.
	317	*/
1462594b	318	if (mca_cfg.bios_cmci_threshold && bios_zero_thresh &&
450cc201 NR	319	(val & MCI_CTL2_CMCI_THRESHOLD_MASK))
450cc201 NR	320	bios_wrong_thresh = 1;
88ccbedd	321	} else {
89cbc767	322	WARN_ON(!test_bit(i, this_cpu_ptr(mce_poll_banks)));
88ccbedd AK	323	}
88ccbedd AK	324	}
ed5c41d3	325	raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
1462594b	326	if (mca_cfg.bios_cmci_threshold && bios_wrong_thresh) {
450cc201 NR	327	pr_info_once(
	328	"bios_cmci_threshold: Some banks do not have valid thresholds set\n");
	329	pr_info_once(
	330	"bios_cmci_threshold: Make sure your BIOS supports this boot option\n");
	331	}
88ccbedd AK	332	}
	333
	334	/*
	335	* Just in case we missed an event during initialization check
	336	* all the CMCI owned banks.
	337	*/
df20e2eb	338	void cmci_recheck(void)
88ccbedd AK	339	{
	340	unsigned long flags;
	341	int banks;
	342
89cbc767	343	if (!mce_available(raw_cpu_ptr(&cpu_info)) \|\| !cmci_supported(&banks))
88ccbedd	344	return;
3f2f0680	345
88ccbedd	346	local_irq_save(flags);
54467353	347	machine_check_poll(0, this_cpu_ptr(&mce_banks_owned));
88ccbedd AK	348	local_irq_restore(flags);
	349	}
	350
c3d1fb56 NR	351	/* Caller must hold the lock on cmci_discover_lock */
	352	static void __cmci_disable_bank(int bank)
	353	{
	354	u64 val;
	355
89cbc767	356	if (!test_bit(bank, this_cpu_ptr(mce_banks_owned)))
c3d1fb56 NR	357	return;
	358	rdmsrl(MSR_IA32_MCx_CTL2(bank), val);
	359	val &= ~MCI_CTL2_CMCI_EN;
	360	wrmsrl(MSR_IA32_MCx_CTL2(bank), val);
89cbc767	361	__clear_bit(bank, this_cpu_ptr(mce_banks_owned));
c3d1fb56 NR	362	}
c3d1fb56 NR	363
88ccbedd AK	364	/*
	365	* Disable CMCI on this CPU for all banks it owns when it goes down.
	366	* This allows other CPUs to claim the banks on rediscovery.
	367	*/
df20e2eb	368	void cmci_clear(void)
88ccbedd	369	{
e5299926	370	unsigned long flags;
88ccbedd AK	371	int i;
88ccbedd AK	372	int banks;
88ccbedd AK	373
	374	if (!cmci_supported(&banks))
	375	return;
ed5c41d3	376	raw_spin_lock_irqsave(&cmci_discover_lock, flags);
c3d1fb56 NR	377	for (i = 0; i < banks; i++)
c3d1fb56 NR	378	__cmci_disable_bank(i);
ed5c41d3	379	raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
88ccbedd AK	380	}
88ccbedd AK	381
7a0c819d	382	static void cmci_rediscover_work_func(void *arg)
85b97637 TC	383	{
	384	int banks;
	385
	386	/* Recheck banks in case CPUs don't all have the same */
	387	if (cmci_supported(&banks))
	388	cmci_discover(banks);
85b97637 TC	389	}
85b97637 TC	390
7a0c819d SB	391	/* After a CPU went down cycle through all the others and rediscover */
7a0c819d SB	392	void cmci_rediscover(void)
88ccbedd	393	{
7a0c819d	394	int banks;
88ccbedd AK	395
	396	if (!cmci_supported(&banks))
	397	return;
88ccbedd	398
7a0c819d	399	on_each_cpu(cmci_rediscover_work_func, NULL, 1);
88ccbedd AK	400	}
	401
	402	/*
	403	* Reenable CMCI on this CPU in case a CPU down failed.
	404	*/
	405	void cmci_reenable(void)
	406	{
	407	int banks;
	408	if (cmci_supported(&banks))
4670a300	409	cmci_discover(banks);
88ccbedd AK	410	}
88ccbedd AK	411
c3d1fb56 NR	412	void cmci_disable_bank(int bank)
	413	{
	414	int banks;
	415	unsigned long flags;
	416
	417	if (!cmci_supported(&banks))
	418	return;
	419
ed5c41d3	420	raw_spin_lock_irqsave(&cmci_discover_lock, flags);
c3d1fb56	421	__cmci_disable_bank(bank);
ed5c41d3	422	raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
c3d1fb56 NR	423	}
c3d1fb56 NR	424
514ec49a	425	static void intel_init_cmci(void)
88ccbedd AK	426	{
	427	int banks;
	428
	429	if (!cmci_supported(&banks))
	430	return;
	431
	432	mce_threshold_vector = intel_threshold_interrupt;
4670a300	433	cmci_discover(banks);
88ccbedd AK	434	/*
	435	* For CPU #0 this runs with still disabled APIC, but that's
	436	* ok because only the vector is set up. We still do another
	437	* check for the banks later for CPU #0 just to make sure
	438	* to not miss any events.
	439	*/
	440	apic_write(APIC_LVTCMCI, THRESHOLD_APIC_VECTOR\|APIC_DM_FIXED);
	441	cmci_recheck();
	442	}
	443
8838eb6c	444	static void intel_init_lmce(void)
88d53867 AR	445	{
	446	u64 val;
	447
	448	if (!lmce_supported())
	449	return;
	450
	451	rdmsrl(MSR_IA32_MCG_EXT_CTL, val);
	452
	453	if (!(val & MCG_EXT_CTL_LMCE_EN))
	454	wrmsrl(MSR_IA32_MCG_EXT_CTL, val \| MCG_EXT_CTL_LMCE_EN);
	455	}
	456
8838eb6c AR	457	static void intel_clear_lmce(void)
	458	{
	459	u64 val;
	460
	461	if (!lmce_supported())
	462	return;
	463
	464	rdmsrl(MSR_IA32_MCG_EXT_CTL, val);
	465	val &= ~MCG_EXT_CTL_LMCE_EN;
	466	wrmsrl(MSR_IA32_MCG_EXT_CTL, val);
	467	}
	468
3f5a7896 TL	469	static void intel_ppin_init(struct cpuinfo_x86 *c)
	470	{
	471	unsigned long long val;
	472
	473	/*
	474	* Even if testing the presence of the MSR would be enough, we don't
	475	* want to risk the situation where other models reuse this MSR for
	476	* other purposes.
	477	*/
	478	switch (c->x86_model) {
	479	case INTEL_FAM6_IVYBRIDGE_X:
	480	case INTEL_FAM6_HASWELL_X:
	481	case INTEL_FAM6_BROADWELL_XEON_D:
	482	case INTEL_FAM6_BROADWELL_X:
	483	case INTEL_FAM6_SKYLAKE_X:
	484	if (rdmsrl_safe(MSR_PPIN_CTL, &val))
	485	return;
	486
	487	if ((val & 3UL) == 1UL) {
	488	/* PPIN available but disabled: */
	489	return;
	490	}
	491
	492	/* If PPIN is disabled, but not locked, try to enable: */
	493	if (!(val & 3UL)) {
	494	wrmsrl_safe(MSR_PPIN_CTL, val \| 2UL);
	495	rdmsrl_safe(MSR_PPIN_CTL, &val);
	496	}
	497
	498	if ((val & 3UL) == 2UL)
	499	set_cpu_cap(c, X86_FEATURE_INTEL_PPIN);
	500	}
	501	}
	502
cc3ca220	503	void mce_intel_feature_init(struct cpuinfo_x86 *c)
1da177e4 LT	504	{
1da177e4 LT	505	intel_init_thermal(c);
88ccbedd	506	intel_init_cmci();
243d657e	507	intel_init_lmce();
3f5a7896	508	intel_ppin_init(c);
1da177e4	509	}
8838eb6c AR	510
	511	void mce_intel_feature_clear(struct cpuinfo_x86 *c)
	512	{
	513	intel_clear_lmce();
	514	}