[mirror_ubuntu-artful-kernel.git] / arch / x86 / kernel / cpu / mtrr / generic.c

/* This only handles 32bit MTRR on 32bit hosts. This is strictly wrong
   because MTRRs can span upto 40 bits (36bits on most modern x86) */ 
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <asm/io.h>
#include <asm/mtrr.h>
#include <asm/msr.h>
#include <asm/system.h>
#include <asm/cpufeature.h>
#include <asm/processor-flags.h>
#include <asm/tlbflush.h>
#include "mtrr.h"

struct mtrr_state {
	struct mtrr_var_range var_ranges[MAX_VAR_RANGES];
	mtrr_type fixed_ranges[NUM_FIXED_RANGES];
	unsigned char enabled;
	unsigned char have_fixed;
	mtrr_type def_type;
};

struct fixed_range_block {
	int base_msr; /* start address of an MTRR block */
	int ranges;   /* number of MTRRs in this block  */
};

static struct fixed_range_block fixed_range_blocks[] = {
	{ MTRRfix64K_00000_MSR, 1 }, /* one  64k MTRR  */
	{ MTRRfix16K_80000_MSR, 2 }, /* two  16k MTRRs */
	{ MTRRfix4K_C0000_MSR,  8 }, /* eight 4k MTRRs */
	{}
};

static unsigned long smp_changes_mask;
static struct mtrr_state mtrr_state = {};

#undef MODULE_PARAM_PREFIX
#define MODULE_PARAM_PREFIX "mtrr."

static int mtrr_show;
module_param_named(show, mtrr_show, bool, 0);

/*  Get the MSR pair relating to a var range  */
static void
get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr)
{
	rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
	rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
}

static void
get_fixed_ranges(mtrr_type * frs)
{
	unsigned int *p = (unsigned int *) frs;
	int i;

	rdmsr(MTRRfix64K_00000_MSR, p[0], p[1]);

	for (i = 0; i < 2; i++)
		rdmsr(MTRRfix16K_80000_MSR + i, p[2 + i * 2], p[3 + i * 2]);
	for (i = 0; i < 8; i++)
		rdmsr(MTRRfix4K_C0000_MSR + i, p[6 + i * 2], p[7 + i * 2]);
}

void mtrr_save_fixed_ranges(void *info)
{
	if (cpu_has_mtrr)
		get_fixed_ranges(mtrr_state.fixed_ranges);
}

static void print_fixed(unsigned base, unsigned step, const mtrr_type*types)
{
	unsigned i;

	for (i = 0; i < 8; ++i, ++types, base += step)
		printk(KERN_INFO "MTRR %05X-%05X %s\n",
			base, base + step - 1, mtrr_attrib_to_str(*types));
}

/*  Grab all of the MTRR state for this CPU into *state  */
void __init get_mtrr_state(void)
{
	unsigned int i;
	struct mtrr_var_range *vrs;
	unsigned lo, dummy;

	vrs = mtrr_state.var_ranges;

	rdmsr(MTRRcap_MSR, lo, dummy);
	mtrr_state.have_fixed = (lo >> 8) & 1;

	for (i = 0; i < num_var_ranges; i++)
		get_mtrr_var_range(i, &vrs[i]);
	if (mtrr_state.have_fixed)
		get_fixed_ranges(mtrr_state.fixed_ranges);

	rdmsr(MTRRdefType_MSR, lo, dummy);
	mtrr_state.def_type = (lo & 0xff);
	mtrr_state.enabled = (lo & 0xc00) >> 10;

	if (mtrr_show) {
		int high_width;

		printk(KERN_INFO "MTRR default type: %s\n", mtrr_attrib_to_str(mtrr_state.def_type));
		if (mtrr_state.have_fixed) {
			printk(KERN_INFO "MTRR fixed ranges %sabled:\n",
			       mtrr_state.enabled & 1 ? "en" : "dis");
			print_fixed(0x00000, 0x10000, mtrr_state.fixed_ranges + 0);
			for (i = 0; i < 2; ++i)
				print_fixed(0x80000 + i * 0x20000, 0x04000, mtrr_state.fixed_ranges + (i + 1) * 8);
			for (i = 0; i < 8; ++i)
				print_fixed(0xC0000 + i * 0x08000, 0x01000, mtrr_state.fixed_ranges + (i + 3) * 8);
		}
		printk(KERN_INFO "MTRR variable ranges %sabled:\n",
		       mtrr_state.enabled & 2 ? "en" : "dis");
		high_width = ((size_or_mask ? ffs(size_or_mask) - 1 : 32) - (32 - PAGE_SHIFT) + 3) / 4;
		for (i = 0; i < num_var_ranges; ++i) {
			if (mtrr_state.var_ranges[i].mask_lo & (1 << 11))
				printk(KERN_INFO "MTRR %u base %0*X%05X000 mask %0*X%05X000 %s\n",
				       i,
				       high_width,
				       mtrr_state.var_ranges[i].base_hi,
				       mtrr_state.var_ranges[i].base_lo >> 12,
				       high_width,
				       mtrr_state.var_ranges[i].mask_hi,
				       mtrr_state.var_ranges[i].mask_lo >> 12,
				       mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 0xff));
			else
				printk(KERN_INFO "MTRR %u disabled\n", i);
		}
	}
}

/*  Some BIOS's are fucked and don't set all MTRRs the same!  */
void __init mtrr_state_warn(void)
{
	unsigned long mask = smp_changes_mask;

	if (!mask)
		return;
	if (mask & MTRR_CHANGE_MASK_FIXED)
		printk(KERN_WARNING "mtrr: your CPUs had inconsistent fixed MTRR settings\n");
	if (mask & MTRR_CHANGE_MASK_VARIABLE)
		printk(KERN_WARNING "mtrr: your CPUs had inconsistent variable MTRR settings\n");
	if (mask & MTRR_CHANGE_MASK_DEFTYPE)
		printk(KERN_WARNING "mtrr: your CPUs had inconsistent MTRRdefType settings\n");
	printk(KERN_INFO "mtrr: probably your BIOS does not setup all CPUs.\n");
	printk(KERN_INFO "mtrr: corrected configuration.\n");
}

/* Doesn't attempt to pass an error out to MTRR users
   because it's quite complicated in some cases and probably not
   worth it because the best error handling is to ignore it. */
void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b)
{
	if (wrmsr_safe(msr, a, b) < 0)
		printk(KERN_ERR
			"MTRR: CPU %u: Writing MSR %x to %x:%x failed\n",
			smp_processor_id(), msr, a, b);
}

/**
 * Enable and allow read/write of extended fixed-range MTRR bits on K8 CPUs
 * see AMD publication no. 24593, chapter 3.2.1 for more information
 */
static inline void k8_enable_fixed_iorrs(void)
{
	unsigned lo, hi;

	rdmsr(MSR_K8_SYSCFG, lo, hi);
	mtrr_wrmsr(MSR_K8_SYSCFG, lo
				| K8_MTRRFIXRANGE_DRAM_ENABLE
				| K8_MTRRFIXRANGE_DRAM_MODIFY, hi);
}

/**
 * Checks and updates an fixed-range MTRR if it differs from the value it
 * should have. If K8 extentions are wanted, update the K8 SYSCFG MSR also.
 * see AMD publication no. 24593, chapter 7.8.1, page 233 for more information
 * \param msr MSR address of the MTTR which should be checked and updated
 * \param changed pointer which indicates whether the MTRR needed to be changed
 * \param msrwords pointer to the MSR values which the MSR should have
 */
static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords)
{
	unsigned lo, hi;

	rdmsr(msr, lo, hi);

	if (lo != msrwords[0] || hi != msrwords[1]) {
		if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
		    boot_cpu_data.x86 == 15 &&
		    ((msrwords[0] | msrwords[1]) & K8_MTRR_RDMEM_WRMEM_MASK))
			k8_enable_fixed_iorrs();
		mtrr_wrmsr(msr, msrwords[0], msrwords[1]);
		*changed = true;
	}
}

int generic_get_free_region(unsigned long base, unsigned long size, int replace_reg)
/*  [SUMMARY] Get a free MTRR.
    <base> The starting (base) address of the region.
    <size> The size (in bytes) of the region.
    [RETURNS] The index of the region on success, else -1 on error.
*/
{
	int i, max;
	mtrr_type ltype;
	unsigned long lbase, lsize;

	max = num_var_ranges;
	if (replace_reg >= 0 && replace_reg < max)
		return replace_reg;
	for (i = 0; i < max; ++i) {
		mtrr_if->get(i, &lbase, &lsize, &ltype);
		if (lsize == 0)
			return i;
	}
	return -ENOSPC;
}

static void generic_get_mtrr(unsigned int reg, unsigned long *base,
			     unsigned long *size, mtrr_type *type)
{
	unsigned int mask_lo, mask_hi, base_lo, base_hi;

	rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi);
	if ((mask_lo & 0x800) == 0) {
		/*  Invalid (i.e. free) range  */
		*base = 0;
		*size = 0;
		*type = 0;
		return;
	}

	rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi);

	/* Work out the shifted address mask. */
	mask_lo = size_or_mask | mask_hi << (32 - PAGE_SHIFT)
	    | mask_lo >> PAGE_SHIFT;

	/* This works correctly if size is a power of two, i.e. a
	   contiguous range. */
	*size = -mask_lo;
	*base = base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT;
	*type = base_lo & 0xff;
}

/**
 * Checks and updates the fixed-range MTRRs if they differ from the saved set
 * \param frs pointer to fixed-range MTRR values, saved by get_fixed_ranges()
 */
static int set_fixed_ranges(mtrr_type * frs)
{
	unsigned long long *saved = (unsigned long long *) frs;
	bool changed = false;
	int block=-1, range;

	while (fixed_range_blocks[++block].ranges)
	    for (range=0; range < fixed_range_blocks[block].ranges; range++)
		set_fixed_range(fixed_range_blocks[block].base_msr + range,
		    &changed, (unsigned int *) saved++);

	return changed;
}

/*  Set the MSR pair relating to a var range. Returns TRUE if
    changes are made  */
static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)
{
	unsigned int lo, hi;
	bool changed = false;

	rdmsr(MTRRphysBase_MSR(index), lo, hi);
	if ((vr->base_lo & 0xfffff0ffUL) != (lo & 0xfffff0ffUL)
	    || (vr->base_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
		(hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
		mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
		changed = true;
	}

	rdmsr(MTRRphysMask_MSR(index), lo, hi);

	if ((vr->mask_lo & 0xfffff800UL) != (lo & 0xfffff800UL)
	    || (vr->mask_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
		(hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
		mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
		changed = true;
	}
	return changed;
}

static u32 deftype_lo, deftype_hi;

static unsigned long set_mtrr_state(void)
/*  [SUMMARY] Set the MTRR state for this CPU.
    <state> The MTRR state information to read.
    <ctxt> Some relevant CPU context.
    [NOTE] The CPU must already be in a safe state for MTRR changes.
    [RETURNS] 0 if no changes made, else a mask indication what was changed.
*/
{
	unsigned int i;
	unsigned long change_mask = 0;

	for (i = 0; i < num_var_ranges; i++)
		if (set_mtrr_var_ranges(i, &mtrr_state.var_ranges[i]))
			change_mask |= MTRR_CHANGE_MASK_VARIABLE;

	if (mtrr_state.have_fixed && set_fixed_ranges(mtrr_state.fixed_ranges))
		change_mask |= MTRR_CHANGE_MASK_FIXED;

	/*  Set_mtrr_restore restores the old value of MTRRdefType,
	   so to set it we fiddle with the saved value  */
	if ((deftype_lo & 0xff) != mtrr_state.def_type
	    || ((deftype_lo & 0xc00) >> 10) != mtrr_state.enabled) {
		deftype_lo = (deftype_lo & ~0xcff) | mtrr_state.def_type | (mtrr_state.enabled << 10);
		change_mask |= MTRR_CHANGE_MASK_DEFTYPE;
	}

	return change_mask;
}


static unsigned long cr4 = 0;
static DEFINE_SPINLOCK(set_atomicity_lock);

/*
 * Since we are disabling the cache don't allow any interrupts - they
 * would run extremely slow and would only increase the pain.  The caller must
 * ensure that local interrupts are disabled and are reenabled after post_set()
 * has been called.
 */

static void prepare_set(void) __acquires(set_atomicity_lock)
{
	unsigned long cr0;

	/*  Note that this is not ideal, since the cache is only flushed/disabled
	   for this CPU while the MTRRs are changed, but changing this requires
	   more invasive changes to the way the kernel boots  */

	spin_lock(&set_atomicity_lock);

	/*  Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */
	cr0 = read_cr0() | X86_CR0_CD;
	write_cr0(cr0);
	wbinvd();

	/*  Save value of CR4 and clear Page Global Enable (bit 7)  */
	if ( cpu_has_pge ) {
		cr4 = read_cr4();
		write_cr4(cr4 & ~X86_CR4_PGE);
	}

	/* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
	__flush_tlb();

	/*  Save MTRR state */
	rdmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);

	/*  Disable MTRRs, and set the default type to uncached  */
	mtrr_wrmsr(MTRRdefType_MSR, deftype_lo & ~0xcff, deftype_hi);
}

static void post_set(void) __releases(set_atomicity_lock)
{
	/*  Flush TLBs (no need to flush caches - they are disabled)  */
	__flush_tlb();

	/* Intel (P6) standard MTRRs */
	mtrr_wrmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);
		
	/*  Enable caches  */
	write_cr0(read_cr0() & 0xbfffffff);

	/*  Restore value of CR4  */
	if ( cpu_has_pge )
		write_cr4(cr4);
	spin_unlock(&set_atomicity_lock);
}

static void generic_set_all(void)
{
	unsigned long mask, count;
	unsigned long flags;

	local_irq_save(flags);
	prepare_set();

	/* Actually set the state */
	mask = set_mtrr_state();

	post_set();
	local_irq_restore(flags);

	/*  Use the atomic bitops to update the global mask  */
	for (count = 0; count < sizeof mask * 8; ++count) {
		if (mask & 0x01)
			set_bit(count, &smp_changes_mask);
		mask >>= 1;
	}
	
}

static void generic_set_mtrr(unsigned int reg, unsigned long base,
			     unsigned long size, mtrr_type type)
/*  [SUMMARY] Set variable MTRR register on the local CPU.
    <reg> The register to set.
    <base> The base address of the region.
    <size> The size of the region. If this is 0 the region is disabled.
    <type> The type of the region.
    [RETURNS] Nothing.
*/
{
	unsigned long flags;
	struct mtrr_var_range *vr;

	vr = &mtrr_state.var_ranges[reg];

	local_irq_save(flags);
	prepare_set();

	if (size == 0) {
		/* The invalid bit is kept in the mask, so we simply clear the
		   relevant mask register to disable a range. */
		mtrr_wrmsr(MTRRphysMask_MSR(reg), 0, 0);
		memset(vr, 0, sizeof(struct mtrr_var_range));
	} else {
		vr->base_lo = base << PAGE_SHIFT | type;
		vr->base_hi = (base & size_and_mask) >> (32 - PAGE_SHIFT);
		vr->mask_lo = -size << PAGE_SHIFT | 0x800;
		vr->mask_hi = (-size & size_and_mask) >> (32 - PAGE_SHIFT);

		mtrr_wrmsr(MTRRphysBase_MSR(reg), vr->base_lo, vr->base_hi);
		mtrr_wrmsr(MTRRphysMask_MSR(reg), vr->mask_lo, vr->mask_hi);
	}

	post_set();
	local_irq_restore(flags);
}

int generic_validate_add_page(unsigned long base, unsigned long size, unsigned int type)
{
	unsigned long lbase, last;

	/*  For Intel PPro stepping <= 7, must be 4 MiB aligned 
	    and not touch 0x70000000->0x7003FFFF */
	if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 &&
	    boot_cpu_data.x86_model == 1 &&
	    boot_cpu_data.x86_mask <= 7) {
		if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) {
			printk(KERN_WARNING "mtrr: base(0x%lx000) is not 4 MiB aligned\n", base);
			return -EINVAL;
		}
		if (!(base + size < 0x70000 || base > 0x7003F) &&
		    (type == MTRR_TYPE_WRCOMB
		     || type == MTRR_TYPE_WRBACK)) {
			printk(KERN_WARNING "mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n");
			return -EINVAL;
		}
	}

	/*  Check upper bits of base and last are equal and lower bits are 0
	    for base and 1 for last  */
	last = base + size - 1;
	for (lbase = base; !(lbase & 1) && (last & 1);
	     lbase = lbase >> 1, last = last >> 1) ;
	if (lbase != last) {
		printk(KERN_WARNING "mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n",
		       base, size);
		return -EINVAL;
	}
	return 0;
}


static int generic_have_wrcomb(void)
{
	unsigned long config, dummy;
	rdmsr(MTRRcap_MSR, config, dummy);
	return (config & (1 << 10));
}

int positive_have_wrcomb(void)
{
	return 1;
}

/* generic structure...
 */
struct mtrr_ops generic_mtrr_ops = {
	.use_intel_if      = 1,
	.set_all	   = generic_set_all,
	.get               = generic_get_mtrr,
	.get_free_region   = generic_get_free_region,
	.set               = generic_set_mtrr,
	.validate_add_page = generic_validate_add_page,
	.have_wrcomb       = generic_have_wrcomb,
};
Commit	Line	Data
1da177e4 LT	1	/* This only handles 32bit MTRR on 32bit hosts. This is strictly wrong
	2	because MTRRs can span upto 40 bits (36bits on most modern x86) */
	3	#include <linux/init.h>
	4	#include <linux/slab.h>
	5	#include <linux/mm.h>
365bff80	6	#include <linux/module.h>
1da177e4 LT	7	#include <asm/io.h>
	8	#include <asm/mtrr.h>
	9	#include <asm/msr.h>
	10	#include <asm/system.h>
	11	#include <asm/cpufeature.h>
7ebad705	12	#include <asm/processor-flags.h>
1da177e4 LT	13	#include <asm/tlbflush.h>
	14	#include "mtrr.h"
	15
	16	struct mtrr_state {
99fc8d42	17	struct mtrr_var_range var_ranges[MAX_VAR_RANGES];
1da177e4 LT	18	mtrr_type fixed_ranges[NUM_FIXED_RANGES];
1da177e4 LT	19	unsigned char enabled;
365bff80	20	unsigned char have_fixed;
1da177e4 LT	21	mtrr_type def_type;
	22	};
	23
de938c51 BK	24	struct fixed_range_block {
	25	int base_msr; /* start address of an MTRR block */
	26	int ranges; /* number of MTRRs in this block */
	27	};
	28
	29	static struct fixed_range_block fixed_range_blocks[] = {
	30	{ MTRRfix64K_00000_MSR, 1 }, /* one 64k MTRR */
	31	{ MTRRfix16K_80000_MSR, 2 }, /* two 16k MTRRs */
	32	{ MTRRfix4K_C0000_MSR, 8 }, /* eight 4k MTRRs */
	33	{}
	34	};
	35
1da177e4 LT	36	static unsigned long smp_changes_mask;
	37	static struct mtrr_state mtrr_state = {};
	38
365bff80 JB	39	#undef MODULE_PARAM_PREFIX
	40	#define MODULE_PARAM_PREFIX "mtrr."
	41
25c16b99	42	static int mtrr_show;
365bff80 JB	43	module_param_named(show, mtrr_show, bool, 0);
365bff80 JB	44
1da177e4	45	/* Get the MSR pair relating to a var range */
bf8c4817	46	static void
1da177e4 LT	47	get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr)
	48	{
	49	rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
	50	rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
	51	}
	52
2b3b4835	53	static void
1da177e4 LT	54	get_fixed_ranges(mtrr_type * frs)
	55	{
	56	unsigned int p = (unsigned int ) frs;
	57	int i;
	58
	59	rdmsr(MTRRfix64K_00000_MSR, p[0], p[1]);
	60
	61	for (i = 0; i < 2; i++)
	62	rdmsr(MTRRfix16K_80000_MSR + i, p[2 + i * 2], p[3 + i * 2]);
	63	for (i = 0; i < 8; i++)
	64	rdmsr(MTRRfix4K_C0000_MSR + i, p[6 + i * 2], p[7 + i * 2]);
	65	}
	66
2b3b4835 BK	67	void mtrr_save_fixed_ranges(void *info)
2b3b4835 BK	68	{
84288ad8 AM	69	if (cpu_has_mtrr)
84288ad8 AM	70	get_fixed_ranges(mtrr_state.fixed_ranges);
2b3b4835 BK	71	}
2b3b4835 BK	72
bf8c4817	73	static void print_fixed(unsigned base, unsigned step, const mtrr_type*types)
365bff80 JB	74	{
	75	unsigned i;
	76
	77	for (i = 0; i < 8; ++i, ++types, base += step)
25c16b99 RD	78	printk(KERN_INFO "MTRR %05X-%05X %s\n",
25c16b99 RD	79	base, base + step - 1, mtrr_attrib_to_str(*types));
365bff80 JB	80	}
365bff80 JB	81
1da177e4	82	/* Grab all of the MTRR state for this CPU into state /
9ef231a4	83	void __init get_mtrr_state(void)
1da177e4 LT	84	{
	85	unsigned int i;
	86	struct mtrr_var_range *vrs;
	87	unsigned lo, dummy;
	88
1da177e4 LT	89	vrs = mtrr_state.var_ranges;
1da177e4 LT	90
365bff80 JB	91	rdmsr(MTRRcap_MSR, lo, dummy);
	92	mtrr_state.have_fixed = (lo >> 8) & 1;
	93
1da177e4 LT	94	for (i = 0; i < num_var_ranges; i++)
1da177e4 LT	95	get_mtrr_var_range(i, &vrs[i]);
365bff80 JB	96	if (mtrr_state.have_fixed)
365bff80 JB	97	get_fixed_ranges(mtrr_state.fixed_ranges);
1da177e4 LT	98
	99	rdmsr(MTRRdefType_MSR, lo, dummy);
	100	mtrr_state.def_type = (lo & 0xff);
	101	mtrr_state.enabled = (lo & 0xc00) >> 10;
365bff80 JB	102
	103	if (mtrr_show) {
	104	int high_width;
	105
	106	printk(KERN_INFO "MTRR default type: %s\n", mtrr_attrib_to_str(mtrr_state.def_type));
	107	if (mtrr_state.have_fixed) {
	108	printk(KERN_INFO "MTRR fixed ranges %sabled:\n",
	109	mtrr_state.enabled & 1 ? "en" : "dis");
	110	print_fixed(0x00000, 0x10000, mtrr_state.fixed_ranges + 0);
	111	for (i = 0; i < 2; ++i)
	112	print_fixed(0x80000 + i * 0x20000, 0x04000, mtrr_state.fixed_ranges + (i + 1) * 8);
	113	for (i = 0; i < 8; ++i)
	114	print_fixed(0xC0000 + i * 0x08000, 0x01000, mtrr_state.fixed_ranges + (i + 3) * 8);
	115	}
	116	printk(KERN_INFO "MTRR variable ranges %sabled:\n",
	117	mtrr_state.enabled & 2 ? "en" : "dis");
	118	high_width = ((size_or_mask ? ffs(size_or_mask) - 1 : 32) - (32 - PAGE_SHIFT) + 3) / 4;
	119	for (i = 0; i < num_var_ranges; ++i) {
	120	if (mtrr_state.var_ranges[i].mask_lo & (1 << 11))
	121	printk(KERN_INFO "MTRR %u base %0X%05X000 mask %0X%05X000 %s\n",
	122	i,
	123	high_width,
	124	mtrr_state.var_ranges[i].base_hi,
	125	mtrr_state.var_ranges[i].base_lo >> 12,
	126	high_width,
	127	mtrr_state.var_ranges[i].mask_hi,
	128	mtrr_state.var_ranges[i].mask_lo >> 12,
	129	mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 0xff));
	130	else
	131	printk(KERN_INFO "MTRR %u disabled\n", i);
	132	}
	133	}
1da177e4 LT	134	}
1da177e4 LT	135
1da177e4 LT	136	/* Some BIOS's are fucked and don't set all MTRRs the same! */
	137	void __init mtrr_state_warn(void)
	138	{
	139	unsigned long mask = smp_changes_mask;
	140
	141	if (!mask)
	142	return;
	143	if (mask & MTRR_CHANGE_MASK_FIXED)
	144	printk(KERN_WARNING "mtrr: your CPUs had inconsistent fixed MTRR settings\n");
	145	if (mask & MTRR_CHANGE_MASK_VARIABLE)
	146	printk(KERN_WARNING "mtrr: your CPUs had inconsistent variable MTRR settings\n");
	147	if (mask & MTRR_CHANGE_MASK_DEFTYPE)
	148	printk(KERN_WARNING "mtrr: your CPUs had inconsistent MTRRdefType settings\n");
	149	printk(KERN_INFO "mtrr: probably your BIOS does not setup all CPUs.\n");
	150	printk(KERN_INFO "mtrr: corrected configuration.\n");
	151	}
	152
	153	/* Doesn't attempt to pass an error out to MTRR users
	154	because it's quite complicated in some cases and probably not
	155	worth it because the best error handling is to ignore it. */
	156	void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b)
	157	{
	158	if (wrmsr_safe(msr, a, b) < 0)
	159	printk(KERN_ERR
	160	"MTRR: CPU %u: Writing MSR %x to %x:%x failed\n",
	161	smp_processor_id(), msr, a, b);
	162	}
	163
de938c51 BK	164	/**
	165	* Enable and allow read/write of extended fixed-range MTRR bits on K8 CPUs
	166	* see AMD publication no. 24593, chapter 3.2.1 for more information
	167	*/
	168	static inline void k8_enable_fixed_iorrs(void)
	169	{
	170	unsigned lo, hi;
	171
	172	rdmsr(MSR_K8_SYSCFG, lo, hi);
	173	mtrr_wrmsr(MSR_K8_SYSCFG, lo
	174	\| K8_MTRRFIXRANGE_DRAM_ENABLE
	175	\| K8_MTRRFIXRANGE_DRAM_MODIFY, hi);
	176	}
	177
	178	/**
	179	* Checks and updates an fixed-range MTRR if it differs from the value it
27b46d76	180	* should have. If K8 extentions are wanted, update the K8 SYSCFG MSR also.
de938c51 BK	181	* see AMD publication no. 24593, chapter 7.8.1, page 233 for more information
	182	* \param msr MSR address of the MTTR which should be checked and updated
	183	* \param changed pointer which indicates whether the MTRR needed to be changed
	184	* \param msrwords pointer to the MSR values which the MSR should have
	185	*/
2d2ee8de	186	static void set_fixed_range(int msr, bool changed, unsigned int msrwords)
de938c51 BK	187	{
	188	unsigned lo, hi;
	189
	190	rdmsr(msr, lo, hi);
	191
	192	if (lo != msrwords[0] \|\| hi != msrwords[1]) {
	193	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
	194	boot_cpu_data.x86 == 15 &&
	195	((msrwords[0] \| msrwords[1]) & K8_MTRR_RDMEM_WRMEM_MASK))
	196	k8_enable_fixed_iorrs();
	197	mtrr_wrmsr(msr, msrwords[0], msrwords[1]);
2d2ee8de	198	*changed = true;
de938c51 BK	199	}
	200	}
	201
365bff80	202	int generic_get_free_region(unsigned long base, unsigned long size, int replace_reg)
1da177e4 LT	203	/* [SUMMARY] Get a free MTRR.
	204	<base> The starting (base) address of the region.
	205	<size> The size (in bytes) of the region.
	206	[RETURNS] The index of the region on success, else -1 on error.
	207	*/
	208	{
	209	int i, max;
	210	mtrr_type ltype;
365bff80	211	unsigned long lbase, lsize;
1da177e4 LT	212
1da177e4 LT	213	max = num_var_ranges;
365bff80 JB	214	if (replace_reg >= 0 && replace_reg < max)
365bff80 JB	215	return replace_reg;
1da177e4 LT	216	for (i = 0; i < max; ++i) {
	217	mtrr_if->get(i, &lbase, &lsize, &ltype);
	218	if (lsize == 0)
	219	return i;
	220	}
	221	return -ENOSPC;
	222	}
	223
408b664a	224	static void generic_get_mtrr(unsigned int reg, unsigned long *base,
365bff80	225	unsigned long size, mtrr_type type)
1da177e4 LT	226	{
	227	unsigned int mask_lo, mask_hi, base_lo, base_hi;
	228
	229	rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi);
	230	if ((mask_lo & 0x800) == 0) {
	231	/* Invalid (i.e. free) range */
	232	*base = 0;
	233	*size = 0;
	234	*type = 0;
	235	return;
	236	}
	237
	238	rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi);
	239
	240	/* Work out the shifted address mask. */
	241	mask_lo = size_or_mask \| mask_hi << (32 - PAGE_SHIFT)
	242	\| mask_lo >> PAGE_SHIFT;
	243
	244	/* This works correctly if size is a power of two, i.e. a
	245	contiguous range. */
	246	*size = -mask_lo;
	247	*base = base_hi << (32 - PAGE_SHIFT) \| base_lo >> PAGE_SHIFT;
	248	*type = base_lo & 0xff;
	249	}
	250
de938c51 BK	251	/**
	252	* Checks and updates the fixed-range MTRRs if they differ from the saved set
	253	* \param frs pointer to fixed-range MTRR values, saved by get_fixed_ranges()
	254	*/
1da177e4 LT	255	static int set_fixed_ranges(mtrr_type * frs)
1da177e4 LT	256	{
de938c51	257	unsigned long long saved = (unsigned long long ) frs;
2d2ee8de	258	bool changed = false;
de938c51	259	int block=-1, range;
1da177e4	260
de938c51 BK	261	while (fixed_range_blocks[++block].ranges)
	262	for (range=0; range < fixed_range_blocks[block].ranges; range++)
	263	set_fixed_range(fixed_range_blocks[block].base_msr + range,
	264	&changed, (unsigned int *) saved++);
1da177e4	265
1da177e4 LT	266	return changed;
	267	}
	268
	269	/* Set the MSR pair relating to a var range. Returns TRUE if
	270	changes are made */
2d2ee8de	271	static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)
1da177e4 LT	272	{
1da177e4 LT	273	unsigned int lo, hi;
2d2ee8de	274	bool changed = false;
1da177e4 LT	275
	276	rdmsr(MTRRphysBase_MSR(index), lo, hi);
	277	if ((vr->base_lo & 0xfffff0ffUL) != (lo & 0xfffff0ffUL)
cf94b62f SS	278	\|\| (vr->base_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
cf94b62f SS	279	(hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
1da177e4	280	mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
2d2ee8de	281	changed = true;
1da177e4 LT	282	}
	283
	284	rdmsr(MTRRphysMask_MSR(index), lo, hi);
	285
	286	if ((vr->mask_lo & 0xfffff800UL) != (lo & 0xfffff800UL)
cf94b62f SS	287	\|\| (vr->mask_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
cf94b62f SS	288	(hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
1da177e4	289	mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
2d2ee8de	290	changed = true;
1da177e4 LT	291	}
	292	return changed;
	293	}
	294
365bff80 JB	295	static u32 deftype_lo, deftype_hi;
	296
	297	static unsigned long set_mtrr_state(void)
1da177e4 LT	298	/* [SUMMARY] Set the MTRR state for this CPU.
	299	<state> The MTRR state information to read.
	300	<ctxt> Some relevant CPU context.
	301	[NOTE] The CPU must already be in a safe state for MTRR changes.
	302	[RETURNS] 0 if no changes made, else a mask indication what was changed.
	303	*/
	304	{
	305	unsigned int i;
	306	unsigned long change_mask = 0;
	307
	308	for (i = 0; i < num_var_ranges; i++)
	309	if (set_mtrr_var_ranges(i, &mtrr_state.var_ranges[i]))
	310	change_mask \|= MTRR_CHANGE_MASK_VARIABLE;
	311
365bff80	312	if (mtrr_state.have_fixed && set_fixed_ranges(mtrr_state.fixed_ranges))
1da177e4 LT	313	change_mask \|= MTRR_CHANGE_MASK_FIXED;
	314
	315	/* Set_mtrr_restore restores the old value of MTRRdefType,
	316	so to set it we fiddle with the saved value */
	317	if ((deftype_lo & 0xff) != mtrr_state.def_type
	318	\|\| ((deftype_lo & 0xc00) >> 10) != mtrr_state.enabled) {
365bff80	319	deftype_lo = (deftype_lo & ~0xcff) \| mtrr_state.def_type \| (mtrr_state.enabled << 10);
1da177e4 LT	320	change_mask \|= MTRR_CHANGE_MASK_DEFTYPE;
	321	}
	322
	323	return change_mask;
	324	}
	325
	326
	327	static unsigned long cr4 = 0;
1da177e4 LT	328	static DEFINE_SPINLOCK(set_atomicity_lock);
	329
	330	/*
	331	* Since we are disabling the cache don't allow any interrupts - they
	332	* would run extremely slow and would only increase the pain. The caller must
	333	* ensure that local interrupts are disabled and are reenabled after post_set()
	334	* has been called.
	335	*/
	336
182daa55	337	static void prepare_set(void) __acquires(set_atomicity_lock)
1da177e4 LT	338	{
	339	unsigned long cr0;
	340
	341	/* Note that this is not ideal, since the cache is only flushed/disabled
	342	for this CPU while the MTRRs are changed, but changing this requires
	343	more invasive changes to the way the kernel boots */
	344
	345	spin_lock(&set_atomicity_lock);
	346
	347	/* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */
7ebad705	348	cr0 = read_cr0() \| X86_CR0_CD;
1da177e4 LT	349	write_cr0(cr0);
	350	wbinvd();
	351
	352	/* Save value of CR4 and clear Page Global Enable (bit 7) */
	353	if ( cpu_has_pge ) {
	354	cr4 = read_cr4();
	355	write_cr4(cr4 & ~X86_CR4_PGE);
	356	}
	357
	358	/* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
	359	__flush_tlb();
	360
	361	/* Save MTRR state */
	362	rdmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);
	363
	364	/* Disable MTRRs, and set the default type to uncached */
365bff80	365	mtrr_wrmsr(MTRRdefType_MSR, deftype_lo & ~0xcff, deftype_hi);
1da177e4 LT	366	}
1da177e4 LT	367
182daa55	368	static void post_set(void) __releases(set_atomicity_lock)
1da177e4 LT	369	{
	370	/* Flush TLBs (no need to flush caches - they are disabled) */
	371	__flush_tlb();
	372
	373	/* Intel (P6) standard MTRRs */
	374	mtrr_wrmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);
	375
	376	/* Enable caches */
	377	write_cr0(read_cr0() & 0xbfffffff);
	378
	379	/* Restore value of CR4 */
	380	if ( cpu_has_pge )
	381	write_cr4(cr4);
	382	spin_unlock(&set_atomicity_lock);
	383	}
	384
	385	static void generic_set_all(void)
	386	{
	387	unsigned long mask, count;
	388	unsigned long flags;
	389
	390	local_irq_save(flags);
	391	prepare_set();
	392
	393	/* Actually set the state */
365bff80	394	mask = set_mtrr_state();
1da177e4 LT	395
	396	post_set();
	397	local_irq_restore(flags);
	398
	399	/* Use the atomic bitops to update the global mask */
	400	for (count = 0; count < sizeof mask * 8; ++count) {
	401	if (mask & 0x01)
	402	set_bit(count, &smp_changes_mask);
	403	mask >>= 1;
	404	}
	405
	406	}
	407
	408	static void generic_set_mtrr(unsigned int reg, unsigned long base,
	409	unsigned long size, mtrr_type type)
	410	/* [SUMMARY] Set variable MTRR register on the local CPU.
	411	<reg> The register to set.
	412	<base> The base address of the region.
	413	<size> The size of the region. If this is 0 the region is disabled.
	414	<type> The type of the region.
1da177e4 LT	415	[RETURNS] Nothing.
	416	*/
	417	{
	418	unsigned long flags;
3b520b23 SL	419	struct mtrr_var_range *vr;
	420
	421	vr = &mtrr_state.var_ranges[reg];
1da177e4 LT	422
	423	local_irq_save(flags);
	424	prepare_set();
	425
	426	if (size == 0) {
	427	/* The invalid bit is kept in the mask, so we simply clear the
	428	relevant mask register to disable a range. */
	429	mtrr_wrmsr(MTRRphysMask_MSR(reg), 0, 0);
3b520b23	430	memset(vr, 0, sizeof(struct mtrr_var_range));
1da177e4	431	} else {
3b520b23 SL	432	vr->base_lo = base << PAGE_SHIFT \| type;
	433	vr->base_hi = (base & size_and_mask) >> (32 - PAGE_SHIFT);
	434	vr->mask_lo = -size << PAGE_SHIFT \| 0x800;
	435	vr->mask_hi = (-size & size_and_mask) >> (32 - PAGE_SHIFT);
	436
	437	mtrr_wrmsr(MTRRphysBase_MSR(reg), vr->base_lo, vr->base_hi);
	438	mtrr_wrmsr(MTRRphysMask_MSR(reg), vr->mask_lo, vr->mask_hi);
1da177e4 LT	439	}
	440
	441	post_set();
	442	local_irq_restore(flags);
	443	}
	444
	445	int generic_validate_add_page(unsigned long base, unsigned long size, unsigned int type)
	446	{
	447	unsigned long lbase, last;
	448
	449	/* For Intel PPro stepping <= 7, must be 4 MiB aligned
	450	and not touch 0x70000000->0x7003FFFF */
	451	if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 &&
	452	boot_cpu_data.x86_model == 1 &&
	453	boot_cpu_data.x86_mask <= 7) {
	454	if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) {
	455	printk(KERN_WARNING "mtrr: base(0x%lx000) is not 4 MiB aligned\n", base);
	456	return -EINVAL;
	457	}
9b483417	458	if (!(base + size < 0x70000 \|\| base > 0x7003F) &&
1da177e4 LT	459	(type == MTRR_TYPE_WRCOMB
	460	\|\| type == MTRR_TYPE_WRBACK)) {
	461	printk(KERN_WARNING "mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n");
	462	return -EINVAL;
	463	}
	464	}
	465
1da177e4 LT	466	/* Check upper bits of base and last are equal and lower bits are 0
	467	for base and 1 for last */
	468	last = base + size - 1;
	469	for (lbase = base; !(lbase & 1) && (last & 1);
	470	lbase = lbase >> 1, last = last >> 1) ;
	471	if (lbase != last) {
	472	printk(KERN_WARNING "mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n",
	473	base, size);
	474	return -EINVAL;
	475	}
	476	return 0;
	477	}
	478
	479
	480	static int generic_have_wrcomb(void)
	481	{
	482	unsigned long config, dummy;
	483	rdmsr(MTRRcap_MSR, config, dummy);
	484	return (config & (1 << 10));
	485	}
	486
	487	int positive_have_wrcomb(void)
	488	{
	489	return 1;
	490	}
	491
	492	/* generic structure...
	493	*/
	494	struct mtrr_ops generic_mtrr_ops = {
	495	.use_intel_if = 1,
	496	.set_all = generic_set_all,
	497	.get = generic_get_mtrr,
	498	.get_free_region = generic_get_free_region,
	499	.set = generic_set_mtrr,
	500	.validate_add_page = generic_validate_add_page,
	501	.have_wrcomb = generic_have_wrcomb,
	502	};