1 /* Generic MTRR (Memory Type Range Register) driver.
3 Copyright (C) 1997-2000 Richard Gooch
4 Copyright (c) 2002 Patrick Mochel
6 This library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Library General Public
8 License as published by the Free Software Foundation; either
9 version 2 of the License, or (at your option) any later version.
11 This library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Library General Public License for more details.
16 You should have received a copy of the GNU Library General Public
17 License along with this library; if not, write to the Free
18 Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 Richard Gooch may be reached by email at rgooch@atnf.csiro.au
21 The postal address is:
22 Richard Gooch, c/o ATNF, P. O. Box 76, Epping, N.S.W., 2121, Australia.
24 Source: "Pentium Pro Family Developer's Manual, Volume 3:
25 Operating System Writer's Guide" (Intel document number 242692),
28 This was cleaned and made readable by Patrick Mochel <mochel@osdl.org>
30 Source: Intel Architecture Software Developers Manual, Volume 3:
31 System Programming Guide; Section 9.11. (1997 edition - PPro).
34 #include <linux/module.h>
35 #include <linux/init.h>
36 #include <linux/pci.h>
37 #include <linux/smp.h>
38 #include <linux/cpu.h>
42 #include <asm/uaccess.h>
43 #include <asm/processor.h>
47 #define MTRR_VERSION "2.0 (20020519)"
49 u32 num_var_ranges
= 0;
51 unsigned int *usage_table
;
52 static DECLARE_MUTEX(main_lock
);
54 u32 size_or_mask
, size_and_mask
;
56 static struct mtrr_ops
* mtrr_ops
[X86_VENDOR_NUM
] = {};
58 struct mtrr_ops
* mtrr_if
= NULL
;
60 static void set_mtrr(unsigned int reg
, unsigned long base
,
61 unsigned long size
, mtrr_type type
);
63 extern int arr3_protected
;
65 void set_mtrr_ops(struct mtrr_ops
* ops
)
67 if (ops
->vendor
&& ops
->vendor
< X86_VENDOR_NUM
)
68 mtrr_ops
[ops
->vendor
] = ops
;
71 /* Returns non-zero if we have the write-combining memory type */
72 static int have_wrcomb(void)
76 if ((dev
= pci_get_class(PCI_CLASS_BRIDGE_HOST
<< 8, NULL
)) != NULL
) {
77 /* ServerWorks LE chipsets have problems with write-combining
78 Don't allow it and leave room for other chipsets to be tagged */
79 if (dev
->vendor
== PCI_VENDOR_ID_SERVERWORKS
&&
80 dev
->device
== PCI_DEVICE_ID_SERVERWORKS_LE
) {
81 printk(KERN_INFO
"mtrr: Serverworks LE detected. Write-combining disabled.\n");
85 /* Intel 450NX errata # 23. Non ascending cachline evictions to
86 write combining memory may resulting in data corruption */
87 if (dev
->vendor
== PCI_VENDOR_ID_INTEL
&&
88 dev
->device
== PCI_DEVICE_ID_INTEL_82451NX
) {
89 printk(KERN_INFO
"mtrr: Intel 450NX MMC detected. Write-combining disabled.\n");
95 return (mtrr_if
->have_wrcomb
? mtrr_if
->have_wrcomb() : 0);
98 /* This function returns the number of variable MTRRs */
99 static void __init
set_num_var_ranges(void)
101 unsigned long config
= 0, dummy
;
104 rdmsr(MTRRcap_MSR
, config
, dummy
);
105 } else if (is_cpu(AMD
))
107 else if (is_cpu(CYRIX
) || is_cpu(CENTAUR
))
109 num_var_ranges
= config
& 0xff;
112 static void __init
init_table(void)
116 max
= num_var_ranges
;
117 if ((usage_table
= kmalloc(max
* sizeof *usage_table
, GFP_KERNEL
))
119 printk(KERN_ERR
"mtrr: could not allocate\n");
122 for (i
= 0; i
< max
; i
++)
126 struct set_mtrr_data
{
129 unsigned long smp_base
;
130 unsigned long smp_size
;
131 unsigned int smp_reg
;
137 static void ipi_handler(void *info
)
138 /* [SUMMARY] Synchronisation handler. Executed by "other" CPUs.
142 struct set_mtrr_data
*data
= info
;
145 local_irq_save(flags
);
147 atomic_dec(&data
->count
);
148 while(!atomic_read(&data
->gate
))
151 /* The master has cleared me to execute */
152 if (data
->smp_reg
!= ~0U)
153 mtrr_if
->set(data
->smp_reg
, data
->smp_base
,
154 data
->smp_size
, data
->smp_type
);
158 atomic_dec(&data
->count
);
159 while(atomic_read(&data
->gate
))
162 atomic_dec(&data
->count
);
163 local_irq_restore(flags
);
169 * set_mtrr - update mtrrs on all processors
170 * @reg: mtrr in question
175 * This is kinda tricky, but fortunately, Intel spelled it out for us cleanly:
177 * 1. Send IPI to do the following:
178 * 2. Disable Interrupts
179 * 3. Wait for all procs to do so
180 * 4. Enter no-fill cache mode
184 * 8. Disable all range registers
185 * 9. Update the MTRRs
186 * 10. Enable all range registers
187 * 11. Flush all TLBs and caches again
188 * 12. Enter normal cache mode and reenable caching
190 * 14. Wait for buddies to catch up
191 * 15. Enable interrupts.
193 * What does that mean for us? Well, first we set data.count to the number
194 * of CPUs. As each CPU disables interrupts, it'll decrement it once. We wait
195 * until it hits 0 and proceed. We set the data.gate flag and reset data.count.
196 * Meanwhile, they are waiting for that flag to be set. Once it's set, each
197 * CPU goes through the transition of updating MTRRs. The CPU vendors may each do it
198 * differently, so we call mtrr_if->set() callback and let them take care of it.
199 * When they're done, they again decrement data->count and wait for data.gate to
201 * When we finish, we wait for data.count to hit 0 and toggle the data.gate flag.
202 * Everyone then enables interrupts and we all continue on.
204 * Note that the mechanism is the same for UP systems, too; all the SMP stuff
207 static void set_mtrr(unsigned int reg
, unsigned long base
,
208 unsigned long size
, mtrr_type type
)
210 struct set_mtrr_data data
;
214 data
.smp_base
= base
;
215 data
.smp_size
= size
;
216 data
.smp_type
= type
;
217 atomic_set(&data
.count
, num_booting_cpus() - 1);
218 atomic_set(&data
.gate
,0);
220 /* Start the ball rolling on other CPUs */
221 if (smp_call_function(ipi_handler
, &data
, 1, 0) != 0)
222 panic("mtrr: timed out waiting for other CPUs\n");
224 local_irq_save(flags
);
226 while(atomic_read(&data
.count
))
229 /* ok, reset count and toggle gate */
230 atomic_set(&data
.count
, num_booting_cpus() - 1);
231 atomic_set(&data
.gate
,1);
233 /* do our MTRR business */
236 * We use this same function to initialize the mtrrs on boot.
237 * The state of the boot cpu's mtrrs has been saved, and we want
238 * to replicate across all the APs.
239 * If we're doing that @reg is set to something special...
242 mtrr_if
->set(reg
,base
,size
,type
);
244 /* wait for the others */
245 while(atomic_read(&data
.count
))
248 atomic_set(&data
.count
, num_booting_cpus() - 1);
249 atomic_set(&data
.gate
,0);
252 * Wait here for everyone to have seen the gate change
253 * So we're the last ones to touch 'data'
255 while(atomic_read(&data
.count
))
258 local_irq_restore(flags
);
262 * mtrr_add_page - Add a memory type region
263 * @base: Physical base address of region in pages (4 KB)
264 * @size: Physical size of region in pages (4 KB)
265 * @type: Type of MTRR desired
266 * @increment: If this is true do usage counting on the region
268 * Memory type region registers control the caching on newer Intel and
269 * non Intel processors. This function allows drivers to request an
270 * MTRR is added. The details and hardware specifics of each processor's
271 * implementation are hidden from the caller, but nevertheless the
272 * caller should expect to need to provide a power of two size on an
273 * equivalent power of two boundary.
275 * If the region cannot be added either because all regions are in use
276 * or the CPU cannot support it a negative value is returned. On success
277 * the register number for this entry is returned, but should be treated
280 * On a multiprocessor machine the changes are made to all processors.
281 * This is required on x86 by the Intel processors.
283 * The available types are
285 * %MTRR_TYPE_UNCACHABLE - No caching
287 * %MTRR_TYPE_WRBACK - Write data back in bursts whenever
289 * %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts
291 * %MTRR_TYPE_WRTHROUGH - Cache reads but not writes
293 * BUGS: Needs a quiet flag for the cases where drivers do not mind
294 * failures and do not wish system log messages to be sent.
297 int mtrr_add_page(unsigned long base
, unsigned long size
,
298 unsigned int type
, char increment
)
309 if ((error
= mtrr_if
->validate_add_page(base
,size
,type
)))
312 if (type
>= MTRR_NUM_TYPES
) {
313 printk(KERN_WARNING
"mtrr: type: %u invalid\n", type
);
317 /* If the type is WC, check that this processor supports it */
318 if ((type
== MTRR_TYPE_WRCOMB
) && !have_wrcomb()) {
320 "mtrr: your processor doesn't support write-combining\n");
324 if (base
& size_or_mask
|| size
& size_or_mask
) {
325 printk(KERN_WARNING
"mtrr: base or size exceeds the MTRR width\n");
331 /* Search for existing MTRR */
333 for (i
= 0; i
< num_var_ranges
; ++i
) {
334 mtrr_if
->get(i
, &lbase
, &lsize
, <ype
);
335 if (base
>= lbase
+ lsize
)
337 if ((base
< lbase
) && (base
+ size
<= lbase
))
339 /* At this point we know there is some kind of overlap/enclosure */
340 if ((base
< lbase
) || (base
+ size
> lbase
+ lsize
)) {
342 "mtrr: 0x%lx000,0x%lx000 overlaps existing"
343 " 0x%lx000,0x%x000\n", base
, size
, lbase
,
347 /* New region is enclosed by an existing region */
349 if (type
== MTRR_TYPE_UNCACHABLE
)
351 printk (KERN_WARNING
"mtrr: type mismatch for %lx000,%lx000 old: %s new: %s\n",
352 base
, size
, mtrr_attrib_to_str(ltype
),
353 mtrr_attrib_to_str(type
));
361 /* Search for an empty MTRR */
362 i
= mtrr_if
->get_free_region(base
, size
);
364 set_mtrr(i
, base
, size
, type
);
367 printk(KERN_INFO
"mtrr: no more MTRRs available\n");
375 * mtrr_add - Add a memory type region
376 * @base: Physical base address of region
377 * @size: Physical size of region
378 * @type: Type of MTRR desired
379 * @increment: If this is true do usage counting on the region
381 * Memory type region registers control the caching on newer Intel and
382 * non Intel processors. This function allows drivers to request an
383 * MTRR is added. The details and hardware specifics of each processor's
384 * implementation are hidden from the caller, but nevertheless the
385 * caller should expect to need to provide a power of two size on an
386 * equivalent power of two boundary.
388 * If the region cannot be added either because all regions are in use
389 * or the CPU cannot support it a negative value is returned. On success
390 * the register number for this entry is returned, but should be treated
393 * On a multiprocessor machine the changes are made to all processors.
394 * This is required on x86 by the Intel processors.
396 * The available types are
398 * %MTRR_TYPE_UNCACHABLE - No caching
400 * %MTRR_TYPE_WRBACK - Write data back in bursts whenever
402 * %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts
404 * %MTRR_TYPE_WRTHROUGH - Cache reads but not writes
406 * BUGS: Needs a quiet flag for the cases where drivers do not mind
407 * failures and do not wish system log messages to be sent.
411 mtrr_add(unsigned long base
, unsigned long size
, unsigned int type
,
414 if ((base
& (PAGE_SIZE
- 1)) || (size
& (PAGE_SIZE
- 1))) {
415 printk(KERN_WARNING
"mtrr: size and base must be multiples of 4 kiB\n");
416 printk(KERN_DEBUG
"mtrr: size: 0x%lx base: 0x%lx\n", size
, base
);
419 return mtrr_add_page(base
>> PAGE_SHIFT
, size
>> PAGE_SHIFT
, type
,
424 * mtrr_del_page - delete a memory type region
425 * @reg: Register returned by mtrr_add
426 * @base: Physical base address
427 * @size: Size of region
429 * If register is supplied then base and size are ignored. This is
430 * how drivers should call it.
432 * Releases an MTRR region. If the usage count drops to zero the
433 * register is freed and the region returns to default state.
434 * On success the register is returned, on failure a negative error
438 int mtrr_del_page(int reg
, unsigned long base
, unsigned long size
)
449 max
= num_var_ranges
;
452 /* Search for existing MTRR */
453 for (i
= 0; i
< max
; ++i
) {
454 mtrr_if
->get(i
, &lbase
, &lsize
, <ype
);
455 if (lbase
== base
&& lsize
== size
) {
461 printk(KERN_DEBUG
"mtrr: no MTRR for %lx000,%lx000 found\n", base
,
467 printk(KERN_WARNING
"mtrr: register: %d too big\n", reg
);
470 if (is_cpu(CYRIX
) && !use_intel()) {
471 if ((reg
== 3) && arr3_protected
) {
472 printk(KERN_WARNING
"mtrr: ARR3 cannot be changed\n");
476 mtrr_if
->get(reg
, &lbase
, &lsize
, <ype
);
478 printk(KERN_WARNING
"mtrr: MTRR %d not used\n", reg
);
481 if (usage_table
[reg
] < 1) {
482 printk(KERN_WARNING
"mtrr: reg: %d has count=0\n", reg
);
485 if (--usage_table
[reg
] < 1)
486 set_mtrr(reg
, 0, 0, 0);
493 * mtrr_del - delete a memory type region
494 * @reg: Register returned by mtrr_add
495 * @base: Physical base address
496 * @size: Size of region
498 * If register is supplied then base and size are ignored. This is
499 * how drivers should call it.
501 * Releases an MTRR region. If the usage count drops to zero the
502 * register is freed and the region returns to default state.
503 * On success the register is returned, on failure a negative error
508 mtrr_del(int reg
, unsigned long base
, unsigned long size
)
510 if ((base
& (PAGE_SIZE
- 1)) || (size
& (PAGE_SIZE
- 1))) {
511 printk(KERN_INFO
"mtrr: size and base must be multiples of 4 kiB\n");
512 printk(KERN_DEBUG
"mtrr: size: 0x%lx base: 0x%lx\n", size
, base
);
515 return mtrr_del_page(reg
, base
>> PAGE_SHIFT
, size
>> PAGE_SHIFT
);
518 EXPORT_SYMBOL(mtrr_add
);
519 EXPORT_SYMBOL(mtrr_del
);
522 * These should be called implicitly, but we can't yet until all the initcall
525 extern void amd_init_mtrr(void);
526 extern void cyrix_init_mtrr(void);
527 extern void centaur_init_mtrr(void);
529 static void __init
init_ifs(void)
536 static void __init
init_other_cpus(void)
541 /* bring up the other processors */
545 finalize_mtrr_state();
557 static struct mtrr_value
* mtrr_state
;
559 static int mtrr_save(struct sys_device
* sysdev
, u32 state
)
562 int size
= num_var_ranges
* sizeof(struct mtrr_value
);
564 mtrr_state
= kmalloc(size
,GFP_ATOMIC
);
566 memset(mtrr_state
,0,size
);
570 for (i
= 0; i
< num_var_ranges
; i
++) {
572 &mtrr_state
[i
].lbase
,
573 &mtrr_state
[i
].lsize
,
574 &mtrr_state
[i
].ltype
);
579 static int mtrr_restore(struct sys_device
* sysdev
)
583 for (i
= 0; i
< num_var_ranges
; i
++) {
584 if (mtrr_state
[i
].lsize
)
588 mtrr_state
[i
].ltype
);
596 static struct sysdev_driver mtrr_sysdev_driver
= {
597 .suspend
= mtrr_save
,
598 .resume
= mtrr_restore
,
603 * mtrr_init - initialize mtrrs on the boot CPU
605 * This needs to be called early; before any of the other CPUs are
606 * initialized (i.e. before smp_init()).
609 static int __init
mtrr_init(void)
614 mtrr_if
= &generic_mtrr_ops
;
615 size_or_mask
= 0xff000000; /* 36 bits */
616 size_and_mask
= 0x00f00000;
618 switch (boot_cpu_data
.x86_vendor
) {
620 /* The original Athlon docs said that
621 total addressable memory is 44 bits wide.
622 It was not really clear whether its MTRRs
623 follow this or not. (Read: 44 or 36 bits).
624 However, "x86-64_overview.pdf" explicitly
625 states that "previous implementations support
626 36 bit MTRRs" and also provides a way to
627 query the width (in bits) of the physical
628 addressable memory on the Hammer family.
630 if (boot_cpu_data
.x86
== 15
631 && (cpuid_eax(0x80000000) >= 0x80000008)) {
633 phys_addr
= cpuid_eax(0x80000008) & 0xff;
635 ~((1 << (phys_addr
- PAGE_SHIFT
)) - 1);
636 size_and_mask
= ~size_or_mask
& 0xfff00000;
638 /* Athlon MTRRs use an Intel-compatible interface for
639 * getting and setting */
641 case X86_VENDOR_CENTAUR
:
642 if (boot_cpu_data
.x86
== 6) {
643 /* VIA Cyrix family have Intel style MTRRs, but don't support PAE */
644 size_or_mask
= 0xfff00000; /* 32 bits */
653 switch (boot_cpu_data
.x86_vendor
) {
655 if (cpu_has_k6_mtrr
) {
656 /* Pre-Athlon (K6) AMD CPU MTRRs */
657 mtrr_if
= mtrr_ops
[X86_VENDOR_AMD
];
658 size_or_mask
= 0xfff00000; /* 32 bits */
662 case X86_VENDOR_CENTAUR
:
663 if (cpu_has_centaur_mcr
) {
664 mtrr_if
= mtrr_ops
[X86_VENDOR_CENTAUR
];
665 size_or_mask
= 0xfff00000; /* 32 bits */
669 case X86_VENDOR_CYRIX
:
670 if (cpu_has_cyrix_arr
) {
671 mtrr_if
= mtrr_ops
[X86_VENDOR_CYRIX
];
672 size_or_mask
= 0xfff00000; /* 32 bits */
680 printk(KERN_INFO
"mtrr: v%s\n",MTRR_VERSION
);
683 set_num_var_ranges();
687 return sysdev_driver_register(&cpu_sysdev_class
,
688 &mtrr_sysdev_driver
);
693 subsys_initcall(mtrr_init
);