1 #include <linux/ioport.h>
2 #include <linux/swap.h>
4 #include <asm/cacheflush.h>
8 #include <asm/page_types.h>
9 #include <asm/sections.h>
10 #include <asm/setup.h>
11 #include <asm/system.h>
12 #include <asm/tlbflush.h>
15 DEFINE_PER_CPU(struct mmu_gather
, mmu_gathers
);
17 unsigned long __initdata e820_table_start
;
18 unsigned long __meminitdata e820_table_end
;
19 unsigned long __meminitdata e820_table_top
;
24 #ifdef CONFIG_DIRECT_GBPAGES
31 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
32 static int disable_nx __cpuinitdata
;
37 * Control non-executable mappings for processes.
42 static int __init
noexec_setup(char *str
)
46 if (!strncmp(str
, "on", 2)) {
47 __supported_pte_mask
|= _PAGE_NX
;
49 } else if (!strncmp(str
, "off", 3)) {
51 __supported_pte_mask
&= ~_PAGE_NX
;
55 early_param("noexec", noexec_setup
);
59 static void __init
set_nx(void)
61 unsigned int v
[4], l
, h
;
63 if (cpu_has_pae
&& (cpuid_eax(0x80000000) > 0x80000001)) {
64 cpuid(0x80000001, &v
[0], &v
[1], &v
[2], &v
[3]);
66 if ((v
[3] & (1 << 20)) && !disable_nx
) {
67 rdmsr(MSR_EFER
, l
, h
);
69 wrmsr(MSR_EFER
, l
, h
);
71 __supported_pte_mask
|= _PAGE_NX
;
76 static inline void set_nx(void)
82 void __cpuinit
check_efer(void)
86 rdmsrl(MSR_EFER
, efer
);
87 if (!(efer
& EFER_NX
) || disable_nx
)
88 __supported_pte_mask
&= ~_PAGE_NX
;
92 static void __init
find_early_table_space(unsigned long end
, int use_pse
,
95 unsigned long puds
, pmds
, ptes
, tables
, start
;
97 puds
= (end
+ PUD_SIZE
- 1) >> PUD_SHIFT
;
98 tables
= roundup(puds
* sizeof(pud_t
), PAGE_SIZE
);
103 extra
= end
- ((end
>>PUD_SHIFT
) << PUD_SHIFT
);
104 pmds
= (extra
+ PMD_SIZE
- 1) >> PMD_SHIFT
;
106 pmds
= (end
+ PMD_SIZE
- 1) >> PMD_SHIFT
;
108 tables
+= roundup(pmds
* sizeof(pmd_t
), PAGE_SIZE
);
113 extra
= end
- ((end
>>PMD_SHIFT
) << PMD_SHIFT
);
117 ptes
= (extra
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
119 ptes
= (end
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
121 tables
+= roundup(ptes
* sizeof(pte_t
), PAGE_SIZE
);
125 tables
+= roundup(__end_of_fixed_addresses
* sizeof(pte_t
), PAGE_SIZE
);
129 * RED-PEN putting page tables only on node 0 could
130 * cause a hotspot and fill up ZONE_DMA. The page tables
131 * need roughly 0.5KB per GB.
135 e820_table_start
= find_e820_area(start
, max_pfn_mapped
<<PAGE_SHIFT
,
137 #else /* CONFIG_X86_64 */
139 e820_table_start
= find_e820_area(start
, end
, tables
, PAGE_SIZE
);
141 if (e820_table_start
== -1UL)
142 panic("Cannot find space for the kernel page tables");
144 e820_table_start
>>= PAGE_SHIFT
;
145 e820_table_end
= e820_table_start
;
146 e820_table_top
= e820_table_start
+ (tables
>> PAGE_SHIFT
);
148 printk(KERN_DEBUG
"kernel direct mapping tables up to %lx @ %lx-%lx\n",
149 end
, e820_table_start
<< PAGE_SHIFT
, e820_table_top
<< PAGE_SHIFT
);
155 unsigned page_size_mask
;
159 #define NR_RANGE_MR 3
160 #else /* CONFIG_X86_64 */
161 #define NR_RANGE_MR 5
164 static int __meminit
save_mr(struct map_range
*mr
, int nr_range
,
165 unsigned long start_pfn
, unsigned long end_pfn
,
166 unsigned long page_size_mask
)
168 if (start_pfn
< end_pfn
) {
169 if (nr_range
>= NR_RANGE_MR
)
170 panic("run out of range for init_memory_mapping\n");
171 mr
[nr_range
].start
= start_pfn
<<PAGE_SHIFT
;
172 mr
[nr_range
].end
= end_pfn
<<PAGE_SHIFT
;
173 mr
[nr_range
].page_size_mask
= page_size_mask
;
181 static void __init
init_gbpages(void)
183 if (direct_gbpages
&& cpu_has_gbpages
)
184 printk(KERN_INFO
"Using GB pages for direct mapping\n");
189 static inline void init_gbpages(void)
195 * Setup the direct mapping of the physical memory at PAGE_OFFSET.
196 * This runs before bootmem is initialized and gets pages directly from
197 * the physical memory. To access them they are temporarily mapped.
199 unsigned long __init_refok
init_memory_mapping(unsigned long start
,
202 unsigned long page_size_mask
= 0;
203 unsigned long start_pfn
, end_pfn
;
204 unsigned long ret
= 0;
207 struct map_range mr
[NR_RANGE_MR
];
209 int use_pse
, use_gbpages
;
211 printk(KERN_INFO
"init_memory_mapping: %016lx-%016lx\n", start
, end
);
216 #ifdef CONFIG_DEBUG_PAGEALLOC
218 * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages.
219 * This will simplify cpa(), which otherwise needs to support splitting
220 * large pages into small in interrupt context, etc.
222 use_pse
= use_gbpages
= 0;
224 use_pse
= cpu_has_pse
;
225 use_gbpages
= direct_gbpages
;
230 printk(KERN_INFO
"NX (Execute Disable) protection: active\n");
232 /* Enable PSE if available */
234 set_in_cr4(X86_CR4_PSE
);
236 /* Enable PGE if available */
238 set_in_cr4(X86_CR4_PGE
);
239 __supported_pte_mask
|= _PAGE_GLOBAL
;
243 page_size_mask
|= 1 << PG_LEVEL_1G
;
245 page_size_mask
|= 1 << PG_LEVEL_2M
;
247 memset(mr
, 0, sizeof(mr
));
250 /* head if not big page alignment ? */
251 start_pfn
= start
>> PAGE_SHIFT
;
252 pos
= start_pfn
<< PAGE_SHIFT
;
255 * Don't use a large page for the first 2/4MB of memory
256 * because there are often fixed size MTRRs in there
257 * and overlapping MTRRs into large pages can cause
261 end_pfn
= 1<<(PMD_SHIFT
- PAGE_SHIFT
);
263 end_pfn
= ((pos
+ (PMD_SIZE
- 1))>>PMD_SHIFT
)
264 << (PMD_SHIFT
- PAGE_SHIFT
);
265 #else /* CONFIG_X86_64 */
266 end_pfn
= ((pos
+ (PMD_SIZE
- 1)) >> PMD_SHIFT
)
267 << (PMD_SHIFT
- PAGE_SHIFT
);
269 if (end_pfn
> (end
>> PAGE_SHIFT
))
270 end_pfn
= end
>> PAGE_SHIFT
;
271 if (start_pfn
< end_pfn
) {
272 nr_range
= save_mr(mr
, nr_range
, start_pfn
, end_pfn
, 0);
273 pos
= end_pfn
<< PAGE_SHIFT
;
276 /* big page (2M) range */
277 start_pfn
= ((pos
+ (PMD_SIZE
- 1))>>PMD_SHIFT
)
278 << (PMD_SHIFT
- PAGE_SHIFT
);
280 end_pfn
= (end
>>PMD_SHIFT
) << (PMD_SHIFT
- PAGE_SHIFT
);
281 #else /* CONFIG_X86_64 */
282 end_pfn
= ((pos
+ (PUD_SIZE
- 1))>>PUD_SHIFT
)
283 << (PUD_SHIFT
- PAGE_SHIFT
);
284 if (end_pfn
> ((end
>>PMD_SHIFT
)<<(PMD_SHIFT
- PAGE_SHIFT
)))
285 end_pfn
= ((end
>>PMD_SHIFT
)<<(PMD_SHIFT
- PAGE_SHIFT
));
288 if (start_pfn
< end_pfn
) {
289 nr_range
= save_mr(mr
, nr_range
, start_pfn
, end_pfn
,
290 page_size_mask
& (1<<PG_LEVEL_2M
));
291 pos
= end_pfn
<< PAGE_SHIFT
;
295 /* big page (1G) range */
296 start_pfn
= ((pos
+ (PUD_SIZE
- 1))>>PUD_SHIFT
)
297 << (PUD_SHIFT
- PAGE_SHIFT
);
298 end_pfn
= (end
>> PUD_SHIFT
) << (PUD_SHIFT
- PAGE_SHIFT
);
299 if (start_pfn
< end_pfn
) {
300 nr_range
= save_mr(mr
, nr_range
, start_pfn
, end_pfn
,
302 ((1<<PG_LEVEL_2M
)|(1<<PG_LEVEL_1G
)));
303 pos
= end_pfn
<< PAGE_SHIFT
;
306 /* tail is not big page (1G) alignment */
307 start_pfn
= ((pos
+ (PMD_SIZE
- 1))>>PMD_SHIFT
)
308 << (PMD_SHIFT
- PAGE_SHIFT
);
309 end_pfn
= (end
>> PMD_SHIFT
) << (PMD_SHIFT
- PAGE_SHIFT
);
310 if (start_pfn
< end_pfn
) {
311 nr_range
= save_mr(mr
, nr_range
, start_pfn
, end_pfn
,
312 page_size_mask
& (1<<PG_LEVEL_2M
));
313 pos
= end_pfn
<< PAGE_SHIFT
;
317 /* tail is not big page (2M) alignment */
318 start_pfn
= pos
>>PAGE_SHIFT
;
319 end_pfn
= end
>>PAGE_SHIFT
;
320 nr_range
= save_mr(mr
, nr_range
, start_pfn
, end_pfn
, 0);
322 /* try to merge same page size and continuous */
323 for (i
= 0; nr_range
> 1 && i
< nr_range
- 1; i
++) {
324 unsigned long old_start
;
325 if (mr
[i
].end
!= mr
[i
+1].start
||
326 mr
[i
].page_size_mask
!= mr
[i
+1].page_size_mask
)
329 old_start
= mr
[i
].start
;
330 memmove(&mr
[i
], &mr
[i
+1],
331 (nr_range
- 1 - i
) * sizeof(struct map_range
));
332 mr
[i
--].start
= old_start
;
336 for (i
= 0; i
< nr_range
; i
++)
337 printk(KERN_DEBUG
" %010lx - %010lx page %s\n",
338 mr
[i
].start
, mr
[i
].end
,
339 (mr
[i
].page_size_mask
& (1<<PG_LEVEL_1G
))?"1G":(
340 (mr
[i
].page_size_mask
& (1<<PG_LEVEL_2M
))?"2M":"4k"));
343 * Find space for the kernel direct mapping tables.
345 * Later we should allocate these tables in the local node of the
346 * memory mapped. Unfortunately this is done currently before the
347 * nodes are discovered.
350 find_early_table_space(end
, use_pse
, use_gbpages
);
353 for (i
= 0; i
< nr_range
; i
++)
354 kernel_physical_mapping_init(mr
[i
].start
, mr
[i
].end
,
355 mr
[i
].page_size_mask
);
357 #else /* CONFIG_X86_64 */
358 for (i
= 0; i
< nr_range
; i
++)
359 ret
= kernel_physical_mapping_init(mr
[i
].start
, mr
[i
].end
,
360 mr
[i
].page_size_mask
);
364 early_ioremap_page_table_range_init();
366 load_cr3(swapper_pg_dir
);
370 if (!after_bootmem
&& !start
) {
374 mmu_cr4_features
= read_cr4();
377 * _brk_end cannot change anymore, but it and _end may be
378 * located on different 2M pages. cleanup_highmap(), however,
379 * can only consider _end when it runs, so destroy any
380 * mappings beyond _brk_end here.
382 pud
= pud_offset(pgd_offset_k(_brk_end
), _brk_end
);
383 pmd
= pmd_offset(pud
, _brk_end
- 1);
384 while (++pmd
<= pmd_offset(pud
, (unsigned long)_end
- 1))
390 if (!after_bootmem
&& e820_table_end
> e820_table_start
)
391 reserve_early(e820_table_start
<< PAGE_SHIFT
,
392 e820_table_end
<< PAGE_SHIFT
, "PGTABLE");
395 early_memtest(start
, end
);
397 return ret
>> PAGE_SHIFT
;
402 * devmem_is_allowed() checks to see if /dev/mem access to a certain address
403 * is valid. The argument is a physical page number.
406 * On x86, access has to be given to the first megabyte of ram because that area
407 * contains bios code and data regions used by X and dosemu and similar apps.
408 * Access has to be given to non-kernel-ram areas as well, these contain the PCI
409 * mmio resources as well as potential bios/acpi data regions.
411 int devmem_is_allowed(unsigned long pagenr
)
415 if (iomem_is_exclusive(pagenr
<< PAGE_SHIFT
))
417 if (!page_is_ram(pagenr
))
422 void free_init_pages(char *what
, unsigned long begin
, unsigned long end
)
424 unsigned long addr
= begin
;
430 * If debugging page accesses then do not free this memory but
431 * mark them not present - any buggy init-section access will
432 * create a kernel page fault:
434 #ifdef CONFIG_DEBUG_PAGEALLOC
435 printk(KERN_INFO
"debug: unmapping init memory %08lx..%08lx\n",
436 begin
, PAGE_ALIGN(end
));
437 set_memory_np(begin
, (end
- begin
) >> PAGE_SHIFT
);
440 * We just marked the kernel text read only above, now that
441 * we are going to free part of that, we need to make that
444 set_memory_rw(begin
, (end
- begin
) >> PAGE_SHIFT
);
446 printk(KERN_INFO
"Freeing %s: %luk freed\n", what
, (end
- begin
) >> 10);
448 for (; addr
< end
; addr
+= PAGE_SIZE
) {
449 ClearPageReserved(virt_to_page(addr
));
450 init_page_count(virt_to_page(addr
));
451 memset((void *)(addr
& ~(PAGE_SIZE
-1)),
452 POISON_FREE_INITMEM
, PAGE_SIZE
);
459 void free_initmem(void)
461 free_init_pages("unused kernel memory",
462 (unsigned long)(&__init_begin
),
463 (unsigned long)(&__init_end
));
466 #ifdef CONFIG_BLK_DEV_INITRD
467 void free_initrd_mem(unsigned long start
, unsigned long end
)
469 free_init_pages("initrd memory", start
, end
);