2 * linux/arch/arm/mm/init.c
4 * Copyright (C) 1995-2005 Russell King
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 #include <linux/kernel.h>
11 #include <linux/errno.h>
12 #include <linux/swap.h>
13 #include <linux/init.h>
14 #include <linux/bootmem.h>
15 #include <linux/mman.h>
16 #include <linux/sched/signal.h>
17 #include <linux/export.h>
18 #include <linux/nodemask.h>
19 #include <linux/initrd.h>
20 #include <linux/of_fdt.h>
21 #include <linux/highmem.h>
22 #include <linux/gfp.h>
23 #include <linux/memblock.h>
24 #include <linux/dma-contiguous.h>
25 #include <linux/sizes.h>
26 #include <linux/stop_machine.h>
29 #include <asm/mach-types.h>
30 #include <asm/memblock.h>
31 #include <asm/memory.h>
33 #include <asm/sections.h>
34 #include <asm/setup.h>
35 #include <asm/system_info.h>
37 #include <asm/fixmap.h>
39 #include <asm/mach/arch.h>
40 #include <asm/mach/map.h>
44 #ifdef CONFIG_CPU_CP15_MMU
45 unsigned long __init
__clear_cr(unsigned long mask
)
47 cr_alignment
= cr_alignment
& ~mask
;
52 static phys_addr_t phys_initrd_start __initdata
= 0;
53 static unsigned long phys_initrd_size __initdata
= 0;
55 static int __init
early_initrd(char *p
)
61 start
= memparse(p
, &endp
);
63 size
= memparse(endp
+ 1, NULL
);
65 phys_initrd_start
= start
;
66 phys_initrd_size
= size
;
70 early_param("initrd", early_initrd
);
72 static int __init
parse_tag_initrd(const struct tag
*tag
)
74 pr_warn("ATAG_INITRD is deprecated; "
75 "please update your bootloader.\n");
76 phys_initrd_start
= __virt_to_phys(tag
->u
.initrd
.start
);
77 phys_initrd_size
= tag
->u
.initrd
.size
;
81 __tagtable(ATAG_INITRD
, parse_tag_initrd
);
83 static int __init
parse_tag_initrd2(const struct tag
*tag
)
85 phys_initrd_start
= tag
->u
.initrd
.start
;
86 phys_initrd_size
= tag
->u
.initrd
.size
;
90 __tagtable(ATAG_INITRD2
, parse_tag_initrd2
);
92 static void __init
find_limits(unsigned long *min
, unsigned long *max_low
,
93 unsigned long *max_high
)
95 *max_low
= PFN_DOWN(memblock_get_current_limit());
96 *min
= PFN_UP(memblock_start_of_DRAM());
97 *max_high
= PFN_DOWN(memblock_end_of_DRAM());
100 #ifdef CONFIG_ZONE_DMA
102 phys_addr_t arm_dma_zone_size __read_mostly
;
103 EXPORT_SYMBOL(arm_dma_zone_size
);
106 * The DMA mask corresponding to the maximum bus address allocatable
107 * using GFP_DMA. The default here places no restriction on DMA
108 * allocations. This must be the smallest DMA mask in the system,
109 * so a successful GFP_DMA allocation will always satisfy this.
111 phys_addr_t arm_dma_limit
;
112 unsigned long arm_dma_pfn_limit
;
114 static void __init
arm_adjust_dma_zone(unsigned long *size
, unsigned long *hole
,
115 unsigned long dma_size
)
117 if (size
[0] <= dma_size
)
120 size
[ZONE_NORMAL
] = size
[0] - dma_size
;
121 size
[ZONE_DMA
] = dma_size
;
122 hole
[ZONE_NORMAL
] = hole
[0];
127 void __init
setup_dma_zone(const struct machine_desc
*mdesc
)
129 #ifdef CONFIG_ZONE_DMA
130 if (mdesc
->dma_zone_size
) {
131 arm_dma_zone_size
= mdesc
->dma_zone_size
;
132 arm_dma_limit
= PHYS_OFFSET
+ arm_dma_zone_size
- 1;
134 arm_dma_limit
= 0xffffffff;
135 arm_dma_pfn_limit
= arm_dma_limit
>> PAGE_SHIFT
;
139 static void __init
zone_sizes_init(unsigned long min
, unsigned long max_low
,
140 unsigned long max_high
)
142 unsigned long zone_size
[MAX_NR_ZONES
], zhole_size
[MAX_NR_ZONES
];
143 struct memblock_region
*reg
;
146 * initialise the zones.
148 memset(zone_size
, 0, sizeof(zone_size
));
151 * The memory size has already been determined. If we need
152 * to do anything fancy with the allocation of this memory
153 * to the zones, now is the time to do it.
155 zone_size
[0] = max_low
- min
;
156 #ifdef CONFIG_HIGHMEM
157 zone_size
[ZONE_HIGHMEM
] = max_high
- max_low
;
161 * Calculate the size of the holes.
162 * holes = node_size - sum(bank_sizes)
164 memcpy(zhole_size
, zone_size
, sizeof(zhole_size
));
165 for_each_memblock(memory
, reg
) {
166 unsigned long start
= memblock_region_memory_base_pfn(reg
);
167 unsigned long end
= memblock_region_memory_end_pfn(reg
);
169 if (start
< max_low
) {
170 unsigned long low_end
= min(end
, max_low
);
171 zhole_size
[0] -= low_end
- start
;
173 #ifdef CONFIG_HIGHMEM
175 unsigned long high_start
= max(start
, max_low
);
176 zhole_size
[ZONE_HIGHMEM
] -= end
- high_start
;
181 #ifdef CONFIG_ZONE_DMA
183 * Adjust the sizes according to any special requirements for
186 if (arm_dma_zone_size
)
187 arm_adjust_dma_zone(zone_size
, zhole_size
,
188 arm_dma_zone_size
>> PAGE_SHIFT
);
191 free_area_init_node(0, zone_size
, min
, zhole_size
);
194 #ifdef CONFIG_HAVE_ARCH_PFN_VALID
195 int pfn_valid(unsigned long pfn
)
197 return memblock_is_map_memory(__pfn_to_phys(pfn
));
199 EXPORT_SYMBOL(pfn_valid
);
202 #ifndef CONFIG_SPARSEMEM
203 static void __init
arm_memory_present(void)
207 static void __init
arm_memory_present(void)
209 struct memblock_region
*reg
;
211 for_each_memblock(memory
, reg
)
212 memory_present(0, memblock_region_memory_base_pfn(reg
),
213 memblock_region_memory_end_pfn(reg
));
217 static bool arm_memblock_steal_permitted
= true;
219 phys_addr_t __init
arm_memblock_steal(phys_addr_t size
, phys_addr_t align
)
223 BUG_ON(!arm_memblock_steal_permitted
);
225 phys
= memblock_alloc_base(size
, align
, MEMBLOCK_ALLOC_ANYWHERE
);
226 memblock_free(phys
, size
);
227 memblock_remove(phys
, size
);
232 static void __init
arm_initrd_init(void)
234 #ifdef CONFIG_BLK_DEV_INITRD
238 /* FDT scan will populate initrd_start */
239 if (initrd_start
&& !phys_initrd_size
) {
240 phys_initrd_start
= __virt_to_phys(initrd_start
);
241 phys_initrd_size
= initrd_end
- initrd_start
;
244 initrd_start
= initrd_end
= 0;
246 if (!phys_initrd_size
)
250 * Round the memory region to page boundaries as per free_initrd_mem()
251 * This allows us to detect whether the pages overlapping the initrd
252 * are in use, but more importantly, reserves the entire set of pages
253 * as we don't want these pages allocated for other purposes.
255 start
= round_down(phys_initrd_start
, PAGE_SIZE
);
256 size
= phys_initrd_size
+ (phys_initrd_start
- start
);
257 size
= round_up(size
, PAGE_SIZE
);
259 if (!memblock_is_region_memory(start
, size
)) {
260 pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region - disabling initrd\n",
265 if (memblock_is_region_reserved(start
, size
)) {
266 pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region - disabling initrd\n",
271 memblock_reserve(start
, size
);
273 /* Now convert initrd to virtual addresses */
274 initrd_start
= __phys_to_virt(phys_initrd_start
);
275 initrd_end
= initrd_start
+ phys_initrd_size
;
279 void __init
arm_memblock_init(const struct machine_desc
*mdesc
)
281 /* Register the kernel text, kernel data and initrd with memblock. */
282 memblock_reserve(__pa(KERNEL_START
), KERNEL_END
- KERNEL_START
);
286 arm_mm_memblock_reserve();
288 /* reserve any platform specific memblock areas */
292 early_init_fdt_reserve_self();
293 early_init_fdt_scan_reserved_mem();
295 /* reserve memory for DMA contiguous allocations */
296 dma_contiguous_reserve(arm_dma_limit
);
298 arm_memblock_steal_permitted
= false;
302 void __init
bootmem_init(void)
304 unsigned long min
, max_low
, max_high
;
306 memblock_allow_resize();
307 max_low
= max_high
= 0;
309 find_limits(&min
, &max_low
, &max_high
);
311 early_memtest((phys_addr_t
)min
<< PAGE_SHIFT
,
312 (phys_addr_t
)max_low
<< PAGE_SHIFT
);
315 * Sparsemem tries to allocate bootmem in memory_present(),
316 * so must be done after the fixed reservations
318 arm_memory_present();
321 * sparse_init() needs the bootmem allocator up and running.
326 * Now free the memory - free_area_init_node needs
327 * the sparse mem_map arrays initialized by sparse_init()
328 * for memmap_init_zone(), otherwise all PFNs are invalid.
330 zone_sizes_init(min
, max_low
, max_high
);
333 * This doesn't seem to be used by the Linux memory manager any
334 * more, but is used by ll_rw_block. If we can get rid of it, we
335 * also get rid of some of the stuff above as well.
338 max_low_pfn
= max_low
;
343 * Poison init memory with an undefined instruction (ARM) or a branch to an
344 * undefined instruction (Thumb).
346 static inline void poison_init_mem(void *s
, size_t count
)
349 for (; count
!= 0; count
-= 4)
354 free_memmap(unsigned long start_pfn
, unsigned long end_pfn
)
356 struct page
*start_pg
, *end_pg
;
357 phys_addr_t pg
, pgend
;
360 * Convert start_pfn/end_pfn to a struct page pointer.
362 start_pg
= pfn_to_page(start_pfn
- 1) + 1;
363 end_pg
= pfn_to_page(end_pfn
- 1) + 1;
366 * Convert to physical addresses, and
367 * round start upwards and end downwards.
369 pg
= PAGE_ALIGN(__pa(start_pg
));
370 pgend
= __pa(end_pg
) & PAGE_MASK
;
373 * If there are free pages between these,
374 * free the section of the memmap array.
377 memblock_free_early(pg
, pgend
- pg
);
381 * The mem_map array can get very big. Free the unused area of the memory map.
383 static void __init
free_unused_memmap(void)
385 unsigned long start
, prev_end
= 0;
386 struct memblock_region
*reg
;
389 * This relies on each bank being in address order.
390 * The banks are sorted previously in bootmem_init().
392 for_each_memblock(memory
, reg
) {
393 start
= memblock_region_memory_base_pfn(reg
);
395 #ifdef CONFIG_SPARSEMEM
397 * Take care not to free memmap entries that don't exist
398 * due to SPARSEMEM sections which aren't present.
401 ALIGN(prev_end
, PAGES_PER_SECTION
));
404 * Align down here since the VM subsystem insists that the
405 * memmap entries are valid from the bank start aligned to
406 * MAX_ORDER_NR_PAGES.
408 start
= round_down(start
, MAX_ORDER_NR_PAGES
);
411 * If we had a previous bank, and there is a space
412 * between the current bank and the previous, free it.
414 if (prev_end
&& prev_end
< start
)
415 free_memmap(prev_end
, start
);
418 * Align up here since the VM subsystem insists that the
419 * memmap entries are valid from the bank end aligned to
420 * MAX_ORDER_NR_PAGES.
422 prev_end
= ALIGN(memblock_region_memory_end_pfn(reg
),
426 #ifdef CONFIG_SPARSEMEM
427 if (!IS_ALIGNED(prev_end
, PAGES_PER_SECTION
))
428 free_memmap(prev_end
,
429 ALIGN(prev_end
, PAGES_PER_SECTION
));
433 #ifdef CONFIG_HIGHMEM
434 static inline void free_area_high(unsigned long pfn
, unsigned long end
)
436 for (; pfn
< end
; pfn
++)
437 free_highmem_page(pfn_to_page(pfn
));
441 static void __init
free_highpages(void)
443 #ifdef CONFIG_HIGHMEM
444 unsigned long max_low
= max_low_pfn
;
445 struct memblock_region
*mem
, *res
;
447 /* set highmem page free */
448 for_each_memblock(memory
, mem
) {
449 unsigned long start
= memblock_region_memory_base_pfn(mem
);
450 unsigned long end
= memblock_region_memory_end_pfn(mem
);
452 /* Ignore complete lowmem entries */
456 if (memblock_is_nomap(mem
))
459 /* Truncate partial highmem entries */
463 /* Find and exclude any reserved regions */
464 for_each_memblock(reserved
, res
) {
465 unsigned long res_start
, res_end
;
467 res_start
= memblock_region_reserved_base_pfn(res
);
468 res_end
= memblock_region_reserved_end_pfn(res
);
472 if (res_start
< start
)
478 if (res_start
!= start
)
479 free_area_high(start
, res_start
);
485 /* And now free anything which remains */
487 free_area_high(start
, end
);
493 * mem_init() marks the free areas in the mem_map and tells us how much
494 * memory is free. This is done after various parts of the system have
495 * claimed their memory after the kernel image.
497 void __init
mem_init(void)
499 #ifdef CONFIG_HAVE_TCM
500 /* These pointers are filled in on TCM detection */
505 set_max_mapnr(pfn_to_page(max_pfn
) - mem_map
);
507 /* this will put all unused low memory onto the freelists */
508 free_unused_memmap();
512 /* now that our DMA memory is actually so designated, we can free it */
513 free_reserved_area(__va(PHYS_OFFSET
), swapper_pg_dir
, -1, NULL
);
518 mem_init_print_info(NULL
);
520 #define MLK(b, t) b, t, ((t) - (b)) >> 10
521 #define MLM(b, t) b, t, ((t) - (b)) >> 20
522 #define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K)
524 pr_notice("Virtual kernel memory layout:\n"
525 " vector : 0x%08lx - 0x%08lx (%4ld kB)\n"
526 #ifdef CONFIG_HAVE_TCM
527 " DTCM : 0x%08lx - 0x%08lx (%4ld kB)\n"
528 " ITCM : 0x%08lx - 0x%08lx (%4ld kB)\n"
530 " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n"
531 " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n"
532 " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n"
533 #ifdef CONFIG_HIGHMEM
534 " pkmap : 0x%08lx - 0x%08lx (%4ld MB)\n"
536 #ifdef CONFIG_MODULES
537 " modules : 0x%08lx - 0x%08lx (%4ld MB)\n"
539 " .text : 0x%p" " - 0x%p" " (%4td kB)\n"
540 " .init : 0x%p" " - 0x%p" " (%4td kB)\n"
541 " .data : 0x%p" " - 0x%p" " (%4td kB)\n"
542 " .bss : 0x%p" " - 0x%p" " (%4td kB)\n",
544 MLK(VECTORS_BASE
, VECTORS_BASE
+ PAGE_SIZE
),
545 #ifdef CONFIG_HAVE_TCM
546 MLK(DTCM_OFFSET
, (unsigned long) dtcm_end
),
547 MLK(ITCM_OFFSET
, (unsigned long) itcm_end
),
549 MLK(FIXADDR_START
, FIXADDR_END
),
550 MLM(VMALLOC_START
, VMALLOC_END
),
551 MLM(PAGE_OFFSET
, (unsigned long)high_memory
),
552 #ifdef CONFIG_HIGHMEM
553 MLM(PKMAP_BASE
, (PKMAP_BASE
) + (LAST_PKMAP
) *
556 #ifdef CONFIG_MODULES
557 MLM(MODULES_VADDR
, MODULES_END
),
560 MLK_ROUNDUP(_text
, _etext
),
561 MLK_ROUNDUP(__init_begin
, __init_end
),
562 MLK_ROUNDUP(_sdata
, _edata
),
563 MLK_ROUNDUP(__bss_start
, __bss_stop
));
570 * Check boundaries twice: Some fundamental inconsistencies can
571 * be detected at build time already.
574 BUILD_BUG_ON(TASK_SIZE
> MODULES_VADDR
);
575 BUG_ON(TASK_SIZE
> MODULES_VADDR
);
578 #ifdef CONFIG_HIGHMEM
579 BUILD_BUG_ON(PKMAP_BASE
+ LAST_PKMAP
* PAGE_SIZE
> PAGE_OFFSET
);
580 BUG_ON(PKMAP_BASE
+ LAST_PKMAP
* PAGE_SIZE
> PAGE_OFFSET
);
583 if (PAGE_SIZE
>= 16384 && get_num_physpages() <= 128) {
584 extern int sysctl_overcommit_memory
;
586 * On a machine this small we won't get
587 * anywhere without overcommit, so turn
590 sysctl_overcommit_memory
= OVERCOMMIT_ALWAYS
;
594 #ifdef CONFIG_STRICT_KERNEL_RWX
595 struct section_perm
{
604 /* First section-aligned location at or after __start_rodata. */
605 extern char __start_rodata_section_aligned
[];
607 static struct section_perm nx_perms
[] = {
608 /* Make pages tables, etc before _stext RW (set NX). */
610 .name
= "pre-text NX",
611 .start
= PAGE_OFFSET
,
612 .end
= (unsigned long)_stext
,
613 .mask
= ~PMD_SECT_XN
,
616 /* Make init RW (set NX). */
619 .start
= (unsigned long)__init_begin
,
620 .end
= (unsigned long)_sdata
,
621 .mask
= ~PMD_SECT_XN
,
624 /* Make rodata NX (set RO in ro_perms below). */
627 .start
= (unsigned long)__start_rodata_section_aligned
,
628 .end
= (unsigned long)__init_begin
,
629 .mask
= ~PMD_SECT_XN
,
634 static struct section_perm ro_perms
[] = {
635 /* Make kernel code and rodata RX (set RO). */
637 .name
= "text/rodata RO",
638 .start
= (unsigned long)_stext
,
639 .end
= (unsigned long)__init_begin
,
640 #ifdef CONFIG_ARM_LPAE
641 .mask
= ~L_PMD_SECT_RDONLY
,
642 .prot
= L_PMD_SECT_RDONLY
,
644 .mask
= ~(PMD_SECT_APX
| PMD_SECT_AP_WRITE
),
645 .prot
= PMD_SECT_APX
| PMD_SECT_AP_WRITE
,
646 .clear
= PMD_SECT_AP_WRITE
,
652 * Updates section permissions only for the current mm (sections are
653 * copied into each mm). During startup, this is the init_mm. Is only
654 * safe to be called with preemption disabled, as under stop_machine().
656 static inline void section_update(unsigned long addr
, pmdval_t mask
,
657 pmdval_t prot
, struct mm_struct
*mm
)
661 pmd
= pmd_offset(pud_offset(pgd_offset(mm
, addr
), addr
), addr
);
663 #ifdef CONFIG_ARM_LPAE
664 pmd
[0] = __pmd((pmd_val(pmd
[0]) & mask
) | prot
);
666 if (addr
& SECTION_SIZE
)
667 pmd
[1] = __pmd((pmd_val(pmd
[1]) & mask
) | prot
);
669 pmd
[0] = __pmd((pmd_val(pmd
[0]) & mask
) | prot
);
671 flush_pmd_entry(pmd
);
672 local_flush_tlb_kernel_range(addr
, addr
+ SECTION_SIZE
);
675 /* Make sure extended page tables are in use. */
676 static inline bool arch_has_strict_perms(void)
678 if (cpu_architecture() < CPU_ARCH_ARMv6
)
681 return !!(get_cr() & CR_XP
);
684 void set_section_perms(struct section_perm
*perms
, int n
, bool set
,
685 struct mm_struct
*mm
)
690 if (!arch_has_strict_perms())
693 for (i
= 0; i
< n
; i
++) {
694 if (!IS_ALIGNED(perms
[i
].start
, SECTION_SIZE
) ||
695 !IS_ALIGNED(perms
[i
].end
, SECTION_SIZE
)) {
696 pr_err("BUG: %s section %lx-%lx not aligned to %lx\n",
697 perms
[i
].name
, perms
[i
].start
, perms
[i
].end
,
702 for (addr
= perms
[i
].start
;
704 addr
+= SECTION_SIZE
)
705 section_update(addr
, perms
[i
].mask
,
706 set
? perms
[i
].prot
: perms
[i
].clear
, mm
);
711 static void update_sections_early(struct section_perm perms
[], int n
)
713 struct task_struct
*t
, *s
;
715 read_lock(&tasklist_lock
);
716 for_each_process(t
) {
717 if (t
->flags
& PF_KTHREAD
)
719 for_each_thread(t
, s
)
720 set_section_perms(perms
, n
, true, s
->mm
);
722 read_unlock(&tasklist_lock
);
723 set_section_perms(perms
, n
, true, current
->active_mm
);
724 set_section_perms(perms
, n
, true, &init_mm
);
727 int __fix_kernmem_perms(void *unused
)
729 update_sections_early(nx_perms
, ARRAY_SIZE(nx_perms
));
733 void fix_kernmem_perms(void)
735 stop_machine(__fix_kernmem_perms
, NULL
, NULL
);
738 int __mark_rodata_ro(void *unused
)
740 update_sections_early(ro_perms
, ARRAY_SIZE(ro_perms
));
744 void mark_rodata_ro(void)
746 stop_machine(__mark_rodata_ro
, NULL
, NULL
);
749 void set_kernel_text_rw(void)
751 set_section_perms(ro_perms
, ARRAY_SIZE(ro_perms
), false,
755 void set_kernel_text_ro(void)
757 set_section_perms(ro_perms
, ARRAY_SIZE(ro_perms
), true,
762 static inline void fix_kernmem_perms(void) { }
763 #endif /* CONFIG_STRICT_KERNEL_RWX */
765 void free_tcmmem(void)
767 #ifdef CONFIG_HAVE_TCM
768 extern char __tcm_start
, __tcm_end
;
770 poison_init_mem(&__tcm_start
, &__tcm_end
- &__tcm_start
);
771 free_reserved_area(&__tcm_start
, &__tcm_end
, -1, "TCM link");
775 void free_initmem(void)
780 poison_init_mem(__init_begin
, __init_end
- __init_begin
);
781 if (!machine_is_integrator() && !machine_is_cintegrator())
782 free_initmem_default(-1);
785 #ifdef CONFIG_BLK_DEV_INITRD
787 static int keep_initrd
;
789 void free_initrd_mem(unsigned long start
, unsigned long end
)
792 if (start
== initrd_start
)
793 start
= round_down(start
, PAGE_SIZE
);
794 if (end
== initrd_end
)
795 end
= round_up(end
, PAGE_SIZE
);
797 poison_init_mem((void *)start
, PAGE_ALIGN(end
) - start
);
798 free_reserved_area((void *)start
, (void *)end
, -1, "initrd");
802 static int __init
keepinitrd_setup(char *__unused
)
808 __setup("keepinitrd", keepinitrd_setup
);