2 * Based on arch/arm/mm/mmu.c
4 * Copyright (C) 1995-2005 Russell King
5 * Copyright (C) 2012 ARM Ltd.
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 #include <linux/cache.h>
21 #include <linux/export.h>
22 #include <linux/kernel.h>
23 #include <linux/errno.h>
24 #include <linux/init.h>
25 #include <linux/ioport.h>
26 #include <linux/kexec.h>
27 #include <linux/libfdt.h>
28 #include <linux/mman.h>
29 #include <linux/nodemask.h>
30 #include <linux/memblock.h>
34 #include <linux/vmalloc.h>
36 #include <asm/barrier.h>
37 #include <asm/cputype.h>
38 #include <asm/fixmap.h>
39 #include <asm/kasan.h>
40 #include <asm/kernel-pgtable.h>
41 #include <asm/sections.h>
42 #include <asm/setup.h>
43 #include <linux/sizes.h>
45 #include <asm/mmu_context.h>
46 #include <asm/ptdump.h>
47 #include <asm/tlbflush.h>
49 #define NO_BLOCK_MAPPINGS BIT(0)
50 #define NO_CONT_MAPPINGS BIT(1)
52 u64 idmap_t0sz
= TCR_T0SZ(VA_BITS
);
53 u64 idmap_ptrs_per_pgd
= PTRS_PER_PGD
;
54 u64 vabits_user __ro_after_init
;
55 EXPORT_SYMBOL(vabits_user
);
57 u64 kimage_voffset __ro_after_init
;
58 EXPORT_SYMBOL(kimage_voffset
);
61 * Empty_zero_page is a special page that is used for zero-initialized data
64 unsigned long empty_zero_page
[PAGE_SIZE
/ sizeof(unsigned long)] __page_aligned_bss
;
65 EXPORT_SYMBOL(empty_zero_page
);
67 static pte_t bm_pte
[PTRS_PER_PTE
] __page_aligned_bss
;
68 static pmd_t bm_pmd
[PTRS_PER_PMD
] __page_aligned_bss __maybe_unused
;
69 static pud_t bm_pud
[PTRS_PER_PUD
] __page_aligned_bss __maybe_unused
;
71 static DEFINE_SPINLOCK(swapper_pgdir_lock
);
73 void set_swapper_pgd(pgd_t
*pgdp
, pgd_t pgd
)
77 spin_lock(&swapper_pgdir_lock
);
78 fixmap_pgdp
= pgd_set_fixmap(__pa_symbol(pgdp
));
79 WRITE_ONCE(*fixmap_pgdp
, pgd
);
81 * We need dsb(ishst) here to ensure the page-table-walker sees
82 * our new entry before set_p?d() returns. The fixmap's
83 * flush_tlb_kernel_range() via clear_fixmap() does this for us.
86 spin_unlock(&swapper_pgdir_lock
);
89 pgprot_t
phys_mem_access_prot(struct file
*file
, unsigned long pfn
,
90 unsigned long size
, pgprot_t vma_prot
)
93 return pgprot_noncached(vma_prot
);
94 else if (file
->f_flags
& O_SYNC
)
95 return pgprot_writecombine(vma_prot
);
98 EXPORT_SYMBOL(phys_mem_access_prot
);
100 static phys_addr_t __init
early_pgtable_alloc(int shift
)
105 phys
= memblock_phys_alloc(PAGE_SIZE
, PAGE_SIZE
);
107 panic("Failed to allocate page table page\n");
110 * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE
111 * slot will be free, so we can (ab)use the FIX_PTE slot to initialise
112 * any level of table.
114 ptr
= pte_set_fixmap(phys
);
116 memset(ptr
, 0, PAGE_SIZE
);
119 * Implicit barriers also ensure the zeroed page is visible to the page
127 static bool pgattr_change_is_safe(u64 old
, u64
new)
130 * The following mapping attributes may be updated in live
131 * kernel mappings without the need for break-before-make.
133 static const pteval_t mask
= PTE_PXN
| PTE_RDONLY
| PTE_WRITE
| PTE_NG
;
135 /* creating or taking down mappings is always safe */
136 if (old
== 0 || new == 0)
139 /* live contiguous mappings may not be manipulated at all */
140 if ((old
| new) & PTE_CONT
)
143 /* Transitioning from Non-Global to Global is unsafe */
144 if (old
& ~new & PTE_NG
)
147 return ((old
^ new) & ~mask
) == 0;
150 static void init_pte(pmd_t
*pmdp
, unsigned long addr
, unsigned long end
,
151 phys_addr_t phys
, pgprot_t prot
)
155 ptep
= pte_set_fixmap_offset(pmdp
, addr
);
157 pte_t old_pte
= READ_ONCE(*ptep
);
159 set_pte(ptep
, pfn_pte(__phys_to_pfn(phys
), prot
));
162 * After the PTE entry has been populated once, we
163 * only allow updates to the permission attributes.
165 BUG_ON(!pgattr_change_is_safe(pte_val(old_pte
),
166 READ_ONCE(pte_val(*ptep
))));
169 } while (ptep
++, addr
+= PAGE_SIZE
, addr
!= end
);
174 static void alloc_init_cont_pte(pmd_t
*pmdp
, unsigned long addr
,
175 unsigned long end
, phys_addr_t phys
,
177 phys_addr_t (*pgtable_alloc
)(int),
181 pmd_t pmd
= READ_ONCE(*pmdp
);
183 BUG_ON(pmd_sect(pmd
));
185 phys_addr_t pte_phys
;
186 BUG_ON(!pgtable_alloc
);
187 pte_phys
= pgtable_alloc(PAGE_SHIFT
);
188 __pmd_populate(pmdp
, pte_phys
, PMD_TYPE_TABLE
);
189 pmd
= READ_ONCE(*pmdp
);
191 BUG_ON(pmd_bad(pmd
));
194 pgprot_t __prot
= prot
;
196 next
= pte_cont_addr_end(addr
, end
);
198 /* use a contiguous mapping if the range is suitably aligned */
199 if ((((addr
| next
| phys
) & ~CONT_PTE_MASK
) == 0) &&
200 (flags
& NO_CONT_MAPPINGS
) == 0)
201 __prot
= __pgprot(pgprot_val(prot
) | PTE_CONT
);
203 init_pte(pmdp
, addr
, next
, phys
, __prot
);
206 } while (addr
= next
, addr
!= end
);
209 static void init_pmd(pud_t
*pudp
, unsigned long addr
, unsigned long end
,
210 phys_addr_t phys
, pgprot_t prot
,
211 phys_addr_t (*pgtable_alloc
)(int), int flags
)
216 pmdp
= pmd_set_fixmap_offset(pudp
, addr
);
218 pmd_t old_pmd
= READ_ONCE(*pmdp
);
220 next
= pmd_addr_end(addr
, end
);
222 /* try section mapping first */
223 if (((addr
| next
| phys
) & ~SECTION_MASK
) == 0 &&
224 (flags
& NO_BLOCK_MAPPINGS
) == 0) {
225 pmd_set_huge(pmdp
, phys
, prot
);
228 * After the PMD entry has been populated once, we
229 * only allow updates to the permission attributes.
231 BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd
),
232 READ_ONCE(pmd_val(*pmdp
))));
234 alloc_init_cont_pte(pmdp
, addr
, next
, phys
, prot
,
235 pgtable_alloc
, flags
);
237 BUG_ON(pmd_val(old_pmd
) != 0 &&
238 pmd_val(old_pmd
) != READ_ONCE(pmd_val(*pmdp
)));
241 } while (pmdp
++, addr
= next
, addr
!= end
);
246 static void alloc_init_cont_pmd(pud_t
*pudp
, unsigned long addr
,
247 unsigned long end
, phys_addr_t phys
,
249 phys_addr_t (*pgtable_alloc
)(int), int flags
)
252 pud_t pud
= READ_ONCE(*pudp
);
255 * Check for initial section mappings in the pgd/pud.
257 BUG_ON(pud_sect(pud
));
259 phys_addr_t pmd_phys
;
260 BUG_ON(!pgtable_alloc
);
261 pmd_phys
= pgtable_alloc(PMD_SHIFT
);
262 __pud_populate(pudp
, pmd_phys
, PUD_TYPE_TABLE
);
263 pud
= READ_ONCE(*pudp
);
265 BUG_ON(pud_bad(pud
));
268 pgprot_t __prot
= prot
;
270 next
= pmd_cont_addr_end(addr
, end
);
272 /* use a contiguous mapping if the range is suitably aligned */
273 if ((((addr
| next
| phys
) & ~CONT_PMD_MASK
) == 0) &&
274 (flags
& NO_CONT_MAPPINGS
) == 0)
275 __prot
= __pgprot(pgprot_val(prot
) | PTE_CONT
);
277 init_pmd(pudp
, addr
, next
, phys
, __prot
, pgtable_alloc
, flags
);
280 } while (addr
= next
, addr
!= end
);
283 static inline bool use_1G_block(unsigned long addr
, unsigned long next
,
286 if (PAGE_SHIFT
!= 12)
289 if (((addr
| next
| phys
) & ~PUD_MASK
) != 0)
295 static void alloc_init_pud(pgd_t
*pgdp
, unsigned long addr
, unsigned long end
,
296 phys_addr_t phys
, pgprot_t prot
,
297 phys_addr_t (*pgtable_alloc
)(int),
302 pgd_t pgd
= READ_ONCE(*pgdp
);
305 phys_addr_t pud_phys
;
306 BUG_ON(!pgtable_alloc
);
307 pud_phys
= pgtable_alloc(PUD_SHIFT
);
308 __pgd_populate(pgdp
, pud_phys
, PUD_TYPE_TABLE
);
309 pgd
= READ_ONCE(*pgdp
);
311 BUG_ON(pgd_bad(pgd
));
313 pudp
= pud_set_fixmap_offset(pgdp
, addr
);
315 pud_t old_pud
= READ_ONCE(*pudp
);
317 next
= pud_addr_end(addr
, end
);
320 * For 4K granule only, attempt to put down a 1GB block
322 if (use_1G_block(addr
, next
, phys
) &&
323 (flags
& NO_BLOCK_MAPPINGS
) == 0) {
324 pud_set_huge(pudp
, phys
, prot
);
327 * After the PUD entry has been populated once, we
328 * only allow updates to the permission attributes.
330 BUG_ON(!pgattr_change_is_safe(pud_val(old_pud
),
331 READ_ONCE(pud_val(*pudp
))));
333 alloc_init_cont_pmd(pudp
, addr
, next
, phys
, prot
,
334 pgtable_alloc
, flags
);
336 BUG_ON(pud_val(old_pud
) != 0 &&
337 pud_val(old_pud
) != READ_ONCE(pud_val(*pudp
)));
340 } while (pudp
++, addr
= next
, addr
!= end
);
345 static void __create_pgd_mapping(pgd_t
*pgdir
, phys_addr_t phys
,
346 unsigned long virt
, phys_addr_t size
,
348 phys_addr_t (*pgtable_alloc
)(int),
351 unsigned long addr
, length
, end
, next
;
352 pgd_t
*pgdp
= pgd_offset_raw(pgdir
, virt
);
355 * If the virtual and physical address don't have the same offset
356 * within a page, we cannot map the region as the caller expects.
358 if (WARN_ON((phys
^ virt
) & ~PAGE_MASK
))
362 addr
= virt
& PAGE_MASK
;
363 length
= PAGE_ALIGN(size
+ (virt
& ~PAGE_MASK
));
367 next
= pgd_addr_end(addr
, end
);
368 alloc_init_pud(pgdp
, addr
, next
, phys
, prot
, pgtable_alloc
,
371 } while (pgdp
++, addr
= next
, addr
!= end
);
374 static phys_addr_t
__pgd_pgtable_alloc(int shift
)
376 void *ptr
= (void *)__get_free_page(PGALLOC_GFP
);
379 /* Ensure the zeroed page is visible to the page table walker */
384 static phys_addr_t
pgd_pgtable_alloc(int shift
)
386 phys_addr_t pa
= __pgd_pgtable_alloc(shift
);
389 * Call proper page table ctor in case later we need to
390 * call core mm functions like apply_to_page_range() on
391 * this pre-allocated page table.
393 * We don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK if pmd is
394 * folded, and if so pgtable_pmd_page_ctor() becomes nop.
396 if (shift
== PAGE_SHIFT
)
397 BUG_ON(!pgtable_page_ctor(phys_to_page(pa
)));
398 else if (shift
== PMD_SHIFT
)
399 BUG_ON(!pgtable_pmd_page_ctor(phys_to_page(pa
)));
405 * This function can only be used to modify existing table entries,
406 * without allocating new levels of table. Note that this permits the
407 * creation of new section or page entries.
409 static void __init
create_mapping_noalloc(phys_addr_t phys
, unsigned long virt
,
410 phys_addr_t size
, pgprot_t prot
)
412 if (virt
< VMALLOC_START
) {
413 pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
417 __create_pgd_mapping(init_mm
.pgd
, phys
, virt
, size
, prot
, NULL
,
421 void __init
create_pgd_mapping(struct mm_struct
*mm
, phys_addr_t phys
,
422 unsigned long virt
, phys_addr_t size
,
423 pgprot_t prot
, bool page_mappings_only
)
427 BUG_ON(mm
== &init_mm
);
429 if (page_mappings_only
)
430 flags
= NO_BLOCK_MAPPINGS
| NO_CONT_MAPPINGS
;
432 __create_pgd_mapping(mm
->pgd
, phys
, virt
, size
, prot
,
433 pgd_pgtable_alloc
, flags
);
436 static void update_mapping_prot(phys_addr_t phys
, unsigned long virt
,
437 phys_addr_t size
, pgprot_t prot
)
439 if (virt
< VMALLOC_START
) {
440 pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n",
445 __create_pgd_mapping(init_mm
.pgd
, phys
, virt
, size
, prot
, NULL
,
448 /* flush the TLBs after updating live kernel mappings */
449 flush_tlb_kernel_range(virt
, virt
+ size
);
452 static void __init
__map_memblock(pgd_t
*pgdp
, phys_addr_t start
,
453 phys_addr_t end
, pgprot_t prot
, int flags
)
455 __create_pgd_mapping(pgdp
, start
, __phys_to_virt(start
), end
- start
,
456 prot
, early_pgtable_alloc
, flags
);
459 void __init
mark_linear_text_alias_ro(void)
462 * Remove the write permissions from the linear alias of .text/.rodata
464 update_mapping_prot(__pa_symbol(_text
), (unsigned long)lm_alias(_text
),
465 (unsigned long)__init_begin
- (unsigned long)_text
,
469 static void __init
map_mem(pgd_t
*pgdp
)
471 phys_addr_t kernel_start
= __pa_symbol(_text
);
472 phys_addr_t kernel_end
= __pa_symbol(__init_begin
);
473 struct memblock_region
*reg
;
476 if (rodata_full
|| debug_pagealloc_enabled())
477 flags
= NO_BLOCK_MAPPINGS
| NO_CONT_MAPPINGS
;
480 * Take care not to create a writable alias for the
481 * read-only text and rodata sections of the kernel image.
482 * So temporarily mark them as NOMAP to skip mappings in
483 * the following for-loop
485 memblock_mark_nomap(kernel_start
, kernel_end
- kernel_start
);
486 #ifdef CONFIG_KEXEC_CORE
488 memblock_mark_nomap(crashk_res
.start
,
489 resource_size(&crashk_res
));
492 /* map all the memory banks */
493 for_each_memblock(memory
, reg
) {
494 phys_addr_t start
= reg
->base
;
495 phys_addr_t end
= start
+ reg
->size
;
499 if (memblock_is_nomap(reg
))
502 __map_memblock(pgdp
, start
, end
, PAGE_KERNEL
, flags
);
506 * Map the linear alias of the [_text, __init_begin) interval
507 * as non-executable now, and remove the write permission in
508 * mark_linear_text_alias_ro() below (which will be called after
509 * alternative patching has completed). This makes the contents
510 * of the region accessible to subsystems such as hibernate,
511 * but protects it from inadvertent modification or execution.
512 * Note that contiguous mappings cannot be remapped in this way,
513 * so we should avoid them here.
515 __map_memblock(pgdp
, kernel_start
, kernel_end
,
516 PAGE_KERNEL
, NO_CONT_MAPPINGS
);
517 memblock_clear_nomap(kernel_start
, kernel_end
- kernel_start
);
519 #ifdef CONFIG_KEXEC_CORE
521 * Use page-level mappings here so that we can shrink the region
522 * in page granularity and put back unused memory to buddy system
523 * through /sys/kernel/kexec_crash_size interface.
525 if (crashk_res
.end
) {
526 __map_memblock(pgdp
, crashk_res
.start
, crashk_res
.end
+ 1,
528 NO_BLOCK_MAPPINGS
| NO_CONT_MAPPINGS
);
529 memblock_clear_nomap(crashk_res
.start
,
530 resource_size(&crashk_res
));
535 void mark_rodata_ro(void)
537 unsigned long section_size
;
540 * mark .rodata as read only. Use __init_begin rather than __end_rodata
541 * to cover NOTES and EXCEPTION_TABLE.
543 section_size
= (unsigned long)__init_begin
- (unsigned long)__start_rodata
;
544 update_mapping_prot(__pa_symbol(__start_rodata
), (unsigned long)__start_rodata
,
545 section_size
, PAGE_KERNEL_RO
);
550 static void __init
map_kernel_segment(pgd_t
*pgdp
, void *va_start
, void *va_end
,
551 pgprot_t prot
, struct vm_struct
*vma
,
552 int flags
, unsigned long vm_flags
)
554 phys_addr_t pa_start
= __pa_symbol(va_start
);
555 unsigned long size
= va_end
- va_start
;
557 BUG_ON(!PAGE_ALIGNED(pa_start
));
558 BUG_ON(!PAGE_ALIGNED(size
));
560 __create_pgd_mapping(pgdp
, pa_start
, (unsigned long)va_start
, size
, prot
,
561 early_pgtable_alloc
, flags
);
563 if (!(vm_flags
& VM_NO_GUARD
))
566 vma
->addr
= va_start
;
567 vma
->phys_addr
= pa_start
;
569 vma
->flags
= VM_MAP
| vm_flags
;
570 vma
->caller
= __builtin_return_address(0);
572 vm_area_add_early(vma
);
575 static int __init
parse_rodata(char *arg
)
577 int ret
= strtobool(arg
, &rodata_enabled
);
583 /* permit 'full' in addition to boolean options */
584 if (strcmp(arg
, "full"))
587 rodata_enabled
= true;
591 early_param("rodata", parse_rodata
);
593 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
594 static int __init
map_entry_trampoline(void)
596 pgprot_t prot
= rodata_enabled
? PAGE_KERNEL_ROX
: PAGE_KERNEL_EXEC
;
597 phys_addr_t pa_start
= __pa_symbol(__entry_tramp_text_start
);
599 /* The trampoline is always mapped and can therefore be global */
600 pgprot_val(prot
) &= ~PTE_NG
;
602 /* Map only the text into the trampoline page table */
603 memset(tramp_pg_dir
, 0, PGD_SIZE
);
604 __create_pgd_mapping(tramp_pg_dir
, pa_start
, TRAMP_VALIAS
, PAGE_SIZE
,
605 prot
, __pgd_pgtable_alloc
, 0);
607 /* Map both the text and data into the kernel page table */
608 __set_fixmap(FIX_ENTRY_TRAMP_TEXT
, pa_start
, prot
);
609 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE
)) {
610 extern char __entry_tramp_data_start
[];
612 __set_fixmap(FIX_ENTRY_TRAMP_DATA
,
613 __pa_symbol(__entry_tramp_data_start
),
619 core_initcall(map_entry_trampoline
);
623 * Create fine-grained mappings for the kernel.
625 static void __init
map_kernel(pgd_t
*pgdp
)
627 static struct vm_struct vmlinux_text
, vmlinux_rodata
, vmlinux_inittext
,
628 vmlinux_initdata
, vmlinux_data
;
631 * External debuggers may need to write directly to the text
632 * mapping to install SW breakpoints. Allow this (only) when
633 * explicitly requested with rodata=off.
635 pgprot_t text_prot
= rodata_enabled
? PAGE_KERNEL_ROX
: PAGE_KERNEL_EXEC
;
638 * Only rodata will be remapped with different permissions later on,
639 * all other segments are allowed to use contiguous mappings.
641 map_kernel_segment(pgdp
, _text
, _etext
, text_prot
, &vmlinux_text
, 0,
643 map_kernel_segment(pgdp
, __start_rodata
, __inittext_begin
, PAGE_KERNEL
,
644 &vmlinux_rodata
, NO_CONT_MAPPINGS
, VM_NO_GUARD
);
645 map_kernel_segment(pgdp
, __inittext_begin
, __inittext_end
, text_prot
,
646 &vmlinux_inittext
, 0, VM_NO_GUARD
);
647 map_kernel_segment(pgdp
, __initdata_begin
, __initdata_end
, PAGE_KERNEL
,
648 &vmlinux_initdata
, 0, VM_NO_GUARD
);
649 map_kernel_segment(pgdp
, _data
, _end
, PAGE_KERNEL
, &vmlinux_data
, 0, 0);
651 if (!READ_ONCE(pgd_val(*pgd_offset_raw(pgdp
, FIXADDR_START
)))) {
653 * The fixmap falls in a separate pgd to the kernel, and doesn't
654 * live in the carveout for the swapper_pg_dir. We can simply
655 * re-use the existing dir for the fixmap.
657 set_pgd(pgd_offset_raw(pgdp
, FIXADDR_START
),
658 READ_ONCE(*pgd_offset_k(FIXADDR_START
)));
659 } else if (CONFIG_PGTABLE_LEVELS
> 3) {
661 * The fixmap shares its top level pgd entry with the kernel
662 * mapping. This can really only occur when we are running
663 * with 16k/4 levels, so we can simply reuse the pud level
666 BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES
));
667 pud_populate(&init_mm
,
668 pud_set_fixmap_offset(pgdp
, FIXADDR_START
),
675 kasan_copy_shadow(pgdp
);
678 void __init
paging_init(void)
680 pgd_t
*pgdp
= pgd_set_fixmap(__pa_symbol(swapper_pg_dir
));
687 cpu_replace_ttbr1(lm_alias(swapper_pg_dir
));
688 init_mm
.pgd
= swapper_pg_dir
;
690 memblock_free(__pa_symbol(init_pg_dir
),
691 __pa_symbol(init_pg_end
) - __pa_symbol(init_pg_dir
));
693 memblock_allow_resize();
697 * Check whether a kernel address is valid (derived from arch/x86/).
699 int kern_addr_valid(unsigned long addr
)
706 if ((((long)addr
) >> VA_BITS
) != -1UL)
709 pgdp
= pgd_offset_k(addr
);
710 if (pgd_none(READ_ONCE(*pgdp
)))
713 pudp
= pud_offset(pgdp
, addr
);
714 pud
= READ_ONCE(*pudp
);
719 return pfn_valid(pud_pfn(pud
));
721 pmdp
= pmd_offset(pudp
, addr
);
722 pmd
= READ_ONCE(*pmdp
);
727 return pfn_valid(pmd_pfn(pmd
));
729 ptep
= pte_offset_kernel(pmdp
, addr
);
730 pte
= READ_ONCE(*ptep
);
734 return pfn_valid(pte_pfn(pte
));
736 #ifdef CONFIG_SPARSEMEM_VMEMMAP
737 #if !ARM64_SWAPPER_USES_SECTION_MAPS
738 int __meminit
vmemmap_populate(unsigned long start
, unsigned long end
, int node
,
739 struct vmem_altmap
*altmap
)
741 return vmemmap_populate_basepages(start
, end
, node
);
743 #else /* !ARM64_SWAPPER_USES_SECTION_MAPS */
744 int __meminit
vmemmap_populate(unsigned long start
, unsigned long end
, int node
,
745 struct vmem_altmap
*altmap
)
747 unsigned long addr
= start
;
754 next
= pmd_addr_end(addr
, end
);
756 pgdp
= vmemmap_pgd_populate(addr
, node
);
760 pudp
= vmemmap_pud_populate(pgdp
, addr
, node
);
764 pmdp
= pmd_offset(pudp
, addr
);
765 if (pmd_none(READ_ONCE(*pmdp
))) {
768 p
= vmemmap_alloc_block_buf(PMD_SIZE
, node
);
772 pmd_set_huge(pmdp
, __pa(p
), __pgprot(PROT_SECT_NORMAL
));
774 vmemmap_verify((pte_t
*)pmdp
, node
, addr
, next
);
775 } while (addr
= next
, addr
!= end
);
779 #endif /* CONFIG_ARM64_64K_PAGES */
780 void vmemmap_free(unsigned long start
, unsigned long end
,
781 struct vmem_altmap
*altmap
)
784 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
786 static inline pud_t
* fixmap_pud(unsigned long addr
)
788 pgd_t
*pgdp
= pgd_offset_k(addr
);
789 pgd_t pgd
= READ_ONCE(*pgdp
);
791 BUG_ON(pgd_none(pgd
) || pgd_bad(pgd
));
793 return pud_offset_kimg(pgdp
, addr
);
796 static inline pmd_t
* fixmap_pmd(unsigned long addr
)
798 pud_t
*pudp
= fixmap_pud(addr
);
799 pud_t pud
= READ_ONCE(*pudp
);
801 BUG_ON(pud_none(pud
) || pud_bad(pud
));
803 return pmd_offset_kimg(pudp
, addr
);
806 static inline pte_t
* fixmap_pte(unsigned long addr
)
808 return &bm_pte
[pte_index(addr
)];
812 * The p*d_populate functions call virt_to_phys implicitly so they can't be used
813 * directly on kernel symbols (bm_p*d). This function is called too early to use
814 * lm_alias so __p*d_populate functions must be used to populate with the
815 * physical address from __pa_symbol.
817 void __init
early_fixmap_init(void)
822 unsigned long addr
= FIXADDR_START
;
824 pgdp
= pgd_offset_k(addr
);
825 pgd
= READ_ONCE(*pgdp
);
826 if (CONFIG_PGTABLE_LEVELS
> 3 &&
827 !(pgd_none(pgd
) || pgd_page_paddr(pgd
) == __pa_symbol(bm_pud
))) {
829 * We only end up here if the kernel mapping and the fixmap
830 * share the top level pgd entry, which should only happen on
831 * 16k/4 levels configurations.
833 BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES
));
834 pudp
= pud_offset_kimg(pgdp
, addr
);
837 __pgd_populate(pgdp
, __pa_symbol(bm_pud
), PUD_TYPE_TABLE
);
838 pudp
= fixmap_pud(addr
);
840 if (pud_none(READ_ONCE(*pudp
)))
841 __pud_populate(pudp
, __pa_symbol(bm_pmd
), PMD_TYPE_TABLE
);
842 pmdp
= fixmap_pmd(addr
);
843 __pmd_populate(pmdp
, __pa_symbol(bm_pte
), PMD_TYPE_TABLE
);
846 * The boot-ioremap range spans multiple pmds, for which
847 * we are not prepared:
849 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN
) >> PMD_SHIFT
)
850 != (__fix_to_virt(FIX_BTMAP_END
) >> PMD_SHIFT
));
852 if ((pmdp
!= fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN
)))
853 || pmdp
!= fixmap_pmd(fix_to_virt(FIX_BTMAP_END
))) {
855 pr_warn("pmdp %p != %p, %p\n",
856 pmdp
, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN
)),
857 fixmap_pmd(fix_to_virt(FIX_BTMAP_END
)));
858 pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
859 fix_to_virt(FIX_BTMAP_BEGIN
));
860 pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n",
861 fix_to_virt(FIX_BTMAP_END
));
863 pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END
);
864 pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN
);
869 * Unusually, this is also called in IRQ context (ghes_iounmap_irq) so if we
870 * ever need to use IPIs for TLB broadcasting, then we're in trouble here.
872 void __set_fixmap(enum fixed_addresses idx
,
873 phys_addr_t phys
, pgprot_t flags
)
875 unsigned long addr
= __fix_to_virt(idx
);
878 BUG_ON(idx
<= FIX_HOLE
|| idx
>= __end_of_fixed_addresses
);
880 ptep
= fixmap_pte(addr
);
882 if (pgprot_val(flags
)) {
883 set_pte(ptep
, pfn_pte(phys
>> PAGE_SHIFT
, flags
));
885 pte_clear(&init_mm
, addr
, ptep
);
886 flush_tlb_kernel_range(addr
, addr
+PAGE_SIZE
);
890 void *__init
__fixmap_remap_fdt(phys_addr_t dt_phys
, int *size
, pgprot_t prot
)
892 const u64 dt_virt_base
= __fix_to_virt(FIX_FDT
);
897 * Check whether the physical FDT address is set and meets the minimum
898 * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be
899 * at least 8 bytes so that we can always access the magic and size
900 * fields of the FDT header after mapping the first chunk, double check
901 * here if that is indeed the case.
903 BUILD_BUG_ON(MIN_FDT_ALIGN
< 8);
904 if (!dt_phys
|| dt_phys
% MIN_FDT_ALIGN
)
908 * Make sure that the FDT region can be mapped without the need to
909 * allocate additional translation table pages, so that it is safe
910 * to call create_mapping_noalloc() this early.
912 * On 64k pages, the FDT will be mapped using PTEs, so we need to
913 * be in the same PMD as the rest of the fixmap.
914 * On 4k pages, we'll use section mappings for the FDT so we only
915 * have to be in the same PUD.
917 BUILD_BUG_ON(dt_virt_base
% SZ_2M
);
919 BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END
) >> SWAPPER_TABLE_SHIFT
!=
920 __fix_to_virt(FIX_BTMAP_BEGIN
) >> SWAPPER_TABLE_SHIFT
);
922 offset
= dt_phys
% SWAPPER_BLOCK_SIZE
;
923 dt_virt
= (void *)dt_virt_base
+ offset
;
925 /* map the first chunk so we can read the size from the header */
926 create_mapping_noalloc(round_down(dt_phys
, SWAPPER_BLOCK_SIZE
),
927 dt_virt_base
, SWAPPER_BLOCK_SIZE
, prot
);
929 if (fdt_magic(dt_virt
) != FDT_MAGIC
)
932 *size
= fdt_totalsize(dt_virt
);
933 if (*size
> MAX_FDT_SIZE
)
936 if (offset
+ *size
> SWAPPER_BLOCK_SIZE
)
937 create_mapping_noalloc(round_down(dt_phys
, SWAPPER_BLOCK_SIZE
), dt_virt_base
,
938 round_up(offset
+ *size
, SWAPPER_BLOCK_SIZE
), prot
);
943 void *__init
fixmap_remap_fdt(phys_addr_t dt_phys
)
948 dt_virt
= __fixmap_remap_fdt(dt_phys
, &size
, PAGE_KERNEL_RO
);
952 memblock_reserve(dt_phys
, size
);
956 int __init
arch_ioremap_pud_supported(void)
959 * Only 4k granule supports level 1 block mappings.
960 * SW table walks can't handle removal of intermediate entries.
962 return IS_ENABLED(CONFIG_ARM64_4K_PAGES
) &&
963 !IS_ENABLED(CONFIG_ARM64_PTDUMP_DEBUGFS
);
966 int __init
arch_ioremap_pmd_supported(void)
968 /* See arch_ioremap_pud_supported() */
969 return !IS_ENABLED(CONFIG_ARM64_PTDUMP_DEBUGFS
);
972 int pud_set_huge(pud_t
*pudp
, phys_addr_t phys
, pgprot_t prot
)
974 pgprot_t sect_prot
= __pgprot(PUD_TYPE_SECT
|
975 pgprot_val(mk_sect_prot(prot
)));
976 pud_t new_pud
= pfn_pud(__phys_to_pfn(phys
), sect_prot
);
978 /* Only allow permission changes for now */
979 if (!pgattr_change_is_safe(READ_ONCE(pud_val(*pudp
)),
983 BUG_ON(phys
& ~PUD_MASK
);
984 set_pud(pudp
, new_pud
);
988 int pmd_set_huge(pmd_t
*pmdp
, phys_addr_t phys
, pgprot_t prot
)
990 pgprot_t sect_prot
= __pgprot(PMD_TYPE_SECT
|
991 pgprot_val(mk_sect_prot(prot
)));
992 pmd_t new_pmd
= pfn_pmd(__phys_to_pfn(phys
), sect_prot
);
994 /* Only allow permission changes for now */
995 if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp
)),
999 BUG_ON(phys
& ~PMD_MASK
);
1000 set_pmd(pmdp
, new_pmd
);
1004 int pud_clear_huge(pud_t
*pudp
)
1006 if (!pud_sect(READ_ONCE(*pudp
)))
1012 int pmd_clear_huge(pmd_t
*pmdp
)
1014 if (!pmd_sect(READ_ONCE(*pmdp
)))
1020 int pmd_free_pte_page(pmd_t
*pmdp
, unsigned long addr
)
1025 pmd
= READ_ONCE(*pmdp
);
1027 if (!pmd_table(pmd
)) {
1032 table
= pte_offset_kernel(pmdp
, addr
);
1034 __flush_tlb_kernel_pgtable(addr
);
1035 pte_free_kernel(NULL
, table
);
1039 int pud_free_pmd_page(pud_t
*pudp
, unsigned long addr
)
1044 unsigned long next
, end
;
1046 pud
= READ_ONCE(*pudp
);
1048 if (!pud_table(pud
)) {
1053 table
= pmd_offset(pudp
, addr
);
1056 end
= addr
+ PUD_SIZE
;
1058 pmd_free_pte_page(pmdp
, next
);
1059 } while (pmdp
++, next
+= PMD_SIZE
, next
!= end
);
1062 __flush_tlb_kernel_pgtable(addr
);
1063 pmd_free(NULL
, table
);
1067 int p4d_free_pud_page(p4d_t
*p4d
, unsigned long addr
)
1069 return 0; /* Don't attempt a block mapping */
1072 #ifdef CONFIG_MEMORY_HOTPLUG
1073 int arch_add_memory(int nid
, u64 start
, u64 size
,
1074 struct mhp_restrictions
*restrictions
)
1078 if (rodata_full
|| debug_pagealloc_enabled())
1079 flags
= NO_BLOCK_MAPPINGS
| NO_CONT_MAPPINGS
;
1081 __create_pgd_mapping(swapper_pg_dir
, start
, __phys_to_virt(start
),
1082 size
, PAGE_KERNEL
, __pgd_pgtable_alloc
, flags
);
1084 return __add_pages(nid
, start
>> PAGE_SHIFT
, size
>> PAGE_SHIFT
,