1 // SPDX-License-Identifier: GPL-2.0-only
3 * Based on arch/arm/mm/mmu.c
5 * Copyright (C) 1995-2005 Russell King
6 * Copyright (C) 2012 ARM Ltd.
9 #include <linux/cache.h>
10 #include <linux/export.h>
11 #include <linux/kernel.h>
12 #include <linux/errno.h>
13 #include <linux/init.h>
14 #include <linux/ioport.h>
15 #include <linux/kexec.h>
16 #include <linux/libfdt.h>
17 #include <linux/mman.h>
18 #include <linux/nodemask.h>
19 #include <linux/memblock.h>
20 #include <linux/memory.h>
24 #include <linux/vmalloc.h>
26 #include <asm/barrier.h>
27 #include <asm/cputype.h>
28 #include <asm/fixmap.h>
29 #include <asm/kasan.h>
30 #include <asm/kernel-pgtable.h>
31 #include <asm/sections.h>
32 #include <asm/setup.h>
33 #include <linux/sizes.h>
35 #include <asm/mmu_context.h>
36 #include <asm/ptdump.h>
37 #include <asm/tlbflush.h>
38 #include <asm/pgalloc.h>
40 #define NO_BLOCK_MAPPINGS BIT(0)
41 #define NO_CONT_MAPPINGS BIT(1)
43 u64 idmap_t0sz
= TCR_T0SZ(VA_BITS_MIN
);
44 u64 idmap_ptrs_per_pgd
= PTRS_PER_PGD
;
46 u64
__section(".mmuoff.data.write") vabits_actual
;
47 EXPORT_SYMBOL(vabits_actual
);
49 u64 kimage_voffset __ro_after_init
;
50 EXPORT_SYMBOL(kimage_voffset
);
53 * Empty_zero_page is a special page that is used for zero-initialized data
56 unsigned long empty_zero_page
[PAGE_SIZE
/ sizeof(unsigned long)] __page_aligned_bss
;
57 EXPORT_SYMBOL(empty_zero_page
);
59 static pte_t bm_pte
[PTRS_PER_PTE
] __page_aligned_bss
;
60 static pmd_t bm_pmd
[PTRS_PER_PMD
] __page_aligned_bss __maybe_unused
;
61 static pud_t bm_pud
[PTRS_PER_PUD
] __page_aligned_bss __maybe_unused
;
63 static DEFINE_SPINLOCK(swapper_pgdir_lock
);
65 void set_swapper_pgd(pgd_t
*pgdp
, pgd_t pgd
)
69 spin_lock(&swapper_pgdir_lock
);
70 fixmap_pgdp
= pgd_set_fixmap(__pa_symbol(pgdp
));
71 WRITE_ONCE(*fixmap_pgdp
, pgd
);
73 * We need dsb(ishst) here to ensure the page-table-walker sees
74 * our new entry before set_p?d() returns. The fixmap's
75 * flush_tlb_kernel_range() via clear_fixmap() does this for us.
78 spin_unlock(&swapper_pgdir_lock
);
81 pgprot_t
phys_mem_access_prot(struct file
*file
, unsigned long pfn
,
82 unsigned long size
, pgprot_t vma_prot
)
85 return pgprot_noncached(vma_prot
);
86 else if (file
->f_flags
& O_SYNC
)
87 return pgprot_writecombine(vma_prot
);
90 EXPORT_SYMBOL(phys_mem_access_prot
);
92 static phys_addr_t __init
early_pgtable_alloc(int shift
)
97 phys
= memblock_phys_alloc(PAGE_SIZE
, PAGE_SIZE
);
99 panic("Failed to allocate page table page\n");
102 * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE
103 * slot will be free, so we can (ab)use the FIX_PTE slot to initialise
104 * any level of table.
106 ptr
= pte_set_fixmap(phys
);
108 memset(ptr
, 0, PAGE_SIZE
);
111 * Implicit barriers also ensure the zeroed page is visible to the page
119 static bool pgattr_change_is_safe(u64 old
, u64
new)
122 * The following mapping attributes may be updated in live
123 * kernel mappings without the need for break-before-make.
125 pteval_t mask
= PTE_PXN
| PTE_RDONLY
| PTE_WRITE
| PTE_NG
;
127 /* creating or taking down mappings is always safe */
128 if (old
== 0 || new == 0)
131 /* live contiguous mappings may not be manipulated at all */
132 if ((old
| new) & PTE_CONT
)
135 /* Transitioning from Non-Global to Global is unsafe */
136 if (old
& ~new & PTE_NG
)
140 * Changing the memory type between Normal and Normal-Tagged is safe
141 * since Tagged is considered a permission attribute from the
142 * mismatched attribute aliases perspective.
144 if (((old
& PTE_ATTRINDX_MASK
) == PTE_ATTRINDX(MT_NORMAL
) ||
145 (old
& PTE_ATTRINDX_MASK
) == PTE_ATTRINDX(MT_NORMAL_TAGGED
)) &&
146 ((new & PTE_ATTRINDX_MASK
) == PTE_ATTRINDX(MT_NORMAL
) ||
147 (new & PTE_ATTRINDX_MASK
) == PTE_ATTRINDX(MT_NORMAL_TAGGED
)))
148 mask
|= PTE_ATTRINDX_MASK
;
150 return ((old
^ new) & ~mask
) == 0;
153 static void init_pte(pmd_t
*pmdp
, unsigned long addr
, unsigned long end
,
154 phys_addr_t phys
, pgprot_t prot
)
158 ptep
= pte_set_fixmap_offset(pmdp
, addr
);
160 pte_t old_pte
= READ_ONCE(*ptep
);
162 set_pte(ptep
, pfn_pte(__phys_to_pfn(phys
), prot
));
165 * After the PTE entry has been populated once, we
166 * only allow updates to the permission attributes.
168 BUG_ON(!pgattr_change_is_safe(pte_val(old_pte
),
169 READ_ONCE(pte_val(*ptep
))));
172 } while (ptep
++, addr
+= PAGE_SIZE
, addr
!= end
);
177 static void alloc_init_cont_pte(pmd_t
*pmdp
, unsigned long addr
,
178 unsigned long end
, phys_addr_t phys
,
180 phys_addr_t (*pgtable_alloc
)(int),
184 pmd_t pmd
= READ_ONCE(*pmdp
);
186 BUG_ON(pmd_sect(pmd
));
188 phys_addr_t pte_phys
;
189 BUG_ON(!pgtable_alloc
);
190 pte_phys
= pgtable_alloc(PAGE_SHIFT
);
191 __pmd_populate(pmdp
, pte_phys
, PMD_TYPE_TABLE
);
192 pmd
= READ_ONCE(*pmdp
);
194 BUG_ON(pmd_bad(pmd
));
197 pgprot_t __prot
= prot
;
199 next
= pte_cont_addr_end(addr
, end
);
201 /* use a contiguous mapping if the range is suitably aligned */
202 if ((((addr
| next
| phys
) & ~CONT_PTE_MASK
) == 0) &&
203 (flags
& NO_CONT_MAPPINGS
) == 0)
204 __prot
= __pgprot(pgprot_val(prot
) | PTE_CONT
);
206 init_pte(pmdp
, addr
, next
, phys
, __prot
);
209 } while (addr
= next
, addr
!= end
);
212 static void init_pmd(pud_t
*pudp
, unsigned long addr
, unsigned long end
,
213 phys_addr_t phys
, pgprot_t prot
,
214 phys_addr_t (*pgtable_alloc
)(int), int flags
)
219 pmdp
= pmd_set_fixmap_offset(pudp
, addr
);
221 pmd_t old_pmd
= READ_ONCE(*pmdp
);
223 next
= pmd_addr_end(addr
, end
);
225 /* try section mapping first */
226 if (((addr
| next
| phys
) & ~SECTION_MASK
) == 0 &&
227 (flags
& NO_BLOCK_MAPPINGS
) == 0) {
228 pmd_set_huge(pmdp
, phys
, prot
);
231 * After the PMD entry has been populated once, we
232 * only allow updates to the permission attributes.
234 BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd
),
235 READ_ONCE(pmd_val(*pmdp
))));
237 alloc_init_cont_pte(pmdp
, addr
, next
, phys
, prot
,
238 pgtable_alloc
, flags
);
240 BUG_ON(pmd_val(old_pmd
) != 0 &&
241 pmd_val(old_pmd
) != READ_ONCE(pmd_val(*pmdp
)));
244 } while (pmdp
++, addr
= next
, addr
!= end
);
249 static void alloc_init_cont_pmd(pud_t
*pudp
, unsigned long addr
,
250 unsigned long end
, phys_addr_t phys
,
252 phys_addr_t (*pgtable_alloc
)(int), int flags
)
255 pud_t pud
= READ_ONCE(*pudp
);
258 * Check for initial section mappings in the pgd/pud.
260 BUG_ON(pud_sect(pud
));
262 phys_addr_t pmd_phys
;
263 BUG_ON(!pgtable_alloc
);
264 pmd_phys
= pgtable_alloc(PMD_SHIFT
);
265 __pud_populate(pudp
, pmd_phys
, PUD_TYPE_TABLE
);
266 pud
= READ_ONCE(*pudp
);
268 BUG_ON(pud_bad(pud
));
271 pgprot_t __prot
= prot
;
273 next
= pmd_cont_addr_end(addr
, end
);
275 /* use a contiguous mapping if the range is suitably aligned */
276 if ((((addr
| next
| phys
) & ~CONT_PMD_MASK
) == 0) &&
277 (flags
& NO_CONT_MAPPINGS
) == 0)
278 __prot
= __pgprot(pgprot_val(prot
) | PTE_CONT
);
280 init_pmd(pudp
, addr
, next
, phys
, __prot
, pgtable_alloc
, flags
);
283 } while (addr
= next
, addr
!= end
);
286 static inline bool use_1G_block(unsigned long addr
, unsigned long next
,
289 if (PAGE_SHIFT
!= 12)
292 if (((addr
| next
| phys
) & ~PUD_MASK
) != 0)
298 static void alloc_init_pud(pgd_t
*pgdp
, unsigned long addr
, unsigned long end
,
299 phys_addr_t phys
, pgprot_t prot
,
300 phys_addr_t (*pgtable_alloc
)(int),
305 p4d_t
*p4dp
= p4d_offset(pgdp
, addr
);
306 p4d_t p4d
= READ_ONCE(*p4dp
);
309 phys_addr_t pud_phys
;
310 BUG_ON(!pgtable_alloc
);
311 pud_phys
= pgtable_alloc(PUD_SHIFT
);
312 __p4d_populate(p4dp
, pud_phys
, PUD_TYPE_TABLE
);
313 p4d
= READ_ONCE(*p4dp
);
315 BUG_ON(p4d_bad(p4d
));
317 pudp
= pud_set_fixmap_offset(p4dp
, addr
);
319 pud_t old_pud
= READ_ONCE(*pudp
);
321 next
= pud_addr_end(addr
, end
);
324 * For 4K granule only, attempt to put down a 1GB block
326 if (use_1G_block(addr
, next
, phys
) &&
327 (flags
& NO_BLOCK_MAPPINGS
) == 0) {
328 pud_set_huge(pudp
, phys
, prot
);
331 * After the PUD entry has been populated once, we
332 * only allow updates to the permission attributes.
334 BUG_ON(!pgattr_change_is_safe(pud_val(old_pud
),
335 READ_ONCE(pud_val(*pudp
))));
337 alloc_init_cont_pmd(pudp
, addr
, next
, phys
, prot
,
338 pgtable_alloc
, flags
);
340 BUG_ON(pud_val(old_pud
) != 0 &&
341 pud_val(old_pud
) != READ_ONCE(pud_val(*pudp
)));
344 } while (pudp
++, addr
= next
, addr
!= end
);
349 static void __create_pgd_mapping(pgd_t
*pgdir
, phys_addr_t phys
,
350 unsigned long virt
, phys_addr_t size
,
352 phys_addr_t (*pgtable_alloc
)(int),
355 unsigned long addr
, end
, next
;
356 pgd_t
*pgdp
= pgd_offset_pgd(pgdir
, virt
);
359 * If the virtual and physical address don't have the same offset
360 * within a page, we cannot map the region as the caller expects.
362 if (WARN_ON((phys
^ virt
) & ~PAGE_MASK
))
366 addr
= virt
& PAGE_MASK
;
367 end
= PAGE_ALIGN(virt
+ size
);
370 next
= pgd_addr_end(addr
, end
);
371 alloc_init_pud(pgdp
, addr
, next
, phys
, prot
, pgtable_alloc
,
374 } while (pgdp
++, addr
= next
, addr
!= end
);
377 static phys_addr_t
__pgd_pgtable_alloc(int shift
)
379 void *ptr
= (void *)__get_free_page(GFP_PGTABLE_KERNEL
);
382 /* Ensure the zeroed page is visible to the page table walker */
387 static phys_addr_t
pgd_pgtable_alloc(int shift
)
389 phys_addr_t pa
= __pgd_pgtable_alloc(shift
);
392 * Call proper page table ctor in case later we need to
393 * call core mm functions like apply_to_page_range() on
394 * this pre-allocated page table.
396 * We don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK if pmd is
397 * folded, and if so pgtable_pmd_page_ctor() becomes nop.
399 if (shift
== PAGE_SHIFT
)
400 BUG_ON(!pgtable_pte_page_ctor(phys_to_page(pa
)));
401 else if (shift
== PMD_SHIFT
)
402 BUG_ON(!pgtable_pmd_page_ctor(phys_to_page(pa
)));
408 * This function can only be used to modify existing table entries,
409 * without allocating new levels of table. Note that this permits the
410 * creation of new section or page entries.
412 static void __init
create_mapping_noalloc(phys_addr_t phys
, unsigned long virt
,
413 phys_addr_t size
, pgprot_t prot
)
415 if ((virt
>= PAGE_END
) && (virt
< VMALLOC_START
)) {
416 pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
420 __create_pgd_mapping(init_mm
.pgd
, phys
, virt
, size
, prot
, NULL
,
424 void __init
create_pgd_mapping(struct mm_struct
*mm
, phys_addr_t phys
,
425 unsigned long virt
, phys_addr_t size
,
426 pgprot_t prot
, bool page_mappings_only
)
430 BUG_ON(mm
== &init_mm
);
432 if (page_mappings_only
)
433 flags
= NO_BLOCK_MAPPINGS
| NO_CONT_MAPPINGS
;
435 __create_pgd_mapping(mm
->pgd
, phys
, virt
, size
, prot
,
436 pgd_pgtable_alloc
, flags
);
439 static void update_mapping_prot(phys_addr_t phys
, unsigned long virt
,
440 phys_addr_t size
, pgprot_t prot
)
442 if ((virt
>= PAGE_END
) && (virt
< VMALLOC_START
)) {
443 pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n",
448 __create_pgd_mapping(init_mm
.pgd
, phys
, virt
, size
, prot
, NULL
,
451 /* flush the TLBs after updating live kernel mappings */
452 flush_tlb_kernel_range(virt
, virt
+ size
);
455 static void __init
__map_memblock(pgd_t
*pgdp
, phys_addr_t start
,
456 phys_addr_t end
, pgprot_t prot
, int flags
)
458 __create_pgd_mapping(pgdp
, start
, __phys_to_virt(start
), end
- start
,
459 prot
, early_pgtable_alloc
, flags
);
462 void __init
mark_linear_text_alias_ro(void)
465 * Remove the write permissions from the linear alias of .text/.rodata
467 update_mapping_prot(__pa_symbol(_stext
), (unsigned long)lm_alias(_stext
),
468 (unsigned long)__init_begin
- (unsigned long)_stext
,
472 static bool crash_mem_map __initdata
;
474 static int __init
enable_crash_mem_map(char *arg
)
477 * Proper parameter parsing is done by reserve_crashkernel(). We only
478 * need to know if the linear map has to avoid block mappings so that
479 * the crashkernel reservations can be unmapped later.
481 crash_mem_map
= true;
485 early_param("crashkernel", enable_crash_mem_map
);
487 static void __init
map_mem(pgd_t
*pgdp
)
489 phys_addr_t kernel_start
= __pa_symbol(_stext
);
490 phys_addr_t kernel_end
= __pa_symbol(__init_begin
);
491 phys_addr_t start
, end
;
495 if (rodata_full
|| crash_mem_map
|| debug_pagealloc_enabled())
496 flags
= NO_BLOCK_MAPPINGS
| NO_CONT_MAPPINGS
;
499 * Take care not to create a writable alias for the
500 * read-only text and rodata sections of the kernel image.
501 * So temporarily mark them as NOMAP to skip mappings in
502 * the following for-loop
504 memblock_mark_nomap(kernel_start
, kernel_end
- kernel_start
);
506 /* map all the memory banks */
507 for_each_mem_range(i
, &start
, &end
) {
511 * The linear map must allow allocation tags reading/writing
512 * if MTE is present. Otherwise, it has the same attributes as
515 __map_memblock(pgdp
, start
, end
, pgprot_tagged(PAGE_KERNEL
),
520 * Map the linear alias of the [_stext, __init_begin) interval
521 * as non-executable now, and remove the write permission in
522 * mark_linear_text_alias_ro() below (which will be called after
523 * alternative patching has completed). This makes the contents
524 * of the region accessible to subsystems such as hibernate,
525 * but protects it from inadvertent modification or execution.
526 * Note that contiguous mappings cannot be remapped in this way,
527 * so we should avoid them here.
529 __map_memblock(pgdp
, kernel_start
, kernel_end
,
530 PAGE_KERNEL
, NO_CONT_MAPPINGS
);
531 memblock_clear_nomap(kernel_start
, kernel_end
- kernel_start
);
534 void mark_rodata_ro(void)
536 unsigned long section_size
;
539 * mark .rodata as read only. Use __init_begin rather than __end_rodata
540 * to cover NOTES and EXCEPTION_TABLE.
542 section_size
= (unsigned long)__init_begin
- (unsigned long)__start_rodata
;
543 update_mapping_prot(__pa_symbol(__start_rodata
), (unsigned long)__start_rodata
,
544 section_size
, PAGE_KERNEL_RO
);
549 static void __init
map_kernel_segment(pgd_t
*pgdp
, void *va_start
, void *va_end
,
550 pgprot_t prot
, struct vm_struct
*vma
,
551 int flags
, unsigned long vm_flags
)
553 phys_addr_t pa_start
= __pa_symbol(va_start
);
554 unsigned long size
= va_end
- va_start
;
556 BUG_ON(!PAGE_ALIGNED(pa_start
));
557 BUG_ON(!PAGE_ALIGNED(size
));
559 __create_pgd_mapping(pgdp
, pa_start
, (unsigned long)va_start
, size
, prot
,
560 early_pgtable_alloc
, flags
);
562 if (!(vm_flags
& VM_NO_GUARD
))
565 vma
->addr
= va_start
;
566 vma
->phys_addr
= pa_start
;
568 vma
->flags
= VM_MAP
| vm_flags
;
569 vma
->caller
= __builtin_return_address(0);
571 vm_area_add_early(vma
);
574 static int __init
parse_rodata(char *arg
)
576 int ret
= strtobool(arg
, &rodata_enabled
);
582 /* permit 'full' in addition to boolean options */
583 if (strcmp(arg
, "full"))
586 rodata_enabled
= true;
590 early_param("rodata", parse_rodata
);
592 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
593 static int __init
map_entry_trampoline(void)
595 pgprot_t prot
= rodata_enabled
? PAGE_KERNEL_ROX
: PAGE_KERNEL_EXEC
;
596 phys_addr_t pa_start
= __pa_symbol(__entry_tramp_text_start
);
598 /* The trampoline is always mapped and can therefore be global */
599 pgprot_val(prot
) &= ~PTE_NG
;
601 /* Map only the text into the trampoline page table */
602 memset(tramp_pg_dir
, 0, PGD_SIZE
);
603 __create_pgd_mapping(tramp_pg_dir
, pa_start
, TRAMP_VALIAS
, PAGE_SIZE
,
604 prot
, __pgd_pgtable_alloc
, 0);
606 /* Map both the text and data into the kernel page table */
607 __set_fixmap(FIX_ENTRY_TRAMP_TEXT
, pa_start
, prot
);
608 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE
)) {
609 extern char __entry_tramp_data_start
[];
611 __set_fixmap(FIX_ENTRY_TRAMP_DATA
,
612 __pa_symbol(__entry_tramp_data_start
),
618 core_initcall(map_entry_trampoline
);
622 * Open coded check for BTI, only for use to determine configuration
623 * for early mappings for before the cpufeature code has run.
625 static bool arm64_early_this_cpu_has_bti(void)
629 if (!IS_ENABLED(CONFIG_ARM64_BTI_KERNEL
))
632 pfr1
= read_sysreg_s(SYS_ID_AA64PFR1_EL1
);
633 return cpuid_feature_extract_unsigned_field(pfr1
,
634 ID_AA64PFR1_BT_SHIFT
);
638 * Create fine-grained mappings for the kernel.
640 static void __init
map_kernel(pgd_t
*pgdp
)
642 static struct vm_struct vmlinux_text
, vmlinux_rodata
, vmlinux_inittext
,
643 vmlinux_initdata
, vmlinux_data
;
646 * External debuggers may need to write directly to the text
647 * mapping to install SW breakpoints. Allow this (only) when
648 * explicitly requested with rodata=off.
650 pgprot_t text_prot
= rodata_enabled
? PAGE_KERNEL_ROX
: PAGE_KERNEL_EXEC
;
653 * If we have a CPU that supports BTI and a kernel built for
654 * BTI then mark the kernel executable text as guarded pages
655 * now so we don't have to rewrite the page tables later.
657 if (arm64_early_this_cpu_has_bti())
658 text_prot
= __pgprot_modify(text_prot
, PTE_GP
, PTE_GP
);
661 * Only rodata will be remapped with different permissions later on,
662 * all other segments are allowed to use contiguous mappings.
664 map_kernel_segment(pgdp
, _stext
, _etext
, text_prot
, &vmlinux_text
, 0,
666 map_kernel_segment(pgdp
, __start_rodata
, __inittext_begin
, PAGE_KERNEL
,
667 &vmlinux_rodata
, NO_CONT_MAPPINGS
, VM_NO_GUARD
);
668 map_kernel_segment(pgdp
, __inittext_begin
, __inittext_end
, text_prot
,
669 &vmlinux_inittext
, 0, VM_NO_GUARD
);
670 map_kernel_segment(pgdp
, __initdata_begin
, __initdata_end
, PAGE_KERNEL
,
671 &vmlinux_initdata
, 0, VM_NO_GUARD
);
672 map_kernel_segment(pgdp
, _data
, _end
, PAGE_KERNEL
, &vmlinux_data
, 0, 0);
674 if (!READ_ONCE(pgd_val(*pgd_offset_pgd(pgdp
, FIXADDR_START
)))) {
676 * The fixmap falls in a separate pgd to the kernel, and doesn't
677 * live in the carveout for the swapper_pg_dir. We can simply
678 * re-use the existing dir for the fixmap.
680 set_pgd(pgd_offset_pgd(pgdp
, FIXADDR_START
),
681 READ_ONCE(*pgd_offset_k(FIXADDR_START
)));
682 } else if (CONFIG_PGTABLE_LEVELS
> 3) {
687 * The fixmap shares its top level pgd entry with the kernel
688 * mapping. This can really only occur when we are running
689 * with 16k/4 levels, so we can simply reuse the pud level
692 BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES
));
693 bm_pgdp
= pgd_offset_pgd(pgdp
, FIXADDR_START
);
694 bm_p4dp
= p4d_offset(bm_pgdp
, FIXADDR_START
);
695 bm_pudp
= pud_set_fixmap_offset(bm_p4dp
, FIXADDR_START
);
696 pud_populate(&init_mm
, bm_pudp
, lm_alias(bm_pmd
));
702 kasan_copy_shadow(pgdp
);
705 void __init
paging_init(void)
707 pgd_t
*pgdp
= pgd_set_fixmap(__pa_symbol(swapper_pg_dir
));
714 cpu_replace_ttbr1(lm_alias(swapper_pg_dir
));
715 init_mm
.pgd
= swapper_pg_dir
;
717 memblock_free(__pa_symbol(init_pg_dir
),
718 __pa_symbol(init_pg_end
) - __pa_symbol(init_pg_dir
));
720 memblock_allow_resize();
724 * Check whether a kernel address is valid (derived from arch/x86/).
726 int kern_addr_valid(unsigned long addr
)
734 addr
= arch_kasan_reset_tag(addr
);
735 if ((((long)addr
) >> VA_BITS
) != -1UL)
738 pgdp
= pgd_offset_k(addr
);
739 if (pgd_none(READ_ONCE(*pgdp
)))
742 p4dp
= p4d_offset(pgdp
, addr
);
743 if (p4d_none(READ_ONCE(*p4dp
)))
746 pudp
= pud_offset(p4dp
, addr
);
747 pud
= READ_ONCE(*pudp
);
752 return pfn_valid(pud_pfn(pud
));
754 pmdp
= pmd_offset(pudp
, addr
);
755 pmd
= READ_ONCE(*pmdp
);
760 return pfn_valid(pmd_pfn(pmd
));
762 ptep
= pte_offset_kernel(pmdp
, addr
);
763 pte
= READ_ONCE(*ptep
);
767 return pfn_valid(pte_pfn(pte
));
770 #ifdef CONFIG_MEMORY_HOTPLUG
771 static void free_hotplug_page_range(struct page
*page
, size_t size
,
772 struct vmem_altmap
*altmap
)
775 vmem_altmap_free(altmap
, size
>> PAGE_SHIFT
);
777 WARN_ON(PageReserved(page
));
778 free_pages((unsigned long)page_address(page
), get_order(size
));
782 static void free_hotplug_pgtable_page(struct page
*page
)
784 free_hotplug_page_range(page
, PAGE_SIZE
, NULL
);
787 static bool pgtable_range_aligned(unsigned long start
, unsigned long end
,
788 unsigned long floor
, unsigned long ceiling
,
801 if (end
- 1 > ceiling
- 1)
806 static void unmap_hotplug_pte_range(pmd_t
*pmdp
, unsigned long addr
,
807 unsigned long end
, bool free_mapped
,
808 struct vmem_altmap
*altmap
)
813 ptep
= pte_offset_kernel(pmdp
, addr
);
814 pte
= READ_ONCE(*ptep
);
818 WARN_ON(!pte_present(pte
));
819 pte_clear(&init_mm
, addr
, ptep
);
820 flush_tlb_kernel_range(addr
, addr
+ PAGE_SIZE
);
822 free_hotplug_page_range(pte_page(pte
),
824 } while (addr
+= PAGE_SIZE
, addr
< end
);
827 static void unmap_hotplug_pmd_range(pud_t
*pudp
, unsigned long addr
,
828 unsigned long end
, bool free_mapped
,
829 struct vmem_altmap
*altmap
)
835 next
= pmd_addr_end(addr
, end
);
836 pmdp
= pmd_offset(pudp
, addr
);
837 pmd
= READ_ONCE(*pmdp
);
841 WARN_ON(!pmd_present(pmd
));
846 * One TLBI should be sufficient here as the PMD_SIZE
847 * range is mapped with a single block entry.
849 flush_tlb_kernel_range(addr
, addr
+ PAGE_SIZE
);
851 free_hotplug_page_range(pmd_page(pmd
),
855 WARN_ON(!pmd_table(pmd
));
856 unmap_hotplug_pte_range(pmdp
, addr
, next
, free_mapped
, altmap
);
857 } while (addr
= next
, addr
< end
);
860 static void unmap_hotplug_pud_range(p4d_t
*p4dp
, unsigned long addr
,
861 unsigned long end
, bool free_mapped
,
862 struct vmem_altmap
*altmap
)
868 next
= pud_addr_end(addr
, end
);
869 pudp
= pud_offset(p4dp
, addr
);
870 pud
= READ_ONCE(*pudp
);
874 WARN_ON(!pud_present(pud
));
879 * One TLBI should be sufficient here as the PUD_SIZE
880 * range is mapped with a single block entry.
882 flush_tlb_kernel_range(addr
, addr
+ PAGE_SIZE
);
884 free_hotplug_page_range(pud_page(pud
),
888 WARN_ON(!pud_table(pud
));
889 unmap_hotplug_pmd_range(pudp
, addr
, next
, free_mapped
, altmap
);
890 } while (addr
= next
, addr
< end
);
893 static void unmap_hotplug_p4d_range(pgd_t
*pgdp
, unsigned long addr
,
894 unsigned long end
, bool free_mapped
,
895 struct vmem_altmap
*altmap
)
901 next
= p4d_addr_end(addr
, end
);
902 p4dp
= p4d_offset(pgdp
, addr
);
903 p4d
= READ_ONCE(*p4dp
);
907 WARN_ON(!p4d_present(p4d
));
908 unmap_hotplug_pud_range(p4dp
, addr
, next
, free_mapped
, altmap
);
909 } while (addr
= next
, addr
< end
);
912 static void unmap_hotplug_range(unsigned long addr
, unsigned long end
,
913 bool free_mapped
, struct vmem_altmap
*altmap
)
919 * altmap can only be used as vmemmap mapping backing memory.
920 * In case the backing memory itself is not being freed, then
921 * altmap is irrelevant. Warn about this inconsistency when
924 WARN_ON(!free_mapped
&& altmap
);
927 next
= pgd_addr_end(addr
, end
);
928 pgdp
= pgd_offset_k(addr
);
929 pgd
= READ_ONCE(*pgdp
);
933 WARN_ON(!pgd_present(pgd
));
934 unmap_hotplug_p4d_range(pgdp
, addr
, next
, free_mapped
, altmap
);
935 } while (addr
= next
, addr
< end
);
938 static void free_empty_pte_table(pmd_t
*pmdp
, unsigned long addr
,
939 unsigned long end
, unsigned long floor
,
940 unsigned long ceiling
)
943 unsigned long i
, start
= addr
;
946 ptep
= pte_offset_kernel(pmdp
, addr
);
947 pte
= READ_ONCE(*ptep
);
950 * This is just a sanity check here which verifies that
951 * pte clearing has been done by earlier unmap loops.
953 WARN_ON(!pte_none(pte
));
954 } while (addr
+= PAGE_SIZE
, addr
< end
);
956 if (!pgtable_range_aligned(start
, end
, floor
, ceiling
, PMD_MASK
))
960 * Check whether we can free the pte page if the rest of the
961 * entries are empty. Overlap with other regions have been
962 * handled by the floor/ceiling check.
964 ptep
= pte_offset_kernel(pmdp
, 0UL);
965 for (i
= 0; i
< PTRS_PER_PTE
; i
++) {
966 if (!pte_none(READ_ONCE(ptep
[i
])))
971 __flush_tlb_kernel_pgtable(start
);
972 free_hotplug_pgtable_page(virt_to_page(ptep
));
975 static void free_empty_pmd_table(pud_t
*pudp
, unsigned long addr
,
976 unsigned long end
, unsigned long floor
,
977 unsigned long ceiling
)
980 unsigned long i
, next
, start
= addr
;
983 next
= pmd_addr_end(addr
, end
);
984 pmdp
= pmd_offset(pudp
, addr
);
985 pmd
= READ_ONCE(*pmdp
);
989 WARN_ON(!pmd_present(pmd
) || !pmd_table(pmd
) || pmd_sect(pmd
));
990 free_empty_pte_table(pmdp
, addr
, next
, floor
, ceiling
);
991 } while (addr
= next
, addr
< end
);
993 if (CONFIG_PGTABLE_LEVELS
<= 2)
996 if (!pgtable_range_aligned(start
, end
, floor
, ceiling
, PUD_MASK
))
1000 * Check whether we can free the pmd page if the rest of the
1001 * entries are empty. Overlap with other regions have been
1002 * handled by the floor/ceiling check.
1004 pmdp
= pmd_offset(pudp
, 0UL);
1005 for (i
= 0; i
< PTRS_PER_PMD
; i
++) {
1006 if (!pmd_none(READ_ONCE(pmdp
[i
])))
1011 __flush_tlb_kernel_pgtable(start
);
1012 free_hotplug_pgtable_page(virt_to_page(pmdp
));
1015 static void free_empty_pud_table(p4d_t
*p4dp
, unsigned long addr
,
1016 unsigned long end
, unsigned long floor
,
1017 unsigned long ceiling
)
1020 unsigned long i
, next
, start
= addr
;
1023 next
= pud_addr_end(addr
, end
);
1024 pudp
= pud_offset(p4dp
, addr
);
1025 pud
= READ_ONCE(*pudp
);
1029 WARN_ON(!pud_present(pud
) || !pud_table(pud
) || pud_sect(pud
));
1030 free_empty_pmd_table(pudp
, addr
, next
, floor
, ceiling
);
1031 } while (addr
= next
, addr
< end
);
1033 if (CONFIG_PGTABLE_LEVELS
<= 3)
1036 if (!pgtable_range_aligned(start
, end
, floor
, ceiling
, PGDIR_MASK
))
1040 * Check whether we can free the pud page if the rest of the
1041 * entries are empty. Overlap with other regions have been
1042 * handled by the floor/ceiling check.
1044 pudp
= pud_offset(p4dp
, 0UL);
1045 for (i
= 0; i
< PTRS_PER_PUD
; i
++) {
1046 if (!pud_none(READ_ONCE(pudp
[i
])))
1051 __flush_tlb_kernel_pgtable(start
);
1052 free_hotplug_pgtable_page(virt_to_page(pudp
));
1055 static void free_empty_p4d_table(pgd_t
*pgdp
, unsigned long addr
,
1056 unsigned long end
, unsigned long floor
,
1057 unsigned long ceiling
)
1063 next
= p4d_addr_end(addr
, end
);
1064 p4dp
= p4d_offset(pgdp
, addr
);
1065 p4d
= READ_ONCE(*p4dp
);
1069 WARN_ON(!p4d_present(p4d
));
1070 free_empty_pud_table(p4dp
, addr
, next
, floor
, ceiling
);
1071 } while (addr
= next
, addr
< end
);
1074 static void free_empty_tables(unsigned long addr
, unsigned long end
,
1075 unsigned long floor
, unsigned long ceiling
)
1081 next
= pgd_addr_end(addr
, end
);
1082 pgdp
= pgd_offset_k(addr
);
1083 pgd
= READ_ONCE(*pgdp
);
1087 WARN_ON(!pgd_present(pgd
));
1088 free_empty_p4d_table(pgdp
, addr
, next
, floor
, ceiling
);
1089 } while (addr
= next
, addr
< end
);
1093 #ifdef CONFIG_SPARSEMEM_VMEMMAP
1094 #if !ARM64_SWAPPER_USES_SECTION_MAPS
1095 int __meminit
vmemmap_populate(unsigned long start
, unsigned long end
, int node
,
1096 struct vmem_altmap
*altmap
)
1098 return vmemmap_populate_basepages(start
, end
, node
, altmap
);
1100 #else /* !ARM64_SWAPPER_USES_SECTION_MAPS */
1101 int __meminit
vmemmap_populate(unsigned long start
, unsigned long end
, int node
,
1102 struct vmem_altmap
*altmap
)
1104 unsigned long addr
= start
;
1112 next
= pmd_addr_end(addr
, end
);
1114 pgdp
= vmemmap_pgd_populate(addr
, node
);
1118 p4dp
= vmemmap_p4d_populate(pgdp
, addr
, node
);
1122 pudp
= vmemmap_pud_populate(p4dp
, addr
, node
);
1126 pmdp
= pmd_offset(pudp
, addr
);
1127 if (pmd_none(READ_ONCE(*pmdp
))) {
1130 p
= vmemmap_alloc_block_buf(PMD_SIZE
, node
, altmap
);
1132 if (vmemmap_populate_basepages(addr
, next
, node
, altmap
))
1137 pmd_set_huge(pmdp
, __pa(p
), __pgprot(PROT_SECT_NORMAL
));
1139 vmemmap_verify((pte_t
*)pmdp
, node
, addr
, next
);
1140 } while (addr
= next
, addr
!= end
);
1144 #endif /* !ARM64_SWAPPER_USES_SECTION_MAPS */
1145 void vmemmap_free(unsigned long start
, unsigned long end
,
1146 struct vmem_altmap
*altmap
)
1148 #ifdef CONFIG_MEMORY_HOTPLUG
1149 WARN_ON((start
< VMEMMAP_START
) || (end
> VMEMMAP_END
));
1151 unmap_hotplug_range(start
, end
, true, altmap
);
1152 free_empty_tables(start
, end
, VMEMMAP_START
, VMEMMAP_END
);
1155 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
1157 static inline pud_t
* fixmap_pud(unsigned long addr
)
1159 pgd_t
*pgdp
= pgd_offset_k(addr
);
1160 p4d_t
*p4dp
= p4d_offset(pgdp
, addr
);
1161 p4d_t p4d
= READ_ONCE(*p4dp
);
1163 BUG_ON(p4d_none(p4d
) || p4d_bad(p4d
));
1165 return pud_offset_kimg(p4dp
, addr
);
1168 static inline pmd_t
* fixmap_pmd(unsigned long addr
)
1170 pud_t
*pudp
= fixmap_pud(addr
);
1171 pud_t pud
= READ_ONCE(*pudp
);
1173 BUG_ON(pud_none(pud
) || pud_bad(pud
));
1175 return pmd_offset_kimg(pudp
, addr
);
1178 static inline pte_t
* fixmap_pte(unsigned long addr
)
1180 return &bm_pte
[pte_index(addr
)];
1184 * The p*d_populate functions call virt_to_phys implicitly so they can't be used
1185 * directly on kernel symbols (bm_p*d). This function is called too early to use
1186 * lm_alias so __p*d_populate functions must be used to populate with the
1187 * physical address from __pa_symbol.
1189 void __init
early_fixmap_init(void)
1195 unsigned long addr
= FIXADDR_START
;
1197 pgdp
= pgd_offset_k(addr
);
1198 p4dp
= p4d_offset(pgdp
, addr
);
1199 p4d
= READ_ONCE(*p4dp
);
1200 if (CONFIG_PGTABLE_LEVELS
> 3 &&
1201 !(p4d_none(p4d
) || p4d_page_paddr(p4d
) == __pa_symbol(bm_pud
))) {
1203 * We only end up here if the kernel mapping and the fixmap
1204 * share the top level pgd entry, which should only happen on
1205 * 16k/4 levels configurations.
1207 BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES
));
1208 pudp
= pud_offset_kimg(p4dp
, addr
);
1211 __p4d_populate(p4dp
, __pa_symbol(bm_pud
), PUD_TYPE_TABLE
);
1212 pudp
= fixmap_pud(addr
);
1214 if (pud_none(READ_ONCE(*pudp
)))
1215 __pud_populate(pudp
, __pa_symbol(bm_pmd
), PMD_TYPE_TABLE
);
1216 pmdp
= fixmap_pmd(addr
);
1217 __pmd_populate(pmdp
, __pa_symbol(bm_pte
), PMD_TYPE_TABLE
);
1220 * The boot-ioremap range spans multiple pmds, for which
1221 * we are not prepared:
1223 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN
) >> PMD_SHIFT
)
1224 != (__fix_to_virt(FIX_BTMAP_END
) >> PMD_SHIFT
));
1226 if ((pmdp
!= fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN
)))
1227 || pmdp
!= fixmap_pmd(fix_to_virt(FIX_BTMAP_END
))) {
1229 pr_warn("pmdp %p != %p, %p\n",
1230 pmdp
, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN
)),
1231 fixmap_pmd(fix_to_virt(FIX_BTMAP_END
)));
1232 pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
1233 fix_to_virt(FIX_BTMAP_BEGIN
));
1234 pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n",
1235 fix_to_virt(FIX_BTMAP_END
));
1237 pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END
);
1238 pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN
);
1243 * Unusually, this is also called in IRQ context (ghes_iounmap_irq) so if we
1244 * ever need to use IPIs for TLB broadcasting, then we're in trouble here.
1246 void __set_fixmap(enum fixed_addresses idx
,
1247 phys_addr_t phys
, pgprot_t flags
)
1249 unsigned long addr
= __fix_to_virt(idx
);
1252 BUG_ON(idx
<= FIX_HOLE
|| idx
>= __end_of_fixed_addresses
);
1254 ptep
= fixmap_pte(addr
);
1256 if (pgprot_val(flags
)) {
1257 set_pte(ptep
, pfn_pte(phys
>> PAGE_SHIFT
, flags
));
1259 pte_clear(&init_mm
, addr
, ptep
);
1260 flush_tlb_kernel_range(addr
, addr
+PAGE_SIZE
);
1264 void *__init
fixmap_remap_fdt(phys_addr_t dt_phys
, int *size
, pgprot_t prot
)
1266 const u64 dt_virt_base
= __fix_to_virt(FIX_FDT
);
1271 * Check whether the physical FDT address is set and meets the minimum
1272 * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be
1273 * at least 8 bytes so that we can always access the magic and size
1274 * fields of the FDT header after mapping the first chunk, double check
1275 * here if that is indeed the case.
1277 BUILD_BUG_ON(MIN_FDT_ALIGN
< 8);
1278 if (!dt_phys
|| dt_phys
% MIN_FDT_ALIGN
)
1282 * Make sure that the FDT region can be mapped without the need to
1283 * allocate additional translation table pages, so that it is safe
1284 * to call create_mapping_noalloc() this early.
1286 * On 64k pages, the FDT will be mapped using PTEs, so we need to
1287 * be in the same PMD as the rest of the fixmap.
1288 * On 4k pages, we'll use section mappings for the FDT so we only
1289 * have to be in the same PUD.
1291 BUILD_BUG_ON(dt_virt_base
% SZ_2M
);
1293 BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END
) >> SWAPPER_TABLE_SHIFT
!=
1294 __fix_to_virt(FIX_BTMAP_BEGIN
) >> SWAPPER_TABLE_SHIFT
);
1296 offset
= dt_phys
% SWAPPER_BLOCK_SIZE
;
1297 dt_virt
= (void *)dt_virt_base
+ offset
;
1299 /* map the first chunk so we can read the size from the header */
1300 create_mapping_noalloc(round_down(dt_phys
, SWAPPER_BLOCK_SIZE
),
1301 dt_virt_base
, SWAPPER_BLOCK_SIZE
, prot
);
1303 if (fdt_magic(dt_virt
) != FDT_MAGIC
)
1306 *size
= fdt_totalsize(dt_virt
);
1307 if (*size
> MAX_FDT_SIZE
)
1310 if (offset
+ *size
> SWAPPER_BLOCK_SIZE
)
1311 create_mapping_noalloc(round_down(dt_phys
, SWAPPER_BLOCK_SIZE
), dt_virt_base
,
1312 round_up(offset
+ *size
, SWAPPER_BLOCK_SIZE
), prot
);
1317 int __init
arch_ioremap_p4d_supported(void)
1322 int __init
arch_ioremap_pud_supported(void)
1325 * Only 4k granule supports level 1 block mappings.
1326 * SW table walks can't handle removal of intermediate entries.
1328 return IS_ENABLED(CONFIG_ARM64_4K_PAGES
) &&
1329 !IS_ENABLED(CONFIG_PTDUMP_DEBUGFS
);
1332 int __init
arch_ioremap_pmd_supported(void)
1334 /* See arch_ioremap_pud_supported() */
1335 return !IS_ENABLED(CONFIG_PTDUMP_DEBUGFS
);
1338 int pud_set_huge(pud_t
*pudp
, phys_addr_t phys
, pgprot_t prot
)
1340 pud_t new_pud
= pfn_pud(__phys_to_pfn(phys
), mk_pud_sect_prot(prot
));
1342 /* Only allow permission changes for now */
1343 if (!pgattr_change_is_safe(READ_ONCE(pud_val(*pudp
)),
1347 VM_BUG_ON(phys
& ~PUD_MASK
);
1348 set_pud(pudp
, new_pud
);
1352 int pmd_set_huge(pmd_t
*pmdp
, phys_addr_t phys
, pgprot_t prot
)
1354 pmd_t new_pmd
= pfn_pmd(__phys_to_pfn(phys
), mk_pmd_sect_prot(prot
));
1356 /* Only allow permission changes for now */
1357 if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp
)),
1361 VM_BUG_ON(phys
& ~PMD_MASK
);
1362 set_pmd(pmdp
, new_pmd
);
1366 int pud_clear_huge(pud_t
*pudp
)
1368 if (!pud_sect(READ_ONCE(*pudp
)))
1374 int pmd_clear_huge(pmd_t
*pmdp
)
1376 if (!pmd_sect(READ_ONCE(*pmdp
)))
1382 int pmd_free_pte_page(pmd_t
*pmdp
, unsigned long addr
)
1387 pmd
= READ_ONCE(*pmdp
);
1389 if (!pmd_table(pmd
)) {
1394 table
= pte_offset_kernel(pmdp
, addr
);
1396 __flush_tlb_kernel_pgtable(addr
);
1397 pte_free_kernel(NULL
, table
);
1401 int pud_free_pmd_page(pud_t
*pudp
, unsigned long addr
)
1406 unsigned long next
, end
;
1408 pud
= READ_ONCE(*pudp
);
1410 if (!pud_table(pud
)) {
1415 table
= pmd_offset(pudp
, addr
);
1418 end
= addr
+ PUD_SIZE
;
1420 pmd_free_pte_page(pmdp
, next
);
1421 } while (pmdp
++, next
+= PMD_SIZE
, next
!= end
);
1424 __flush_tlb_kernel_pgtable(addr
);
1425 pmd_free(NULL
, table
);
1429 int p4d_free_pud_page(p4d_t
*p4d
, unsigned long addr
)
1431 return 0; /* Don't attempt a block mapping */
1434 #ifdef CONFIG_MEMORY_HOTPLUG
1435 static void __remove_pgd_mapping(pgd_t
*pgdir
, unsigned long start
, u64 size
)
1437 unsigned long end
= start
+ size
;
1439 WARN_ON(pgdir
!= init_mm
.pgd
);
1440 WARN_ON((start
< PAGE_OFFSET
) || (end
> PAGE_END
));
1442 unmap_hotplug_range(start
, end
, false, NULL
);
1443 free_empty_tables(start
, end
, PAGE_OFFSET
, PAGE_END
);
1446 static bool inside_linear_region(u64 start
, u64 size
)
1448 u64 start_linear_pa
= __pa(_PAGE_OFFSET(vabits_actual
));
1449 u64 end_linear_pa
= __pa(PAGE_END
- 1);
1451 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE
)) {
1453 * Check for a wrap, it is possible because of randomized linear
1454 * mapping the start physical address is actually bigger than
1455 * the end physical address. In this case set start to zero
1456 * because [0, end_linear_pa] range must still be able to cover
1457 * all addressable physical addresses.
1459 if (start_linear_pa
> end_linear_pa
)
1460 start_linear_pa
= 0;
1463 WARN_ON(start_linear_pa
> end_linear_pa
);
1466 * Linear mapping region is the range [PAGE_OFFSET..(PAGE_END - 1)]
1467 * accommodating both its ends but excluding PAGE_END. Max physical
1468 * range which can be mapped inside this linear mapping range, must
1469 * also be derived from its end points.
1471 return start
>= start_linear_pa
&& (start
+ size
- 1) <= end_linear_pa
;
1474 int arch_add_memory(int nid
, u64 start
, u64 size
,
1475 struct mhp_params
*params
)
1479 if (!inside_linear_region(start
, size
)) {
1480 pr_err("[%llx %llx] is outside linear mapping region\n", start
, start
+ size
);
1484 if (rodata_full
|| debug_pagealloc_enabled())
1485 flags
= NO_BLOCK_MAPPINGS
| NO_CONT_MAPPINGS
;
1487 __create_pgd_mapping(swapper_pg_dir
, start
, __phys_to_virt(start
),
1488 size
, params
->pgprot
, __pgd_pgtable_alloc
,
1491 memblock_clear_nomap(start
, size
);
1493 ret
= __add_pages(nid
, start
>> PAGE_SHIFT
, size
>> PAGE_SHIFT
,
1496 __remove_pgd_mapping(swapper_pg_dir
,
1497 __phys_to_virt(start
), size
);
1501 void arch_remove_memory(int nid
, u64 start
, u64 size
,
1502 struct vmem_altmap
*altmap
)
1504 unsigned long start_pfn
= start
>> PAGE_SHIFT
;
1505 unsigned long nr_pages
= size
>> PAGE_SHIFT
;
1507 __remove_pages(start_pfn
, nr_pages
, altmap
);
1508 __remove_pgd_mapping(swapper_pg_dir
, __phys_to_virt(start
), size
);
1512 * This memory hotplug notifier helps prevent boot memory from being
1513 * inadvertently removed as it blocks pfn range offlining process in
1514 * __offline_pages(). Hence this prevents both offlining as well as
1515 * removal process for boot memory which is initially always online.
1516 * In future if and when boot memory could be removed, this notifier
1517 * should be dropped and free_hotplug_page_range() should handle any
1518 * reserved pages allocated during boot.
1520 static int prevent_bootmem_remove_notifier(struct notifier_block
*nb
,
1521 unsigned long action
, void *data
)
1523 struct mem_section
*ms
;
1524 struct memory_notify
*arg
= data
;
1525 unsigned long end_pfn
= arg
->start_pfn
+ arg
->nr_pages
;
1526 unsigned long pfn
= arg
->start_pfn
;
1528 if ((action
!= MEM_GOING_OFFLINE
) && (action
!= MEM_OFFLINE
))
1531 for (; pfn
< end_pfn
; pfn
+= PAGES_PER_SECTION
) {
1532 unsigned long start
= PFN_PHYS(pfn
);
1533 unsigned long end
= start
+ (1UL << PA_SECTION_SHIFT
);
1535 ms
= __pfn_to_section(pfn
);
1536 if (!early_section(ms
))
1539 if (action
== MEM_GOING_OFFLINE
) {
1541 * Boot memory removal is not supported. Prevent
1542 * it via blocking any attempted offline request
1543 * for the boot memory and just report it.
1545 pr_warn("Boot memory [%lx %lx] offlining attempted\n", start
, end
);
1547 } else if (action
== MEM_OFFLINE
) {
1549 * This should have never happened. Boot memory
1550 * offlining should have been prevented by this
1551 * very notifier. Probably some memory removal
1552 * procedure might have changed which would then
1553 * require further debug.
1555 pr_err("Boot memory [%lx %lx] offlined\n", start
, end
);
1558 * Core memory hotplug does not process a return
1559 * code from the notifier for MEM_OFFLINE events.
1560 * The error condition has been reported. Return
1561 * from here as if ignored.
1569 static struct notifier_block prevent_bootmem_remove_nb
= {
1570 .notifier_call
= prevent_bootmem_remove_notifier
,
1574 * This ensures that boot memory sections on the platform are online
1575 * from early boot. Memory sections could not be prevented from being
1576 * offlined, unless for some reason they are not online to begin with.
1577 * This helps validate the basic assumption on which the above memory
1578 * event notifier works to prevent boot memory section offlining and
1579 * its possible removal.
1581 static void validate_bootmem_online(void)
1583 phys_addr_t start
, end
, addr
;
1584 struct mem_section
*ms
;
1588 * Scanning across all memblock might be expensive
1589 * on some big memory systems. Hence enable this
1590 * validation only with DEBUG_VM.
1592 if (!IS_ENABLED(CONFIG_DEBUG_VM
))
1595 for_each_mem_range(i
, &start
, &end
) {
1596 for (addr
= start
; addr
< end
; addr
+= (1UL << PA_SECTION_SHIFT
)) {
1597 ms
= __pfn_to_section(PHYS_PFN(addr
));
1600 * All memory ranges in the system at this point
1601 * should have been marked as early sections.
1603 WARN_ON(!early_section(ms
));
1606 * Memory notifier mechanism here to prevent boot
1607 * memory offlining depends on the fact that each
1608 * early section memory on the system is initially
1609 * online. Otherwise a given memory section which
1610 * is already offline will be overlooked and can
1611 * be removed completely. Call out such sections.
1613 if (!online_section(ms
))
1614 pr_err("Boot memory [%llx %llx] is offline, can be removed\n",
1615 addr
, addr
+ (1UL << PA_SECTION_SHIFT
));
1620 static int __init
prevent_bootmem_remove_init(void)
1624 if (!IS_ENABLED(CONFIG_MEMORY_HOTREMOVE
))
1627 validate_bootmem_online();
1628 ret
= register_memory_notifier(&prevent_bootmem_remove_nb
);
1630 pr_err("%s: Notifier registration failed %d\n", __func__
, ret
);
1634 early_initcall(prevent_bootmem_remove_init
);