1 // SPDX-License-Identifier: GPL-2.0
4 #include <linux/hugetlb.h>
5 #include <asm/pgalloc.h>
6 #include <asm/pgtable.h>
8 #include <asm/fixmap.h>
11 #ifdef CONFIG_DYNAMIC_PHYSICAL_MASK
12 phys_addr_t physical_mask __ro_after_init
= (1ULL << __PHYSICAL_MASK_SHIFT
) - 1;
13 EXPORT_SYMBOL(physical_mask
);
16 #define PGALLOC_GFP (GFP_KERNEL_ACCOUNT | __GFP_ZERO)
19 #define PGALLOC_USER_GFP __GFP_HIGHMEM
21 #define PGALLOC_USER_GFP 0
24 gfp_t __userpte_alloc_gfp
= PGALLOC_GFP
| PGALLOC_USER_GFP
;
26 pte_t
*pte_alloc_one_kernel(struct mm_struct
*mm
)
28 return (pte_t
*)__get_free_page(PGALLOC_GFP
& ~__GFP_ACCOUNT
);
31 pgtable_t
pte_alloc_one(struct mm_struct
*mm
)
35 pte
= alloc_pages(__userpte_alloc_gfp
, 0);
38 if (!pgtable_page_ctor(pte
)) {
45 static int __init
setup_userpte(char *arg
)
51 * "userpte=nohigh" disables allocation of user pagetables in
54 if (strcmp(arg
, "nohigh") == 0)
55 __userpte_alloc_gfp
&= ~__GFP_HIGHMEM
;
60 early_param("userpte", setup_userpte
);
62 void ___pte_free_tlb(struct mmu_gather
*tlb
, struct page
*pte
)
64 pgtable_page_dtor(pte
);
65 paravirt_release_pte(page_to_pfn(pte
));
66 paravirt_tlb_remove_table(tlb
, pte
);
69 #if CONFIG_PGTABLE_LEVELS > 2
70 void ___pmd_free_tlb(struct mmu_gather
*tlb
, pmd_t
*pmd
)
72 struct page
*page
= virt_to_page(pmd
);
73 paravirt_release_pmd(__pa(pmd
) >> PAGE_SHIFT
);
75 * NOTE! For PAE, any changes to the top page-directory-pointer-table
76 * entries need a full cr3 reload to flush.
79 tlb
->need_flush_all
= 1;
81 pgtable_pmd_page_dtor(page
);
82 paravirt_tlb_remove_table(tlb
, page
);
85 #if CONFIG_PGTABLE_LEVELS > 3
86 void ___pud_free_tlb(struct mmu_gather
*tlb
, pud_t
*pud
)
88 paravirt_release_pud(__pa(pud
) >> PAGE_SHIFT
);
89 paravirt_tlb_remove_table(tlb
, virt_to_page(pud
));
92 #if CONFIG_PGTABLE_LEVELS > 4
93 void ___p4d_free_tlb(struct mmu_gather
*tlb
, p4d_t
*p4d
)
95 paravirt_release_p4d(__pa(p4d
) >> PAGE_SHIFT
);
96 paravirt_tlb_remove_table(tlb
, virt_to_page(p4d
));
98 #endif /* CONFIG_PGTABLE_LEVELS > 4 */
99 #endif /* CONFIG_PGTABLE_LEVELS > 3 */
100 #endif /* CONFIG_PGTABLE_LEVELS > 2 */
102 static inline void pgd_list_add(pgd_t
*pgd
)
104 struct page
*page
= virt_to_page(pgd
);
106 list_add(&page
->lru
, &pgd_list
);
109 static inline void pgd_list_del(pgd_t
*pgd
)
111 struct page
*page
= virt_to_page(pgd
);
113 list_del(&page
->lru
);
116 #define UNSHARED_PTRS_PER_PGD \
117 (SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD)
118 #define MAX_UNSHARED_PTRS_PER_PGD \
119 max_t(size_t, KERNEL_PGD_BOUNDARY, PTRS_PER_PGD)
122 static void pgd_set_mm(pgd_t
*pgd
, struct mm_struct
*mm
)
124 virt_to_page(pgd
)->pt_mm
= mm
;
127 struct mm_struct
*pgd_page_get_mm(struct page
*page
)
132 static void pgd_ctor(struct mm_struct
*mm
, pgd_t
*pgd
)
134 /* If the pgd points to a shared pagetable level (either the
135 ptes in non-PAE, or shared PMD in PAE), then just copy the
136 references from swapper_pg_dir. */
137 if (CONFIG_PGTABLE_LEVELS
== 2 ||
138 (CONFIG_PGTABLE_LEVELS
== 3 && SHARED_KERNEL_PMD
) ||
139 CONFIG_PGTABLE_LEVELS
>= 4) {
140 clone_pgd_range(pgd
+ KERNEL_PGD_BOUNDARY
,
141 swapper_pg_dir
+ KERNEL_PGD_BOUNDARY
,
145 /* list required to sync kernel mapping updates */
146 if (!SHARED_KERNEL_PMD
) {
152 static void pgd_dtor(pgd_t
*pgd
)
154 if (SHARED_KERNEL_PMD
)
157 spin_lock(&pgd_lock
);
159 spin_unlock(&pgd_lock
);
163 * List of all pgd's needed for non-PAE so it can invalidate entries
164 * in both cached and uncached pgd's; not needed for PAE since the
165 * kernel pmd is shared. If PAE were not to share the pmd a similar
166 * tactic would be needed. This is essentially codepath-based locking
167 * against pageattr.c; it is the unique case in which a valid change
168 * of kernel pagetables can't be lazily synchronized by vmalloc faults.
169 * vmalloc faults work because attached pagetables are never freed.
173 #ifdef CONFIG_X86_PAE
175 * In PAE mode, we need to do a cr3 reload (=tlb flush) when
176 * updating the top-level pagetable entries to guarantee the
177 * processor notices the update. Since this is expensive, and
178 * all 4 top-level entries are used almost immediately in a
179 * new process's life, we just pre-populate them here.
181 * Also, if we're in a paravirt environment where the kernel pmd is
182 * not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate
183 * and initialize the kernel pmds here.
185 #define PREALLOCATED_PMDS UNSHARED_PTRS_PER_PGD
186 #define MAX_PREALLOCATED_PMDS MAX_UNSHARED_PTRS_PER_PGD
189 * We allocate separate PMDs for the kernel part of the user page-table
190 * when PTI is enabled. We need them to map the per-process LDT into the
191 * user-space page-table.
193 #define PREALLOCATED_USER_PMDS (static_cpu_has(X86_FEATURE_PTI) ? \
195 #define MAX_PREALLOCATED_USER_PMDS KERNEL_PGD_PTRS
197 void pud_populate(struct mm_struct
*mm
, pud_t
*pudp
, pmd_t
*pmd
)
199 paravirt_alloc_pmd(mm
, __pa(pmd
) >> PAGE_SHIFT
);
201 /* Note: almost everything apart from _PAGE_PRESENT is
202 reserved at the pmd (PDPT) level. */
203 set_pud(pudp
, __pud(__pa(pmd
) | _PAGE_PRESENT
));
206 * According to Intel App note "TLBs, Paging-Structure Caches,
207 * and Their Invalidation", April 2007, document 317080-001,
208 * section 8.1: in PAE mode we explicitly have to flush the
209 * TLB via cr3 if the top-level pgd is changed...
213 #else /* !CONFIG_X86_PAE */
215 /* No need to prepopulate any pagetable entries in non-PAE modes. */
216 #define PREALLOCATED_PMDS 0
217 #define MAX_PREALLOCATED_PMDS 0
218 #define PREALLOCATED_USER_PMDS 0
219 #define MAX_PREALLOCATED_USER_PMDS 0
220 #endif /* CONFIG_X86_PAE */
222 static void free_pmds(struct mm_struct
*mm
, pmd_t
*pmds
[], int count
)
226 for (i
= 0; i
< count
; i
++)
228 pgtable_pmd_page_dtor(virt_to_page(pmds
[i
]));
229 free_page((unsigned long)pmds
[i
]);
234 static int preallocate_pmds(struct mm_struct
*mm
, pmd_t
*pmds
[], int count
)
238 gfp_t gfp
= PGALLOC_GFP
;
241 gfp
&= ~__GFP_ACCOUNT
;
243 for (i
= 0; i
< count
; i
++) {
244 pmd_t
*pmd
= (pmd_t
*)__get_free_page(gfp
);
247 if (pmd
&& !pgtable_pmd_page_ctor(virt_to_page(pmd
))) {
248 free_page((unsigned long)pmd
);
258 free_pmds(mm
, pmds
, count
);
266 * Mop up any pmd pages which may still be attached to the pgd.
267 * Normally they will be freed by munmap/exit_mmap, but any pmd we
268 * preallocate which never got a corresponding vma will need to be
271 static void mop_up_one_pmd(struct mm_struct
*mm
, pgd_t
*pgdp
)
275 if (pgd_val(pgd
) != 0) {
276 pmd_t
*pmd
= (pmd_t
*)pgd_page_vaddr(pgd
);
280 paravirt_release_pmd(pgd_val(pgd
) >> PAGE_SHIFT
);
286 static void pgd_mop_up_pmds(struct mm_struct
*mm
, pgd_t
*pgdp
)
290 for (i
= 0; i
< PREALLOCATED_PMDS
; i
++)
291 mop_up_one_pmd(mm
, &pgdp
[i
]);
293 #ifdef CONFIG_PAGE_TABLE_ISOLATION
295 if (!static_cpu_has(X86_FEATURE_PTI
))
298 pgdp
= kernel_to_user_pgdp(pgdp
);
300 for (i
= 0; i
< PREALLOCATED_USER_PMDS
; i
++)
301 mop_up_one_pmd(mm
, &pgdp
[i
+ KERNEL_PGD_BOUNDARY
]);
305 static void pgd_prepopulate_pmd(struct mm_struct
*mm
, pgd_t
*pgd
, pmd_t
*pmds
[])
311 if (PREALLOCATED_PMDS
== 0) /* Work around gcc-3.4.x bug */
314 p4d
= p4d_offset(pgd
, 0);
315 pud
= pud_offset(p4d
, 0);
317 for (i
= 0; i
< PREALLOCATED_PMDS
; i
++, pud
++) {
318 pmd_t
*pmd
= pmds
[i
];
320 if (i
>= KERNEL_PGD_BOUNDARY
)
321 memcpy(pmd
, (pmd_t
*)pgd_page_vaddr(swapper_pg_dir
[i
]),
322 sizeof(pmd_t
) * PTRS_PER_PMD
);
324 pud_populate(mm
, pud
, pmd
);
328 #ifdef CONFIG_PAGE_TABLE_ISOLATION
329 static void pgd_prepopulate_user_pmd(struct mm_struct
*mm
,
330 pgd_t
*k_pgd
, pmd_t
*pmds
[])
332 pgd_t
*s_pgd
= kernel_to_user_pgdp(swapper_pg_dir
);
333 pgd_t
*u_pgd
= kernel_to_user_pgdp(k_pgd
);
338 u_p4d
= p4d_offset(u_pgd
, 0);
339 u_pud
= pud_offset(u_p4d
, 0);
341 s_pgd
+= KERNEL_PGD_BOUNDARY
;
342 u_pud
+= KERNEL_PGD_BOUNDARY
;
344 for (i
= 0; i
< PREALLOCATED_USER_PMDS
; i
++, u_pud
++, s_pgd
++) {
345 pmd_t
*pmd
= pmds
[i
];
347 memcpy(pmd
, (pmd_t
*)pgd_page_vaddr(*s_pgd
),
348 sizeof(pmd_t
) * PTRS_PER_PMD
);
350 pud_populate(mm
, u_pud
, pmd
);
355 static void pgd_prepopulate_user_pmd(struct mm_struct
*mm
,
356 pgd_t
*k_pgd
, pmd_t
*pmds
[])
361 * Xen paravirt assumes pgd table should be in one page. 64 bit kernel also
362 * assumes that pgd should be in one page.
364 * But kernel with PAE paging that is not running as a Xen domain
365 * only needs to allocate 32 bytes for pgd instead of one page.
367 #ifdef CONFIG_X86_PAE
369 #include <linux/slab.h>
371 #define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
374 static struct kmem_cache
*pgd_cache
;
376 void __init
pgd_cache_init(void)
379 * When PAE kernel is running as a Xen domain, it does not use
380 * shared kernel pmd. And this requires a whole page for pgd.
382 if (!SHARED_KERNEL_PMD
)
386 * when PAE kernel is not running as a Xen domain, it uses
387 * shared kernel pmd. Shared kernel pmd does not require a whole
388 * page for pgd. We are able to just allocate a 32-byte for pgd.
389 * During boot time, we create a 32-byte slab for pgd table allocation.
391 pgd_cache
= kmem_cache_create("pgd_cache", PGD_SIZE
, PGD_ALIGN
,
395 static inline pgd_t
*_pgd_alloc(void)
398 * If no SHARED_KERNEL_PMD, PAE kernel is running as a Xen domain.
399 * We allocate one page for pgd.
401 if (!SHARED_KERNEL_PMD
)
402 return (pgd_t
*)__get_free_pages(PGALLOC_GFP
,
403 PGD_ALLOCATION_ORDER
);
406 * Now PAE kernel is not running as a Xen domain. We can allocate
407 * a 32-byte slab for pgd to save memory space.
409 return kmem_cache_alloc(pgd_cache
, PGALLOC_GFP
);
412 static inline void _pgd_free(pgd_t
*pgd
)
414 if (!SHARED_KERNEL_PMD
)
415 free_pages((unsigned long)pgd
, PGD_ALLOCATION_ORDER
);
417 kmem_cache_free(pgd_cache
, pgd
);
421 void __init
pgd_cache_init(void)
425 static inline pgd_t
*_pgd_alloc(void)
427 return (pgd_t
*)__get_free_pages(PGALLOC_GFP
, PGD_ALLOCATION_ORDER
);
430 static inline void _pgd_free(pgd_t
*pgd
)
432 free_pages((unsigned long)pgd
, PGD_ALLOCATION_ORDER
);
434 #endif /* CONFIG_X86_PAE */
436 pgd_t
*pgd_alloc(struct mm_struct
*mm
)
439 pmd_t
*u_pmds
[MAX_PREALLOCATED_USER_PMDS
];
440 pmd_t
*pmds
[MAX_PREALLOCATED_PMDS
];
449 if (preallocate_pmds(mm
, pmds
, PREALLOCATED_PMDS
) != 0)
452 if (preallocate_pmds(mm
, u_pmds
, PREALLOCATED_USER_PMDS
) != 0)
455 if (paravirt_pgd_alloc(mm
) != 0)
456 goto out_free_user_pmds
;
459 * Make sure that pre-populating the pmds is atomic with
460 * respect to anything walking the pgd_list, so that they
461 * never see a partially populated pgd.
463 spin_lock(&pgd_lock
);
466 pgd_prepopulate_pmd(mm
, pgd
, pmds
);
467 pgd_prepopulate_user_pmd(mm
, pgd
, u_pmds
);
469 spin_unlock(&pgd_lock
);
474 free_pmds(mm
, u_pmds
, PREALLOCATED_USER_PMDS
);
476 free_pmds(mm
, pmds
, PREALLOCATED_PMDS
);
483 void pgd_free(struct mm_struct
*mm
, pgd_t
*pgd
)
485 pgd_mop_up_pmds(mm
, pgd
);
487 paravirt_pgd_free(mm
, pgd
);
492 * Used to set accessed or dirty bits in the page table entries
493 * on other architectures. On x86, the accessed and dirty bits
494 * are tracked by hardware. However, do_wp_page calls this function
495 * to also make the pte writeable at the same time the dirty bit is
496 * set. In that case we do actually need to write the PTE.
498 int ptep_set_access_flags(struct vm_area_struct
*vma
,
499 unsigned long address
, pte_t
*ptep
,
500 pte_t entry
, int dirty
)
502 int changed
= !pte_same(*ptep
, entry
);
504 if (changed
&& dirty
)
505 set_pte(ptep
, entry
);
510 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
511 int pmdp_set_access_flags(struct vm_area_struct
*vma
,
512 unsigned long address
, pmd_t
*pmdp
,
513 pmd_t entry
, int dirty
)
515 int changed
= !pmd_same(*pmdp
, entry
);
517 VM_BUG_ON(address
& ~HPAGE_PMD_MASK
);
519 if (changed
&& dirty
) {
520 set_pmd(pmdp
, entry
);
522 * We had a write-protection fault here and changed the pmd
523 * to to more permissive. No need to flush the TLB for that,
524 * #PF is architecturally guaranteed to do that and in the
525 * worst-case we'll generate a spurious fault.
532 int pudp_set_access_flags(struct vm_area_struct
*vma
, unsigned long address
,
533 pud_t
*pudp
, pud_t entry
, int dirty
)
535 int changed
= !pud_same(*pudp
, entry
);
537 VM_BUG_ON(address
& ~HPAGE_PUD_MASK
);
539 if (changed
&& dirty
) {
540 set_pud(pudp
, entry
);
542 * We had a write-protection fault here and changed the pud
543 * to to more permissive. No need to flush the TLB for that,
544 * #PF is architecturally guaranteed to do that and in the
545 * worst-case we'll generate a spurious fault.
553 int ptep_test_and_clear_young(struct vm_area_struct
*vma
,
554 unsigned long addr
, pte_t
*ptep
)
558 if (pte_young(*ptep
))
559 ret
= test_and_clear_bit(_PAGE_BIT_ACCESSED
,
560 (unsigned long *) &ptep
->pte
);
565 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
566 int pmdp_test_and_clear_young(struct vm_area_struct
*vma
,
567 unsigned long addr
, pmd_t
*pmdp
)
571 if (pmd_young(*pmdp
))
572 ret
= test_and_clear_bit(_PAGE_BIT_ACCESSED
,
573 (unsigned long *)pmdp
);
577 int pudp_test_and_clear_young(struct vm_area_struct
*vma
,
578 unsigned long addr
, pud_t
*pudp
)
582 if (pud_young(*pudp
))
583 ret
= test_and_clear_bit(_PAGE_BIT_ACCESSED
,
584 (unsigned long *)pudp
);
590 int ptep_clear_flush_young(struct vm_area_struct
*vma
,
591 unsigned long address
, pte_t
*ptep
)
594 * On x86 CPUs, clearing the accessed bit without a TLB flush
595 * doesn't cause data corruption. [ It could cause incorrect
596 * page aging and the (mistaken) reclaim of hot pages, but the
597 * chance of that should be relatively low. ]
599 * So as a performance optimization don't flush the TLB when
600 * clearing the accessed bit, it will eventually be flushed by
601 * a context switch or a VM operation anyway. [ In the rare
602 * event of it not getting flushed for a long time the delay
603 * shouldn't really matter because there's no real memory
604 * pressure for swapout to react to. ]
606 return ptep_test_and_clear_young(vma
, address
, ptep
);
609 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
610 int pmdp_clear_flush_young(struct vm_area_struct
*vma
,
611 unsigned long address
, pmd_t
*pmdp
)
615 VM_BUG_ON(address
& ~HPAGE_PMD_MASK
);
617 young
= pmdp_test_and_clear_young(vma
, address
, pmdp
);
619 flush_tlb_range(vma
, address
, address
+ HPAGE_PMD_SIZE
);
626 * reserve_top_address - reserves a hole in the top of kernel address space
627 * @reserve - size of hole to reserve
629 * Can be used to relocate the fixmap area and poke a hole in the top
630 * of kernel address space to make room for a hypervisor.
632 void __init
reserve_top_address(unsigned long reserve
)
635 BUG_ON(fixmaps_set
> 0);
636 __FIXADDR_TOP
= round_down(-reserve
, 1 << PMD_SHIFT
) - PAGE_SIZE
;
637 printk(KERN_INFO
"Reserving virtual address space above 0x%08lx (rounded to 0x%08lx)\n",
638 -reserve
, __FIXADDR_TOP
+ PAGE_SIZE
);
644 void __native_set_fixmap(enum fixed_addresses idx
, pte_t pte
)
646 unsigned long address
= __fix_to_virt(idx
);
650 * Ensure that the static initial page tables are covering the
653 BUILD_BUG_ON(__end_of_permanent_fixed_addresses
>
654 (FIXMAP_PMD_NUM
* PTRS_PER_PTE
));
657 if (idx
>= __end_of_fixed_addresses
) {
661 set_pte_vaddr(address
, pte
);
665 void native_set_fixmap(enum fixed_addresses idx
, phys_addr_t phys
,
668 /* Sanitize 'prot' against any unsupported bits: */
669 pgprot_val(flags
) &= __default_kernel_pte_mask
;
671 __native_set_fixmap(idx
, pfn_pte(phys
>> PAGE_SHIFT
, flags
));
674 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
675 #ifdef CONFIG_X86_5LEVEL
677 * p4d_set_huge - setup kernel P4D mapping
679 * No 512GB pages yet -- always return 0
681 int p4d_set_huge(p4d_t
*p4d
, phys_addr_t addr
, pgprot_t prot
)
687 * p4d_clear_huge - clear kernel P4D mapping when it is set
689 * No 512GB pages yet -- always return 0
691 int p4d_clear_huge(p4d_t
*p4d
)
698 * pud_set_huge - setup kernel PUD mapping
700 * MTRRs can override PAT memory types with 4KiB granularity. Therefore, this
701 * function sets up a huge page only if any of the following conditions are met:
703 * - MTRRs are disabled, or
705 * - MTRRs are enabled and the range is completely covered by a single MTRR, or
707 * - MTRRs are enabled and the corresponding MTRR memory type is WB, which
708 * has no effect on the requested PAT memory type.
710 * Callers should try to decrease page size (1GB -> 2MB -> 4K) if the bigger
711 * page mapping attempt fails.
713 * Returns 1 on success and 0 on failure.
715 int pud_set_huge(pud_t
*pud
, phys_addr_t addr
, pgprot_t prot
)
719 mtrr
= mtrr_type_lookup(addr
, addr
+ PUD_SIZE
, &uniform
);
720 if ((mtrr
!= MTRR_TYPE_INVALID
) && (!uniform
) &&
721 (mtrr
!= MTRR_TYPE_WRBACK
))
724 /* Bail out if we are we on a populated non-leaf entry: */
725 if (pud_present(*pud
) && !pud_huge(*pud
))
728 prot
= pgprot_4k_2_large(prot
);
730 set_pte((pte_t
*)pud
, pfn_pte(
731 (u64
)addr
>> PAGE_SHIFT
,
732 __pgprot(pgprot_val(prot
) | _PAGE_PSE
)));
738 * pmd_set_huge - setup kernel PMD mapping
740 * See text over pud_set_huge() above.
742 * Returns 1 on success and 0 on failure.
744 int pmd_set_huge(pmd_t
*pmd
, phys_addr_t addr
, pgprot_t prot
)
748 mtrr
= mtrr_type_lookup(addr
, addr
+ PMD_SIZE
, &uniform
);
749 if ((mtrr
!= MTRR_TYPE_INVALID
) && (!uniform
) &&
750 (mtrr
!= MTRR_TYPE_WRBACK
)) {
751 pr_warn_once("%s: Cannot satisfy [mem %#010llx-%#010llx] with a huge-page mapping due to MTRR override.\n",
752 __func__
, addr
, addr
+ PMD_SIZE
);
756 /* Bail out if we are we on a populated non-leaf entry: */
757 if (pmd_present(*pmd
) && !pmd_huge(*pmd
))
760 prot
= pgprot_4k_2_large(prot
);
762 set_pte((pte_t
*)pmd
, pfn_pte(
763 (u64
)addr
>> PAGE_SHIFT
,
764 __pgprot(pgprot_val(prot
) | _PAGE_PSE
)));
770 * pud_clear_huge - clear kernel PUD mapping when it is set
772 * Returns 1 on success and 0 on failure (no PUD map is found).
774 int pud_clear_huge(pud_t
*pud
)
776 if (pud_large(*pud
)) {
785 * pmd_clear_huge - clear kernel PMD mapping when it is set
787 * Returns 1 on success and 0 on failure (no PMD map is found).
789 int pmd_clear_huge(pmd_t
*pmd
)
791 if (pmd_large(*pmd
)) {
800 * Until we support 512GB pages, skip them in the vmap area.
802 int p4d_free_pud_page(p4d_t
*p4d
, unsigned long addr
)
809 * pud_free_pmd_page - Clear pud entry and free pmd page.
810 * @pud: Pointer to a PUD.
811 * @addr: Virtual address associated with pud.
813 * Context: The pud range has been unmapped and TLB purged.
814 * Return: 1 if clearing the entry succeeded. 0 otherwise.
816 * NOTE: Callers must allow a single page allocation.
818 int pud_free_pmd_page(pud_t
*pud
, unsigned long addr
)
824 pmd
= (pmd_t
*)pud_page_vaddr(*pud
);
825 pmd_sv
= (pmd_t
*)__get_free_page(GFP_KERNEL
);
829 for (i
= 0; i
< PTRS_PER_PMD
; i
++) {
831 if (!pmd_none(pmd
[i
]))
837 /* INVLPG to clear all paging-structure caches */
838 flush_tlb_kernel_range(addr
, addr
+ PAGE_SIZE
-1);
840 for (i
= 0; i
< PTRS_PER_PMD
; i
++) {
841 if (!pmd_none(pmd_sv
[i
])) {
842 pte
= (pte_t
*)pmd_page_vaddr(pmd_sv
[i
]);
843 free_page((unsigned long)pte
);
847 free_page((unsigned long)pmd_sv
);
848 free_page((unsigned long)pmd
);
854 * pmd_free_pte_page - Clear pmd entry and free pte page.
855 * @pmd: Pointer to a PMD.
856 * @addr: Virtual address associated with pmd.
858 * Context: The pmd range has been unmapped and TLB purged.
859 * Return: 1 if clearing the entry succeeded. 0 otherwise.
861 int pmd_free_pte_page(pmd_t
*pmd
, unsigned long addr
)
865 pte
= (pte_t
*)pmd_page_vaddr(*pmd
);
868 /* INVLPG to clear all paging-structure caches */
869 flush_tlb_kernel_range(addr
, addr
+ PAGE_SIZE
-1);
871 free_page((unsigned long)pte
);
876 #else /* !CONFIG_X86_64 */
878 int pud_free_pmd_page(pud_t
*pud
, unsigned long addr
)
880 return pud_none(*pud
);
884 * Disable free page handling on x86-PAE. This assures that ioremap()
885 * does not update sync'd pmd entries. See vmalloc_sync_one().
887 int pmd_free_pte_page(pmd_t
*pmd
, unsigned long addr
)
889 return pmd_none(*pmd
);
892 #endif /* CONFIG_X86_64 */
893 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */