1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_PGALLOC_H
3 #define _ASM_X86_PGALLOC_H
5 #include <linux/threads.h>
6 #include <linux/mm.h> /* for struct page */
7 #include <linux/pagemap.h>
9 static inline int __paravirt_pgd_alloc(struct mm_struct
*mm
) { return 0; }
11 #ifdef CONFIG_PARAVIRT
12 #include <asm/paravirt.h>
14 #define paravirt_pgd_alloc(mm) __paravirt_pgd_alloc(mm)
15 static inline void paravirt_pgd_free(struct mm_struct
*mm
, pgd_t
*pgd
) {}
16 static inline void paravirt_alloc_pte(struct mm_struct
*mm
, unsigned long pfn
) {}
17 static inline void paravirt_alloc_pmd(struct mm_struct
*mm
, unsigned long pfn
) {}
18 static inline void paravirt_alloc_pmd_clone(unsigned long pfn
, unsigned long clonepfn
,
19 unsigned long start
, unsigned long count
) {}
20 static inline void paravirt_alloc_pud(struct mm_struct
*mm
, unsigned long pfn
) {}
21 static inline void paravirt_alloc_p4d(struct mm_struct
*mm
, unsigned long pfn
) {}
22 static inline void paravirt_release_pte(unsigned long pfn
) {}
23 static inline void paravirt_release_pmd(unsigned long pfn
) {}
24 static inline void paravirt_release_pud(unsigned long pfn
) {}
25 static inline void paravirt_release_p4d(unsigned long pfn
) {}
29 * Flags to use when allocating a user page table page.
31 extern gfp_t __userpte_alloc_gfp
;
33 #ifdef CONFIG_PAGE_TABLE_ISOLATION
35 * Instead of one PGD, we acquire two PGDs. Being order-1, it is
36 * both 8k in size and 8k-aligned. That lets us just flip bit 12
37 * in a pointer to swap between the two 4k halves.
39 #define PGD_ALLOCATION_ORDER 1
41 #define PGD_ALLOCATION_ORDER 0
45 * Allocate and free page tables.
47 extern pgd_t
*pgd_alloc(struct mm_struct
*);
48 extern void pgd_free(struct mm_struct
*mm
, pgd_t
*pgd
);
50 extern pte_t
*pte_alloc_one_kernel(struct mm_struct
*, unsigned long);
51 extern pgtable_t
pte_alloc_one(struct mm_struct
*, unsigned long);
53 /* Should really implement gc for free page table pages. This could be
54 done with a reference count in struct page. */
56 static inline void pte_free_kernel(struct mm_struct
*mm
, pte_t
*pte
)
58 BUG_ON((unsigned long)pte
& (PAGE_SIZE
-1));
59 free_page((unsigned long)pte
);
62 static inline void pte_free(struct mm_struct
*mm
, struct page
*pte
)
64 pgtable_page_dtor(pte
);
68 extern void ___pte_free_tlb(struct mmu_gather
*tlb
, struct page
*pte
);
70 static inline void __pte_free_tlb(struct mmu_gather
*tlb
, struct page
*pte
,
71 unsigned long address
)
73 ___pte_free_tlb(tlb
, pte
);
76 static inline void pmd_populate_kernel(struct mm_struct
*mm
,
77 pmd_t
*pmd
, pte_t
*pte
)
79 paravirt_alloc_pte(mm
, __pa(pte
) >> PAGE_SHIFT
);
80 set_pmd(pmd
, __pmd(__pa(pte
) | _PAGE_TABLE
));
83 static inline void pmd_populate(struct mm_struct
*mm
, pmd_t
*pmd
,
86 unsigned long pfn
= page_to_pfn(pte
);
88 paravirt_alloc_pte(mm
, pfn
);
89 set_pmd(pmd
, __pmd(((pteval_t
)pfn
<< PAGE_SHIFT
) | _PAGE_TABLE
));
92 #define pmd_pgtable(pmd) pmd_page(pmd)
94 #if CONFIG_PGTABLE_LEVELS > 2
95 static inline pmd_t
*pmd_alloc_one(struct mm_struct
*mm
, unsigned long addr
)
98 gfp_t gfp
= GFP_KERNEL_ACCOUNT
| __GFP_ZERO
;
101 gfp
&= ~__GFP_ACCOUNT
;
102 page
= alloc_pages(gfp
, 0);
105 if (!pgtable_pmd_page_ctor(page
)) {
106 __free_pages(page
, 0);
109 return (pmd_t
*)page_address(page
);
112 static inline void pmd_free(struct mm_struct
*mm
, pmd_t
*pmd
)
114 BUG_ON((unsigned long)pmd
& (PAGE_SIZE
-1));
115 pgtable_pmd_page_dtor(virt_to_page(pmd
));
116 free_page((unsigned long)pmd
);
119 extern void ___pmd_free_tlb(struct mmu_gather
*tlb
, pmd_t
*pmd
);
121 static inline void __pmd_free_tlb(struct mmu_gather
*tlb
, pmd_t
*pmd
,
122 unsigned long address
)
124 ___pmd_free_tlb(tlb
, pmd
);
127 #ifdef CONFIG_X86_PAE
128 extern void pud_populate(struct mm_struct
*mm
, pud_t
*pudp
, pmd_t
*pmd
);
129 #else /* !CONFIG_X86_PAE */
130 static inline void pud_populate(struct mm_struct
*mm
, pud_t
*pud
, pmd_t
*pmd
)
132 paravirt_alloc_pmd(mm
, __pa(pmd
) >> PAGE_SHIFT
);
133 set_pud(pud
, __pud(_PAGE_TABLE
| __pa(pmd
)));
135 #endif /* CONFIG_X86_PAE */
137 #if CONFIG_PGTABLE_LEVELS > 3
138 static inline void p4d_populate(struct mm_struct
*mm
, p4d_t
*p4d
, pud_t
*pud
)
140 paravirt_alloc_pud(mm
, __pa(pud
) >> PAGE_SHIFT
);
141 set_p4d(p4d
, __p4d(_PAGE_TABLE
| __pa(pud
)));
144 static inline pud_t
*pud_alloc_one(struct mm_struct
*mm
, unsigned long addr
)
146 gfp_t gfp
= GFP_KERNEL_ACCOUNT
;
149 gfp
&= ~__GFP_ACCOUNT
;
150 return (pud_t
*)get_zeroed_page(gfp
);
153 static inline void pud_free(struct mm_struct
*mm
, pud_t
*pud
)
155 BUG_ON((unsigned long)pud
& (PAGE_SIZE
-1));
156 free_page((unsigned long)pud
);
159 extern void ___pud_free_tlb(struct mmu_gather
*tlb
, pud_t
*pud
);
161 static inline void __pud_free_tlb(struct mmu_gather
*tlb
, pud_t
*pud
,
162 unsigned long address
)
164 ___pud_free_tlb(tlb
, pud
);
167 #if CONFIG_PGTABLE_LEVELS > 4
168 static inline void pgd_populate(struct mm_struct
*mm
, pgd_t
*pgd
, p4d_t
*p4d
)
170 paravirt_alloc_p4d(mm
, __pa(p4d
) >> PAGE_SHIFT
);
171 set_pgd(pgd
, __pgd(_PAGE_TABLE
| __pa(p4d
)));
174 static inline p4d_t
*p4d_alloc_one(struct mm_struct
*mm
, unsigned long addr
)
176 gfp_t gfp
= GFP_KERNEL_ACCOUNT
;
179 gfp
&= ~__GFP_ACCOUNT
;
180 return (p4d_t
*)get_zeroed_page(gfp
);
183 static inline void p4d_free(struct mm_struct
*mm
, p4d_t
*p4d
)
185 if (!pgtable_l5_enabled
)
188 BUG_ON((unsigned long)p4d
& (PAGE_SIZE
-1));
189 free_page((unsigned long)p4d
);
192 extern void ___p4d_free_tlb(struct mmu_gather
*tlb
, p4d_t
*p4d
);
194 static inline void __p4d_free_tlb(struct mmu_gather
*tlb
, p4d_t
*p4d
,
195 unsigned long address
)
197 ___p4d_free_tlb(tlb
, p4d
);
200 #endif /* CONFIG_PGTABLE_LEVELS > 4 */
201 #endif /* CONFIG_PGTABLE_LEVELS > 3 */
202 #endif /* CONFIG_PGTABLE_LEVELS > 2 */
204 #endif /* _ASM_X86_PGALLOC_H */