1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_PGALLOC_H
3 #define _ASM_X86_PGALLOC_H
5 #include <linux/threads.h>
6 #include <linux/mm.h> /* for struct page */
7 #include <linux/pagemap.h>
9 static inline int __paravirt_pgd_alloc(struct mm_struct
*mm
) { return 0; }
11 #ifdef CONFIG_PARAVIRT
12 #include <asm/paravirt.h>
14 #define paravirt_pgd_alloc(mm) __paravirt_pgd_alloc(mm)
15 static inline void paravirt_pgd_free(struct mm_struct
*mm
, pgd_t
*pgd
) {}
16 static inline void paravirt_alloc_pte(struct mm_struct
*mm
, unsigned long pfn
) {}
17 static inline void paravirt_alloc_pmd(struct mm_struct
*mm
, unsigned long pfn
) {}
18 static inline void paravirt_alloc_pmd_clone(unsigned long pfn
, unsigned long clonepfn
,
19 unsigned long start
, unsigned long count
) {}
20 static inline void paravirt_alloc_pud(struct mm_struct
*mm
, unsigned long pfn
) {}
21 static inline void paravirt_alloc_p4d(struct mm_struct
*mm
, unsigned long pfn
) {}
22 static inline void paravirt_release_pte(unsigned long pfn
) {}
23 static inline void paravirt_release_pmd(unsigned long pfn
) {}
24 static inline void paravirt_release_pud(unsigned long pfn
) {}
25 static inline void paravirt_release_p4d(unsigned long pfn
) {}
29 * Flags to use when allocating a user page table page.
31 extern gfp_t __userpte_alloc_gfp
;
34 * Allocate and free page tables.
36 extern pgd_t
*pgd_alloc(struct mm_struct
*);
37 extern void pgd_free(struct mm_struct
*mm
, pgd_t
*pgd
);
39 extern pte_t
*pte_alloc_one_kernel(struct mm_struct
*, unsigned long);
40 extern pgtable_t
pte_alloc_one(struct mm_struct
*, unsigned long);
42 /* Should really implement gc for free page table pages. This could be
43 done with a reference count in struct page. */
45 static inline void pte_free_kernel(struct mm_struct
*mm
, pte_t
*pte
)
47 BUG_ON((unsigned long)pte
& (PAGE_SIZE
-1));
48 free_page((unsigned long)pte
);
51 static inline void pte_free(struct mm_struct
*mm
, struct page
*pte
)
53 pgtable_page_dtor(pte
);
57 extern void ___pte_free_tlb(struct mmu_gather
*tlb
, struct page
*pte
);
59 static inline void __pte_free_tlb(struct mmu_gather
*tlb
, struct page
*pte
,
60 unsigned long address
)
62 ___pte_free_tlb(tlb
, pte
);
65 static inline void pmd_populate_kernel(struct mm_struct
*mm
,
66 pmd_t
*pmd
, pte_t
*pte
)
68 paravirt_alloc_pte(mm
, __pa(pte
) >> PAGE_SHIFT
);
69 set_pmd(pmd
, __pmd(__pa(pte
) | _PAGE_TABLE
));
72 static inline void pmd_populate(struct mm_struct
*mm
, pmd_t
*pmd
,
75 unsigned long pfn
= page_to_pfn(pte
);
77 paravirt_alloc_pte(mm
, pfn
);
78 set_pmd(pmd
, __pmd(((pteval_t
)pfn
<< PAGE_SHIFT
) | _PAGE_TABLE
));
81 #define pmd_pgtable(pmd) pmd_page(pmd)
83 #if CONFIG_PGTABLE_LEVELS > 2
84 static inline pmd_t
*pmd_alloc_one(struct mm_struct
*mm
, unsigned long addr
)
87 gfp_t gfp
= GFP_KERNEL_ACCOUNT
| __GFP_ZERO
;
90 gfp
&= ~__GFP_ACCOUNT
;
91 page
= alloc_pages(gfp
, 0);
94 if (!pgtable_pmd_page_ctor(page
)) {
95 __free_pages(page
, 0);
98 return (pmd_t
*)page_address(page
);
101 static inline void pmd_free(struct mm_struct
*mm
, pmd_t
*pmd
)
103 BUG_ON((unsigned long)pmd
& (PAGE_SIZE
-1));
104 pgtable_pmd_page_dtor(virt_to_page(pmd
));
105 free_page((unsigned long)pmd
);
108 extern void ___pmd_free_tlb(struct mmu_gather
*tlb
, pmd_t
*pmd
);
110 static inline void __pmd_free_tlb(struct mmu_gather
*tlb
, pmd_t
*pmd
,
111 unsigned long address
)
113 ___pmd_free_tlb(tlb
, pmd
);
116 #ifdef CONFIG_X86_PAE
117 extern void pud_populate(struct mm_struct
*mm
, pud_t
*pudp
, pmd_t
*pmd
);
118 #else /* !CONFIG_X86_PAE */
119 static inline void pud_populate(struct mm_struct
*mm
, pud_t
*pud
, pmd_t
*pmd
)
121 paravirt_alloc_pmd(mm
, __pa(pmd
) >> PAGE_SHIFT
);
122 set_pud(pud
, __pud(_PAGE_TABLE
| __pa(pmd
)));
124 #endif /* CONFIG_X86_PAE */
126 #if CONFIG_PGTABLE_LEVELS > 3
127 static inline void p4d_populate(struct mm_struct
*mm
, p4d_t
*p4d
, pud_t
*pud
)
129 paravirt_alloc_pud(mm
, __pa(pud
) >> PAGE_SHIFT
);
130 set_p4d(p4d
, __p4d(_PAGE_TABLE
| __pa(pud
)));
133 static inline pud_t
*pud_alloc_one(struct mm_struct
*mm
, unsigned long addr
)
135 gfp_t gfp
= GFP_KERNEL_ACCOUNT
;
138 gfp
&= ~__GFP_ACCOUNT
;
139 return (pud_t
*)get_zeroed_page(gfp
);
142 static inline void pud_free(struct mm_struct
*mm
, pud_t
*pud
)
144 BUG_ON((unsigned long)pud
& (PAGE_SIZE
-1));
145 free_page((unsigned long)pud
);
148 extern void ___pud_free_tlb(struct mmu_gather
*tlb
, pud_t
*pud
);
150 static inline void __pud_free_tlb(struct mmu_gather
*tlb
, pud_t
*pud
,
151 unsigned long address
)
153 ___pud_free_tlb(tlb
, pud
);
156 #if CONFIG_PGTABLE_LEVELS > 4
157 static inline void pgd_populate(struct mm_struct
*mm
, pgd_t
*pgd
, p4d_t
*p4d
)
159 paravirt_alloc_p4d(mm
, __pa(p4d
) >> PAGE_SHIFT
);
160 set_pgd(pgd
, __pgd(_PAGE_TABLE
| __pa(p4d
)));
163 static inline p4d_t
*p4d_alloc_one(struct mm_struct
*mm
, unsigned long addr
)
165 gfp_t gfp
= GFP_KERNEL_ACCOUNT
;
168 gfp
&= ~__GFP_ACCOUNT
;
169 return (p4d_t
*)get_zeroed_page(gfp
);
172 static inline void p4d_free(struct mm_struct
*mm
, p4d_t
*p4d
)
174 BUG_ON((unsigned long)p4d
& (PAGE_SIZE
-1));
175 free_page((unsigned long)p4d
);
178 extern void ___p4d_free_tlb(struct mmu_gather
*tlb
, p4d_t
*p4d
);
180 static inline void __p4d_free_tlb(struct mmu_gather
*tlb
, p4d_t
*p4d
,
181 unsigned long address
)
183 ___p4d_free_tlb(tlb
, p4d
);
186 #endif /* CONFIG_PGTABLE_LEVELS > 4 */
187 #endif /* CONFIG_PGTABLE_LEVELS > 3 */
188 #endif /* CONFIG_PGTABLE_LEVELS > 2 */
190 #endif /* _ASM_X86_PGALLOC_H */