[mirror_ubuntu-bionic-kernel.git] / arch / x86 / include / asm / pgalloc.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PGALLOC_H
#define _ASM_X86_PGALLOC_H

#include <linux/threads.h>
#include <linux/mm.h>		/* for struct page */
#include <linux/pagemap.h>

static inline int  __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; }

#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#else
#define paravirt_pgd_alloc(mm)	__paravirt_pgd_alloc(mm)
static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) {}
static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn)	{}
static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)	{}
static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn,
					    unsigned long start, unsigned long count) {}
static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn)	{}
static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn)	{}
static inline void paravirt_release_pte(unsigned long pfn) {}
static inline void paravirt_release_pmd(unsigned long pfn) {}
static inline void paravirt_release_pud(unsigned long pfn) {}
static inline void paravirt_release_p4d(unsigned long pfn) {}
#endif

/*
 * Flags to use when allocating a user page table page.
 */
extern gfp_t __userpte_alloc_gfp;

/*
 * Allocate and free page tables.
 */
extern pgd_t *pgd_alloc(struct mm_struct *);
extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);

extern pte_t *pte_alloc_one_kernel(struct mm_struct *, unsigned long);
extern pgtable_t pte_alloc_one(struct mm_struct *, unsigned long);

/* Should really implement gc for free page table pages. This could be
   done with a reference count in struct page. */

static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
{
	BUG_ON((unsigned long)pte & (PAGE_SIZE-1));
	free_page((unsigned long)pte);
}

static inline void pte_free(struct mm_struct *mm, struct page *pte)
{
	pgtable_page_dtor(pte);
	__free_page(pte);
}

extern void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte);

static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte,
				  unsigned long address)
{
	___pte_free_tlb(tlb, pte);
}

static inline void pmd_populate_kernel(struct mm_struct *mm,
				       pmd_t *pmd, pte_t *pte)
{
	paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT);
	set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE));
}

static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
				struct page *pte)
{
	unsigned long pfn = page_to_pfn(pte);

	paravirt_alloc_pte(mm, pfn);
	set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE));
}

#define pmd_pgtable(pmd) pmd_page(pmd)

#if CONFIG_PGTABLE_LEVELS > 2
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
{
	struct page *page;
	gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO;

	if (mm == &init_mm)
		gfp &= ~__GFP_ACCOUNT;
	page = alloc_pages(gfp, 0);
	if (!page)
		return NULL;
	if (!pgtable_pmd_page_ctor(page)) {
		__free_pages(page, 0);
		return NULL;
	}
	return (pmd_t *)page_address(page);
}

static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
{
	BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
	pgtable_pmd_page_dtor(virt_to_page(pmd));
	free_page((unsigned long)pmd);
}

extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd);

static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
				  unsigned long address)
{
	___pmd_free_tlb(tlb, pmd);
}

#ifdef CONFIG_X86_PAE
extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd);
#else	/* !CONFIG_X86_PAE */
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
{
	paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
	set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd)));
}
#endif	/* CONFIG_X86_PAE */

#if CONFIG_PGTABLE_LEVELS > 3
static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)
{
	paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT);
	set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud)));
}

static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
{
	gfp_t gfp = GFP_KERNEL_ACCOUNT;

	if (mm == &init_mm)
		gfp &= ~__GFP_ACCOUNT;
	return (pud_t *)get_zeroed_page(gfp);
}

static inline void pud_free(struct mm_struct *mm, pud_t *pud)
{
	BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
	free_page((unsigned long)pud);
}

extern void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud);

static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
				  unsigned long address)
{
	___pud_free_tlb(tlb, pud);
}

#if CONFIG_PGTABLE_LEVELS > 4
static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)
{
	paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT);
	set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d)));
}

static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr)
{
	gfp_t gfp = GFP_KERNEL_ACCOUNT;

	if (mm == &init_mm)
		gfp &= ~__GFP_ACCOUNT;
	return (p4d_t *)get_zeroed_page(gfp);
}

static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)
{
	BUG_ON((unsigned long)p4d & (PAGE_SIZE-1));
	free_page((unsigned long)p4d);
}

extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d);

static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,
				  unsigned long address)
{
	___p4d_free_tlb(tlb, p4d);
}

#endif	/* CONFIG_PGTABLE_LEVELS > 4 */
#endif	/* CONFIG_PGTABLE_LEVELS > 3 */
#endif	/* CONFIG_PGTABLE_LEVELS > 2 */

#endif /* _ASM_X86_PGALLOC_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
1965aae3 PA	2	#ifndef _ASM_X86_PGALLOC_H
1965aae3 PA	3	#define _ASM_X86_PGALLOC_H
4f76cd38 JF	4
	5	#include <linux/threads.h>
	6	#include <linux/mm.h> /* for struct page */
	7	#include <linux/pagemap.h>
	8
eba0045f JF	9	static inline int __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; }
eba0045f JF	10
1d262d3a JF	11	#ifdef CONFIG_PARAVIRT
	12	#include <asm/paravirt.h>
	13	#else
eba0045f JF	14	#define paravirt_pgd_alloc(mm) __paravirt_pgd_alloc(mm)
eba0045f JF	15	static inline void paravirt_pgd_free(struct mm_struct mm, pgd_t pgd) {}
286cd494 JF	16	static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) {}
	17	static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) {}
	18	static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn,
	19	unsigned long start, unsigned long count) {}
	20	static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) {}
335437fb	21	static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn) {}
286cd494 JF	22	static inline void paravirt_release_pte(unsigned long pfn) {}
	23	static inline void paravirt_release_pmd(unsigned long pfn) {}
	24	static inline void paravirt_release_pud(unsigned long pfn) {}
335437fb	25	static inline void paravirt_release_p4d(unsigned long pfn) {}
1d262d3a JF	26	#endif
1d262d3a JF	27
14315592 IC	28	/*
	29	* Flags to use when allocating a user page table page.
	30	*/
	31	extern gfp_t __userpte_alloc_gfp;
	32
4f76cd38 JF	33	/*
	34	* Allocate and free page tables.
	35	*/
	36	extern pgd_t pgd_alloc(struct mm_struct );
	37	extern void pgd_free(struct mm_struct mm, pgd_t pgd);
	38
	39	extern pte_t pte_alloc_one_kernel(struct mm_struct , unsigned long);
	40	extern pgtable_t pte_alloc_one(struct mm_struct *, unsigned long);
	41
397f687a JF	42	/* Should really implement gc for free page table pages. This could be
	43	done with a reference count in struct page. */
	44
	45	static inline void pte_free_kernel(struct mm_struct mm, pte_t pte)
	46	{
	47	BUG_ON((unsigned long)pte & (PAGE_SIZE-1));
	48	free_page((unsigned long)pte);
	49	}
	50
	51	static inline void pte_free(struct mm_struct mm, struct page pte)
	52	{
42ef73fe	53	pgtable_page_dtor(pte);
397f687a JF	54	__free_page(pte);
	55	}
	56
9e1b32ca BH	57	extern void ___pte_free_tlb(struct mmu_gather tlb, struct page pte);
	58
	59	static inline void __pte_free_tlb(struct mmu_gather tlb, struct page pte,
	60	unsigned long address)
	61	{
	62	___pte_free_tlb(tlb, pte);
	63	}
397f687a	64
170fdff7 JF	65	static inline void pmd_populate_kernel(struct mm_struct *mm,
	66	pmd_t pmd, pte_t pte)
	67	{
6944a9c8	68	paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT);
170fdff7 JF	69	set_pmd(pmd, __pmd(__pa(pte) \| _PAGE_TABLE));
	70	}
	71
	72	static inline void pmd_populate(struct mm_struct mm, pmd_t pmd,
	73	struct page *pte)
	74	{
	75	unsigned long pfn = page_to_pfn(pte);
	76
6944a9c8	77	paravirt_alloc_pte(mm, pfn);
170fdff7 JF	78	set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) \| _PAGE_TABLE));
	79	}
	80
	81	#define pmd_pgtable(pmd) pmd_page(pmd)
	82
98233368	83	#if CONFIG_PGTABLE_LEVELS > 2
170fdff7 JF	84	static inline pmd_t pmd_alloc_one(struct mm_struct mm, unsigned long addr)
170fdff7 JF	85	{
9491846f	86	struct page *page;
3e79ec7d VD	87	gfp_t gfp = GFP_KERNEL_ACCOUNT \| __GFP_ZERO;
	88
	89	if (mm == &init_mm)
	90	gfp &= ~__GFP_ACCOUNT;
	91	page = alloc_pages(gfp, 0);
9491846f KS	92	if (!page)
	93	return NULL;
	94	if (!pgtable_pmd_page_ctor(page)) {
	95	__free_pages(page, 0);
	96	return NULL;
	97	}
	98	return (pmd_t *)page_address(page);
170fdff7 JF	99	}
	100
	101	static inline void pmd_free(struct mm_struct mm, pmd_t pmd)
	102	{
	103	BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
9491846f	104	pgtable_pmd_page_dtor(virt_to_page(pmd));
170fdff7 JF	105	free_page((unsigned long)pmd);
	106	}
	107
9e1b32ca BH	108	extern void ___pmd_free_tlb(struct mmu_gather tlb, pmd_t pmd);
	109
	110	static inline void __pmd_free_tlb(struct mmu_gather tlb, pmd_t pmd,
b595076a	111	unsigned long address)
9e1b32ca BH	112	{
	113	___pmd_free_tlb(tlb, pmd);
	114	}
170fdff7	115
5a5f8f42 JF	116	#ifdef CONFIG_X86_PAE
	117	extern void pud_populate(struct mm_struct mm, pud_t pudp, pmd_t *pmd);
	118	#else /* !CONFIG_X86_PAE */
	119	static inline void pud_populate(struct mm_struct mm, pud_t pud, pmd_t *pmd)
	120	{
6944a9c8	121	paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
5a5f8f42 JF	122	set_pud(pud, __pud(_PAGE_TABLE \| __pa(pmd)));
	123	}
	124	#endif /* CONFIG_X86_PAE */
	125
98233368	126	#if CONFIG_PGTABLE_LEVELS > 3
f2a6a705	127	static inline void p4d_populate(struct mm_struct mm, p4d_t p4d, pud_t *pud)
5a5f8f42	128	{
2761fa09	129	paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT);
f2a6a705	130	set_p4d(p4d, __p4d(_PAGE_TABLE \| __pa(pud)));
5a5f8f42 JF	131	}
	132
	133	static inline pud_t pud_alloc_one(struct mm_struct mm, unsigned long addr)
	134	{
3e79ec7d VD	135	gfp_t gfp = GFP_KERNEL_ACCOUNT;
	136
	137	if (mm == &init_mm)
	138	gfp &= ~__GFP_ACCOUNT;
	139	return (pud_t *)get_zeroed_page(gfp);
5a5f8f42 JF	140	}
	141
	142	static inline void pud_free(struct mm_struct mm, pud_t pud)
	143	{
	144	BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
	145	free_page((unsigned long)pud);
	146	}
	147
9e1b32ca BH	148	extern void ___pud_free_tlb(struct mmu_gather tlb, pud_t pud);
	149
	150	static inline void __pud_free_tlb(struct mmu_gather tlb, pud_t pud,
	151	unsigned long address)
	152	{
	153	___pud_free_tlb(tlb, pud);
	154	}
	155
f2a6a705 KS	156	#if CONFIG_PGTABLE_LEVELS > 4
	157	static inline void pgd_populate(struct mm_struct mm, pgd_t pgd, p4d_t *p4d)
	158	{
	159	paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT);
	160	set_pgd(pgd, __pgd(_PAGE_TABLE \| __pa(p4d)));
	161	}
	162
	163	static inline p4d_t p4d_alloc_one(struct mm_struct mm, unsigned long addr)
	164	{
	165	gfp_t gfp = GFP_KERNEL_ACCOUNT;
	166
	167	if (mm == &init_mm)
	168	gfp &= ~__GFP_ACCOUNT;
	169	return (p4d_t *)get_zeroed_page(gfp);
	170	}
	171
	172	static inline void p4d_free(struct mm_struct mm, p4d_t p4d)
	173	{
	174	BUG_ON((unsigned long)p4d & (PAGE_SIZE-1));
	175	free_page((unsigned long)p4d);
	176	}
	177
	178	extern void ___p4d_free_tlb(struct mmu_gather tlb, p4d_t p4d);
	179
	180	static inline void __p4d_free_tlb(struct mmu_gather tlb, p4d_t p4d,
	181	unsigned long address)
	182	{
	183	___p4d_free_tlb(tlb, p4d);
	184	}
	185
	186	#endif /* CONFIG_PGTABLE_LEVELS > 4 */
98233368 KS	187	#endif /* CONFIG_PGTABLE_LEVELS > 3 */
98233368 KS	188	#endif /* CONFIG_PGTABLE_LEVELS > 2 */
4f76cd38	189
1965aae3	190	#endif /* _ASM_X86_PGALLOC_H */