[mirror_ubuntu-artful-kernel.git] / arch / powerpc / include / asm / book3s / 64 / pgalloc.h

#ifndef _ASM_POWERPC_BOOK3S_64_PGALLOC_H
#define _ASM_POWERPC_BOOK3S_64_PGALLOC_H
/*
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include <linux/slab.h>
#include <linux/cpumask.h>
#include <linux/percpu.h>

struct vmemmap_backing {
	struct vmemmap_backing *list;
	unsigned long phys;
	unsigned long virt_addr;
};
extern struct vmemmap_backing *vmemmap_list;

/*
 * Functions that deal with pagetables that could be at any level of
 * the table need to be passed an "index_size" so they know how to
 * handle allocation.  For PTE pages (which are linked to a struct
 * page for now, and drawn from the main get_free_pages() pool), the
 * allocation size will be (2^index_size * sizeof(pointer)) and
 * allocations are drawn from the kmem_cache in PGT_CACHE(index_size).
 *
 * The maximum index size needs to be big enough to allow any
 * pagetable sizes we need, but small enough to fit in the low bits of
 * any page table pointer.  In other words all pagetables, even tiny
 * ones, must be aligned to allow at least enough low 0 bits to
 * contain this value.  This value is also used as a mask, so it must
 * be one less than a power of two.
 */
#define MAX_PGTABLE_INDEX_SIZE	0xf

extern struct kmem_cache *pgtable_cache[];
#define PGT_CACHE(shift) ({				\
			BUG_ON(!(shift));		\
			pgtable_cache[(shift) - 1];	\
		})

#define PGALLOC_GFP GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO

extern pte_t *pte_fragment_alloc(struct mm_struct *, unsigned long, int);
extern void pte_fragment_free(unsigned long *, int);
extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift);
#ifdef CONFIG_SMP
extern void __tlb_remove_table(void *_table);
#endif

static inline pgd_t *radix__pgd_alloc(struct mm_struct *mm)
{
#ifdef CONFIG_PPC_64K_PAGES
	return (pgd_t *)__get_free_page(pgtable_gfp_flags(mm, PGALLOC_GFP));
#else
	struct page *page;
	page = alloc_pages(pgtable_gfp_flags(mm, PGALLOC_GFP | __GFP_RETRY_MAYFAIL),
				4);
	if (!page)
		return NULL;
	return (pgd_t *) page_address(page);
#endif
}

static inline void radix__pgd_free(struct mm_struct *mm, pgd_t *pgd)
{
#ifdef CONFIG_PPC_64K_PAGES
	free_page((unsigned long)pgd);
#else
	free_pages((unsigned long)pgd, 4);
#endif
}

static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{
	if (radix_enabled())
		return radix__pgd_alloc(mm);
	return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE),
		pgtable_gfp_flags(mm, GFP_KERNEL));
}

static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
{
	if (radix_enabled())
		return radix__pgd_free(mm, pgd);
	kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd);
}

static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
{
	pgd_set(pgd, __pgtable_ptr_val(pud) | PGD_VAL_BITS);
}

static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
{
	return kmem_cache_alloc(PGT_CACHE(PUD_INDEX_SIZE),
		pgtable_gfp_flags(mm, GFP_KERNEL));
}

static inline void pud_free(struct mm_struct *mm, pud_t *pud)
{
	kmem_cache_free(PGT_CACHE(PUD_INDEX_SIZE), pud);
}

static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
{
	pud_set(pud, __pgtable_ptr_val(pmd) | PUD_VAL_BITS);
}

static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
                                  unsigned long address)
{
	/*
	 * By now all the pud entries should be none entries. So go
	 * ahead and flush the page walk cache
	 */
	flush_tlb_pgtable(tlb, address);
        pgtable_free_tlb(tlb, pud, PUD_INDEX_SIZE);
}

static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
{
	return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX),
		pgtable_gfp_flags(mm, GFP_KERNEL));
}

static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
{
	kmem_cache_free(PGT_CACHE(PMD_CACHE_INDEX), pmd);
}

static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
                                  unsigned long address)
{
	/*
	 * By now all the pud entries should be none entries. So go
	 * ahead and flush the page walk cache
	 */
	flush_tlb_pgtable(tlb, address);
        return pgtable_free_tlb(tlb, pmd, PMD_CACHE_INDEX);
}

static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
				       pte_t *pte)
{
	pmd_set(pmd, __pgtable_ptr_val(pte) | PMD_VAL_BITS);
}

static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
				pgtable_t pte_page)
{
	pmd_set(pmd, __pgtable_ptr_val(pte_page) | PMD_VAL_BITS);
}

static inline pgtable_t pmd_pgtable(pmd_t pmd)
{
	return (pgtable_t)pmd_page_vaddr(pmd);
}

#ifdef CONFIG_PPC_4K_PAGES
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
					  unsigned long address)
{
	return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
}

static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
				      unsigned long address)
{
	struct page *page;
	pte_t *pte;

	pte = (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT);
	if (!pte)
		return NULL;
	page = virt_to_page(pte);
	if (!pgtable_page_ctor(page)) {
		__free_page(page);
		return NULL;
	}
	return pte;
}
#else /* if CONFIG_PPC_64K_PAGES */

static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
					  unsigned long address)
{
	return (pte_t *)pte_fragment_alloc(mm, address, 1);
}

static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
				      unsigned long address)
{
	return (pgtable_t)pte_fragment_alloc(mm, address, 0);
}
#endif

static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
{
	pte_fragment_free((unsigned long *)pte, 1);
}

static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
{
	pte_fragment_free((unsigned long *)ptepage, 0);
}

static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
				  unsigned long address)
{
	/*
	 * By now all the pud entries should be none entries. So go
	 * ahead and flush the page walk cache
	 */
	flush_tlb_pgtable(tlb, address);
	pgtable_free_tlb(tlb, table, 0);
}

#define check_pgt_cache()	do { } while (0)

#endif /* _ASM_POWERPC_BOOK3S_64_PGALLOC_H */
Commit	Line	Data
75a9b8a6 AK	1	#ifndef _ASM_POWERPC_BOOK3S_64_PGALLOC_H
75a9b8a6 AK	2	#define _ASM_POWERPC_BOOK3S_64_PGALLOC_H
101ad5c6 AK	3	/*
	4	* This program is free software; you can redistribute it and/or
	5	* modify it under the terms of the GNU General Public License
	6	* as published by the Free Software Foundation; either version
	7	* 2 of the License, or (at your option) any later version.
	8	*/
	9
	10	#include <linux/slab.h>
	11	#include <linux/cpumask.h>
	12	#include <linux/percpu.h>
	13
	14	struct vmemmap_backing {
	15	struct vmemmap_backing *list;
	16	unsigned long phys;
	17	unsigned long virt_addr;
	18	};
	19	extern struct vmemmap_backing *vmemmap_list;
	20
	21	/*
	22	* Functions that deal with pagetables that could be at any level of
	23	* the table need to be passed an "index_size" so they know how to
	24	* handle allocation. For PTE pages (which are linked to a struct
	25	* page for now, and drawn from the main get_free_pages() pool), the
	26	* allocation size will be (2^index_size * sizeof(pointer)) and
	27	* allocations are drawn from the kmem_cache in PGT_CACHE(index_size).
	28	*
	29	* The maximum index size needs to be big enough to allow any
	30	* pagetable sizes we need, but small enough to fit in the low bits of
	31	* any page table pointer. In other words all pagetables, even tiny
	32	* ones, must be aligned to allow at least enough low 0 bits to
	33	* contain this value. This value is also used as a mask, so it must
	34	* be one less than a power of two.
	35	*/
	36	#define MAX_PGTABLE_INDEX_SIZE 0xf
	37
	38	extern struct kmem_cache *pgtable_cache[];
	39	#define PGT_CACHE(shift) ({ \
	40	BUG_ON(!(shift)); \
	41	pgtable_cache[(shift) - 1]; \
	42	})
	43
2379a23e	44	#define PGALLOC_GFP GFP_KERNEL \| __GFP_NOTRACK \| __GFP_ZERO
934828ed AK	45
	46	extern pte_t pte_fragment_alloc(struct mm_struct , unsigned long, int);
	47	extern void pte_fragment_free(unsigned long *, int);
	48	extern void pgtable_free_tlb(struct mmu_gather tlb, void table, int shift);
	49	#ifdef CONFIG_SMP
	50	extern void __tlb_remove_table(void *_table);
	51	#endif
	52
a2f41eb9 AK	53	static inline pgd_t radix__pgd_alloc(struct mm_struct mm)
	54	{
	55	#ifdef CONFIG_PPC_64K_PAGES
de3b8761	56	return (pgd_t *)__get_free_page(pgtable_gfp_flags(mm, PGALLOC_GFP));
a2f41eb9 AK	57	#else
a2f41eb9 AK	58	struct page *page;
dcda9b04	59	page = alloc_pages(pgtable_gfp_flags(mm, PGALLOC_GFP \| __GFP_RETRY_MAYFAIL),
de3b8761	60	4);
a2f41eb9 AK	61	if (!page)
	62	return NULL;
	63	return (pgd_t *) page_address(page);
	64	#endif
	65	}
	66
	67	static inline void radix__pgd_free(struct mm_struct mm, pgd_t pgd)
	68	{
	69	#ifdef CONFIG_PPC_64K_PAGES
	70	free_page((unsigned long)pgd);
	71	#else
	72	free_pages((unsigned long)pgd, 4);
	73	#endif
	74	}
	75
101ad5c6 AK	76	static inline pgd_t pgd_alloc(struct mm_struct mm)
101ad5c6 AK	77	{
a2f41eb9 AK	78	if (radix_enabled())
a2f41eb9 AK	79	return radix__pgd_alloc(mm);
de3b8761 BS	80	return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE),
de3b8761 BS	81	pgtable_gfp_flags(mm, GFP_KERNEL));
101ad5c6 AK	82	}
	83
	84	static inline void pgd_free(struct mm_struct mm, pgd_t pgd)
	85	{
a2f41eb9 AK	86	if (radix_enabled())
a2f41eb9 AK	87	return radix__pgd_free(mm, pgd);
101ad5c6 AK	88	kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd);
	89	}
	90
75a9b8a6 AK	91	static inline void pgd_populate(struct mm_struct mm, pgd_t pgd, pud_t *pud)
75a9b8a6 AK	92	{
a2f41eb9	93	pgd_set(pgd, __pgtable_ptr_val(pud) \| PGD_VAL_BITS);
75a9b8a6	94	}
101ad5c6 AK	95
	96	static inline pud_t pud_alloc_one(struct mm_struct mm, unsigned long addr)
	97	{
de3b8761 BS	98	return kmem_cache_alloc(PGT_CACHE(PUD_INDEX_SIZE),
de3b8761 BS	99	pgtable_gfp_flags(mm, GFP_KERNEL));
101ad5c6 AK	100	}
	101
	102	static inline void pud_free(struct mm_struct mm, pud_t pud)
	103	{
	104	kmem_cache_free(PGT_CACHE(PUD_INDEX_SIZE), pud);
	105	}
	106
	107	static inline void pud_populate(struct mm_struct mm, pud_t pud, pmd_t *pmd)
	108	{
a2f41eb9	109	pud_set(pud, __pgtable_ptr_val(pmd) \| PUD_VAL_BITS);
101ad5c6 AK	110	}
101ad5c6 AK	111
934828ed AK	112	static inline void __pud_free_tlb(struct mmu_gather tlb, pud_t pud,
	113	unsigned long address)
	114	{
a145abf1 AK	115	/*
	116	* By now all the pud entries should be none entries. So go
	117	* ahead and flush the page walk cache
	118	*/
	119	flush_tlb_pgtable(tlb, address);
934828ed AK	120	pgtable_free_tlb(tlb, pud, PUD_INDEX_SIZE);
	121	}
	122
	123	static inline pmd_t pmd_alloc_one(struct mm_struct mm, unsigned long addr)
	124	{
de3b8761 BS	125	return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX),
de3b8761 BS	126	pgtable_gfp_flags(mm, GFP_KERNEL));
934828ed AK	127	}
	128
	129	static inline void pmd_free(struct mm_struct mm, pmd_t pmd)
	130	{
	131	kmem_cache_free(PGT_CACHE(PMD_CACHE_INDEX), pmd);
	132	}
	133
	134	static inline void __pmd_free_tlb(struct mmu_gather tlb, pmd_t pmd,
	135	unsigned long address)
	136	{
a145abf1 AK	137	/*
	138	* By now all the pud entries should be none entries. So go
	139	* ahead and flush the page walk cache
	140	*/
	141	flush_tlb_pgtable(tlb, address);
934828ed AK	142	return pgtable_free_tlb(tlb, pmd, PMD_CACHE_INDEX);
	143	}
	144
101ad5c6 AK	145	static inline void pmd_populate_kernel(struct mm_struct mm, pmd_t pmd,
	146	pte_t *pte)
	147	{
a2f41eb9	148	pmd_set(pmd, __pgtable_ptr_val(pte) \| PMD_VAL_BITS);
101ad5c6	149	}
934828ed	150
101ad5c6 AK	151	static inline void pmd_populate(struct mm_struct mm, pmd_t pmd,
	152	pgtable_t pte_page)
	153	{
a2f41eb9	154	pmd_set(pmd, __pgtable_ptr_val(pte_page) \| PMD_VAL_BITS);
101ad5c6 AK	155	}
101ad5c6 AK	156
75a9b8a6 AK	157	static inline pgtable_t pmd_pgtable(pmd_t pmd)
75a9b8a6 AK	158	{
934828ed	159	return (pgtable_t)pmd_page_vaddr(pmd);
75a9b8a6	160	}
101ad5c6	161
934828ed	162	#ifdef CONFIG_PPC_4K_PAGES
101ad5c6 AK	163	static inline pte_t pte_alloc_one_kernel(struct mm_struct mm,
	164	unsigned long address)
	165	{
32d6bd90	166	return (pte_t *)__get_free_page(GFP_KERNEL \| __GFP_ZERO);
101ad5c6 AK	167	}
	168
	169	static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
	170	unsigned long address)
	171	{
	172	struct page *page;
	173	pte_t *pte;
	174
de3b8761	175	pte = (pte_t *)__get_free_page(GFP_KERNEL \| __GFP_ZERO \| __GFP_ACCOUNT);
101ad5c6 AK	176	if (!pte)
	177	return NULL;
	178	page = virt_to_page(pte);
	179	if (!pgtable_page_ctor(page)) {
	180	__free_page(page);
	181	return NULL;
	182	}
934828ed	183	return pte;
101ad5c6	184	}
101ad5c6 AK	185	#else /* if CONFIG_PPC_64K_PAGES */
101ad5c6 AK	186
101ad5c6 AK	187	static inline pte_t pte_alloc_one_kernel(struct mm_struct mm,
	188	unsigned long address)
	189	{
74701d59	190	return (pte_t *)pte_fragment_alloc(mm, address, 1);
101ad5c6 AK	191	}
	192
	193	static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
934828ed	194	unsigned long address)
101ad5c6	195	{
74701d59	196	return (pgtable_t)pte_fragment_alloc(mm, address, 0);
101ad5c6	197	}
934828ed	198	#endif
101ad5c6 AK	199
	200	static inline void pte_free_kernel(struct mm_struct mm, pte_t pte)
	201	{
74701d59	202	pte_fragment_free((unsigned long *)pte, 1);
101ad5c6 AK	203	}
	204
	205	static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
	206	{
74701d59	207	pte_fragment_free((unsigned long *)ptepage, 0);
101ad5c6 AK	208	}
	209
	210	static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
	211	unsigned long address)
	212	{
a145abf1 AK	213	/*
	214	* By now all the pud entries should be none entries. So go
	215	* ahead and flush the page walk cache
	216	*/
	217	flush_tlb_pgtable(tlb, address);
101ad5c6 AK	218	pgtable_free_tlb(tlb, table, 0);
101ad5c6 AK	219	}
101ad5c6 AK	220
	221	#define check_pgt_cache() do { } while (0)
	222
75a9b8a6	223	#endif /* _ASM_POWERPC_BOOK3S_64_PGALLOC_H */