[mirror_ubuntu-eoan-kernel.git] / arch / x86 / mm / pgtable_32.c

/*
 *  linux/arch/i386/mm/pgtable.c
 */

#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/mm.h>
#include <linux/nmi.h>
#include <linux/swap.h>
#include <linux/smp.h>
#include <linux/highmem.h>
#include <linux/slab.h>
#include <linux/pagemap.h>
#include <linux/spinlock.h>
#include <linux/module.h>
#include <linux/quicklist.h>

#include <asm/system.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/fixmap.h>
#include <asm/e820.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>

void show_mem(void)
{
	int total = 0, reserved = 0;
	int shared = 0, cached = 0;
	int highmem = 0;
	struct page *page;
	pg_data_t *pgdat;
	unsigned long i;
	unsigned long flags;

	printk(KERN_INFO "Mem-info:\n");
	show_free_areas();
	printk(KERN_INFO "Free swap:       %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
	for_each_online_pgdat(pgdat) {
		pgdat_resize_lock(pgdat, &flags);
		for (i = 0; i < pgdat->node_spanned_pages; ++i) {
			if (unlikely(i % MAX_ORDER_NR_PAGES == 0))
				touch_nmi_watchdog();
			page = pgdat_page_nr(pgdat, i);
			total++;
			if (PageHighMem(page))
				highmem++;
			if (PageReserved(page))
				reserved++;
			else if (PageSwapCache(page))
				cached++;
			else if (page_count(page))
				shared += page_count(page) - 1;
		}
		pgdat_resize_unlock(pgdat, &flags);
	}
	printk(KERN_INFO "%d pages of RAM\n", total);
	printk(KERN_INFO "%d pages of HIGHMEM\n", highmem);
	printk(KERN_INFO "%d reserved pages\n", reserved);
	printk(KERN_INFO "%d pages shared\n", shared);
	printk(KERN_INFO "%d pages swap cached\n", cached);

	printk(KERN_INFO "%lu pages dirty\n", global_page_state(NR_FILE_DIRTY));
	printk(KERN_INFO "%lu pages writeback\n",
					global_page_state(NR_WRITEBACK));
	printk(KERN_INFO "%lu pages mapped\n", global_page_state(NR_FILE_MAPPED));
	printk(KERN_INFO "%lu pages slab\n",
		global_page_state(NR_SLAB_RECLAIMABLE) +
		global_page_state(NR_SLAB_UNRECLAIMABLE));
	printk(KERN_INFO "%lu pages pagetables\n",
					global_page_state(NR_PAGETABLE));
}

/*
 * Associate a virtual page frame with a given physical page frame 
 * and protection flags for that frame.
 */ 
static void set_pte_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	pgd = swapper_pg_dir + pgd_index(vaddr);
	if (pgd_none(*pgd)) {
		BUG();
		return;
	}
	pud = pud_offset(pgd, vaddr);
	if (pud_none(*pud)) {
		BUG();
		return;
	}
	pmd = pmd_offset(pud, vaddr);
	if (pmd_none(*pmd)) {
		BUG();
		return;
	}
	pte = pte_offset_kernel(pmd, vaddr);
	if (pgprot_val(flags))
		set_pte_present(&init_mm, vaddr, pte, pfn_pte(pfn, flags));
	else
		pte_clear(&init_mm, vaddr, pte);

	/*
	 * It's enough to flush this one mapping.
	 * (PGE mappings get flushed as well)
	 */
	__flush_tlb_one(vaddr);
}

/*
 * Associate a large virtual page frame with a given physical page frame 
 * and protection flags for that frame. pfn is for the base of the page,
 * vaddr is what the page gets mapped to - both must be properly aligned. 
 * The pmd must already be instantiated. Assumes PAE mode.
 */ 
void set_pmd_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;

	if (vaddr & (PMD_SIZE-1)) {		/* vaddr is misaligned */
		printk(KERN_WARNING "set_pmd_pfn: vaddr misaligned\n");
		return; /* BUG(); */
	}
	if (pfn & (PTRS_PER_PTE-1)) {		/* pfn is misaligned */
		printk(KERN_WARNING "set_pmd_pfn: pfn misaligned\n");
		return; /* BUG(); */
	}
	pgd = swapper_pg_dir + pgd_index(vaddr);
	if (pgd_none(*pgd)) {
		printk(KERN_WARNING "set_pmd_pfn: pgd_none\n");
		return; /* BUG(); */
	}
	pud = pud_offset(pgd, vaddr);
	pmd = pmd_offset(pud, vaddr);
	set_pmd(pmd, pfn_pmd(pfn, flags));
	/*
	 * It's enough to flush this one mapping.
	 * (PGE mappings get flushed as well)
	 */
	__flush_tlb_one(vaddr);
}

static int fixmaps;
unsigned long __FIXADDR_TOP = 0xfffff000;
EXPORT_SYMBOL(__FIXADDR_TOP);

void __set_fixmap (enum fixed_addresses idx, unsigned long phys, pgprot_t flags)
{
	unsigned long address = __fix_to_virt(idx);

	if (idx >= __end_of_fixed_addresses) {
		BUG();
		return;
	}
	set_pte_pfn(address, phys >> PAGE_SHIFT, flags);
	fixmaps++;
}

/**
 * reserve_top_address - reserves a hole in the top of kernel address space
 * @reserve - size of hole to reserve
 *
 * Can be used to relocate the fixmap area and poke a hole in the top
 * of kernel address space to make room for a hypervisor.
 */
void reserve_top_address(unsigned long reserve)
{
	BUG_ON(fixmaps > 0);
	printk(KERN_INFO "Reserving virtual address space above 0x%08x\n",
	       (int)-reserve);
	__FIXADDR_TOP = -reserve - PAGE_SIZE;
	__VMALLOC_RESERVE += reserve;
}

pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
{
	return (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
}

struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
	struct page *pte;

#ifdef CONFIG_HIGHPTE
	pte = alloc_pages(GFP_KERNEL|__GFP_HIGHMEM|__GFP_REPEAT|__GFP_ZERO, 0);
#else
	pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
#endif
	return pte;
}

/*
 * List of all pgd's needed for non-PAE so it can invalidate entries
 * in both cached and uncached pgd's; not needed for PAE since the
 * kernel pmd is shared. If PAE were not to share the pmd a similar
 * tactic would be needed. This is essentially codepath-based locking
 * against pageattr.c; it is the unique case in which a valid change
 * of kernel pagetables can't be lazily synchronized by vmalloc faults.
 * vmalloc faults work because attached pagetables are never freed.
 * -- wli
 */
static inline void pgd_list_add(pgd_t *pgd)
{
	struct page *page = virt_to_page(pgd);

	list_add(&page->lru, &pgd_list);
}

static inline void pgd_list_del(pgd_t *pgd)
{
	struct page *page = virt_to_page(pgd);

	list_del(&page->lru);
}


#if (PTRS_PER_PMD == 1)
/* Non-PAE pgd constructor */
static void pgd_ctor(void *pgd)
{
	unsigned long flags;

	/* !PAE, no pagetable sharing */
	memset(pgd, 0, USER_PTRS_PER_PGD*sizeof(pgd_t));

	spin_lock_irqsave(&pgd_lock, flags);

	/* must happen under lock */
	clone_pgd_range((pgd_t *)pgd + USER_PTRS_PER_PGD,
			swapper_pg_dir + USER_PTRS_PER_PGD,
			KERNEL_PGD_PTRS);
	paravirt_alloc_pd_clone(__pa(pgd) >> PAGE_SHIFT,
				__pa(swapper_pg_dir) >> PAGE_SHIFT,
				USER_PTRS_PER_PGD,
				KERNEL_PGD_PTRS);
	pgd_list_add(pgd);
	spin_unlock_irqrestore(&pgd_lock, flags);
}
#else  /* PTRS_PER_PMD > 1 */
/* PAE pgd constructor */
static void pgd_ctor(void *pgd)
{
	/* PAE, kernel PMD may be shared */

	if (SHARED_KERNEL_PMD) {
		clone_pgd_range((pgd_t *)pgd + USER_PTRS_PER_PGD,
				swapper_pg_dir + USER_PTRS_PER_PGD,
				KERNEL_PGD_PTRS);
	} else {
		unsigned long flags;

		memset(pgd, 0, USER_PTRS_PER_PGD*sizeof(pgd_t));
		spin_lock_irqsave(&pgd_lock, flags);
		pgd_list_add(pgd);
		spin_unlock_irqrestore(&pgd_lock, flags);
	}
}
#endif	/* PTRS_PER_PMD */

static void pgd_dtor(void *pgd)
{
	unsigned long flags; /* can be called from interrupt context */

	if (SHARED_KERNEL_PMD)
		return;

	spin_lock_irqsave(&pgd_lock, flags);
	pgd_list_del(pgd);
	spin_unlock_irqrestore(&pgd_lock, flags);
}

#define UNSHARED_PTRS_PER_PGD				\
	(SHARED_KERNEL_PMD ? USER_PTRS_PER_PGD : PTRS_PER_PGD)

#ifdef CONFIG_X86_PAE
/*
 * Mop up any pmd pages which may still be attached to the pgd.
 * Normally they will be freed by munmap/exit_mmap, but any pmd we
 * preallocate which never got a corresponding vma will need to be
 * freed manually.
 */
static void pgd_mop_up_pmds(pgd_t *pgdp)
{
	int i;

	for(i = 0; i < UNSHARED_PTRS_PER_PGD; i++) {
		pgd_t pgd = pgdp[i];

		if (pgd_val(pgd) != 0) {
			pmd_t *pmd = (pmd_t *)pgd_page_vaddr(pgd);

			pgdp[i] = native_make_pgd(0);

			paravirt_release_pd(pgd_val(pgd) >> PAGE_SHIFT);
			pmd_free(pmd);
		}
	}
}

/*
 * In PAE mode, we need to do a cr3 reload (=tlb flush) when
 * updating the top-level pagetable entries to guarantee the
 * processor notices the update.  Since this is expensive, and
 * all 4 top-level entries are used almost immediately in a
 * new process's life, we just pre-populate them here.
 *
 * Also, if we're in a paravirt environment where the kernel pmd is
 * not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate
 * and initialize the kernel pmds here.
 */
static int pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd)
{
	pud_t *pud;
	unsigned long addr;
	int i;

	pud = pud_offset(pgd, 0);
 	for (addr = i = 0; i < UNSHARED_PTRS_PER_PGD;
	     i++, pud++, addr += PUD_SIZE) {
		pmd_t *pmd = pmd_alloc_one(mm, addr);

		if (!pmd) {
			pgd_mop_up_pmds(pgd);
			return 0;
		}

		if (i >= USER_PTRS_PER_PGD)
			memcpy(pmd, (pmd_t *)pgd_page_vaddr(swapper_pg_dir[i]),
			       sizeof(pmd_t) * PTRS_PER_PMD);

		pud_populate(mm, pud, pmd);
	}

	return 1;
}
#else  /* !CONFIG_X86_PAE */
/* No need to prepopulate any pagetable entries in non-PAE modes. */
static int pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd)
{
	return 1;
}

static void pgd_mop_up_pmds(pgd_t *pgd)
{
}
#endif	/* CONFIG_X86_PAE */

pgd_t *pgd_alloc(struct mm_struct *mm)
{
	pgd_t *pgd = quicklist_alloc(0, GFP_KERNEL, pgd_ctor);

	mm->pgd = pgd;		/* so that alloc_pd can use it */

	if (pgd && !pgd_prepopulate_pmd(mm, pgd)) {
		quicklist_free(0, pgd_dtor, pgd);
		pgd = NULL;
	}

	return pgd;
}

void pgd_free(pgd_t *pgd)
{
	pgd_mop_up_pmds(pgd);
	quicklist_free(0, pgd_dtor, pgd);
}

void check_pgt_cache(void)
{
	quicklist_trim(0, pgd_dtor, 25, 16);
}

void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
{
	paravirt_release_pt(page_to_pfn(pte));
	tlb_remove_page(tlb, pte);
}

#ifdef CONFIG_X86_PAE

void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
{
	/* This is called just after the pmd has been detached from
	   the pgd, which requires a full tlb flush to be recognized
	   by the CPU.  Rather than incurring multiple tlb flushes
	   while the address space is being pulled down, make the tlb
	   gathering machinery do a full flush when we're done. */
	tlb->fullmm = 1;

	paravirt_release_pd(__pa(pmd) >> PAGE_SHIFT);
	tlb_remove_page(tlb, virt_to_page(pmd));
}

#endif
Commit	Line	Data
1da177e4 LT	1	/*
	2	* linux/arch/i386/mm/pgtable.c
	3	*/
	4
1da177e4 LT	5	#include <linux/sched.h>
	6	#include <linux/kernel.h>
	7	#include <linux/errno.h>
	8	#include <linux/mm.h>
27eb0b28	9	#include <linux/nmi.h>
1da177e4 LT	10	#include <linux/swap.h>
	11	#include <linux/smp.h>
	12	#include <linux/highmem.h>
	13	#include <linux/slab.h>
	14	#include <linux/pagemap.h>
	15	#include <linux/spinlock.h>
052e7994	16	#include <linux/module.h>
f1d1a842	17	#include <linux/quicklist.h>
1da177e4 LT	18
	19	#include <asm/system.h>
	20	#include <asm/pgtable.h>
	21	#include <asm/pgalloc.h>
	22	#include <asm/fixmap.h>
	23	#include <asm/e820.h>
	24	#include <asm/tlb.h>
	25	#include <asm/tlbflush.h>
	26
	27	void show_mem(void)
	28	{
	29	int total = 0, reserved = 0;
	30	int shared = 0, cached = 0;
	31	int highmem = 0;
	32	struct page *page;
	33	pg_data_t *pgdat;
	34	unsigned long i;
208d54e5	35	unsigned long flags;
1da177e4	36
f90e7185	37	printk(KERN_INFO "Mem-info:\n");
1da177e4	38	show_free_areas();
f90e7185	39	printk(KERN_INFO "Free swap: %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
ec936fc5	40	for_each_online_pgdat(pgdat) {
208d54e5	41	pgdat_resize_lock(pgdat, &flags);
1da177e4	42	for (i = 0; i < pgdat->node_spanned_pages; ++i) {
27eb0b28 PB	43	if (unlikely(i % MAX_ORDER_NR_PAGES == 0))
27eb0b28 PB	44	touch_nmi_watchdog();
408fde81	45	page = pgdat_page_nr(pgdat, i);
1da177e4 LT	46	total++;
	47	if (PageHighMem(page))
	48	highmem++;
	49	if (PageReserved(page))
	50	reserved++;
	51	else if (PageSwapCache(page))
	52	cached++;
	53	else if (page_count(page))
	54	shared += page_count(page) - 1;
	55	}
208d54e5	56	pgdat_resize_unlock(pgdat, &flags);
1da177e4	57	}
f90e7185 CL	58	printk(KERN_INFO "%d pages of RAM\n", total);
	59	printk(KERN_INFO "%d pages of HIGHMEM\n", highmem);
	60	printk(KERN_INFO "%d reserved pages\n", reserved);
	61	printk(KERN_INFO "%d pages shared\n", shared);
	62	printk(KERN_INFO "%d pages swap cached\n", cached);
6f4e1e50	63
b1e7a8fd	64	printk(KERN_INFO "%lu pages dirty\n", global_page_state(NR_FILE_DIRTY));
ce866b34 CL	65	printk(KERN_INFO "%lu pages writeback\n",
ce866b34 CL	66	global_page_state(NR_WRITEBACK));
65ba55f5	67	printk(KERN_INFO "%lu pages mapped\n", global_page_state(NR_FILE_MAPPED));
972d1a7b CL	68	printk(KERN_INFO "%lu pages slab\n",
	69	global_page_state(NR_SLAB_RECLAIMABLE) +
	70	global_page_state(NR_SLAB_UNRECLAIMABLE));
df849a15 CL	71	printk(KERN_INFO "%lu pages pagetables\n",
df849a15 CL	72	global_page_state(NR_PAGETABLE));
1da177e4 LT	73	}
	74
	75	/*
	76	* Associate a virtual page frame with a given physical page frame
	77	* and protection flags for that frame.
	78	*/
	79	static void set_pte_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags)
	80	{
	81	pgd_t *pgd;
	82	pud_t *pud;
	83	pmd_t *pmd;
	84	pte_t *pte;
	85
	86	pgd = swapper_pg_dir + pgd_index(vaddr);
	87	if (pgd_none(*pgd)) {
	88	BUG();
	89	return;
	90	}
	91	pud = pud_offset(pgd, vaddr);
	92	if (pud_none(*pud)) {
	93	BUG();
	94	return;
	95	}
	96	pmd = pmd_offset(pud, vaddr);
	97	if (pmd_none(*pmd)) {
	98	BUG();
	99	return;
	100	}
	101	pte = pte_offset_kernel(pmd, vaddr);
b0bfece4	102	if (pgprot_val(flags))
aa506dc7	103	set_pte_present(&init_mm, vaddr, pte, pfn_pte(pfn, flags));
b0bfece4 JB	104	else
b0bfece4 JB	105	pte_clear(&init_mm, vaddr, pte);
1da177e4 LT	106
	107	/*
	108	* It's enough to flush this one mapping.
	109	* (PGE mappings get flushed as well)
	110	*/
	111	__flush_tlb_one(vaddr);
	112	}
	113
	114	/*
	115	* Associate a large virtual page frame with a given physical page frame
	116	* and protection flags for that frame. pfn is for the base of the page,
	117	* vaddr is what the page gets mapped to - both must be properly aligned.
	118	* The pmd must already be instantiated. Assumes PAE mode.
	119	*/
	120	void set_pmd_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags)
	121	{
	122	pgd_t *pgd;
	123	pud_t *pud;
	124	pmd_t *pmd;
	125
	126	if (vaddr & (PMD_SIZE-1)) { /* vaddr is misaligned */
f90e7185	127	printk(KERN_WARNING "set_pmd_pfn: vaddr misaligned\n");
1da177e4 LT	128	return; /* BUG(); */
	129	}
	130	if (pfn & (PTRS_PER_PTE-1)) { /* pfn is misaligned */
f90e7185	131	printk(KERN_WARNING "set_pmd_pfn: pfn misaligned\n");
1da177e4 LT	132	return; /* BUG(); */
	133	}
	134	pgd = swapper_pg_dir + pgd_index(vaddr);
	135	if (pgd_none(*pgd)) {
f90e7185	136	printk(KERN_WARNING "set_pmd_pfn: pgd_none\n");
1da177e4 LT	137	return; /* BUG(); */
	138	}
	139	pud = pud_offset(pgd, vaddr);
	140	pmd = pmd_offset(pud, vaddr);
	141	set_pmd(pmd, pfn_pmd(pfn, flags));
	142	/*
	143	* It's enough to flush this one mapping.
	144	* (PGE mappings get flushed as well)
	145	*/
	146	__flush_tlb_one(vaddr);
	147	}
	148
052e7994	149	static int fixmaps;
052e7994 JF	150	unsigned long __FIXADDR_TOP = 0xfffff000;
052e7994 JF	151	EXPORT_SYMBOL(__FIXADDR_TOP);
052e7994	152
1da177e4 LT	153	void __set_fixmap (enum fixed_addresses idx, unsigned long phys, pgprot_t flags)
	154	{
	155	unsigned long address = __fix_to_virt(idx);
	156
	157	if (idx >= __end_of_fixed_addresses) {
	158	BUG();
	159	return;
	160	}
	161	set_pte_pfn(address, phys >> PAGE_SHIFT, flags);
052e7994 JF	162	fixmaps++;
	163	}
	164
	165	/**
	166	* reserve_top_address - reserves a hole in the top of kernel address space
	167	* @reserve - size of hole to reserve
	168	*
	169	* Can be used to relocate the fixmap area and poke a hole in the top
	170	* of kernel address space to make room for a hypervisor.
	171	*/
	172	void reserve_top_address(unsigned long reserve)
	173	{
	174	BUG_ON(fixmaps > 0);
7ce0bcfd ZA	175	printk(KERN_INFO "Reserving virtual address space above 0x%08x\n",
7ce0bcfd ZA	176	(int)-reserve);
052e7994 JF	177	__FIXADDR_TOP = -reserve - PAGE_SIZE;
052e7994 JF	178	__VMALLOC_RESERVE += reserve;
1da177e4 LT	179	}
	180
	181	pte_t pte_alloc_one_kernel(struct mm_struct mm, unsigned long address)
	182	{
	183	return (pte_t *)__get_free_page(GFP_KERNEL\|__GFP_REPEAT\|__GFP_ZERO);
	184	}
	185
	186	struct page pte_alloc_one(struct mm_struct mm, unsigned long address)
	187	{
	188	struct page *pte;
	189
	190	#ifdef CONFIG_HIGHPTE
	191	pte = alloc_pages(GFP_KERNEL\|__GFP_HIGHMEM\|__GFP_REPEAT\|__GFP_ZERO, 0);
	192	#else
	193	pte = alloc_pages(GFP_KERNEL\|__GFP_REPEAT\|__GFP_ZERO, 0);
	194	#endif
	195	return pte;
	196	}
	197
1da177e4 LT	198	/*
	199	* List of all pgd's needed for non-PAE so it can invalidate entries
	200	* in both cached and uncached pgd's; not needed for PAE since the
	201	* kernel pmd is shared. If PAE were not to share the pmd a similar
	202	* tactic would be needed. This is essentially codepath-based locking
	203	* against pageattr.c; it is the unique case in which a valid change
	204	* of kernel pagetables can't be lazily synchronized by vmalloc faults.
	205	* vmalloc faults work because attached pagetables are never freed.
1da177e4 LT	206	* -- wli
1da177e4 LT	207	*/
1da177e4 LT	208	static inline void pgd_list_add(pgd_t *pgd)
	209	{
	210	struct page *page = virt_to_page(pgd);
e3ed910d JF	211
e3ed910d JF	212	list_add(&page->lru, &pgd_list);
1da177e4 LT	213	}
	214
	215	static inline void pgd_list_del(pgd_t *pgd)
	216	{
e3ed910d JF	217	struct page *page = virt_to_page(pgd);
	218
	219	list_del(&page->lru);
1da177e4 LT	220	}
1da177e4 LT	221
f1d1a842 CL	222
f1d1a842 CL	223
5311ab62 JF	224	#if (PTRS_PER_PMD == 1)
5311ab62 JF	225	/* Non-PAE pgd constructor */
2378569d	226	static void pgd_ctor(void *pgd)
1da177e4 LT	227	{
	228	unsigned long flags;
	229
5311ab62 JF	230	/* !PAE, no pagetable sharing */
	231	memset(pgd, 0, USER_PTRS_PER_PGD*sizeof(pgd_t));
	232
	233	spin_lock_irqsave(&pgd_lock, flags);
1da177e4	234
5311ab62	235	/* must happen under lock */
d7271b14	236	clone_pgd_range((pgd_t *)pgd + USER_PTRS_PER_PGD,
1da177e4	237	swapper_pg_dir + USER_PTRS_PER_PGD,
d7271b14	238	KERNEL_PGD_PTRS);
c119ecce	239	paravirt_alloc_pd_clone(__pa(pgd) >> PAGE_SHIFT,
5311ab62 JF	240	__pa(swapper_pg_dir) >> PAGE_SHIFT,
	241	USER_PTRS_PER_PGD,
	242	KERNEL_PGD_PTRS);
1da177e4 LT	243	pgd_list_add(pgd);
1da177e4 LT	244	spin_unlock_irqrestore(&pgd_lock, flags);
1da177e4	245	}
5311ab62 JF	246	#else /* PTRS_PER_PMD > 1 */
5311ab62 JF	247	/* PAE pgd constructor */
2378569d	248	static void pgd_ctor(void *pgd)
5311ab62 JF	249	{
	250	/* PAE, kernel PMD may be shared */
	251
	252	if (SHARED_KERNEL_PMD) {
	253	clone_pgd_range((pgd_t *)pgd + USER_PTRS_PER_PGD,
	254	swapper_pg_dir + USER_PTRS_PER_PGD,
	255	KERNEL_PGD_PTRS);
	256	} else {
	257	unsigned long flags;
	258
	259	memset(pgd, 0, USER_PTRS_PER_PGD*sizeof(pgd_t));
	260	spin_lock_irqsave(&pgd_lock, flags);
	261	pgd_list_add(pgd);
	262	spin_unlock_irqrestore(&pgd_lock, flags);
	263	}
	264	}
	265	#endif /* PTRS_PER_PMD */
1da177e4	266
2378569d	267	static void pgd_dtor(void *pgd)
1da177e4 LT	268	{
	269	unsigned long flags; /* can be called from interrupt context */
	270
f1d1a842 CL	271	if (SHARED_KERNEL_PMD)
f1d1a842 CL	272	return;
5311ab62	273
1da177e4 LT	274	spin_lock_irqsave(&pgd_lock, flags);
	275	pgd_list_del(pgd);
	276	spin_unlock_irqrestore(&pgd_lock, flags);
	277	}
	278
5311ab62 JF	279	#define UNSHARED_PTRS_PER_PGD \
	280	(SHARED_KERNEL_PMD ? USER_PTRS_PER_PGD : PTRS_PER_PGD)
	281
8fe3deef JF	282	#ifdef CONFIG_X86_PAE
	283	/*
	284	* Mop up any pmd pages which may still be attached to the pgd.
	285	* Normally they will be freed by munmap/exit_mmap, but any pmd we
	286	* preallocate which never got a corresponding vma will need to be
	287	* freed manually.
	288	*/
	289	static void pgd_mop_up_pmds(pgd_t *pgdp)
	290	{
	291	int i;
	292
508bebbb	293	for(i = 0; i < UNSHARED_PTRS_PER_PGD; i++) {
8fe3deef JF	294	pgd_t pgd = pgdp[i];
	295
	296	if (pgd_val(pgd) != 0) {
	297	pmd_t pmd = (pmd_t )pgd_page_vaddr(pgd);
	298
	299	pgdp[i] = native_make_pgd(0);
	300
	301	paravirt_release_pd(pgd_val(pgd) >> PAGE_SHIFT);
	302	pmd_free(pmd);
	303	}
	304	}
	305	}
	306
	307	/*
	308	* In PAE mode, we need to do a cr3 reload (=tlb flush) when
	309	* updating the top-level pagetable entries to guarantee the
	310	* processor notices the update. Since this is expensive, and
	311	* all 4 top-level entries are used almost immediately in a
	312	* new process's life, we just pre-populate them here.
508bebbb JF	313	*
	314	* Also, if we're in a paravirt environment where the kernel pmd is
	315	* not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate
	316	* and initialize the kernel pmds here.
8fe3deef JF	317	*/
	318	static int pgd_prepopulate_pmd(struct mm_struct mm, pgd_t pgd)
	319	{
	320	pud_t *pud;
	321	unsigned long addr;
	322	int i;
	323
	324	pud = pud_offset(pgd, 0);
508bebbb JF	325	for (addr = i = 0; i < UNSHARED_PTRS_PER_PGD;
508bebbb JF	326	i++, pud++, addr += PUD_SIZE) {
8fe3deef JF	327	pmd_t *pmd = pmd_alloc_one(mm, addr);
	328
	329	if (!pmd) {
	330	pgd_mop_up_pmds(pgd);
	331	return 0;
	332	}
	333
508bebbb JF	334	if (i >= USER_PTRS_PER_PGD)
	335	memcpy(pmd, (pmd_t *)pgd_page_vaddr(swapper_pg_dir[i]),
	336	sizeof(pmd_t) * PTRS_PER_PMD);
	337
8fe3deef JF	338	pud_populate(mm, pud, pmd);
	339	}
	340
	341	return 1;
	342	}
	343	#else /* !CONFIG_X86_PAE */
	344	/* No need to prepopulate any pagetable entries in non-PAE modes. */
	345	static int pgd_prepopulate_pmd(struct mm_struct mm, pgd_t pgd)
	346	{
	347	return 1;
	348	}
	349
	350	static void pgd_mop_up_pmds(pgd_t *pgd)
	351	{
	352	}
	353	#endif /* CONFIG_X86_PAE */
	354
1da177e4 LT	355	pgd_t pgd_alloc(struct mm_struct mm)
1da177e4 LT	356	{
f1d1a842	357	pgd_t *pgd = quicklist_alloc(0, GFP_KERNEL, pgd_ctor);
1da177e4	358
6c435456 JF	359	mm->pgd = pgd; /* so that alloc_pd can use it */
6c435456 JF	360
8fe3deef JF	361	if (pgd && !pgd_prepopulate_pmd(mm, pgd)) {
	362	quicklist_free(0, pgd_dtor, pgd);
	363	pgd = NULL;
	364	}
	365
1da177e4	366	return pgd;
1da177e4 LT	367	}
	368
	369	void pgd_free(pgd_t *pgd)
	370	{
8fe3deef	371	pgd_mop_up_pmds(pgd);
f1d1a842	372	quicklist_free(0, pgd_dtor, pgd);
1da177e4	373	}
f1d1a842 CL	374
	375	void check_pgt_cache(void)
	376	{
	377	quicklist_trim(0, pgd_dtor, 25, 16);
	378	}
5aa05085 IM	379
	380	void __pte_free_tlb(struct mmu_gather tlb, struct page pte)
	381	{
	382	paravirt_release_pt(page_to_pfn(pte));
	383	tlb_remove_page(tlb, pte);
	384	}
	385
	386	#ifdef CONFIG_X86_PAE
	387
	388	void __pmd_free_tlb(struct mmu_gather tlb, pmd_t pmd)
	389	{
	390	/* This is called just after the pmd has been detached from
	391	the pgd, which requires a full tlb flush to be recognized
	392	by the CPU. Rather than incurring multiple tlb flushes
	393	while the address space is being pulled down, make the tlb
	394	gathering machinery do a full flush when we're done. */
	395	tlb->fullmm = 1;
	396
	397	paravirt_release_pd(__pa(pmd) >> PAGE_SHIFT);
	398	tlb_remove_page(tlb, virt_to_page(pmd));
	399	}
	400
	401	#endif