[mirror_ubuntu-artful-kernel.git] / arch / x86 / mm / hugetlbpage.c

/*
 * IA-32 Huge TLB Page Support for Kernel.
 *
 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
 */

#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
#include <linux/err.h>
#include <linux/sysctl.h>
#include <asm/mman.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>
#include <asm/pgalloc.h>

#if 0	/* This is just for testing */
struct page *
follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
{
	unsigned long start = address;
	int length = 1;
	int nr;
	struct page *page;
	struct vm_area_struct *vma;

	vma = find_vma(mm, addr);
	if (!vma || !is_vm_hugetlb_page(vma))
		return ERR_PTR(-EINVAL);

	pte = huge_pte_offset(mm, address);

	/* hugetlb should be locked, and hence, prefaulted */
	WARN_ON(!pte || pte_none(*pte));

	page = &pte_page(*pte)[vpfn % (HPAGE_SIZE/PAGE_SIZE)];

	WARN_ON(!PageHead(page));

	return page;
}

int pmd_huge(pmd_t pmd)
{
	return 0;
}

int pud_huge(pud_t pud)
{
	return 0;
}

struct page *
follow_huge_pmd(struct mm_struct *mm, unsigned long address,
		pmd_t *pmd, int write)
{
	return NULL;
}

#else

struct page *
follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
{
	return ERR_PTR(-EINVAL);
}

int pmd_huge(pmd_t pmd)
{
	return !!(pmd_val(pmd) & _PAGE_PSE);
}

int pud_huge(pud_t pud)
{
	return !!(pud_val(pud) & _PAGE_PSE);
}

#endif

/* x86_64 also uses this file */

#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
		unsigned long addr, unsigned long len,
		unsigned long pgoff, unsigned long flags)
{
	struct hstate *h = hstate_file(file);
	struct vm_unmapped_area_info info;

	info.flags = 0;
	info.length = len;
	info.low_limit = TASK_UNMAPPED_BASE;
	info.high_limit = TASK_SIZE;
	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	info.align_offset = 0;
	return vm_unmapped_area(&info);
}

static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
		unsigned long addr0, unsigned long len,
		unsigned long pgoff, unsigned long flags)
{
	struct hstate *h = hstate_file(file);
	struct vm_unmapped_area_info info;
	unsigned long addr;

	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
	info.length = len;
	info.low_limit = PAGE_SIZE;
	info.high_limit = current->mm->mmap_base;
	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	info.align_offset = 0;
	addr = vm_unmapped_area(&info);

	/*
	 * A failed mmap() very likely causes application failure,
	 * so fall back to the bottom-up function here. This scenario
	 * can happen with large stack limits and large mmap()
	 * allocations.
	 */
	if (addr & ~PAGE_MASK) {
		VM_BUG_ON(addr != -ENOMEM);
		info.flags = 0;
		info.low_limit = TASK_UNMAPPED_BASE;
		info.high_limit = TASK_SIZE;
		addr = vm_unmapped_area(&info);
	}

	return addr;
}

unsigned long
hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
		unsigned long len, unsigned long pgoff, unsigned long flags)
{
	struct hstate *h = hstate_file(file);
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma;

	if (len & ~huge_page_mask(h))
		return -EINVAL;
	if (len > TASK_SIZE)
		return -ENOMEM;

	if (flags & MAP_FIXED) {
		if (prepare_hugepage_range(file, addr, len))
			return -EINVAL;
		return addr;
	}

	if (addr) {
		addr = ALIGN(addr, huge_page_size(h));
		vma = find_vma(mm, addr);
		if (TASK_SIZE - len >= addr &&
		    (!vma || addr + len <= vma->vm_start))
			return addr;
	}
	if (mm->get_unmapped_area == arch_get_unmapped_area)
		return hugetlb_get_unmapped_area_bottomup(file, addr, len,
				pgoff, flags);
	else
		return hugetlb_get_unmapped_area_topdown(file, addr, len,
				pgoff, flags);
}

#endif /*HAVE_ARCH_HUGETLB_UNMAPPED_AREA*/

#ifdef CONFIG_X86_64
static __init int setup_hugepagesz(char *opt)
{
	unsigned long ps = memparse(opt, &opt);
	if (ps == PMD_SIZE) {
		hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
	} else if (ps == PUD_SIZE && cpu_has_gbpages) {
		hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
	} else {
		printk(KERN_ERR "hugepagesz: Unsupported page size %lu M\n",
			ps >> 20);
		return 0;
	}
	return 1;
}
__setup("hugepagesz=", setup_hugepagesz);
#endif
Commit	Line	Data
1da177e4 LT	1	/*
	2	* IA-32 Huge TLB Page Support for Kernel.
	3	*
	4	* Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
	5	*/
	6
1da177e4 LT	7	#include <linux/init.h>
	8	#include <linux/fs.h>
	9	#include <linux/mm.h>
	10	#include <linux/hugetlb.h>
	11	#include <linux/pagemap.h>
1da177e4 LT	12	#include <linux/err.h>
	13	#include <linux/sysctl.h>
	14	#include <asm/mman.h>
	15	#include <asm/tlb.h>
	16	#include <asm/tlbflush.h>
a5a19c63	17	#include <asm/pgalloc.h>
1da177e4	18
1da177e4 LT	19	#if 0 /* This is just for testing */
	20	struct page *
	21	follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
	22	{
	23	unsigned long start = address;
	24	int length = 1;
	25	int nr;
	26	struct page *page;
	27	struct vm_area_struct *vma;
	28
	29	vma = find_vma(mm, addr);
	30	if (!vma \|\| !is_vm_hugetlb_page(vma))
	31	return ERR_PTR(-EINVAL);
	32
	33	pte = huge_pte_offset(mm, address);
	34
	35	/* hugetlb should be locked, and hence, prefaulted */
	36	WARN_ON(!pte \|\| pte_none(*pte));
	37
	38	page = &pte_page(*pte)[vpfn % (HPAGE_SIZE/PAGE_SIZE)];
	39
25e59881	40	WARN_ON(!PageHead(page));
1da177e4 LT	41
	42	return page;
	43	}
	44
	45	int pmd_huge(pmd_t pmd)
	46	{
	47	return 0;
	48	}
	49
ceb86879 AK	50	int pud_huge(pud_t pud)
	51	{
	52	return 0;
	53	}
	54
1da177e4 LT	55	struct page *
	56	follow_huge_pmd(struct mm_struct *mm, unsigned long address,
	57	pmd_t *pmd, int write)
	58	{
	59	return NULL;
	60	}
	61
	62	#else
	63
	64	struct page *
	65	follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
	66	{
	67	return ERR_PTR(-EINVAL);
	68	}
	69
	70	int pmd_huge(pmd_t pmd)
	71	{
	72	return !!(pmd_val(pmd) & _PAGE_PSE);
	73	}
	74
ceb86879 AK	75	int pud_huge(pud_t pud)
ceb86879 AK	76	{
39c11e6c	77	return !!(pud_val(pud) & _PAGE_PSE);
ceb86879 AK	78	}
ceb86879 AK	79
1da177e4 LT	80	#endif
1da177e4 LT	81
1da177e4 LT	82	/* x86_64 also uses this file */
	83
	84	#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
	85	static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
	86	unsigned long addr, unsigned long len,
	87	unsigned long pgoff, unsigned long flags)
	88	{
39c11e6c	89	struct hstate *h = hstate_file(file);
cdc17344 ML	90	struct vm_unmapped_area_info info;
	91
	92	info.flags = 0;
	93	info.length = len;
	94	info.low_limit = TASK_UNMAPPED_BASE;
	95	info.high_limit = TASK_SIZE;
	96	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	97	info.align_offset = 0;
	98	return vm_unmapped_area(&info);
1da177e4 LT	99	}
	100
	101	static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
	102	unsigned long addr0, unsigned long len,
	103	unsigned long pgoff, unsigned long flags)
	104	{
39c11e6c	105	struct hstate *h = hstate_file(file);
cdc17344 ML	106	struct vm_unmapped_area_info info;
cdc17344 ML	107	unsigned long addr;
1da177e4	108
cdc17344 ML	109	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
	110	info.length = len;
	111	info.low_limit = PAGE_SIZE;
	112	info.high_limit = current->mm->mmap_base;
	113	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	114	info.align_offset = 0;
	115	addr = vm_unmapped_area(&info);
1da177e4	116
1da177e4 LT	117	/*
	118	* A failed mmap() very likely causes application failure,
	119	* so fall back to the bottom-up function here. This scenario
	120	* can happen with large stack limits and large mmap()
	121	* allocations.
	122	*/
cdc17344 ML	123	if (addr & ~PAGE_MASK) {
	124	VM_BUG_ON(addr != -ENOMEM);
	125	info.flags = 0;
	126	info.low_limit = TASK_UNMAPPED_BASE;
	127	info.high_limit = TASK_SIZE;
	128	addr = vm_unmapped_area(&info);
	129	}
1da177e4 LT	130
	131	return addr;
	132	}
	133
	134	unsigned long
	135	hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
	136	unsigned long len, unsigned long pgoff, unsigned long flags)
	137	{
39c11e6c	138	struct hstate *h = hstate_file(file);
1da177e4 LT	139	struct mm_struct *mm = current->mm;
	140	struct vm_area_struct *vma;
	141
39c11e6c	142	if (len & ~huge_page_mask(h))
1da177e4 LT	143	return -EINVAL;
	144	if (len > TASK_SIZE)
	145	return -ENOMEM;
	146
5a8130f2	147	if (flags & MAP_FIXED) {
a5516438	148	if (prepare_hugepage_range(file, addr, len))
5a8130f2 BH	149	return -EINVAL;
	150	return addr;
	151	}
	152
1da177e4	153	if (addr) {
39c11e6c	154	addr = ALIGN(addr, huge_page_size(h));
1da177e4 LT	155	vma = find_vma(mm, addr);
	156	if (TASK_SIZE - len >= addr &&
	157	(!vma \|\| addr + len <= vma->vm_start))
	158	return addr;
	159	}
	160	if (mm->get_unmapped_area == arch_get_unmapped_area)
	161	return hugetlb_get_unmapped_area_bottomup(file, addr, len,
	162	pgoff, flags);
	163	else
	164	return hugetlb_get_unmapped_area_topdown(file, addr, len,
	165	pgoff, flags);
	166	}
	167
	168	#endif /HAVE_ARCH_HUGETLB_UNMAPPED_AREA/
	169
b4718e62 AK	170	#ifdef CONFIG_X86_64
	171	static __init int setup_hugepagesz(char *opt)
	172	{
	173	unsigned long ps = memparse(opt, &opt);
	174	if (ps == PMD_SIZE) {
	175	hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
	176	} else if (ps == PUD_SIZE && cpu_has_gbpages) {
	177	hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
	178	} else {
	179	printk(KERN_ERR "hugepagesz: Unsupported page size %lu M\n",
	180	ps >> 20);
	181	return 0;
	182	}
	183	return 1;
	184	}
	185	__setup("hugepagesz=", setup_hugepagesz);
	186	#endif