[mirror_ubuntu-bionic-kernel.git] / arch / x86 / mm / hugetlbpage.c

// SPDX-License-Identifier: GPL-2.0
/*
 * IA-32 Huge TLB Page Support for Kernel.
 *
 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
 */

#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/sched/mm.h>
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
#include <linux/err.h>
#include <linux/sysctl.h>
#include <linux/compat.h>
#include <asm/mman.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>
#include <asm/pgalloc.h>
#include <asm/elf.h>
#include <asm/mpx.h>

#if 0	/* This is just for testing */
struct page *
follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
{
	unsigned long start = address;
	int length = 1;
	int nr;
	struct page *page;
	struct vm_area_struct *vma;

	vma = find_vma(mm, addr);
	if (!vma || !is_vm_hugetlb_page(vma))
		return ERR_PTR(-EINVAL);

	pte = huge_pte_offset(mm, address, vma_mmu_pagesize(vma));

	/* hugetlb should be locked, and hence, prefaulted */
	WARN_ON(!pte || pte_none(*pte));

	page = &pte_page(*pte)[vpfn % (HPAGE_SIZE/PAGE_SIZE)];

	WARN_ON(!PageHead(page));

	return page;
}

int pmd_huge(pmd_t pmd)
{
	return 0;
}

int pud_huge(pud_t pud)
{
	return 0;
}

#else

/*
 * pmd_huge() returns 1 if @pmd is hugetlb related entry, that is normal
 * hugetlb entry or non-present (migration or hwpoisoned) hugetlb entry.
 * Otherwise, returns 0.
 */
int pmd_huge(pmd_t pmd)
{
	return !pmd_none(pmd) &&
		(pmd_val(pmd) & (_PAGE_PRESENT|_PAGE_PSE)) != _PAGE_PRESENT;
}

int pud_huge(pud_t pud)
{
	return !!(pud_val(pud) & _PAGE_PSE);
}
#endif

#ifdef CONFIG_HUGETLB_PAGE
static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
		unsigned long addr, unsigned long len,
		unsigned long pgoff, unsigned long flags)
{
	struct hstate *h = hstate_file(file);
	struct vm_unmapped_area_info info;

	info.flags = 0;
	info.length = len;
	info.low_limit = get_mmap_base(1);

	/*
	 * If hint address is above DEFAULT_MAP_WINDOW, look for unmapped area
	 * in the full address space.
	 */
	info.high_limit = in_compat_syscall() ?
		task_size_32bit() : task_size_64bit(addr > DEFAULT_MAP_WINDOW);

	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	info.align_offset = 0;
	return vm_unmapped_area(&info);
}

static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
		unsigned long addr, unsigned long len,
		unsigned long pgoff, unsigned long flags)
{
	struct hstate *h = hstate_file(file);
	struct vm_unmapped_area_info info;

	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
	info.length = len;
	info.low_limit = PAGE_SIZE;
	info.high_limit = get_mmap_base(0);

	/*
	 * If hint address is above DEFAULT_MAP_WINDOW, look for unmapped area
	 * in the full address space.
	 */
	if (addr > DEFAULT_MAP_WINDOW && !in_compat_syscall())
		info.high_limit += TASK_SIZE_MAX - DEFAULT_MAP_WINDOW;

	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	info.align_offset = 0;
	addr = vm_unmapped_area(&info);

	/*
	 * A failed mmap() very likely causes application failure,
	 * so fall back to the bottom-up function here. This scenario
	 * can happen with large stack limits and large mmap()
	 * allocations.
	 */
	if (addr & ~PAGE_MASK) {
		VM_BUG_ON(addr != -ENOMEM);
		info.flags = 0;
		info.low_limit = TASK_UNMAPPED_BASE;
		info.high_limit = TASK_SIZE_LOW;
		addr = vm_unmapped_area(&info);
	}

	return addr;
}

unsigned long
hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
		unsigned long len, unsigned long pgoff, unsigned long flags)
{
	struct hstate *h = hstate_file(file);
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma;

	if (len & ~huge_page_mask(h))
		return -EINVAL;

	addr = mpx_unmapped_area_check(addr, len, flags);
	if (IS_ERR_VALUE(addr))
		return addr;

	if (len > TASK_SIZE)
		return -ENOMEM;

	if (flags & MAP_FIXED) {
		if (prepare_hugepage_range(file, addr, len))
			return -EINVAL;
		return addr;
	}

	if (addr) {
		addr = ALIGN(addr, huge_page_size(h));
		vma = find_vma(mm, addr);
		if (TASK_SIZE - len >= addr &&
		    (!vma || addr + len <= vm_start_gap(vma)))
			return addr;
	}
	if (mm->get_unmapped_area == arch_get_unmapped_area)
		return hugetlb_get_unmapped_area_bottomup(file, addr, len,
				pgoff, flags);
	else
		return hugetlb_get_unmapped_area_topdown(file, addr, len,
				pgoff, flags);
}
#endif /* CONFIG_HUGETLB_PAGE */

#ifdef CONFIG_X86_64
static __init int setup_hugepagesz(char *opt)
{
	unsigned long ps = memparse(opt, &opt);
	if (ps == PMD_SIZE) {
		hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
	} else if (ps == PUD_SIZE && boot_cpu_has(X86_FEATURE_GBPAGES)) {
		hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
	} else {
		hugetlb_bad_size();
		printk(KERN_ERR "hugepagesz: Unsupported page size %lu M\n",
			ps >> 20);
		return 0;
	}
	return 1;
}
__setup("hugepagesz=", setup_hugepagesz);

#if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA)
static __init int gigantic_pages_init(void)
{
	/* With compaction or CMA we can allocate gigantic pages at runtime */
	if (boot_cpu_has(X86_FEATURE_GBPAGES) && !size_to_hstate(1UL << PUD_SHIFT))
		hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
	return 0;
}
arch_initcall(gigantic_pages_init);
#endif
#endif
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
1da177e4 LT	2	/*
	3	* IA-32 Huge TLB Page Support for Kernel.
	4	*
	5	* Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
	6	*/
	7
1da177e4 LT	8	#include <linux/init.h>
	9	#include <linux/fs.h>
	10	#include <linux/mm.h>
01042607	11	#include <linux/sched/mm.h>
1da177e4 LT	12	#include <linux/hugetlb.h>
1da177e4 LT	13	#include <linux/pagemap.h>
1da177e4 LT	14	#include <linux/err.h>
1da177e4 LT	15	#include <linux/sysctl.h>
e13b73dd	16	#include <linux/compat.h>
1da177e4 LT	17	#include <asm/mman.h>
	18	#include <asm/tlb.h>
	19	#include <asm/tlbflush.h>
a5a19c63	20	#include <asm/pgalloc.h>
e13b73dd	21	#include <asm/elf.h>
44b04912	22	#include <asm/mpx.h>
1da177e4	23
1da177e4 LT	24	#if 0 /* This is just for testing */
	25	struct page *
	26	follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
	27	{
	28	unsigned long start = address;
	29	int length = 1;
	30	int nr;
	31	struct page *page;
	32	struct vm_area_struct *vma;
	33
	34	vma = find_vma(mm, addr);
	35	if (!vma \|\| !is_vm_hugetlb_page(vma))
	36	return ERR_PTR(-EINVAL);
	37
7868a208	38	pte = huge_pte_offset(mm, address, vma_mmu_pagesize(vma));
1da177e4 LT	39
	40	/* hugetlb should be locked, and hence, prefaulted */
	41	WARN_ON(!pte \|\| pte_none(*pte));
	42
	43	page = &pte_page(*pte)[vpfn % (HPAGE_SIZE/PAGE_SIZE)];
	44
25e59881	45	WARN_ON(!PageHead(page));
1da177e4 LT	46
	47	return page;
	48	}
	49
	50	int pmd_huge(pmd_t pmd)
	51	{
	52	return 0;
	53	}
	54
ceb86879 AK	55	int pud_huge(pud_t pud)
	56	{
	57	return 0;
	58	}
	59
1da177e4 LT	60	#else
1da177e4 LT	61
cbef8478 NH	62	/*
	63	* pmd_huge() returns 1 if @pmd is hugetlb related entry, that is normal
	64	* hugetlb entry or non-present (migration or hwpoisoned) hugetlb entry.
	65	* Otherwise, returns 0.
	66	*/
1da177e4 LT	67	int pmd_huge(pmd_t pmd)
1da177e4 LT	68	{
cbef8478 NH	69	return !pmd_none(pmd) &&
cbef8478 NH	70	(pmd_val(pmd) & (_PAGE_PRESENT\|_PAGE_PSE)) != _PAGE_PRESENT;
1da177e4 LT	71	}
1da177e4 LT	72
ceb86879 AK	73	int pud_huge(pud_t pud)
ceb86879 AK	74	{
39c11e6c	75	return !!(pud_val(pud) & _PAGE_PSE);
ceb86879	76	}
1da177e4 LT	77	#endif
1da177e4 LT	78
fd8526ad	79	#ifdef CONFIG_HUGETLB_PAGE
1da177e4 LT	80	static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
	81	unsigned long addr, unsigned long len,
	82	unsigned long pgoff, unsigned long flags)
	83	{
39c11e6c	84	struct hstate *h = hstate_file(file);
cdc17344 ML	85	struct vm_unmapped_area_info info;
	86
	87	info.flags = 0;
	88	info.length = len;
e13b73dd	89	info.low_limit = get_mmap_base(1);
b569bab7 KS	90
	91	/*
	92	* If hint address is above DEFAULT_MAP_WINDOW, look for unmapped area
	93	* in the full address space.
	94	*/
e13b73dd	95	info.high_limit = in_compat_syscall() ?
b569bab7 KS	96	task_size_32bit() : task_size_64bit(addr > DEFAULT_MAP_WINDOW);
b569bab7 KS	97
cdc17344 ML	98	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	99	info.align_offset = 0;
	100	return vm_unmapped_area(&info);
1da177e4 LT	101	}
	102
	103	static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
b569bab7	104	unsigned long addr, unsigned long len,
1da177e4 LT	105	unsigned long pgoff, unsigned long flags)
1da177e4 LT	106	{
39c11e6c	107	struct hstate *h = hstate_file(file);
cdc17344	108	struct vm_unmapped_area_info info;
1da177e4	109
cdc17344 ML	110	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
	111	info.length = len;
	112	info.low_limit = PAGE_SIZE;
e13b73dd	113	info.high_limit = get_mmap_base(0);
b569bab7 KS	114
	115	/*
	116	* If hint address is above DEFAULT_MAP_WINDOW, look for unmapped area
	117	* in the full address space.
	118	*/
	119	if (addr > DEFAULT_MAP_WINDOW && !in_compat_syscall())
	120	info.high_limit += TASK_SIZE_MAX - DEFAULT_MAP_WINDOW;
	121
cdc17344 ML	122	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	123	info.align_offset = 0;
	124	addr = vm_unmapped_area(&info);
1da177e4	125
1da177e4 LT	126	/*
	127	* A failed mmap() very likely causes application failure,
	128	* so fall back to the bottom-up function here. This scenario
	129	* can happen with large stack limits and large mmap()
	130	* allocations.
	131	*/
cdc17344 ML	132	if (addr & ~PAGE_MASK) {
	133	VM_BUG_ON(addr != -ENOMEM);
	134	info.flags = 0;
	135	info.low_limit = TASK_UNMAPPED_BASE;
b569bab7	136	info.high_limit = TASK_SIZE_LOW;
cdc17344 ML	137	addr = vm_unmapped_area(&info);
cdc17344 ML	138	}
1da177e4 LT	139
	140	return addr;
	141	}
	142
	143	unsigned long
	144	hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
	145	unsigned long len, unsigned long pgoff, unsigned long flags)
	146	{
39c11e6c	147	struct hstate *h = hstate_file(file);
1da177e4 LT	148	struct mm_struct *mm = current->mm;
	149	struct vm_area_struct *vma;
	150
39c11e6c	151	if (len & ~huge_page_mask(h))
1da177e4	152	return -EINVAL;
44b04912 KS	153
	154	addr = mpx_unmapped_area_check(addr, len, flags);
	155	if (IS_ERR_VALUE(addr))
	156	return addr;
	157
1da177e4 LT	158	if (len > TASK_SIZE)
	159	return -ENOMEM;
	160
5a8130f2	161	if (flags & MAP_FIXED) {
a5516438	162	if (prepare_hugepage_range(file, addr, len))
5a8130f2 BH	163	return -EINVAL;
	164	return addr;
	165	}
	166
1da177e4	167	if (addr) {
39c11e6c	168	addr = ALIGN(addr, huge_page_size(h));
1da177e4 LT	169	vma = find_vma(mm, addr);
1da177e4 LT	170	if (TASK_SIZE - len >= addr &&
1be7107f	171	(!vma \|\| addr + len <= vm_start_gap(vma)))
1da177e4 LT	172	return addr;
	173	}
	174	if (mm->get_unmapped_area == arch_get_unmapped_area)
	175	return hugetlb_get_unmapped_area_bottomup(file, addr, len,
	176	pgoff, flags);
	177	else
	178	return hugetlb_get_unmapped_area_topdown(file, addr, len,
	179	pgoff, flags);
	180	}
fd8526ad	181	#endif /* CONFIG_HUGETLB_PAGE */
1da177e4	182
b4718e62 AK	183	#ifdef CONFIG_X86_64
	184	static __init int setup_hugepagesz(char *opt)
	185	{
	186	unsigned long ps = memparse(opt, &opt);
	187	if (ps == PMD_SIZE) {
	188	hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
b8291adc	189	} else if (ps == PUD_SIZE && boot_cpu_has(X86_FEATURE_GBPAGES)) {
b4718e62 AK	190	hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
b4718e62 AK	191	} else {
2b18e532	192	hugetlb_bad_size();
b4718e62 AK	193	printk(KERN_ERR "hugepagesz: Unsupported page size %lu M\n",
	194	ps >> 20);
	195	return 0;
	196	}
	197	return 1;
	198	}
	199	__setup("hugepagesz=", setup_hugepagesz);
ece84b39	200
080fe206	201	#if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) \|\| defined(CONFIG_CMA)
ece84b39 KS	202	static __init int gigantic_pages_init(void)
ece84b39 KS	203	{
080fe206	204	/* With compaction or CMA we can allocate gigantic pages at runtime */
b8291adc	205	if (boot_cpu_has(X86_FEATURE_GBPAGES) && !size_to_hstate(1UL << PUD_SHIFT))
ece84b39 KS	206	hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
	207	return 0;
	208	}
	209	arch_initcall(gigantic_pages_init);
	210	#endif
b4718e62	211	#endif