[mirror_ubuntu-jammy-kernel.git] / arch / tile / mm / hugetlbpage.c

/*
 * Copyright 2010 Tilera Corporation. All Rights Reserved.
 *
 *   This program is free software; you can redistribute it and/or
 *   modify it under the terms of the GNU General Public License
 *   as published by the Free Software Foundation, version 2.
 *
 *   This program is distributed in the hope that it will be useful, but
 *   WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
 *   NON INFRINGEMENT.  See the GNU General Public License for
 *   more details.
 *
 * TILE Huge TLB Page Support for Kernel.
 * Taken from i386 hugetlb implementation:
 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
 */

#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/sched/mm.h>
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/sysctl.h>
#include <linux/mman.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>
#include <asm/setup.h>

#ifdef CONFIG_HUGETLB_SUPER_PAGES

/*
 * Provide an additional huge page size (in addition to the regular default
 * huge page size) if no "hugepagesz" arguments are specified.
 * Note that it must be smaller than the default huge page size so
 * that it's possible to allocate them on demand from the buddy allocator.
 * You can change this to 64K (on a 16K build), 256K, 1M, or 4M,
 * or not define it at all.
 */
#define ADDITIONAL_HUGE_SIZE (1024 * 1024UL)

/* "Extra" page-size multipliers, one per level of the page table. */
int huge_shift[HUGE_SHIFT_ENTRIES] = {
#ifdef ADDITIONAL_HUGE_SIZE
#define ADDITIONAL_HUGE_SHIFT __builtin_ctzl(ADDITIONAL_HUGE_SIZE / PAGE_SIZE)
	[HUGE_SHIFT_PAGE] = ADDITIONAL_HUGE_SHIFT
#endif
};

#endif

pte_t *huge_pte_alloc(struct mm_struct *mm,
		      unsigned long addr, unsigned long sz)
{
	pgd_t *pgd;
	pud_t *pud;

	addr &= -sz;   /* Mask off any low bits in the address. */

	pgd = pgd_offset(mm, addr);
	pud = pud_alloc(mm, pgd, addr);

#ifdef CONFIG_HUGETLB_SUPER_PAGES
	if (sz >= PGDIR_SIZE) {
		BUG_ON(sz != PGDIR_SIZE &&
		       sz != PGDIR_SIZE << huge_shift[HUGE_SHIFT_PGDIR]);
		return (pte_t *)pud;
	} else {
		pmd_t *pmd = pmd_alloc(mm, pud, addr);
		if (sz >= PMD_SIZE) {
			BUG_ON(sz != PMD_SIZE &&
			       sz != (PMD_SIZE << huge_shift[HUGE_SHIFT_PMD]));
			return (pte_t *)pmd;
		}
		else {
			if (sz != PAGE_SIZE << huge_shift[HUGE_SHIFT_PAGE])
				panic("Unexpected page size %#lx\n", sz);
			return pte_alloc_map(mm, pmd, addr);
		}
	}
#else
	BUG_ON(sz != PMD_SIZE);
	return (pte_t *) pmd_alloc(mm, pud, addr);
#endif
}

static pte_t *get_pte(pte_t *base, int index, int level)
{
	pte_t *ptep = base + index;
#ifdef CONFIG_HUGETLB_SUPER_PAGES
	if (!pte_present(*ptep) && huge_shift[level] != 0) {
		unsigned long mask = -1UL << huge_shift[level];
		pte_t *super_ptep = base + (index & mask);
		pte_t pte = *super_ptep;
		if (pte_present(pte) && pte_super(pte))
			ptep = super_ptep;
	}
#endif
	return ptep;
}

pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
#ifdef CONFIG_HUGETLB_SUPER_PAGES
	pte_t *pte;
#endif

	/* Get the top-level page table entry. */
	pgd = (pgd_t *)get_pte((pte_t *)mm->pgd, pgd_index(addr), 0);

	/* We don't have four levels. */
	pud = pud_offset(pgd, addr);
#ifndef __PAGETABLE_PUD_FOLDED
# error support fourth page table level
#endif
	if (!pud_present(*pud))
		return NULL;

	/* Check for an L0 huge PTE, if we have three levels. */
#ifndef __PAGETABLE_PMD_FOLDED
	if (pud_huge(*pud))
		return (pte_t *)pud;

	pmd = (pmd_t *)get_pte((pte_t *)pud_page_vaddr(*pud),
			       pmd_index(addr), 1);
	if (!pmd_present(*pmd))
		return NULL;
#else
	pmd = pmd_offset(pud, addr);
#endif

	/* Check for an L1 huge PTE. */
	if (pmd_huge(*pmd))
		return (pte_t *)pmd;

#ifdef CONFIG_HUGETLB_SUPER_PAGES
	/* Check for an L2 huge PTE. */
	pte = get_pte((pte_t *)pmd_page_vaddr(*pmd), pte_index(addr), 2);
	if (!pte_present(*pte))
		return NULL;
	if (pte_super(*pte))
		return pte;
#endif

	return NULL;
}

int pmd_huge(pmd_t pmd)
{
	return !!(pmd_val(pmd) & _PAGE_HUGE_PAGE);
}

int pud_huge(pud_t pud)
{
	return !!(pud_val(pud) & _PAGE_HUGE_PAGE);
}

#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
		unsigned long addr, unsigned long len,
		unsigned long pgoff, unsigned long flags)
{
	struct hstate *h = hstate_file(file);
	struct vm_unmapped_area_info info;

	info.flags = 0;
	info.length = len;
	info.low_limit = TASK_UNMAPPED_BASE;
	info.high_limit = TASK_SIZE;
	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	info.align_offset = 0;
	return vm_unmapped_area(&info);
}

static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
		unsigned long addr0, unsigned long len,
		unsigned long pgoff, unsigned long flags)
{
	struct hstate *h = hstate_file(file);
	struct vm_unmapped_area_info info;
	unsigned long addr;

	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
	info.length = len;
	info.low_limit = PAGE_SIZE;
	info.high_limit = current->mm->mmap_base;
	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	info.align_offset = 0;
	addr = vm_unmapped_area(&info);

	/*
	 * A failed mmap() very likely causes application failure,
	 * so fall back to the bottom-up function here. This scenario
	 * can happen with large stack limits and large mmap()
	 * allocations.
	 */
	if (addr & ~PAGE_MASK) {
		VM_BUG_ON(addr != -ENOMEM);
		info.flags = 0;
		info.low_limit = TASK_UNMAPPED_BASE;
		info.high_limit = TASK_SIZE;
		addr = vm_unmapped_area(&info);
	}

	return addr;
}

unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
		unsigned long len, unsigned long pgoff, unsigned long flags)
{
	struct hstate *h = hstate_file(file);
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma;

	if (len & ~huge_page_mask(h))
		return -EINVAL;
	if (len > TASK_SIZE)
		return -ENOMEM;

	if (flags & MAP_FIXED) {
		if (prepare_hugepage_range(file, addr, len))
			return -EINVAL;
		return addr;
	}

	if (addr) {
		addr = ALIGN(addr, huge_page_size(h));
		vma = find_vma(mm, addr);
		if (TASK_SIZE - len >= addr &&
		    (!vma || addr + len <= vm_start_gap(vma)))
			return addr;
	}
	if (current->mm->get_unmapped_area == arch_get_unmapped_area)
		return hugetlb_get_unmapped_area_bottomup(file, addr, len,
				pgoff, flags);
	else
		return hugetlb_get_unmapped_area_topdown(file, addr, len,
				pgoff, flags);
}
#endif /* HAVE_ARCH_HUGETLB_UNMAPPED_AREA */

#ifdef CONFIG_HUGETLB_SUPER_PAGES
static __init int __setup_hugepagesz(unsigned long ps)
{
	int log_ps = __builtin_ctzl(ps);
	int level, base_shift;

	if ((1UL << log_ps) != ps || (log_ps & 1) != 0) {
		pr_warn("Not enabling %ld byte huge pages; must be a power of four\n",
			ps);
		return -EINVAL;
	}

	if (ps > 64*1024*1024*1024UL) {
		pr_warn("Not enabling %ld MB huge pages; largest legal value is 64 GB\n",
			ps >> 20);
		return -EINVAL;
	} else if (ps >= PUD_SIZE) {
		static long hv_jpage_size;
		if (hv_jpage_size == 0)
			hv_jpage_size = hv_sysconf(HV_SYSCONF_PAGE_SIZE_JUMBO);
		if (hv_jpage_size != PUD_SIZE) {
			pr_warn("Not enabling >= %ld MB huge pages: hypervisor reports size %ld\n",
				PUD_SIZE >> 20, hv_jpage_size);
			return -EINVAL;
		}
		level = 0;
		base_shift = PUD_SHIFT;
	} else if (ps >= PMD_SIZE) {
		level = 1;
		base_shift = PMD_SHIFT;
	} else if (ps > PAGE_SIZE) {
		level = 2;
		base_shift = PAGE_SHIFT;
	} else {
		pr_err("hugepagesz: huge page size %ld too small\n", ps);
		return -EINVAL;
	}

	if (log_ps != base_shift) {
		int shift_val = log_ps - base_shift;
		if (huge_shift[level] != 0) {
			int old_shift = base_shift + huge_shift[level];
			pr_warn("Not enabling %ld MB huge pages; already have size %ld MB\n",
				ps >> 20, (1UL << old_shift) >> 20);
			return -EINVAL;
		}
		if (hv_set_pte_super_shift(level, shift_val) != 0) {
			pr_warn("Not enabling %ld MB huge pages; no hypervisor support\n",
				ps >> 20);
			return -EINVAL;
		}
		printk(KERN_DEBUG "Enabled %ld MB huge pages\n", ps >> 20);
		huge_shift[level] = shift_val;
	}

	hugetlb_add_hstate(log_ps - PAGE_SHIFT);

	return 0;
}

static bool saw_hugepagesz;

static __init int setup_hugepagesz(char *opt)
{
	int rc;

	if (!saw_hugepagesz) {
		saw_hugepagesz = true;
		memset(huge_shift, 0, sizeof(huge_shift));
	}
	rc = __setup_hugepagesz(memparse(opt, NULL));
	if (rc)
		hugetlb_bad_size();
	return rc;
}
__setup("hugepagesz=", setup_hugepagesz);

#ifdef ADDITIONAL_HUGE_SIZE
/*
 * Provide an additional huge page size if no "hugepagesz" args are given.
 * In that case, all the cores have properly set up their hv super_shift
 * already, but we need to notify the hugetlb code to enable the
 * new huge page size from the Linux point of view.
 */
static __init int add_default_hugepagesz(void)
{
	if (!saw_hugepagesz) {
		BUILD_BUG_ON(ADDITIONAL_HUGE_SIZE >= PMD_SIZE ||
			     ADDITIONAL_HUGE_SIZE <= PAGE_SIZE);
		BUILD_BUG_ON((PAGE_SIZE << ADDITIONAL_HUGE_SHIFT) !=
			     ADDITIONAL_HUGE_SIZE);
		BUILD_BUG_ON(ADDITIONAL_HUGE_SHIFT & 1);
		hugetlb_add_hstate(ADDITIONAL_HUGE_SHIFT);
	}
	return 0;
}
arch_initcall(add_default_hugepagesz);
#endif

#endif /* CONFIG_HUGETLB_SUPER_PAGES */
Commit	Line	Data
867e359b CM	1	/*
	2	* Copyright 2010 Tilera Corporation. All Rights Reserved.
	3	*
	4	* This program is free software; you can redistribute it and/or
	5	* modify it under the terms of the GNU General Public License
	6	* as published by the Free Software Foundation, version 2.
	7	*
	8	* This program is distributed in the hope that it will be useful, but
	9	* WITHOUT ANY WARRANTY; without even the implied warranty of
	10	* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
	11	* NON INFRINGEMENT. See the GNU General Public License for
	12	* more details.
	13	*
	14	* TILE Huge TLB Page Support for Kernel.
	15	* Taken from i386 hugetlb implementation:
	16	* Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
	17	*/
	18
	19	#include <linux/init.h>
	20	#include <linux/fs.h>
	21	#include <linux/mm.h>
01042607	22	#include <linux/sched/mm.h>
867e359b CM	23	#include <linux/hugetlb.h>
867e359b CM	24	#include <linux/pagemap.h>
867e359b CM	25	#include <linux/slab.h>
	26	#include <linux/err.h>
	27	#include <linux/sysctl.h>
	28	#include <linux/mman.h>
	29	#include <asm/tlb.h>
	30	#include <asm/tlbflush.h>
621b1955 CM	31	#include <asm/setup.h>
	32
	33	#ifdef CONFIG_HUGETLB_SUPER_PAGES
	34
	35	/*
	36	* Provide an additional huge page size (in addition to the regular default
	37	* huge page size) if no "hugepagesz" arguments are specified.
	38	* Note that it must be smaller than the default huge page size so
	39	* that it's possible to allocate them on demand from the buddy allocator.
	40	* You can change this to 64K (on a 16K build), 256K, 1M, or 4M,
	41	* or not define it at all.
	42	*/
	43	#define ADDITIONAL_HUGE_SIZE (1024 * 1024UL)
	44
	45	/* "Extra" page-size multipliers, one per level of the page table. */
	46	int huge_shift[HUGE_SHIFT_ENTRIES] = {
	47	#ifdef ADDITIONAL_HUGE_SIZE
	48	#define ADDITIONAL_HUGE_SHIFT __builtin_ctzl(ADDITIONAL_HUGE_SIZE / PAGE_SIZE)
	49	[HUGE_SHIFT_PAGE] = ADDITIONAL_HUGE_SHIFT
	50	#endif
	51	};
	52
621b1955	53	#endif
867e359b CM	54
	55	pte_t huge_pte_alloc(struct mm_struct mm,
	56	unsigned long addr, unsigned long sz)
	57	{
	58	pgd_t *pgd;
	59	pud_t *pud;
867e359b	60
621b1955	61	addr &= -sz; /* Mask off any low bits in the address. */
867e359b CM	62
	63	pgd = pgd_offset(mm, addr);
	64	pud = pud_alloc(mm, pgd, addr);
867e359b	65
621b1955 CM	66	#ifdef CONFIG_HUGETLB_SUPER_PAGES
	67	if (sz >= PGDIR_SIZE) {
	68	BUG_ON(sz != PGDIR_SIZE &&
	69	sz != PGDIR_SIZE << huge_shift[HUGE_SHIFT_PGDIR]);
	70	return (pte_t *)pud;
	71	} else {
	72	pmd_t *pmd = pmd_alloc(mm, pud, addr);
	73	if (sz >= PMD_SIZE) {
	74	BUG_ON(sz != PMD_SIZE &&
	75	sz != (PMD_SIZE << huge_shift[HUGE_SHIFT_PMD]));
	76	return (pte_t *)pmd;
	77	}
	78	else {
	79	if (sz != PAGE_SIZE << huge_shift[HUGE_SHIFT_PAGE])
	80	panic("Unexpected page size %#lx\n", sz);
3ed3a4f0	81	return pte_alloc_map(mm, pmd, addr);
621b1955 CM	82	}
	83	}
	84	#else
	85	BUG_ON(sz != PMD_SIZE);
	86	return (pte_t *) pmd_alloc(mm, pud, addr);
	87	#endif
867e359b CM	88	}
867e359b CM	89
621b1955	90	static pte_t get_pte(pte_t base, int index, int level)
867e359b	91	{
621b1955 CM	92	pte_t *ptep = base + index;
	93	#ifdef CONFIG_HUGETLB_SUPER_PAGES
	94	if (!pte_present(*ptep) && huge_shift[level] != 0) {
	95	unsigned long mask = -1UL << huge_shift[level];
	96	pte_t *super_ptep = base + (index & mask);
	97	pte_t pte = *super_ptep;
	98	if (pte_present(pte) && pte_super(pte))
	99	ptep = super_ptep;
867e359b	100	}
621b1955 CM	101	#endif
621b1955 CM	102	return ptep;
867e359b CM	103	}
867e359b CM	104
621b1955	105	pte_t huge_pte_offset(struct mm_struct mm, unsigned long addr)
867e359b	106	{
621b1955 CM	107	pgd_t *pgd;
	108	pud_t *pud;
	109	pmd_t *pmd;
	110	#ifdef CONFIG_HUGETLB_SUPER_PAGES
	111	pte_t *pte;
	112	#endif
867e359b	113
621b1955 CM	114	/* Get the top-level page table entry. */
621b1955 CM	115	pgd = (pgd_t )get_pte((pte_t )mm->pgd, pgd_index(addr), 0);
867e359b	116
621b1955 CM	117	/* We don't have four levels. */
	118	pud = pud_offset(pgd, addr);
	119	#ifndef __PAGETABLE_PUD_FOLDED
	120	# error support fourth page table level
	121	#endif
a0bd12d7 CM	122	if (!pud_present(*pud))
a0bd12d7 CM	123	return NULL;
867e359b	124
621b1955 CM	125	/* Check for an L0 huge PTE, if we have three levels. */
	126	#ifndef __PAGETABLE_PMD_FOLDED
	127	if (pud_huge(*pud))
	128	return (pte_t *)pud;
867e359b	129
621b1955 CM	130	pmd = (pmd_t )get_pte((pte_t )pud_page_vaddr(*pud),
	131	pmd_index(addr), 1);
	132	if (!pmd_present(*pmd))
	133	return NULL;
	134	#else
	135	pmd = pmd_offset(pud, addr);
	136	#endif
867e359b	137
621b1955 CM	138	/* Check for an L1 huge PTE. */
	139	if (pmd_huge(*pmd))
	140	return (pte_t *)pmd;
	141
	142	#ifdef CONFIG_HUGETLB_SUPER_PAGES
	143	/* Check for an L2 huge PTE. */
	144	pte = get_pte((pte_t )pmd_page_vaddr(pmd), pte_index(addr), 2);
	145	if (!pte_present(*pte))
	146	return NULL;
	147	if (pte_super(*pte))
	148	return pte;
	149	#endif
867e359b	150
867e359b CM	151	return NULL;
	152	}
	153
867e359b CM	154	int pmd_huge(pmd_t pmd)
	155	{
	156	return !!(pmd_val(pmd) & _PAGE_HUGE_PAGE);
	157	}
	158
	159	int pud_huge(pud_t pud)
	160	{
	161	return !!(pud_val(pud) & _PAGE_HUGE_PAGE);
	162	}
	163
867e359b CM	164	#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
	165	static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
	166	unsigned long addr, unsigned long len,
	167	unsigned long pgoff, unsigned long flags)
	168	{
	169	struct hstate *h = hstate_file(file);
dd529596 ML	170	struct vm_unmapped_area_info info;
	171
	172	info.flags = 0;
	173	info.length = len;
	174	info.low_limit = TASK_UNMAPPED_BASE;
	175	info.high_limit = TASK_SIZE;
	176	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	177	info.align_offset = 0;
	178	return vm_unmapped_area(&info);
867e359b CM	179	}
	180
	181	static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
	182	unsigned long addr0, unsigned long len,
	183	unsigned long pgoff, unsigned long flags)
	184	{
	185	struct hstate *h = hstate_file(file);
dd529596 ML	186	struct vm_unmapped_area_info info;
dd529596 ML	187	unsigned long addr;
867e359b	188
dd529596 ML	189	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
	190	info.length = len;
	191	info.low_limit = PAGE_SIZE;
	192	info.high_limit = current->mm->mmap_base;
	193	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	194	info.align_offset = 0;
	195	addr = vm_unmapped_area(&info);
867e359b	196
867e359b CM	197	/*
	198	* A failed mmap() very likely causes application failure,
	199	* so fall back to the bottom-up function here. This scenario
	200	* can happen with large stack limits and large mmap()
	201	* allocations.
	202	*/
dd529596 ML	203	if (addr & ~PAGE_MASK) {
	204	VM_BUG_ON(addr != -ENOMEM);
	205	info.flags = 0;
	206	info.low_limit = TASK_UNMAPPED_BASE;
	207	info.high_limit = TASK_SIZE;
	208	addr = vm_unmapped_area(&info);
	209	}
867e359b CM	210
	211	return addr;
	212	}
	213
	214	unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
	215	unsigned long len, unsigned long pgoff, unsigned long flags)
	216	{
	217	struct hstate *h = hstate_file(file);
	218	struct mm_struct *mm = current->mm;
	219	struct vm_area_struct *vma;
	220
	221	if (len & ~huge_page_mask(h))
	222	return -EINVAL;
	223	if (len > TASK_SIZE)
	224	return -ENOMEM;
	225
	226	if (flags & MAP_FIXED) {
	227	if (prepare_hugepage_range(file, addr, len))
	228	return -EINVAL;
	229	return addr;
	230	}
	231
	232	if (addr) {
	233	addr = ALIGN(addr, huge_page_size(h));
	234	vma = find_vma(mm, addr);
	235	if (TASK_SIZE - len >= addr &&
1be7107f	236	(!vma \|\| addr + len <= vm_start_gap(vma)))
867e359b CM	237	return addr;
	238	}
	239	if (current->mm->get_unmapped_area == arch_get_unmapped_area)
	240	return hugetlb_get_unmapped_area_bottomup(file, addr, len,
	241	pgoff, flags);
	242	else
	243	return hugetlb_get_unmapped_area_topdown(file, addr, len,
	244	pgoff, flags);
	245	}
621b1955	246	#endif /* HAVE_ARCH_HUGETLB_UNMAPPED_AREA */
867e359b	247
621b1955 CM	248	#ifdef CONFIG_HUGETLB_SUPER_PAGES
621b1955 CM	249	static __init int __setup_hugepagesz(unsigned long ps)
867e359b	250	{
621b1955 CM	251	int log_ps = __builtin_ctzl(ps);
	252	int level, base_shift;
	253
	254	if ((1UL << log_ps) != ps \|\| (log_ps & 1) != 0) {
f4743673 JP	255	pr_warn("Not enabling %ld byte huge pages; must be a power of four\n",
f4743673 JP	256	ps);
621b1955 CM	257	return -EINVAL;
	258	}
	259
	260	if (ps > 6410241024*1024UL) {
f4743673 JP	261	pr_warn("Not enabling %ld MB huge pages; largest legal value is 64 GB\n",
f4743673 JP	262	ps >> 20);
621b1955 CM	263	return -EINVAL;
	264	} else if (ps >= PUD_SIZE) {
	265	static long hv_jpage_size;
	266	if (hv_jpage_size == 0)
	267	hv_jpage_size = hv_sysconf(HV_SYSCONF_PAGE_SIZE_JUMBO);
	268	if (hv_jpage_size != PUD_SIZE) {
f4743673	269	pr_warn("Not enabling >= %ld MB huge pages: hypervisor reports size %ld\n",
621b1955 CM	270	PUD_SIZE >> 20, hv_jpage_size);
	271	return -EINVAL;
	272	}
	273	level = 0;
	274	base_shift = PUD_SHIFT;
	275	} else if (ps >= PMD_SIZE) {
	276	level = 1;
	277	base_shift = PMD_SHIFT;
	278	} else if (ps > PAGE_SIZE) {
	279	level = 2;
	280	base_shift = PAGE_SHIFT;
867e359b	281	} else {
621b1955 CM	282	pr_err("hugepagesz: huge page size %ld too small\n", ps);
621b1955 CM	283	return -EINVAL;
867e359b	284	}
621b1955 CM	285
	286	if (log_ps != base_shift) {
	287	int shift_val = log_ps - base_shift;
	288	if (huge_shift[level] != 0) {
	289	int old_shift = base_shift + huge_shift[level];
f4743673	290	pr_warn("Not enabling %ld MB huge pages; already have size %ld MB\n",
621b1955 CM	291	ps >> 20, (1UL << old_shift) >> 20);
	292	return -EINVAL;
	293	}
	294	if (hv_set_pte_super_shift(level, shift_val) != 0) {
f4743673 JP	295	pr_warn("Not enabling %ld MB huge pages; no hypervisor support\n",
f4743673 JP	296	ps >> 20);
621b1955 CM	297	return -EINVAL;
	298	}
	299	printk(KERN_DEBUG "Enabled %ld MB huge pages\n", ps >> 20);
	300	huge_shift[level] = shift_val;
	301	}
	302
	303	hugetlb_add_hstate(log_ps - PAGE_SHIFT);
	304
	305	return 0;
	306	}
	307
	308	static bool saw_hugepagesz;
	309
	310	static __init int setup_hugepagesz(char *opt)
	311	{
b3d424f1 VT	312	int rc;
b3d424f1 VT	313
621b1955 CM	314	if (!saw_hugepagesz) {
	315	saw_hugepagesz = true;
	316	memset(huge_shift, 0, sizeof(huge_shift));
	317	}
b3d424f1 VT	318	rc = __setup_hugepagesz(memparse(opt, NULL));
	319	if (rc)
	320	hugetlb_bad_size();
	321	return rc;
867e359b CM	322	}
	323	__setup("hugepagesz=", setup_hugepagesz);
	324
621b1955 CM	325	#ifdef ADDITIONAL_HUGE_SIZE
	326	/*
	327	* Provide an additional huge page size if no "hugepagesz" args are given.
	328	* In that case, all the cores have properly set up their hv super_shift
	329	* already, but we need to notify the hugetlb code to enable the
	330	* new huge page size from the Linux point of view.
	331	*/
	332	static __init int add_default_hugepagesz(void)
	333	{
	334	if (!saw_hugepagesz) {
	335	BUILD_BUG_ON(ADDITIONAL_HUGE_SIZE >= PMD_SIZE \|\|
	336	ADDITIONAL_HUGE_SIZE <= PAGE_SIZE);
	337	BUILD_BUG_ON((PAGE_SIZE << ADDITIONAL_HUGE_SHIFT) !=
	338	ADDITIONAL_HUGE_SIZE);
	339	BUILD_BUG_ON(ADDITIONAL_HUGE_SHIFT & 1);
	340	hugetlb_add_hstate(ADDITIONAL_HUGE_SHIFT);
	341	}
	342	return 0;
	343	}
	344	arch_initcall(add_default_hugepagesz);
	345	#endif
	346
	347	#endif /* CONFIG_HUGETLB_SUPER_PAGES */