2 * IA-32 Huge TLB Page Support for Kernel.
4 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
7 #include <linux/init.h>
10 #include <linux/hugetlb.h>
11 #include <linux/pagemap.h>
12 #include <linux/err.h>
13 #include <linux/sysctl.h>
16 #include <asm/tlbflush.h>
17 #include <asm/pgalloc.h>
19 #if 0 /* This is just for testing */
21 follow_huge_addr(struct mm_struct
*mm
, unsigned long address
, int write
)
23 unsigned long start
= address
;
27 struct vm_area_struct
*vma
;
29 vma
= find_vma(mm
, addr
);
30 if (!vma
|| !is_vm_hugetlb_page(vma
))
31 return ERR_PTR(-EINVAL
);
33 pte
= huge_pte_offset(mm
, address
);
35 /* hugetlb should be locked, and hence, prefaulted */
36 WARN_ON(!pte
|| pte_none(*pte
));
38 page
= &pte_page(*pte
)[vpfn
% (HPAGE_SIZE
/PAGE_SIZE
)];
40 WARN_ON(!PageHead(page
));
45 int pmd_huge(pmd_t pmd
)
50 int pud_huge(pud_t pud
)
56 follow_huge_pmd(struct mm_struct
*mm
, unsigned long address
,
57 pmd_t
*pmd
, int write
)
65 follow_huge_addr(struct mm_struct
*mm
, unsigned long address
, int write
)
67 return ERR_PTR(-EINVAL
);
70 int pmd_huge(pmd_t pmd
)
72 return !!(pmd_val(pmd
) & _PAGE_PSE
);
75 int pud_huge(pud_t pud
)
77 return !!(pud_val(pud
) & _PAGE_PSE
);
82 /* x86_64 also uses this file */
84 #ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
85 static unsigned long hugetlb_get_unmapped_area_bottomup(struct file
*file
,
86 unsigned long addr
, unsigned long len
,
87 unsigned long pgoff
, unsigned long flags
)
89 struct hstate
*h
= hstate_file(file
);
90 struct vm_unmapped_area_info info
;
94 info
.low_limit
= TASK_UNMAPPED_BASE
;
95 info
.high_limit
= TASK_SIZE
;
96 info
.align_mask
= PAGE_MASK
& ~huge_page_mask(h
);
97 info
.align_offset
= 0;
98 return vm_unmapped_area(&info
);
101 static unsigned long hugetlb_get_unmapped_area_topdown(struct file
*file
,
102 unsigned long addr0
, unsigned long len
,
103 unsigned long pgoff
, unsigned long flags
)
105 struct hstate
*h
= hstate_file(file
);
106 struct vm_unmapped_area_info info
;
109 info
.flags
= VM_UNMAPPED_AREA_TOPDOWN
;
111 info
.low_limit
= PAGE_SIZE
;
112 info
.high_limit
= current
->mm
->mmap_base
;
113 info
.align_mask
= PAGE_MASK
& ~huge_page_mask(h
);
114 info
.align_offset
= 0;
115 addr
= vm_unmapped_area(&info
);
118 * A failed mmap() very likely causes application failure,
119 * so fall back to the bottom-up function here. This scenario
120 * can happen with large stack limits and large mmap()
123 if (addr
& ~PAGE_MASK
) {
124 VM_BUG_ON(addr
!= -ENOMEM
);
126 info
.low_limit
= TASK_UNMAPPED_BASE
;
127 info
.high_limit
= TASK_SIZE
;
128 addr
= vm_unmapped_area(&info
);
135 hugetlb_get_unmapped_area(struct file
*file
, unsigned long addr
,
136 unsigned long len
, unsigned long pgoff
, unsigned long flags
)
138 struct hstate
*h
= hstate_file(file
);
139 struct mm_struct
*mm
= current
->mm
;
140 struct vm_area_struct
*vma
;
142 if (len
& ~huge_page_mask(h
))
147 if (flags
& MAP_FIXED
) {
148 if (prepare_hugepage_range(file
, addr
, len
))
154 addr
= ALIGN(addr
, huge_page_size(h
));
155 vma
= find_vma(mm
, addr
);
156 if (TASK_SIZE
- len
>= addr
&&
157 (!vma
|| addr
+ len
<= vma
->vm_start
))
160 if (mm
->get_unmapped_area
== arch_get_unmapped_area
)
161 return hugetlb_get_unmapped_area_bottomup(file
, addr
, len
,
164 return hugetlb_get_unmapped_area_topdown(file
, addr
, len
,
168 #endif /*HAVE_ARCH_HUGETLB_UNMAPPED_AREA*/
171 static __init
int setup_hugepagesz(char *opt
)
173 unsigned long ps
= memparse(opt
, &opt
);
174 if (ps
== PMD_SIZE
) {
175 hugetlb_add_hstate(PMD_SHIFT
- PAGE_SHIFT
);
176 } else if (ps
== PUD_SIZE
&& cpu_has_gbpages
) {
177 hugetlb_add_hstate(PUD_SHIFT
- PAGE_SHIFT
);
179 printk(KERN_ERR
"hugepagesz: Unsupported page size %lu M\n",
185 __setup("hugepagesz=", setup_hugepagesz
);