]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * Copyright (C) 2009 Red Hat, Inc. | |
3 | * | |
4 | * This work is licensed under the terms of the GNU GPL, version 2. See | |
5 | * the COPYING file in the top-level directory. | |
6 | */ | |
7 | ||
8 | #include <linux/mm.h> | |
9 | #include <linux/sched.h> | |
10 | #include <linux/highmem.h> | |
11 | #include <linux/hugetlb.h> | |
12 | #include <linux/mmu_notifier.h> | |
13 | #include <linux/rmap.h> | |
14 | #include <linux/swap.h> | |
15 | #include <asm/tlb.h> | |
16 | #include <asm/pgalloc.h> | |
17 | #include "internal.h" | |
18 | ||
19 | unsigned long transparent_hugepage_flags __read_mostly = | |
20 | (1<<TRANSPARENT_HUGEPAGE_FLAG); | |
21 | ||
22 | #ifdef CONFIG_SYSFS | |
23 | static ssize_t double_flag_show(struct kobject *kobj, | |
24 | struct kobj_attribute *attr, char *buf, | |
25 | enum transparent_hugepage_flag enabled, | |
26 | enum transparent_hugepage_flag req_madv) | |
27 | { | |
28 | if (test_bit(enabled, &transparent_hugepage_flags)) { | |
29 | VM_BUG_ON(test_bit(req_madv, &transparent_hugepage_flags)); | |
30 | return sprintf(buf, "[always] madvise never\n"); | |
31 | } else if (test_bit(req_madv, &transparent_hugepage_flags)) | |
32 | return sprintf(buf, "always [madvise] never\n"); | |
33 | else | |
34 | return sprintf(buf, "always madvise [never]\n"); | |
35 | } | |
36 | static ssize_t double_flag_store(struct kobject *kobj, | |
37 | struct kobj_attribute *attr, | |
38 | const char *buf, size_t count, | |
39 | enum transparent_hugepage_flag enabled, | |
40 | enum transparent_hugepage_flag req_madv) | |
41 | { | |
42 | if (!memcmp("always", buf, | |
43 | min(sizeof("always")-1, count))) { | |
44 | set_bit(enabled, &transparent_hugepage_flags); | |
45 | clear_bit(req_madv, &transparent_hugepage_flags); | |
46 | } else if (!memcmp("madvise", buf, | |
47 | min(sizeof("madvise")-1, count))) { | |
48 | clear_bit(enabled, &transparent_hugepage_flags); | |
49 | set_bit(req_madv, &transparent_hugepage_flags); | |
50 | } else if (!memcmp("never", buf, | |
51 | min(sizeof("never")-1, count))) { | |
52 | clear_bit(enabled, &transparent_hugepage_flags); | |
53 | clear_bit(req_madv, &transparent_hugepage_flags); | |
54 | } else | |
55 | return -EINVAL; | |
56 | ||
57 | return count; | |
58 | } | |
59 | ||
60 | static ssize_t enabled_show(struct kobject *kobj, | |
61 | struct kobj_attribute *attr, char *buf) | |
62 | { | |
63 | return double_flag_show(kobj, attr, buf, | |
64 | TRANSPARENT_HUGEPAGE_FLAG, | |
65 | TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG); | |
66 | } | |
67 | static ssize_t enabled_store(struct kobject *kobj, | |
68 | struct kobj_attribute *attr, | |
69 | const char *buf, size_t count) | |
70 | { | |
71 | return double_flag_store(kobj, attr, buf, count, | |
72 | TRANSPARENT_HUGEPAGE_FLAG, | |
73 | TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG); | |
74 | } | |
75 | static struct kobj_attribute enabled_attr = | |
76 | __ATTR(enabled, 0644, enabled_show, enabled_store); | |
77 | ||
78 | static ssize_t single_flag_show(struct kobject *kobj, | |
79 | struct kobj_attribute *attr, char *buf, | |
80 | enum transparent_hugepage_flag flag) | |
81 | { | |
82 | if (test_bit(flag, &transparent_hugepage_flags)) | |
83 | return sprintf(buf, "[yes] no\n"); | |
84 | else | |
85 | return sprintf(buf, "yes [no]\n"); | |
86 | } | |
87 | static ssize_t single_flag_store(struct kobject *kobj, | |
88 | struct kobj_attribute *attr, | |
89 | const char *buf, size_t count, | |
90 | enum transparent_hugepage_flag flag) | |
91 | { | |
92 | if (!memcmp("yes", buf, | |
93 | min(sizeof("yes")-1, count))) { | |
94 | set_bit(flag, &transparent_hugepage_flags); | |
95 | } else if (!memcmp("no", buf, | |
96 | min(sizeof("no")-1, count))) { | |
97 | clear_bit(flag, &transparent_hugepage_flags); | |
98 | } else | |
99 | return -EINVAL; | |
100 | ||
101 | return count; | |
102 | } | |
103 | ||
104 | /* | |
105 | * Currently defrag only disables __GFP_NOWAIT for allocation. A blind | |
106 | * __GFP_REPEAT is too aggressive, it's never worth swapping tons of | |
107 | * memory just to allocate one more hugepage. | |
108 | */ | |
109 | static ssize_t defrag_show(struct kobject *kobj, | |
110 | struct kobj_attribute *attr, char *buf) | |
111 | { | |
112 | return double_flag_show(kobj, attr, buf, | |
113 | TRANSPARENT_HUGEPAGE_DEFRAG_FLAG, | |
114 | TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG); | |
115 | } | |
116 | static ssize_t defrag_store(struct kobject *kobj, | |
117 | struct kobj_attribute *attr, | |
118 | const char *buf, size_t count) | |
119 | { | |
120 | return double_flag_store(kobj, attr, buf, count, | |
121 | TRANSPARENT_HUGEPAGE_DEFRAG_FLAG, | |
122 | TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG); | |
123 | } | |
124 | static struct kobj_attribute defrag_attr = | |
125 | __ATTR(defrag, 0644, defrag_show, defrag_store); | |
126 | ||
127 | #ifdef CONFIG_DEBUG_VM | |
128 | static ssize_t debug_cow_show(struct kobject *kobj, | |
129 | struct kobj_attribute *attr, char *buf) | |
130 | { | |
131 | return single_flag_show(kobj, attr, buf, | |
132 | TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG); | |
133 | } | |
134 | static ssize_t debug_cow_store(struct kobject *kobj, | |
135 | struct kobj_attribute *attr, | |
136 | const char *buf, size_t count) | |
137 | { | |
138 | return single_flag_store(kobj, attr, buf, count, | |
139 | TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG); | |
140 | } | |
141 | static struct kobj_attribute debug_cow_attr = | |
142 | __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store); | |
143 | #endif /* CONFIG_DEBUG_VM */ | |
144 | ||
145 | static struct attribute *hugepage_attr[] = { | |
146 | &enabled_attr.attr, | |
147 | &defrag_attr.attr, | |
148 | #ifdef CONFIG_DEBUG_VM | |
149 | &debug_cow_attr.attr, | |
150 | #endif | |
151 | NULL, | |
152 | }; | |
153 | ||
154 | static struct attribute_group hugepage_attr_group = { | |
155 | .attrs = hugepage_attr, | |
156 | .name = "transparent_hugepage", | |
157 | }; | |
158 | #endif /* CONFIG_SYSFS */ | |
159 | ||
160 | static int __init hugepage_init(void) | |
161 | { | |
162 | #ifdef CONFIG_SYSFS | |
163 | int err; | |
164 | ||
165 | err = sysfs_create_group(mm_kobj, &hugepage_attr_group); | |
166 | if (err) | |
167 | printk(KERN_ERR "hugepage: register sysfs failed\n"); | |
168 | #endif | |
169 | return 0; | |
170 | } | |
171 | module_init(hugepage_init) | |
172 | ||
173 | static int __init setup_transparent_hugepage(char *str) | |
174 | { | |
175 | int ret = 0; | |
176 | if (!str) | |
177 | goto out; | |
178 | if (!strcmp(str, "always")) { | |
179 | set_bit(TRANSPARENT_HUGEPAGE_FLAG, | |
180 | &transparent_hugepage_flags); | |
181 | clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, | |
182 | &transparent_hugepage_flags); | |
183 | ret = 1; | |
184 | } else if (!strcmp(str, "madvise")) { | |
185 | clear_bit(TRANSPARENT_HUGEPAGE_FLAG, | |
186 | &transparent_hugepage_flags); | |
187 | set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, | |
188 | &transparent_hugepage_flags); | |
189 | ret = 1; | |
190 | } else if (!strcmp(str, "never")) { | |
191 | clear_bit(TRANSPARENT_HUGEPAGE_FLAG, | |
192 | &transparent_hugepage_flags); | |
193 | clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, | |
194 | &transparent_hugepage_flags); | |
195 | ret = 1; | |
196 | } | |
197 | out: | |
198 | if (!ret) | |
199 | printk(KERN_WARNING | |
200 | "transparent_hugepage= cannot parse, ignored\n"); | |
201 | return ret; | |
202 | } | |
203 | __setup("transparent_hugepage=", setup_transparent_hugepage); | |
204 | ||
205 | static void prepare_pmd_huge_pte(pgtable_t pgtable, | |
206 | struct mm_struct *mm) | |
207 | { | |
208 | assert_spin_locked(&mm->page_table_lock); | |
209 | ||
210 | /* FIFO */ | |
211 | if (!mm->pmd_huge_pte) | |
212 | INIT_LIST_HEAD(&pgtable->lru); | |
213 | else | |
214 | list_add(&pgtable->lru, &mm->pmd_huge_pte->lru); | |
215 | mm->pmd_huge_pte = pgtable; | |
216 | } | |
217 | ||
218 | static inline pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) | |
219 | { | |
220 | if (likely(vma->vm_flags & VM_WRITE)) | |
221 | pmd = pmd_mkwrite(pmd); | |
222 | return pmd; | |
223 | } | |
224 | ||
225 | static int __do_huge_pmd_anonymous_page(struct mm_struct *mm, | |
226 | struct vm_area_struct *vma, | |
227 | unsigned long haddr, pmd_t *pmd, | |
228 | struct page *page) | |
229 | { | |
230 | int ret = 0; | |
231 | pgtable_t pgtable; | |
232 | ||
233 | VM_BUG_ON(!PageCompound(page)); | |
234 | pgtable = pte_alloc_one(mm, haddr); | |
235 | if (unlikely(!pgtable)) { | |
236 | put_page(page); | |
237 | return VM_FAULT_OOM; | |
238 | } | |
239 | ||
240 | clear_huge_page(page, haddr, HPAGE_PMD_NR); | |
241 | __SetPageUptodate(page); | |
242 | ||
243 | spin_lock(&mm->page_table_lock); | |
244 | if (unlikely(!pmd_none(*pmd))) { | |
245 | spin_unlock(&mm->page_table_lock); | |
246 | put_page(page); | |
247 | pte_free(mm, pgtable); | |
248 | } else { | |
249 | pmd_t entry; | |
250 | entry = mk_pmd(page, vma->vm_page_prot); | |
251 | entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); | |
252 | entry = pmd_mkhuge(entry); | |
253 | /* | |
254 | * The spinlocking to take the lru_lock inside | |
255 | * page_add_new_anon_rmap() acts as a full memory | |
256 | * barrier to be sure clear_huge_page writes become | |
257 | * visible after the set_pmd_at() write. | |
258 | */ | |
259 | page_add_new_anon_rmap(page, vma, haddr); | |
260 | set_pmd_at(mm, haddr, pmd, entry); | |
261 | prepare_pmd_huge_pte(pgtable, mm); | |
262 | add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR); | |
263 | spin_unlock(&mm->page_table_lock); | |
264 | } | |
265 | ||
266 | return ret; | |
267 | } | |
268 | ||
269 | static inline struct page *alloc_hugepage(int defrag) | |
270 | { | |
271 | return alloc_pages(GFP_TRANSHUGE & ~(defrag ? 0 : __GFP_WAIT), | |
272 | HPAGE_PMD_ORDER); | |
273 | } | |
274 | ||
275 | int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, | |
276 | unsigned long address, pmd_t *pmd, | |
277 | unsigned int flags) | |
278 | { | |
279 | struct page *page; | |
280 | unsigned long haddr = address & HPAGE_PMD_MASK; | |
281 | pte_t *pte; | |
282 | ||
283 | if (haddr >= vma->vm_start && haddr + HPAGE_PMD_SIZE <= vma->vm_end) { | |
284 | if (unlikely(anon_vma_prepare(vma))) | |
285 | return VM_FAULT_OOM; | |
286 | page = alloc_hugepage(transparent_hugepage_defrag(vma)); | |
287 | if (unlikely(!page)) | |
288 | goto out; | |
289 | ||
290 | return __do_huge_pmd_anonymous_page(mm, vma, haddr, pmd, page); | |
291 | } | |
292 | out: | |
293 | /* | |
294 | * Use __pte_alloc instead of pte_alloc_map, because we can't | |
295 | * run pte_offset_map on the pmd, if an huge pmd could | |
296 | * materialize from under us from a different thread. | |
297 | */ | |
298 | if (unlikely(__pte_alloc(mm, vma, pmd, address))) | |
299 | return VM_FAULT_OOM; | |
300 | /* if an huge pmd materialized from under us just retry later */ | |
301 | if (unlikely(pmd_trans_huge(*pmd))) | |
302 | return 0; | |
303 | /* | |
304 | * A regular pmd is established and it can't morph into a huge pmd | |
305 | * from under us anymore at this point because we hold the mmap_sem | |
306 | * read mode and khugepaged takes it in write mode. So now it's | |
307 | * safe to run pte_offset_map(). | |
308 | */ | |
309 | pte = pte_offset_map(pmd, address); | |
310 | return handle_pte_fault(mm, vma, address, pte, pmd, flags); | |
311 | } | |
312 | ||
313 | int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, | |
314 | pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, | |
315 | struct vm_area_struct *vma) | |
316 | { | |
317 | struct page *src_page; | |
318 | pmd_t pmd; | |
319 | pgtable_t pgtable; | |
320 | int ret; | |
321 | ||
322 | ret = -ENOMEM; | |
323 | pgtable = pte_alloc_one(dst_mm, addr); | |
324 | if (unlikely(!pgtable)) | |
325 | goto out; | |
326 | ||
327 | spin_lock(&dst_mm->page_table_lock); | |
328 | spin_lock_nested(&src_mm->page_table_lock, SINGLE_DEPTH_NESTING); | |
329 | ||
330 | ret = -EAGAIN; | |
331 | pmd = *src_pmd; | |
332 | if (unlikely(!pmd_trans_huge(pmd))) { | |
333 | pte_free(dst_mm, pgtable); | |
334 | goto out_unlock; | |
335 | } | |
336 | if (unlikely(pmd_trans_splitting(pmd))) { | |
337 | /* split huge page running from under us */ | |
338 | spin_unlock(&src_mm->page_table_lock); | |
339 | spin_unlock(&dst_mm->page_table_lock); | |
340 | pte_free(dst_mm, pgtable); | |
341 | ||
342 | wait_split_huge_page(vma->anon_vma, src_pmd); /* src_vma */ | |
343 | goto out; | |
344 | } | |
345 | src_page = pmd_page(pmd); | |
346 | VM_BUG_ON(!PageHead(src_page)); | |
347 | get_page(src_page); | |
348 | page_dup_rmap(src_page); | |
349 | add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); | |
350 | ||
351 | pmdp_set_wrprotect(src_mm, addr, src_pmd); | |
352 | pmd = pmd_mkold(pmd_wrprotect(pmd)); | |
353 | set_pmd_at(dst_mm, addr, dst_pmd, pmd); | |
354 | prepare_pmd_huge_pte(pgtable, dst_mm); | |
355 | ||
356 | ret = 0; | |
357 | out_unlock: | |
358 | spin_unlock(&src_mm->page_table_lock); | |
359 | spin_unlock(&dst_mm->page_table_lock); | |
360 | out: | |
361 | return ret; | |
362 | } | |
363 | ||
364 | /* no "address" argument so destroys page coloring of some arch */ | |
365 | pgtable_t get_pmd_huge_pte(struct mm_struct *mm) | |
366 | { | |
367 | pgtable_t pgtable; | |
368 | ||
369 | assert_spin_locked(&mm->page_table_lock); | |
370 | ||
371 | /* FIFO */ | |
372 | pgtable = mm->pmd_huge_pte; | |
373 | if (list_empty(&pgtable->lru)) | |
374 | mm->pmd_huge_pte = NULL; | |
375 | else { | |
376 | mm->pmd_huge_pte = list_entry(pgtable->lru.next, | |
377 | struct page, lru); | |
378 | list_del(&pgtable->lru); | |
379 | } | |
380 | return pgtable; | |
381 | } | |
382 | ||
383 | static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm, | |
384 | struct vm_area_struct *vma, | |
385 | unsigned long address, | |
386 | pmd_t *pmd, pmd_t orig_pmd, | |
387 | struct page *page, | |
388 | unsigned long haddr) | |
389 | { | |
390 | pgtable_t pgtable; | |
391 | pmd_t _pmd; | |
392 | int ret = 0, i; | |
393 | struct page **pages; | |
394 | ||
395 | pages = kmalloc(sizeof(struct page *) * HPAGE_PMD_NR, | |
396 | GFP_KERNEL); | |
397 | if (unlikely(!pages)) { | |
398 | ret |= VM_FAULT_OOM; | |
399 | goto out; | |
400 | } | |
401 | ||
402 | for (i = 0; i < HPAGE_PMD_NR; i++) { | |
403 | pages[i] = alloc_page_vma(GFP_HIGHUSER_MOVABLE, | |
404 | vma, address); | |
405 | if (unlikely(!pages[i])) { | |
406 | while (--i >= 0) | |
407 | put_page(pages[i]); | |
408 | kfree(pages); | |
409 | ret |= VM_FAULT_OOM; | |
410 | goto out; | |
411 | } | |
412 | } | |
413 | ||
414 | for (i = 0; i < HPAGE_PMD_NR; i++) { | |
415 | copy_user_highpage(pages[i], page + i, | |
416 | haddr + PAGE_SHIFT*i, vma); | |
417 | __SetPageUptodate(pages[i]); | |
418 | cond_resched(); | |
419 | } | |
420 | ||
421 | spin_lock(&mm->page_table_lock); | |
422 | if (unlikely(!pmd_same(*pmd, orig_pmd))) | |
423 | goto out_free_pages; | |
424 | VM_BUG_ON(!PageHead(page)); | |
425 | ||
426 | pmdp_clear_flush_notify(vma, haddr, pmd); | |
427 | /* leave pmd empty until pte is filled */ | |
428 | ||
429 | pgtable = get_pmd_huge_pte(mm); | |
430 | pmd_populate(mm, &_pmd, pgtable); | |
431 | ||
432 | for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { | |
433 | pte_t *pte, entry; | |
434 | entry = mk_pte(pages[i], vma->vm_page_prot); | |
435 | entry = maybe_mkwrite(pte_mkdirty(entry), vma); | |
436 | page_add_new_anon_rmap(pages[i], vma, haddr); | |
437 | pte = pte_offset_map(&_pmd, haddr); | |
438 | VM_BUG_ON(!pte_none(*pte)); | |
439 | set_pte_at(mm, haddr, pte, entry); | |
440 | pte_unmap(pte); | |
441 | } | |
442 | kfree(pages); | |
443 | ||
444 | mm->nr_ptes++; | |
445 | smp_wmb(); /* make pte visible before pmd */ | |
446 | pmd_populate(mm, pmd, pgtable); | |
447 | page_remove_rmap(page); | |
448 | spin_unlock(&mm->page_table_lock); | |
449 | ||
450 | ret |= VM_FAULT_WRITE; | |
451 | put_page(page); | |
452 | ||
453 | out: | |
454 | return ret; | |
455 | ||
456 | out_free_pages: | |
457 | spin_unlock(&mm->page_table_lock); | |
458 | for (i = 0; i < HPAGE_PMD_NR; i++) | |
459 | put_page(pages[i]); | |
460 | kfree(pages); | |
461 | goto out; | |
462 | } | |
463 | ||
464 | int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, | |
465 | unsigned long address, pmd_t *pmd, pmd_t orig_pmd) | |
466 | { | |
467 | int ret = 0; | |
468 | struct page *page, *new_page; | |
469 | unsigned long haddr; | |
470 | ||
471 | VM_BUG_ON(!vma->anon_vma); | |
472 | spin_lock(&mm->page_table_lock); | |
473 | if (unlikely(!pmd_same(*pmd, orig_pmd))) | |
474 | goto out_unlock; | |
475 | ||
476 | page = pmd_page(orig_pmd); | |
477 | VM_BUG_ON(!PageCompound(page) || !PageHead(page)); | |
478 | haddr = address & HPAGE_PMD_MASK; | |
479 | if (page_mapcount(page) == 1) { | |
480 | pmd_t entry; | |
481 | entry = pmd_mkyoung(orig_pmd); | |
482 | entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); | |
483 | if (pmdp_set_access_flags(vma, haddr, pmd, entry, 1)) | |
484 | update_mmu_cache(vma, address, entry); | |
485 | ret |= VM_FAULT_WRITE; | |
486 | goto out_unlock; | |
487 | } | |
488 | get_page(page); | |
489 | spin_unlock(&mm->page_table_lock); | |
490 | ||
491 | if (transparent_hugepage_enabled(vma) && | |
492 | !transparent_hugepage_debug_cow()) | |
493 | new_page = alloc_hugepage(transparent_hugepage_defrag(vma)); | |
494 | else | |
495 | new_page = NULL; | |
496 | ||
497 | if (unlikely(!new_page)) { | |
498 | ret = do_huge_pmd_wp_page_fallback(mm, vma, address, | |
499 | pmd, orig_pmd, page, haddr); | |
500 | put_page(page); | |
501 | goto out; | |
502 | } | |
503 | ||
504 | copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR); | |
505 | __SetPageUptodate(new_page); | |
506 | ||
507 | spin_lock(&mm->page_table_lock); | |
508 | put_page(page); | |
509 | if (unlikely(!pmd_same(*pmd, orig_pmd))) | |
510 | put_page(new_page); | |
511 | else { | |
512 | pmd_t entry; | |
513 | VM_BUG_ON(!PageHead(page)); | |
514 | entry = mk_pmd(new_page, vma->vm_page_prot); | |
515 | entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); | |
516 | entry = pmd_mkhuge(entry); | |
517 | pmdp_clear_flush_notify(vma, haddr, pmd); | |
518 | page_add_new_anon_rmap(new_page, vma, haddr); | |
519 | set_pmd_at(mm, haddr, pmd, entry); | |
520 | update_mmu_cache(vma, address, entry); | |
521 | page_remove_rmap(page); | |
522 | put_page(page); | |
523 | ret |= VM_FAULT_WRITE; | |
524 | } | |
525 | out_unlock: | |
526 | spin_unlock(&mm->page_table_lock); | |
527 | out: | |
528 | return ret; | |
529 | } | |
530 | ||
531 | struct page *follow_trans_huge_pmd(struct mm_struct *mm, | |
532 | unsigned long addr, | |
533 | pmd_t *pmd, | |
534 | unsigned int flags) | |
535 | { | |
536 | struct page *page = NULL; | |
537 | ||
538 | assert_spin_locked(&mm->page_table_lock); | |
539 | ||
540 | if (flags & FOLL_WRITE && !pmd_write(*pmd)) | |
541 | goto out; | |
542 | ||
543 | page = pmd_page(*pmd); | |
544 | VM_BUG_ON(!PageHead(page)); | |
545 | if (flags & FOLL_TOUCH) { | |
546 | pmd_t _pmd; | |
547 | /* | |
548 | * We should set the dirty bit only for FOLL_WRITE but | |
549 | * for now the dirty bit in the pmd is meaningless. | |
550 | * And if the dirty bit will become meaningful and | |
551 | * we'll only set it with FOLL_WRITE, an atomic | |
552 | * set_bit will be required on the pmd to set the | |
553 | * young bit, instead of the current set_pmd_at. | |
554 | */ | |
555 | _pmd = pmd_mkyoung(pmd_mkdirty(*pmd)); | |
556 | set_pmd_at(mm, addr & HPAGE_PMD_MASK, pmd, _pmd); | |
557 | } | |
558 | page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT; | |
559 | VM_BUG_ON(!PageCompound(page)); | |
560 | if (flags & FOLL_GET) | |
561 | get_page(page); | |
562 | ||
563 | out: | |
564 | return page; | |
565 | } | |
566 | ||
567 | int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, | |
568 | pmd_t *pmd) | |
569 | { | |
570 | int ret = 0; | |
571 | ||
572 | spin_lock(&tlb->mm->page_table_lock); | |
573 | if (likely(pmd_trans_huge(*pmd))) { | |
574 | if (unlikely(pmd_trans_splitting(*pmd))) { | |
575 | spin_unlock(&tlb->mm->page_table_lock); | |
576 | wait_split_huge_page(vma->anon_vma, | |
577 | pmd); | |
578 | } else { | |
579 | struct page *page; | |
580 | pgtable_t pgtable; | |
581 | pgtable = get_pmd_huge_pte(tlb->mm); | |
582 | page = pmd_page(*pmd); | |
583 | pmd_clear(pmd); | |
584 | page_remove_rmap(page); | |
585 | VM_BUG_ON(page_mapcount(page) < 0); | |
586 | add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); | |
587 | VM_BUG_ON(!PageHead(page)); | |
588 | spin_unlock(&tlb->mm->page_table_lock); | |
589 | tlb_remove_page(tlb, page); | |
590 | pte_free(tlb->mm, pgtable); | |
591 | ret = 1; | |
592 | } | |
593 | } else | |
594 | spin_unlock(&tlb->mm->page_table_lock); | |
595 | ||
596 | return ret; | |
597 | } | |
598 | ||
599 | pmd_t *page_check_address_pmd(struct page *page, | |
600 | struct mm_struct *mm, | |
601 | unsigned long address, | |
602 | enum page_check_address_pmd_flag flag) | |
603 | { | |
604 | pgd_t *pgd; | |
605 | pud_t *pud; | |
606 | pmd_t *pmd, *ret = NULL; | |
607 | ||
608 | if (address & ~HPAGE_PMD_MASK) | |
609 | goto out; | |
610 | ||
611 | pgd = pgd_offset(mm, address); | |
612 | if (!pgd_present(*pgd)) | |
613 | goto out; | |
614 | ||
615 | pud = pud_offset(pgd, address); | |
616 | if (!pud_present(*pud)) | |
617 | goto out; | |
618 | ||
619 | pmd = pmd_offset(pud, address); | |
620 | if (pmd_none(*pmd)) | |
621 | goto out; | |
622 | if (pmd_page(*pmd) != page) | |
623 | goto out; | |
624 | VM_BUG_ON(flag == PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG && | |
625 | pmd_trans_splitting(*pmd)); | |
626 | if (pmd_trans_huge(*pmd)) { | |
627 | VM_BUG_ON(flag == PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG && | |
628 | !pmd_trans_splitting(*pmd)); | |
629 | ret = pmd; | |
630 | } | |
631 | out: | |
632 | return ret; | |
633 | } | |
634 | ||
635 | static int __split_huge_page_splitting(struct page *page, | |
636 | struct vm_area_struct *vma, | |
637 | unsigned long address) | |
638 | { | |
639 | struct mm_struct *mm = vma->vm_mm; | |
640 | pmd_t *pmd; | |
641 | int ret = 0; | |
642 | ||
643 | spin_lock(&mm->page_table_lock); | |
644 | pmd = page_check_address_pmd(page, mm, address, | |
645 | PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG); | |
646 | if (pmd) { | |
647 | /* | |
648 | * We can't temporarily set the pmd to null in order | |
649 | * to split it, the pmd must remain marked huge at all | |
650 | * times or the VM won't take the pmd_trans_huge paths | |
651 | * and it won't wait on the anon_vma->root->lock to | |
652 | * serialize against split_huge_page*. | |
653 | */ | |
654 | pmdp_splitting_flush_notify(vma, address, pmd); | |
655 | ret = 1; | |
656 | } | |
657 | spin_unlock(&mm->page_table_lock); | |
658 | ||
659 | return ret; | |
660 | } | |
661 | ||
662 | static void __split_huge_page_refcount(struct page *page) | |
663 | { | |
664 | int i; | |
665 | unsigned long head_index = page->index; | |
666 | struct zone *zone = page_zone(page); | |
667 | ||
668 | /* prevent PageLRU to go away from under us, and freeze lru stats */ | |
669 | spin_lock_irq(&zone->lru_lock); | |
670 | compound_lock(page); | |
671 | ||
672 | for (i = 1; i < HPAGE_PMD_NR; i++) { | |
673 | struct page *page_tail = page + i; | |
674 | ||
675 | /* tail_page->_count cannot change */ | |
676 | atomic_sub(atomic_read(&page_tail->_count), &page->_count); | |
677 | BUG_ON(page_count(page) <= 0); | |
678 | atomic_add(page_mapcount(page) + 1, &page_tail->_count); | |
679 | BUG_ON(atomic_read(&page_tail->_count) <= 0); | |
680 | ||
681 | /* after clearing PageTail the gup refcount can be released */ | |
682 | smp_mb(); | |
683 | ||
684 | page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; | |
685 | page_tail->flags |= (page->flags & | |
686 | ((1L << PG_referenced) | | |
687 | (1L << PG_swapbacked) | | |
688 | (1L << PG_mlocked) | | |
689 | (1L << PG_uptodate))); | |
690 | page_tail->flags |= (1L << PG_dirty); | |
691 | ||
692 | /* | |
693 | * 1) clear PageTail before overwriting first_page | |
694 | * 2) clear PageTail before clearing PageHead for VM_BUG_ON | |
695 | */ | |
696 | smp_wmb(); | |
697 | ||
698 | /* | |
699 | * __split_huge_page_splitting() already set the | |
700 | * splitting bit in all pmd that could map this | |
701 | * hugepage, that will ensure no CPU can alter the | |
702 | * mapcount on the head page. The mapcount is only | |
703 | * accounted in the head page and it has to be | |
704 | * transferred to all tail pages in the below code. So | |
705 | * for this code to be safe, the split the mapcount | |
706 | * can't change. But that doesn't mean userland can't | |
707 | * keep changing and reading the page contents while | |
708 | * we transfer the mapcount, so the pmd splitting | |
709 | * status is achieved setting a reserved bit in the | |
710 | * pmd, not by clearing the present bit. | |
711 | */ | |
712 | BUG_ON(page_mapcount(page_tail)); | |
713 | page_tail->_mapcount = page->_mapcount; | |
714 | ||
715 | BUG_ON(page_tail->mapping); | |
716 | page_tail->mapping = page->mapping; | |
717 | ||
718 | page_tail->index = ++head_index; | |
719 | ||
720 | BUG_ON(!PageAnon(page_tail)); | |
721 | BUG_ON(!PageUptodate(page_tail)); | |
722 | BUG_ON(!PageDirty(page_tail)); | |
723 | BUG_ON(!PageSwapBacked(page_tail)); | |
724 | ||
725 | lru_add_page_tail(zone, page, page_tail); | |
726 | } | |
727 | ||
728 | ClearPageCompound(page); | |
729 | compound_unlock(page); | |
730 | spin_unlock_irq(&zone->lru_lock); | |
731 | ||
732 | for (i = 1; i < HPAGE_PMD_NR; i++) { | |
733 | struct page *page_tail = page + i; | |
734 | BUG_ON(page_count(page_tail) <= 0); | |
735 | /* | |
736 | * Tail pages may be freed if there wasn't any mapping | |
737 | * like if add_to_swap() is running on a lru page that | |
738 | * had its mapping zapped. And freeing these pages | |
739 | * requires taking the lru_lock so we do the put_page | |
740 | * of the tail pages after the split is complete. | |
741 | */ | |
742 | put_page(page_tail); | |
743 | } | |
744 | ||
745 | /* | |
746 | * Only the head page (now become a regular page) is required | |
747 | * to be pinned by the caller. | |
748 | */ | |
749 | BUG_ON(page_count(page) <= 0); | |
750 | } | |
751 | ||
752 | static int __split_huge_page_map(struct page *page, | |
753 | struct vm_area_struct *vma, | |
754 | unsigned long address) | |
755 | { | |
756 | struct mm_struct *mm = vma->vm_mm; | |
757 | pmd_t *pmd, _pmd; | |
758 | int ret = 0, i; | |
759 | pgtable_t pgtable; | |
760 | unsigned long haddr; | |
761 | ||
762 | spin_lock(&mm->page_table_lock); | |
763 | pmd = page_check_address_pmd(page, mm, address, | |
764 | PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG); | |
765 | if (pmd) { | |
766 | pgtable = get_pmd_huge_pte(mm); | |
767 | pmd_populate(mm, &_pmd, pgtable); | |
768 | ||
769 | for (i = 0, haddr = address; i < HPAGE_PMD_NR; | |
770 | i++, haddr += PAGE_SIZE) { | |
771 | pte_t *pte, entry; | |
772 | BUG_ON(PageCompound(page+i)); | |
773 | entry = mk_pte(page + i, vma->vm_page_prot); | |
774 | entry = maybe_mkwrite(pte_mkdirty(entry), vma); | |
775 | if (!pmd_write(*pmd)) | |
776 | entry = pte_wrprotect(entry); | |
777 | else | |
778 | BUG_ON(page_mapcount(page) != 1); | |
779 | if (!pmd_young(*pmd)) | |
780 | entry = pte_mkold(entry); | |
781 | pte = pte_offset_map(&_pmd, haddr); | |
782 | BUG_ON(!pte_none(*pte)); | |
783 | set_pte_at(mm, haddr, pte, entry); | |
784 | pte_unmap(pte); | |
785 | } | |
786 | ||
787 | mm->nr_ptes++; | |
788 | smp_wmb(); /* make pte visible before pmd */ | |
789 | /* | |
790 | * Up to this point the pmd is present and huge and | |
791 | * userland has the whole access to the hugepage | |
792 | * during the split (which happens in place). If we | |
793 | * overwrite the pmd with the not-huge version | |
794 | * pointing to the pte here (which of course we could | |
795 | * if all CPUs were bug free), userland could trigger | |
796 | * a small page size TLB miss on the small sized TLB | |
797 | * while the hugepage TLB entry is still established | |
798 | * in the huge TLB. Some CPU doesn't like that. See | |
799 | * http://support.amd.com/us/Processor_TechDocs/41322.pdf, | |
800 | * Erratum 383 on page 93. Intel should be safe but is | |
801 | * also warns that it's only safe if the permission | |
802 | * and cache attributes of the two entries loaded in | |
803 | * the two TLB is identical (which should be the case | |
804 | * here). But it is generally safer to never allow | |
805 | * small and huge TLB entries for the same virtual | |
806 | * address to be loaded simultaneously. So instead of | |
807 | * doing "pmd_populate(); flush_tlb_range();" we first | |
808 | * mark the current pmd notpresent (atomically because | |
809 | * here the pmd_trans_huge and pmd_trans_splitting | |
810 | * must remain set at all times on the pmd until the | |
811 | * split is complete for this pmd), then we flush the | |
812 | * SMP TLB and finally we write the non-huge version | |
813 | * of the pmd entry with pmd_populate. | |
814 | */ | |
815 | set_pmd_at(mm, address, pmd, pmd_mknotpresent(*pmd)); | |
816 | flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); | |
817 | pmd_populate(mm, pmd, pgtable); | |
818 | ret = 1; | |
819 | } | |
820 | spin_unlock(&mm->page_table_lock); | |
821 | ||
822 | return ret; | |
823 | } | |
824 | ||
825 | /* must be called with anon_vma->root->lock hold */ | |
826 | static void __split_huge_page(struct page *page, | |
827 | struct anon_vma *anon_vma) | |
828 | { | |
829 | int mapcount, mapcount2; | |
830 | struct anon_vma_chain *avc; | |
831 | ||
832 | BUG_ON(!PageHead(page)); | |
833 | BUG_ON(PageTail(page)); | |
834 | ||
835 | mapcount = 0; | |
836 | list_for_each_entry(avc, &anon_vma->head, same_anon_vma) { | |
837 | struct vm_area_struct *vma = avc->vma; | |
838 | unsigned long addr = vma_address(page, vma); | |
839 | BUG_ON(is_vma_temporary_stack(vma)); | |
840 | if (addr == -EFAULT) | |
841 | continue; | |
842 | mapcount += __split_huge_page_splitting(page, vma, addr); | |
843 | } | |
844 | /* | |
845 | * It is critical that new vmas are added to the tail of the | |
846 | * anon_vma list. This guarantes that if copy_huge_pmd() runs | |
847 | * and establishes a child pmd before | |
848 | * __split_huge_page_splitting() freezes the parent pmd (so if | |
849 | * we fail to prevent copy_huge_pmd() from running until the | |
850 | * whole __split_huge_page() is complete), we will still see | |
851 | * the newly established pmd of the child later during the | |
852 | * walk, to be able to set it as pmd_trans_splitting too. | |
853 | */ | |
854 | if (mapcount != page_mapcount(page)) | |
855 | printk(KERN_ERR "mapcount %d page_mapcount %d\n", | |
856 | mapcount, page_mapcount(page)); | |
857 | BUG_ON(mapcount != page_mapcount(page)); | |
858 | ||
859 | __split_huge_page_refcount(page); | |
860 | ||
861 | mapcount2 = 0; | |
862 | list_for_each_entry(avc, &anon_vma->head, same_anon_vma) { | |
863 | struct vm_area_struct *vma = avc->vma; | |
864 | unsigned long addr = vma_address(page, vma); | |
865 | BUG_ON(is_vma_temporary_stack(vma)); | |
866 | if (addr == -EFAULT) | |
867 | continue; | |
868 | mapcount2 += __split_huge_page_map(page, vma, addr); | |
869 | } | |
870 | if (mapcount != mapcount2) | |
871 | printk(KERN_ERR "mapcount %d mapcount2 %d page_mapcount %d\n", | |
872 | mapcount, mapcount2, page_mapcount(page)); | |
873 | BUG_ON(mapcount != mapcount2); | |
874 | } | |
875 | ||
876 | int split_huge_page(struct page *page) | |
877 | { | |
878 | struct anon_vma *anon_vma; | |
879 | int ret = 1; | |
880 | ||
881 | BUG_ON(!PageAnon(page)); | |
882 | anon_vma = page_lock_anon_vma(page); | |
883 | if (!anon_vma) | |
884 | goto out; | |
885 | ret = 0; | |
886 | if (!PageCompound(page)) | |
887 | goto out_unlock; | |
888 | ||
889 | BUG_ON(!PageSwapBacked(page)); | |
890 | __split_huge_page(page, anon_vma); | |
891 | ||
892 | BUG_ON(PageCompound(page)); | |
893 | out_unlock: | |
894 | page_unlock_anon_vma(anon_vma); | |
895 | out: | |
896 | return ret; | |
897 | } | |
898 | ||
899 | int hugepage_madvise(unsigned long *vm_flags) | |
900 | { | |
901 | /* | |
902 | * Be somewhat over-protective like KSM for now! | |
903 | */ | |
904 | if (*vm_flags & (VM_HUGEPAGE | VM_SHARED | VM_MAYSHARE | | |
905 | VM_PFNMAP | VM_IO | VM_DONTEXPAND | | |
906 | VM_RESERVED | VM_HUGETLB | VM_INSERTPAGE | | |
907 | VM_MIXEDMAP | VM_SAO)) | |
908 | return -EINVAL; | |
909 | ||
910 | *vm_flags |= VM_HUGEPAGE; | |
911 | ||
912 | return 0; | |
913 | } | |
914 | ||
915 | void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd) | |
916 | { | |
917 | struct page *page; | |
918 | ||
919 | spin_lock(&mm->page_table_lock); | |
920 | if (unlikely(!pmd_trans_huge(*pmd))) { | |
921 | spin_unlock(&mm->page_table_lock); | |
922 | return; | |
923 | } | |
924 | page = pmd_page(*pmd); | |
925 | VM_BUG_ON(!page_count(page)); | |
926 | get_page(page); | |
927 | spin_unlock(&mm->page_table_lock); | |
928 | ||
929 | split_huge_page(page); | |
930 | ||
931 | put_page(page); | |
932 | BUG_ON(pmd_trans_huge(*pmd)); | |
933 | } |