2 * Copyright (C) 2009 Red Hat, Inc.
4 * This work is licensed under the terms of the GNU GPL, version 2. See
5 * the COPYING file in the top-level directory.
9 #include <linux/sched.h>
10 #include <linux/highmem.h>
11 #include <linux/hugetlb.h>
12 #include <linux/mmu_notifier.h>
13 #include <linux/rmap.h>
14 #include <linux/swap.h>
15 #include <linux/mm_inline.h>
16 #include <linux/kthread.h>
17 #include <linux/khugepaged.h>
18 #include <linux/freezer.h>
19 #include <linux/mman.h>
21 #include <asm/pgalloc.h>
25 * By default transparent hugepage support is enabled for all mappings
26 * and khugepaged scans all mappings. Defrag is only invoked by
27 * khugepaged hugepage allocations and by page faults inside
28 * MADV_HUGEPAGE regions to avoid the risk of slowing down short lived
31 unsigned long transparent_hugepage_flags __read_mostly
=
32 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS
33 (1<<TRANSPARENT_HUGEPAGE_FLAG
)|
35 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
36 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG
)|
38 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG
)|
39 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG
);
41 /* default scan 8*512 pte (or vmas) every 30 second */
42 static unsigned int khugepaged_pages_to_scan __read_mostly
= HPAGE_PMD_NR
*8;
43 static unsigned int khugepaged_pages_collapsed
;
44 static unsigned int khugepaged_full_scans
;
45 static unsigned int khugepaged_scan_sleep_millisecs __read_mostly
= 10000;
46 /* during fragmentation poll the hugepage allocator once every minute */
47 static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly
= 60000;
48 static struct task_struct
*khugepaged_thread __read_mostly
;
49 static DEFINE_MUTEX(khugepaged_mutex
);
50 static DEFINE_SPINLOCK(khugepaged_mm_lock
);
51 static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait
);
53 * default collapse hugepages if there is at least one pte mapped like
54 * it would have happened if the vma was large enough during page
57 static unsigned int khugepaged_max_ptes_none __read_mostly
= HPAGE_PMD_NR
-1;
59 static int khugepaged(void *none
);
60 static int mm_slots_hash_init(void);
61 static int khugepaged_slab_init(void);
62 static void khugepaged_slab_free(void);
64 #define MM_SLOTS_HASH_HEADS 1024
65 static struct hlist_head
*mm_slots_hash __read_mostly
;
66 static struct kmem_cache
*mm_slot_cache __read_mostly
;
69 * struct mm_slot - hash lookup from mm to mm_slot
70 * @hash: hash collision list
71 * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head
72 * @mm: the mm that this information is valid for
75 struct hlist_node hash
;
76 struct list_head mm_node
;
81 * struct khugepaged_scan - cursor for scanning
82 * @mm_head: the head of the mm list to scan
83 * @mm_slot: the current mm_slot we are scanning
84 * @address: the next address inside that to be scanned
86 * There is only the one khugepaged_scan instance of this cursor structure.
88 struct khugepaged_scan
{
89 struct list_head mm_head
;
90 struct mm_slot
*mm_slot
;
91 unsigned long address
;
93 .mm_head
= LIST_HEAD_INIT(khugepaged_scan
.mm_head
),
97 static int set_recommended_min_free_kbytes(void)
101 unsigned long recommended_min
;
102 extern int min_free_kbytes
;
104 if (!test_bit(TRANSPARENT_HUGEPAGE_FLAG
,
105 &transparent_hugepage_flags
) &&
106 !test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG
,
107 &transparent_hugepage_flags
))
110 for_each_populated_zone(zone
)
113 /* Make sure at least 2 hugepages are free for MIGRATE_RESERVE */
114 recommended_min
= pageblock_nr_pages
* nr_zones
* 2;
117 * Make sure that on average at least two pageblocks are almost free
118 * of another type, one for a migratetype to fall back to and a
119 * second to avoid subsequent fallbacks of other types There are 3
120 * MIGRATE_TYPES we care about.
122 recommended_min
+= pageblock_nr_pages
* nr_zones
*
123 MIGRATE_PCPTYPES
* MIGRATE_PCPTYPES
;
125 /* don't ever allow to reserve more than 5% of the lowmem */
126 recommended_min
= min(recommended_min
,
127 (unsigned long) nr_free_buffer_pages() / 20);
128 recommended_min
<<= (PAGE_SHIFT
-10);
130 if (recommended_min
> min_free_kbytes
)
131 min_free_kbytes
= recommended_min
;
132 setup_per_zone_wmarks();
135 late_initcall(set_recommended_min_free_kbytes
);
137 static int start_khugepaged(void)
140 if (khugepaged_enabled()) {
142 if (unlikely(!mm_slot_cache
|| !mm_slots_hash
)) {
146 mutex_lock(&khugepaged_mutex
);
147 if (!khugepaged_thread
)
148 khugepaged_thread
= kthread_run(khugepaged
, NULL
,
150 if (unlikely(IS_ERR(khugepaged_thread
))) {
152 "khugepaged: kthread_run(khugepaged) failed\n");
153 err
= PTR_ERR(khugepaged_thread
);
154 khugepaged_thread
= NULL
;
156 wakeup
= !list_empty(&khugepaged_scan
.mm_head
);
157 mutex_unlock(&khugepaged_mutex
);
159 wake_up_interruptible(&khugepaged_wait
);
161 set_recommended_min_free_kbytes();
164 wake_up_interruptible(&khugepaged_wait
);
171 static ssize_t
double_flag_show(struct kobject
*kobj
,
172 struct kobj_attribute
*attr
, char *buf
,
173 enum transparent_hugepage_flag enabled
,
174 enum transparent_hugepage_flag req_madv
)
176 if (test_bit(enabled
, &transparent_hugepage_flags
)) {
177 VM_BUG_ON(test_bit(req_madv
, &transparent_hugepage_flags
));
178 return sprintf(buf
, "[always] madvise never\n");
179 } else if (test_bit(req_madv
, &transparent_hugepage_flags
))
180 return sprintf(buf
, "always [madvise] never\n");
182 return sprintf(buf
, "always madvise [never]\n");
184 static ssize_t
double_flag_store(struct kobject
*kobj
,
185 struct kobj_attribute
*attr
,
186 const char *buf
, size_t count
,
187 enum transparent_hugepage_flag enabled
,
188 enum transparent_hugepage_flag req_madv
)
190 if (!memcmp("always", buf
,
191 min(sizeof("always")-1, count
))) {
192 set_bit(enabled
, &transparent_hugepage_flags
);
193 clear_bit(req_madv
, &transparent_hugepage_flags
);
194 } else if (!memcmp("madvise", buf
,
195 min(sizeof("madvise")-1, count
))) {
196 clear_bit(enabled
, &transparent_hugepage_flags
);
197 set_bit(req_madv
, &transparent_hugepage_flags
);
198 } else if (!memcmp("never", buf
,
199 min(sizeof("never")-1, count
))) {
200 clear_bit(enabled
, &transparent_hugepage_flags
);
201 clear_bit(req_madv
, &transparent_hugepage_flags
);
208 static ssize_t
enabled_show(struct kobject
*kobj
,
209 struct kobj_attribute
*attr
, char *buf
)
211 return double_flag_show(kobj
, attr
, buf
,
212 TRANSPARENT_HUGEPAGE_FLAG
,
213 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG
);
215 static ssize_t
enabled_store(struct kobject
*kobj
,
216 struct kobj_attribute
*attr
,
217 const char *buf
, size_t count
)
221 ret
= double_flag_store(kobj
, attr
, buf
, count
,
222 TRANSPARENT_HUGEPAGE_FLAG
,
223 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG
);
226 int err
= start_khugepaged();
232 (test_bit(TRANSPARENT_HUGEPAGE_FLAG
,
233 &transparent_hugepage_flags
) ||
234 test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG
,
235 &transparent_hugepage_flags
)))
236 set_recommended_min_free_kbytes();
240 static struct kobj_attribute enabled_attr
=
241 __ATTR(enabled
, 0644, enabled_show
, enabled_store
);
243 static ssize_t
single_flag_show(struct kobject
*kobj
,
244 struct kobj_attribute
*attr
, char *buf
,
245 enum transparent_hugepage_flag flag
)
247 if (test_bit(flag
, &transparent_hugepage_flags
))
248 return sprintf(buf
, "[yes] no\n");
250 return sprintf(buf
, "yes [no]\n");
252 static ssize_t
single_flag_store(struct kobject
*kobj
,
253 struct kobj_attribute
*attr
,
254 const char *buf
, size_t count
,
255 enum transparent_hugepage_flag flag
)
257 if (!memcmp("yes", buf
,
258 min(sizeof("yes")-1, count
))) {
259 set_bit(flag
, &transparent_hugepage_flags
);
260 } else if (!memcmp("no", buf
,
261 min(sizeof("no")-1, count
))) {
262 clear_bit(flag
, &transparent_hugepage_flags
);
270 * Currently defrag only disables __GFP_NOWAIT for allocation. A blind
271 * __GFP_REPEAT is too aggressive, it's never worth swapping tons of
272 * memory just to allocate one more hugepage.
274 static ssize_t
defrag_show(struct kobject
*kobj
,
275 struct kobj_attribute
*attr
, char *buf
)
277 return double_flag_show(kobj
, attr
, buf
,
278 TRANSPARENT_HUGEPAGE_DEFRAG_FLAG
,
279 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG
);
281 static ssize_t
defrag_store(struct kobject
*kobj
,
282 struct kobj_attribute
*attr
,
283 const char *buf
, size_t count
)
285 return double_flag_store(kobj
, attr
, buf
, count
,
286 TRANSPARENT_HUGEPAGE_DEFRAG_FLAG
,
287 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG
);
289 static struct kobj_attribute defrag_attr
=
290 __ATTR(defrag
, 0644, defrag_show
, defrag_store
);
292 #ifdef CONFIG_DEBUG_VM
293 static ssize_t
debug_cow_show(struct kobject
*kobj
,
294 struct kobj_attribute
*attr
, char *buf
)
296 return single_flag_show(kobj
, attr
, buf
,
297 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG
);
299 static ssize_t
debug_cow_store(struct kobject
*kobj
,
300 struct kobj_attribute
*attr
,
301 const char *buf
, size_t count
)
303 return single_flag_store(kobj
, attr
, buf
, count
,
304 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG
);
306 static struct kobj_attribute debug_cow_attr
=
307 __ATTR(debug_cow
, 0644, debug_cow_show
, debug_cow_store
);
308 #endif /* CONFIG_DEBUG_VM */
310 static struct attribute
*hugepage_attr
[] = {
313 #ifdef CONFIG_DEBUG_VM
314 &debug_cow_attr
.attr
,
319 static struct attribute_group hugepage_attr_group
= {
320 .attrs
= hugepage_attr
,
323 static ssize_t
scan_sleep_millisecs_show(struct kobject
*kobj
,
324 struct kobj_attribute
*attr
,
327 return sprintf(buf
, "%u\n", khugepaged_scan_sleep_millisecs
);
330 static ssize_t
scan_sleep_millisecs_store(struct kobject
*kobj
,
331 struct kobj_attribute
*attr
,
332 const char *buf
, size_t count
)
337 err
= strict_strtoul(buf
, 10, &msecs
);
338 if (err
|| msecs
> UINT_MAX
)
341 khugepaged_scan_sleep_millisecs
= msecs
;
342 wake_up_interruptible(&khugepaged_wait
);
346 static struct kobj_attribute scan_sleep_millisecs_attr
=
347 __ATTR(scan_sleep_millisecs
, 0644, scan_sleep_millisecs_show
,
348 scan_sleep_millisecs_store
);
350 static ssize_t
alloc_sleep_millisecs_show(struct kobject
*kobj
,
351 struct kobj_attribute
*attr
,
354 return sprintf(buf
, "%u\n", khugepaged_alloc_sleep_millisecs
);
357 static ssize_t
alloc_sleep_millisecs_store(struct kobject
*kobj
,
358 struct kobj_attribute
*attr
,
359 const char *buf
, size_t count
)
364 err
= strict_strtoul(buf
, 10, &msecs
);
365 if (err
|| msecs
> UINT_MAX
)
368 khugepaged_alloc_sleep_millisecs
= msecs
;
369 wake_up_interruptible(&khugepaged_wait
);
373 static struct kobj_attribute alloc_sleep_millisecs_attr
=
374 __ATTR(alloc_sleep_millisecs
, 0644, alloc_sleep_millisecs_show
,
375 alloc_sleep_millisecs_store
);
377 static ssize_t
pages_to_scan_show(struct kobject
*kobj
,
378 struct kobj_attribute
*attr
,
381 return sprintf(buf
, "%u\n", khugepaged_pages_to_scan
);
383 static ssize_t
pages_to_scan_store(struct kobject
*kobj
,
384 struct kobj_attribute
*attr
,
385 const char *buf
, size_t count
)
390 err
= strict_strtoul(buf
, 10, &pages
);
391 if (err
|| !pages
|| pages
> UINT_MAX
)
394 khugepaged_pages_to_scan
= pages
;
398 static struct kobj_attribute pages_to_scan_attr
=
399 __ATTR(pages_to_scan
, 0644, pages_to_scan_show
,
400 pages_to_scan_store
);
402 static ssize_t
pages_collapsed_show(struct kobject
*kobj
,
403 struct kobj_attribute
*attr
,
406 return sprintf(buf
, "%u\n", khugepaged_pages_collapsed
);
408 static struct kobj_attribute pages_collapsed_attr
=
409 __ATTR_RO(pages_collapsed
);
411 static ssize_t
full_scans_show(struct kobject
*kobj
,
412 struct kobj_attribute
*attr
,
415 return sprintf(buf
, "%u\n", khugepaged_full_scans
);
417 static struct kobj_attribute full_scans_attr
=
418 __ATTR_RO(full_scans
);
420 static ssize_t
khugepaged_defrag_show(struct kobject
*kobj
,
421 struct kobj_attribute
*attr
, char *buf
)
423 return single_flag_show(kobj
, attr
, buf
,
424 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG
);
426 static ssize_t
khugepaged_defrag_store(struct kobject
*kobj
,
427 struct kobj_attribute
*attr
,
428 const char *buf
, size_t count
)
430 return single_flag_store(kobj
, attr
, buf
, count
,
431 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG
);
433 static struct kobj_attribute khugepaged_defrag_attr
=
434 __ATTR(defrag
, 0644, khugepaged_defrag_show
,
435 khugepaged_defrag_store
);
438 * max_ptes_none controls if khugepaged should collapse hugepages over
439 * any unmapped ptes in turn potentially increasing the memory
440 * footprint of the vmas. When max_ptes_none is 0 khugepaged will not
441 * reduce the available free memory in the system as it
442 * runs. Increasing max_ptes_none will instead potentially reduce the
443 * free memory in the system during the khugepaged scan.
445 static ssize_t
khugepaged_max_ptes_none_show(struct kobject
*kobj
,
446 struct kobj_attribute
*attr
,
449 return sprintf(buf
, "%u\n", khugepaged_max_ptes_none
);
451 static ssize_t
khugepaged_max_ptes_none_store(struct kobject
*kobj
,
452 struct kobj_attribute
*attr
,
453 const char *buf
, size_t count
)
456 unsigned long max_ptes_none
;
458 err
= strict_strtoul(buf
, 10, &max_ptes_none
);
459 if (err
|| max_ptes_none
> HPAGE_PMD_NR
-1)
462 khugepaged_max_ptes_none
= max_ptes_none
;
466 static struct kobj_attribute khugepaged_max_ptes_none_attr
=
467 __ATTR(max_ptes_none
, 0644, khugepaged_max_ptes_none_show
,
468 khugepaged_max_ptes_none_store
);
470 static struct attribute
*khugepaged_attr
[] = {
471 &khugepaged_defrag_attr
.attr
,
472 &khugepaged_max_ptes_none_attr
.attr
,
473 &pages_to_scan_attr
.attr
,
474 &pages_collapsed_attr
.attr
,
475 &full_scans_attr
.attr
,
476 &scan_sleep_millisecs_attr
.attr
,
477 &alloc_sleep_millisecs_attr
.attr
,
481 static struct attribute_group khugepaged_attr_group
= {
482 .attrs
= khugepaged_attr
,
483 .name
= "khugepaged",
485 #endif /* CONFIG_SYSFS */
487 static int __init
hugepage_init(void)
491 static struct kobject
*hugepage_kobj
;
495 if (!has_transparent_hugepage()) {
496 transparent_hugepage_flags
= 0;
502 hugepage_kobj
= kobject_create_and_add("transparent_hugepage", mm_kobj
);
503 if (unlikely(!hugepage_kobj
)) {
504 printk(KERN_ERR
"hugepage: failed kobject create\n");
508 err
= sysfs_create_group(hugepage_kobj
, &hugepage_attr_group
);
510 printk(KERN_ERR
"hugepage: failed register hugeage group\n");
514 err
= sysfs_create_group(hugepage_kobj
, &khugepaged_attr_group
);
516 printk(KERN_ERR
"hugepage: failed register hugeage group\n");
521 err
= khugepaged_slab_init();
525 err
= mm_slots_hash_init();
527 khugepaged_slab_free();
532 * By default disable transparent hugepages on smaller systems,
533 * where the extra memory used could hurt more than TLB overhead
534 * is likely to save. The admin can still enable it through /sys.
536 if (totalram_pages
< (512 << (20 - PAGE_SHIFT
)))
537 transparent_hugepage_flags
= 0;
541 set_recommended_min_free_kbytes();
546 module_init(hugepage_init
)
548 static int __init
setup_transparent_hugepage(char *str
)
553 if (!strcmp(str
, "always")) {
554 set_bit(TRANSPARENT_HUGEPAGE_FLAG
,
555 &transparent_hugepage_flags
);
556 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG
,
557 &transparent_hugepage_flags
);
559 } else if (!strcmp(str
, "madvise")) {
560 clear_bit(TRANSPARENT_HUGEPAGE_FLAG
,
561 &transparent_hugepage_flags
);
562 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG
,
563 &transparent_hugepage_flags
);
565 } else if (!strcmp(str
, "never")) {
566 clear_bit(TRANSPARENT_HUGEPAGE_FLAG
,
567 &transparent_hugepage_flags
);
568 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG
,
569 &transparent_hugepage_flags
);
575 "transparent_hugepage= cannot parse, ignored\n");
578 __setup("transparent_hugepage=", setup_transparent_hugepage
);
580 static void prepare_pmd_huge_pte(pgtable_t pgtable
,
581 struct mm_struct
*mm
)
583 assert_spin_locked(&mm
->page_table_lock
);
586 if (!mm
->pmd_huge_pte
)
587 INIT_LIST_HEAD(&pgtable
->lru
);
589 list_add(&pgtable
->lru
, &mm
->pmd_huge_pte
->lru
);
590 mm
->pmd_huge_pte
= pgtable
;
593 static inline pmd_t
maybe_pmd_mkwrite(pmd_t pmd
, struct vm_area_struct
*vma
)
595 if (likely(vma
->vm_flags
& VM_WRITE
))
596 pmd
= pmd_mkwrite(pmd
);
600 static int __do_huge_pmd_anonymous_page(struct mm_struct
*mm
,
601 struct vm_area_struct
*vma
,
602 unsigned long haddr
, pmd_t
*pmd
,
608 VM_BUG_ON(!PageCompound(page
));
609 pgtable
= pte_alloc_one(mm
, haddr
);
610 if (unlikely(!pgtable
)) {
611 mem_cgroup_uncharge_page(page
);
616 clear_huge_page(page
, haddr
, HPAGE_PMD_NR
);
617 __SetPageUptodate(page
);
619 spin_lock(&mm
->page_table_lock
);
620 if (unlikely(!pmd_none(*pmd
))) {
621 spin_unlock(&mm
->page_table_lock
);
622 mem_cgroup_uncharge_page(page
);
624 pte_free(mm
, pgtable
);
627 entry
= mk_pmd(page
, vma
->vm_page_prot
);
628 entry
= maybe_pmd_mkwrite(pmd_mkdirty(entry
), vma
);
629 entry
= pmd_mkhuge(entry
);
631 * The spinlocking to take the lru_lock inside
632 * page_add_new_anon_rmap() acts as a full memory
633 * barrier to be sure clear_huge_page writes become
634 * visible after the set_pmd_at() write.
636 page_add_new_anon_rmap(page
, vma
, haddr
);
637 set_pmd_at(mm
, haddr
, pmd
, entry
);
638 prepare_pmd_huge_pte(pgtable
, mm
);
639 add_mm_counter(mm
, MM_ANONPAGES
, HPAGE_PMD_NR
);
640 spin_unlock(&mm
->page_table_lock
);
646 static inline gfp_t
alloc_hugepage_gfpmask(int defrag
, gfp_t extra_gfp
)
648 return (GFP_TRANSHUGE
& ~(defrag
? 0 : __GFP_WAIT
)) | extra_gfp
;
651 static inline struct page
*alloc_hugepage_vma(int defrag
,
652 struct vm_area_struct
*vma
,
653 unsigned long haddr
, int nd
,
656 return alloc_pages_vma(alloc_hugepage_gfpmask(defrag
, extra_gfp
),
657 HPAGE_PMD_ORDER
, vma
, haddr
, nd
);
661 static inline struct page
*alloc_hugepage(int defrag
)
663 return alloc_pages(alloc_hugepage_gfpmask(defrag
, 0),
668 int do_huge_pmd_anonymous_page(struct mm_struct
*mm
, struct vm_area_struct
*vma
,
669 unsigned long address
, pmd_t
*pmd
,
673 unsigned long haddr
= address
& HPAGE_PMD_MASK
;
676 if (haddr
>= vma
->vm_start
&& haddr
+ HPAGE_PMD_SIZE
<= vma
->vm_end
) {
677 if (unlikely(anon_vma_prepare(vma
)))
679 if (unlikely(khugepaged_enter(vma
)))
681 page
= alloc_hugepage_vma(transparent_hugepage_defrag(vma
),
682 vma
, haddr
, numa_node_id(), 0);
683 if (unlikely(!page
)) {
684 count_vm_event(THP_FAULT_FALLBACK
);
687 count_vm_event(THP_FAULT_ALLOC
);
688 if (unlikely(mem_cgroup_newpage_charge(page
, mm
, GFP_KERNEL
))) {
693 return __do_huge_pmd_anonymous_page(mm
, vma
, haddr
, pmd
, page
);
697 * Use __pte_alloc instead of pte_alloc_map, because we can't
698 * run pte_offset_map on the pmd, if an huge pmd could
699 * materialize from under us from a different thread.
701 if (unlikely(__pte_alloc(mm
, vma
, pmd
, address
)))
703 /* if an huge pmd materialized from under us just retry later */
704 if (unlikely(pmd_trans_huge(*pmd
)))
707 * A regular pmd is established and it can't morph into a huge pmd
708 * from under us anymore at this point because we hold the mmap_sem
709 * read mode and khugepaged takes it in write mode. So now it's
710 * safe to run pte_offset_map().
712 pte
= pte_offset_map(pmd
, address
);
713 return handle_pte_fault(mm
, vma
, address
, pte
, pmd
, flags
);
716 int copy_huge_pmd(struct mm_struct
*dst_mm
, struct mm_struct
*src_mm
,
717 pmd_t
*dst_pmd
, pmd_t
*src_pmd
, unsigned long addr
,
718 struct vm_area_struct
*vma
)
720 struct page
*src_page
;
726 pgtable
= pte_alloc_one(dst_mm
, addr
);
727 if (unlikely(!pgtable
))
730 spin_lock(&dst_mm
->page_table_lock
);
731 spin_lock_nested(&src_mm
->page_table_lock
, SINGLE_DEPTH_NESTING
);
735 if (unlikely(!pmd_trans_huge(pmd
))) {
736 pte_free(dst_mm
, pgtable
);
739 if (unlikely(pmd_trans_splitting(pmd
))) {
740 /* split huge page running from under us */
741 spin_unlock(&src_mm
->page_table_lock
);
742 spin_unlock(&dst_mm
->page_table_lock
);
743 pte_free(dst_mm
, pgtable
);
745 wait_split_huge_page(vma
->anon_vma
, src_pmd
); /* src_vma */
748 src_page
= pmd_page(pmd
);
749 VM_BUG_ON(!PageHead(src_page
));
751 page_dup_rmap(src_page
);
752 add_mm_counter(dst_mm
, MM_ANONPAGES
, HPAGE_PMD_NR
);
754 pmdp_set_wrprotect(src_mm
, addr
, src_pmd
);
755 pmd
= pmd_mkold(pmd_wrprotect(pmd
));
756 set_pmd_at(dst_mm
, addr
, dst_pmd
, pmd
);
757 prepare_pmd_huge_pte(pgtable
, dst_mm
);
761 spin_unlock(&src_mm
->page_table_lock
);
762 spin_unlock(&dst_mm
->page_table_lock
);
767 /* no "address" argument so destroys page coloring of some arch */
768 pgtable_t
get_pmd_huge_pte(struct mm_struct
*mm
)
772 assert_spin_locked(&mm
->page_table_lock
);
775 pgtable
= mm
->pmd_huge_pte
;
776 if (list_empty(&pgtable
->lru
))
777 mm
->pmd_huge_pte
= NULL
;
779 mm
->pmd_huge_pte
= list_entry(pgtable
->lru
.next
,
781 list_del(&pgtable
->lru
);
786 static int do_huge_pmd_wp_page_fallback(struct mm_struct
*mm
,
787 struct vm_area_struct
*vma
,
788 unsigned long address
,
789 pmd_t
*pmd
, pmd_t orig_pmd
,
798 pages
= kmalloc(sizeof(struct page
*) * HPAGE_PMD_NR
,
800 if (unlikely(!pages
)) {
805 for (i
= 0; i
< HPAGE_PMD_NR
; i
++) {
806 pages
[i
] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE
|
808 vma
, address
, page_to_nid(page
));
809 if (unlikely(!pages
[i
] ||
810 mem_cgroup_newpage_charge(pages
[i
], mm
,
814 mem_cgroup_uncharge_start();
816 mem_cgroup_uncharge_page(pages
[i
]);
819 mem_cgroup_uncharge_end();
826 for (i
= 0; i
< HPAGE_PMD_NR
; i
++) {
827 copy_user_highpage(pages
[i
], page
+ i
,
828 haddr
+ PAGE_SHIFT
*i
, vma
);
829 __SetPageUptodate(pages
[i
]);
833 spin_lock(&mm
->page_table_lock
);
834 if (unlikely(!pmd_same(*pmd
, orig_pmd
)))
836 VM_BUG_ON(!PageHead(page
));
838 pmdp_clear_flush_notify(vma
, haddr
, pmd
);
839 /* leave pmd empty until pte is filled */
841 pgtable
= get_pmd_huge_pte(mm
);
842 pmd_populate(mm
, &_pmd
, pgtable
);
844 for (i
= 0; i
< HPAGE_PMD_NR
; i
++, haddr
+= PAGE_SIZE
) {
846 entry
= mk_pte(pages
[i
], vma
->vm_page_prot
);
847 entry
= maybe_mkwrite(pte_mkdirty(entry
), vma
);
848 page_add_new_anon_rmap(pages
[i
], vma
, haddr
);
849 pte
= pte_offset_map(&_pmd
, haddr
);
850 VM_BUG_ON(!pte_none(*pte
));
851 set_pte_at(mm
, haddr
, pte
, entry
);
857 smp_wmb(); /* make pte visible before pmd */
858 pmd_populate(mm
, pmd
, pgtable
);
859 page_remove_rmap(page
);
860 spin_unlock(&mm
->page_table_lock
);
862 ret
|= VM_FAULT_WRITE
;
869 spin_unlock(&mm
->page_table_lock
);
870 mem_cgroup_uncharge_start();
871 for (i
= 0; i
< HPAGE_PMD_NR
; i
++) {
872 mem_cgroup_uncharge_page(pages
[i
]);
875 mem_cgroup_uncharge_end();
880 int do_huge_pmd_wp_page(struct mm_struct
*mm
, struct vm_area_struct
*vma
,
881 unsigned long address
, pmd_t
*pmd
, pmd_t orig_pmd
)
884 struct page
*page
, *new_page
;
887 VM_BUG_ON(!vma
->anon_vma
);
888 spin_lock(&mm
->page_table_lock
);
889 if (unlikely(!pmd_same(*pmd
, orig_pmd
)))
892 page
= pmd_page(orig_pmd
);
893 VM_BUG_ON(!PageCompound(page
) || !PageHead(page
));
894 haddr
= address
& HPAGE_PMD_MASK
;
895 if (page_mapcount(page
) == 1) {
897 entry
= pmd_mkyoung(orig_pmd
);
898 entry
= maybe_pmd_mkwrite(pmd_mkdirty(entry
), vma
);
899 if (pmdp_set_access_flags(vma
, haddr
, pmd
, entry
, 1))
900 update_mmu_cache(vma
, address
, entry
);
901 ret
|= VM_FAULT_WRITE
;
905 spin_unlock(&mm
->page_table_lock
);
907 if (transparent_hugepage_enabled(vma
) &&
908 !transparent_hugepage_debug_cow())
909 new_page
= alloc_hugepage_vma(transparent_hugepage_defrag(vma
),
910 vma
, haddr
, numa_node_id(), 0);
914 if (unlikely(!new_page
)) {
915 count_vm_event(THP_FAULT_FALLBACK
);
916 ret
= do_huge_pmd_wp_page_fallback(mm
, vma
, address
,
917 pmd
, orig_pmd
, page
, haddr
);
921 count_vm_event(THP_FAULT_ALLOC
);
923 if (unlikely(mem_cgroup_newpage_charge(new_page
, mm
, GFP_KERNEL
))) {
930 copy_user_huge_page(new_page
, page
, haddr
, vma
, HPAGE_PMD_NR
);
931 __SetPageUptodate(new_page
);
933 spin_lock(&mm
->page_table_lock
);
935 if (unlikely(!pmd_same(*pmd
, orig_pmd
))) {
936 mem_cgroup_uncharge_page(new_page
);
940 VM_BUG_ON(!PageHead(page
));
941 entry
= mk_pmd(new_page
, vma
->vm_page_prot
);
942 entry
= maybe_pmd_mkwrite(pmd_mkdirty(entry
), vma
);
943 entry
= pmd_mkhuge(entry
);
944 pmdp_clear_flush_notify(vma
, haddr
, pmd
);
945 page_add_new_anon_rmap(new_page
, vma
, haddr
);
946 set_pmd_at(mm
, haddr
, pmd
, entry
);
947 update_mmu_cache(vma
, address
, entry
);
948 page_remove_rmap(page
);
950 ret
|= VM_FAULT_WRITE
;
953 spin_unlock(&mm
->page_table_lock
);
958 struct page
*follow_trans_huge_pmd(struct mm_struct
*mm
,
963 struct page
*page
= NULL
;
965 assert_spin_locked(&mm
->page_table_lock
);
967 if (flags
& FOLL_WRITE
&& !pmd_write(*pmd
))
970 page
= pmd_page(*pmd
);
971 VM_BUG_ON(!PageHead(page
));
972 if (flags
& FOLL_TOUCH
) {
975 * We should set the dirty bit only for FOLL_WRITE but
976 * for now the dirty bit in the pmd is meaningless.
977 * And if the dirty bit will become meaningful and
978 * we'll only set it with FOLL_WRITE, an atomic
979 * set_bit will be required on the pmd to set the
980 * young bit, instead of the current set_pmd_at.
982 _pmd
= pmd_mkyoung(pmd_mkdirty(*pmd
));
983 set_pmd_at(mm
, addr
& HPAGE_PMD_MASK
, pmd
, _pmd
);
985 page
+= (addr
& ~HPAGE_PMD_MASK
) >> PAGE_SHIFT
;
986 VM_BUG_ON(!PageCompound(page
));
987 if (flags
& FOLL_GET
)
994 int zap_huge_pmd(struct mmu_gather
*tlb
, struct vm_area_struct
*vma
,
999 spin_lock(&tlb
->mm
->page_table_lock
);
1000 if (likely(pmd_trans_huge(*pmd
))) {
1001 if (unlikely(pmd_trans_splitting(*pmd
))) {
1002 spin_unlock(&tlb
->mm
->page_table_lock
);
1003 wait_split_huge_page(vma
->anon_vma
,
1008 pgtable
= get_pmd_huge_pte(tlb
->mm
);
1009 page
= pmd_page(*pmd
);
1011 page_remove_rmap(page
);
1012 VM_BUG_ON(page_mapcount(page
) < 0);
1013 add_mm_counter(tlb
->mm
, MM_ANONPAGES
, -HPAGE_PMD_NR
);
1014 VM_BUG_ON(!PageHead(page
));
1015 spin_unlock(&tlb
->mm
->page_table_lock
);
1016 tlb_remove_page(tlb
, page
);
1017 pte_free(tlb
->mm
, pgtable
);
1021 spin_unlock(&tlb
->mm
->page_table_lock
);
1026 int mincore_huge_pmd(struct vm_area_struct
*vma
, pmd_t
*pmd
,
1027 unsigned long addr
, unsigned long end
,
1032 spin_lock(&vma
->vm_mm
->page_table_lock
);
1033 if (likely(pmd_trans_huge(*pmd
))) {
1034 ret
= !pmd_trans_splitting(*pmd
);
1035 spin_unlock(&vma
->vm_mm
->page_table_lock
);
1037 wait_split_huge_page(vma
->anon_vma
, pmd
);
1040 * All logical pages in the range are present
1041 * if backed by a huge page.
1043 memset(vec
, 1, (end
- addr
) >> PAGE_SHIFT
);
1046 spin_unlock(&vma
->vm_mm
->page_table_lock
);
1051 int change_huge_pmd(struct vm_area_struct
*vma
, pmd_t
*pmd
,
1052 unsigned long addr
, pgprot_t newprot
)
1054 struct mm_struct
*mm
= vma
->vm_mm
;
1057 spin_lock(&mm
->page_table_lock
);
1058 if (likely(pmd_trans_huge(*pmd
))) {
1059 if (unlikely(pmd_trans_splitting(*pmd
))) {
1060 spin_unlock(&mm
->page_table_lock
);
1061 wait_split_huge_page(vma
->anon_vma
, pmd
);
1065 entry
= pmdp_get_and_clear(mm
, addr
, pmd
);
1066 entry
= pmd_modify(entry
, newprot
);
1067 set_pmd_at(mm
, addr
, pmd
, entry
);
1068 spin_unlock(&vma
->vm_mm
->page_table_lock
);
1069 flush_tlb_range(vma
, addr
, addr
+ HPAGE_PMD_SIZE
);
1073 spin_unlock(&vma
->vm_mm
->page_table_lock
);
1078 pmd_t
*page_check_address_pmd(struct page
*page
,
1079 struct mm_struct
*mm
,
1080 unsigned long address
,
1081 enum page_check_address_pmd_flag flag
)
1085 pmd_t
*pmd
, *ret
= NULL
;
1087 if (address
& ~HPAGE_PMD_MASK
)
1090 pgd
= pgd_offset(mm
, address
);
1091 if (!pgd_present(*pgd
))
1094 pud
= pud_offset(pgd
, address
);
1095 if (!pud_present(*pud
))
1098 pmd
= pmd_offset(pud
, address
);
1101 if (pmd_page(*pmd
) != page
)
1104 * split_vma() may create temporary aliased mappings. There is
1105 * no risk as long as all huge pmd are found and have their
1106 * splitting bit set before __split_huge_page_refcount
1107 * runs. Finding the same huge pmd more than once during the
1108 * same rmap walk is not a problem.
1110 if (flag
== PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG
&&
1111 pmd_trans_splitting(*pmd
))
1113 if (pmd_trans_huge(*pmd
)) {
1114 VM_BUG_ON(flag
== PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG
&&
1115 !pmd_trans_splitting(*pmd
));
1122 static int __split_huge_page_splitting(struct page
*page
,
1123 struct vm_area_struct
*vma
,
1124 unsigned long address
)
1126 struct mm_struct
*mm
= vma
->vm_mm
;
1130 spin_lock(&mm
->page_table_lock
);
1131 pmd
= page_check_address_pmd(page
, mm
, address
,
1132 PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG
);
1135 * We can't temporarily set the pmd to null in order
1136 * to split it, the pmd must remain marked huge at all
1137 * times or the VM won't take the pmd_trans_huge paths
1138 * and it won't wait on the anon_vma->root->lock to
1139 * serialize against split_huge_page*.
1141 pmdp_splitting_flush_notify(vma
, address
, pmd
);
1144 spin_unlock(&mm
->page_table_lock
);
1149 static void __split_huge_page_refcount(struct page
*page
)
1152 unsigned long head_index
= page
->index
;
1153 struct zone
*zone
= page_zone(page
);
1156 /* prevent PageLRU to go away from under us, and freeze lru stats */
1157 spin_lock_irq(&zone
->lru_lock
);
1158 compound_lock(page
);
1160 for (i
= 1; i
< HPAGE_PMD_NR
; i
++) {
1161 struct page
*page_tail
= page
+ i
;
1163 /* tail_page->_count cannot change */
1164 atomic_sub(atomic_read(&page_tail
->_count
), &page
->_count
);
1165 BUG_ON(page_count(page
) <= 0);
1166 atomic_add(page_mapcount(page
) + 1, &page_tail
->_count
);
1167 BUG_ON(atomic_read(&page_tail
->_count
) <= 0);
1169 /* after clearing PageTail the gup refcount can be released */
1173 * retain hwpoison flag of the poisoned tail page:
1174 * fix for the unsuitable process killed on Guest Machine(KVM)
1175 * by the memory-failure.
1177 page_tail
->flags
&= ~PAGE_FLAGS_CHECK_AT_PREP
| __PG_HWPOISON
;
1178 page_tail
->flags
|= (page
->flags
&
1179 ((1L << PG_referenced
) |
1180 (1L << PG_swapbacked
) |
1181 (1L << PG_mlocked
) |
1182 (1L << PG_uptodate
)));
1183 page_tail
->flags
|= (1L << PG_dirty
);
1186 * 1) clear PageTail before overwriting first_page
1187 * 2) clear PageTail before clearing PageHead for VM_BUG_ON
1192 * __split_huge_page_splitting() already set the
1193 * splitting bit in all pmd that could map this
1194 * hugepage, that will ensure no CPU can alter the
1195 * mapcount on the head page. The mapcount is only
1196 * accounted in the head page and it has to be
1197 * transferred to all tail pages in the below code. So
1198 * for this code to be safe, the split the mapcount
1199 * can't change. But that doesn't mean userland can't
1200 * keep changing and reading the page contents while
1201 * we transfer the mapcount, so the pmd splitting
1202 * status is achieved setting a reserved bit in the
1203 * pmd, not by clearing the present bit.
1205 BUG_ON(page_mapcount(page_tail
));
1206 page_tail
->_mapcount
= page
->_mapcount
;
1208 BUG_ON(page_tail
->mapping
);
1209 page_tail
->mapping
= page
->mapping
;
1211 page_tail
->index
= ++head_index
;
1213 BUG_ON(!PageAnon(page_tail
));
1214 BUG_ON(!PageUptodate(page_tail
));
1215 BUG_ON(!PageDirty(page_tail
));
1216 BUG_ON(!PageSwapBacked(page_tail
));
1218 mem_cgroup_split_huge_fixup(page
, page_tail
);
1220 lru_add_page_tail(zone
, page
, page_tail
);
1223 __dec_zone_page_state(page
, NR_ANON_TRANSPARENT_HUGEPAGES
);
1224 __mod_zone_page_state(zone
, NR_ANON_PAGES
, HPAGE_PMD_NR
);
1227 * A hugepage counts for HPAGE_PMD_NR pages on the LRU statistics,
1228 * so adjust those appropriately if this page is on the LRU.
1230 if (PageLRU(page
)) {
1231 zonestat
= NR_LRU_BASE
+ page_lru(page
);
1232 __mod_zone_page_state(zone
, zonestat
, -(HPAGE_PMD_NR
-1));
1235 ClearPageCompound(page
);
1236 compound_unlock(page
);
1237 spin_unlock_irq(&zone
->lru_lock
);
1239 for (i
= 1; i
< HPAGE_PMD_NR
; i
++) {
1240 struct page
*page_tail
= page
+ i
;
1241 BUG_ON(page_count(page_tail
) <= 0);
1243 * Tail pages may be freed if there wasn't any mapping
1244 * like if add_to_swap() is running on a lru page that
1245 * had its mapping zapped. And freeing these pages
1246 * requires taking the lru_lock so we do the put_page
1247 * of the tail pages after the split is complete.
1249 put_page(page_tail
);
1253 * Only the head page (now become a regular page) is required
1254 * to be pinned by the caller.
1256 BUG_ON(page_count(page
) <= 0);
1259 static int __split_huge_page_map(struct page
*page
,
1260 struct vm_area_struct
*vma
,
1261 unsigned long address
)
1263 struct mm_struct
*mm
= vma
->vm_mm
;
1267 unsigned long haddr
;
1269 spin_lock(&mm
->page_table_lock
);
1270 pmd
= page_check_address_pmd(page
, mm
, address
,
1271 PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG
);
1273 pgtable
= get_pmd_huge_pte(mm
);
1274 pmd_populate(mm
, &_pmd
, pgtable
);
1276 for (i
= 0, haddr
= address
; i
< HPAGE_PMD_NR
;
1277 i
++, haddr
+= PAGE_SIZE
) {
1279 BUG_ON(PageCompound(page
+i
));
1280 entry
= mk_pte(page
+ i
, vma
->vm_page_prot
);
1281 entry
= maybe_mkwrite(pte_mkdirty(entry
), vma
);
1282 if (!pmd_write(*pmd
))
1283 entry
= pte_wrprotect(entry
);
1285 BUG_ON(page_mapcount(page
) != 1);
1286 if (!pmd_young(*pmd
))
1287 entry
= pte_mkold(entry
);
1288 pte
= pte_offset_map(&_pmd
, haddr
);
1289 BUG_ON(!pte_none(*pte
));
1290 set_pte_at(mm
, haddr
, pte
, entry
);
1295 smp_wmb(); /* make pte visible before pmd */
1297 * Up to this point the pmd is present and huge and
1298 * userland has the whole access to the hugepage
1299 * during the split (which happens in place). If we
1300 * overwrite the pmd with the not-huge version
1301 * pointing to the pte here (which of course we could
1302 * if all CPUs were bug free), userland could trigger
1303 * a small page size TLB miss on the small sized TLB
1304 * while the hugepage TLB entry is still established
1305 * in the huge TLB. Some CPU doesn't like that. See
1306 * http://support.amd.com/us/Processor_TechDocs/41322.pdf,
1307 * Erratum 383 on page 93. Intel should be safe but is
1308 * also warns that it's only safe if the permission
1309 * and cache attributes of the two entries loaded in
1310 * the two TLB is identical (which should be the case
1311 * here). But it is generally safer to never allow
1312 * small and huge TLB entries for the same virtual
1313 * address to be loaded simultaneously. So instead of
1314 * doing "pmd_populate(); flush_tlb_range();" we first
1315 * mark the current pmd notpresent (atomically because
1316 * here the pmd_trans_huge and pmd_trans_splitting
1317 * must remain set at all times on the pmd until the
1318 * split is complete for this pmd), then we flush the
1319 * SMP TLB and finally we write the non-huge version
1320 * of the pmd entry with pmd_populate.
1322 set_pmd_at(mm
, address
, pmd
, pmd_mknotpresent(*pmd
));
1323 flush_tlb_range(vma
, address
, address
+ HPAGE_PMD_SIZE
);
1324 pmd_populate(mm
, pmd
, pgtable
);
1327 spin_unlock(&mm
->page_table_lock
);
1332 /* must be called with anon_vma->root->lock hold */
1333 static void __split_huge_page(struct page
*page
,
1334 struct anon_vma
*anon_vma
)
1336 int mapcount
, mapcount2
;
1337 struct anon_vma_chain
*avc
;
1339 BUG_ON(!PageHead(page
));
1340 BUG_ON(PageTail(page
));
1343 list_for_each_entry(avc
, &anon_vma
->head
, same_anon_vma
) {
1344 struct vm_area_struct
*vma
= avc
->vma
;
1345 unsigned long addr
= vma_address(page
, vma
);
1346 BUG_ON(is_vma_temporary_stack(vma
));
1347 if (addr
== -EFAULT
)
1349 mapcount
+= __split_huge_page_splitting(page
, vma
, addr
);
1352 * It is critical that new vmas are added to the tail of the
1353 * anon_vma list. This guarantes that if copy_huge_pmd() runs
1354 * and establishes a child pmd before
1355 * __split_huge_page_splitting() freezes the parent pmd (so if
1356 * we fail to prevent copy_huge_pmd() from running until the
1357 * whole __split_huge_page() is complete), we will still see
1358 * the newly established pmd of the child later during the
1359 * walk, to be able to set it as pmd_trans_splitting too.
1361 if (mapcount
!= page_mapcount(page
))
1362 printk(KERN_ERR
"mapcount %d page_mapcount %d\n",
1363 mapcount
, page_mapcount(page
));
1364 BUG_ON(mapcount
!= page_mapcount(page
));
1366 __split_huge_page_refcount(page
);
1369 list_for_each_entry(avc
, &anon_vma
->head
, same_anon_vma
) {
1370 struct vm_area_struct
*vma
= avc
->vma
;
1371 unsigned long addr
= vma_address(page
, vma
);
1372 BUG_ON(is_vma_temporary_stack(vma
));
1373 if (addr
== -EFAULT
)
1375 mapcount2
+= __split_huge_page_map(page
, vma
, addr
);
1377 if (mapcount
!= mapcount2
)
1378 printk(KERN_ERR
"mapcount %d mapcount2 %d page_mapcount %d\n",
1379 mapcount
, mapcount2
, page_mapcount(page
));
1380 BUG_ON(mapcount
!= mapcount2
);
1383 int split_huge_page(struct page
*page
)
1385 struct anon_vma
*anon_vma
;
1388 BUG_ON(!PageAnon(page
));
1389 anon_vma
= page_lock_anon_vma(page
);
1393 if (!PageCompound(page
))
1396 BUG_ON(!PageSwapBacked(page
));
1397 __split_huge_page(page
, anon_vma
);
1398 count_vm_event(THP_SPLIT
);
1400 BUG_ON(PageCompound(page
));
1402 page_unlock_anon_vma(anon_vma
);
1407 int hugepage_madvise(struct vm_area_struct
*vma
,
1408 unsigned long *vm_flags
, int advice
)
1413 * Be somewhat over-protective like KSM for now!
1415 if (*vm_flags
& (VM_HUGEPAGE
|
1416 VM_SHARED
| VM_MAYSHARE
|
1417 VM_PFNMAP
| VM_IO
| VM_DONTEXPAND
|
1418 VM_RESERVED
| VM_HUGETLB
| VM_INSERTPAGE
|
1419 VM_MIXEDMAP
| VM_SAO
))
1421 *vm_flags
&= ~VM_NOHUGEPAGE
;
1422 *vm_flags
|= VM_HUGEPAGE
;
1424 * If the vma become good for khugepaged to scan,
1425 * register it here without waiting a page fault that
1426 * may not happen any time soon.
1428 if (unlikely(khugepaged_enter_vma_merge(vma
)))
1431 case MADV_NOHUGEPAGE
:
1433 * Be somewhat over-protective like KSM for now!
1435 if (*vm_flags
& (VM_NOHUGEPAGE
|
1436 VM_SHARED
| VM_MAYSHARE
|
1437 VM_PFNMAP
| VM_IO
| VM_DONTEXPAND
|
1438 VM_RESERVED
| VM_HUGETLB
| VM_INSERTPAGE
|
1439 VM_MIXEDMAP
| VM_SAO
))
1441 *vm_flags
&= ~VM_HUGEPAGE
;
1442 *vm_flags
|= VM_NOHUGEPAGE
;
1444 * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning
1445 * this vma even if we leave the mm registered in khugepaged if
1446 * it got registered before VM_NOHUGEPAGE was set.
1454 static int __init
khugepaged_slab_init(void)
1456 mm_slot_cache
= kmem_cache_create("khugepaged_mm_slot",
1457 sizeof(struct mm_slot
),
1458 __alignof__(struct mm_slot
), 0, NULL
);
1465 static void __init
khugepaged_slab_free(void)
1467 kmem_cache_destroy(mm_slot_cache
);
1468 mm_slot_cache
= NULL
;
1471 static inline struct mm_slot
*alloc_mm_slot(void)
1473 if (!mm_slot_cache
) /* initialization failed */
1475 return kmem_cache_zalloc(mm_slot_cache
, GFP_KERNEL
);
1478 static inline void free_mm_slot(struct mm_slot
*mm_slot
)
1480 kmem_cache_free(mm_slot_cache
, mm_slot
);
1483 static int __init
mm_slots_hash_init(void)
1485 mm_slots_hash
= kzalloc(MM_SLOTS_HASH_HEADS
* sizeof(struct hlist_head
),
1493 static void __init
mm_slots_hash_free(void)
1495 kfree(mm_slots_hash
);
1496 mm_slots_hash
= NULL
;
1500 static struct mm_slot
*get_mm_slot(struct mm_struct
*mm
)
1502 struct mm_slot
*mm_slot
;
1503 struct hlist_head
*bucket
;
1504 struct hlist_node
*node
;
1506 bucket
= &mm_slots_hash
[((unsigned long)mm
/ sizeof(struct mm_struct
))
1507 % MM_SLOTS_HASH_HEADS
];
1508 hlist_for_each_entry(mm_slot
, node
, bucket
, hash
) {
1509 if (mm
== mm_slot
->mm
)
1515 static void insert_to_mm_slots_hash(struct mm_struct
*mm
,
1516 struct mm_slot
*mm_slot
)
1518 struct hlist_head
*bucket
;
1520 bucket
= &mm_slots_hash
[((unsigned long)mm
/ sizeof(struct mm_struct
))
1521 % MM_SLOTS_HASH_HEADS
];
1523 hlist_add_head(&mm_slot
->hash
, bucket
);
1526 static inline int khugepaged_test_exit(struct mm_struct
*mm
)
1528 return atomic_read(&mm
->mm_users
) == 0;
1531 int __khugepaged_enter(struct mm_struct
*mm
)
1533 struct mm_slot
*mm_slot
;
1536 mm_slot
= alloc_mm_slot();
1540 /* __khugepaged_exit() must not run from under us */
1541 VM_BUG_ON(khugepaged_test_exit(mm
));
1542 if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE
, &mm
->flags
))) {
1543 free_mm_slot(mm_slot
);
1547 spin_lock(&khugepaged_mm_lock
);
1548 insert_to_mm_slots_hash(mm
, mm_slot
);
1550 * Insert just behind the scanning cursor, to let the area settle
1553 wakeup
= list_empty(&khugepaged_scan
.mm_head
);
1554 list_add_tail(&mm_slot
->mm_node
, &khugepaged_scan
.mm_head
);
1555 spin_unlock(&khugepaged_mm_lock
);
1557 atomic_inc(&mm
->mm_count
);
1559 wake_up_interruptible(&khugepaged_wait
);
1564 int khugepaged_enter_vma_merge(struct vm_area_struct
*vma
)
1566 unsigned long hstart
, hend
;
1569 * Not yet faulted in so we will register later in the
1570 * page fault if needed.
1573 if (vma
->vm_file
|| vma
->vm_ops
)
1574 /* khugepaged not yet working on file or special mappings */
1576 VM_BUG_ON(is_linear_pfn_mapping(vma
) || is_pfn_mapping(vma
));
1577 hstart
= (vma
->vm_start
+ ~HPAGE_PMD_MASK
) & HPAGE_PMD_MASK
;
1578 hend
= vma
->vm_end
& HPAGE_PMD_MASK
;
1580 return khugepaged_enter(vma
);
1584 void __khugepaged_exit(struct mm_struct
*mm
)
1586 struct mm_slot
*mm_slot
;
1589 spin_lock(&khugepaged_mm_lock
);
1590 mm_slot
= get_mm_slot(mm
);
1591 if (mm_slot
&& khugepaged_scan
.mm_slot
!= mm_slot
) {
1592 hlist_del(&mm_slot
->hash
);
1593 list_del(&mm_slot
->mm_node
);
1598 spin_unlock(&khugepaged_mm_lock
);
1599 clear_bit(MMF_VM_HUGEPAGE
, &mm
->flags
);
1600 free_mm_slot(mm_slot
);
1602 } else if (mm_slot
) {
1603 spin_unlock(&khugepaged_mm_lock
);
1605 * This is required to serialize against
1606 * khugepaged_test_exit() (which is guaranteed to run
1607 * under mmap sem read mode). Stop here (after we
1608 * return all pagetables will be destroyed) until
1609 * khugepaged has finished working on the pagetables
1610 * under the mmap_sem.
1612 down_write(&mm
->mmap_sem
);
1613 up_write(&mm
->mmap_sem
);
1615 spin_unlock(&khugepaged_mm_lock
);
1618 static void release_pte_page(struct page
*page
)
1620 /* 0 stands for page_is_file_cache(page) == false */
1621 dec_zone_page_state(page
, NR_ISOLATED_ANON
+ 0);
1623 putback_lru_page(page
);
1626 static void release_pte_pages(pte_t
*pte
, pte_t
*_pte
)
1628 while (--_pte
>= pte
) {
1629 pte_t pteval
= *_pte
;
1630 if (!pte_none(pteval
))
1631 release_pte_page(pte_page(pteval
));
1635 static void release_all_pte_pages(pte_t
*pte
)
1637 release_pte_pages(pte
, pte
+ HPAGE_PMD_NR
);
1640 static int __collapse_huge_page_isolate(struct vm_area_struct
*vma
,
1641 unsigned long address
,
1646 int referenced
= 0, isolated
= 0, none
= 0;
1647 for (_pte
= pte
; _pte
< pte
+HPAGE_PMD_NR
;
1648 _pte
++, address
+= PAGE_SIZE
) {
1649 pte_t pteval
= *_pte
;
1650 if (pte_none(pteval
)) {
1651 if (++none
<= khugepaged_max_ptes_none
)
1654 release_pte_pages(pte
, _pte
);
1658 if (!pte_present(pteval
) || !pte_write(pteval
)) {
1659 release_pte_pages(pte
, _pte
);
1662 page
= vm_normal_page(vma
, address
, pteval
);
1663 if (unlikely(!page
)) {
1664 release_pte_pages(pte
, _pte
);
1667 VM_BUG_ON(PageCompound(page
));
1668 BUG_ON(!PageAnon(page
));
1669 VM_BUG_ON(!PageSwapBacked(page
));
1671 /* cannot use mapcount: can't collapse if there's a gup pin */
1672 if (page_count(page
) != 1) {
1673 release_pte_pages(pte
, _pte
);
1677 * We can do it before isolate_lru_page because the
1678 * page can't be freed from under us. NOTE: PG_lock
1679 * is needed to serialize against split_huge_page
1680 * when invoked from the VM.
1682 if (!trylock_page(page
)) {
1683 release_pte_pages(pte
, _pte
);
1687 * Isolate the page to avoid collapsing an hugepage
1688 * currently in use by the VM.
1690 if (isolate_lru_page(page
)) {
1692 release_pte_pages(pte
, _pte
);
1695 /* 0 stands for page_is_file_cache(page) == false */
1696 inc_zone_page_state(page
, NR_ISOLATED_ANON
+ 0);
1697 VM_BUG_ON(!PageLocked(page
));
1698 VM_BUG_ON(PageLRU(page
));
1700 /* If there is no mapped pte young don't collapse the page */
1701 if (pte_young(pteval
) || PageReferenced(page
) ||
1702 mmu_notifier_test_young(vma
->vm_mm
, address
))
1705 if (unlikely(!referenced
))
1706 release_all_pte_pages(pte
);
1713 static void __collapse_huge_page_copy(pte_t
*pte
, struct page
*page
,
1714 struct vm_area_struct
*vma
,
1715 unsigned long address
,
1719 for (_pte
= pte
; _pte
< pte
+HPAGE_PMD_NR
; _pte
++) {
1720 pte_t pteval
= *_pte
;
1721 struct page
*src_page
;
1723 if (pte_none(pteval
)) {
1724 clear_user_highpage(page
, address
);
1725 add_mm_counter(vma
->vm_mm
, MM_ANONPAGES
, 1);
1727 src_page
= pte_page(pteval
);
1728 copy_user_highpage(page
, src_page
, address
, vma
);
1729 VM_BUG_ON(page_mapcount(src_page
) != 1);
1730 VM_BUG_ON(page_count(src_page
) != 2);
1731 release_pte_page(src_page
);
1733 * ptl mostly unnecessary, but preempt has to
1734 * be disabled to update the per-cpu stats
1735 * inside page_remove_rmap().
1739 * paravirt calls inside pte_clear here are
1742 pte_clear(vma
->vm_mm
, address
, _pte
);
1743 page_remove_rmap(src_page
);
1745 free_page_and_swap_cache(src_page
);
1748 address
+= PAGE_SIZE
;
1753 static void collapse_huge_page(struct mm_struct
*mm
,
1754 unsigned long address
,
1755 struct page
**hpage
,
1756 struct vm_area_struct
*vma
,
1764 struct page
*new_page
;
1767 unsigned long hstart
, hend
;
1769 VM_BUG_ON(address
& ~HPAGE_PMD_MASK
);
1773 if (unlikely(mem_cgroup_newpage_charge(new_page
, mm
, GFP_KERNEL
))) {
1774 up_read(&mm
->mmap_sem
);
1780 * Allocate the page while the vma is still valid and under
1781 * the mmap_sem read mode so there is no memory allocation
1782 * later when we take the mmap_sem in write mode. This is more
1783 * friendly behavior (OTOH it may actually hide bugs) to
1784 * filesystems in userland with daemons allocating memory in
1785 * the userland I/O paths. Allocating memory with the
1786 * mmap_sem in read mode is good idea also to allow greater
1789 new_page
= alloc_hugepage_vma(khugepaged_defrag(), vma
, address
,
1790 node
, __GFP_OTHER_NODE
);
1791 if (unlikely(!new_page
)) {
1792 up_read(&mm
->mmap_sem
);
1793 count_vm_event(THP_COLLAPSE_ALLOC_FAILED
);
1794 *hpage
= ERR_PTR(-ENOMEM
);
1797 count_vm_event(THP_COLLAPSE_ALLOC
);
1798 if (unlikely(mem_cgroup_newpage_charge(new_page
, mm
, GFP_KERNEL
))) {
1799 up_read(&mm
->mmap_sem
);
1805 /* after allocating the hugepage upgrade to mmap_sem write mode */
1806 up_read(&mm
->mmap_sem
);
1809 * Prevent all access to pagetables with the exception of
1810 * gup_fast later hanlded by the ptep_clear_flush and the VM
1811 * handled by the anon_vma lock + PG_lock.
1813 down_write(&mm
->mmap_sem
);
1814 if (unlikely(khugepaged_test_exit(mm
)))
1817 vma
= find_vma(mm
, address
);
1818 hstart
= (vma
->vm_start
+ ~HPAGE_PMD_MASK
) & HPAGE_PMD_MASK
;
1819 hend
= vma
->vm_end
& HPAGE_PMD_MASK
;
1820 if (address
< hstart
|| address
+ HPAGE_PMD_SIZE
> hend
)
1823 if ((!(vma
->vm_flags
& VM_HUGEPAGE
) && !khugepaged_always()) ||
1824 (vma
->vm_flags
& VM_NOHUGEPAGE
))
1827 /* VM_PFNMAP vmas may have vm_ops null but vm_file set */
1828 if (!vma
->anon_vma
|| vma
->vm_ops
|| vma
->vm_file
)
1830 if (is_vma_temporary_stack(vma
))
1832 VM_BUG_ON(is_linear_pfn_mapping(vma
) || is_pfn_mapping(vma
));
1834 pgd
= pgd_offset(mm
, address
);
1835 if (!pgd_present(*pgd
))
1838 pud
= pud_offset(pgd
, address
);
1839 if (!pud_present(*pud
))
1842 pmd
= pmd_offset(pud
, address
);
1843 /* pmd can't go away or become huge under us */
1844 if (!pmd_present(*pmd
) || pmd_trans_huge(*pmd
))
1847 anon_vma_lock(vma
->anon_vma
);
1849 pte
= pte_offset_map(pmd
, address
);
1850 ptl
= pte_lockptr(mm
, pmd
);
1852 spin_lock(&mm
->page_table_lock
); /* probably unnecessary */
1854 * After this gup_fast can't run anymore. This also removes
1855 * any huge TLB entry from the CPU so we won't allow
1856 * huge and small TLB entries for the same virtual address
1857 * to avoid the risk of CPU bugs in that area.
1859 _pmd
= pmdp_clear_flush_notify(vma
, address
, pmd
);
1860 spin_unlock(&mm
->page_table_lock
);
1863 isolated
= __collapse_huge_page_isolate(vma
, address
, pte
);
1866 if (unlikely(!isolated
)) {
1868 spin_lock(&mm
->page_table_lock
);
1869 BUG_ON(!pmd_none(*pmd
));
1870 set_pmd_at(mm
, address
, pmd
, _pmd
);
1871 spin_unlock(&mm
->page_table_lock
);
1872 anon_vma_unlock(vma
->anon_vma
);
1877 * All pages are isolated and locked so anon_vma rmap
1878 * can't run anymore.
1880 anon_vma_unlock(vma
->anon_vma
);
1882 __collapse_huge_page_copy(pte
, new_page
, vma
, address
, ptl
);
1884 __SetPageUptodate(new_page
);
1885 pgtable
= pmd_pgtable(_pmd
);
1886 VM_BUG_ON(page_count(pgtable
) != 1);
1887 VM_BUG_ON(page_mapcount(pgtable
) != 0);
1889 _pmd
= mk_pmd(new_page
, vma
->vm_page_prot
);
1890 _pmd
= maybe_pmd_mkwrite(pmd_mkdirty(_pmd
), vma
);
1891 _pmd
= pmd_mkhuge(_pmd
);
1894 * spin_lock() below is not the equivalent of smp_wmb(), so
1895 * this is needed to avoid the copy_huge_page writes to become
1896 * visible after the set_pmd_at() write.
1900 spin_lock(&mm
->page_table_lock
);
1901 BUG_ON(!pmd_none(*pmd
));
1902 page_add_new_anon_rmap(new_page
, vma
, address
);
1903 set_pmd_at(mm
, address
, pmd
, _pmd
);
1904 update_mmu_cache(vma
, address
, entry
);
1905 prepare_pmd_huge_pte(pgtable
, mm
);
1907 spin_unlock(&mm
->page_table_lock
);
1912 khugepaged_pages_collapsed
++;
1914 up_write(&mm
->mmap_sem
);
1918 mem_cgroup_uncharge_page(new_page
);
1925 static int khugepaged_scan_pmd(struct mm_struct
*mm
,
1926 struct vm_area_struct
*vma
,
1927 unsigned long address
,
1928 struct page
**hpage
)
1934 int ret
= 0, referenced
= 0, none
= 0;
1936 unsigned long _address
;
1940 VM_BUG_ON(address
& ~HPAGE_PMD_MASK
);
1942 pgd
= pgd_offset(mm
, address
);
1943 if (!pgd_present(*pgd
))
1946 pud
= pud_offset(pgd
, address
);
1947 if (!pud_present(*pud
))
1950 pmd
= pmd_offset(pud
, address
);
1951 if (!pmd_present(*pmd
) || pmd_trans_huge(*pmd
))
1954 pte
= pte_offset_map_lock(mm
, pmd
, address
, &ptl
);
1955 for (_address
= address
, _pte
= pte
; _pte
< pte
+HPAGE_PMD_NR
;
1956 _pte
++, _address
+= PAGE_SIZE
) {
1957 pte_t pteval
= *_pte
;
1958 if (pte_none(pteval
)) {
1959 if (++none
<= khugepaged_max_ptes_none
)
1964 if (!pte_present(pteval
) || !pte_write(pteval
))
1966 page
= vm_normal_page(vma
, _address
, pteval
);
1967 if (unlikely(!page
))
1970 * Chose the node of the first page. This could
1971 * be more sophisticated and look at more pages,
1972 * but isn't for now.
1975 node
= page_to_nid(page
);
1976 VM_BUG_ON(PageCompound(page
));
1977 if (!PageLRU(page
) || PageLocked(page
) || !PageAnon(page
))
1979 /* cannot use mapcount: can't collapse if there's a gup pin */
1980 if (page_count(page
) != 1)
1982 if (pte_young(pteval
) || PageReferenced(page
) ||
1983 mmu_notifier_test_young(vma
->vm_mm
, address
))
1989 pte_unmap_unlock(pte
, ptl
);
1991 /* collapse_huge_page will return with the mmap_sem released */
1992 collapse_huge_page(mm
, address
, hpage
, vma
, node
);
1997 static void collect_mm_slot(struct mm_slot
*mm_slot
)
1999 struct mm_struct
*mm
= mm_slot
->mm
;
2001 VM_BUG_ON(!spin_is_locked(&khugepaged_mm_lock
));
2003 if (khugepaged_test_exit(mm
)) {
2005 hlist_del(&mm_slot
->hash
);
2006 list_del(&mm_slot
->mm_node
);
2009 * Not strictly needed because the mm exited already.
2011 * clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
2014 /* khugepaged_mm_lock actually not necessary for the below */
2015 free_mm_slot(mm_slot
);
2020 static unsigned int khugepaged_scan_mm_slot(unsigned int pages
,
2021 struct page
**hpage
)
2023 struct mm_slot
*mm_slot
;
2024 struct mm_struct
*mm
;
2025 struct vm_area_struct
*vma
;
2029 VM_BUG_ON(!spin_is_locked(&khugepaged_mm_lock
));
2031 if (khugepaged_scan
.mm_slot
)
2032 mm_slot
= khugepaged_scan
.mm_slot
;
2034 mm_slot
= list_entry(khugepaged_scan
.mm_head
.next
,
2035 struct mm_slot
, mm_node
);
2036 khugepaged_scan
.address
= 0;
2037 khugepaged_scan
.mm_slot
= mm_slot
;
2039 spin_unlock(&khugepaged_mm_lock
);
2042 down_read(&mm
->mmap_sem
);
2043 if (unlikely(khugepaged_test_exit(mm
)))
2046 vma
= find_vma(mm
, khugepaged_scan
.address
);
2049 for (; vma
; vma
= vma
->vm_next
) {
2050 unsigned long hstart
, hend
;
2053 if (unlikely(khugepaged_test_exit(mm
))) {
2058 if ((!(vma
->vm_flags
& VM_HUGEPAGE
) &&
2059 !khugepaged_always()) ||
2060 (vma
->vm_flags
& VM_NOHUGEPAGE
)) {
2065 /* VM_PFNMAP vmas may have vm_ops null but vm_file set */
2066 if (!vma
->anon_vma
|| vma
->vm_ops
|| vma
->vm_file
)
2068 if (is_vma_temporary_stack(vma
))
2071 VM_BUG_ON(is_linear_pfn_mapping(vma
) || is_pfn_mapping(vma
));
2073 hstart
= (vma
->vm_start
+ ~HPAGE_PMD_MASK
) & HPAGE_PMD_MASK
;
2074 hend
= vma
->vm_end
& HPAGE_PMD_MASK
;
2077 if (khugepaged_scan
.address
> hend
)
2079 if (khugepaged_scan
.address
< hstart
)
2080 khugepaged_scan
.address
= hstart
;
2081 VM_BUG_ON(khugepaged_scan
.address
& ~HPAGE_PMD_MASK
);
2083 while (khugepaged_scan
.address
< hend
) {
2086 if (unlikely(khugepaged_test_exit(mm
)))
2087 goto breakouterloop
;
2089 VM_BUG_ON(khugepaged_scan
.address
< hstart
||
2090 khugepaged_scan
.address
+ HPAGE_PMD_SIZE
>
2092 ret
= khugepaged_scan_pmd(mm
, vma
,
2093 khugepaged_scan
.address
,
2095 /* move to next address */
2096 khugepaged_scan
.address
+= HPAGE_PMD_SIZE
;
2097 progress
+= HPAGE_PMD_NR
;
2099 /* we released mmap_sem so break loop */
2100 goto breakouterloop_mmap_sem
;
2101 if (progress
>= pages
)
2102 goto breakouterloop
;
2106 up_read(&mm
->mmap_sem
); /* exit_mmap will destroy ptes after this */
2107 breakouterloop_mmap_sem
:
2109 spin_lock(&khugepaged_mm_lock
);
2110 VM_BUG_ON(khugepaged_scan
.mm_slot
!= mm_slot
);
2112 * Release the current mm_slot if this mm is about to die, or
2113 * if we scanned all vmas of this mm.
2115 if (khugepaged_test_exit(mm
) || !vma
) {
2117 * Make sure that if mm_users is reaching zero while
2118 * khugepaged runs here, khugepaged_exit will find
2119 * mm_slot not pointing to the exiting mm.
2121 if (mm_slot
->mm_node
.next
!= &khugepaged_scan
.mm_head
) {
2122 khugepaged_scan
.mm_slot
= list_entry(
2123 mm_slot
->mm_node
.next
,
2124 struct mm_slot
, mm_node
);
2125 khugepaged_scan
.address
= 0;
2127 khugepaged_scan
.mm_slot
= NULL
;
2128 khugepaged_full_scans
++;
2131 collect_mm_slot(mm_slot
);
2137 static int khugepaged_has_work(void)
2139 return !list_empty(&khugepaged_scan
.mm_head
) &&
2140 khugepaged_enabled();
2143 static int khugepaged_wait_event(void)
2145 return !list_empty(&khugepaged_scan
.mm_head
) ||
2146 !khugepaged_enabled();
2149 static void khugepaged_do_scan(struct page
**hpage
)
2151 unsigned int progress
= 0, pass_through_head
= 0;
2152 unsigned int pages
= khugepaged_pages_to_scan
;
2154 barrier(); /* write khugepaged_pages_to_scan to local stack */
2156 while (progress
< pages
) {
2161 *hpage
= alloc_hugepage(khugepaged_defrag());
2162 if (unlikely(!*hpage
)) {
2163 count_vm_event(THP_COLLAPSE_ALLOC_FAILED
);
2166 count_vm_event(THP_COLLAPSE_ALLOC
);
2173 if (unlikely(kthread_should_stop() || freezing(current
)))
2176 spin_lock(&khugepaged_mm_lock
);
2177 if (!khugepaged_scan
.mm_slot
)
2178 pass_through_head
++;
2179 if (khugepaged_has_work() &&
2180 pass_through_head
< 2)
2181 progress
+= khugepaged_scan_mm_slot(pages
- progress
,
2185 spin_unlock(&khugepaged_mm_lock
);
2189 static void khugepaged_alloc_sleep(void)
2192 add_wait_queue(&khugepaged_wait
, &wait
);
2193 schedule_timeout_interruptible(
2195 khugepaged_alloc_sleep_millisecs
));
2196 remove_wait_queue(&khugepaged_wait
, &wait
);
2200 static struct page
*khugepaged_alloc_hugepage(void)
2205 hpage
= alloc_hugepage(khugepaged_defrag());
2207 count_vm_event(THP_COLLAPSE_ALLOC_FAILED
);
2208 khugepaged_alloc_sleep();
2210 count_vm_event(THP_COLLAPSE_ALLOC
);
2211 } while (unlikely(!hpage
) &&
2212 likely(khugepaged_enabled()));
2217 static void khugepaged_loop(void)
2224 while (likely(khugepaged_enabled())) {
2226 hpage
= khugepaged_alloc_hugepage();
2227 if (unlikely(!hpage
)) {
2228 count_vm_event(THP_COLLAPSE_ALLOC_FAILED
);
2231 count_vm_event(THP_COLLAPSE_ALLOC
);
2233 if (IS_ERR(hpage
)) {
2234 khugepaged_alloc_sleep();
2239 khugepaged_do_scan(&hpage
);
2245 if (unlikely(kthread_should_stop()))
2247 if (khugepaged_has_work()) {
2249 if (!khugepaged_scan_sleep_millisecs
)
2251 add_wait_queue(&khugepaged_wait
, &wait
);
2252 schedule_timeout_interruptible(
2254 khugepaged_scan_sleep_millisecs
));
2255 remove_wait_queue(&khugepaged_wait
, &wait
);
2256 } else if (khugepaged_enabled())
2257 wait_event_freezable(khugepaged_wait
,
2258 khugepaged_wait_event());
2262 static int khugepaged(void *none
)
2264 struct mm_slot
*mm_slot
;
2267 set_user_nice(current
, 19);
2269 /* serialize with start_khugepaged() */
2270 mutex_lock(&khugepaged_mutex
);
2273 mutex_unlock(&khugepaged_mutex
);
2274 VM_BUG_ON(khugepaged_thread
!= current
);
2276 VM_BUG_ON(khugepaged_thread
!= current
);
2278 mutex_lock(&khugepaged_mutex
);
2279 if (!khugepaged_enabled())
2281 if (unlikely(kthread_should_stop()))
2285 spin_lock(&khugepaged_mm_lock
);
2286 mm_slot
= khugepaged_scan
.mm_slot
;
2287 khugepaged_scan
.mm_slot
= NULL
;
2289 collect_mm_slot(mm_slot
);
2290 spin_unlock(&khugepaged_mm_lock
);
2292 khugepaged_thread
= NULL
;
2293 mutex_unlock(&khugepaged_mutex
);
2298 void __split_huge_page_pmd(struct mm_struct
*mm
, pmd_t
*pmd
)
2302 spin_lock(&mm
->page_table_lock
);
2303 if (unlikely(!pmd_trans_huge(*pmd
))) {
2304 spin_unlock(&mm
->page_table_lock
);
2307 page
= pmd_page(*pmd
);
2308 VM_BUG_ON(!page_count(page
));
2310 spin_unlock(&mm
->page_table_lock
);
2312 split_huge_page(page
);
2315 BUG_ON(pmd_trans_huge(*pmd
));
2318 static void split_huge_page_address(struct mm_struct
*mm
,
2319 unsigned long address
)
2325 VM_BUG_ON(!(address
& ~HPAGE_PMD_MASK
));
2327 pgd
= pgd_offset(mm
, address
);
2328 if (!pgd_present(*pgd
))
2331 pud
= pud_offset(pgd
, address
);
2332 if (!pud_present(*pud
))
2335 pmd
= pmd_offset(pud
, address
);
2336 if (!pmd_present(*pmd
))
2339 * Caller holds the mmap_sem write mode, so a huge pmd cannot
2340 * materialize from under us.
2342 split_huge_page_pmd(mm
, pmd
);
2345 void __vma_adjust_trans_huge(struct vm_area_struct
*vma
,
2346 unsigned long start
,
2351 * If the new start address isn't hpage aligned and it could
2352 * previously contain an hugepage: check if we need to split
2355 if (start
& ~HPAGE_PMD_MASK
&&
2356 (start
& HPAGE_PMD_MASK
) >= vma
->vm_start
&&
2357 (start
& HPAGE_PMD_MASK
) + HPAGE_PMD_SIZE
<= vma
->vm_end
)
2358 split_huge_page_address(vma
->vm_mm
, start
);
2361 * If the new end address isn't hpage aligned and it could
2362 * previously contain an hugepage: check if we need to split
2365 if (end
& ~HPAGE_PMD_MASK
&&
2366 (end
& HPAGE_PMD_MASK
) >= vma
->vm_start
&&
2367 (end
& HPAGE_PMD_MASK
) + HPAGE_PMD_SIZE
<= vma
->vm_end
)
2368 split_huge_page_address(vma
->vm_mm
, end
);
2371 * If we're also updating the vma->vm_next->vm_start, if the new
2372 * vm_next->vm_start isn't page aligned and it could previously
2373 * contain an hugepage: check if we need to split an huge pmd.
2375 if (adjust_next
> 0) {
2376 struct vm_area_struct
*next
= vma
->vm_next
;
2377 unsigned long nstart
= next
->vm_start
;
2378 nstart
+= adjust_next
<< PAGE_SHIFT
;
2379 if (nstart
& ~HPAGE_PMD_MASK
&&
2380 (nstart
& HPAGE_PMD_MASK
) >= next
->vm_start
&&
2381 (nstart
& HPAGE_PMD_MASK
) + HPAGE_PMD_SIZE
<= next
->vm_end
)
2382 split_huge_page_address(next
->vm_mm
, nstart
);