]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - mm/migrate.c
Merge tag 'phy-for-4.6-rc' of git://git.kernel.org/pub/scm/linux/kernel/git/kishon...
[mirror_ubuntu-zesty-kernel.git] / mm / migrate.c
1 /*
2 * Memory Migration functionality - linux/mm/migrate.c
3 *
4 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
5 *
6 * Page migration was first developed in the context of the memory hotplug
7 * project. The main authors of the migration code are:
8 *
9 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
10 * Hirokazu Takahashi <taka@valinux.co.jp>
11 * Dave Hansen <haveblue@us.ibm.com>
12 * Christoph Lameter
13 */
14
15 #include <linux/migrate.h>
16 #include <linux/export.h>
17 #include <linux/swap.h>
18 #include <linux/swapops.h>
19 #include <linux/pagemap.h>
20 #include <linux/buffer_head.h>
21 #include <linux/mm_inline.h>
22 #include <linux/nsproxy.h>
23 #include <linux/pagevec.h>
24 #include <linux/ksm.h>
25 #include <linux/rmap.h>
26 #include <linux/topology.h>
27 #include <linux/cpu.h>
28 #include <linux/cpuset.h>
29 #include <linux/writeback.h>
30 #include <linux/mempolicy.h>
31 #include <linux/vmalloc.h>
32 #include <linux/security.h>
33 #include <linux/backing-dev.h>
34 #include <linux/syscalls.h>
35 #include <linux/hugetlb.h>
36 #include <linux/hugetlb_cgroup.h>
37 #include <linux/gfp.h>
38 #include <linux/balloon_compaction.h>
39 #include <linux/mmu_notifier.h>
40 #include <linux/page_idle.h>
41 #include <linux/page_owner.h>
42
43 #include <asm/tlbflush.h>
44
45 #define CREATE_TRACE_POINTS
46 #include <trace/events/migrate.h>
47
48 #include "internal.h"
49
50 /*
51 * migrate_prep() needs to be called before we start compiling a list of pages
52 * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is
53 * undesirable, use migrate_prep_local()
54 */
55 int migrate_prep(void)
56 {
57 /*
58 * Clear the LRU lists so pages can be isolated.
59 * Note that pages may be moved off the LRU after we have
60 * drained them. Those pages will fail to migrate like other
61 * pages that may be busy.
62 */
63 lru_add_drain_all();
64
65 return 0;
66 }
67
68 /* Do the necessary work of migrate_prep but not if it involves other CPUs */
69 int migrate_prep_local(void)
70 {
71 lru_add_drain();
72
73 return 0;
74 }
75
76 /*
77 * Put previously isolated pages back onto the appropriate lists
78 * from where they were once taken off for compaction/migration.
79 *
80 * This function shall be used whenever the isolated pageset has been
81 * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range()
82 * and isolate_huge_page().
83 */
84 void putback_movable_pages(struct list_head *l)
85 {
86 struct page *page;
87 struct page *page2;
88
89 list_for_each_entry_safe(page, page2, l, lru) {
90 if (unlikely(PageHuge(page))) {
91 putback_active_hugepage(page);
92 continue;
93 }
94 list_del(&page->lru);
95 dec_zone_page_state(page, NR_ISOLATED_ANON +
96 page_is_file_cache(page));
97 if (unlikely(isolated_balloon_page(page)))
98 balloon_page_putback(page);
99 else
100 putback_lru_page(page);
101 }
102 }
103
104 /*
105 * Restore a potential migration pte to a working pte entry
106 */
107 static int remove_migration_pte(struct page *new, struct vm_area_struct *vma,
108 unsigned long addr, void *old)
109 {
110 struct mm_struct *mm = vma->vm_mm;
111 swp_entry_t entry;
112 pmd_t *pmd;
113 pte_t *ptep, pte;
114 spinlock_t *ptl;
115
116 if (unlikely(PageHuge(new))) {
117 ptep = huge_pte_offset(mm, addr);
118 if (!ptep)
119 goto out;
120 ptl = huge_pte_lockptr(hstate_vma(vma), mm, ptep);
121 } else {
122 pmd = mm_find_pmd(mm, addr);
123 if (!pmd)
124 goto out;
125
126 ptep = pte_offset_map(pmd, addr);
127
128 /*
129 * Peek to check is_swap_pte() before taking ptlock? No, we
130 * can race mremap's move_ptes(), which skips anon_vma lock.
131 */
132
133 ptl = pte_lockptr(mm, pmd);
134 }
135
136 spin_lock(ptl);
137 pte = *ptep;
138 if (!is_swap_pte(pte))
139 goto unlock;
140
141 entry = pte_to_swp_entry(pte);
142
143 if (!is_migration_entry(entry) ||
144 migration_entry_to_page(entry) != old)
145 goto unlock;
146
147 get_page(new);
148 pte = pte_mkold(mk_pte(new, vma->vm_page_prot));
149 if (pte_swp_soft_dirty(*ptep))
150 pte = pte_mksoft_dirty(pte);
151
152 /* Recheck VMA as permissions can change since migration started */
153 if (is_write_migration_entry(entry))
154 pte = maybe_mkwrite(pte, vma);
155
156 #ifdef CONFIG_HUGETLB_PAGE
157 if (PageHuge(new)) {
158 pte = pte_mkhuge(pte);
159 pte = arch_make_huge_pte(pte, vma, new, 0);
160 }
161 #endif
162 flush_dcache_page(new);
163 set_pte_at(mm, addr, ptep, pte);
164
165 if (PageHuge(new)) {
166 if (PageAnon(new))
167 hugepage_add_anon_rmap(new, vma, addr);
168 else
169 page_dup_rmap(new, true);
170 } else if (PageAnon(new))
171 page_add_anon_rmap(new, vma, addr, false);
172 else
173 page_add_file_rmap(new);
174
175 if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))
176 mlock_vma_page(new);
177
178 /* No need to invalidate - it was non-present before */
179 update_mmu_cache(vma, addr, ptep);
180 unlock:
181 pte_unmap_unlock(ptep, ptl);
182 out:
183 return SWAP_AGAIN;
184 }
185
186 /*
187 * Get rid of all migration entries and replace them by
188 * references to the indicated page.
189 */
190 void remove_migration_ptes(struct page *old, struct page *new, bool locked)
191 {
192 struct rmap_walk_control rwc = {
193 .rmap_one = remove_migration_pte,
194 .arg = old,
195 };
196
197 if (locked)
198 rmap_walk_locked(new, &rwc);
199 else
200 rmap_walk(new, &rwc);
201 }
202
203 /*
204 * Something used the pte of a page under migration. We need to
205 * get to the page and wait until migration is finished.
206 * When we return from this function the fault will be retried.
207 */
208 void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
209 spinlock_t *ptl)
210 {
211 pte_t pte;
212 swp_entry_t entry;
213 struct page *page;
214
215 spin_lock(ptl);
216 pte = *ptep;
217 if (!is_swap_pte(pte))
218 goto out;
219
220 entry = pte_to_swp_entry(pte);
221 if (!is_migration_entry(entry))
222 goto out;
223
224 page = migration_entry_to_page(entry);
225
226 /*
227 * Once radix-tree replacement of page migration started, page_count
228 * *must* be zero. And, we don't want to call wait_on_page_locked()
229 * against a page without get_page().
230 * So, we use get_page_unless_zero(), here. Even failed, page fault
231 * will occur again.
232 */
233 if (!get_page_unless_zero(page))
234 goto out;
235 pte_unmap_unlock(ptep, ptl);
236 wait_on_page_locked(page);
237 put_page(page);
238 return;
239 out:
240 pte_unmap_unlock(ptep, ptl);
241 }
242
243 void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
244 unsigned long address)
245 {
246 spinlock_t *ptl = pte_lockptr(mm, pmd);
247 pte_t *ptep = pte_offset_map(pmd, address);
248 __migration_entry_wait(mm, ptep, ptl);
249 }
250
251 void migration_entry_wait_huge(struct vm_area_struct *vma,
252 struct mm_struct *mm, pte_t *pte)
253 {
254 spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte);
255 __migration_entry_wait(mm, pte, ptl);
256 }
257
258 #ifdef CONFIG_BLOCK
259 /* Returns true if all buffers are successfully locked */
260 static bool buffer_migrate_lock_buffers(struct buffer_head *head,
261 enum migrate_mode mode)
262 {
263 struct buffer_head *bh = head;
264
265 /* Simple case, sync compaction */
266 if (mode != MIGRATE_ASYNC) {
267 do {
268 get_bh(bh);
269 lock_buffer(bh);
270 bh = bh->b_this_page;
271
272 } while (bh != head);
273
274 return true;
275 }
276
277 /* async case, we cannot block on lock_buffer so use trylock_buffer */
278 do {
279 get_bh(bh);
280 if (!trylock_buffer(bh)) {
281 /*
282 * We failed to lock the buffer and cannot stall in
283 * async migration. Release the taken locks
284 */
285 struct buffer_head *failed_bh = bh;
286 put_bh(failed_bh);
287 bh = head;
288 while (bh != failed_bh) {
289 unlock_buffer(bh);
290 put_bh(bh);
291 bh = bh->b_this_page;
292 }
293 return false;
294 }
295
296 bh = bh->b_this_page;
297 } while (bh != head);
298 return true;
299 }
300 #else
301 static inline bool buffer_migrate_lock_buffers(struct buffer_head *head,
302 enum migrate_mode mode)
303 {
304 return true;
305 }
306 #endif /* CONFIG_BLOCK */
307
308 /*
309 * Replace the page in the mapping.
310 *
311 * The number of remaining references must be:
312 * 1 for anonymous pages without a mapping
313 * 2 for pages with a mapping
314 * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
315 */
316 int migrate_page_move_mapping(struct address_space *mapping,
317 struct page *newpage, struct page *page,
318 struct buffer_head *head, enum migrate_mode mode,
319 int extra_count)
320 {
321 struct zone *oldzone, *newzone;
322 int dirty;
323 int expected_count = 1 + extra_count;
324 void **pslot;
325
326 if (!mapping) {
327 /* Anonymous page without mapping */
328 if (page_count(page) != expected_count)
329 return -EAGAIN;
330
331 /* No turning back from here */
332 newpage->index = page->index;
333 newpage->mapping = page->mapping;
334 if (PageSwapBacked(page))
335 SetPageSwapBacked(newpage);
336
337 return MIGRATEPAGE_SUCCESS;
338 }
339
340 oldzone = page_zone(page);
341 newzone = page_zone(newpage);
342
343 spin_lock_irq(&mapping->tree_lock);
344
345 pslot = radix_tree_lookup_slot(&mapping->page_tree,
346 page_index(page));
347
348 expected_count += 1 + page_has_private(page);
349 if (page_count(page) != expected_count ||
350 radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) {
351 spin_unlock_irq(&mapping->tree_lock);
352 return -EAGAIN;
353 }
354
355 if (!page_ref_freeze(page, expected_count)) {
356 spin_unlock_irq(&mapping->tree_lock);
357 return -EAGAIN;
358 }
359
360 /*
361 * In the async migration case of moving a page with buffers, lock the
362 * buffers using trylock before the mapping is moved. If the mapping
363 * was moved, we later failed to lock the buffers and could not move
364 * the mapping back due to an elevated page count, we would have to
365 * block waiting on other references to be dropped.
366 */
367 if (mode == MIGRATE_ASYNC && head &&
368 !buffer_migrate_lock_buffers(head, mode)) {
369 page_ref_unfreeze(page, expected_count);
370 spin_unlock_irq(&mapping->tree_lock);
371 return -EAGAIN;
372 }
373
374 /*
375 * Now we know that no one else is looking at the page:
376 * no turning back from here.
377 */
378 newpage->index = page->index;
379 newpage->mapping = page->mapping;
380 if (PageSwapBacked(page))
381 SetPageSwapBacked(newpage);
382
383 get_page(newpage); /* add cache reference */
384 if (PageSwapCache(page)) {
385 SetPageSwapCache(newpage);
386 set_page_private(newpage, page_private(page));
387 }
388
389 /* Move dirty while page refs frozen and newpage not yet exposed */
390 dirty = PageDirty(page);
391 if (dirty) {
392 ClearPageDirty(page);
393 SetPageDirty(newpage);
394 }
395
396 radix_tree_replace_slot(pslot, newpage);
397
398 /*
399 * Drop cache reference from old page by unfreezing
400 * to one less reference.
401 * We know this isn't the last reference.
402 */
403 page_ref_unfreeze(page, expected_count - 1);
404
405 spin_unlock(&mapping->tree_lock);
406 /* Leave irq disabled to prevent preemption while updating stats */
407
408 /*
409 * If moved to a different zone then also account
410 * the page for that zone. Other VM counters will be
411 * taken care of when we establish references to the
412 * new page and drop references to the old page.
413 *
414 * Note that anonymous pages are accounted for
415 * via NR_FILE_PAGES and NR_ANON_PAGES if they
416 * are mapped to swap space.
417 */
418 if (newzone != oldzone) {
419 __dec_zone_state(oldzone, NR_FILE_PAGES);
420 __inc_zone_state(newzone, NR_FILE_PAGES);
421 if (PageSwapBacked(page) && !PageSwapCache(page)) {
422 __dec_zone_state(oldzone, NR_SHMEM);
423 __inc_zone_state(newzone, NR_SHMEM);
424 }
425 if (dirty && mapping_cap_account_dirty(mapping)) {
426 __dec_zone_state(oldzone, NR_FILE_DIRTY);
427 __inc_zone_state(newzone, NR_FILE_DIRTY);
428 }
429 }
430 local_irq_enable();
431
432 return MIGRATEPAGE_SUCCESS;
433 }
434
435 /*
436 * The expected number of remaining references is the same as that
437 * of migrate_page_move_mapping().
438 */
439 int migrate_huge_page_move_mapping(struct address_space *mapping,
440 struct page *newpage, struct page *page)
441 {
442 int expected_count;
443 void **pslot;
444
445 spin_lock_irq(&mapping->tree_lock);
446
447 pslot = radix_tree_lookup_slot(&mapping->page_tree,
448 page_index(page));
449
450 expected_count = 2 + page_has_private(page);
451 if (page_count(page) != expected_count ||
452 radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) {
453 spin_unlock_irq(&mapping->tree_lock);
454 return -EAGAIN;
455 }
456
457 if (!page_ref_freeze(page, expected_count)) {
458 spin_unlock_irq(&mapping->tree_lock);
459 return -EAGAIN;
460 }
461
462 newpage->index = page->index;
463 newpage->mapping = page->mapping;
464
465 get_page(newpage);
466
467 radix_tree_replace_slot(pslot, newpage);
468
469 page_ref_unfreeze(page, expected_count - 1);
470
471 spin_unlock_irq(&mapping->tree_lock);
472
473 return MIGRATEPAGE_SUCCESS;
474 }
475
476 /*
477 * Gigantic pages are so large that we do not guarantee that page++ pointer
478 * arithmetic will work across the entire page. We need something more
479 * specialized.
480 */
481 static void __copy_gigantic_page(struct page *dst, struct page *src,
482 int nr_pages)
483 {
484 int i;
485 struct page *dst_base = dst;
486 struct page *src_base = src;
487
488 for (i = 0; i < nr_pages; ) {
489 cond_resched();
490 copy_highpage(dst, src);
491
492 i++;
493 dst = mem_map_next(dst, dst_base, i);
494 src = mem_map_next(src, src_base, i);
495 }
496 }
497
498 static void copy_huge_page(struct page *dst, struct page *src)
499 {
500 int i;
501 int nr_pages;
502
503 if (PageHuge(src)) {
504 /* hugetlbfs page */
505 struct hstate *h = page_hstate(src);
506 nr_pages = pages_per_huge_page(h);
507
508 if (unlikely(nr_pages > MAX_ORDER_NR_PAGES)) {
509 __copy_gigantic_page(dst, src, nr_pages);
510 return;
511 }
512 } else {
513 /* thp page */
514 BUG_ON(!PageTransHuge(src));
515 nr_pages = hpage_nr_pages(src);
516 }
517
518 for (i = 0; i < nr_pages; i++) {
519 cond_resched();
520 copy_highpage(dst + i, src + i);
521 }
522 }
523
524 /*
525 * Copy the page to its new location
526 */
527 void migrate_page_copy(struct page *newpage, struct page *page)
528 {
529 int cpupid;
530
531 if (PageHuge(page) || PageTransHuge(page))
532 copy_huge_page(newpage, page);
533 else
534 copy_highpage(newpage, page);
535
536 if (PageError(page))
537 SetPageError(newpage);
538 if (PageReferenced(page))
539 SetPageReferenced(newpage);
540 if (PageUptodate(page))
541 SetPageUptodate(newpage);
542 if (TestClearPageActive(page)) {
543 VM_BUG_ON_PAGE(PageUnevictable(page), page);
544 SetPageActive(newpage);
545 } else if (TestClearPageUnevictable(page))
546 SetPageUnevictable(newpage);
547 if (PageChecked(page))
548 SetPageChecked(newpage);
549 if (PageMappedToDisk(page))
550 SetPageMappedToDisk(newpage);
551
552 /* Move dirty on pages not done by migrate_page_move_mapping() */
553 if (PageDirty(page))
554 SetPageDirty(newpage);
555
556 if (page_is_young(page))
557 set_page_young(newpage);
558 if (page_is_idle(page))
559 set_page_idle(newpage);
560
561 /*
562 * Copy NUMA information to the new page, to prevent over-eager
563 * future migrations of this same page.
564 */
565 cpupid = page_cpupid_xchg_last(page, -1);
566 page_cpupid_xchg_last(newpage, cpupid);
567
568 ksm_migrate_page(newpage, page);
569 /*
570 * Please do not reorder this without considering how mm/ksm.c's
571 * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache().
572 */
573 if (PageSwapCache(page))
574 ClearPageSwapCache(page);
575 ClearPagePrivate(page);
576 set_page_private(page, 0);
577
578 /*
579 * If any waiters have accumulated on the new page then
580 * wake them up.
581 */
582 if (PageWriteback(newpage))
583 end_page_writeback(newpage);
584
585 copy_page_owner(page, newpage);
586
587 mem_cgroup_migrate(page, newpage);
588 }
589
590 /************************************************************
591 * Migration functions
592 ***********************************************************/
593
594 /*
595 * Common logic to directly migrate a single page suitable for
596 * pages that do not use PagePrivate/PagePrivate2.
597 *
598 * Pages are locked upon entry and exit.
599 */
600 int migrate_page(struct address_space *mapping,
601 struct page *newpage, struct page *page,
602 enum migrate_mode mode)
603 {
604 int rc;
605
606 BUG_ON(PageWriteback(page)); /* Writeback must be complete */
607
608 rc = migrate_page_move_mapping(mapping, newpage, page, NULL, mode, 0);
609
610 if (rc != MIGRATEPAGE_SUCCESS)
611 return rc;
612
613 migrate_page_copy(newpage, page);
614 return MIGRATEPAGE_SUCCESS;
615 }
616 EXPORT_SYMBOL(migrate_page);
617
618 #ifdef CONFIG_BLOCK
619 /*
620 * Migration function for pages with buffers. This function can only be used
621 * if the underlying filesystem guarantees that no other references to "page"
622 * exist.
623 */
624 int buffer_migrate_page(struct address_space *mapping,
625 struct page *newpage, struct page *page, enum migrate_mode mode)
626 {
627 struct buffer_head *bh, *head;
628 int rc;
629
630 if (!page_has_buffers(page))
631 return migrate_page(mapping, newpage, page, mode);
632
633 head = page_buffers(page);
634
635 rc = migrate_page_move_mapping(mapping, newpage, page, head, mode, 0);
636
637 if (rc != MIGRATEPAGE_SUCCESS)
638 return rc;
639
640 /*
641 * In the async case, migrate_page_move_mapping locked the buffers
642 * with an IRQ-safe spinlock held. In the sync case, the buffers
643 * need to be locked now
644 */
645 if (mode != MIGRATE_ASYNC)
646 BUG_ON(!buffer_migrate_lock_buffers(head, mode));
647
648 ClearPagePrivate(page);
649 set_page_private(newpage, page_private(page));
650 set_page_private(page, 0);
651 put_page(page);
652 get_page(newpage);
653
654 bh = head;
655 do {
656 set_bh_page(bh, newpage, bh_offset(bh));
657 bh = bh->b_this_page;
658
659 } while (bh != head);
660
661 SetPagePrivate(newpage);
662
663 migrate_page_copy(newpage, page);
664
665 bh = head;
666 do {
667 unlock_buffer(bh);
668 put_bh(bh);
669 bh = bh->b_this_page;
670
671 } while (bh != head);
672
673 return MIGRATEPAGE_SUCCESS;
674 }
675 EXPORT_SYMBOL(buffer_migrate_page);
676 #endif
677
678 /*
679 * Writeback a page to clean the dirty state
680 */
681 static int writeout(struct address_space *mapping, struct page *page)
682 {
683 struct writeback_control wbc = {
684 .sync_mode = WB_SYNC_NONE,
685 .nr_to_write = 1,
686 .range_start = 0,
687 .range_end = LLONG_MAX,
688 .for_reclaim = 1
689 };
690 int rc;
691
692 if (!mapping->a_ops->writepage)
693 /* No write method for the address space */
694 return -EINVAL;
695
696 if (!clear_page_dirty_for_io(page))
697 /* Someone else already triggered a write */
698 return -EAGAIN;
699
700 /*
701 * A dirty page may imply that the underlying filesystem has
702 * the page on some queue. So the page must be clean for
703 * migration. Writeout may mean we loose the lock and the
704 * page state is no longer what we checked for earlier.
705 * At this point we know that the migration attempt cannot
706 * be successful.
707 */
708 remove_migration_ptes(page, page, false);
709
710 rc = mapping->a_ops->writepage(page, &wbc);
711
712 if (rc != AOP_WRITEPAGE_ACTIVATE)
713 /* unlocked. Relock */
714 lock_page(page);
715
716 return (rc < 0) ? -EIO : -EAGAIN;
717 }
718
719 /*
720 * Default handling if a filesystem does not provide a migration function.
721 */
722 static int fallback_migrate_page(struct address_space *mapping,
723 struct page *newpage, struct page *page, enum migrate_mode mode)
724 {
725 if (PageDirty(page)) {
726 /* Only writeback pages in full synchronous migration */
727 if (mode != MIGRATE_SYNC)
728 return -EBUSY;
729 return writeout(mapping, page);
730 }
731
732 /*
733 * Buffers may be managed in a filesystem specific way.
734 * We must have no buffers or drop them.
735 */
736 if (page_has_private(page) &&
737 !try_to_release_page(page, GFP_KERNEL))
738 return -EAGAIN;
739
740 return migrate_page(mapping, newpage, page, mode);
741 }
742
743 /*
744 * Move a page to a newly allocated page
745 * The page is locked and all ptes have been successfully removed.
746 *
747 * The new page will have replaced the old page if this function
748 * is successful.
749 *
750 * Return value:
751 * < 0 - error code
752 * MIGRATEPAGE_SUCCESS - success
753 */
754 static int move_to_new_page(struct page *newpage, struct page *page,
755 enum migrate_mode mode)
756 {
757 struct address_space *mapping;
758 int rc;
759
760 VM_BUG_ON_PAGE(!PageLocked(page), page);
761 VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
762
763 mapping = page_mapping(page);
764 if (!mapping)
765 rc = migrate_page(mapping, newpage, page, mode);
766 else if (mapping->a_ops->migratepage)
767 /*
768 * Most pages have a mapping and most filesystems provide a
769 * migratepage callback. Anonymous pages are part of swap
770 * space which also has its own migratepage callback. This
771 * is the most common path for page migration.
772 */
773 rc = mapping->a_ops->migratepage(mapping, newpage, page, mode);
774 else
775 rc = fallback_migrate_page(mapping, newpage, page, mode);
776
777 /*
778 * When successful, old pagecache page->mapping must be cleared before
779 * page is freed; but stats require that PageAnon be left as PageAnon.
780 */
781 if (rc == MIGRATEPAGE_SUCCESS) {
782 if (!PageAnon(page))
783 page->mapping = NULL;
784 }
785 return rc;
786 }
787
788 static int __unmap_and_move(struct page *page, struct page *newpage,
789 int force, enum migrate_mode mode)
790 {
791 int rc = -EAGAIN;
792 int page_was_mapped = 0;
793 struct anon_vma *anon_vma = NULL;
794
795 if (!trylock_page(page)) {
796 if (!force || mode == MIGRATE_ASYNC)
797 goto out;
798
799 /*
800 * It's not safe for direct compaction to call lock_page.
801 * For example, during page readahead pages are added locked
802 * to the LRU. Later, when the IO completes the pages are
803 * marked uptodate and unlocked. However, the queueing
804 * could be merging multiple pages for one bio (e.g.
805 * mpage_readpages). If an allocation happens for the
806 * second or third page, the process can end up locking
807 * the same page twice and deadlocking. Rather than
808 * trying to be clever about what pages can be locked,
809 * avoid the use of lock_page for direct compaction
810 * altogether.
811 */
812 if (current->flags & PF_MEMALLOC)
813 goto out;
814
815 lock_page(page);
816 }
817
818 if (PageWriteback(page)) {
819 /*
820 * Only in the case of a full synchronous migration is it
821 * necessary to wait for PageWriteback. In the async case,
822 * the retry loop is too short and in the sync-light case,
823 * the overhead of stalling is too much
824 */
825 if (mode != MIGRATE_SYNC) {
826 rc = -EBUSY;
827 goto out_unlock;
828 }
829 if (!force)
830 goto out_unlock;
831 wait_on_page_writeback(page);
832 }
833
834 /*
835 * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
836 * we cannot notice that anon_vma is freed while we migrates a page.
837 * This get_anon_vma() delays freeing anon_vma pointer until the end
838 * of migration. File cache pages are no problem because of page_lock()
839 * File Caches may use write_page() or lock_page() in migration, then,
840 * just care Anon page here.
841 *
842 * Only page_get_anon_vma() understands the subtleties of
843 * getting a hold on an anon_vma from outside one of its mms.
844 * But if we cannot get anon_vma, then we won't need it anyway,
845 * because that implies that the anon page is no longer mapped
846 * (and cannot be remapped so long as we hold the page lock).
847 */
848 if (PageAnon(page) && !PageKsm(page))
849 anon_vma = page_get_anon_vma(page);
850
851 /*
852 * Block others from accessing the new page when we get around to
853 * establishing additional references. We are usually the only one
854 * holding a reference to newpage at this point. We used to have a BUG
855 * here if trylock_page(newpage) fails, but would like to allow for
856 * cases where there might be a race with the previous use of newpage.
857 * This is much like races on refcount of oldpage: just don't BUG().
858 */
859 if (unlikely(!trylock_page(newpage)))
860 goto out_unlock;
861
862 if (unlikely(isolated_balloon_page(page))) {
863 /*
864 * A ballooned page does not need any special attention from
865 * physical to virtual reverse mapping procedures.
866 * Skip any attempt to unmap PTEs or to remap swap cache,
867 * in order to avoid burning cycles at rmap level, and perform
868 * the page migration right away (proteced by page lock).
869 */
870 rc = balloon_page_migrate(newpage, page, mode);
871 goto out_unlock_both;
872 }
873
874 /*
875 * Corner case handling:
876 * 1. When a new swap-cache page is read into, it is added to the LRU
877 * and treated as swapcache but it has no rmap yet.
878 * Calling try_to_unmap() against a page->mapping==NULL page will
879 * trigger a BUG. So handle it here.
880 * 2. An orphaned page (see truncate_complete_page) might have
881 * fs-private metadata. The page can be picked up due to memory
882 * offlining. Everywhere else except page reclaim, the page is
883 * invisible to the vm, so the page can not be migrated. So try to
884 * free the metadata, so the page can be freed.
885 */
886 if (!page->mapping) {
887 VM_BUG_ON_PAGE(PageAnon(page), page);
888 if (page_has_private(page)) {
889 try_to_free_buffers(page);
890 goto out_unlock_both;
891 }
892 } else if (page_mapped(page)) {
893 /* Establish migration ptes */
894 VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma,
895 page);
896 try_to_unmap(page,
897 TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
898 page_was_mapped = 1;
899 }
900
901 if (!page_mapped(page))
902 rc = move_to_new_page(newpage, page, mode);
903
904 if (page_was_mapped)
905 remove_migration_ptes(page,
906 rc == MIGRATEPAGE_SUCCESS ? newpage : page, false);
907
908 out_unlock_both:
909 unlock_page(newpage);
910 out_unlock:
911 /* Drop an anon_vma reference if we took one */
912 if (anon_vma)
913 put_anon_vma(anon_vma);
914 unlock_page(page);
915 out:
916 return rc;
917 }
918
919 /*
920 * gcc 4.7 and 4.8 on arm get an ICEs when inlining unmap_and_move(). Work
921 * around it.
922 */
923 #if (GCC_VERSION >= 40700 && GCC_VERSION < 40900) && defined(CONFIG_ARM)
924 #define ICE_noinline noinline
925 #else
926 #define ICE_noinline
927 #endif
928
929 /*
930 * Obtain the lock on page, remove all ptes and migrate the page
931 * to the newly allocated page in newpage.
932 */
933 static ICE_noinline int unmap_and_move(new_page_t get_new_page,
934 free_page_t put_new_page,
935 unsigned long private, struct page *page,
936 int force, enum migrate_mode mode,
937 enum migrate_reason reason)
938 {
939 int rc = MIGRATEPAGE_SUCCESS;
940 int *result = NULL;
941 struct page *newpage;
942
943 newpage = get_new_page(page, private, &result);
944 if (!newpage)
945 return -ENOMEM;
946
947 if (page_count(page) == 1) {
948 /* page was freed from under us. So we are done. */
949 goto out;
950 }
951
952 if (unlikely(PageTransHuge(page))) {
953 lock_page(page);
954 rc = split_huge_page(page);
955 unlock_page(page);
956 if (rc)
957 goto out;
958 }
959
960 rc = __unmap_and_move(page, newpage, force, mode);
961 if (rc == MIGRATEPAGE_SUCCESS) {
962 put_new_page = NULL;
963 set_page_owner_migrate_reason(newpage, reason);
964 }
965
966 out:
967 if (rc != -EAGAIN) {
968 /*
969 * A page that has been migrated has all references
970 * removed and will be freed. A page that has not been
971 * migrated will have kepts its references and be
972 * restored.
973 */
974 list_del(&page->lru);
975 dec_zone_page_state(page, NR_ISOLATED_ANON +
976 page_is_file_cache(page));
977 /* Soft-offlined page shouldn't go through lru cache list */
978 if (reason == MR_MEMORY_FAILURE) {
979 put_page(page);
980 if (!test_set_page_hwpoison(page))
981 num_poisoned_pages_inc();
982 } else
983 putback_lru_page(page);
984 }
985
986 /*
987 * If migration was not successful and there's a freeing callback, use
988 * it. Otherwise, putback_lru_page() will drop the reference grabbed
989 * during isolation.
990 */
991 if (put_new_page)
992 put_new_page(newpage, private);
993 else if (unlikely(__is_movable_balloon_page(newpage))) {
994 /* drop our reference, page already in the balloon */
995 put_page(newpage);
996 } else
997 putback_lru_page(newpage);
998
999 if (result) {
1000 if (rc)
1001 *result = rc;
1002 else
1003 *result = page_to_nid(newpage);
1004 }
1005 return rc;
1006 }
1007
1008 /*
1009 * Counterpart of unmap_and_move_page() for hugepage migration.
1010 *
1011 * This function doesn't wait the completion of hugepage I/O
1012 * because there is no race between I/O and migration for hugepage.
1013 * Note that currently hugepage I/O occurs only in direct I/O
1014 * where no lock is held and PG_writeback is irrelevant,
1015 * and writeback status of all subpages are counted in the reference
1016 * count of the head page (i.e. if all subpages of a 2MB hugepage are
1017 * under direct I/O, the reference of the head page is 512 and a bit more.)
1018 * This means that when we try to migrate hugepage whose subpages are
1019 * doing direct I/O, some references remain after try_to_unmap() and
1020 * hugepage migration fails without data corruption.
1021 *
1022 * There is also no race when direct I/O is issued on the page under migration,
1023 * because then pte is replaced with migration swap entry and direct I/O code
1024 * will wait in the page fault for migration to complete.
1025 */
1026 static int unmap_and_move_huge_page(new_page_t get_new_page,
1027 free_page_t put_new_page, unsigned long private,
1028 struct page *hpage, int force,
1029 enum migrate_mode mode, int reason)
1030 {
1031 int rc = -EAGAIN;
1032 int *result = NULL;
1033 int page_was_mapped = 0;
1034 struct page *new_hpage;
1035 struct anon_vma *anon_vma = NULL;
1036
1037 /*
1038 * Movability of hugepages depends on architectures and hugepage size.
1039 * This check is necessary because some callers of hugepage migration
1040 * like soft offline and memory hotremove don't walk through page
1041 * tables or check whether the hugepage is pmd-based or not before
1042 * kicking migration.
1043 */
1044 if (!hugepage_migration_supported(page_hstate(hpage))) {
1045 putback_active_hugepage(hpage);
1046 return -ENOSYS;
1047 }
1048
1049 new_hpage = get_new_page(hpage, private, &result);
1050 if (!new_hpage)
1051 return -ENOMEM;
1052
1053 if (!trylock_page(hpage)) {
1054 if (!force || mode != MIGRATE_SYNC)
1055 goto out;
1056 lock_page(hpage);
1057 }
1058
1059 if (PageAnon(hpage))
1060 anon_vma = page_get_anon_vma(hpage);
1061
1062 if (unlikely(!trylock_page(new_hpage)))
1063 goto put_anon;
1064
1065 if (page_mapped(hpage)) {
1066 try_to_unmap(hpage,
1067 TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
1068 page_was_mapped = 1;
1069 }
1070
1071 if (!page_mapped(hpage))
1072 rc = move_to_new_page(new_hpage, hpage, mode);
1073
1074 if (page_was_mapped)
1075 remove_migration_ptes(hpage,
1076 rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, false);
1077
1078 unlock_page(new_hpage);
1079
1080 put_anon:
1081 if (anon_vma)
1082 put_anon_vma(anon_vma);
1083
1084 if (rc == MIGRATEPAGE_SUCCESS) {
1085 hugetlb_cgroup_migrate(hpage, new_hpage);
1086 put_new_page = NULL;
1087 set_page_owner_migrate_reason(new_hpage, reason);
1088 }
1089
1090 unlock_page(hpage);
1091 out:
1092 if (rc != -EAGAIN)
1093 putback_active_hugepage(hpage);
1094
1095 /*
1096 * If migration was not successful and there's a freeing callback, use
1097 * it. Otherwise, put_page() will drop the reference grabbed during
1098 * isolation.
1099 */
1100 if (put_new_page)
1101 put_new_page(new_hpage, private);
1102 else
1103 putback_active_hugepage(new_hpage);
1104
1105 if (result) {
1106 if (rc)
1107 *result = rc;
1108 else
1109 *result = page_to_nid(new_hpage);
1110 }
1111 return rc;
1112 }
1113
1114 /*
1115 * migrate_pages - migrate the pages specified in a list, to the free pages
1116 * supplied as the target for the page migration
1117 *
1118 * @from: The list of pages to be migrated.
1119 * @get_new_page: The function used to allocate free pages to be used
1120 * as the target of the page migration.
1121 * @put_new_page: The function used to free target pages if migration
1122 * fails, or NULL if no special handling is necessary.
1123 * @private: Private data to be passed on to get_new_page()
1124 * @mode: The migration mode that specifies the constraints for
1125 * page migration, if any.
1126 * @reason: The reason for page migration.
1127 *
1128 * The function returns after 10 attempts or if no pages are movable any more
1129 * because the list has become empty or no retryable pages exist any more.
1130 * The caller should call putback_movable_pages() to return pages to the LRU
1131 * or free list only if ret != 0.
1132 *
1133 * Returns the number of pages that were not migrated, or an error code.
1134 */
1135 int migrate_pages(struct list_head *from, new_page_t get_new_page,
1136 free_page_t put_new_page, unsigned long private,
1137 enum migrate_mode mode, int reason)
1138 {
1139 int retry = 1;
1140 int nr_failed = 0;
1141 int nr_succeeded = 0;
1142 int pass = 0;
1143 struct page *page;
1144 struct page *page2;
1145 int swapwrite = current->flags & PF_SWAPWRITE;
1146 int rc;
1147
1148 if (!swapwrite)
1149 current->flags |= PF_SWAPWRITE;
1150
1151 for(pass = 0; pass < 10 && retry; pass++) {
1152 retry = 0;
1153
1154 list_for_each_entry_safe(page, page2, from, lru) {
1155 cond_resched();
1156
1157 if (PageHuge(page))
1158 rc = unmap_and_move_huge_page(get_new_page,
1159 put_new_page, private, page,
1160 pass > 2, mode, reason);
1161 else
1162 rc = unmap_and_move(get_new_page, put_new_page,
1163 private, page, pass > 2, mode,
1164 reason);
1165
1166 switch(rc) {
1167 case -ENOMEM:
1168 goto out;
1169 case -EAGAIN:
1170 retry++;
1171 break;
1172 case MIGRATEPAGE_SUCCESS:
1173 nr_succeeded++;
1174 break;
1175 default:
1176 /*
1177 * Permanent failure (-EBUSY, -ENOSYS, etc.):
1178 * unlike -EAGAIN case, the failed page is
1179 * removed from migration page list and not
1180 * retried in the next outer loop.
1181 */
1182 nr_failed++;
1183 break;
1184 }
1185 }
1186 }
1187 nr_failed += retry;
1188 rc = nr_failed;
1189 out:
1190 if (nr_succeeded)
1191 count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
1192 if (nr_failed)
1193 count_vm_events(PGMIGRATE_FAIL, nr_failed);
1194 trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason);
1195
1196 if (!swapwrite)
1197 current->flags &= ~PF_SWAPWRITE;
1198
1199 return rc;
1200 }
1201
1202 #ifdef CONFIG_NUMA
1203 /*
1204 * Move a list of individual pages
1205 */
1206 struct page_to_node {
1207 unsigned long addr;
1208 struct page *page;
1209 int node;
1210 int status;
1211 };
1212
1213 static struct page *new_page_node(struct page *p, unsigned long private,
1214 int **result)
1215 {
1216 struct page_to_node *pm = (struct page_to_node *)private;
1217
1218 while (pm->node != MAX_NUMNODES && pm->page != p)
1219 pm++;
1220
1221 if (pm->node == MAX_NUMNODES)
1222 return NULL;
1223
1224 *result = &pm->status;
1225
1226 if (PageHuge(p))
1227 return alloc_huge_page_node(page_hstate(compound_head(p)),
1228 pm->node);
1229 else
1230 return __alloc_pages_node(pm->node,
1231 GFP_HIGHUSER_MOVABLE | __GFP_THISNODE, 0);
1232 }
1233
1234 /*
1235 * Move a set of pages as indicated in the pm array. The addr
1236 * field must be set to the virtual address of the page to be moved
1237 * and the node number must contain a valid target node.
1238 * The pm array ends with node = MAX_NUMNODES.
1239 */
1240 static int do_move_page_to_node_array(struct mm_struct *mm,
1241 struct page_to_node *pm,
1242 int migrate_all)
1243 {
1244 int err;
1245 struct page_to_node *pp;
1246 LIST_HEAD(pagelist);
1247
1248 down_read(&mm->mmap_sem);
1249
1250 /*
1251 * Build a list of pages to migrate
1252 */
1253 for (pp = pm; pp->node != MAX_NUMNODES; pp++) {
1254 struct vm_area_struct *vma;
1255 struct page *page;
1256
1257 err = -EFAULT;
1258 vma = find_vma(mm, pp->addr);
1259 if (!vma || pp->addr < vma->vm_start || !vma_migratable(vma))
1260 goto set_status;
1261
1262 /* FOLL_DUMP to ignore special (like zero) pages */
1263 page = follow_page(vma, pp->addr,
1264 FOLL_GET | FOLL_SPLIT | FOLL_DUMP);
1265
1266 err = PTR_ERR(page);
1267 if (IS_ERR(page))
1268 goto set_status;
1269
1270 err = -ENOENT;
1271 if (!page)
1272 goto set_status;
1273
1274 pp->page = page;
1275 err = page_to_nid(page);
1276
1277 if (err == pp->node)
1278 /*
1279 * Node already in the right place
1280 */
1281 goto put_and_set;
1282
1283 err = -EACCES;
1284 if (page_mapcount(page) > 1 &&
1285 !migrate_all)
1286 goto put_and_set;
1287
1288 if (PageHuge(page)) {
1289 if (PageHead(page))
1290 isolate_huge_page(page, &pagelist);
1291 goto put_and_set;
1292 }
1293
1294 err = isolate_lru_page(page);
1295 if (!err) {
1296 list_add_tail(&page->lru, &pagelist);
1297 inc_zone_page_state(page, NR_ISOLATED_ANON +
1298 page_is_file_cache(page));
1299 }
1300 put_and_set:
1301 /*
1302 * Either remove the duplicate refcount from
1303 * isolate_lru_page() or drop the page ref if it was
1304 * not isolated.
1305 */
1306 put_page(page);
1307 set_status:
1308 pp->status = err;
1309 }
1310
1311 err = 0;
1312 if (!list_empty(&pagelist)) {
1313 err = migrate_pages(&pagelist, new_page_node, NULL,
1314 (unsigned long)pm, MIGRATE_SYNC, MR_SYSCALL);
1315 if (err)
1316 putback_movable_pages(&pagelist);
1317 }
1318
1319 up_read(&mm->mmap_sem);
1320 return err;
1321 }
1322
1323 /*
1324 * Migrate an array of page address onto an array of nodes and fill
1325 * the corresponding array of status.
1326 */
1327 static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
1328 unsigned long nr_pages,
1329 const void __user * __user *pages,
1330 const int __user *nodes,
1331 int __user *status, int flags)
1332 {
1333 struct page_to_node *pm;
1334 unsigned long chunk_nr_pages;
1335 unsigned long chunk_start;
1336 int err;
1337
1338 err = -ENOMEM;
1339 pm = (struct page_to_node *)__get_free_page(GFP_KERNEL);
1340 if (!pm)
1341 goto out;
1342
1343 migrate_prep();
1344
1345 /*
1346 * Store a chunk of page_to_node array in a page,
1347 * but keep the last one as a marker
1348 */
1349 chunk_nr_pages = (PAGE_SIZE / sizeof(struct page_to_node)) - 1;
1350
1351 for (chunk_start = 0;
1352 chunk_start < nr_pages;
1353 chunk_start += chunk_nr_pages) {
1354 int j;
1355
1356 if (chunk_start + chunk_nr_pages > nr_pages)
1357 chunk_nr_pages = nr_pages - chunk_start;
1358
1359 /* fill the chunk pm with addrs and nodes from user-space */
1360 for (j = 0; j < chunk_nr_pages; j++) {
1361 const void __user *p;
1362 int node;
1363
1364 err = -EFAULT;
1365 if (get_user(p, pages + j + chunk_start))
1366 goto out_pm;
1367 pm[j].addr = (unsigned long) p;
1368
1369 if (get_user(node, nodes + j + chunk_start))
1370 goto out_pm;
1371
1372 err = -ENODEV;
1373 if (node < 0 || node >= MAX_NUMNODES)
1374 goto out_pm;
1375
1376 if (!node_state(node, N_MEMORY))
1377 goto out_pm;
1378
1379 err = -EACCES;
1380 if (!node_isset(node, task_nodes))
1381 goto out_pm;
1382
1383 pm[j].node = node;
1384 }
1385
1386 /* End marker for this chunk */
1387 pm[chunk_nr_pages].node = MAX_NUMNODES;
1388
1389 /* Migrate this chunk */
1390 err = do_move_page_to_node_array(mm, pm,
1391 flags & MPOL_MF_MOVE_ALL);
1392 if (err < 0)
1393 goto out_pm;
1394
1395 /* Return status information */
1396 for (j = 0; j < chunk_nr_pages; j++)
1397 if (put_user(pm[j].status, status + j + chunk_start)) {
1398 err = -EFAULT;
1399 goto out_pm;
1400 }
1401 }
1402 err = 0;
1403
1404 out_pm:
1405 free_page((unsigned long)pm);
1406 out:
1407 return err;
1408 }
1409
1410 /*
1411 * Determine the nodes of an array of pages and store it in an array of status.
1412 */
1413 static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
1414 const void __user **pages, int *status)
1415 {
1416 unsigned long i;
1417
1418 down_read(&mm->mmap_sem);
1419
1420 for (i = 0; i < nr_pages; i++) {
1421 unsigned long addr = (unsigned long)(*pages);
1422 struct vm_area_struct *vma;
1423 struct page *page;
1424 int err = -EFAULT;
1425
1426 vma = find_vma(mm, addr);
1427 if (!vma || addr < vma->vm_start)
1428 goto set_status;
1429
1430 /* FOLL_DUMP to ignore special (like zero) pages */
1431 page = follow_page(vma, addr, FOLL_DUMP);
1432
1433 err = PTR_ERR(page);
1434 if (IS_ERR(page))
1435 goto set_status;
1436
1437 err = page ? page_to_nid(page) : -ENOENT;
1438 set_status:
1439 *status = err;
1440
1441 pages++;
1442 status++;
1443 }
1444
1445 up_read(&mm->mmap_sem);
1446 }
1447
1448 /*
1449 * Determine the nodes of a user array of pages and store it in
1450 * a user array of status.
1451 */
1452 static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
1453 const void __user * __user *pages,
1454 int __user *status)
1455 {
1456 #define DO_PAGES_STAT_CHUNK_NR 16
1457 const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
1458 int chunk_status[DO_PAGES_STAT_CHUNK_NR];
1459
1460 while (nr_pages) {
1461 unsigned long chunk_nr;
1462
1463 chunk_nr = nr_pages;
1464 if (chunk_nr > DO_PAGES_STAT_CHUNK_NR)
1465 chunk_nr = DO_PAGES_STAT_CHUNK_NR;
1466
1467 if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages)))
1468 break;
1469
1470 do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);
1471
1472 if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
1473 break;
1474
1475 pages += chunk_nr;
1476 status += chunk_nr;
1477 nr_pages -= chunk_nr;
1478 }
1479 return nr_pages ? -EFAULT : 0;
1480 }
1481
1482 /*
1483 * Move a list of pages in the address space of the currently executing
1484 * process.
1485 */
1486 SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
1487 const void __user * __user *, pages,
1488 const int __user *, nodes,
1489 int __user *, status, int, flags)
1490 {
1491 const struct cred *cred = current_cred(), *tcred;
1492 struct task_struct *task;
1493 struct mm_struct *mm;
1494 int err;
1495 nodemask_t task_nodes;
1496
1497 /* Check flags */
1498 if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
1499 return -EINVAL;
1500
1501 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
1502 return -EPERM;
1503
1504 /* Find the mm_struct */
1505 rcu_read_lock();
1506 task = pid ? find_task_by_vpid(pid) : current;
1507 if (!task) {
1508 rcu_read_unlock();
1509 return -ESRCH;
1510 }
1511 get_task_struct(task);
1512
1513 /*
1514 * Check if this process has the right to modify the specified
1515 * process. The right exists if the process has administrative
1516 * capabilities, superuser privileges or the same
1517 * userid as the target process.
1518 */
1519 tcred = __task_cred(task);
1520 if (!uid_eq(cred->euid, tcred->suid) && !uid_eq(cred->euid, tcred->uid) &&
1521 !uid_eq(cred->uid, tcred->suid) && !uid_eq(cred->uid, tcred->uid) &&
1522 !capable(CAP_SYS_NICE)) {
1523 rcu_read_unlock();
1524 err = -EPERM;
1525 goto out;
1526 }
1527 rcu_read_unlock();
1528
1529 err = security_task_movememory(task);
1530 if (err)
1531 goto out;
1532
1533 task_nodes = cpuset_mems_allowed(task);
1534 mm = get_task_mm(task);
1535 put_task_struct(task);
1536
1537 if (!mm)
1538 return -EINVAL;
1539
1540 if (nodes)
1541 err = do_pages_move(mm, task_nodes, nr_pages, pages,
1542 nodes, status, flags);
1543 else
1544 err = do_pages_stat(mm, nr_pages, pages, status);
1545
1546 mmput(mm);
1547 return err;
1548
1549 out:
1550 put_task_struct(task);
1551 return err;
1552 }
1553
1554 #ifdef CONFIG_NUMA_BALANCING
1555 /*
1556 * Returns true if this is a safe migration target node for misplaced NUMA
1557 * pages. Currently it only checks the watermarks which crude
1558 */
1559 static bool migrate_balanced_pgdat(struct pglist_data *pgdat,
1560 unsigned long nr_migrate_pages)
1561 {
1562 int z;
1563 for (z = pgdat->nr_zones - 1; z >= 0; z--) {
1564 struct zone *zone = pgdat->node_zones + z;
1565
1566 if (!populated_zone(zone))
1567 continue;
1568
1569 if (!zone_reclaimable(zone))
1570 continue;
1571
1572 /* Avoid waking kswapd by allocating pages_to_migrate pages. */
1573 if (!zone_watermark_ok(zone, 0,
1574 high_wmark_pages(zone) +
1575 nr_migrate_pages,
1576 0, 0))
1577 continue;
1578 return true;
1579 }
1580 return false;
1581 }
1582
1583 static struct page *alloc_misplaced_dst_page(struct page *page,
1584 unsigned long data,
1585 int **result)
1586 {
1587 int nid = (int) data;
1588 struct page *newpage;
1589
1590 newpage = __alloc_pages_node(nid,
1591 (GFP_HIGHUSER_MOVABLE |
1592 __GFP_THISNODE | __GFP_NOMEMALLOC |
1593 __GFP_NORETRY | __GFP_NOWARN) &
1594 ~__GFP_RECLAIM, 0);
1595
1596 return newpage;
1597 }
1598
1599 /*
1600 * page migration rate limiting control.
1601 * Do not migrate more than @pages_to_migrate in a @migrate_interval_millisecs
1602 * window of time. Default here says do not migrate more than 1280M per second.
1603 */
1604 static unsigned int migrate_interval_millisecs __read_mostly = 100;
1605 static unsigned int ratelimit_pages __read_mostly = 128 << (20 - PAGE_SHIFT);
1606
1607 /* Returns true if the node is migrate rate-limited after the update */
1608 static bool numamigrate_update_ratelimit(pg_data_t *pgdat,
1609 unsigned long nr_pages)
1610 {
1611 /*
1612 * Rate-limit the amount of data that is being migrated to a node.
1613 * Optimal placement is no good if the memory bus is saturated and
1614 * all the time is being spent migrating!
1615 */
1616 if (time_after(jiffies, pgdat->numabalancing_migrate_next_window)) {
1617 spin_lock(&pgdat->numabalancing_migrate_lock);
1618 pgdat->numabalancing_migrate_nr_pages = 0;
1619 pgdat->numabalancing_migrate_next_window = jiffies +
1620 msecs_to_jiffies(migrate_interval_millisecs);
1621 spin_unlock(&pgdat->numabalancing_migrate_lock);
1622 }
1623 if (pgdat->numabalancing_migrate_nr_pages > ratelimit_pages) {
1624 trace_mm_numa_migrate_ratelimit(current, pgdat->node_id,
1625 nr_pages);
1626 return true;
1627 }
1628
1629 /*
1630 * This is an unlocked non-atomic update so errors are possible.
1631 * The consequences are failing to migrate when we potentiall should
1632 * have which is not severe enough to warrant locking. If it is ever
1633 * a problem, it can be converted to a per-cpu counter.
1634 */
1635 pgdat->numabalancing_migrate_nr_pages += nr_pages;
1636 return false;
1637 }
1638
1639 static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
1640 {
1641 int page_lru;
1642
1643 VM_BUG_ON_PAGE(compound_order(page) && !PageTransHuge(page), page);
1644
1645 /* Avoid migrating to a node that is nearly full */
1646 if (!migrate_balanced_pgdat(pgdat, 1UL << compound_order(page)))
1647 return 0;
1648
1649 if (isolate_lru_page(page))
1650 return 0;
1651
1652 /*
1653 * migrate_misplaced_transhuge_page() skips page migration's usual
1654 * check on page_count(), so we must do it here, now that the page
1655 * has been isolated: a GUP pin, or any other pin, prevents migration.
1656 * The expected page count is 3: 1 for page's mapcount and 1 for the
1657 * caller's pin and 1 for the reference taken by isolate_lru_page().
1658 */
1659 if (PageTransHuge(page) && page_count(page) != 3) {
1660 putback_lru_page(page);
1661 return 0;
1662 }
1663
1664 page_lru = page_is_file_cache(page);
1665 mod_zone_page_state(page_zone(page), NR_ISOLATED_ANON + page_lru,
1666 hpage_nr_pages(page));
1667
1668 /*
1669 * Isolating the page has taken another reference, so the
1670 * caller's reference can be safely dropped without the page
1671 * disappearing underneath us during migration.
1672 */
1673 put_page(page);
1674 return 1;
1675 }
1676
1677 bool pmd_trans_migrating(pmd_t pmd)
1678 {
1679 struct page *page = pmd_page(pmd);
1680 return PageLocked(page);
1681 }
1682
1683 /*
1684 * Attempt to migrate a misplaced page to the specified destination
1685 * node. Caller is expected to have an elevated reference count on
1686 * the page that will be dropped by this function before returning.
1687 */
1688 int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma,
1689 int node)
1690 {
1691 pg_data_t *pgdat = NODE_DATA(node);
1692 int isolated;
1693 int nr_remaining;
1694 LIST_HEAD(migratepages);
1695
1696 /*
1697 * Don't migrate file pages that are mapped in multiple processes
1698 * with execute permissions as they are probably shared libraries.
1699 */
1700 if (page_mapcount(page) != 1 && page_is_file_cache(page) &&
1701 (vma->vm_flags & VM_EXEC))
1702 goto out;
1703
1704 /*
1705 * Rate-limit the amount of data that is being migrated to a node.
1706 * Optimal placement is no good if the memory bus is saturated and
1707 * all the time is being spent migrating!
1708 */
1709 if (numamigrate_update_ratelimit(pgdat, 1))
1710 goto out;
1711
1712 isolated = numamigrate_isolate_page(pgdat, page);
1713 if (!isolated)
1714 goto out;
1715
1716 list_add(&page->lru, &migratepages);
1717 nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_page,
1718 NULL, node, MIGRATE_ASYNC,
1719 MR_NUMA_MISPLACED);
1720 if (nr_remaining) {
1721 if (!list_empty(&migratepages)) {
1722 list_del(&page->lru);
1723 dec_zone_page_state(page, NR_ISOLATED_ANON +
1724 page_is_file_cache(page));
1725 putback_lru_page(page);
1726 }
1727 isolated = 0;
1728 } else
1729 count_vm_numa_event(NUMA_PAGE_MIGRATE);
1730 BUG_ON(!list_empty(&migratepages));
1731 return isolated;
1732
1733 out:
1734 put_page(page);
1735 return 0;
1736 }
1737 #endif /* CONFIG_NUMA_BALANCING */
1738
1739 #if defined(CONFIG_NUMA_BALANCING) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
1740 /*
1741 * Migrates a THP to a given target node. page must be locked and is unlocked
1742 * before returning.
1743 */
1744 int migrate_misplaced_transhuge_page(struct mm_struct *mm,
1745 struct vm_area_struct *vma,
1746 pmd_t *pmd, pmd_t entry,
1747 unsigned long address,
1748 struct page *page, int node)
1749 {
1750 spinlock_t *ptl;
1751 pg_data_t *pgdat = NODE_DATA(node);
1752 int isolated = 0;
1753 struct page *new_page = NULL;
1754 int page_lru = page_is_file_cache(page);
1755 unsigned long mmun_start = address & HPAGE_PMD_MASK;
1756 unsigned long mmun_end = mmun_start + HPAGE_PMD_SIZE;
1757 pmd_t orig_entry;
1758
1759 /*
1760 * Rate-limit the amount of data that is being migrated to a node.
1761 * Optimal placement is no good if the memory bus is saturated and
1762 * all the time is being spent migrating!
1763 */
1764 if (numamigrate_update_ratelimit(pgdat, HPAGE_PMD_NR))
1765 goto out_dropref;
1766
1767 new_page = alloc_pages_node(node,
1768 (GFP_TRANSHUGE | __GFP_THISNODE) & ~__GFP_RECLAIM,
1769 HPAGE_PMD_ORDER);
1770 if (!new_page)
1771 goto out_fail;
1772 prep_transhuge_page(new_page);
1773
1774 isolated = numamigrate_isolate_page(pgdat, page);
1775 if (!isolated) {
1776 put_page(new_page);
1777 goto out_fail;
1778 }
1779 /*
1780 * We are not sure a pending tlb flush here is for a huge page
1781 * mapping or not. Hence use the tlb range variant
1782 */
1783 if (mm_tlb_flush_pending(mm))
1784 flush_tlb_range(vma, mmun_start, mmun_end);
1785
1786 /* Prepare a page as a migration target */
1787 __SetPageLocked(new_page);
1788 SetPageSwapBacked(new_page);
1789
1790 /* anon mapping, we can simply copy page->mapping to the new page: */
1791 new_page->mapping = page->mapping;
1792 new_page->index = page->index;
1793 migrate_page_copy(new_page, page);
1794 WARN_ON(PageLRU(new_page));
1795
1796 /* Recheck the target PMD */
1797 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
1798 ptl = pmd_lock(mm, pmd);
1799 if (unlikely(!pmd_same(*pmd, entry) || page_count(page) != 2)) {
1800 fail_putback:
1801 spin_unlock(ptl);
1802 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
1803
1804 /* Reverse changes made by migrate_page_copy() */
1805 if (TestClearPageActive(new_page))
1806 SetPageActive(page);
1807 if (TestClearPageUnevictable(new_page))
1808 SetPageUnevictable(page);
1809
1810 unlock_page(new_page);
1811 put_page(new_page); /* Free it */
1812
1813 /* Retake the callers reference and putback on LRU */
1814 get_page(page);
1815 putback_lru_page(page);
1816 mod_zone_page_state(page_zone(page),
1817 NR_ISOLATED_ANON + page_lru, -HPAGE_PMD_NR);
1818
1819 goto out_unlock;
1820 }
1821
1822 orig_entry = *pmd;
1823 entry = mk_pmd(new_page, vma->vm_page_prot);
1824 entry = pmd_mkhuge(entry);
1825 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
1826
1827 /*
1828 * Clear the old entry under pagetable lock and establish the new PTE.
1829 * Any parallel GUP will either observe the old page blocking on the
1830 * page lock, block on the page table lock or observe the new page.
1831 * The SetPageUptodate on the new page and page_add_new_anon_rmap
1832 * guarantee the copy is visible before the pagetable update.
1833 */
1834 flush_cache_range(vma, mmun_start, mmun_end);
1835 page_add_anon_rmap(new_page, vma, mmun_start, true);
1836 pmdp_huge_clear_flush_notify(vma, mmun_start, pmd);
1837 set_pmd_at(mm, mmun_start, pmd, entry);
1838 update_mmu_cache_pmd(vma, address, &entry);
1839
1840 if (page_count(page) != 2) {
1841 set_pmd_at(mm, mmun_start, pmd, orig_entry);
1842 flush_pmd_tlb_range(vma, mmun_start, mmun_end);
1843 mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
1844 update_mmu_cache_pmd(vma, address, &entry);
1845 page_remove_rmap(new_page, true);
1846 goto fail_putback;
1847 }
1848
1849 mlock_migrate_page(new_page, page);
1850 page_remove_rmap(page, true);
1851 set_page_owner_migrate_reason(new_page, MR_NUMA_MISPLACED);
1852
1853 spin_unlock(ptl);
1854 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
1855
1856 /* Take an "isolate" reference and put new page on the LRU. */
1857 get_page(new_page);
1858 putback_lru_page(new_page);
1859
1860 unlock_page(new_page);
1861 unlock_page(page);
1862 put_page(page); /* Drop the rmap reference */
1863 put_page(page); /* Drop the LRU isolation reference */
1864
1865 count_vm_events(PGMIGRATE_SUCCESS, HPAGE_PMD_NR);
1866 count_vm_numa_events(NUMA_PAGE_MIGRATE, HPAGE_PMD_NR);
1867
1868 mod_zone_page_state(page_zone(page),
1869 NR_ISOLATED_ANON + page_lru,
1870 -HPAGE_PMD_NR);
1871 return isolated;
1872
1873 out_fail:
1874 count_vm_events(PGMIGRATE_FAIL, HPAGE_PMD_NR);
1875 out_dropref:
1876 ptl = pmd_lock(mm, pmd);
1877 if (pmd_same(*pmd, entry)) {
1878 entry = pmd_modify(entry, vma->vm_page_prot);
1879 set_pmd_at(mm, mmun_start, pmd, entry);
1880 update_mmu_cache_pmd(vma, address, &entry);
1881 }
1882 spin_unlock(ptl);
1883
1884 out_unlock:
1885 unlock_page(page);
1886 put_page(page);
1887 return 0;
1888 }
1889 #endif /* CONFIG_NUMA_BALANCING */
1890
1891 #endif /* CONFIG_NUMA */