UBUNTU: Ubuntu-4.15.0-96.97

[mirror_ubuntu-bionic-kernel.git] / mm / migrate.c

diff --git a/mm/migrate.c b/mm/migrate.c
index 4d0be47a322a8a33491c94072100ebcec829becd..09047de4961d036595db0804538656167702c5f1 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -247,10 +247,8 @@ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
                                pte = swp_entry_to_pte(entry);
                        } else if (is_device_public_page(new)) {
                                pte = pte_mkdevmap(pte);
-                               flush_dcache_page(new);
                        }
-               } else
-                       flush_dcache_page(new);
+               }
 
 #ifdef CONFIG_HUGETLB_PAGE
                if (PageHuge(new)) {
@@ -274,6 +272,9 @@ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
                if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))
                        mlock_vma_page(new);
 
+               if (PageTransHuge(page) && PageMlocked(page))
+                       clear_page_mlock(page);
+
                /* No need to invalidate - it was non-present before */
                update_mmu_cache(vma, pvmw.address, pvmw.pte);
        }
@@ -968,6 +969,13 @@ static int move_to_new_page(struct page *newpage, struct page *page,
                 */
                if (!PageMappingFlags(page))
                        page->mapping = NULL;
+
+               if (unlikely(is_zone_device_page(newpage))) {
+                       if (is_device_public_page(newpage))
+                               flush_dcache_page(newpage);
+               } else
+                       flush_dcache_page(newpage);
+
        }
 out:
        return rc;
@@ -1103,10 +1111,13 @@ out:
         * If migration is successful, decrease refcount of the newpage
         * which will not free the page because new page owner increased
         * refcounter. As well, if it is LRU page, add the page to LRU
-        * list in here.
+        * list in here. Use the old state of the isolated source page to
+        * determine if we migrated a LRU page. newpage was already unlocked
+        * and possibly modified by its owner - don't rely on the page
+        * state.
         */
        if (rc == MIGRATEPAGE_SUCCESS) {
-               if (unlikely(__PageMovable(newpage)))
+               if (unlikely(!is_lru))
                        put_page(newpage);
                else
                        putback_lru_page(newpage);
@@ -1297,6 +1308,16 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
                lock_page(hpage);
        }
 
+       /*
+        * Check for pages which are in the process of being freed.  Without
+        * page_mapping() set, hugetlbfs specific move page routine will not
+        * be called and we could leak usage counts for subpools.
+        */
+       if (page_private(hpage) && !page_mapping(hpage)) {
+               rc = -EBUSY;
+               goto out_unlock;
+       }
+
        if (PageAnon(hpage))
                anon_vma = page_get_anon_vma(hpage);
 
@@ -1328,6 +1349,7 @@ put_anon:
                set_page_owner_migrate_reason(new_hpage, reason);
        }
 
+out_unlock:
        unlock_page(hpage);
 out:
        if (rc != -EAGAIN)
@@ -2071,15 +2093,26 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
        entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
 
        /*
-        * Clear the old entry under pagetable lock and establish the new PTE.
-        * Any parallel GUP will either observe the old page blocking on the
-        * page lock, block on the page table lock or observe the new page.
-        * The SetPageUptodate on the new page and page_add_new_anon_rmap
-        * guarantee the copy is visible before the pagetable update.
+        * Overwrite the old entry under pagetable lock and establish
+        * the new PTE. Any parallel GUP will either observe the old
+        * page blocking on the page lock, block on the page table
+        * lock or observe the new page. The SetPageUptodate on the
+        * new page and page_add_new_anon_rmap guarantee the copy is
+        * visible before the pagetable update.
         */
        flush_cache_range(vma, mmun_start, mmun_end);
        page_add_anon_rmap(new_page, vma, mmun_start, true);
-       pmdp_huge_clear_flush_notify(vma, mmun_start, pmd);
+       /*
+        * At this point the pmd is numa/protnone (i.e. non present) and the TLB
+        * has already been flushed globally.  So no TLB can be currently
+        * caching this non present pmd mapping.  There's no need to clear the
+        * pmd before doing set_pmd_at(), nor to flush the TLB after
+        * set_pmd_at().  Clearing the pmd here would introduce a race
+        * condition against MADV_DONTNEED, because MADV_DONTNEED only holds the
+        * mmap_sem for reading.  If the pmd is set to NULL at any given time,
+        * MADV_DONTNEED won't wait on the pmd lock and it'll skip clearing this
+        * pmd.
+        */
        set_pmd_at(mm, mmun_start, pmd, entry);
        update_mmu_cache_pmd(vma, address, &entry);
 
@@ -2093,7 +2126,7 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
         * No need to double call mmu_notifier->invalidate_range() callback as
         * the above pmdp_huge_clear_flush_notify() did already call it.
         */
-       mmu_notifier_invalidate_range_only_end(mm, mmun_start, mmun_end);
+       mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
 
        /* Take an "isolate" reference and put new page on the LRU. */
        get_page(new_page);
@@ -2351,16 +2384,13 @@ next:
  */
 static void migrate_vma_collect(struct migrate_vma *migrate)
 {
-       struct mm_walk mm_walk;
-
-       mm_walk.pmd_entry = migrate_vma_collect_pmd;
-       mm_walk.pte_entry = NULL;
-       mm_walk.pte_hole = migrate_vma_collect_hole;
-       mm_walk.hugetlb_entry = NULL;
-       mm_walk.test_walk = NULL;
-       mm_walk.vma = migrate->vma;
-       mm_walk.mm = migrate->vma->vm_mm;
-       mm_walk.private = migrate;
+       struct mm_walk mm_walk = {
+               .pmd_entry = migrate_vma_collect_pmd,
+               .pte_hole = migrate_vma_collect_hole,
+               .vma = migrate->vma,
+               .mm = migrate->vma->vm_mm,
+               .private = migrate,
+       };
 
        mmu_notifier_invalidate_range_start(mm_walk.mm,
                                            migrate->start,