Revert "rmap: do not call mmu_notifier_invalidate_page() under ptl"

[mirror_ubuntu-bionic-kernel.git] / mm / rmap.c

diff --git a/mm/rmap.c b/mm/rmap.c
index c1286d47aa1fad7fee7ea5bb865a2dc7efd672f2..c8993c63eb259b3a5302a058ce231d1290fc9b66 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -888,10 +888,10 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
                .flags = PVMW_SYNC,
        };
        int *cleaned = arg;
-       bool invalidation_needed = false;
 
        while (page_vma_mapped_walk(&pvmw)) {
                int ret = 0;
+               address = pvmw.address;
                if (pvmw.pte) {
                        pte_t entry;
                        pte_t *pte = pvmw.pte;
@@ -899,11 +899,11 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
                        if (!pte_dirty(*pte) && !pte_write(*pte))
                                continue;
 
-                       flush_cache_page(vma, pvmw.address, pte_pfn(*pte));
-                       entry = ptep_clear_flush(vma, pvmw.address, pte);
+                       flush_cache_page(vma, address, pte_pfn(*pte));
+                       entry = ptep_clear_flush(vma, address, pte);
                        entry = pte_wrprotect(entry);
                        entry = pte_mkclean(entry);
-                       set_pte_at(vma->vm_mm, pvmw.address, pte, entry);
+                       set_pte_at(vma->vm_mm, address, pte, entry);
                        ret = 1;
                } else {
 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
@@ -913,11 +913,11 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
                        if (!pmd_dirty(*pmd) && !pmd_write(*pmd))
                                continue;
 
-                       flush_cache_page(vma, pvmw.address, page_to_pfn(page));
-                       entry = pmdp_huge_clear_flush(vma, pvmw.address, pmd);
+                       flush_cache_page(vma, address, page_to_pfn(page));
+                       entry = pmdp_huge_clear_flush(vma, address, pmd);
                        entry = pmd_wrprotect(entry);
                        entry = pmd_mkclean(entry);
-                       set_pmd_at(vma->vm_mm, pvmw.address, pmd, entry);
+                       set_pmd_at(vma->vm_mm, address, pmd, entry);
                        ret = 1;
 #else
                        /* unexpected pmd-mapped page? */
@@ -926,16 +926,11 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
                }
 
                if (ret) {
+                       mmu_notifier_invalidate_page(vma->vm_mm, address);
                        (*cleaned)++;
-                       invalidation_needed = true;
                }
        }
 
-       if (invalidation_needed) {
-               mmu_notifier_invalidate_range(vma->vm_mm, address,
-                               address + (1UL << compound_order(page)));
-       }
-
        return true;
 }
 
@@ -1328,7 +1323,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
        };
        pte_t pteval;
        struct page *subpage;
-       bool ret = true, invalidation_needed = false;
+       bool ret = true;
        enum ttu_flags flags = (enum ttu_flags)arg;
 
        /* munlock has nothing to gain from examining un-locked vmas */
@@ -1368,9 +1363,11 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
                VM_BUG_ON_PAGE(!pvmw.pte, page);
 
                subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);
+               address = pvmw.address;
+
 
                if (!(flags & TTU_IGNORE_ACCESS)) {
-                       if (ptep_clear_flush_young_notify(vma, pvmw.address,
+                       if (ptep_clear_flush_young_notify(vma, address,
                                                pvmw.pte)) {
                                ret = false;
                                page_vma_mapped_walk_done(&pvmw);
@@ -1379,7 +1376,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
                }
 
                /* Nuke the page table entry. */
-               flush_cache_page(vma, pvmw.address, pte_pfn(*pvmw.pte));
+               flush_cache_page(vma, address, pte_pfn(*pvmw.pte));
                if (should_defer_flush(mm, flags)) {
                        /*
                         * We clear the PTE but do not flush so potentially
@@ -1389,12 +1386,11 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
                         * transition on a cached TLB entry is written through
                         * and traps if the PTE is unmapped.
                         */
-                       pteval = ptep_get_and_clear(mm, pvmw.address,
-                                                   pvmw.pte);
+                       pteval = ptep_get_and_clear(mm, address, pvmw.pte);
 
                        set_tlb_ubc_flush_pending(mm, pte_dirty(pteval));
                } else {
-                       pteval = ptep_clear_flush(vma, pvmw.address, pvmw.pte);
+                       pteval = ptep_clear_flush(vma, address, pvmw.pte);
                }
 
                /* Move the dirty bit to the page. Now the pte is gone. */
@@ -1409,12 +1405,12 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
                        if (PageHuge(page)) {
                                int nr = 1 << compound_order(page);
                                hugetlb_count_sub(nr, mm);
-                               set_huge_swap_pte_at(mm, pvmw.address,
+                               set_huge_swap_pte_at(mm, address,
                                                     pvmw.pte, pteval,
                                                     vma_mmu_pagesize(vma));
                        } else {
                                dec_mm_counter(mm, mm_counter(page));
-                               set_pte_at(mm, pvmw.address, pvmw.pte, pteval);
+                               set_pte_at(mm, address, pvmw.pte, pteval);
                        }
 
                } else if (pte_unused(pteval)) {
@@ -1438,7 +1434,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
                        swp_pte = swp_entry_to_pte(entry);
                        if (pte_soft_dirty(pteval))
                                swp_pte = pte_swp_mksoft_dirty(swp_pte);
-                       set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte);
+                       set_pte_at(mm, address, pvmw.pte, swp_pte);
                } else if (PageAnon(page)) {
                        swp_entry_t entry = { .val = page_private(subpage) };
                        pte_t swp_pte;
@@ -1464,7 +1460,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
                                 * If the page was redirtied, it cannot be
                                 * discarded. Remap the page to page table.
                                 */
-                               set_pte_at(mm, pvmw.address, pvmw.pte, pteval);
+                               set_pte_at(mm, address, pvmw.pte, pteval);
                                SetPageSwapBacked(page);
                                ret = false;
                                page_vma_mapped_walk_done(&pvmw);
@@ -1472,7 +1468,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
                        }
 
                        if (swap_duplicate(entry) < 0) {
-                               set_pte_at(mm, pvmw.address, pvmw.pte, pteval);
+                               set_pte_at(mm, address, pvmw.pte, pteval);
                                ret = false;
                                page_vma_mapped_walk_done(&pvmw);
                                break;
@@ -1488,18 +1484,14 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
                        swp_pte = swp_entry_to_pte(entry);
                        if (pte_soft_dirty(pteval))
                                swp_pte = pte_swp_mksoft_dirty(swp_pte);
-                       set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte);
+                       set_pte_at(mm, address, pvmw.pte, swp_pte);
                } else
                        dec_mm_counter(mm, mm_counter_file(page));
 discard:
                page_remove_rmap(subpage, PageHuge(page));
                put_page(page);
-               invalidation_needed = true;
+               mmu_notifier_invalidate_page(mm, address);
        }
-
-       if (invalidation_needed)
-               mmu_notifier_invalidate_range(mm, address,
-                               address + (1UL << compound_order(page)));
        return ret;
 }