mm/mmu_gather: track page size with mmu gather and force flush if page size change

This allows an arch which needs to do special handing with respect to
different page size when flushing tlb to implement the same in mmu
gather.

Link: http://lkml.kernel.org/r/1465049193-22197-3-git-send-email-aneesh.kumar@linux.vnet.ibm.com
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Hugh Dickins <hughd@google.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Minchan Kim <minchan.kim@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h
index a9d2aee3826f470d6966fcbcad1ef8fa822782cf..1e25cd80589e7370eee54dedb18d355d3bf66e48 100644 (file)
--- a/arch/arm/include/asm/tlb.h
+++ b/arch/arm/include/asm/tlb.h
@@ -225,12 +225,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
        }
 }
 
+static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
+                                         struct page *page, int page_size)
+{
+       return __tlb_remove_page(tlb, page);
+}
+
 static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,
                                         struct page *page)
 {
        return __tlb_remove_page(tlb, page);
 }
 
+static inline void tlb_remove_page_size(struct mmu_gather *tlb,
+                                       struct page *page, int page_size)
+{
+       return tlb_remove_page(tlb, page);
+}
+
 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
        unsigned long addr)
 {

diff --git a/arch/ia64/include/asm/tlb.h b/arch/ia64/include/asm/tlb.h
index e7da41aa91100be88bf8a723f410e1d3ef8d8b31..77e541cf0e5d560b4325511f2c18d050d2ca5ddf 100644 (file)
--- a/arch/ia64/include/asm/tlb.h
+++ b/arch/ia64/include/asm/tlb.h
@@ -242,12 +242,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
        }
 }
 
+static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
+                                         struct page *page, int page_size)
+{
+       return __tlb_remove_page(tlb, page);
+}
+
 static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,
                                         struct page *page)
 {
        return __tlb_remove_page(tlb, page);
 }
 
+static inline void tlb_remove_page_size(struct mmu_gather *tlb,
+                                       struct page *page, int page_size)
+{
+       return tlb_remove_page(tlb, page);
+}
+
 /*
  * Remove TLB entry for PTE mapped at virtual address ADDRESS.  This is called for any
  * PTE, not just those pointing to (normal) physical memory.

diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h
index 30759b5608497407a699877304d356edc8d9c014..15711de1040358098fd864caad1d1f3b472b2301 100644 (file)
--- a/arch/s390/include/asm/tlb.h
+++ b/arch/s390/include/asm/tlb.h
@@ -98,11 +98,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
        free_page_and_swap_cache(page);
 }
 
+static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
+                                         struct page *page, int page_size)
+{
+       return __tlb_remove_page(tlb, page);
+}
+
 static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,
                                         struct page *page)
 {
        return __tlb_remove_page(tlb, page);
 }
+
+static inline void tlb_remove_page_size(struct mmu_gather *tlb,
+                                       struct page *page, int page_size)
+{
+       return tlb_remove_page(tlb, page);
+}
+
 /*
  * pte_free_tlb frees a pte table and clears the CRSTE for the
  * page table from the tlb.

diff --git a/arch/sh/include/asm/tlb.h b/arch/sh/include/asm/tlb.h
index 21ae8f5546b20a0bde08d5892d2d6d1fca7bf994..025cdb1032f6f9db7f30cfa54ccf10047772ee83 100644 (file)
--- a/arch/sh/include/asm/tlb.h
+++ b/arch/sh/include/asm/tlb.h
@@ -109,12 +109,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
        __tlb_remove_page(tlb, page);
 }
 
+static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
+                                         struct page *page, int page_size)
+{
+       return __tlb_remove_page(tlb, page);
+}
+
 static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,
                                         struct page *page)
 {
        return __tlb_remove_page(tlb, page);
 }
 
+static inline void tlb_remove_page_size(struct mmu_gather *tlb,
+                                       struct page *page, int page_size)
+{
+       return tlb_remove_page(tlb, page);
+}
+
 #define pte_free_tlb(tlb, ptep, addr)  pte_free((tlb)->mm, ptep)
 #define pmd_free_tlb(tlb, pmdp, addr)  pmd_free((tlb)->mm, pmdp)
 #define pud_free_tlb(tlb, pudp, addr)  pud_free((tlb)->mm, pudp)

diff --git a/arch/um/include/asm/tlb.h b/arch/um/include/asm/tlb.h
index 3dc4cbb3c2c0a6e872d1b371eb8f9e1f0835deee..821ff0acfe17f7f9e4264be04d7d346731bd6b4e 100644 (file)
--- a/arch/um/include/asm/tlb.h
+++ b/arch/um/include/asm/tlb.h
@@ -110,12 +110,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
        __tlb_remove_page(tlb, page);
 }
 
+static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
+                                         struct page *page, int page_size)
+{
+       return __tlb_remove_page(tlb, page);
+}
+
 static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,
                                         struct page *page)
 {
        return __tlb_remove_page(tlb, page);
 }
 
+static inline void tlb_remove_page_size(struct mmu_gather *tlb,
+                                       struct page *page, int page_size)
+{
+       return tlb_remove_page(tlb, page);
+}
+
 /**
  * tlb_remove_tlb_entry - remember a pte unmapping for later tlb invalidation.
  *

diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index 7b899a46a4cb8efa13e2dad9d22820b3b1389ed5..c6d6671876080e9bdc48b820bb5b5d8d362a470b 100644 (file)
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -112,6 +112,7 @@ struct mmu_gather {
         * that that we can adjust the range after the flush
         */
        unsigned long addr;
+       int page_size;
 };
 
 #define HAVE_GENERIC_MMU_GATHER
@@ -120,7 +121,8 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long
 void tlb_flush_mmu(struct mmu_gather *tlb);
 void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start,
                                                        unsigned long end);
-bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page);
+extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,
+                                  int page_size);
 
 static inline void __tlb_adjust_range(struct mmu_gather *tlb,
                                      unsigned long address)
@@ -145,23 +147,36 @@ static inline void __tlb_reset_range(struct mmu_gather *tlb)
        }
 }
 
+static inline void tlb_remove_page_size(struct mmu_gather *tlb,
+                                       struct page *page, int page_size)
+{
+       if (__tlb_remove_page_size(tlb, page, page_size)) {
+               tlb_flush_mmu(tlb);
+               tlb->page_size = page_size;
+               __tlb_adjust_range(tlb, tlb->addr);
+               __tlb_remove_page_size(tlb, page, page_size);
+       }
+}
+
+static bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
+{
+       return __tlb_remove_page_size(tlb, page, PAGE_SIZE);
+}
+
 /* tlb_remove_page
  *     Similar to __tlb_remove_page but will call tlb_flush_mmu() itself when
  *     required.
  */
 static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
 {
-       if (__tlb_remove_page(tlb, page)) {
-               tlb_flush_mmu(tlb);
-               __tlb_adjust_range(tlb, tlb->addr);
-               __tlb_remove_page(tlb, page);
-       }
+       return tlb_remove_page_size(tlb, page, PAGE_SIZE);
 }
 
 static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb, struct page *page)
 {
        /* active->nr should be zero when we call this */
        VM_BUG_ON_PAGE(tlb->active->nr, page);
+       tlb->page_size = PAGE_SIZE;
        __tlb_adjust_range(tlb, tlb->addr);
        return __tlb_remove_page(tlb, page);
 }

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 343a2b7e57aa25f7d02709ff13ab8a5e47c68a87..23d1bf42fef105e85e8e452d663cd6c2d5d6fdac 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1689,7 +1689,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
                pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd));
                atomic_long_dec(&tlb->mm->nr_ptes);
                spin_unlock(ptl);
-               tlb_remove_page(tlb, page);
+               tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE);
        }
        return 1;
 }

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 524c078ce67b2708c10de1504d279159420256d6..a9a8c313d133b97273ef073e72dcb7e3edf19690 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3250,7 +3250,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
                page_remove_rmap(page, true);
 
                spin_unlock(ptl);
-               tlb_remove_page(tlb, page);
+               tlb_remove_page_size(tlb, page, huge_page_size(h));
                /*
                 * Bail out after unmapping reference page if supplied
                 */

diff --git a/mm/memory.c b/mm/memory.c
index 12f31501c3231813086c64102911b4f7404bdbb0..a329149e1c5448087a8d4fd1b42f1c23ad393bd1 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -233,6 +233,7 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
        tlb->batch = NULL;
 #endif
+       tlb->page_size = 0;
 
        __tlb_reset_range(tlb);
 }
@@ -294,12 +295,19 @@ void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long e
  *     When out of page slots we must call tlb_flush_mmu().
  *returns true if the caller should flush.
  */
-bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
+bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size)
 {
        struct mmu_gather_batch *batch;
 
        VM_BUG_ON(!tlb->end);
 
+       if (!tlb->page_size)
+               tlb->page_size = page_size;
+       else {
+               if (page_size != tlb->page_size)
+                       return true;
+       }
+
        batch = tlb->active;
        if (batch->nr == batch->max) {
                if (!tlb_next_batch(tlb))