powerpc/mm/radix: Improve TLB/PWC flushes

BugLink: http://bugs.launchpad.net/bugs/1709220
At the moment we have to rather sub-optimal flushing behaviours:

 - flush_tlb_mm() will flush the PWC which is unnecessary (for example
   when doing a fork)

 - A large unmap will call flush_tlb_pwc() multiple times causing us
   to perform that fairly expensive operation repeatedly. This happens
   often in batches of 3 on every new process.

So we change flush_tlb_mm() to only flush the TLB, and we use the
existing "need_flush_all" flag in struct mmu_gather to indicate
that the PWC needs flushing.

Unfortunately, flush_tlb_range() still needs to do a full flush
for now as it's used by the THP collapsing. We will fix that later.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Reviewed-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
(backported from commit a46cc7a90fd8d95bfbb2b27080efe872a1a51db4 linux-next)
Signed-off-by: Joseph Salisbury <joseph.salisbury@canonical.com>
Acked-by: Stefan Bader <stefan.bader@canonical.com>
Acked-by: Seth Forshee <seth.forshee@canonical.com>
Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>

diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index cc7fbde4f53cac09b257f9a7ff4f1202af7c3c98..7196999cdc8292310e2ef941bdc46855df75a77b 100644 (file)
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -22,22 +22,20 @@ extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end
 
 extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
 extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
-extern void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
 extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
                                              int psize);
 extern void radix__tlb_flush(struct mmu_gather *tlb);
 #ifdef CONFIG_SMP
 extern void radix__flush_tlb_mm(struct mm_struct *mm);
 extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
-extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
 extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
                                        int psize);
 #else
 #define radix__flush_tlb_mm(mm)                radix__local_flush_tlb_mm(mm)
 #define radix__flush_tlb_page(vma,addr)        radix__local_flush_tlb_page(vma,addr)
 #define radix__flush_tlb_page_psize(mm,addr,p) radix__local_flush_tlb_page_psize(mm,addr,p)
-#define radix__flush_tlb_pwc(tlb, addr)        radix__local_flush_tlb_pwc(tlb, addr)
 #endif
+extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
 extern void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
                                     unsigned long page_size);
 extern void radix__flush_tlb_lpid(unsigned long lpid);

diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index f1af3ab5200f7b7c61898d2a3c74aa779019d2ef..a11c3eab4ad9f515b7f9988a50eef3b559e94573 100644 (file)
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -134,31 +134,23 @@ void radix__local_flush_tlb_mm(struct mm_struct *mm)
        preempt_disable();
        pid = mm->context.id;
        if (pid != MMU_NO_CONTEXT)
-               _tlbiel_pid(pid, RIC_FLUSH_ALL);
+               _tlbiel_pid(pid, RIC_FLUSH_TLB);
        preempt_enable();
 }
 EXPORT_SYMBOL(radix__local_flush_tlb_mm);
 
-void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
+#ifndef CONFIG_SMP
+static void radix__local_flush_all_mm(struct mm_struct *mm)
 {
        unsigned long pid;
-       struct mm_struct *mm = tlb->mm;
-       /*
-        * If we are doing a full mm flush, we will do a tlb flush
-        * with RIC_FLUSH_ALL later.
-        */
-       if (tlb->fullmm)
-               return;
 
        preempt_disable();
-
        pid = mm->context.id;
        if (pid != MMU_NO_CONTEXT)
-               _tlbiel_pid(pid, RIC_FLUSH_PWC);
-
+               _tlbiel_pid(pid, RIC_FLUSH_ALL);
        preempt_enable();
 }
-EXPORT_SYMBOL(radix__local_flush_tlb_pwc);
+#endif /* CONFIG_SMP */
 
 void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
                                       int psize)
@@ -195,51 +187,36 @@ void radix__flush_tlb_mm(struct mm_struct *mm)
        if (unlikely(pid == MMU_NO_CONTEXT))
                goto no_context;
 
-       if (!mm_is_thread_local(mm)) {
-               int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
-
-               if (lock_tlbie)
-                       raw_spin_lock(&native_tlbie_lock);
-               _tlbie_pid(pid, RIC_FLUSH_ALL);
-               if (lock_tlbie)
-                       raw_spin_unlock(&native_tlbie_lock);
-       } else
-               _tlbiel_pid(pid, RIC_FLUSH_ALL);
+       if (!mm_is_thread_local(mm))
+               _tlbie_pid(pid, RIC_FLUSH_TLB);
+       else
+               _tlbiel_pid(pid, RIC_FLUSH_TLB);
 no_context:
        preempt_enable();
 }
 EXPORT_SYMBOL(radix__flush_tlb_mm);
 
-void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
+static void radix__flush_all_mm(struct mm_struct *mm)
 {
        unsigned long pid;
-       struct mm_struct *mm = tlb->mm;
 
-       /*
-        * If we are doing a full mm flush, we will do a tlb flush
-        * with RIC_FLUSH_ALL later.
-        */
-       if (tlb->fullmm)
-               return;
        preempt_disable();
-
        pid = mm->context.id;
        if (unlikely(pid == MMU_NO_CONTEXT))
                goto no_context;
 
-       if (!mm_is_thread_local(mm)) {
-               int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
-
-               if (lock_tlbie)
-                       raw_spin_lock(&native_tlbie_lock);
-               _tlbie_pid(pid, RIC_FLUSH_PWC);
-               if (lock_tlbie)
-                       raw_spin_unlock(&native_tlbie_lock);
-       } else
-               _tlbiel_pid(pid, RIC_FLUSH_PWC);
+       if (!mm_is_thread_local(mm))
+               _tlbie_pid(pid, RIC_FLUSH_ALL);
+       else
+               _tlbiel_pid(pid, RIC_FLUSH_ALL);
 no_context:
        preempt_enable();
 }
+
+void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
+{
+       tlb->need_flush_all = 1;
+}
 EXPORT_SYMBOL(radix__flush_tlb_pwc);
 
 void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
@@ -277,6 +254,8 @@ void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
 }
 EXPORT_SYMBOL(radix__flush_tlb_page);
 
+#else /* CONFIG_SMP */
+#define radix__flush_all_mm radix__local_flush_all_mm
 #endif /* CONFIG_SMP */
 
 void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end)
@@ -300,7 +279,12 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 
 {
        struct mm_struct *mm = vma->vm_mm;
-       radix__flush_tlb_mm(mm);
+
+       /*
+        * This is currently used when collapsing THPs so we need to
+        * flush the PWC. We should fix this.
+        */
+       radix__flush_all_mm(mm);
 }
 EXPORT_SYMBOL(radix__flush_tlb_range);
 
@@ -331,7 +315,10 @@ void radix__tlb_flush(struct mmu_gather *tlb)
         */
        if (psize != -1 && !tlb->fullmm && !tlb->need_flush_all)
                radix__flush_tlb_range_psize(mm, tlb->start, tlb->end, psize);
-       else
+       else if (tlb->need_flush_all) {
+               tlb->need_flush_all = 0;
+               radix__flush_all_mm(mm);
+       } else
                radix__flush_tlb_mm(mm);
 }