powerpc/mm/radix: Improve TLB/PWC flushes

[mirror_ubuntu-zesty-kernel.git] / arch / powerpc / mm / tlb-radix.c

diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index f1af3ab5200f7b7c61898d2a3c74aa779019d2ef..a11c3eab4ad9f515b7f9988a50eef3b559e94573 100644 (file)
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -134,31 +134,23 @@ void radix__local_flush_tlb_mm(struct mm_struct *mm)
        preempt_disable();
        pid = mm->context.id;
        if (pid != MMU_NO_CONTEXT)
-               _tlbiel_pid(pid, RIC_FLUSH_ALL);
+               _tlbiel_pid(pid, RIC_FLUSH_TLB);
        preempt_enable();
 }
 EXPORT_SYMBOL(radix__local_flush_tlb_mm);
 
-void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
+#ifndef CONFIG_SMP
+static void radix__local_flush_all_mm(struct mm_struct *mm)
 {
        unsigned long pid;
-       struct mm_struct *mm = tlb->mm;
-       /*
-        * If we are doing a full mm flush, we will do a tlb flush
-        * with RIC_FLUSH_ALL later.
-        */
-       if (tlb->fullmm)
-               return;
 
        preempt_disable();
-
        pid = mm->context.id;
        if (pid != MMU_NO_CONTEXT)
-               _tlbiel_pid(pid, RIC_FLUSH_PWC);
-
+               _tlbiel_pid(pid, RIC_FLUSH_ALL);
        preempt_enable();
 }
-EXPORT_SYMBOL(radix__local_flush_tlb_pwc);
+#endif /* CONFIG_SMP */
 
 void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
                                       int psize)
@@ -195,51 +187,36 @@ void radix__flush_tlb_mm(struct mm_struct *mm)
        if (unlikely(pid == MMU_NO_CONTEXT))
                goto no_context;
 
-       if (!mm_is_thread_local(mm)) {
-               int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
-
-               if (lock_tlbie)
-                       raw_spin_lock(&native_tlbie_lock);
-               _tlbie_pid(pid, RIC_FLUSH_ALL);
-               if (lock_tlbie)
-                       raw_spin_unlock(&native_tlbie_lock);
-       } else
-               _tlbiel_pid(pid, RIC_FLUSH_ALL);
+       if (!mm_is_thread_local(mm))
+               _tlbie_pid(pid, RIC_FLUSH_TLB);
+       else
+               _tlbiel_pid(pid, RIC_FLUSH_TLB);
 no_context:
        preempt_enable();
 }
 EXPORT_SYMBOL(radix__flush_tlb_mm);
 
-void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
+static void radix__flush_all_mm(struct mm_struct *mm)
 {
        unsigned long pid;
-       struct mm_struct *mm = tlb->mm;
 
-       /*
-        * If we are doing a full mm flush, we will do a tlb flush
-        * with RIC_FLUSH_ALL later.
-        */
-       if (tlb->fullmm)
-               return;
        preempt_disable();
-
        pid = mm->context.id;
        if (unlikely(pid == MMU_NO_CONTEXT))
                goto no_context;
 
-       if (!mm_is_thread_local(mm)) {
-               int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
-
-               if (lock_tlbie)
-                       raw_spin_lock(&native_tlbie_lock);
-               _tlbie_pid(pid, RIC_FLUSH_PWC);
-               if (lock_tlbie)
-                       raw_spin_unlock(&native_tlbie_lock);
-       } else
-               _tlbiel_pid(pid, RIC_FLUSH_PWC);
+       if (!mm_is_thread_local(mm))
+               _tlbie_pid(pid, RIC_FLUSH_ALL);
+       else
+               _tlbiel_pid(pid, RIC_FLUSH_ALL);
 no_context:
        preempt_enable();
 }
+
+void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
+{
+       tlb->need_flush_all = 1;
+}
 EXPORT_SYMBOL(radix__flush_tlb_pwc);
 
 void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
@@ -277,6 +254,8 @@ void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
 }
 EXPORT_SYMBOL(radix__flush_tlb_page);
 
+#else /* CONFIG_SMP */
+#define radix__flush_all_mm radix__local_flush_all_mm
 #endif /* CONFIG_SMP */
 
 void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end)
@@ -300,7 +279,12 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 
 {
        struct mm_struct *mm = vma->vm_mm;
-       radix__flush_tlb_mm(mm);
+
+       /*
+        * This is currently used when collapsing THPs so we need to
+        * flush the PWC. We should fix this.
+        */
+       radix__flush_all_mm(mm);
 }
 EXPORT_SYMBOL(radix__flush_tlb_range);
 
@@ -331,7 +315,10 @@ void radix__tlb_flush(struct mmu_gather *tlb)
         */
        if (psize != -1 && !tlb->fullmm && !tlb->need_flush_all)
                radix__flush_tlb_range_psize(mm, tlb->start, tlb->end, psize);
-       else
+       else if (tlb->need_flush_all) {
+               tlb->need_flush_all = 0;
+               radix__flush_all_mm(mm);
+       } else
                radix__flush_tlb_mm(mm);
 }