powerpc/mm/radix: Optimise tlbiel flush all case

[mirror_ubuntu-zesty-kernel.git] / arch / powerpc / mm / tlb-radix.c

diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index 61b79119065ff3030162df12446a67c712cd6649..5e531e46ca3af30e6b0b3e209c4015f8baea77e1 100644 (file)
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -34,10 +34,8 @@ static inline void __tlbiel_pid(unsigned long pid, int set,
        prs = 1; /* process scoped */
        r = 1;   /* raidx format */
 
-       asm volatile("ptesync": : :"memory");
        asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
                     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
-       asm volatile("ptesync": : :"memory");
 }
 
 /*
@@ -47,12 +45,23 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
 {
        int set;
 
-       for (set = 0; set < POWER9_TLB_SETS_RADIX ; set++) {
+       asm volatile("ptesync": : :"memory");
+
+       /*
+        * Flush the first set of the TLB, and if we're doing a RIC_FLUSH_ALL,
+        * also flush the entire Page Walk Cache.
+        */
+       __tlbiel_pid(pid, 0, ric);
+
+       if (ric == RIC_FLUSH_ALL)
+               /* For the remaining sets, just flush the TLB */
+               ric = RIC_FLUSH_TLB;
+
+       for (set = 1; set < POWER9_TLB_SETS_RADIX ; set++)
                __tlbiel_pid(pid, set, ric);
-       }
-       if (cpu_has_feature(CPU_FTR_POWER9_DD1))
-               asm volatile(PPC_INVALIDATE_ERAT : : :"memory");
-       return;
+
+       asm volatile("ptesync": : :"memory");
+       asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
 }
 
 static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
@@ -85,8 +94,6 @@ static inline void _tlbiel_va(unsigned long va, unsigned long pid,
        asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
                     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
        asm volatile("ptesync": : :"memory");
-       if (cpu_has_feature(CPU_FTR_POWER9_DD1))
-               asm volatile(PPC_INVALIDATE_ERAT : : :"memory");
 }
 
 static inline void _tlbie_va(unsigned long va, unsigned long pid,
@@ -133,6 +140,12 @@ void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
 {
        unsigned long pid;
        struct mm_struct *mm = tlb->mm;
+       /*
+        * If we are doing a full mm flush, we will do a tlb flush
+        * with RIC_FLUSH_ALL later.
+        */
+       if (tlb->fullmm)
+               return;
 
        preempt_disable();
 
@@ -199,6 +212,12 @@ void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
        unsigned long pid;
        struct mm_struct *mm = tlb->mm;
 
+       /*
+        * If we are doing a full mm flush, we will do a tlb flush
+        * with RIC_FLUSH_ALL later.
+        */
+       if (tlb->fullmm)
+               return;
        preempt_disable();
 
        pid = mm->context.id;