powerpc/mm/thp: Use tlbiel if possible

If we know that user address space has never executed on other cpus
we could use tlbiel.

Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>

diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h
index 15c9150a58cc8ecd84b75a60935d7b3780b2f5be..e5c0919acca470536007c985e0f6cbd6f7c7a9bd 100644 (file)
--- a/arch/powerpc/include/asm/machdep.h
+++ b/arch/powerpc/include/asm/machdep.h
@@ -60,7 +60,7 @@ struct machdep_calls {
        void            (*hugepage_invalidate)(unsigned long vsid,
                                               unsigned long addr,
                                               unsigned char *hpte_slot_array,
-                                              int psize, int ssize);
+                                              int psize, int ssize, int local);
        /* special for kexec, to be called in real mode, linear mapping is
         * destroyed as well */
        void            (*hpte_clear_all)(void);

diff --git a/arch/powerpc/include/asm/tlbflush.h b/arch/powerpc/include/asm/tlbflush.h
index 19550d346feadcfeb6da843bd4f0677b1f79e949..4d3ecd8d8929c1bd26e799b4f94413a0cbafd396 100644 (file)
--- a/arch/powerpc/include/asm/tlbflush.h
+++ b/arch/powerpc/include/asm/tlbflush.h
@@ -128,7 +128,8 @@ extern void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize,
                            int ssize, int local);
 extern void flush_hash_range(unsigned long number, int local);
 extern void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
-                               pmd_t *pmdp, unsigned int psize, int ssize);
+                               pmd_t *pmdp, unsigned int psize, int ssize,
+                               int local);
 
 static inline void local_flush_tlb_mm(struct mm_struct *mm)
 {

diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index 558e50bac6f716158646b55d8aaa9b4db726dd0f..13700911b522b1cfdc8518c4566fd3b7ab859f4e 100644 (file)
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -425,7 +425,7 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
 static void native_hugepage_invalidate(unsigned long vsid,
                                       unsigned long addr,
                                       unsigned char *hpte_slot_array,
-                                      int psize, int ssize)
+                                      int psize, int ssize, int local)
 {
        int i;
        struct hash_pte *hptep;
@@ -471,7 +471,7 @@ static void native_hugepage_invalidate(unsigned long vsid,
                 * instruction compares entry_VA in tlb with the VA specified
                 * here
                 */
-               tlbie(vpn, psize, actual_psize, ssize, 0);
+               tlbie(vpn, psize, actual_psize, ssize, local);
        }
        local_irq_restore(flags);
 }

diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 6c2076c65d7c7ff8d164781646b0c7c8e4a47599..68211d398fdb18021bd06e5f3c0965ec0bd3cf4f 100644 (file)
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -1317,7 +1317,7 @@ void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize,
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
-                        pmd_t *pmdp, unsigned int psize, int ssize)
+                        pmd_t *pmdp, unsigned int psize, int ssize, int local)
 {
        int i, max_hpte_count, valid;
        unsigned long s_addr;
@@ -1334,9 +1334,11 @@ void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
        if (!hpte_slot_array)
                return;
 
-       if (ppc_md.hugepage_invalidate)
-               return ppc_md.hugepage_invalidate(vsid, s_addr, hpte_slot_array,
-                                                 psize, ssize);
+       if (ppc_md.hugepage_invalidate) {
+               ppc_md.hugepage_invalidate(vsid, s_addr, hpte_slot_array,
+                                          psize, ssize, local);
+               goto tm_abort;
+       }
        /*
         * No bluk hpte removal support, invalidate each entry
         */
@@ -1362,8 +1364,24 @@ void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
                slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
                slot += hidx & _PTEIDX_GROUP_IX;
                ppc_md.hpte_invalidate(slot, vpn, psize,
-                                      MMU_PAGE_16M, ssize, 0);
+                                      MMU_PAGE_16M, ssize, local);
+       }
+tm_abort:
+#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
+       /* Transactions are not aborted by tlbiel, only tlbie.
+        * Without, syncing a page back to a block device w/ PIO could pick up
+        * transactional data (bad!) so we force an abort here.  Before the
+        * sync the page will be made read-only, which will flush_hash_page.
+        * BIG ISSUE here: if the kernel uses a page from userspace without
+        * unmapping it first, it may see the speculated version.
+        */
+       if (local && cpu_has_feature(CPU_FTR_TM) &&
+           current->thread.regs &&
+           MSR_TM_ACTIVE(current->thread.regs->msr)) {
+               tm_enable();
+               tm_abort(TM_CAUSE_TLBI);
        }
+#endif
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 

diff --git a/arch/powerpc/mm/hugepage-hash64.c b/arch/powerpc/mm/hugepage-hash64.c
index 1b3ad46a71b5f4fcbd790b995138e7e22334936f..3a648cd363ae84c5f39352a64e25019fefd0d01c 100644 (file)
--- a/arch/powerpc/mm/hugepage-hash64.c
+++ b/arch/powerpc/mm/hugepage-hash64.c
@@ -95,7 +95,7 @@ int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid,
                 */
                if ((old_pmd & _PAGE_HASHPTE) && !(old_pmd & _PAGE_COMBO))
                        flush_hash_hugepage(vsid, ea, pmdp, MMU_PAGE_64K,
-                                           ssize);
+                                           ssize, local);
        }
 
        valid = hpte_valid(hpte_slot_array, index);

diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
index c175c990580ebbc78f4dcd4c1bcc92a052ab9bda..eea9fa1f8ae79b37fccc6b0b8a2f620cacd7347c 100644 (file)
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -739,9 +739,10 @@ void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
 void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
                            pmd_t *pmdp, unsigned long old_pmd)
 {
-       int ssize;
+       int ssize, local = 0;
        unsigned int psize;
        unsigned long vsid;
+       const struct cpumask *tmp;
 
        /* get the base page size,vsid and segment size */
 #ifdef CONFIG_DEBUG_VM
@@ -762,7 +763,11 @@ void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
                ssize = mmu_kernel_ssize;
        }
 
-       return flush_hash_hugepage(vsid, addr, pmdp, psize, ssize);
+       tmp = cpumask_of(smp_processor_id());
+       if (cpumask_equal(mm_cpumask(mm), tmp))
+               local = 1;
+
+       return flush_hash_hugepage(vsid, addr, pmdp, psize, ssize, local);
 }
 
 static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot)

diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c
index d214a012b026982cc913c9334f61093e6cef88b2..832f221840f2fe61fbe59b1e8c222739274eff8c 100644 (file)
--- a/arch/powerpc/platforms/pseries/lpar.c
+++ b/arch/powerpc/platforms/pseries/lpar.c
@@ -442,7 +442,7 @@ static void __pSeries_lpar_hugepage_invalidate(unsigned long *slot,
 static void pSeries_lpar_hugepage_invalidate(unsigned long vsid,
                                             unsigned long addr,
                                             unsigned char *hpte_slot_array,
-                                            int psize, int ssize)
+                                            int psize, int ssize, int local)
 {
        int i, index = 0;
        unsigned long s_addr = addr;