x86/speculation: Rework speculative_store_bypass_update()

[mirror_ubuntu-artful-kernel.git] / mm / page_vma_mapped.c

diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c
index 8ec6ba230bb9dabba967489f0ba8f15ca61b8806..6b9311631aa1209cb8af2c71ae371e9b7542dab8 100644 (file)
--- a/mm/page_vma_mapped.c
+++ b/mm/page_vma_mapped.c
@@ -6,17 +6,6 @@
 
 #include "internal.h"
 
-static inline bool check_pmd(struct page_vma_mapped_walk *pvmw)
-{
-       pmd_t pmde;
-       /*
-        * Make sure we don't re-load pmd between present and !trans_huge check.
-        * We need a consistent view.
-        */
-       pmde = READ_ONCE(*pvmw->pmd);
-       return pmd_present(pmde) && !pmd_trans_huge(pmde);
-}
-
 static inline bool not_found(struct page_vma_mapped_walk *pvmw)
 {
        page_vma_mapped_walk_done(pvmw);
@@ -106,6 +95,7 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw)
        pgd_t *pgd;
        p4d_t *p4d;
        pud_t *pud;
+       pmd_t pmde;
 
        /* The only possible pmd mapping has been handled on last iteration */
        if (pvmw->pmd && !pvmw->pte)
@@ -138,7 +128,13 @@ restart:
        if (!pud_present(*pud))
                return false;
        pvmw->pmd = pmd_offset(pud, pvmw->address);
-       if (pmd_trans_huge(*pvmw->pmd)) {
+       /*
+        * Make sure the pmd value isn't cached in a register by the
+        * compiler and used as a stale value after we've observed a
+        * subsequent update.
+        */
+       pmde = READ_ONCE(*pvmw->pmd);
+       if (pmd_trans_huge(pmde)) {
                pvmw->ptl = pmd_lock(mm, pvmw->pmd);
                if (!pmd_present(*pvmw->pmd))
                        return not_found(pvmw);
@@ -153,9 +149,8 @@ restart:
                        spin_unlock(pvmw->ptl);
                        pvmw->ptl = NULL;
                }
-       } else {
-               if (!check_pmd(pvmw))
-                       return false;
+       } else if (!pmd_present(pmde)) {
+               return false;
        }
        if (!map_pte(pvmw))
                goto next_pte;