x86/mm: provide pmdp_establish() helper

[mirror_ubuntu-bionic-kernel.git] / arch / x86 / include / asm / pgtable-3level.h

diff --git a/arch/x86/include/asm/pgtable-3level.h b/arch/x86/include/asm/pgtable-3level.h
index bc4af5453802559af4c1f14a2e3a3dbaf96e57dc..a564084c6141d42f603b257160a34685e1bffdbe 100644 (file)
--- a/arch/x86/include/asm/pgtable-3level.h
+++ b/arch/x86/include/asm/pgtable-3level.h
@@ -98,6 +98,9 @@ static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
 
 static inline void native_set_pud(pud_t *pudp, pud_t pud)
 {
+#ifdef CONFIG_PAGE_TABLE_ISOLATION
+       pud.p4d.pgd = pti_set_user_pgtbl(&pudp->p4d.pgd, pud.p4d.pgd);
+#endif
        set_64bit((unsigned long long *)(pudp), native_pud_val(pud));
 }
 
@@ -158,7 +161,6 @@ static inline pte_t native_ptep_get_and_clear(pte_t *ptep)
 #define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp)
 #endif
 
-#ifdef CONFIG_SMP
 union split_pmd {
        struct {
                u32 pmd_low;
@@ -166,6 +168,8 @@ union split_pmd {
        };
        pmd_t pmd;
 };
+
+#ifdef CONFIG_SMP
 static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp)
 {
        union split_pmd res, *orig = (union split_pmd *)pmdp;
@@ -181,6 +185,40 @@ static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp)
 #define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp)
 #endif
 
+#ifndef pmdp_establish
+#define pmdp_establish pmdp_establish
+static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
+               unsigned long address, pmd_t *pmdp, pmd_t pmd)
+{
+       pmd_t old;
+
+       /*
+        * If pmd has present bit cleared we can get away without expensive
+        * cmpxchg64: we can update pmdp half-by-half without racing with
+        * anybody.
+        */
+       if (!(pmd_val(pmd) & _PAGE_PRESENT)) {
+               union split_pmd old, new, *ptr;
+
+               ptr = (union split_pmd *)pmdp;
+
+               new.pmd = pmd;
+
+               /* xchg acts as a barrier before setting of the high bits */
+               old.pmd_low = xchg(&ptr->pmd_low, new.pmd_low);
+               old.pmd_high = ptr->pmd_high;
+               ptr->pmd_high = new.pmd_high;
+               return old.pmd;
+       }
+
+       do {
+               old = *pmdp;
+       } while (cmpxchg64(&pmdp->pmd, old.pmd, pmd.pmd) != old.pmd);
+
+       return old;
+}
+#endif
+
 #ifdef CONFIG_SMP
 union split_pud {
        struct {
@@ -194,6 +232,10 @@ static inline pud_t native_pudp_get_and_clear(pud_t *pudp)
 {
        union split_pud res, *orig = (union split_pud *)pudp;
 
+#ifdef CONFIG_PAGE_TABLE_ISOLATION
+       pti_set_user_pgtbl(&pudp->p4d.pgd, __pgd(0));
+#endif
+
        /* xchg acts as a barrier before setting of the high bits */
        res.pud_low = xchg(&orig->pud_low, 0);
        res.pud_high = orig->pud_high;
@@ -206,12 +248,43 @@ static inline pud_t native_pudp_get_and_clear(pud_t *pudp)
 #endif
 
 /* Encode and de-code a swap entry */
+#define SWP_TYPE_BITS          5
+
+#define SWP_OFFSET_FIRST_BIT   (_PAGE_BIT_PROTNONE + 1)
+
+/* We always extract/encode the offset by shifting it all the way up, and then down again */
+#define SWP_OFFSET_SHIFT       (SWP_OFFSET_FIRST_BIT + SWP_TYPE_BITS)
+
 #define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > 5)
 #define __swp_type(x)                  (((x).val) & 0x1f)
 #define __swp_offset(x)                        ((x).val >> 5)
 #define __swp_entry(type, offset)      ((swp_entry_t){(type) | (offset) << 5})
-#define __pte_to_swp_entry(pte)                ((swp_entry_t){ (pte).pte_high })
-#define __swp_entry_to_pte(x)          ((pte_t){ { .pte_high = (x).val } })
+
+/*
+ * Normally, __swp_entry() converts from arch-independent swp_entry_t to
+ * arch-dependent swp_entry_t, and __swp_entry_to_pte() just stores the result
+ * to pte. But here we have 32bit swp_entry_t and 64bit pte, and need to use the
+ * whole 64 bits. Thus, we shift the "real" arch-dependent conversion to
+ * __swp_entry_to_pte() through the following helper macro based on 64bit
+ * __swp_entry().
+ */
+#define __swp_pteval_entry(type, offset) ((pteval_t) { \
+       (~(pteval_t)(offset) << SWP_OFFSET_SHIFT >> SWP_TYPE_BITS) \
+       | ((pteval_t)(type) << (64 - SWP_TYPE_BITS)) })
+
+#define __swp_entry_to_pte(x)  ((pte_t){ .pte = \
+               __swp_pteval_entry(__swp_type(x), __swp_offset(x)) })
+/*
+ * Analogically, __pte_to_swp_entry() doesn't just extract the arch-dependent
+ * swp_entry_t, but also has to convert it from 64bit to the 32bit
+ * intermediate representation, using the following macros based on 64bit
+ * __swp_type() and __swp_offset().
+ */
+#define __pteval_swp_type(x) ((unsigned long)((x).pte >> (64 - SWP_TYPE_BITS)))
+#define __pteval_swp_offset(x) ((unsigned long)(~((x).pte) << SWP_TYPE_BITS >> SWP_OFFSET_SHIFT))
+
+#define __pte_to_swp_entry(pte)        (__swp_entry(__pteval_swp_type(pte), \
+                                            __pteval_swp_offset(pte)))
 
 #define gup_get_pte gup_get_pte
 /*
@@ -260,4 +333,6 @@ static inline pte_t gup_get_pte(pte_t *ptep)
        return pte;
 }
 
+#include <asm/pgtable-invert.h>
+
 #endif /* _ASM_X86_PGTABLE_3LEVEL_H */