powerpc/mm: use _raw variant of page table accessors

This switch few of the page table accessor to use the __raw variant
and does the cpu to big endian conversion of constants. This helps in
generating better code.

For ex: a pgd_none(pgd) check with and without fix is listed below

Without fix:
------------
   2240:	20 00 61 eb 	ld      r27,32(r1)
/* PGD level */
typedef struct { __be64 pgd; } pgd_t;
static inline unsigned long pgd_val(pgd_t x)
{
	return be64_to_cpu(x.pgd);

    2244:	22 00 66 78 	rldicl  r6,r3,32,32
    2248:	3e 40 7d 54 	rotlwi  r29,r3,8
    224c:	0e c0 7d 50 	rlwimi  r29,r3,24,0,7
    2250:	3e 40 c5 54 	rotlwi  r5,r6,8
    2254:	2e c4 7d 50 	rlwimi  r29,r3,24,16,23
    2258:	0e c0 c5 50 	rlwimi  r5,r6,24,0,7
    225c:	2e c4 c5 50 	rlwimi  r5,r6,24,16,23
    2260:	c6 07 bd 7b 	rldicr  r29,r29,32,31
    2264:	78 2b bd 7f 	or      r29,r29,r5
		if (pgd_none(pgd))
    2268:	00 00 bd 2f 	cmpdi   cr7,r29,0
    226c:	54 03 9e 41 	beq     cr7,25c0 <__get_user_pages_fast+0x500>

With fix:
---------
    2370:	20 00 61 eb 	ld      r27,32(r1)
		if (pgd_none(pgd))
    2374:	00 00 bd 2f 	cmpdi   cr7,r29,0
    2378:	a8 03 9e 41 	beq     cr7,2720 <__get_user_pages_fast+0x530>
			break;
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Acked-by: Balbir Singh <bsingharora@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>

diff --git a/arch/powerpc/include/asm/book3s/64/pgtable-4k.h b/arch/powerpc/include/asm/book3s/64/pgtable-4k.h
index 71e9abced493767bcd44bca1a4fa249c044be97a..9db83b4e017da3085b3daa88a30fcd4ca75f6905 100644 (file)
--- a/arch/powerpc/include/asm/book3s/64/pgtable-4k.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable-4k.h
@@ -11,7 +11,7 @@ static inline int pmd_huge(pmd_t pmd)
         * leaf pte for huge page
         */
        if (radix_enabled())
-               return !!(pmd_val(pmd) & _PAGE_PTE);
+               return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE));
        return 0;
 }
 
@@ -21,7 +21,7 @@ static inline int pud_huge(pud_t pud)
         * leaf pte for huge page
         */
        if (radix_enabled())
-               return !!(pud_val(pud) & _PAGE_PTE);
+               return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE));
        return 0;
 }
 
@@ -31,7 +31,7 @@ static inline int pgd_huge(pgd_t pgd)
         * leaf pte for huge page
         */
        if (radix_enabled())
-               return !!(pgd_val(pgd) & _PAGE_PTE);
+               return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PTE));
        return 0;
 }
 #define pgd_huge pgd_huge

diff --git a/arch/powerpc/include/asm/book3s/64/pgtable-64k.h b/arch/powerpc/include/asm/book3s/64/pgtable-64k.h
index cb2d0a5fa3f8b230b27637a6908091e8814ec166..0d2845b447633cf5f0842bc769c8b63236c88792 100644 (file)
--- a/arch/powerpc/include/asm/book3s/64/pgtable-64k.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable-64k.h
@@ -15,7 +15,7 @@ static inline int pmd_huge(pmd_t pmd)
        /*
         * leaf pte for huge page
         */
-       return !!(pmd_val(pmd) & _PAGE_PTE);
+       return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE));
 }
 
 static inline int pud_huge(pud_t pud)
@@ -23,7 +23,7 @@ static inline int pud_huge(pud_t pud)
        /*
         * leaf pte for huge page
         */
-       return !!(pud_val(pud) & _PAGE_PTE);
+       return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE));
 }
 
 static inline int pgd_huge(pgd_t pgd)
@@ -31,7 +31,7 @@ static inline int pgd_huge(pgd_t pgd)
        /*
         * leaf pte for huge page
         */
-       return !!(pgd_val(pgd) & _PAGE_PTE);
+       return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PTE));
 }
 #define pgd_huge pgd_huge
 

diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h
index ab84c89c9e982129c3cde79792548c6078d42065..263bf39ced406e9b8a64f5c416924783fc36b143 100644 (file)
--- a/arch/powerpc/include/asm/book3s/64/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable.h
@@ -318,7 +318,7 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
 {
        unsigned long old;
 
-       if ((pte_val(*ptep) & (_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0)
+       if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0)
                return 0;
        old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0);
        return (old & _PAGE_ACCESSED) != 0;
@@ -336,8 +336,7 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
 static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
                                      pte_t *ptep)
 {
-
-       if ((pte_val(*ptep) & _PAGE_WRITE) == 0)
+       if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_WRITE)) == 0)
                return;
 
        pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 0);
@@ -346,7 +345,7 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
 static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
                                           unsigned long addr, pte_t *ptep)
 {
-       if ((pte_val(*ptep) & _PAGE_WRITE) == 0)
+       if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_WRITE)) == 0)
                return;
 
        pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 1);
@@ -365,17 +364,35 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
 {
        pte_update(mm, addr, ptep, ~0UL, 0, 0);
 }
-static inline int pte_write(pte_t pte)         { return !!(pte_val(pte) & _PAGE_WRITE);}
-static inline int pte_dirty(pte_t pte)         { return !!(pte_val(pte) & _PAGE_DIRTY); }
-static inline int pte_young(pte_t pte)         { return !!(pte_val(pte) & _PAGE_ACCESSED); }
-static inline int pte_special(pte_t pte)       { return !!(pte_val(pte) & _PAGE_SPECIAL); }
+
+static inline int pte_write(pte_t pte)
+{
+       return !!(pte_raw(pte) & cpu_to_be64(_PAGE_WRITE));
+}
+
+static inline int pte_dirty(pte_t pte)
+{
+       return !!(pte_raw(pte) & cpu_to_be64(_PAGE_DIRTY));
+}
+
+static inline int pte_young(pte_t pte)
+{
+       return !!(pte_raw(pte) & cpu_to_be64(_PAGE_ACCESSED));
+}
+
+static inline int pte_special(pte_t pte)
+{
+       return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SPECIAL));
+}
+
 static inline pgprot_t pte_pgprot(pte_t pte)   { return __pgprot(pte_val(pte) & PAGE_PROT_BITS); }
 
 #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
 static inline bool pte_soft_dirty(pte_t pte)
 {
-       return !!(pte_val(pte) & _PAGE_SOFT_DIRTY);
+       return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SOFT_DIRTY));
 }
+
 static inline pte_t pte_mksoft_dirty(pte_t pte)
 {
        return __pte(pte_val(pte) | _PAGE_SOFT_DIRTY);
@@ -395,14 +412,14 @@ static inline pte_t pte_clear_soft_dirty(pte_t pte)
  */
 static inline int pte_protnone(pte_t pte)
 {
-       return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PRIVILEGED)) ==
-               (_PAGE_PRESENT | _PAGE_PRIVILEGED);
+       return (pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_PRIVILEGED)) ==
+               cpu_to_be64(_PAGE_PRESENT | _PAGE_PRIVILEGED);
 }
 #endif /* CONFIG_NUMA_BALANCING */
 
 static inline int pte_present(pte_t pte)
 {
-       return !!(pte_val(pte) & _PAGE_PRESENT);
+       return !!(pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT));
 }
 /*
  * Conversion functions: convert a page and protection to a page entry,
@@ -474,7 +491,7 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 
 static inline bool pte_user(pte_t pte)
 {
-       return !(pte_val(pte) & _PAGE_PRIVILEGED);
+       return !(pte_raw(pte) & cpu_to_be64(_PAGE_PRIVILEGED));
 }
 
 /* Encode and de-code a swap entry */
@@ -517,10 +534,12 @@ static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
 {
        return __pte(pte_val(pte) | _PAGE_SWP_SOFT_DIRTY);
 }
+
 static inline bool pte_swp_soft_dirty(pte_t pte)
 {
-       return !!(pte_val(pte) & _PAGE_SWP_SOFT_DIRTY);
+       return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SWP_SOFT_DIRTY));
 }
+
 static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
 {
        return __pte(pte_val(pte) & ~_PAGE_SWP_SOFT_DIRTY);
@@ -626,8 +645,16 @@ static inline void pmd_clear(pmd_t *pmdp)
        *pmdp = __pmd(0);
 }
 
-#define pmd_none(pmd)          (!pmd_val(pmd))
-#define        pmd_present(pmd)        (!pmd_none(pmd))
+static inline int pmd_none(pmd_t pmd)
+{
+       return !pmd_raw(pmd);
+}
+
+static inline int pmd_present(pmd_t pmd)
+{
+
+       return !pmd_none(pmd);
+}
 
 static inline int pmd_bad(pmd_t pmd)
 {
@@ -646,19 +673,26 @@ static inline void pud_clear(pud_t *pudp)
        *pudp = __pud(0);
 }
 
-#define pud_none(pud)          (!pud_val(pud))
-#define pud_present(pud)       (pud_val(pud) != 0)
+static inline int pud_none(pud_t pud)
+{
+       return !pud_raw(pud);
+}
+
+static inline int pud_present(pud_t pud)
+{
+       return !pud_none(pud);
+}
 
 extern struct page *pud_page(pud_t pud);
 extern struct page *pmd_page(pmd_t pmd);
 static inline pte_t pud_pte(pud_t pud)
 {
-       return __pte(pud_val(pud));
+       return __pte_raw(pud_raw(pud));
 }
 
 static inline pud_t pte_pud(pte_t pte)
 {
-       return __pud(pte_val(pte));
+       return __pud_raw(pte_raw(pte));
 }
 #define pud_write(pud)         pte_write(pud_pte(pud))
 
@@ -681,17 +715,24 @@ static inline void pgd_clear(pgd_t *pgdp)
        *pgdp = __pgd(0);
 }
 
-#define pgd_none(pgd)          (!pgd_val(pgd))
-#define pgd_present(pgd)       (!pgd_none(pgd))
+static inline int pgd_none(pgd_t pgd)
+{
+       return !pgd_raw(pgd);
+}
+
+static inline int pgd_present(pgd_t pgd)
+{
+       return !pgd_none(pgd);
+}
 
 static inline pte_t pgd_pte(pgd_t pgd)
 {
-       return __pte(pgd_val(pgd));
+       return __pte_raw(pgd_raw(pgd));
 }
 
 static inline pgd_t pte_pgd(pte_t pte)
 {
-       return __pgd(pte_val(pte));
+       return __pgd_raw(pte_raw(pte));
 }
 
 static inline int pgd_bad(pgd_t pgd)
@@ -783,12 +824,12 @@ struct page *realmode_pfn_to_page(unsigned long pfn);
 
 static inline pte_t pmd_pte(pmd_t pmd)
 {
-       return __pte(pmd_val(pmd));
+       return __pte_raw(pmd_raw(pmd));
 }
 
 static inline pmd_t pte_pmd(pte_t pte)
 {
-       return __pmd(pte_val(pte));
+       return __pmd_raw(pte_raw(pte));
 }
 
 static inline pte_t *pmdp_ptep(pmd_t *pmd)
@@ -849,7 +890,7 @@ pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp,
 
 static inline int pmd_large(pmd_t pmd)
 {
-       return !!(pmd_val(pmd) & _PAGE_PTE);
+       return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE));
 }
 
 static inline pmd_t pmd_mknotpresent(pmd_t pmd)
@@ -865,7 +906,7 @@ static inline int __pmdp_test_and_clear_young(struct mm_struct *mm,
 {
        unsigned long old;
 
-       if ((pmd_val(*pmdp) & (_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0)
+       if ((pmd_raw(*pmdp) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0)
                return 0;
        old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED, 0);
        return ((old & _PAGE_ACCESSED) != 0);
@@ -876,7 +917,7 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr,
                                      pmd_t *pmdp)
 {
 
-       if ((pmd_val(*pmdp) & _PAGE_WRITE) == 0)
+       if ((pmd_raw(*pmdp) & cpu_to_be64(_PAGE_WRITE)) == 0)
                return;
 
        pmd_hugepage_update(mm, addr, pmdp, _PAGE_WRITE, 0);

diff --git a/arch/powerpc/include/asm/pgtable-be-types.h b/arch/powerpc/include/asm/pgtable-be-types.h
index e2bf208605b16a0de7e976970570caca970d6e5b..49c0a5a80efa2948764e247d3cbd76fc6fdf06a0 100644 (file)
--- a/arch/powerpc/include/asm/pgtable-be-types.h
+++ b/arch/powerpc/include/asm/pgtable-be-types.h
@@ -6,6 +6,7 @@
 /* PTE level */
 typedef struct { __be64 pte; } pte_t;
 #define __pte(x)       ((pte_t) { cpu_to_be64(x) })
+#define __pte_raw(x)   ((pte_t) { (x) })
 static inline unsigned long pte_val(pte_t x)
 {
        return be64_to_cpu(x.pte);
@@ -20,6 +21,7 @@ static inline __be64 pte_raw(pte_t x)
 #ifdef CONFIG_PPC64
 typedef struct { __be64 pmd; } pmd_t;
 #define __pmd(x)       ((pmd_t) { cpu_to_be64(x) })
+#define __pmd_raw(x)   ((pmd_t) { (x) })
 static inline unsigned long pmd_val(pmd_t x)
 {
        return be64_to_cpu(x.pmd);
@@ -37,21 +39,34 @@ static inline __be64 pmd_raw(pmd_t x)
 #if defined(CONFIG_PPC_BOOK3S_64) || !defined(CONFIG_PPC_64K_PAGES)
 typedef struct { __be64 pud; } pud_t;
 #define __pud(x)       ((pud_t) { cpu_to_be64(x) })
+#define __pud_raw(x)   ((pud_t) { (x) })
 static inline unsigned long pud_val(pud_t x)
 {
        return be64_to_cpu(x.pud);
 }
+
+static inline __be64 pud_raw(pud_t x)
+{
+       return x.pud;
+}
+
 #endif /* CONFIG_PPC_BOOK3S_64 || !CONFIG_PPC_64K_PAGES */
 #endif /* CONFIG_PPC64 */
 
 /* PGD level */
 typedef struct { __be64 pgd; } pgd_t;
 #define __pgd(x)       ((pgd_t) { cpu_to_be64(x) })
+#define __pgd_raw(x)   ((pgd_t) { (x) })
 static inline unsigned long pgd_val(pgd_t x)
 {
        return be64_to_cpu(x.pgd);
 }
 
+static inline __be64 pgd_raw(pgd_t x)
+{
+       return x.pgd;
+}
+
 /* Page protection bits */
 typedef struct { unsigned long pgprot; } pgprot_t;
 #define pgprot_val(x)  ((x).pgprot)