sparc64: add support for folded p4d page tables

Implement primitives necessary for the 4th level folding, add walks of p4d
level where appropriate and replace 5level-fixup.h with pgtable-nop4d.h.

Signed-off-by: Mike Rapoport <rppt@linux.ibm.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h
index 48abccba49915f18dbdade44ccefb778a91404b9..b0d90a1774a2c3995bbdb62b0b3105e53a7c0924 100644 (file)
--- a/arch/sparc/include/asm/pgalloc_64.h
+++ b/arch/sparc/include/asm/pgalloc_64.h
@@ -16,12 +16,12 @@
 
 extern struct kmem_cache *pgtable_cache;
 
-static inline void __pgd_populate(pgd_t *pgd, pud_t *pud)
+static inline void __p4d_populate(p4d_t *p4d, pud_t *pud)
 {
-       pgd_set(pgd, pud);
+       p4d_set(p4d, pud);
 }
 
-#define pgd_populate(MM, PGD, PUD)     __pgd_populate(PGD, PUD)
+#define p4d_populate(MM, P4D, PUD)     __p4d_populate(P4D, PUD)
 
 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 {

diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 1599de7305327cbe506504e229232c4dc1f50aeb..1450e113a7d220ed94a5055415a5d9f7352376d3 100644 (file)
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -13,7 +13,7 @@
  * the SpitFire page tables.
  */
 
-#include <asm-generic/5level-fixup.h>
+#include <asm-generic/pgtable-nop4d.h>
 #include <linux/compiler.h>
 #include <linux/const.h>
 #include <asm/types.h>
@@ -810,9 +810,9 @@ static inline int pmd_present(pmd_t pmd)
 
 #define pud_bad(pud)                   (pud_val(pud) & ~PAGE_MASK)
 
-#define pgd_none(pgd)                  (!pgd_val(pgd))
+#define p4d_none(p4d)                  (!p4d_val(p4d))
 
-#define pgd_bad(pgd)                   (pgd_val(pgd) & ~PAGE_MASK)
+#define p4d_bad(p4d)                   (p4d_val(p4d) & ~PAGE_MASK)
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 void set_pmd_at(struct mm_struct *mm, unsigned long addr,
@@ -859,13 +859,13 @@ static inline unsigned long pud_page_vaddr(pud_t pud)
 #define pmd_clear(pmdp)                        (pmd_val(*(pmdp)) = 0UL)
 #define pud_present(pud)               (pud_val(pud) != 0U)
 #define pud_clear(pudp)                        (pud_val(*(pudp)) = 0UL)
-#define pgd_page_vaddr(pgd)            \
-       ((unsigned long) __va(pgd_val(pgd)))
-#define pgd_present(pgd)               (pgd_val(pgd) != 0U)
-#define pgd_clear(pgdp)                        (pgd_val(*(pgdp)) = 0UL)
+#define p4d_page_vaddr(p4d)            \
+       ((unsigned long) __va(p4d_val(p4d)))
+#define p4d_present(p4d)               (p4d_val(p4d) != 0U)
+#define p4d_clear(p4dp)                        (p4d_val(*(p4dp)) = 0UL)
 
 /* only used by the stubbed out hugetlb gup code, should never be called */
-#define pgd_page(pgd)                  NULL
+#define p4d_page(p4d)                  NULL
 
 static inline unsigned long pud_large(pud_t pud)
 {
@@ -884,8 +884,8 @@ static inline unsigned long pud_pfn(pud_t pud)
 /* Same in both SUN4V and SUN4U.  */
 #define pte_none(pte)                  (!pte_val(pte))
 
-#define pgd_set(pgdp, pudp)    \
-       (pgd_val(*(pgdp)) = (__pa((unsigned long) (pudp))))
+#define p4d_set(p4dp, pudp)    \
+       (p4d_val(*(p4dp)) = (__pa((unsigned long) (pudp))))
 
 /* to find an entry in a page-table-directory. */
 #define pgd_index(address)     (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
@@ -896,8 +896,8 @@ static inline unsigned long pud_pfn(pud_t pud)
 
 /* Find an entry in the third-level page table.. */
 #define pud_index(address)     (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
-#define pud_offset(pgdp, address)      \
-       ((pud_t *) pgd_page_vaddr(*(pgdp)) + pud_index(address))
+#define pud_offset(p4dp, address)      \
+       ((pud_t *) p4d_page_vaddr(*(p4dp)) + pud_index(address))
 
 /* Find an entry in the second-level page table.. */
 #define pmd_offset(pudp, address)      \

diff --git a/arch/sparc/kernel/signal32.c b/arch/sparc/kernel/signal32.c
index a237810aa9f4fd04c0fac83831bf47350b0511b3..2a734ecd0a4018636fb4f7b2770d4d32d0783d49 100644 (file)
--- a/arch/sparc/kernel/signal32.c
+++ b/arch/sparc/kernel/signal32.c
@@ -299,6 +299,7 @@ static void flush_signal_insns(unsigned long address)
        unsigned long pstate, paddr;
        pte_t *ptep, pte;
        pgd_t *pgdp;
+       p4d_t *p4dp;
        pud_t *pudp;
        pmd_t *pmdp;
 
@@ -318,7 +319,10 @@ static void flush_signal_insns(unsigned long address)
        pgdp = pgd_offset(current->mm, address);
        if (pgd_none(*pgdp))
                goto out_irqs_on;
-       pudp = pud_offset(pgdp, address);
+       p4dp = p4d_offset(pgdp, address);
+       if (p4d_none(*p4dp))
+               goto out_irqs_on;
+       pudp = pud_offset(p4dp, address);
        if (pud_none(*pudp))
                goto out_irqs_on;
        pmdp = pmd_offset(pudp, address);

diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c
index a8275fea4b70c02019718fb0dfe93df1f6e12e6d..eb94557983383e8d2f8d1b0ee17a14cd64d9cc2f 100644 (file)
--- a/arch/sparc/kernel/smp_64.c
+++ b/arch/sparc/kernel/smp_64.c
@@ -1621,6 +1621,7 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
 static void __init pcpu_populate_pte(unsigned long addr)
 {
        pgd_t *pgd = pgd_offset_k(addr);
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
 
@@ -1633,7 +1634,17 @@ static void __init pcpu_populate_pte(unsigned long addr)
                pgd_populate(&init_mm, pgd, new);
        }
 
-       pud = pud_offset(pgd, addr);
+       p4d = p4d_offset(pgd, addr);
+       if (p4d_none(*p4d)) {
+               pud_t *new;
+
+               new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
+               if (!new)
+                       goto err_alloc;
+               p4d_populate(&init_mm, p4d, new);
+       }
+
+       pud = pud_offset(p4d, addr);
        if (pud_none(*pud)) {
                pmd_t *new;
 

diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c
index 2371fb6b97e4dd6e463349fb55583ee3cafdf43e..8b7ddbd14b658c4377bd8c61c5fc0c0abced38b2 100644 (file)
--- a/arch/sparc/mm/fault_64.c
+++ b/arch/sparc/mm/fault_64.c
@@ -80,6 +80,7 @@ static void __kprobes bad_kernel_pc(struct pt_regs *regs, unsigned long vaddr)
 static unsigned int get_user_insn(unsigned long tpc)
 {
        pgd_t *pgdp = pgd_offset(current->mm, tpc);
+       p4d_t *p4dp;
        pud_t *pudp;
        pmd_t *pmdp;
        pte_t *ptep, pte;
@@ -88,7 +89,10 @@ static unsigned int get_user_insn(unsigned long tpc)
 
        if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp)))
                goto out;
-       pudp = pud_offset(pgdp, tpc);
+       p4dp = p4d_offset(pgdp, tpc);
+       if (p4d_none(*p4dp) || unlikely(p4d_bad(*p4dp)))
+               goto out;
+       pudp = pud_offset(p4dp, tpc);
        if (pud_none(*pudp) || unlikely(pud_bad(*pudp)))
                goto out;
 

diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
index f78793a06bbd22af4f74422f4c80040e09f0813b..7b9fa861b67c270adcea581c8a52144af205330b 100644 (file)
--- a/arch/sparc/mm/hugetlbpage.c
+++ b/arch/sparc/mm/hugetlbpage.c
@@ -277,11 +277,13 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
                        unsigned long addr, unsigned long sz)
 {
        pgd_t *pgd;
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
 
        pgd = pgd_offset(mm, addr);
-       pud = pud_alloc(mm, pgd, addr);
+       p4d = p4d_offset(pgd, addr);
+       pud = pud_alloc(mm, p4d, addr);
        if (!pud)
                return NULL;
        if (sz >= PUD_SIZE)
@@ -298,13 +300,17 @@ pte_t *huge_pte_offset(struct mm_struct *mm,
                       unsigned long addr, unsigned long sz)
 {
        pgd_t *pgd;
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
 
        pgd = pgd_offset(mm, addr);
        if (pgd_none(*pgd))
                return NULL;
-       pud = pud_offset(pgd, addr);
+       p4d = p4d_offset(pgd, addr);
+       if (p4d_none(*p4d))
+               return NULL;
+       pud = pud_offset(p4d, addr);
        if (pud_none(*pud))
                return NULL;
        if (is_hugetlb_pud(*pud))
@@ -449,7 +455,7 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
        mm_dec_nr_pmds(tlb->mm);
 }
 
-static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
+static void hugetlb_free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
                                   unsigned long addr, unsigned long end,
                                   unsigned long floor, unsigned long ceiling)
 {
@@ -458,7 +464,7 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
        unsigned long start;
 
        start = addr;
-       pud = pud_offset(pgd, addr);
+       pud = pud_offset(p4d, addr);
        do {
                next = pud_addr_end(addr, end);
                if (pud_none_or_clear_bad(pud))
@@ -481,8 +487,8 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
        if (end - 1 > ceiling - 1)
                return;
 
-       pud = pud_offset(pgd, start);
-       pgd_clear(pgd);
+       pud = pud_offset(p4d, start);
+       p4d_clear(p4d);
        pud_free_tlb(tlb, pud, start);
        mm_dec_nr_puds(tlb->mm);
 }
@@ -492,6 +498,7 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb,
                            unsigned long floor, unsigned long ceiling)
 {
        pgd_t *pgd;
+       p4d_t *p4d;
        unsigned long next;
 
        addr &= PMD_MASK;
@@ -511,10 +518,11 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb,
                return;
 
        pgd = pgd_offset(tlb->mm, addr);
+       p4d = p4d_offset(pgd, addr);
        do {
-               next = pgd_addr_end(addr, end);
-               if (pgd_none_or_clear_bad(pgd))
+               next = p4d_addr_end(addr, end);
+               if (p4d_none_or_clear_bad(p4d))
                        continue;
-               hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
-       } while (pgd++, addr = next, addr != end);
+               hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling);
+       } while (p4d++, addr = next, addr != end);
 }

diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 4b099dd7a7671566054c3d2c2c51a0ddc7da2b76..4f02c6c95ac5512c562066628137d60758d76e49 100644 (file)
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -530,7 +530,8 @@ void __kprobes flush_icache_range(unsigned long start, unsigned long end)
                                paddr = kaddr & mask;
                        else {
                                pgd_t *pgdp = pgd_offset_k(kaddr);
-                               pud_t *pudp = pud_offset(pgdp, kaddr);
+                               p4d_t *p4dp = p4d_offset(pgdp, kaddr);
+                               pud_t *pudp = pud_offset(p4dp, kaddr);
                                pmd_t *pmdp = pmd_offset(pudp, kaddr);
                                pte_t *ptep = pte_offset_kernel(pmdp, kaddr);
 
@@ -1653,6 +1654,7 @@ static unsigned long max_phys_bits = 40;
 bool kern_addr_valid(unsigned long addr)
 {
        pgd_t *pgd;
+       p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *pte;
@@ -1674,7 +1676,11 @@ bool kern_addr_valid(unsigned long addr)
        if (pgd_none(*pgd))
                return 0;
 
-       pud = pud_offset(pgd, addr);
+       p4d = p4d_offset(pgd, addr);
+       if (p4d_none(*p4d))
+               return 0;
+
+       pud = pud_offset(p4d, addr);
        if (pud_none(*pud))
                return 0;
 
@@ -1800,6 +1806,7 @@ static unsigned long __ref kernel_map_range(unsigned long pstart,
        while (vstart < vend) {
                unsigned long this_end, paddr = __pa(vstart);
                pgd_t *pgd = pgd_offset_k(vstart);
+               p4d_t *p4d;
                pud_t *pud;
                pmd_t *pmd;
                pte_t *pte;
@@ -1814,7 +1821,20 @@ static unsigned long __ref kernel_map_range(unsigned long pstart,
                        alloc_bytes += PAGE_SIZE;
                        pgd_populate(&init_mm, pgd, new);
                }
-               pud = pud_offset(pgd, vstart);
+
+               p4d = p4d_offset(pgd, vstart);
+               if (p4d_none(*p4d)) {
+                       pud_t *new;
+
+                       new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE,
+                                                 PAGE_SIZE);
+                       if (!new)
+                               goto err_alloc;
+                       alloc_bytes += PAGE_SIZE;
+                       p4d_populate(&init_mm, p4d, new);
+               }
+
+               pud = pud_offset(p4d, vstart);
                if (pud_none(*pud)) {
                        pmd_t *new;
 
@@ -2612,13 +2632,18 @@ int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
        for (; vstart < vend; vstart += PMD_SIZE) {
                pgd_t *pgd = vmemmap_pgd_populate(vstart, node);
                unsigned long pte;
+               p4d_t *p4d;
                pud_t *pud;
                pmd_t *pmd;
 
                if (!pgd)
                        return -ENOMEM;
 
-               pud = vmemmap_pud_populate(pgd, vstart, node);
+               p4d = vmemmap_p4d_populate(pgd, vstart, node);
+               if (!p4d)
+                       return -ENOMEM;
+
+               pud = vmemmap_pud_populate(p4d, vstart, node);
                if (!pud)
                        return -ENOMEM;