Merge branch 'i2c/for-current' of git://git.kernel.org/pub/scm/linux/kernel/git/wsa...

[mirror_ubuntu-bionic-kernel.git] / mm / pagewalk.c

diff --git a/mm/pagewalk.c b/mm/pagewalk.c
index 207244489a681d10f16c318c0a6ff0423b5fe30b..60f7856e508fb90e6010feadad2233f4d148341e 100644 (file)
--- a/mm/pagewalk.c
+++ b/mm/pagewalk.c
@@ -69,23 +69,41 @@ again:
        return err;
 }
 
-static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
+static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
                          struct mm_walk *walk)
 {
        pud_t *pud;
        unsigned long next;
        int err = 0;
 
-       pud = pud_offset(pgd, addr);
+       pud = pud_offset(p4d, addr);
        do {
+ again:
                next = pud_addr_end(addr, end);
-               if (pud_none_or_clear_bad(pud)) {
+               if (pud_none(*pud) || !walk->vma) {
                        if (walk->pte_hole)
                                err = walk->pte_hole(addr, next, walk);
                        if (err)
                                break;
                        continue;
                }
+
+               if (walk->pud_entry) {
+                       spinlock_t *ptl = pud_trans_huge_lock(pud, walk->vma);
+
+                       if (ptl) {
+                               err = walk->pud_entry(pud, addr, next, walk);
+                               spin_unlock(ptl);
+                               if (err)
+                                       break;
+                               continue;
+                       }
+               }
+
+               split_huge_pud(walk->vma, pud, addr);
+               if (pud_none(*pud))
+                       goto again;
+
                if (walk->pmd_entry || walk->pte_entry)
                        err = walk_pmd_range(pud, addr, next, walk);
                if (err)
@@ -95,6 +113,32 @@ static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
        return err;
 }
 
+static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
+                         struct mm_walk *walk)
+{
+       p4d_t *p4d;
+       unsigned long next;
+       int err = 0;
+
+       p4d = p4d_offset(pgd, addr);
+       do {
+               next = p4d_addr_end(addr, end);
+               if (p4d_none_or_clear_bad(p4d)) {
+                       if (walk->pte_hole)
+                               err = walk->pte_hole(addr, next, walk);
+                       if (err)
+                               break;
+                       continue;
+               }
+               if (walk->pmd_entry || walk->pte_entry)
+                       err = walk_pud_range(p4d, addr, next, walk);
+               if (err)
+                       break;
+       } while (p4d++, addr = next, addr != end);
+
+       return err;
+}
+
 static int walk_pgd_range(unsigned long addr, unsigned long end,
                          struct mm_walk *walk)
 {
@@ -113,7 +157,7 @@ static int walk_pgd_range(unsigned long addr, unsigned long end,
                        continue;
                }
                if (walk->pmd_entry || walk->pte_entry)
-                       err = walk_pud_range(pgd, addr, next, walk);
+                       err = walk_p4d_range(pgd, addr, next, walk);
                if (err)
                        break;
        } while (pgd++, addr = next, addr != end);