mm/follow_page_mask: add support for hugepage directory entry

[mirror_ubuntu-artful-kernel.git] / mm / gup.c

diff --git a/mm/gup.c b/mm/gup.c
index fe95a37a417256be4463105832614fbd892106ec..9e472cb835b5195af0cfe208a040ec864c8202e1 100644 (file)
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -226,6 +226,14 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma,
                        return page;
                return no_page_table(vma, flags);
        }
+       if (is_hugepd(__hugepd(pmd_val(*pmd)))) {
+               page = follow_huge_pd(vma, address,
+                                     __hugepd(pmd_val(*pmd)), flags,
+                                     PMD_SHIFT);
+               if (page)
+                       return page;
+               return no_page_table(vma, flags);
+       }
        if (pmd_devmap(*pmd)) {
                ptl = pmd_lock(mm, pmd);
                page = follow_devmap_pmd(vma, address, pmd, flags);
@@ -292,6 +300,14 @@ static struct page *follow_pud_mask(struct vm_area_struct *vma,
                        return page;
                return no_page_table(vma, flags);
        }
+       if (is_hugepd(__hugepd(pud_val(*pud)))) {
+               page = follow_huge_pd(vma, address,
+                                     __hugepd(pud_val(*pud)), flags,
+                                     PUD_SHIFT);
+               if (page)
+                       return page;
+               return no_page_table(vma, flags);
+       }
        if (pud_devmap(*pud)) {
                ptl = pud_lock(mm, pud);
                page = follow_devmap_pud(vma, address, pud, flags);
@@ -311,6 +327,7 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma,
                                    unsigned int flags, unsigned int *page_mask)
 {
        p4d_t *p4d;
+       struct page *page;
 
        p4d = p4d_offset(pgdp, address);
        if (p4d_none(*p4d))
@@ -319,6 +336,14 @@ static struct page *follow_p4d_mask(struct vm_area_struct *vma,
        if (unlikely(p4d_bad(*p4d)))
                return no_page_table(vma, flags);
 
+       if (is_hugepd(__hugepd(p4d_val(*p4d)))) {
+               page = follow_huge_pd(vma, address,
+                                     __hugepd(p4d_val(*p4d)), flags,
+                                     P4D_SHIFT);
+               if (page)
+                       return page;
+               return no_page_table(vma, flags);
+       }
        return follow_pud_mask(vma, address, p4d, flags, page_mask);
 }
 
@@ -363,6 +388,14 @@ struct page *follow_page_mask(struct vm_area_struct *vma,
                        return page;
                return no_page_table(vma, flags);
        }
+       if (is_hugepd(__hugepd(pgd_val(*pgd)))) {
+               page = follow_huge_pd(vma, address,
+                                     __hugepd(pgd_val(*pgd)), flags,
+                                     PGDIR_SHIFT);
+               if (page)
+                       return page;
+               return no_page_table(vma, flags);
+       }
 
        return follow_p4d_mask(vma, address, pgd, flags, page_mask);
 }