mm, gup: ensure real head page is ref-counted when using hugepages

[mirror_ubuntu-artful-kernel.git] / mm / gup.c

diff --git a/mm/gup.c b/mm/gup.c
index 1e68bf21330299cd1e04aae8e28f10d082d87115..23f01c40c88f63cc88ff62548a4d120b0c406a8e 100644 (file)
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -1423,8 +1423,7 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
                return __gup_device_huge_pmd(orig, addr, end, pages, nr);
 
        refs = 0;
-       head = pmd_page(orig);
-       page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
+       page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
        do {
                pages[*nr] = page;
                (*nr)++;
@@ -1432,6 +1431,7 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
                refs++;
        } while (addr += PAGE_SIZE, addr != end);
 
+       head = compound_head(pmd_page(orig));
        if (!page_cache_add_speculative(head, refs)) {
                *nr -= refs;
                return 0;
@@ -1461,8 +1461,7 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
                return __gup_device_huge_pud(orig, addr, end, pages, nr);
 
        refs = 0;
-       head = pud_page(orig);
-       page = head + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
+       page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
        do {
                pages[*nr] = page;
                (*nr)++;
@@ -1470,6 +1469,7 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
                refs++;
        } while (addr += PAGE_SIZE, addr != end);
 
+       head = compound_head(pud_page(orig));
        if (!page_cache_add_speculative(head, refs)) {
                *nr -= refs;
                return 0;
@@ -1498,8 +1498,7 @@ static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
 
        BUILD_BUG_ON(pgd_devmap(orig));
        refs = 0;
-       head = pgd_page(orig);
-       page = head + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
+       page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
        do {
                pages[*nr] = page;
                (*nr)++;
@@ -1507,6 +1506,7 @@ static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
                refs++;
        } while (addr += PAGE_SIZE, addr != end);
 
+       head = compound_head(pgd_page(orig));
        if (!page_cache_add_speculative(head, refs)) {
                *nr -= refs;
                return 0;