mm: hugetlb: introduce page_huge_active

[mirror_ubuntu-zesty-kernel.git] / mm / hugetlb.c

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 995c8d65a95c019d9b86719cd5e0935144b5cc05..05407831016baf550e3527acf996125c26133a43 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -924,6 +924,31 @@ struct hstate *size_to_hstate(unsigned long size)
        return NULL;
 }
 
+/*
+ * Test to determine whether the hugepage is "active/in-use" (i.e. being linked
+ * to hstate->hugepage_activelist.)
+ *
+ * This function can be called for tail pages, but never returns true for them.
+ */
+bool page_huge_active(struct page *page)
+{
+       VM_BUG_ON_PAGE(!PageHuge(page), page);
+       return PageHead(page) && PagePrivate(&page[1]);
+}
+
+/* never called for tail page */
+static void set_page_huge_active(struct page *page)
+{
+       VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
+       SetPagePrivate(&page[1]);
+}
+
+static void clear_page_huge_active(struct page *page)
+{
+       VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
+       ClearPagePrivate(&page[1]);
+}
+
 void free_huge_page(struct page *page)
 {
        /*
@@ -952,6 +977,7 @@ void free_huge_page(struct page *page)
                restore_reserve = true;
 
        spin_lock(&hugetlb_lock);
+       clear_page_huge_active(page);
        hugetlb_cgroup_uncharge_page(hstate_index(h),
                                     pages_per_huge_page(h), page);
        if (restore_reserve)
@@ -2972,6 +2998,7 @@ retry_avoidcopy:
        copy_user_huge_page(new_page, old_page, address, vma,
                            pages_per_huge_page(h));
        __SetPageUptodate(new_page);
+       set_page_huge_active(new_page);
 
        mmun_start = address & huge_page_mask(h);
        mmun_end = mmun_start + huge_page_size(h);
@@ -3084,6 +3111,7 @@ retry:
                }
                clear_huge_page(page, address, pages_per_huge_page(h));
                __SetPageUptodate(page);
+               set_page_huge_active(page);
 
                if (vma->vm_flags & VM_MAYSHARE) {
                        int err;
@@ -3913,19 +3941,26 @@ int dequeue_hwpoisoned_huge_page(struct page *hpage)
 
 bool isolate_huge_page(struct page *page, struct list_head *list)
 {
+       bool ret = true;
+
        VM_BUG_ON_PAGE(!PageHead(page), page);
-       if (!get_page_unless_zero(page))
-               return false;
        spin_lock(&hugetlb_lock);
+       if (!page_huge_active(page) || !get_page_unless_zero(page)) {
+               ret = false;
+               goto unlock;
+       }
+       clear_page_huge_active(page);
        list_move_tail(&page->lru, list);
+unlock:
        spin_unlock(&hugetlb_lock);
-       return true;
+       return ret;
 }
 
 void putback_active_hugepage(struct page *page)
 {
        VM_BUG_ON_PAGE(!PageHead(page), page);
        spin_lock(&hugetlb_lock);
+       set_page_huge_active(page);
        list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
        spin_unlock(&hugetlb_lock);
        put_page(page);