hugetlb: use same fault hash key for shared and private mappings

BugLink: https://bugs.launchpad.net/bugs/1838576
commit 1b426bac66e6cc83c9f2d92b96e4e72acf43419a upstream.

hugetlb uses a fault mutex hash table to prevent page faults of the
same pages concurrently.  The key for shared and private mappings is
different.  Shared keys off address_space and file index.  Private keys
off mm and virtual address.  Consider a private mappings of a populated
hugetlbfs file.  A fault will map the page from the file and if needed
do a COW to map a writable page.

Hugetlbfs hole punch uses the fault mutex to prevent mappings of file
pages.  It uses the address_space file index key.  However, private
mappings will use a different key and could race with this code to map
the file page.  This causes problems (BUG) for the page cache remove
code as it expects the page to be unmapped.  A sample stack is:

page dumped because: VM_BUG_ON_PAGE(page_mapped(page))
kernel BUG at mm/filemap.c:169!
...
RIP: 0010:unaccount_page_cache_page+0x1b8/0x200
...
Call Trace:
__delete_from_page_cache+0x39/0x220
delete_from_page_cache+0x45/0x70
remove_inode_hugepages+0x13c/0x380
? __add_to_page_cache_locked+0x162/0x380
hugetlbfs_fallocate+0x403/0x540
? _cond_resched+0x15/0x30
? __inode_security_revalidate+0x5d/0x70
? selinux_file_permission+0x100/0x130
vfs_fallocate+0x13f/0x270
ksys_fallocate+0x3c/0x80
__x64_sys_fallocate+0x1a/0x20
do_syscall_64+0x5b/0x180
entry_SYSCALL_64_after_hwframe+0x44/0xa9

There seems to be another potential COW issue/race with this approach
of different private and shared keys as noted in commit 8382d914ebf7
("mm, hugetlb: improve page-fault scalability").

Since every hugetlb mapping (even anon and private) is actually a file
mapping, just use the address_space index key for all mappings.  This
results in potentially more hash collisions.  However, this should not
be the common case.

Link: http://lkml.kernel.org/r/20190328234704.27083-3-mike.kravetz@oracle.com
Link: http://lkml.kernel.org/r/20190412165235.t4sscoujczfhuiyt@linux-r8p5
Fixes: b5cec28d36f5 ("hugetlbfs: truncate_hugepages() takes a range of pages")
Signed-off-by: Mike Kravetz <mike.kravetz@oracle.com>
Reviewed-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Reviewed-by: Davidlohr Bueso <dbueso@suse.de>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: "Kirill A . Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Signed-off-by: Kamal Mostafa <kamal@canonical.com>
Signed-off-by: Khalid Elmously <khalid.elmously@canonical.com>

diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
index 57b7a14e8c42e509ce41ac442557d75a2d19a65f..4b9d5def37be3594a2aa3b6ca2445e7c5e3d99cd 100644 (file)
--- a/fs/hugetlbfs/inode.c
+++ b/fs/hugetlbfs/inode.c
@@ -436,9 +436,7 @@ static void remove_inode_hugepages(struct inode *inode, loff_t lstart,
                        u32 hash;
 
                        index = page->index;
-                       hash = hugetlb_fault_mutex_hash(h, current->mm,
-                                                       &pseudo_vma,
-                                                       mapping, index, 0);
+                       hash = hugetlb_fault_mutex_hash(h, mapping, index, 0);
                        mutex_lock(&hugetlb_fault_mutex_table[hash]);
 
                        /*
@@ -557,7 +555,6 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset,
        struct address_space *mapping = inode->i_mapping;
        struct hstate *h = hstate_inode(inode);
        struct vm_area_struct pseudo_vma;
-       struct mm_struct *mm = current->mm;
        loff_t hpage_size = huge_page_size(h);
        unsigned long hpage_shift = huge_page_shift(h);
        pgoff_t start, index, end;
@@ -621,8 +618,7 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset,
                addr = index * hpage_size;
 
                /* mutex taken here, fault path and hole punch */
-               hash = hugetlb_fault_mutex_hash(h, mm, &pseudo_vma, mapping,
-                                               index, addr);
+               hash = hugetlb_fault_mutex_hash(h, mapping, index, addr);
                mutex_lock(&hugetlb_fault_mutex_table[hash]);
 
                /* See if already present in mapping to avoid alloc/free */

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 7aa2de25c09cf4d330dc9e3a71ca65c24f2e0d34..50a07235032fd74491cb5e8fc9b780d202a602dd 100644 (file)
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -122,9 +122,7 @@ void putback_active_hugepage(struct page *page);
 void free_huge_page(struct page *page);
 void hugetlb_fix_reserve_counts(struct inode *inode);
 extern struct mutex *hugetlb_fault_mutex_table;
-u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
-                               struct vm_area_struct *vma,
-                               struct address_space *mapping,
+u32 hugetlb_fault_mutex_hash(struct hstate *h, struct address_space *mapping,
                                pgoff_t idx, unsigned long address);
 
 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud);

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index e3f672369f52c5e4432822a8956f61aad1bb3463..8255949949e39c6fbfb521c674e0f186e8ebb5b1 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3734,8 +3734,7 @@ retry:
                         * handling userfault.  Reacquire after handling
                         * fault to make calling code simpler.
                         */
-                       hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping,
-                                                       idx, address);
+                       hash = hugetlb_fault_mutex_hash(h, mapping, idx, address);
                        mutex_unlock(&hugetlb_fault_mutex_table[hash]);
                        ret = handle_userfault(&vmf, VM_UFFD_MISSING);
                        mutex_lock(&hugetlb_fault_mutex_table[hash]);
@@ -3847,21 +3846,14 @@ backout_unlocked:
 }
 
 #ifdef CONFIG_SMP
-u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
-                           struct vm_area_struct *vma,
-                           struct address_space *mapping,
+u32 hugetlb_fault_mutex_hash(struct hstate *h, struct address_space *mapping,
                            pgoff_t idx, unsigned long address)
 {
        unsigned long key[2];
        u32 hash;
 
-       if (vma->vm_flags & VM_SHARED) {
-               key[0] = (unsigned long) mapping;
-               key[1] = idx;
-       } else {
-               key[0] = (unsigned long) mm;
-               key[1] = address >> huge_page_shift(h);
-       }
+       key[0] = (unsigned long) mapping;
+       key[1] = idx;
 
        hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0);
 
@@ -3872,9 +3864,7 @@ u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
  * For uniprocesor systems we always use a single mutex, so just
  * return 0 and avoid the hashing overhead.
  */
-u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
-                           struct vm_area_struct *vma,
-                           struct address_space *mapping,
+u32 hugetlb_fault_mutex_hash(struct hstate *h, struct address_space *mapping,
                            pgoff_t idx, unsigned long address)
 {
        return 0;
@@ -3920,7 +3910,7 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
         * get spurious allocation failures if two CPUs race to instantiate
         * the same page in the page cache.
         */
-       hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping, idx, address);
+       hash = hugetlb_fault_mutex_hash(h, mapping, idx, address);
        mutex_lock(&hugetlb_fault_mutex_table[hash]);
 
        entry = huge_ptep_get(ptep);

diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index 5d70fdbd8bc0134acfbd603850d53788de9c6448..d3b4a78d79b6c7209b152d82d6f0bef33e042bd5 100644 (file)
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -272,8 +272,7 @@ retry:
                 */
                idx = linear_page_index(dst_vma, dst_addr);
                mapping = dst_vma->vm_file->f_mapping;
-               hash = hugetlb_fault_mutex_hash(h, dst_mm, dst_vma, mapping,
-                                                               idx, dst_addr);
+               hash = hugetlb_fault_mutex_hash(h, mapping, idx, dst_addr);
                mutex_lock(&hugetlb_fault_mutex_table[hash]);
 
                err = -ENOMEM;