mm: gup: use get_user_pages_unlocked within get_user_pages_fast

This allows the get_user_pages_fast slow path to release the mmap_sem
before blocking.

Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Reviewed-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andres Lagar-Cavilla <andreslc@google.com>
Cc: Peter Feiner <pfeiner@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/mips/mm/gup.c b/arch/mips/mm/gup.c
index 70795a67a27622df94a6dcac714e930d7bf85557..349995d19c7f2c85ee1eeb83d7882d9558d96e03 100644 (file)
--- a/arch/mips/mm/gup.c
+++ b/arch/mips/mm/gup.c
@@ -301,11 +301,9 @@ slow_irqon:
        start += nr << PAGE_SHIFT;
        pages += nr;
 
-       down_read(&mm->mmap_sem);
-       ret = get_user_pages(current, mm, start,
-                               (end - start) >> PAGE_SHIFT,
-                               write, 0, pages, NULL);
-       up_read(&mm->mmap_sem);
+       ret = get_user_pages_unlocked(current, mm, start,
+                                     (end - start) >> PAGE_SHIFT,
+                                     write, 0, pages);
 
        /* Have to be a bit careful with return values */
        if (nr > 0) {

diff --git a/arch/s390/mm/gup.c b/arch/s390/mm/gup.c
index 639fce464008854cf3d23c22f6dcbe4a970add68..5c586c78ca8deb3486df1563232b34d46c6a4027 100644 (file)
--- a/arch/s390/mm/gup.c
+++ b/arch/s390/mm/gup.c
@@ -235,10 +235,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write,
        /* Try to get the remaining pages with get_user_pages */
        start += nr << PAGE_SHIFT;
        pages += nr;
-       down_read(&mm->mmap_sem);
-       ret = get_user_pages(current, mm, start,
-                            nr_pages - nr, write, 0, pages, NULL);
-       up_read(&mm->mmap_sem);
+       ret = get_user_pages_unlocked(current, mm, start,
+                            nr_pages - nr, write, 0, pages);
        /* Have to be a bit careful with return values */
        if (nr > 0)
                ret = (ret < 0) ? nr : ret + nr;

diff --git a/arch/sh/mm/gup.c b/arch/sh/mm/gup.c
index 37458f38b22093d9bf1b6468577fb39de3f54013..e15f52a17b6c1c3650ccb1e2549e3e2bbc7679e1 100644 (file)
--- a/arch/sh/mm/gup.c
+++ b/arch/sh/mm/gup.c
@@ -257,10 +257,8 @@ slow_irqon:
                start += nr << PAGE_SHIFT;
                pages += nr;
 
-               down_read(&mm->mmap_sem);
-               ret = get_user_pages(current, mm, start,
-                       (end - start) >> PAGE_SHIFT, write, 0, pages, NULL);
-               up_read(&mm->mmap_sem);
+               ret = get_user_pages_unlocked(current, mm, start,
+                       (end - start) >> PAGE_SHIFT, write, 0, pages);
 
                /* Have to be a bit careful with return values */
                if (nr > 0) {

diff --git a/arch/sparc/mm/gup.c b/arch/sparc/mm/gup.c
index ae6ce383d4df6e3188cc547d365f366364f873e9..2e5c4fc2daa91efa1dd4325ca001169fc37b4d89 100644 (file)
--- a/arch/sparc/mm/gup.c
+++ b/arch/sparc/mm/gup.c
@@ -249,10 +249,8 @@ slow:
                start += nr << PAGE_SHIFT;
                pages += nr;
 
-               down_read(&mm->mmap_sem);
-               ret = get_user_pages(current, mm, start,
-                       (end - start) >> PAGE_SHIFT, write, 0, pages, NULL);
-               up_read(&mm->mmap_sem);
+               ret = get_user_pages_unlocked(current, mm, start,
+                       (end - start) >> PAGE_SHIFT, write, 0, pages);
 
                /* Have to be a bit careful with return values */
                if (nr > 0) {

diff --git a/arch/x86/mm/gup.c b/arch/x86/mm/gup.c
index 224b14235e967b62052c81eec48b2daf335f94fc..89df70e0caa6b95427bf5996a601510d34e5265b 100644 (file)
--- a/arch/x86/mm/gup.c
+++ b/arch/x86/mm/gup.c
@@ -388,10 +388,9 @@ slow_irqon:
                start += nr << PAGE_SHIFT;
                pages += nr;
 
-               down_read(&mm->mmap_sem);
-               ret = get_user_pages(current, mm, start,
-                       (end - start) >> PAGE_SHIFT, write, 0, pages, NULL);
-               up_read(&mm->mmap_sem);
+               ret = get_user_pages_unlocked(current, mm, start,
+                                             (end - start) >> PAGE_SHIFT,
+                                             write, 0, pages);
 
                /* Have to be a bit careful with return values */
                if (nr > 0) {

diff --git a/mm/gup.c b/mm/gup.c
index dad5875fb766b5e1d0f8044fce03826dcd782384..c2da1163986aa52badc10c0eabca39540e458aee 100644 (file)
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -1243,10 +1243,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write,
                start += nr << PAGE_SHIFT;
                pages += nr;
 
-               down_read(&mm->mmap_sem);
-               ret = get_user_pages(current, mm, start,
-                                    nr_pages - nr, write, 0, pages, NULL);
-               up_read(&mm->mmap_sem);
+               ret = get_user_pages_unlocked(current, mm, start,
+                                             nr_pages - nr, write, 0, pages);
 
                /* Have to be a bit careful with return values */
                if (nr > 0) {

diff --git a/mm/util.c b/mm/util.c
index fec39d4509a958763685fb6b70499590f9e363fb..f3ef639c4857e5bc954732227adeeea355f1c9a9 100644 (file)
--- a/mm/util.c
+++ b/mm/util.c
@@ -240,14 +240,8 @@ int __weak get_user_pages_fast(unsigned long start,
                                int nr_pages, int write, struct page **pages)
 {
        struct mm_struct *mm = current->mm;
-       int ret;
-
-       down_read(&mm->mmap_sem);
-       ret = get_user_pages(current, mm, start, nr_pages,
-                                       write, 0, pages, NULL);
-       up_read(&mm->mmap_sem);
-
-       return ret;
+       return get_user_pages_unlocked(current, mm, start, nr_pages,
+                                      write, 0, pages);
 }
 EXPORT_SYMBOL_GPL(get_user_pages_fast);