x86/speculation: Rework speculative_store_bypass_update()

[mirror_ubuntu-artful-kernel.git] / mm / gup.c

diff --git a/mm/gup.c b/mm/gup.c
index 23f01c40c88f63cc88ff62548a4d120b0c406a8e..4a789f1c6a2792d4842b223e2fce271b69b2e67c 100644 (file)
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -1618,6 +1618,47 @@ static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,
        return 1;
 }
 
+static void gup_pgd_range(unsigned long addr, unsigned long end,
+               int write, struct page **pages, int *nr)
+{
+       unsigned long next;
+       pgd_t *pgdp;
+
+       pgdp = pgd_offset(current->mm, addr);
+       do {
+               pgd_t pgd = READ_ONCE(*pgdp);
+
+               next = pgd_addr_end(addr, end);
+               if (pgd_none(pgd))
+                       return;
+               if (unlikely(pgd_huge(pgd))) {
+                       if (!gup_huge_pgd(pgd, pgdp, addr, next, write,
+                                         pages, nr))
+                               return;
+               } else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
+                       if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
+                                        PGDIR_SHIFT, next, write, pages, nr))
+                               return;
+               } else if (!gup_p4d_range(pgd, addr, next, write, pages, nr))
+                       return;
+       } while (pgdp++, addr = next, addr != end);
+}
+
+#ifndef gup_fast_permitted
+/*
+ * Check if it's allowed to use __get_user_pages_fast() for the range, or
+ * we need to fall back to the slow version:
+ */
+bool gup_fast_permitted(unsigned long start, int nr_pages, int write)
+{
+       unsigned long len, end;
+
+       len = (unsigned long) nr_pages << PAGE_SHIFT;
+       end = start + len;
+       return end >= start;
+}
+#endif
+
 /*
  * Like get_user_pages_fast() except it's IRQ-safe in that it won't fall back to
  * the regular GUP. It will only return non-negative values.
@@ -1625,10 +1666,8 @@ static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,
 int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
                          struct page **pages)
 {
-       struct mm_struct *mm = current->mm;
        unsigned long addr, len, end;
-       unsigned long next, flags;
-       pgd_t *pgdp;
+       unsigned long flags;
        int nr = 0;
 
        start &= PAGE_MASK;
@@ -1652,45 +1691,15 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
         * block IPIs that come from THPs splitting.
         */
 
-       local_irq_save(flags);
-       pgdp = pgd_offset(mm, addr);
-       do {
-               pgd_t pgd = READ_ONCE(*pgdp);
-
-               next = pgd_addr_end(addr, end);
-               if (pgd_none(pgd))
-                       break;
-               if (unlikely(pgd_huge(pgd))) {
-                       if (!gup_huge_pgd(pgd, pgdp, addr, next, write,
-                                         pages, &nr))
-                               break;
-               } else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
-                       if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
-                                        PGDIR_SHIFT, next, write, pages, &nr))
-                               break;
-               } else if (!gup_p4d_range(pgd, addr, next, write, pages, &nr))
-                       break;
-       } while (pgdp++, addr = next, addr != end);
-       local_irq_restore(flags);
+       if (gup_fast_permitted(start, nr_pages, write)) {
+               local_irq_save(flags);
+               gup_pgd_range(addr, end, write, pages, &nr);
+               local_irq_restore(flags);
+       }
 
        return nr;
 }
 
-#ifndef gup_fast_permitted
-/*
- * Check if it's allowed to use __get_user_pages_fast() for the range, or
- * we need to fall back to the slow version:
- */
-bool gup_fast_permitted(unsigned long start, int nr_pages, int write)
-{
-       unsigned long len, end;
-
-       len = (unsigned long) nr_pages << PAGE_SHIFT;
-       end = start + len;
-       return end >= start;
-}
-#endif
-
 /**
  * get_user_pages_fast() - pin user pages in memory
  * @start:     starting user address
@@ -1710,12 +1719,22 @@ bool gup_fast_permitted(unsigned long start, int nr_pages, int write)
 int get_user_pages_fast(unsigned long start, int nr_pages, int write,
                        struct page **pages)
 {
+       unsigned long addr, len, end;
        int nr = 0, ret = 0;
 
        start &= PAGE_MASK;
+       addr = start;
+       len = (unsigned long) nr_pages << PAGE_SHIFT;
+       end = start + len;
+
+       if (unlikely(!access_ok(write ? VERIFY_WRITE : VERIFY_READ,
+                                       (void __user *)start, len)))
+               return 0;
 
        if (gup_fast_permitted(start, nr_pages, write)) {
-               nr = __get_user_pages_fast(start, nr_pages, write, pages);
+               local_irq_disable();
+               gup_pgd_range(addr, end, write, pages, &nr);
+               local_irq_enable();
                ret = nr;
        }