patches/kernel/0050-mm-x86-mm-Fix-performance-regression-in-get_user_pag.patch

   1 From 7c5d42f31bf68647dd00ac2fef9057d113e8072d Mon Sep 17 00:00:00 2001
   2 From: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
   3 Date: Sat, 9 Sep 2017 00:56:03 +0300
   4 Subject: [PATCH 050/241] mm, x86/mm: Fix performance regression in
   5  get_user_pages_fast()
   6 MIME-Version: 1.0
   7 Content-Type: text/plain; charset=UTF-8
   8 Content-Transfer-Encoding: 8bit
   9
  10 CVE-2017-5754
  11
  12 The 0-day test bot found a performance regression that was tracked down to
  13 switching x86 to the generic get_user_pages_fast() implementation:
  14
  15   http://lkml.kernel.org/r/20170710024020.GA26389@yexl-desktop
  16
  17 The regression was caused by the fact that we now use local_irq_save() +
  18 local_irq_restore() in get_user_pages_fast() to disable interrupts.
  19 In x86 implementation local_irq_disable() + local_irq_enable() was used.
  20
  21 The fix is to make get_user_pages_fast() use local_irq_disable(),
  22 leaving local_irq_save() for __get_user_pages_fast() that can be called
  23 with interrupts disabled.
  24
  25 Numbers for pinning a gigabyte of memory, one page a time, 20 repeats:
  26
  27   Before:  Average: 14.91 ms, stddev: 0.45 ms
  28   After:   Average: 10.76 ms, stddev: 0.18 ms
  29
  30 Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
  31 Cc: Andrew Morton <akpm@linux-foundation.org>
  32 Cc: Huang Ying <ying.huang@intel.com>
  33 Cc: Jonathan Corbet <corbet@lwn.net>
  34 Cc: Linus Torvalds <torvalds@linux-foundation.org>
  35 Cc: Peter Zijlstra <peterz@infradead.org>
  36 Cc: Thomas Gleixner <tglx@linutronix.de>
  37 Cc: Thorsten Leemhuis <regressions@leemhuis.info>
  38 Cc: linux-mm@kvack.org
  39 Fixes: e585513b76f7 ("x86/mm/gup: Switch GUP to the generic get_user_page_fast() implementation")
  40 Link: http://lkml.kernel.org/r/20170908215603.9189-3-kirill.shutemov@linux.intel.com
  41 Signed-off-by: Ingo Molnar <mingo@kernel.org>
  42 (cherry picked from commit 5b65c4677a57a1d4414212f9995aa0e46a21ff80)
  43 Signed-off-by: Andy Whitcroft <apw@canonical.com>
  44 Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>
  45 (cherry picked from commit 5241f4b2c68284612e34910305f3234e4a64701b)
  46 Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
  47 ---
  48  mm/gup.c | 97 ++++++++++++++++++++++++++++++++++++++--------------------------
  49  1 file changed, 58 insertions(+), 39 deletions(-)
  50
  51 diff --git a/mm/gup.c b/mm/gup.c
  52 index 23f01c40c88f..4a789f1c6a27 100644
  53 --- a/mm/gup.c
  54 +++ b/mm/gup.c
  55 @@ -1618,6 +1618,47 @@ static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,
  56         return 1;
  57  }
  58
  59 +static void gup_pgd_range(unsigned long addr, unsigned long end,
  60 +               int write, struct page **pages, int *nr)
  61 +{
  62 +       unsigned long next;
  63 +       pgd_t *pgdp;
  64 +
  65 +       pgdp = pgd_offset(current->mm, addr);
  66 +       do {
  67 +               pgd_t pgd = READ_ONCE(*pgdp);
  68 +
  69 +               next = pgd_addr_end(addr, end);
  70 +               if (pgd_none(pgd))
  71 +                       return;
  72 +               if (unlikely(pgd_huge(pgd))) {
  73 +                       if (!gup_huge_pgd(pgd, pgdp, addr, next, write,
  74 +                                         pages, nr))
  75 +                               return;
  76 +               } else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
  77 +                       if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
  78 +                                        PGDIR_SHIFT, next, write, pages, nr))
  79 +                               return;
  80 +               } else if (!gup_p4d_range(pgd, addr, next, write, pages, nr))
  81 +                       return;
  82 +       } while (pgdp++, addr = next, addr != end);
  83 +}
  84 +
  85 +#ifndef gup_fast_permitted
  86 +/*
  87 + * Check if it's allowed to use __get_user_pages_fast() for the range, or
  88 + * we need to fall back to the slow version:
  89 + */
  90 +bool gup_fast_permitted(unsigned long start, int nr_pages, int write)
  91 +{
  92 +       unsigned long len, end;
  93 +
  94 +       len = (unsigned long) nr_pages << PAGE_SHIFT;
  95 +       end = start + len;
  96 +       return end >= start;
  97 +}
  98 +#endif
  99 +
 100  /*
 101   * Like get_user_pages_fast() except it's IRQ-safe in that it won't fall back to
 102   * the regular GUP. It will only return non-negative values.
 103 @@ -1625,10 +1666,8 @@ static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,
 104  int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
 105                           struct page **pages)
 106  {
 107 -       struct mm_struct *mm = current->mm;
 108         unsigned long addr, len, end;
 109 -       unsigned long next, flags;
 110 -       pgd_t *pgdp;
 111 +       unsigned long flags;
 112         int nr = 0;
 113
 114         start &= PAGE_MASK;
 115 @@ -1652,45 +1691,15 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
 116          * block IPIs that come from THPs splitting.
 117          */
 118
 119 -       local_irq_save(flags);
 120 -       pgdp = pgd_offset(mm, addr);
 121 -       do {
 122 -               pgd_t pgd = READ_ONCE(*pgdp);
 123 -
 124 -               next = pgd_addr_end(addr, end);
 125 -               if (pgd_none(pgd))
 126 -                       break;
 127 -               if (unlikely(pgd_huge(pgd))) {
 128 -                       if (!gup_huge_pgd(pgd, pgdp, addr, next, write,
 129 -                                         pages, &nr))
 130 -                               break;
 131 -               } else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
 132 -                       if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
 133 -                                        PGDIR_SHIFT, next, write, pages, &nr))
 134 -                               break;
 135 -               } else if (!gup_p4d_range(pgd, addr, next, write, pages, &nr))
 136 -                       break;
 137 -       } while (pgdp++, addr = next, addr != end);
 138 -       local_irq_restore(flags);
 139 +       if (gup_fast_permitted(start, nr_pages, write)) {
 140 +               local_irq_save(flags);
 141 +               gup_pgd_range(addr, end, write, pages, &nr);
 142 +               local_irq_restore(flags);
 143 +       }
 144
 145         return nr;
 146  }
 147
 148 -#ifndef gup_fast_permitted
 149 -/*
 150 - * Check if it's allowed to use __get_user_pages_fast() for the range, or
 151 - * we need to fall back to the slow version:
 152 - */
 153 -bool gup_fast_permitted(unsigned long start, int nr_pages, int write)
 154 -{
 155 -       unsigned long len, end;
 156 -
 157 -       len = (unsigned long) nr_pages << PAGE_SHIFT;
 158 -       end = start + len;
 159 -       return end >= start;
 160 -}
 161 -#endif
 162 -
 163  /**
 164   * get_user_pages_fast() - pin user pages in memory
 165   * @start:     starting user address
 166 @@ -1710,12 +1719,22 @@ bool gup_fast_permitted(unsigned long start, int nr_pages, int write)
 167  int get_user_pages_fast(unsigned long start, int nr_pages, int write,
 168                         struct page **pages)
 169  {
 170 +       unsigned long addr, len, end;
 171         int nr = 0, ret = 0;
 172
 173         start &= PAGE_MASK;
 174 +       addr = start;
 175 +       len = (unsigned long) nr_pages << PAGE_SHIFT;
 176 +       end = start + len;
 177 +
 178 +       if (unlikely(!access_ok(write ? VERIFY_WRITE : VERIFY_READ,
 179 +                                       (void __user *)start, len)))
 180 +               return 0;
 181
 182         if (gup_fast_permitted(start, nr_pages, write)) {
 183 -               nr = __get_user_pages_fast(start, nr_pages, write, pages);
 184 +               local_irq_disable();
 185 +               gup_pgd_range(addr, end, write, pages, &nr);
 186 +               local_irq_enable();
 187                 ret = nr;
 188         }
 189
 190 --
 191 2.14.2
 192