arm64: uaccess: Don't bother eliding access_ok checks in __{get, put}_user

Commit 84624087dd7e upstream.

access_ok isn't an expensive operation once the addr_limit for the current
thread has been loaded into the cache. Given that the initial access_ok
check preceding a sequence of __{get,put}_user operations will take
the brunt of the miss, we can make the __* variants identical to the
full-fat versions, which brings with it the benefits of address masking.

The likely cost in these sequences will be from toggling PAN/UAO, which
we can address later by implementing the *_unsafe versions.

Reviewed-by: Robin Murphy <robin.murphy@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
(cherry picked from commit 28d8886d985ccc84e4e8f567c60f491acb1ae51c)

CVE-2017-5753
CVE-2017-5715
CVE-2017-5754

Signed-off-by: Paolo Pisati <paolo.pisati@canonical.com>
Acked-by: Brad Figg <brad.figg@canonical.com>
Acked-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>
Signed-off-by: Khalid Elmously <khalid.elmously@canonical.com>

diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 2f5dd32f5aef3ae010f5f4b3f9d443a4d23ff15a..0687bb084719a412d3324ea13f27c52d8498621d 100644 (file)
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -294,28 +294,33 @@ do {                                                                      \
        (x) = (__force __typeof__(*(ptr)))__gu_val;                     \
 } while (0)
 
-#define __get_user(x, ptr)                                             \
+#define __get_user_check(x, ptr, err)                                  \
 ({                                                                     \
-       int __gu_err = 0;                                               \
-       __get_user_err((x), (ptr), __gu_err);                           \
-       __gu_err;                                                       \
+       __typeof__(*(ptr)) __user *__p = (ptr);                         \
+       might_fault();                                                  \
+       if (access_ok(VERIFY_READ, __p, sizeof(*__p))) {                \
+               __p = uaccess_mask_ptr(__p);                            \
+               __get_user_err((x), __p, (err));                        \
+       } else {                                                        \
+               (x) = 0; (err) = -EFAULT;                               \
+       }                                                               \
 })
 
 #define __get_user_error(x, ptr, err)                                  \
 ({                                                                     \
-       __get_user_err((x), (ptr), (err));                              \
+       __get_user_check((x), (ptr), (err));                            \
        (void)0;                                                        \
 })
 
-#define get_user(x, ptr)                                               \
+#define __get_user(x, ptr)                                             \
 ({                                                                     \
-       __typeof__(*(ptr)) __user *__p = (ptr);                         \
-       might_fault();                                                  \
-       access_ok(VERIFY_READ, __p, sizeof(*__p)) ?                     \
-               __p = uaccess_mask_ptr(__p), __get_user((x), __p) :     \
-               ((x) = 0, -EFAULT);                                     \
+       int __gu_err = 0;                                               \
+       __get_user_check((x), (ptr), __gu_err);                         \
+       __gu_err;                                                       \
 })
 
+#define get_user       __get_user
+
 #define __put_user_asm(instr, alt_instr, reg, x, addr, err, feature)   \
        asm volatile(                                                   \
        "1:"ALTERNATIVE(instr "     " reg "1, [%2]\n",                  \
@@ -358,28 +363,33 @@ do {                                                                      \
        uaccess_disable_not_uao();                                      \
 } while (0)
 
-#define __put_user(x, ptr)                                             \
+#define __put_user_check(x, ptr, err)                                  \
 ({                                                                     \
-       int __pu_err = 0;                                               \
-       __put_user_err((x), (ptr), __pu_err);                           \
-       __pu_err;                                                       \
+       __typeof__(*(ptr)) __user *__p = (ptr);                         \
+       might_fault();                                                  \
+       if (access_ok(VERIFY_WRITE, __p, sizeof(*__p))) {               \
+               __p = uaccess_mask_ptr(__p);                            \
+               __put_user_err((x), __p, (err));                        \
+       } else  {                                                       \
+               (err) = -EFAULT;                                        \
+       }                                                               \
 })
 
 #define __put_user_error(x, ptr, err)                                  \
 ({                                                                     \
-       __put_user_err((x), (ptr), (err));                              \
+       __put_user_check((x), (ptr), (err));                            \
        (void)0;                                                        \
 })
 
-#define put_user(x, ptr)                                               \
+#define __put_user(x, ptr)                                             \
 ({                                                                     \
-       __typeof__(*(ptr)) __user *__p = (ptr);                         \
-       might_fault();                                                  \
-       access_ok(VERIFY_WRITE, __p, sizeof(*__p)) ?                    \
-               __p = uaccess_mask_ptr(__p), __put_user((x), __p) :     \
-               -EFAULT;                                                \
+       int __pu_err = 0;                                               \
+       __put_user_check((x), (ptr), __pu_err);                         \
+       __pu_err;                                                       \
 })
 
+#define put_user       __put_user
+
 extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n);
 #define raw_copy_from_user __arch_copy_from_user
 extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n);