mm/maccess.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Access kernel or user memory without faulting.
   4  */
   5 #include <linux/export.h>
   6 #include <linux/mm.h>
   7 #include <linux/uaccess.h>
   8
   9 bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src,
  10                 size_t size)
  11 {
  12         return true;
  13 }
  14
  15 #ifdef HAVE_GET_KERNEL_NOFAULT
  16
  17 #define copy_from_kernel_nofault_loop(dst, src, len, type, err_label)   \
  18         while (len >= sizeof(type)) {                                   \
  19                 __get_kernel_nofault(dst, src, type, err_label);                \
  20                 dst += sizeof(type);                                    \
  21                 src += sizeof(type);                                    \
  22                 len -= sizeof(type);                                    \
  23         }
  24
  25 long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
  26 {
  27         unsigned long align = 0;
  28
  29         if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
  30                 align = (unsigned long)dst | (unsigned long)src;
  31
  32         if (!copy_from_kernel_nofault_allowed(src, size))
  33                 return -ERANGE;
  34
  35         pagefault_disable();
  36         if (!(align & 7))
  37                 copy_from_kernel_nofault_loop(dst, src, size, u64, Efault);
  38         if (!(align & 3))
  39                 copy_from_kernel_nofault_loop(dst, src, size, u32, Efault);
  40         if (!(align & 1))
  41                 copy_from_kernel_nofault_loop(dst, src, size, u16, Efault);
  42         copy_from_kernel_nofault_loop(dst, src, size, u8, Efault);
  43         pagefault_enable();
  44         return 0;
  45 Efault:
  46         pagefault_enable();
  47         return -EFAULT;
  48 }
  49 EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
  50
  51 #define copy_to_kernel_nofault_loop(dst, src, len, type, err_label)     \
  52         while (len >= sizeof(type)) {                                   \
  53                 __put_kernel_nofault(dst, src, type, err_label);                \
  54                 dst += sizeof(type);                                    \
  55                 src += sizeof(type);                                    \
  56                 len -= sizeof(type);                                    \
  57         }
  58
  59 long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
  60 {
  61         unsigned long align = 0;
  62
  63         if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
  64                 align = (unsigned long)dst | (unsigned long)src;
  65
  66         pagefault_disable();
  67         if (!(align & 7))
  68                 copy_to_kernel_nofault_loop(dst, src, size, u64, Efault);
  69         if (!(align & 3))
  70                 copy_to_kernel_nofault_loop(dst, src, size, u32, Efault);
  71         if (!(align & 1))
  72                 copy_to_kernel_nofault_loop(dst, src, size, u16, Efault);
  73         copy_to_kernel_nofault_loop(dst, src, size, u8, Efault);
  74         pagefault_enable();
  75         return 0;
  76 Efault:
  77         pagefault_enable();
  78         return -EFAULT;
  79 }
  80
  81 long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
  82 {
  83         const void *src = unsafe_addr;
  84
  85         if (unlikely(count <= 0))
  86                 return 0;
  87         if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
  88                 return -ERANGE;
  89
  90         pagefault_disable();
  91         do {
  92                 __get_kernel_nofault(dst, src, u8, Efault);
  93                 dst++;
  94                 src++;
  95         } while (dst[-1] && src - unsafe_addr < count);
  96         pagefault_enable();
  97
  98         dst[-1] = '\0';
  99         return src - unsafe_addr;
 100 Efault:
 101         pagefault_enable();
 102         dst[-1] = '\0';
 103         return -EFAULT;
 104 }
 105 #else /* HAVE_GET_KERNEL_NOFAULT */
 106 /**
 107  * copy_from_kernel_nofault(): safely attempt to read from kernel-space
 108  * @dst: pointer to the buffer that shall take the data
 109  * @src: address to read from
 110  * @size: size of the data chunk
 111  *
 112  * Safely read from kernel address @src to the buffer at @dst.  If a kernel
 113  * fault happens, handle that and return -EFAULT.  If @src is not a valid kernel
 114  * address, return -ERANGE.
 115  *
 116  * We ensure that the copy_from_user is executed in atomic context so that
 117  * do_page_fault() doesn't attempt to take mmap_lock.  This makes
 118  * copy_from_kernel_nofault() suitable for use within regions where the caller
 119  * already holds mmap_lock, or other locks which nest inside mmap_lock.
 120  */
 121 long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
 122 {
 123         long ret;
 124         mm_segment_t old_fs = get_fs();
 125
 126         if (!copy_from_kernel_nofault_allowed(src, size))
 127                 return -ERANGE;
 128
 129         set_fs(KERNEL_DS);
 130         pagefault_disable();
 131         ret = __copy_from_user_inatomic(dst, (__force const void __user *)src,
 132                         size);
 133         pagefault_enable();
 134         set_fs(old_fs);
 135
 136         if (ret)
 137                 return -EFAULT;
 138         return 0;
 139 }
 140 EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
 141
 142 /**
 143  * copy_to_kernel_nofault(): safely attempt to write to a location
 144  * @dst: address to write to
 145  * @src: pointer to the data that shall be written
 146  * @size: size of the data chunk
 147  *
 148  * Safely write to address @dst from the buffer at @src.  If a kernel fault
 149  * happens, handle that and return -EFAULT.
 150  */
 151 long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
 152 {
 153         long ret;
 154         mm_segment_t old_fs = get_fs();
 155
 156         set_fs(KERNEL_DS);
 157         pagefault_disable();
 158         ret = __copy_to_user_inatomic((__force void __user *)dst, src, size);
 159         pagefault_enable();
 160         set_fs(old_fs);
 161
 162         if (ret)
 163                 return -EFAULT;
 164         return 0;
 165 }
 166
 167 /**
 168  * strncpy_from_kernel_nofault: - Copy a NUL terminated string from unsafe
 169  *                               address.
 170  * @dst:   Destination address, in kernel space.  This buffer must be at
 171  *         least @count bytes long.
 172  * @unsafe_addr: Unsafe address.
 173  * @count: Maximum number of bytes to copy, including the trailing NUL.
 174  *
 175  * Copies a NUL-terminated string from unsafe address to kernel buffer.
 176  *
 177  * On success, returns the length of the string INCLUDING the trailing NUL.
 178  *
 179  * If access fails, returns -EFAULT (some data may have been copied and the
 180  * trailing NUL added).  If @unsafe_addr is not a valid kernel address, return
 181  * -ERANGE.
 182  *
 183  * If @count is smaller than the length of the string, copies @count-1 bytes,
 184  * sets the last byte of @dst buffer to NUL and returns @count.
 185  */
 186 long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
 187 {
 188         mm_segment_t old_fs = get_fs();
 189         const void *src = unsafe_addr;
 190         long ret;
 191
 192         if (unlikely(count <= 0))
 193                 return 0;
 194         if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
 195                 return -ERANGE;
 196
 197         set_fs(KERNEL_DS);
 198         pagefault_disable();
 199
 200         do {
 201                 ret = __get_user(*dst++, (const char __user __force *)src++);
 202         } while (dst[-1] && ret == 0 && src - unsafe_addr < count);
 203
 204         dst[-1] = '\0';
 205         pagefault_enable();
 206         set_fs(old_fs);
 207
 208         return ret ? -EFAULT : src - unsafe_addr;
 209 }
 210 #endif /* HAVE_GET_KERNEL_NOFAULT */
 211
 212 /**
 213  * copy_from_user_nofault(): safely attempt to read from a user-space location
 214  * @dst: pointer to the buffer that shall take the data
 215  * @src: address to read from. This must be a user address.
 216  * @size: size of the data chunk
 217  *
 218  * Safely read from user address @src to the buffer at @dst. If a kernel fault
 219  * happens, handle that and return -EFAULT.
 220  */
 221 long copy_from_user_nofault(void *dst, const void __user *src, size_t size)
 222 {
 223         long ret = -EFAULT;
 224         mm_segment_t old_fs = force_uaccess_begin();
 225
 226         if (access_ok(src, size)) {
 227                 pagefault_disable();
 228                 ret = __copy_from_user_inatomic(dst, src, size);
 229                 pagefault_enable();
 230         }
 231         force_uaccess_end(old_fs);
 232
 233         if (ret)
 234                 return -EFAULT;
 235         return 0;
 236 }
 237 EXPORT_SYMBOL_GPL(copy_from_user_nofault);
 238
 239 /**
 240  * copy_to_user_nofault(): safely attempt to write to a user-space location
 241  * @dst: address to write to
 242  * @src: pointer to the data that shall be written
 243  * @size: size of the data chunk
 244  *
 245  * Safely write to address @dst from the buffer at @src.  If a kernel fault
 246  * happens, handle that and return -EFAULT.
 247  */
 248 long copy_to_user_nofault(void __user *dst, const void *src, size_t size)
 249 {
 250         long ret = -EFAULT;
 251         mm_segment_t old_fs = force_uaccess_begin();
 252
 253         if (access_ok(dst, size)) {
 254                 pagefault_disable();
 255                 ret = __copy_to_user_inatomic(dst, src, size);
 256                 pagefault_enable();
 257         }
 258         force_uaccess_end(old_fs);
 259
 260         if (ret)
 261                 return -EFAULT;
 262         return 0;
 263 }
 264 EXPORT_SYMBOL_GPL(copy_to_user_nofault);
 265
 266 /**
 267  * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user
 268  *                              address.
 269  * @dst:   Destination address, in kernel space.  This buffer must be at
 270  *         least @count bytes long.
 271  * @unsafe_addr: Unsafe user address.
 272  * @count: Maximum number of bytes to copy, including the trailing NUL.
 273  *
 274  * Copies a NUL-terminated string from unsafe user address to kernel buffer.
 275  *
 276  * On success, returns the length of the string INCLUDING the trailing NUL.
 277  *
 278  * If access fails, returns -EFAULT (some data may have been copied
 279  * and the trailing NUL added).
 280  *
 281  * If @count is smaller than the length of the string, copies @count-1 bytes,
 282  * sets the last byte of @dst buffer to NUL and returns @count.
 283  */
 284 long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
 285                               long count)
 286 {
 287         mm_segment_t old_fs;
 288         long ret;
 289
 290         if (unlikely(count <= 0))
 291                 return 0;
 292
 293         old_fs = force_uaccess_begin();
 294         pagefault_disable();
 295         ret = strncpy_from_user(dst, unsafe_addr, count);
 296         pagefault_enable();
 297         force_uaccess_end(old_fs);
 298
 299         if (ret >= count) {
 300                 ret = count;
 301                 dst[ret - 1] = '\0';
 302         } else if (ret > 0) {
 303                 ret++;
 304         }
 305
 306         return ret;
 307 }
 308
 309 /**
 310  * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL.
 311  * @unsafe_addr: The string to measure.
 312  * @count: Maximum count (including NUL)
 313  *
 314  * Get the size of a NUL-terminated string in user space without pagefault.
 315  *
 316  * Returns the size of the string INCLUDING the terminating NUL.
 317  *
 318  * If the string is too long, returns a number larger than @count. User
 319  * has to check the return value against "> count".
 320  * On exception (or invalid count), returns 0.
 321  *
 322  * Unlike strnlen_user, this can be used from IRQ handler etc. because
 323  * it disables pagefaults.
 324  */
 325 long strnlen_user_nofault(const void __user *unsafe_addr, long count)
 326 {
 327         mm_segment_t old_fs;
 328         int ret;
 329
 330         old_fs = force_uaccess_begin();
 331         pagefault_disable();
 332         ret = strnlen_user(unsafe_addr, count);
 333         pagefault_enable();
 334         force_uaccess_end(old_fs);
 335
 336         return ret;
 337 }