mm/maccess.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Access kernel memory without faulting.
   4  */
   5 #include <linux/export.h>
   6 #include <linux/mm.h>
   7 #include <linux/uaccess.h>
   8
   9 static __always_inline long
  10 probe_read_common(void *dst, const void __user *src, size_t size)
  11 {
  12         long ret;
  13
  14         pagefault_disable();
  15         ret = __copy_from_user_inatomic(dst, src, size);
  16         pagefault_enable();
  17
  18         return ret ? -EFAULT : 0;
  19 }
  20
  21 static __always_inline long
  22 probe_write_common(void __user *dst, const void *src, size_t size)
  23 {
  24         long ret;
  25
  26         pagefault_disable();
  27         ret = __copy_to_user_inatomic(dst, src, size);
  28         pagefault_enable();
  29
  30         return ret ? -EFAULT : 0;
  31 }
  32
  33 /**
  34  * probe_kernel_read(): safely attempt to read from a kernel-space location
  35  * @dst: pointer to the buffer that shall take the data
  36  * @src: address to read from
  37  * @size: size of the data chunk
  38  *
  39  * Safely read from address @src to the buffer at @dst.  If a kernel fault
  40  * happens, handle that and return -EFAULT.
  41  *
  42  * We ensure that the copy_from_user is executed in atomic context so that
  43  * do_page_fault() doesn't attempt to take mmap_sem.  This makes
  44  * probe_kernel_read() suitable for use within regions where the caller
  45  * already holds mmap_sem, or other locks which nest inside mmap_sem.
  46  */
  47
  48 long __weak probe_kernel_read(void *dst, const void *src, size_t size)
  49     __attribute__((alias("__probe_kernel_read")));
  50
  51 long __probe_kernel_read(void *dst, const void *src, size_t size)
  52 {
  53         long ret;
  54         mm_segment_t old_fs = get_fs();
  55
  56         set_fs(KERNEL_DS);
  57         ret = probe_read_common(dst, (__force const void __user *)src, size);
  58         set_fs(old_fs);
  59
  60         return ret;
  61 }
  62 EXPORT_SYMBOL_GPL(probe_kernel_read);
  63
  64 /**
  65  * probe_user_read(): safely attempt to read from a user-space location
  66  * @dst: pointer to the buffer that shall take the data
  67  * @src: address to read from. This must be a user address.
  68  * @size: size of the data chunk
  69  *
  70  * Safely read from user address @src to the buffer at @dst. If a kernel fault
  71  * happens, handle that and return -EFAULT.
  72  */
  73
  74 long __weak probe_user_read(void *dst, const void __user *src, size_t size)
  75     __attribute__((alias("__probe_user_read")));
  76
  77 long __probe_user_read(void *dst, const void __user *src, size_t size)
  78 {
  79         long ret = -EFAULT;
  80         mm_segment_t old_fs = get_fs();
  81
  82         set_fs(USER_DS);
  83         if (access_ok(src, size))
  84                 ret = probe_read_common(dst, src, size);
  85         set_fs(old_fs);
  86
  87         return ret;
  88 }
  89 EXPORT_SYMBOL_GPL(probe_user_read);
  90
  91 /**
  92  * probe_kernel_write(): safely attempt to write to a location
  93  * @dst: address to write to
  94  * @src: pointer to the data that shall be written
  95  * @size: size of the data chunk
  96  *
  97  * Safely write to address @dst from the buffer at @src.  If a kernel fault
  98  * happens, handle that and return -EFAULT.
  99  */
 100
 101 long __weak probe_kernel_write(void *dst, const void *src, size_t size)
 102     __attribute__((alias("__probe_kernel_write")));
 103
 104 long __probe_kernel_write(void *dst, const void *src, size_t size)
 105 {
 106         long ret;
 107         mm_segment_t old_fs = get_fs();
 108
 109         set_fs(KERNEL_DS);
 110         ret = probe_write_common((__force void __user *)dst, src, size);
 111         set_fs(old_fs);
 112
 113         return ret;
 114 }
 115 EXPORT_SYMBOL_GPL(probe_kernel_write);
 116
 117 /**
 118  * probe_user_write(): safely attempt to write to a user-space location
 119  * @dst: address to write to
 120  * @src: pointer to the data that shall be written
 121  * @size: size of the data chunk
 122  *
 123  * Safely write to address @dst from the buffer at @src.  If a kernel fault
 124  * happens, handle that and return -EFAULT.
 125  */
 126
 127 long __weak probe_user_write(void __user *dst, const void *src, size_t size)
 128     __attribute__((alias("__probe_user_write")));
 129
 130 long __probe_user_write(void __user *dst, const void *src, size_t size)
 131 {
 132         long ret = -EFAULT;
 133         mm_segment_t old_fs = get_fs();
 134
 135         set_fs(USER_DS);
 136         if (access_ok(dst, size))
 137                 ret = probe_write_common(dst, src, size);
 138         set_fs(old_fs);
 139
 140         return ret;
 141 }
 142 EXPORT_SYMBOL_GPL(probe_user_write);
 143
 144 /**
 145  * strncpy_from_unsafe: - Copy a NUL terminated string from unsafe address.
 146  * @dst:   Destination address, in kernel space.  This buffer must be at
 147  *         least @count bytes long.
 148  * @unsafe_addr: Unsafe address.
 149  * @count: Maximum number of bytes to copy, including the trailing NUL.
 150  *
 151  * Copies a NUL-terminated string from unsafe address to kernel buffer.
 152  *
 153  * On success, returns the length of the string INCLUDING the trailing NUL.
 154  *
 155  * If access fails, returns -EFAULT (some data may have been copied
 156  * and the trailing NUL added).
 157  *
 158  * If @count is smaller than the length of the string, copies @count-1 bytes,
 159  * sets the last byte of @dst buffer to NUL and returns @count.
 160  */
 161 long strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count)
 162 {
 163         mm_segment_t old_fs = get_fs();
 164         const void *src = unsafe_addr;
 165         long ret;
 166
 167         if (unlikely(count <= 0))
 168                 return 0;
 169
 170         set_fs(KERNEL_DS);
 171         pagefault_disable();
 172
 173         do {
 174                 ret = __get_user(*dst++, (const char __user __force *)src++);
 175         } while (dst[-1] && ret == 0 && src - unsafe_addr < count);
 176
 177         dst[-1] = '\0';
 178         pagefault_enable();
 179         set_fs(old_fs);
 180
 181         return ret ? -EFAULT : src - unsafe_addr;
 182 }
 183
 184 /**
 185  * strncpy_from_unsafe_user: - Copy a NUL terminated string from unsafe user
 186  *                              address.
 187  * @dst:   Destination address, in kernel space.  This buffer must be at
 188  *         least @count bytes long.
 189  * @unsafe_addr: Unsafe user address.
 190  * @count: Maximum number of bytes to copy, including the trailing NUL.
 191  *
 192  * Copies a NUL-terminated string from unsafe user address to kernel buffer.
 193  *
 194  * On success, returns the length of the string INCLUDING the trailing NUL.
 195  *
 196  * If access fails, returns -EFAULT (some data may have been copied
 197  * and the trailing NUL added).
 198  *
 199  * If @count is smaller than the length of the string, copies @count-1 bytes,
 200  * sets the last byte of @dst buffer to NUL and returns @count.
 201  */
 202 long strncpy_from_unsafe_user(char *dst, const void __user *unsafe_addr,
 203                               long count)
 204 {
 205         mm_segment_t old_fs = get_fs();
 206         long ret;
 207
 208         if (unlikely(count <= 0))
 209                 return 0;
 210
 211         set_fs(USER_DS);
 212         pagefault_disable();
 213         ret = strncpy_from_user(dst, unsafe_addr, count);
 214         pagefault_enable();
 215         set_fs(old_fs);
 216
 217         if (ret >= count) {
 218                 ret = count;
 219                 dst[ret - 1] = '\0';
 220         } else if (ret > 0) {
 221                 ret++;
 222         }
 223
 224         return ret;
 225 }
 226
 227 /**
 228  * strnlen_unsafe_user: - Get the size of a user string INCLUDING final NUL.
 229  * @unsafe_addr: The string to measure.
 230  * @count: Maximum count (including NUL)
 231  *
 232  * Get the size of a NUL-terminated string in user space without pagefault.
 233  *
 234  * Returns the size of the string INCLUDING the terminating NUL.
 235  *
 236  * If the string is too long, returns a number larger than @count. User
 237  * has to check the return value against "> count".
 238  * On exception (or invalid count), returns 0.
 239  *
 240  * Unlike strnlen_user, this can be used from IRQ handler etc. because
 241  * it disables pagefaults.
 242  */
 243 long strnlen_unsafe_user(const void __user *unsafe_addr, long count)
 244 {
 245         mm_segment_t old_fs = get_fs();
 246         int ret;
 247
 248         set_fs(USER_DS);
 249         pagefault_disable();
 250         ret = strnlen_user(unsafe_addr, count);
 251         pagefault_enable();
 252         set_fs(old_fs);
 253
 254         return ret;
 255 }