]>
Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
c33fa9f5 | 2 | /* |
3f0acb1e | 3 | * Access kernel or user memory without faulting. |
c33fa9f5 | 4 | */ |
b95f1b31 | 5 | #include <linux/export.h> |
c33fa9f5 | 6 | #include <linux/mm.h> |
7c7fcf76 | 7 | #include <linux/uaccess.h> |
c33fa9f5 | 8 | |
98a23609 | 9 | bool __weak probe_kernel_read_allowed(const void *unsafe_src, size_t size) |
eab0c608 CH |
10 | { |
11 | return true; | |
12 | } | |
13 | ||
b58294ea CH |
14 | #ifdef HAVE_GET_KERNEL_NOFAULT |
15 | ||
16 | #define probe_kernel_read_loop(dst, src, len, type, err_label) \ | |
17 | while (len >= sizeof(type)) { \ | |
18 | __get_kernel_nofault(dst, src, type, err_label); \ | |
19 | dst += sizeof(type); \ | |
20 | src += sizeof(type); \ | |
21 | len -= sizeof(type); \ | |
22 | } | |
23 | ||
24 | long probe_kernel_read(void *dst, const void *src, size_t size) | |
25 | { | |
26 | if (!probe_kernel_read_allowed(src, size)) | |
27 | return -EFAULT; | |
28 | ||
29 | pagefault_disable(); | |
30 | probe_kernel_read_loop(dst, src, size, u64, Efault); | |
31 | probe_kernel_read_loop(dst, src, size, u32, Efault); | |
32 | probe_kernel_read_loop(dst, src, size, u16, Efault); | |
33 | probe_kernel_read_loop(dst, src, size, u8, Efault); | |
34 | pagefault_enable(); | |
35 | return 0; | |
36 | Efault: | |
37 | pagefault_enable(); | |
38 | return -EFAULT; | |
39 | } | |
40 | EXPORT_SYMBOL_GPL(probe_kernel_read); | |
41 | ||
42 | #define probe_kernel_write_loop(dst, src, len, type, err_label) \ | |
43 | while (len >= sizeof(type)) { \ | |
44 | __put_kernel_nofault(dst, src, type, err_label); \ | |
45 | dst += sizeof(type); \ | |
46 | src += sizeof(type); \ | |
47 | len -= sizeof(type); \ | |
48 | } | |
49 | ||
50 | long probe_kernel_write(void *dst, const void *src, size_t size) | |
51 | { | |
52 | pagefault_disable(); | |
53 | probe_kernel_write_loop(dst, src, size, u64, Efault); | |
54 | probe_kernel_write_loop(dst, src, size, u32, Efault); | |
55 | probe_kernel_write_loop(dst, src, size, u16, Efault); | |
56 | probe_kernel_write_loop(dst, src, size, u8, Efault); | |
57 | pagefault_enable(); | |
58 | return 0; | |
59 | Efault: | |
60 | pagefault_enable(); | |
61 | return -EFAULT; | |
62 | } | |
63 | ||
64 | long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) | |
65 | { | |
66 | const void *src = unsafe_addr; | |
67 | ||
68 | if (unlikely(count <= 0)) | |
69 | return 0; | |
70 | if (!probe_kernel_read_allowed(unsafe_addr, count)) | |
71 | return -EFAULT; | |
72 | ||
73 | pagefault_disable(); | |
74 | do { | |
75 | __get_kernel_nofault(dst, src, u8, Efault); | |
76 | dst++; | |
77 | src++; | |
78 | } while (dst[-1] && src - unsafe_addr < count); | |
79 | pagefault_enable(); | |
80 | ||
81 | dst[-1] = '\0'; | |
82 | return src - unsafe_addr; | |
83 | Efault: | |
84 | pagefault_enable(); | |
85 | dst[-1] = '\0'; | |
86 | return -EFAULT; | |
87 | } | |
88 | #else /* HAVE_GET_KERNEL_NOFAULT */ | |
c33fa9f5 | 89 | /** |
98a23609 | 90 | * probe_kernel_read(): safely attempt to read from kernel-space |
4f6de12b CH |
91 | * @dst: pointer to the buffer that shall take the data |
92 | * @src: address to read from | |
93 | * @size: size of the data chunk | |
94 | * | |
95 | * Safely read from kernel address @src to the buffer at @dst. If a kernel | |
96 | * fault happens, handle that and return -EFAULT. | |
0ab32b6f AM |
97 | * |
98 | * We ensure that the copy_from_user is executed in atomic context so that | |
c1e8d7c6 | 99 | * do_page_fault() doesn't attempt to take mmap_lock. This makes |
0ab32b6f | 100 | * probe_kernel_read() suitable for use within regions where the caller |
c1e8d7c6 | 101 | * already holds mmap_lock, or other locks which nest inside mmap_lock. |
c33fa9f5 | 102 | */ |
98a23609 | 103 | long probe_kernel_read(void *dst, const void *src, size_t size) |
c33fa9f5 IM |
104 | { |
105 | long ret; | |
b4b8ac52 | 106 | mm_segment_t old_fs = get_fs(); |
c33fa9f5 | 107 | |
98a23609 | 108 | if (!probe_kernel_read_allowed(src, size)) |
eab0c608 CH |
109 | return -EFAULT; |
110 | ||
b4b8ac52 | 111 | set_fs(KERNEL_DS); |
cd030905 CH |
112 | pagefault_disable(); |
113 | ret = __copy_from_user_inatomic(dst, (__force const void __user *)src, | |
114 | size); | |
115 | pagefault_enable(); | |
b4b8ac52 | 116 | set_fs(old_fs); |
c33fa9f5 | 117 | |
cd030905 CH |
118 | if (ret) |
119 | return -EFAULT; | |
120 | return 0; | |
c33fa9f5 | 121 | } |
98a23609 | 122 | EXPORT_SYMBOL_GPL(probe_kernel_read); |
c33fa9f5 IM |
123 | |
124 | /** | |
125 | * probe_kernel_write(): safely attempt to write to a location | |
126 | * @dst: address to write to | |
127 | * @src: pointer to the data that shall be written | |
128 | * @size: size of the data chunk | |
129 | * | |
130 | * Safely write to address @dst from the buffer at @src. If a kernel fault | |
131 | * happens, handle that and return -EFAULT. | |
132 | */ | |
48c49c0e | 133 | long probe_kernel_write(void *dst, const void *src, size_t size) |
c33fa9f5 IM |
134 | { |
135 | long ret; | |
b4b8ac52 | 136 | mm_segment_t old_fs = get_fs(); |
c33fa9f5 | 137 | |
b4b8ac52 | 138 | set_fs(KERNEL_DS); |
cd030905 CH |
139 | pagefault_disable(); |
140 | ret = __copy_to_user_inatomic((__force void __user *)dst, src, size); | |
141 | pagefault_enable(); | |
b4b8ac52 | 142 | set_fs(old_fs); |
c33fa9f5 | 143 | |
cd030905 CH |
144 | if (ret) |
145 | return -EFAULT; | |
146 | return 0; | |
c33fa9f5 | 147 | } |
dbb7ee0e | 148 | |
4f6de12b | 149 | /** |
c4cb1644 | 150 | * strncpy_from_kernel_nofault: - Copy a NUL terminated string from unsafe |
4f6de12b CH |
151 | * address. |
152 | * @dst: Destination address, in kernel space. This buffer must be at | |
153 | * least @count bytes long. | |
154 | * @unsafe_addr: Unsafe address. | |
155 | * @count: Maximum number of bytes to copy, including the trailing NUL. | |
156 | * | |
157 | * Copies a NUL-terminated string from unsafe address to kernel buffer. | |
158 | * | |
159 | * On success, returns the length of the string INCLUDING the trailing NUL. | |
160 | * | |
161 | * If access fails, returns -EFAULT (some data may have been copied | |
162 | * and the trailing NUL added). | |
163 | * | |
164 | * If @count is smaller than the length of the string, copies @count-1 bytes, | |
165 | * sets the last byte of @dst buffer to NUL and returns @count. | |
166 | */ | |
eab0c608 | 167 | long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) |
dbb7ee0e AS |
168 | { |
169 | mm_segment_t old_fs = get_fs(); | |
170 | const void *src = unsafe_addr; | |
171 | long ret; | |
172 | ||
173 | if (unlikely(count <= 0)) | |
174 | return 0; | |
98a23609 | 175 | if (!probe_kernel_read_allowed(unsafe_addr, count)) |
eab0c608 | 176 | return -EFAULT; |
dbb7ee0e AS |
177 | |
178 | set_fs(KERNEL_DS); | |
179 | pagefault_disable(); | |
180 | ||
181 | do { | |
bd28b145 | 182 | ret = __get_user(*dst++, (const char __user __force *)src++); |
dbb7ee0e AS |
183 | } while (dst[-1] && ret == 0 && src - unsafe_addr < count); |
184 | ||
185 | dst[-1] = '\0'; | |
186 | pagefault_enable(); | |
187 | set_fs(old_fs); | |
188 | ||
9dd861d5 | 189 | return ret ? -EFAULT : src - unsafe_addr; |
dbb7ee0e | 190 | } |
b58294ea | 191 | #endif /* HAVE_GET_KERNEL_NOFAULT */ |
3d708182 | 192 | |
fc3562d7 CH |
193 | /** |
194 | * probe_user_read(): safely attempt to read from a user-space location | |
195 | * @dst: pointer to the buffer that shall take the data | |
196 | * @src: address to read from. This must be a user address. | |
197 | * @size: size of the data chunk | |
198 | * | |
199 | * Safely read from user address @src to the buffer at @dst. If a kernel fault | |
200 | * happens, handle that and return -EFAULT. | |
201 | */ | |
202 | long probe_user_read(void *dst, const void __user *src, size_t size) | |
203 | { | |
204 | long ret = -EFAULT; | |
205 | mm_segment_t old_fs = get_fs(); | |
206 | ||
207 | set_fs(USER_DS); | |
208 | if (access_ok(src, size)) { | |
209 | pagefault_disable(); | |
210 | ret = __copy_from_user_inatomic(dst, src, size); | |
211 | pagefault_enable(); | |
212 | } | |
213 | set_fs(old_fs); | |
214 | ||
215 | if (ret) | |
216 | return -EFAULT; | |
217 | return 0; | |
218 | } | |
219 | EXPORT_SYMBOL_GPL(probe_user_read); | |
220 | ||
221 | /** | |
222 | * probe_user_write(): safely attempt to write to a user-space location | |
223 | * @dst: address to write to | |
224 | * @src: pointer to the data that shall be written | |
225 | * @size: size of the data chunk | |
226 | * | |
227 | * Safely write to address @dst from the buffer at @src. If a kernel fault | |
228 | * happens, handle that and return -EFAULT. | |
229 | */ | |
230 | long probe_user_write(void __user *dst, const void *src, size_t size) | |
231 | { | |
232 | long ret = -EFAULT; | |
233 | mm_segment_t old_fs = get_fs(); | |
234 | ||
235 | set_fs(USER_DS); | |
236 | if (access_ok(dst, size)) { | |
237 | pagefault_disable(); | |
238 | ret = __copy_to_user_inatomic(dst, src, size); | |
239 | pagefault_enable(); | |
240 | } | |
241 | set_fs(old_fs); | |
242 | ||
243 | if (ret) | |
244 | return -EFAULT; | |
245 | return 0; | |
246 | } | |
247 | EXPORT_SYMBOL_GPL(probe_user_write); | |
248 | ||
3d708182 | 249 | /** |
bd88bb5d | 250 | * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user |
3d708182 MH |
251 | * address. |
252 | * @dst: Destination address, in kernel space. This buffer must be at | |
253 | * least @count bytes long. | |
254 | * @unsafe_addr: Unsafe user address. | |
255 | * @count: Maximum number of bytes to copy, including the trailing NUL. | |
256 | * | |
257 | * Copies a NUL-terminated string from unsafe user address to kernel buffer. | |
258 | * | |
259 | * On success, returns the length of the string INCLUDING the trailing NUL. | |
260 | * | |
261 | * If access fails, returns -EFAULT (some data may have been copied | |
262 | * and the trailing NUL added). | |
263 | * | |
264 | * If @count is smaller than the length of the string, copies @count-1 bytes, | |
265 | * sets the last byte of @dst buffer to NUL and returns @count. | |
266 | */ | |
bd88bb5d | 267 | long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr, |
3d708182 MH |
268 | long count) |
269 | { | |
270 | mm_segment_t old_fs = get_fs(); | |
271 | long ret; | |
272 | ||
273 | if (unlikely(count <= 0)) | |
274 | return 0; | |
275 | ||
276 | set_fs(USER_DS); | |
277 | pagefault_disable(); | |
278 | ret = strncpy_from_user(dst, unsafe_addr, count); | |
279 | pagefault_enable(); | |
280 | set_fs(old_fs); | |
281 | ||
282 | if (ret >= count) { | |
283 | ret = count; | |
284 | dst[ret - 1] = '\0'; | |
285 | } else if (ret > 0) { | |
286 | ret++; | |
287 | } | |
288 | ||
289 | return ret; | |
290 | } | |
291 | ||
292 | /** | |
02dddb16 | 293 | * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL. |
3d708182 MH |
294 | * @unsafe_addr: The string to measure. |
295 | * @count: Maximum count (including NUL) | |
296 | * | |
297 | * Get the size of a NUL-terminated string in user space without pagefault. | |
298 | * | |
299 | * Returns the size of the string INCLUDING the terminating NUL. | |
300 | * | |
301 | * If the string is too long, returns a number larger than @count. User | |
302 | * has to check the return value against "> count". | |
303 | * On exception (or invalid count), returns 0. | |
304 | * | |
305 | * Unlike strnlen_user, this can be used from IRQ handler etc. because | |
306 | * it disables pagefaults. | |
307 | */ | |
02dddb16 | 308 | long strnlen_user_nofault(const void __user *unsafe_addr, long count) |
3d708182 MH |
309 | { |
310 | mm_segment_t old_fs = get_fs(); | |
311 | int ret; | |
312 | ||
313 | set_fs(USER_DS); | |
314 | pagefault_disable(); | |
315 | ret = strnlen_user(unsafe_addr, count); | |
316 | pagefault_enable(); | |
317 | set_fs(old_fs); | |
318 | ||
319 | return ret; | |
320 | } |