]>
Commit | Line | Data |
---|---|---|
3f50dbc1 | 1 | /* |
1da177e4 LT |
2 | * User address space access functions. |
3 | * The non inlined parts of asm-i386/uaccess.h are here. | |
4 | * | |
5 | * Copyright 1997 Andi Kleen <ak@muc.de> | |
6 | * Copyright 1997 Linus Torvalds | |
7 | */ | |
1da177e4 LT |
8 | #include <linux/mm.h> |
9 | #include <linux/highmem.h> | |
10 | #include <linux/blkdev.h> | |
11 | #include <linux/module.h> | |
3fcfab16 | 12 | #include <linux/backing-dev.h> |
b6a8b316 | 13 | #include <linux/interrupt.h> |
1da177e4 LT |
14 | #include <asm/uaccess.h> |
15 | #include <asm/mmx.h> | |
16 | ||
8bfcb396 TP |
17 | #ifdef CONFIG_X86_INTEL_USERCOPY |
18 | /* | |
19 | * Alignment at which movsl is preferred for bulk memory copies. | |
20 | */ | |
21 | struct movsl_mask movsl_mask __read_mostly; | |
22 | #endif | |
23 | ||
1da177e4 LT |
24 | static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n) |
25 | { | |
26 | #ifdef CONFIG_X86_INTEL_USERCOPY | |
27 | if (n >= 64 && ((a1 ^ a2) & movsl_mask.mask)) | |
28 | return 0; | |
29 | #endif | |
30 | return 1; | |
31 | } | |
3f50dbc1 PC |
32 | #define movsl_is_ok(a1, a2, n) \ |
33 | __movsl_is_ok((unsigned long)(a1), (unsigned long)(a2), (n)) | |
1da177e4 LT |
34 | |
35 | /* | |
36 | * Copy a null terminated string from userspace. | |
37 | */ | |
38 | ||
3f50dbc1 | 39 | #define __do_strncpy_from_user(dst, src, count, res) \ |
1da177e4 LT |
40 | do { \ |
41 | int __d0, __d1, __d2; \ | |
3ee1afa3 | 42 | might_fault(); \ |
1da177e4 LT |
43 | __asm__ __volatile__( \ |
44 | " testl %1,%1\n" \ | |
45 | " jz 2f\n" \ | |
46 | "0: lodsb\n" \ | |
47 | " stosb\n" \ | |
48 | " testb %%al,%%al\n" \ | |
49 | " jz 1f\n" \ | |
50 | " decl %1\n" \ | |
51 | " jnz 0b\n" \ | |
52 | "1: subl %1,%0\n" \ | |
53 | "2:\n" \ | |
54 | ".section .fixup,\"ax\"\n" \ | |
55 | "3: movl %5,%0\n" \ | |
56 | " jmp 2b\n" \ | |
57 | ".previous\n" \ | |
28777441 | 58 | _ASM_EXTABLE(0b,3b) \ |
e0a96129 | 59 | : "=&d"(res), "=&c"(count), "=&a" (__d0), "=&S" (__d1), \ |
1da177e4 LT |
60 | "=&D" (__d2) \ |
61 | : "i"(-EFAULT), "0"(count), "1"(count), "3"(src), "4"(dst) \ | |
62 | : "memory"); \ | |
63 | } while (0) | |
64 | ||
65 | /** | |
66 | * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking. | |
67 | * @dst: Destination address, in kernel space. This buffer must be at | |
68 | * least @count bytes long. | |
69 | * @src: Source address, in user space. | |
70 | * @count: Maximum number of bytes to copy, including the trailing NUL. | |
3f50dbc1 | 71 | * |
1da177e4 LT |
72 | * Copies a NUL-terminated string from userspace to kernel space. |
73 | * Caller must check the specified block with access_ok() before calling | |
74 | * this function. | |
75 | * | |
76 | * On success, returns the length of the string (not including the trailing | |
77 | * NUL). | |
78 | * | |
79 | * If access to userspace fails, returns -EFAULT (some data may have been | |
80 | * copied). | |
81 | * | |
82 | * If @count is smaller than the length of the string, copies @count bytes | |
83 | * and returns @count. | |
84 | */ | |
85 | long | |
86 | __strncpy_from_user(char *dst, const char __user *src, long count) | |
87 | { | |
88 | long res; | |
89 | __do_strncpy_from_user(dst, src, count, res); | |
90 | return res; | |
91 | } | |
129f6946 | 92 | EXPORT_SYMBOL(__strncpy_from_user); |
1da177e4 LT |
93 | |
94 | /** | |
95 | * strncpy_from_user: - Copy a NUL terminated string from userspace. | |
96 | * @dst: Destination address, in kernel space. This buffer must be at | |
97 | * least @count bytes long. | |
98 | * @src: Source address, in user space. | |
99 | * @count: Maximum number of bytes to copy, including the trailing NUL. | |
3f50dbc1 | 100 | * |
1da177e4 LT |
101 | * Copies a NUL-terminated string from userspace to kernel space. |
102 | * | |
103 | * On success, returns the length of the string (not including the trailing | |
104 | * NUL). | |
105 | * | |
106 | * If access to userspace fails, returns -EFAULT (some data may have been | |
107 | * copied). | |
108 | * | |
109 | * If @count is smaller than the length of the string, copies @count bytes | |
110 | * and returns @count. | |
111 | */ | |
112 | long | |
113 | strncpy_from_user(char *dst, const char __user *src, long count) | |
114 | { | |
115 | long res = -EFAULT; | |
116 | if (access_ok(VERIFY_READ, src, 1)) | |
117 | __do_strncpy_from_user(dst, src, count, res); | |
118 | return res; | |
119 | } | |
129f6946 | 120 | EXPORT_SYMBOL(strncpy_from_user); |
1da177e4 LT |
121 | |
122 | /* | |
123 | * Zero Userspace | |
124 | */ | |
125 | ||
126 | #define __do_clear_user(addr,size) \ | |
127 | do { \ | |
128 | int __d0; \ | |
3ee1afa3 | 129 | might_fault(); \ |
3f50dbc1 | 130 | __asm__ __volatile__( \ |
1da177e4 LT |
131 | "0: rep; stosl\n" \ |
132 | " movl %2,%0\n" \ | |
133 | "1: rep; stosb\n" \ | |
134 | "2:\n" \ | |
135 | ".section .fixup,\"ax\"\n" \ | |
136 | "3: lea 0(%2,%0,4),%0\n" \ | |
137 | " jmp 2b\n" \ | |
138 | ".previous\n" \ | |
28777441 PA |
139 | _ASM_EXTABLE(0b,3b) \ |
140 | _ASM_EXTABLE(1b,2b) \ | |
1da177e4 LT |
141 | : "=&c"(size), "=&D" (__d0) \ |
142 | : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \ | |
143 | } while (0) | |
144 | ||
145 | /** | |
146 | * clear_user: - Zero a block of memory in user space. | |
147 | * @to: Destination address, in user space. | |
148 | * @n: Number of bytes to zero. | |
149 | * | |
150 | * Zero a block of memory in user space. | |
151 | * | |
152 | * Returns number of bytes that could not be cleared. | |
153 | * On success, this will be zero. | |
154 | */ | |
155 | unsigned long | |
156 | clear_user(void __user *to, unsigned long n) | |
157 | { | |
1d18ef48 | 158 | might_fault(); |
1da177e4 LT |
159 | if (access_ok(VERIFY_WRITE, to, n)) |
160 | __do_clear_user(to, n); | |
161 | return n; | |
162 | } | |
129f6946 | 163 | EXPORT_SYMBOL(clear_user); |
1da177e4 LT |
164 | |
165 | /** | |
166 | * __clear_user: - Zero a block of memory in user space, with less checking. | |
167 | * @to: Destination address, in user space. | |
168 | * @n: Number of bytes to zero. | |
169 | * | |
170 | * Zero a block of memory in user space. Caller must check | |
171 | * the specified block with access_ok() before calling this function. | |
172 | * | |
173 | * Returns number of bytes that could not be cleared. | |
174 | * On success, this will be zero. | |
175 | */ | |
176 | unsigned long | |
177 | __clear_user(void __user *to, unsigned long n) | |
178 | { | |
179 | __do_clear_user(to, n); | |
180 | return n; | |
181 | } | |
129f6946 | 182 | EXPORT_SYMBOL(__clear_user); |
1da177e4 LT |
183 | |
184 | /** | |
35e38a6e | 185 | * strnlen_user: - Get the size of a string in user space. |
1da177e4 LT |
186 | * @s: The string to measure. |
187 | * @n: The maximum valid length | |
188 | * | |
189 | * Get the size of a NUL-terminated string in user space. | |
190 | * | |
191 | * Returns the size of the string INCLUDING the terminating NUL. | |
192 | * On exception, returns 0. | |
193 | * If the string is too long, returns a value greater than @n. | |
194 | */ | |
195 | long strnlen_user(const char __user *s, long n) | |
196 | { | |
197 | unsigned long mask = -__addr_ok(s); | |
198 | unsigned long res, tmp; | |
199 | ||
3ee1afa3 | 200 | might_fault(); |
1da177e4 LT |
201 | |
202 | __asm__ __volatile__( | |
203 | " testl %0, %0\n" | |
204 | " jz 3f\n" | |
205 | " andl %0,%%ecx\n" | |
206 | "0: repne; scasb\n" | |
207 | " setne %%al\n" | |
208 | " subl %%ecx,%0\n" | |
209 | " addl %0,%%eax\n" | |
210 | "1:\n" | |
211 | ".section .fixup,\"ax\"\n" | |
212 | "2: xorl %%eax,%%eax\n" | |
213 | " jmp 1b\n" | |
214 | "3: movb $1,%%al\n" | |
215 | " jmp 1b\n" | |
216 | ".previous\n" | |
217 | ".section __ex_table,\"a\"\n" | |
218 | " .align 4\n" | |
219 | " .long 0b,2b\n" | |
220 | ".previous" | |
e0a96129 | 221 | :"=&r" (n), "=&D" (s), "=&a" (res), "=&c" (tmp) |
1da177e4 LT |
222 | :"0" (n), "1" (s), "2" (0), "3" (mask) |
223 | :"cc"); | |
224 | return res & mask; | |
225 | } | |
129f6946 | 226 | EXPORT_SYMBOL(strnlen_user); |
1da177e4 LT |
227 | |
228 | #ifdef CONFIG_X86_INTEL_USERCOPY | |
229 | static unsigned long | |
230 | __copy_user_intel(void __user *to, const void *from, unsigned long size) | |
231 | { | |
232 | int d0, d1; | |
233 | __asm__ __volatile__( | |
234 | " .align 2,0x90\n" | |
235 | "1: movl 32(%4), %%eax\n" | |
236 | " cmpl $67, %0\n" | |
237 | " jbe 3f\n" | |
238 | "2: movl 64(%4), %%eax\n" | |
239 | " .align 2,0x90\n" | |
240 | "3: movl 0(%4), %%eax\n" | |
241 | "4: movl 4(%4), %%edx\n" | |
242 | "5: movl %%eax, 0(%3)\n" | |
243 | "6: movl %%edx, 4(%3)\n" | |
244 | "7: movl 8(%4), %%eax\n" | |
245 | "8: movl 12(%4),%%edx\n" | |
246 | "9: movl %%eax, 8(%3)\n" | |
247 | "10: movl %%edx, 12(%3)\n" | |
248 | "11: movl 16(%4), %%eax\n" | |
249 | "12: movl 20(%4), %%edx\n" | |
250 | "13: movl %%eax, 16(%3)\n" | |
251 | "14: movl %%edx, 20(%3)\n" | |
252 | "15: movl 24(%4), %%eax\n" | |
253 | "16: movl 28(%4), %%edx\n" | |
254 | "17: movl %%eax, 24(%3)\n" | |
255 | "18: movl %%edx, 28(%3)\n" | |
256 | "19: movl 32(%4), %%eax\n" | |
257 | "20: movl 36(%4), %%edx\n" | |
258 | "21: movl %%eax, 32(%3)\n" | |
259 | "22: movl %%edx, 36(%3)\n" | |
260 | "23: movl 40(%4), %%eax\n" | |
261 | "24: movl 44(%4), %%edx\n" | |
262 | "25: movl %%eax, 40(%3)\n" | |
263 | "26: movl %%edx, 44(%3)\n" | |
264 | "27: movl 48(%4), %%eax\n" | |
265 | "28: movl 52(%4), %%edx\n" | |
266 | "29: movl %%eax, 48(%3)\n" | |
267 | "30: movl %%edx, 52(%3)\n" | |
268 | "31: movl 56(%4), %%eax\n" | |
269 | "32: movl 60(%4), %%edx\n" | |
270 | "33: movl %%eax, 56(%3)\n" | |
271 | "34: movl %%edx, 60(%3)\n" | |
272 | " addl $-64, %0\n" | |
273 | " addl $64, %4\n" | |
274 | " addl $64, %3\n" | |
275 | " cmpl $63, %0\n" | |
276 | " ja 1b\n" | |
277 | "35: movl %0, %%eax\n" | |
278 | " shrl $2, %0\n" | |
279 | " andl $3, %%eax\n" | |
280 | " cld\n" | |
281 | "99: rep; movsl\n" | |
282 | "36: movl %%eax, %0\n" | |
283 | "37: rep; movsb\n" | |
284 | "100:\n" | |
285 | ".section .fixup,\"ax\"\n" | |
286 | "101: lea 0(%%eax,%0,4),%0\n" | |
287 | " jmp 100b\n" | |
288 | ".previous\n" | |
289 | ".section __ex_table,\"a\"\n" | |
290 | " .align 4\n" | |
291 | " .long 1b,100b\n" | |
292 | " .long 2b,100b\n" | |
293 | " .long 3b,100b\n" | |
294 | " .long 4b,100b\n" | |
295 | " .long 5b,100b\n" | |
296 | " .long 6b,100b\n" | |
297 | " .long 7b,100b\n" | |
298 | " .long 8b,100b\n" | |
299 | " .long 9b,100b\n" | |
300 | " .long 10b,100b\n" | |
301 | " .long 11b,100b\n" | |
302 | " .long 12b,100b\n" | |
303 | " .long 13b,100b\n" | |
304 | " .long 14b,100b\n" | |
305 | " .long 15b,100b\n" | |
306 | " .long 16b,100b\n" | |
307 | " .long 17b,100b\n" | |
308 | " .long 18b,100b\n" | |
309 | " .long 19b,100b\n" | |
310 | " .long 20b,100b\n" | |
311 | " .long 21b,100b\n" | |
312 | " .long 22b,100b\n" | |
313 | " .long 23b,100b\n" | |
314 | " .long 24b,100b\n" | |
315 | " .long 25b,100b\n" | |
316 | " .long 26b,100b\n" | |
317 | " .long 27b,100b\n" | |
318 | " .long 28b,100b\n" | |
319 | " .long 29b,100b\n" | |
320 | " .long 30b,100b\n" | |
321 | " .long 31b,100b\n" | |
322 | " .long 32b,100b\n" | |
323 | " .long 33b,100b\n" | |
324 | " .long 34b,100b\n" | |
325 | " .long 35b,100b\n" | |
326 | " .long 36b,100b\n" | |
327 | " .long 37b,100b\n" | |
328 | " .long 99b,101b\n" | |
329 | ".previous" | |
330 | : "=&c"(size), "=&D" (d0), "=&S" (d1) | |
331 | : "1"(to), "2"(from), "0"(size) | |
332 | : "eax", "edx", "memory"); | |
333 | return size; | |
334 | } | |
335 | ||
336 | static unsigned long | |
337 | __copy_user_zeroing_intel(void *to, const void __user *from, unsigned long size) | |
338 | { | |
339 | int d0, d1; | |
340 | __asm__ __volatile__( | |
341 | " .align 2,0x90\n" | |
342 | "0: movl 32(%4), %%eax\n" | |
3f50dbc1 PC |
343 | " cmpl $67, %0\n" |
344 | " jbe 2f\n" | |
1da177e4 | 345 | "1: movl 64(%4), %%eax\n" |
3f50dbc1 PC |
346 | " .align 2,0x90\n" |
347 | "2: movl 0(%4), %%eax\n" | |
348 | "21: movl 4(%4), %%edx\n" | |
349 | " movl %%eax, 0(%3)\n" | |
350 | " movl %%edx, 4(%3)\n" | |
351 | "3: movl 8(%4), %%eax\n" | |
352 | "31: movl 12(%4),%%edx\n" | |
353 | " movl %%eax, 8(%3)\n" | |
1da177e4 LT |
354 | " movl %%edx, 12(%3)\n" |
355 | "4: movl 16(%4), %%eax\n" | |
356 | "41: movl 20(%4), %%edx\n" | |
357 | " movl %%eax, 16(%3)\n" | |
358 | " movl %%edx, 20(%3)\n" | |
359 | "10: movl 24(%4), %%eax\n" | |
360 | "51: movl 28(%4), %%edx\n" | |
361 | " movl %%eax, 24(%3)\n" | |
362 | " movl %%edx, 28(%3)\n" | |
363 | "11: movl 32(%4), %%eax\n" | |
364 | "61: movl 36(%4), %%edx\n" | |
365 | " movl %%eax, 32(%3)\n" | |
366 | " movl %%edx, 36(%3)\n" | |
367 | "12: movl 40(%4), %%eax\n" | |
368 | "71: movl 44(%4), %%edx\n" | |
369 | " movl %%eax, 40(%3)\n" | |
370 | " movl %%edx, 44(%3)\n" | |
371 | "13: movl 48(%4), %%eax\n" | |
372 | "81: movl 52(%4), %%edx\n" | |
373 | " movl %%eax, 48(%3)\n" | |
374 | " movl %%edx, 52(%3)\n" | |
375 | "14: movl 56(%4), %%eax\n" | |
376 | "91: movl 60(%4), %%edx\n" | |
377 | " movl %%eax, 56(%3)\n" | |
378 | " movl %%edx, 60(%3)\n" | |
3f50dbc1 PC |
379 | " addl $-64, %0\n" |
380 | " addl $64, %4\n" | |
381 | " addl $64, %3\n" | |
382 | " cmpl $63, %0\n" | |
383 | " ja 0b\n" | |
384 | "5: movl %0, %%eax\n" | |
385 | " shrl $2, %0\n" | |
386 | " andl $3, %%eax\n" | |
387 | " cld\n" | |
388 | "6: rep; movsl\n" | |
1da177e4 | 389 | " movl %%eax,%0\n" |
3f50dbc1 PC |
390 | "7: rep; movsb\n" |
391 | "8:\n" | |
1da177e4 | 392 | ".section .fixup,\"ax\"\n" |
3f50dbc1 PC |
393 | "9: lea 0(%%eax,%0,4),%0\n" |
394 | "16: pushl %0\n" | |
395 | " pushl %%eax\n" | |
1da177e4 | 396 | " xorl %%eax,%%eax\n" |
3f50dbc1 PC |
397 | " rep; stosb\n" |
398 | " popl %%eax\n" | |
399 | " popl %0\n" | |
400 | " jmp 8b\n" | |
401 | ".previous\n" | |
1da177e4 | 402 | ".section __ex_table,\"a\"\n" |
3f50dbc1 PC |
403 | " .align 4\n" |
404 | " .long 0b,16b\n" | |
1da177e4 LT |
405 | " .long 1b,16b\n" |
406 | " .long 2b,16b\n" | |
407 | " .long 21b,16b\n" | |
3f50dbc1 | 408 | " .long 3b,16b\n" |
1da177e4 | 409 | " .long 31b,16b\n" |
3f50dbc1 | 410 | " .long 4b,16b\n" |
1da177e4 LT |
411 | " .long 41b,16b\n" |
412 | " .long 10b,16b\n" | |
413 | " .long 51b,16b\n" | |
414 | " .long 11b,16b\n" | |
415 | " .long 61b,16b\n" | |
416 | " .long 12b,16b\n" | |
417 | " .long 71b,16b\n" | |
418 | " .long 13b,16b\n" | |
419 | " .long 81b,16b\n" | |
420 | " .long 14b,16b\n" | |
421 | " .long 91b,16b\n" | |
3f50dbc1 PC |
422 | " .long 6b,9b\n" |
423 | " .long 7b,16b\n" | |
424 | ".previous" | |
1da177e4 LT |
425 | : "=&c"(size), "=&D" (d0), "=&S" (d1) |
426 | : "1"(to), "2"(from), "0"(size) | |
427 | : "eax", "edx", "memory"); | |
428 | return size; | |
429 | } | |
c22ce143 HY |
430 | |
431 | /* | |
432 | * Non Temporal Hint version of __copy_user_zeroing_intel. It is cache aware. | |
433 | * hyoshiok@miraclelinux.com | |
434 | */ | |
435 | ||
436 | static unsigned long __copy_user_zeroing_intel_nocache(void *to, | |
437 | const void __user *from, unsigned long size) | |
438 | { | |
3f50dbc1 | 439 | int d0, d1; |
c22ce143 HY |
440 | |
441 | __asm__ __volatile__( | |
442 | " .align 2,0x90\n" | |
443 | "0: movl 32(%4), %%eax\n" | |
444 | " cmpl $67, %0\n" | |
445 | " jbe 2f\n" | |
446 | "1: movl 64(%4), %%eax\n" | |
447 | " .align 2,0x90\n" | |
448 | "2: movl 0(%4), %%eax\n" | |
449 | "21: movl 4(%4), %%edx\n" | |
450 | " movnti %%eax, 0(%3)\n" | |
451 | " movnti %%edx, 4(%3)\n" | |
452 | "3: movl 8(%4), %%eax\n" | |
453 | "31: movl 12(%4),%%edx\n" | |
454 | " movnti %%eax, 8(%3)\n" | |
455 | " movnti %%edx, 12(%3)\n" | |
456 | "4: movl 16(%4), %%eax\n" | |
457 | "41: movl 20(%4), %%edx\n" | |
458 | " movnti %%eax, 16(%3)\n" | |
459 | " movnti %%edx, 20(%3)\n" | |
460 | "10: movl 24(%4), %%eax\n" | |
461 | "51: movl 28(%4), %%edx\n" | |
462 | " movnti %%eax, 24(%3)\n" | |
463 | " movnti %%edx, 28(%3)\n" | |
464 | "11: movl 32(%4), %%eax\n" | |
465 | "61: movl 36(%4), %%edx\n" | |
466 | " movnti %%eax, 32(%3)\n" | |
467 | " movnti %%edx, 36(%3)\n" | |
468 | "12: movl 40(%4), %%eax\n" | |
469 | "71: movl 44(%4), %%edx\n" | |
470 | " movnti %%eax, 40(%3)\n" | |
471 | " movnti %%edx, 44(%3)\n" | |
472 | "13: movl 48(%4), %%eax\n" | |
473 | "81: movl 52(%4), %%edx\n" | |
474 | " movnti %%eax, 48(%3)\n" | |
475 | " movnti %%edx, 52(%3)\n" | |
476 | "14: movl 56(%4), %%eax\n" | |
477 | "91: movl 60(%4), %%edx\n" | |
478 | " movnti %%eax, 56(%3)\n" | |
479 | " movnti %%edx, 60(%3)\n" | |
480 | " addl $-64, %0\n" | |
481 | " addl $64, %4\n" | |
482 | " addl $64, %3\n" | |
483 | " cmpl $63, %0\n" | |
484 | " ja 0b\n" | |
485 | " sfence \n" | |
486 | "5: movl %0, %%eax\n" | |
487 | " shrl $2, %0\n" | |
488 | " andl $3, %%eax\n" | |
489 | " cld\n" | |
490 | "6: rep; movsl\n" | |
491 | " movl %%eax,%0\n" | |
492 | "7: rep; movsb\n" | |
493 | "8:\n" | |
494 | ".section .fixup,\"ax\"\n" | |
495 | "9: lea 0(%%eax,%0,4),%0\n" | |
496 | "16: pushl %0\n" | |
497 | " pushl %%eax\n" | |
498 | " xorl %%eax,%%eax\n" | |
499 | " rep; stosb\n" | |
500 | " popl %%eax\n" | |
501 | " popl %0\n" | |
502 | " jmp 8b\n" | |
503 | ".previous\n" | |
504 | ".section __ex_table,\"a\"\n" | |
505 | " .align 4\n" | |
506 | " .long 0b,16b\n" | |
507 | " .long 1b,16b\n" | |
508 | " .long 2b,16b\n" | |
509 | " .long 21b,16b\n" | |
510 | " .long 3b,16b\n" | |
511 | " .long 31b,16b\n" | |
512 | " .long 4b,16b\n" | |
513 | " .long 41b,16b\n" | |
514 | " .long 10b,16b\n" | |
515 | " .long 51b,16b\n" | |
516 | " .long 11b,16b\n" | |
517 | " .long 61b,16b\n" | |
518 | " .long 12b,16b\n" | |
519 | " .long 71b,16b\n" | |
520 | " .long 13b,16b\n" | |
521 | " .long 81b,16b\n" | |
522 | " .long 14b,16b\n" | |
523 | " .long 91b,16b\n" | |
524 | " .long 6b,9b\n" | |
525 | " .long 7b,16b\n" | |
526 | ".previous" | |
527 | : "=&c"(size), "=&D" (d0), "=&S" (d1) | |
528 | : "1"(to), "2"(from), "0"(size) | |
529 | : "eax", "edx", "memory"); | |
530 | return size; | |
531 | } | |
532 | ||
7c12d811 N |
533 | static unsigned long __copy_user_intel_nocache(void *to, |
534 | const void __user *from, unsigned long size) | |
535 | { | |
3f50dbc1 | 536 | int d0, d1; |
7c12d811 N |
537 | |
538 | __asm__ __volatile__( | |
539 | " .align 2,0x90\n" | |
540 | "0: movl 32(%4), %%eax\n" | |
541 | " cmpl $67, %0\n" | |
542 | " jbe 2f\n" | |
543 | "1: movl 64(%4), %%eax\n" | |
544 | " .align 2,0x90\n" | |
545 | "2: movl 0(%4), %%eax\n" | |
546 | "21: movl 4(%4), %%edx\n" | |
547 | " movnti %%eax, 0(%3)\n" | |
548 | " movnti %%edx, 4(%3)\n" | |
549 | "3: movl 8(%4), %%eax\n" | |
550 | "31: movl 12(%4),%%edx\n" | |
551 | " movnti %%eax, 8(%3)\n" | |
552 | " movnti %%edx, 12(%3)\n" | |
553 | "4: movl 16(%4), %%eax\n" | |
554 | "41: movl 20(%4), %%edx\n" | |
555 | " movnti %%eax, 16(%3)\n" | |
556 | " movnti %%edx, 20(%3)\n" | |
557 | "10: movl 24(%4), %%eax\n" | |
558 | "51: movl 28(%4), %%edx\n" | |
559 | " movnti %%eax, 24(%3)\n" | |
560 | " movnti %%edx, 28(%3)\n" | |
561 | "11: movl 32(%4), %%eax\n" | |
562 | "61: movl 36(%4), %%edx\n" | |
563 | " movnti %%eax, 32(%3)\n" | |
564 | " movnti %%edx, 36(%3)\n" | |
565 | "12: movl 40(%4), %%eax\n" | |
566 | "71: movl 44(%4), %%edx\n" | |
567 | " movnti %%eax, 40(%3)\n" | |
568 | " movnti %%edx, 44(%3)\n" | |
569 | "13: movl 48(%4), %%eax\n" | |
570 | "81: movl 52(%4), %%edx\n" | |
571 | " movnti %%eax, 48(%3)\n" | |
572 | " movnti %%edx, 52(%3)\n" | |
573 | "14: movl 56(%4), %%eax\n" | |
574 | "91: movl 60(%4), %%edx\n" | |
575 | " movnti %%eax, 56(%3)\n" | |
576 | " movnti %%edx, 60(%3)\n" | |
577 | " addl $-64, %0\n" | |
578 | " addl $64, %4\n" | |
579 | " addl $64, %3\n" | |
580 | " cmpl $63, %0\n" | |
581 | " ja 0b\n" | |
582 | " sfence \n" | |
583 | "5: movl %0, %%eax\n" | |
584 | " shrl $2, %0\n" | |
585 | " andl $3, %%eax\n" | |
586 | " cld\n" | |
587 | "6: rep; movsl\n" | |
588 | " movl %%eax,%0\n" | |
589 | "7: rep; movsb\n" | |
590 | "8:\n" | |
591 | ".section .fixup,\"ax\"\n" | |
592 | "9: lea 0(%%eax,%0,4),%0\n" | |
593 | "16: jmp 8b\n" | |
594 | ".previous\n" | |
595 | ".section __ex_table,\"a\"\n" | |
596 | " .align 4\n" | |
597 | " .long 0b,16b\n" | |
598 | " .long 1b,16b\n" | |
599 | " .long 2b,16b\n" | |
600 | " .long 21b,16b\n" | |
601 | " .long 3b,16b\n" | |
602 | " .long 31b,16b\n" | |
603 | " .long 4b,16b\n" | |
604 | " .long 41b,16b\n" | |
605 | " .long 10b,16b\n" | |
606 | " .long 51b,16b\n" | |
607 | " .long 11b,16b\n" | |
608 | " .long 61b,16b\n" | |
609 | " .long 12b,16b\n" | |
610 | " .long 71b,16b\n" | |
611 | " .long 13b,16b\n" | |
612 | " .long 81b,16b\n" | |
613 | " .long 14b,16b\n" | |
614 | " .long 91b,16b\n" | |
615 | " .long 6b,9b\n" | |
616 | " .long 7b,16b\n" | |
617 | ".previous" | |
618 | : "=&c"(size), "=&D" (d0), "=&S" (d1) | |
619 | : "1"(to), "2"(from), "0"(size) | |
620 | : "eax", "edx", "memory"); | |
621 | return size; | |
622 | } | |
623 | ||
1da177e4 | 624 | #else |
c22ce143 | 625 | |
1da177e4 LT |
626 | /* |
627 | * Leave these declared but undefined. They should not be any references to | |
628 | * them | |
629 | */ | |
c22ce143 HY |
630 | unsigned long __copy_user_zeroing_intel(void *to, const void __user *from, |
631 | unsigned long size); | |
632 | unsigned long __copy_user_intel(void __user *to, const void *from, | |
633 | unsigned long size); | |
634 | unsigned long __copy_user_zeroing_intel_nocache(void *to, | |
635 | const void __user *from, unsigned long size); | |
1da177e4 LT |
636 | #endif /* CONFIG_X86_INTEL_USERCOPY */ |
637 | ||
638 | /* Generic arbitrary sized copy. */ | |
3f50dbc1 | 639 | #define __copy_user(to, from, size) \ |
1da177e4 LT |
640 | do { \ |
641 | int __d0, __d1, __d2; \ | |
642 | __asm__ __volatile__( \ | |
643 | " cmp $7,%0\n" \ | |
644 | " jbe 1f\n" \ | |
645 | " movl %1,%0\n" \ | |
646 | " negl %0\n" \ | |
647 | " andl $7,%0\n" \ | |
648 | " subl %0,%3\n" \ | |
649 | "4: rep; movsb\n" \ | |
650 | " movl %3,%0\n" \ | |
651 | " shrl $2,%0\n" \ | |
652 | " andl $3,%3\n" \ | |
653 | " .align 2,0x90\n" \ | |
654 | "0: rep; movsl\n" \ | |
655 | " movl %3,%0\n" \ | |
656 | "1: rep; movsb\n" \ | |
657 | "2:\n" \ | |
658 | ".section .fixup,\"ax\"\n" \ | |
659 | "5: addl %3,%0\n" \ | |
660 | " jmp 2b\n" \ | |
661 | "3: lea 0(%3,%0,4),%0\n" \ | |
662 | " jmp 2b\n" \ | |
663 | ".previous\n" \ | |
664 | ".section __ex_table,\"a\"\n" \ | |
665 | " .align 4\n" \ | |
666 | " .long 4b,5b\n" \ | |
667 | " .long 0b,3b\n" \ | |
668 | " .long 1b,2b\n" \ | |
669 | ".previous" \ | |
670 | : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \ | |
671 | : "3"(size), "0"(size), "1"(to), "2"(from) \ | |
672 | : "memory"); \ | |
673 | } while (0) | |
674 | ||
3f50dbc1 | 675 | #define __copy_user_zeroing(to, from, size) \ |
1da177e4 LT |
676 | do { \ |
677 | int __d0, __d1, __d2; \ | |
678 | __asm__ __volatile__( \ | |
679 | " cmp $7,%0\n" \ | |
680 | " jbe 1f\n" \ | |
681 | " movl %1,%0\n" \ | |
682 | " negl %0\n" \ | |
683 | " andl $7,%0\n" \ | |
684 | " subl %0,%3\n" \ | |
685 | "4: rep; movsb\n" \ | |
686 | " movl %3,%0\n" \ | |
687 | " shrl $2,%0\n" \ | |
688 | " andl $3,%3\n" \ | |
689 | " .align 2,0x90\n" \ | |
690 | "0: rep; movsl\n" \ | |
691 | " movl %3,%0\n" \ | |
692 | "1: rep; movsb\n" \ | |
693 | "2:\n" \ | |
694 | ".section .fixup,\"ax\"\n" \ | |
695 | "5: addl %3,%0\n" \ | |
696 | " jmp 6f\n" \ | |
697 | "3: lea 0(%3,%0,4),%0\n" \ | |
698 | "6: pushl %0\n" \ | |
699 | " pushl %%eax\n" \ | |
700 | " xorl %%eax,%%eax\n" \ | |
701 | " rep; stosb\n" \ | |
702 | " popl %%eax\n" \ | |
703 | " popl %0\n" \ | |
704 | " jmp 2b\n" \ | |
705 | ".previous\n" \ | |
706 | ".section __ex_table,\"a\"\n" \ | |
707 | " .align 4\n" \ | |
708 | " .long 4b,5b\n" \ | |
709 | " .long 0b,3b\n" \ | |
710 | " .long 1b,6b\n" \ | |
711 | ".previous" \ | |
712 | : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \ | |
713 | : "3"(size), "0"(size), "1"(to), "2"(from) \ | |
714 | : "memory"); \ | |
715 | } while (0) | |
716 | ||
c22ce143 HY |
717 | unsigned long __copy_to_user_ll(void __user *to, const void *from, |
718 | unsigned long n) | |
1da177e4 | 719 | { |
1da177e4 LT |
720 | #ifndef CONFIG_X86_WP_WORKS_OK |
721 | if (unlikely(boot_cpu_data.wp_works_ok == 0) && | |
3f50dbc1 | 722 | ((unsigned long)to) < TASK_SIZE) { |
b6a8b316 TG |
723 | /* |
724 | * When we are in an atomic section (see | |
725 | * mm/filemap.c:file_read_actor), return the full | |
726 | * length to take the slow path. | |
727 | */ | |
728 | if (in_atomic()) | |
729 | return n; | |
730 | ||
3f50dbc1 | 731 | /* |
1da177e4 LT |
732 | * CPU does not honor the WP bit when writing |
733 | * from supervisory mode, and due to preemption or SMP, | |
734 | * the page tables can change at any time. | |
735 | * Do it manually. Manfred <manfred@colorfullife.com> | |
736 | */ | |
737 | while (n) { | |
3f50dbc1 | 738 | unsigned long offset = ((unsigned long)to)%PAGE_SIZE; |
1da177e4 LT |
739 | unsigned long len = PAGE_SIZE - offset; |
740 | int retval; | |
741 | struct page *pg; | |
742 | void *maddr; | |
3f50dbc1 | 743 | |
1da177e4 LT |
744 | if (len > n) |
745 | len = n; | |
746 | ||
747 | survive: | |
748 | down_read(¤t->mm->mmap_sem); | |
749 | retval = get_user_pages(current, current->mm, | |
3f50dbc1 | 750 | (unsigned long)to, 1, 1, 0, &pg, NULL); |
1da177e4 | 751 | |
b460cbc5 | 752 | if (retval == -ENOMEM && is_global_init(current)) { |
1da177e4 | 753 | up_read(¤t->mm->mmap_sem); |
8aa7e847 | 754 | congestion_wait(BLK_RW_ASYNC, HZ/50); |
1da177e4 LT |
755 | goto survive; |
756 | } | |
757 | ||
758 | if (retval != 1) { | |
759 | up_read(¤t->mm->mmap_sem); | |
3f50dbc1 PC |
760 | break; |
761 | } | |
1da177e4 LT |
762 | |
763 | maddr = kmap_atomic(pg, KM_USER0); | |
764 | memcpy(maddr + offset, from, len); | |
765 | kunmap_atomic(maddr, KM_USER0); | |
766 | set_page_dirty_lock(pg); | |
767 | put_page(pg); | |
768 | up_read(¤t->mm->mmap_sem); | |
769 | ||
770 | from += len; | |
771 | to += len; | |
772 | n -= len; | |
773 | } | |
774 | return n; | |
775 | } | |
776 | #endif | |
777 | if (movsl_is_ok(to, from, n)) | |
778 | __copy_user(to, from, n); | |
779 | else | |
780 | n = __copy_user_intel(to, from, n); | |
781 | return n; | |
782 | } | |
129f6946 | 783 | EXPORT_SYMBOL(__copy_to_user_ll); |
1da177e4 | 784 | |
c22ce143 HY |
785 | unsigned long __copy_from_user_ll(void *to, const void __user *from, |
786 | unsigned long n) | |
1da177e4 | 787 | { |
1da177e4 LT |
788 | if (movsl_is_ok(to, from, n)) |
789 | __copy_user_zeroing(to, from, n); | |
790 | else | |
791 | n = __copy_user_zeroing_intel(to, from, n); | |
792 | return n; | |
793 | } | |
129f6946 | 794 | EXPORT_SYMBOL(__copy_from_user_ll); |
1da177e4 | 795 | |
7c12d811 N |
796 | unsigned long __copy_from_user_ll_nozero(void *to, const void __user *from, |
797 | unsigned long n) | |
798 | { | |
7c12d811 N |
799 | if (movsl_is_ok(to, from, n)) |
800 | __copy_user(to, from, n); | |
801 | else | |
802 | n = __copy_user_intel((void __user *)to, | |
803 | (const void *)from, n); | |
804 | return n; | |
805 | } | |
806 | EXPORT_SYMBOL(__copy_from_user_ll_nozero); | |
807 | ||
c22ce143 HY |
808 | unsigned long __copy_from_user_ll_nocache(void *to, const void __user *from, |
809 | unsigned long n) | |
810 | { | |
c22ce143 | 811 | #ifdef CONFIG_X86_INTEL_USERCOPY |
3f50dbc1 PC |
812 | if (n > 64 && cpu_has_xmm2) |
813 | n = __copy_user_zeroing_intel_nocache(to, from, n); | |
c22ce143 HY |
814 | else |
815 | __copy_user_zeroing(to, from, n); | |
816 | #else | |
3f50dbc1 | 817 | __copy_user_zeroing(to, from, n); |
c22ce143 HY |
818 | #endif |
819 | return n; | |
820 | } | |
914c8269 | 821 | EXPORT_SYMBOL(__copy_from_user_ll_nocache); |
c22ce143 | 822 | |
7c12d811 N |
823 | unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from, |
824 | unsigned long n) | |
825 | { | |
7c12d811 | 826 | #ifdef CONFIG_X86_INTEL_USERCOPY |
3f50dbc1 PC |
827 | if (n > 64 && cpu_has_xmm2) |
828 | n = __copy_user_intel_nocache(to, from, n); | |
7c12d811 N |
829 | else |
830 | __copy_user(to, from, n); | |
831 | #else | |
3f50dbc1 | 832 | __copy_user(to, from, n); |
7c12d811 N |
833 | #endif |
834 | return n; | |
835 | } | |
914c8269 | 836 | EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero); |
7c12d811 | 837 | |
1da177e4 LT |
838 | /** |
839 | * copy_to_user: - Copy a block of data into user space. | |
840 | * @to: Destination address, in user space. | |
841 | * @from: Source address, in kernel space. | |
842 | * @n: Number of bytes to copy. | |
843 | * | |
844 | * Context: User context only. This function may sleep. | |
845 | * | |
846 | * Copy data from kernel space to user space. | |
847 | * | |
848 | * Returns number of bytes that could not be copied. | |
849 | * On success, this will be zero. | |
850 | */ | |
851 | unsigned long | |
852 | copy_to_user(void __user *to, const void *from, unsigned long n) | |
853 | { | |
1da177e4 LT |
854 | if (access_ok(VERIFY_WRITE, to, n)) |
855 | n = __copy_to_user(to, from, n); | |
856 | return n; | |
857 | } | |
858 | EXPORT_SYMBOL(copy_to_user); | |
859 | ||
860 | /** | |
861 | * copy_from_user: - Copy a block of data from user space. | |
862 | * @to: Destination address, in kernel space. | |
863 | * @from: Source address, in user space. | |
864 | * @n: Number of bytes to copy. | |
865 | * | |
866 | * Context: User context only. This function may sleep. | |
867 | * | |
868 | * Copy data from user space to kernel space. | |
869 | * | |
870 | * Returns number of bytes that could not be copied. | |
871 | * On success, this will be zero. | |
872 | * | |
873 | * If some data could not be copied, this function will pad the copied | |
874 | * data to the requested size using zero bytes. | |
875 | */ | |
876 | unsigned long | |
877 | copy_from_user(void *to, const void __user *from, unsigned long n) | |
878 | { | |
1da177e4 LT |
879 | if (access_ok(VERIFY_READ, from, n)) |
880 | n = __copy_from_user(to, from, n); | |
881 | else | |
882 | memset(to, 0, n); | |
883 | return n; | |
884 | } | |
885 | EXPORT_SYMBOL(copy_from_user); |