]>
Commit | Line | Data |
---|---|---|
ad2fc2cd VM |
1 | /* |
2 | * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com> | |
3 | * Copyright 2002 Andi Kleen, SuSE Labs. | |
1da177e4 | 4 | * Subject to the GNU Public License v2. |
ad2fc2cd VM |
5 | * |
6 | * Functions to copy from and to user space. | |
7 | */ | |
1da177e4 | 8 | |
8d379dad JB |
9 | #include <linux/linkage.h> |
10 | #include <asm/dwarf2.h> | |
11 | ||
7bcd3f34 AK |
12 | #define FIX_ALIGNMENT 1 |
13 | ||
3022d734 AK |
14 | #include <asm/current.h> |
15 | #include <asm/asm-offsets.h> | |
16 | #include <asm/thread_info.h> | |
17 | #include <asm/cpufeature.h> | |
18 | ||
19 | .macro ALTERNATIVE_JUMP feature,orig,alt | |
20 | 0: | |
21 | .byte 0xe9 /* 32bit jump */ | |
22 | .long \orig-1f /* by default jump to orig */ | |
23 | 1: | |
24 | .section .altinstr_replacement,"ax" | |
ad2fc2cd | 25 | 2: .byte 0xe9 /* near jump with 32bit immediate */ |
3022d734 AK |
26 | .long \alt-1b /* offset */ /* or alternatively to alt */ |
27 | .previous | |
28 | .section .altinstructions,"a" | |
29 | .align 8 | |
30 | .quad 0b | |
31 | .quad 2b | |
ad2fc2cd | 32 | .byte \feature /* when feature is set */ |
3022d734 AK |
33 | .byte 5 |
34 | .byte 5 | |
35 | .previous | |
36 | .endm | |
1da177e4 | 37 | |
ad2fc2cd VM |
38 | .macro ALIGN_DESTINATION |
39 | #ifdef FIX_ALIGNMENT | |
40 | /* check for bad alignment of destination */ | |
41 | movl %edi,%ecx | |
42 | andl $7,%ecx | |
43 | jz 102f /* already aligned */ | |
44 | subl $8,%ecx | |
45 | negl %ecx | |
46 | subl %ecx,%edx | |
47 | 100: movb (%rsi),%al | |
48 | 101: movb %al,(%rdi) | |
49 | incq %rsi | |
50 | incq %rdi | |
51 | decl %ecx | |
52 | jnz 100b | |
53 | 102: | |
54 | .section .fixup,"ax" | |
afd962a9 | 55 | 103: addl %ecx,%edx /* ecx is zerorest also */ |
ad2fc2cd VM |
56 | jmp copy_user_handle_tail |
57 | .previous | |
58 | ||
59 | .section __ex_table,"a" | |
60 | .align 8 | |
61 | .quad 100b,103b | |
62 | .quad 101b,103b | |
63 | .previous | |
64 | #endif | |
65 | .endm | |
66 | ||
67 | /* Standard copy_to_user with segment limit checking */ | |
8d379dad JB |
68 | ENTRY(copy_to_user) |
69 | CFI_STARTPROC | |
1da177e4 LT |
70 | GET_THREAD_INFO(%rax) |
71 | movq %rdi,%rcx | |
72 | addq %rdx,%rcx | |
ad2fc2cd | 73 | jc bad_to_user |
26ccb8a7 | 74 | cmpq TI_addr_limit(%rax),%rcx |
1da177e4 | 75 | jae bad_to_user |
3022d734 | 76 | ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string |
8d379dad | 77 | CFI_ENDPROC |
3fd382ce | 78 | ENDPROC(copy_to_user) |
7bcd3f34 | 79 | |
ad2fc2cd VM |
80 | /* Standard copy_from_user with segment limit checking */ |
81 | ENTRY(copy_from_user) | |
3022d734 | 82 | CFI_STARTPROC |
ad2fc2cd VM |
83 | GET_THREAD_INFO(%rax) |
84 | movq %rsi,%rcx | |
85 | addq %rdx,%rcx | |
86 | jc bad_from_user | |
87 | cmpq TI_addr_limit(%rax),%rcx | |
88 | jae bad_from_user | |
3022d734 AK |
89 | ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string |
90 | CFI_ENDPROC | |
ad2fc2cd | 91 | ENDPROC(copy_from_user) |
3022d734 | 92 | |
ad2fc2cd | 93 | ENTRY(copy_user_generic) |
3022d734 | 94 | CFI_STARTPROC |
3022d734 AK |
95 | ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string |
96 | CFI_ENDPROC | |
ad2fc2cd | 97 | ENDPROC(copy_user_generic) |
1da177e4 | 98 | |
ad2fc2cd | 99 | ENTRY(__copy_from_user_inatomic) |
8d379dad | 100 | CFI_STARTPROC |
3022d734 | 101 | ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string |
8d379dad | 102 | CFI_ENDPROC |
ad2fc2cd VM |
103 | ENDPROC(__copy_from_user_inatomic) |
104 | ||
1da177e4 LT |
105 | .section .fixup,"ax" |
106 | /* must zero dest */ | |
ad2fc2cd | 107 | ENTRY(bad_from_user) |
1da177e4 | 108 | bad_from_user: |
8d379dad | 109 | CFI_STARTPROC |
1da177e4 LT |
110 | movl %edx,%ecx |
111 | xorl %eax,%eax | |
112 | rep | |
113 | stosb | |
114 | bad_to_user: | |
ad2fc2cd | 115 | movl %edx,%eax |
1da177e4 | 116 | ret |
8d379dad | 117 | CFI_ENDPROC |
ad2fc2cd | 118 | ENDPROC(bad_from_user) |
1da177e4 | 119 | .previous |
ad2fc2cd | 120 | |
1da177e4 | 121 | /* |
3022d734 | 122 | * copy_user_generic_unrolled - memory copy with exception handling. |
ad2fc2cd VM |
123 | * This version is for CPUs like P4 that don't have efficient micro |
124 | * code for rep movsq | |
125 | * | |
126 | * Input: | |
1da177e4 LT |
127 | * rdi destination |
128 | * rsi source | |
129 | * rdx count | |
130 | * | |
ad2fc2cd VM |
131 | * Output: |
132 | * eax uncopied bytes or 0 if successfull. | |
1da177e4 | 133 | */ |
3022d734 | 134 | ENTRY(copy_user_generic_unrolled) |
8d379dad | 135 | CFI_STARTPROC |
ad2fc2cd VM |
136 | cmpl $8,%edx |
137 | jb 20f /* less then 8 bytes, go to byte copy loop */ | |
138 | ALIGN_DESTINATION | |
139 | movl %edx,%ecx | |
140 | andl $63,%edx | |
141 | shrl $6,%ecx | |
142 | jz 17f | |
143 | 1: movq (%rsi),%r8 | |
144 | 2: movq 1*8(%rsi),%r9 | |
145 | 3: movq 2*8(%rsi),%r10 | |
146 | 4: movq 3*8(%rsi),%r11 | |
147 | 5: movq %r8,(%rdi) | |
148 | 6: movq %r9,1*8(%rdi) | |
149 | 7: movq %r10,2*8(%rdi) | |
150 | 8: movq %r11,3*8(%rdi) | |
151 | 9: movq 4*8(%rsi),%r8 | |
152 | 10: movq 5*8(%rsi),%r9 | |
153 | 11: movq 6*8(%rsi),%r10 | |
154 | 12: movq 7*8(%rsi),%r11 | |
155 | 13: movq %r8,4*8(%rdi) | |
156 | 14: movq %r9,5*8(%rdi) | |
157 | 15: movq %r10,6*8(%rdi) | |
158 | 16: movq %r11,7*8(%rdi) | |
7bcd3f34 AK |
159 | leaq 64(%rsi),%rsi |
160 | leaq 64(%rdi),%rdi | |
7bcd3f34 | 161 | decl %ecx |
ad2fc2cd VM |
162 | jnz 1b |
163 | 17: movl %edx,%ecx | |
164 | andl $7,%edx | |
165 | shrl $3,%ecx | |
166 | jz 20f | |
167 | 18: movq (%rsi),%r8 | |
168 | 19: movq %r8,(%rdi) | |
7bcd3f34 | 169 | leaq 8(%rsi),%rsi |
ad2fc2cd VM |
170 | leaq 8(%rdi),%rdi |
171 | decl %ecx | |
172 | jnz 18b | |
173 | 20: andl %edx,%edx | |
174 | jz 23f | |
7bcd3f34 | 175 | movl %edx,%ecx |
ad2fc2cd VM |
176 | 21: movb (%rsi),%al |
177 | 22: movb %al,(%rdi) | |
7bcd3f34 | 178 | incq %rsi |
ad2fc2cd | 179 | incq %rdi |
7bcd3f34 | 180 | decl %ecx |
ad2fc2cd VM |
181 | jnz 21b |
182 | 23: xor %eax,%eax | |
7bcd3f34 AK |
183 | ret |
184 | ||
ad2fc2cd VM |
185 | .section .fixup,"ax" |
186 | 30: shll $6,%ecx | |
187 | addl %ecx,%edx | |
188 | jmp 60f | |
27cb0a75 | 189 | 40: lea (%rdx,%rcx,8),%rdx |
ad2fc2cd VM |
190 | jmp 60f |
191 | 50: movl %ecx,%edx | |
192 | 60: jmp copy_user_handle_tail /* ecx is zerorest also */ | |
193 | .previous | |
7bcd3f34 | 194 | |
7bcd3f34 AK |
195 | .section __ex_table,"a" |
196 | .align 8 | |
ad2fc2cd VM |
197 | .quad 1b,30b |
198 | .quad 2b,30b | |
199 | .quad 3b,30b | |
200 | .quad 4b,30b | |
201 | .quad 5b,30b | |
202 | .quad 6b,30b | |
203 | .quad 7b,30b | |
204 | .quad 8b,30b | |
205 | .quad 9b,30b | |
206 | .quad 10b,30b | |
207 | .quad 11b,30b | |
208 | .quad 12b,30b | |
209 | .quad 13b,30b | |
210 | .quad 14b,30b | |
211 | .quad 15b,30b | |
212 | .quad 16b,30b | |
213 | .quad 18b,40b | |
214 | .quad 19b,40b | |
215 | .quad 21b,50b | |
216 | .quad 22b,50b | |
7bcd3f34 | 217 | .previous |
8d379dad | 218 | CFI_ENDPROC |
ad2fc2cd | 219 | ENDPROC(copy_user_generic_unrolled) |
8d379dad | 220 | |
ad2fc2cd VM |
221 | /* Some CPUs run faster using the string copy instructions. |
222 | * This is also a lot simpler. Use them when possible. | |
223 | * | |
224 | * Only 4GB of copy is supported. This shouldn't be a problem | |
225 | * because the kernel normally only writes from/to page sized chunks | |
226 | * even if user space passed a longer buffer. | |
227 | * And more would be dangerous because both Intel and AMD have | |
228 | * errata with rep movsq > 4GB. If someone feels the need to fix | |
229 | * this please consider this. | |
230 | * | |
231 | * Input: | |
232 | * rdi destination | |
233 | * rsi source | |
234 | * rdx count | |
235 | * | |
236 | * Output: | |
237 | * eax uncopied bytes or 0 if successful. | |
238 | */ | |
3022d734 | 239 | ENTRY(copy_user_generic_string) |
8d379dad | 240 | CFI_STARTPROC |
ad2fc2cd VM |
241 | andl %edx,%edx |
242 | jz 4f | |
243 | cmpl $8,%edx | |
244 | jb 2f /* less than 8 bytes, go to byte copy loop */ | |
245 | ALIGN_DESTINATION | |
1da177e4 LT |
246 | movl %edx,%ecx |
247 | shrl $3,%ecx | |
ad2fc2cd VM |
248 | andl $7,%edx |
249 | 1: rep | |
3022d734 | 250 | movsq |
ad2fc2cd VM |
251 | 2: movl %edx,%ecx |
252 | 3: rep | |
253 | movsb | |
254 | 4: xorl %eax,%eax | |
1da177e4 | 255 | ret |
3022d734 | 256 | |
ad2fc2cd | 257 | .section .fixup,"ax" |
27cb0a75 | 258 | 11: lea (%rdx,%rcx,8),%rcx |
ad2fc2cd VM |
259 | 12: movl %ecx,%edx /* ecx is zerorest also */ |
260 | jmp copy_user_handle_tail | |
261 | .previous | |
2cbc9ee3 | 262 | |
1da177e4 | 263 | .section __ex_table,"a" |
ad2fc2cd VM |
264 | .align 8 |
265 | .quad 1b,11b | |
266 | .quad 3b,12b | |
1da177e4 | 267 | .previous |
ad2fc2cd VM |
268 | CFI_ENDPROC |
269 | ENDPROC(copy_user_generic_string) |