]> git.proxmox.com Git - mirror_ubuntu-eoan-kernel.git/blame - arch/metag/lib/usercopy.c
projects / mirror_ubuntu-eoan-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc
[mirror_ubuntu-eoan-kernel.git] / arch / metag / lib / usercopy.c
CommitLineData
373cd784
JH		 1/*
2 * User address space access functions.
3 * The non-inlined parts of asm-metag/uaccess.h are here.
4 *
5 * Copyright (C) 2006, Imagination Technologies.
6 * Copyright (C) 2000, Axis Communications AB.
7 *
8 * Written by Hans-Peter Nilsson.
9 * Pieces used from memcpy, originally by Kenny Ranerup long time ago.
10 * Modified for Meta by Will Newton.
11 */
12
9da3ee9a 13#include <linux/export.h>
373cd784
JH		 14#include <linux/uaccess.h>
15#include <asm/cache.h> /* def of L1_CACHE_BYTES */
16
17#define USE_RAPF
18#define RAPF_MIN_BUF_SIZE (3*L1_CACHE_BYTES)
19
20
21/* The "double write" in this code is because the Meta will not fault
22 * immediately unless the memory pipe is forced to by e.g. a data stall or
23 * another memory op. The second write should be discarded by the write
24 * combiner so should have virtually no cost.
25 */
26
27#define __asm_copy_user_cont(to, from, ret, COPY, FIXUP, TENTRY) \
28 asm volatile ( \
29 COPY \
30 "1:\n" \
31 " .section .fixup,\"ax\"\n" \
373cd784
JH		 32 FIXUP \
33 " MOVT D1Ar1,#HI(1b)\n" \
34 " JUMP D1Ar1,#LO(1b)\n" \
35 " .previous\n" \
36 " .section __ex_table,\"a\"\n" \
37 TENTRY \
38 " .previous\n" \
39 : "=r" (to), "=r" (from), "=r" (ret) \
40 : "0" (to), "1" (from), "2" (ret) \
41 : "D1Ar1", "memory")
42
43
44#define __asm_copy_to_user_1(to, from, ret) \
45 __asm_copy_user_cont(to, from, ret, \
46 " GETB D1Ar1,[%1++]\n" \
47 " SETB [%0],D1Ar1\n" \
48 "2: SETB [%0++],D1Ar1\n", \
49 "3: ADD %2,%2,#1\n", \
50 " .long 2b,3b\n")
51
52#define __asm_copy_to_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
53 __asm_copy_user_cont(to, from, ret, \
54 " GETW D1Ar1,[%1++]\n" \
55 " SETW [%0],D1Ar1\n" \
56 "2: SETW [%0++],D1Ar1\n" COPY, \
57 "3: ADD %2,%2,#2\n" FIXUP, \
58 " .long 2b,3b\n" TENTRY)
59
60#define __asm_copy_to_user_2(to, from, ret) \
61 __asm_copy_to_user_2x_cont(to, from, ret, "", "", "")
62
63#define __asm_copy_to_user_3(to, from, ret) \
64 __asm_copy_to_user_2x_cont(to, from, ret, \
65 " GETB D1Ar1,[%1++]\n" \
66 " SETB [%0],D1Ar1\n" \
67 "4: SETB [%0++],D1Ar1\n", \
68 "5: ADD %2,%2,#1\n", \
69 " .long 4b,5b\n")
70
71#define __asm_copy_to_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
72 __asm_copy_user_cont(to, from, ret, \
73 " GETD D1Ar1,[%1++]\n" \
74 " SETD [%0],D1Ar1\n" \
75 "2: SETD [%0++],D1Ar1\n" COPY, \
76 "3: ADD %2,%2,#4\n" FIXUP, \
77 " .long 2b,3b\n" TENTRY)
78
79#define __asm_copy_to_user_4(to, from, ret) \
80 __asm_copy_to_user_4x_cont(to, from, ret, "", "", "")
81
82#define __asm_copy_to_user_5(to, from, ret) \
83 __asm_copy_to_user_4x_cont(to, from, ret, \
84 " GETB D1Ar1,[%1++]\n" \
85 " SETB [%0],D1Ar1\n" \
86 "4: SETB [%0++],D1Ar1\n", \
87 "5: ADD %2,%2,#1\n", \
88 " .long 4b,5b\n")
89
90#define __asm_copy_to_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
91 __asm_copy_to_user_4x_cont(to, from, ret, \
92 " GETW D1Ar1,[%1++]\n" \
93 " SETW [%0],D1Ar1\n" \
94 "4: SETW [%0++],D1Ar1\n" COPY, \
95 "5: ADD %2,%2,#2\n" FIXUP, \
96 " .long 4b,5b\n" TENTRY)
97
98#define __asm_copy_to_user_6(to, from, ret) \
99 __asm_copy_to_user_6x_cont(to, from, ret, "", "", "")
100
101#define __asm_copy_to_user_7(to, from, ret) \
102 __asm_copy_to_user_6x_cont(to, from, ret, \
103 " GETB D1Ar1,[%1++]\n" \
104 " SETB [%0],D1Ar1\n" \
105 "6: SETB [%0++],D1Ar1\n", \
106 "7: ADD %2,%2,#1\n", \
107 " .long 6b,7b\n")
108
109#define __asm_copy_to_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
110 __asm_copy_to_user_4x_cont(to, from, ret, \
111 " GETD D1Ar1,[%1++]\n" \
112 " SETD [%0],D1Ar1\n" \
113 "4: SETD [%0++],D1Ar1\n" COPY, \
114 "5: ADD %2,%2,#4\n" FIXUP, \
115 " .long 4b,5b\n" TENTRY)
116
117#define __asm_copy_to_user_8(to, from, ret) \
118 __asm_copy_to_user_8x_cont(to, from, ret, "", "", "")
119
120#define __asm_copy_to_user_9(to, from, ret) \
121 __asm_copy_to_user_8x_cont(to, from, ret, \
122 " GETB D1Ar1,[%1++]\n" \
123 " SETB [%0],D1Ar1\n" \
124 "6: SETB [%0++],D1Ar1\n", \
125 "7: ADD %2,%2,#1\n", \
126 " .long 6b,7b\n")
127
128#define __asm_copy_to_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
129 __asm_copy_to_user_8x_cont(to, from, ret, \
130 " GETW D1Ar1,[%1++]\n" \
131 " SETW [%0],D1Ar1\n" \
132 "6: SETW [%0++],D1Ar1\n" COPY, \
133 "7: ADD %2,%2,#2\n" FIXUP, \
134 " .long 6b,7b\n" TENTRY)
135
136#define __asm_copy_to_user_10(to, from, ret) \
137 __asm_copy_to_user_10x_cont(to, from, ret, "", "", "")
138
139#define __asm_copy_to_user_11(to, from, ret) \
140 __asm_copy_to_user_10x_cont(to, from, ret, \
141 " GETB D1Ar1,[%1++]\n" \
142 " SETB [%0],D1Ar1\n" \
143 "8: SETB [%0++],D1Ar1\n", \
144 "9: ADD %2,%2,#1\n", \
145 " .long 8b,9b\n")
146
147#define __asm_copy_to_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
148 __asm_copy_to_user_8x_cont(to, from, ret, \
149 " GETD D1Ar1,[%1++]\n" \
150 " SETD [%0],D1Ar1\n" \
151 "6: SETD [%0++],D1Ar1\n" COPY, \
152 "7: ADD %2,%2,#4\n" FIXUP, \
153 " .long 6b,7b\n" TENTRY)
154#define __asm_copy_to_user_12(to, from, ret) \
155 __asm_copy_to_user_12x_cont(to, from, ret, "", "", "")
156
157#define __asm_copy_to_user_13(to, from, ret) \
158 __asm_copy_to_user_12x_cont(to, from, ret, \
159 " GETB D1Ar1,[%1++]\n" \
160 " SETB [%0],D1Ar1\n" \
161 "8: SETB [%0++],D1Ar1\n", \
162 "9: ADD %2,%2,#1\n", \
163 " .long 8b,9b\n")
164
165#define __asm_copy_to_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
166 __asm_copy_to_user_12x_cont(to, from, ret, \
167 " GETW D1Ar1,[%1++]\n" \
168 " SETW [%0],D1Ar1\n" \
169 "8: SETW [%0++],D1Ar1\n" COPY, \
170 "9: ADD %2,%2,#2\n" FIXUP, \
171 " .long 8b,9b\n" TENTRY)
172
173#define __asm_copy_to_user_14(to, from, ret) \
174 __asm_copy_to_user_14x_cont(to, from, ret, "", "", "")
175
176#define __asm_copy_to_user_15(to, from, ret) \
177 __asm_copy_to_user_14x_cont(to, from, ret, \
178 " GETB D1Ar1,[%1++]\n" \
179 " SETB [%0],D1Ar1\n" \
180 "10: SETB [%0++],D1Ar1\n", \
181 "11: ADD %2,%2,#1\n", \
182 " .long 10b,11b\n")
183
184#define __asm_copy_to_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
185 __asm_copy_to_user_12x_cont(to, from, ret, \
186 " GETD D1Ar1,[%1++]\n" \
187 " SETD [%0],D1Ar1\n" \
188 "8: SETD [%0++],D1Ar1\n" COPY, \
189 "9: ADD %2,%2,#4\n" FIXUP, \
190 " .long 8b,9b\n" TENTRY)
191
192#define __asm_copy_to_user_16(to, from, ret) \
193 __asm_copy_to_user_16x_cont(to, from, ret, "", "", "")
194
195#define __asm_copy_to_user_8x64(to, from, ret) \
196 asm volatile ( \
197 " GETL D0Ar2,D1Ar1,[%1++]\n" \
198 " SETL [%0],D0Ar2,D1Ar1\n" \
199 "2: SETL [%0++],D0Ar2,D1Ar1\n" \
200 "1:\n" \
201 " .section .fixup,\"ax\"\n" \
202 "3: ADD %2,%2,#8\n" \
203 " MOVT D0Ar2,#HI(1b)\n" \
204 " JUMP D0Ar2,#LO(1b)\n" \
205 " .previous\n" \
206 " .section __ex_table,\"a\"\n" \
207 " .long 2b,3b\n" \
208 " .previous\n" \
209 : "=r" (to), "=r" (from), "=r" (ret) \
210 : "0" (to), "1" (from), "2" (ret) \
211 : "D1Ar1", "D0Ar2", "memory")
212
213/*
214 * optimized copying loop using RAPF when 64 bit aligned
215 *
216 * n will be automatically decremented inside the loop
217 * ret will be left intact. if error occurs we will rewind
218 * so that the original non optimized code will fill up
219 * this value correctly.
220 *
221 * on fault:
222 * > n will hold total number of uncopied bytes
223 *
224 * > {'to','from'} will be rewind back so that
225 * the non-optimized code will do the proper fix up
226 *
227 * DCACHE drops the cacheline which helps in reducing cache
228 * pollution.
229 *
230 * We introduce an extra SETL at the end of the loop to
231 * ensure we don't fall off the loop before we catch all
232 * erros.
233 *
234 * NOTICE:
235 * LSM_STEP in TXSTATUS must be cleared in fix up code.
236 * since we're using M{S,G}ETL, a fault might happen at
237 * any address in the middle of M{S,G}ETL causing
238 * the value of LSM_STEP to be incorrect which can
239 * cause subsequent use of M{S,G}ET{L,D} to go wrong.
240 * ie: if LSM_STEP was 1 when a fault occurs, the
241 * next call to M{S,G}ET{L,D} will skip the first
242 * copy/getting as it think that the first 1 has already
243 * been done.
244 *
245 */
246#define __asm_copy_user_64bit_rapf_loop( \
247 to, from, ret, n, id, FIXUP) \
248 asm volatile ( \
249 ".balign 8\n" \
250 "MOV RAPF, %1\n" \
251 "MSETL [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n" \
252 "MOV D0Ar6, #0\n" \
253 "LSR D1Ar5, %3, #6\n" \
254 "SUB TXRPT, D1Ar5, #2\n" \
255 "MOV RAPF, %1\n" \
256 "$Lloop"id":\n" \
257 "ADD RAPF, %1, #64\n" \
258 "21:\n" \
259 "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
260 "22:\n" \
261 "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
373cd784 262 "23:\n" \
b884a190 263 "SUB %3, %3, #32\n" \
373cd784 264 "24:\n" \
b884a190
JH		 265 "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
266 "25:\n" \
373cd784 267 "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
b884a190 268 "26:\n" \
373cd784
JH		 269 "SUB %3, %3, #32\n" \
270 "DCACHE [%1+#-64], D0Ar6\n" \
271 "BR $Lloop"id"\n" \
272 \
273 "MOV RAPF, %1\n" \
b884a190 274 "27:\n" \
373cd784 275 "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
b884a190 276 "28:\n" \
373cd784 277 "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
b884a190 278 "29:\n" \
373cd784 279 "SUB %3, %3, #32\n" \
b884a190 280 "30:\n" \
373cd784 281 "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
b884a190 282 "31:\n" \
373cd784 283 "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
b884a190 284 "32:\n" \
373cd784 285 "SUB %0, %0, #8\n" \
b884a190 286 "33:\n" \
373cd784
JH		 287 "SETL [%0++], D0.7, D1.7\n" \
288 "SUB %3, %3, #32\n" \
289 "1:" \
290 "DCACHE [%1+#-64], D0Ar6\n" \
291 "GETL D0Ar6, D1Ar5, [A0StP+#-40]\n" \
292 "GETL D0FrT, D1RtP, [A0StP+#-32]\n" \
293 "GETL D0.5, D1.5, [A0StP+#-24]\n" \
294 "GETL D0.6, D1.6, [A0StP+#-16]\n" \
295 "GETL D0.7, D1.7, [A0StP+#-8]\n" \
296 "SUB A0StP, A0StP, #40\n" \
297 " .section .fixup,\"ax\"\n" \
298 "4:\n" \
299 " ADD %0, %0, #8\n" \
300 "3:\n" \
301 " MOV D0Ar2, TXSTATUS\n" \
302 " MOV D1Ar1, TXSTATUS\n" \
303 " AND D1Ar1, D1Ar1, #0xFFFFF8FF\n" \
304 " MOV TXSTATUS, D1Ar1\n" \
305 FIXUP \
306 " MOVT D0Ar2,#HI(1b)\n" \
307 " JUMP D0Ar2,#LO(1b)\n" \
308 " .previous\n" \
309 " .section __ex_table,\"a\"\n" \
310 " .long 21b,3b\n" \
311 " .long 22b,3b\n" \
312 " .long 23b,3b\n" \
313 " .long 24b,3b\n" \
314 " .long 25b,3b\n" \
315 " .long 26b,3b\n" \
316 " .long 27b,3b\n" \
317 " .long 28b,3b\n" \
b884a190
JH		 318 " .long 29b,3b\n" \
319 " .long 30b,3b\n" \
320 " .long 31b,3b\n" \
321 " .long 32b,3b\n" \
322 " .long 33b,4b\n" \
373cd784
JH		 323 " .previous\n" \
324 : "=r" (to), "=r" (from), "=r" (ret), "=d" (n) \
325 : "0" (to), "1" (from), "2" (ret), "3" (n) \
fd40eee1 326 : "D1Ar1", "D0Ar2", "cc", "memory")
373cd784
JH		 327
328/* rewind 'to' and 'from' pointers when a fault occurs
329 *
330 * Rationale:
331 * A fault always occurs on writing to user buffer. A fault
332 * is at a single address, so we need to rewind by only 4
333 * bytes.
334 * Since we do a complete read from kernel buffer before
335 * writing, we need to rewind it also. The amount to be
336 * rewind equals the number of faulty writes in MSETD
337 * which is: [4 - (LSM_STEP-1)]*8
338 * LSM_STEP is bits 10:8 in TXSTATUS which is already read
339 * and stored in D0Ar2
340 *
341 * NOTE: If a fault occurs at the last operation in M{G,S}ETL
342 * LSM_STEP will be 0. ie: we do 4 writes in our case, if
343 * a fault happens at the 4th write, LSM_STEP will be 0
344 * instead of 4. The code copes with that.
345 *
346 * n is updated by the number of successful writes, which is:
347 * n = n - (LSM_STEP-1)*8
348 */
349#define __asm_copy_to_user_64bit_rapf_loop(to, from, ret, n, id)\
350 __asm_copy_user_64bit_rapf_loop(to, from, ret, n, id, \
351 "LSR D0Ar2, D0Ar2, #8\n" \
fd40eee1 352 "ANDS D0Ar2, D0Ar2, #0x7\n" \
373cd784
JH		 353 "ADDZ D0Ar2, D0Ar2, #4\n" \
354 "SUB D0Ar2, D0Ar2, #1\n" \
355 "MOV D1Ar1, #4\n" \
356 "SUB D0Ar2, D1Ar1, D0Ar2\n" \
357 "LSL D0Ar2, D0Ar2, #3\n" \
358 "LSL D1Ar1, D1Ar1, #3\n" \
359 "SUB D1Ar1, D1Ar1, D0Ar2\n" \
360 "SUB %0, %0, #8\n" \
361 "SUB %1, %1,D0Ar2\n" \
362 "SUB %3, %3, D1Ar1\n")
363
364/*
365 * optimized copying loop using RAPF when 32 bit aligned
366 *
367 * n will be automatically decremented inside the loop
368 * ret will be left intact. if error occurs we will rewind
369 * so that the original non optimized code will fill up
370 * this value correctly.
371 *
372 * on fault:
373 * > n will hold total number of uncopied bytes
374 *
375 * > {'to','from'} will be rewind back so that
376 * the non-optimized code will do the proper fix up
377 *
378 * DCACHE drops the cacheline which helps in reducing cache
379 * pollution.
380 *
381 * We introduce an extra SETD at the end of the loop to
382 * ensure we don't fall off the loop before we catch all
383 * erros.
384 *
385 * NOTICE:
386 * LSM_STEP in TXSTATUS must be cleared in fix up code.
387 * since we're using M{S,G}ETL, a fault might happen at
388 * any address in the middle of M{S,G}ETL causing
389 * the value of LSM_STEP to be incorrect which can
390 * cause subsequent use of M{S,G}ET{L,D} to go wrong.
391 * ie: if LSM_STEP was 1 when a fault occurs, the
392 * next call to M{S,G}ET{L,D} will skip the first
393 * copy/getting as it think that the first 1 has already
394 * been done.
395 *
396 */
397#define __asm_copy_user_32bit_rapf_loop( \
398 to, from, ret, n, id, FIXUP) \
399 asm volatile ( \
400 ".balign 8\n" \
401 "MOV RAPF, %1\n" \
402 "MSETL [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n" \
403 "MOV D0Ar6, #0\n" \
404 "LSR D1Ar5, %3, #6\n" \
405 "SUB TXRPT, D1Ar5, #2\n" \
406 "MOV RAPF, %1\n" \
407 "$Lloop"id":\n" \
408 "ADD RAPF, %1, #64\n" \
409 "21:\n" \
410 "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
411 "22:\n" \
412 "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
373cd784 413 "23:\n" \
373cd784 414 "SUB %3, %3, #16\n" \
b884a190 415 "24:\n" \
373cd784 416 "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
b884a190 417 "25:\n" \
373cd784 418 "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
b884a190 419 "26:\n" \
373cd784
JH		 420 "SUB %3, %3, #16\n" \
421 "27:\n" \
422 "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
423 "28:\n" \
424 "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
b884a190
JH		 425 "29:\n" \
426 "SUB %3, %3, #16\n" \
427 "30:\n" \
428 "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
429 "31:\n" \
430 "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
431 "32:\n" \
373cd784
JH		 432 "SUB %3, %3, #16\n" \
433 "DCACHE [%1+#-64], D0Ar6\n" \
434 "BR $Lloop"id"\n" \
435 \
436 "MOV RAPF, %1\n" \
b884a190 437 "33:\n" \
373cd784 438 "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
b884a190 439 "34:\n" \
373cd784 440 "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
b884a190 441 "35:\n" \
373cd784 442 "SUB %3, %3, #16\n" \
b884a190 443 "36:\n" \
373cd784 444 "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
b884a190 445 "37:\n" \
373cd784 446 "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
b884a190 447 "38:\n" \
373cd784 448 "SUB %3, %3, #16\n" \
b884a190 449 "39:\n" \
373cd784 450 "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
b884a190 451 "40:\n" \
373cd784 452 "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
b884a190 453 "41:\n" \
373cd784 454 "SUB %3, %3, #16\n" \
b884a190 455 "42:\n" \
373cd784 456 "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \
b884a190 457 "43:\n" \
373cd784 458 "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \
b884a190 459 "44:\n" \
373cd784 460 "SUB %0, %0, #4\n" \
b884a190 461 "45:\n" \
373cd784
JH		 462 "SETD [%0++], D0.7\n" \
463 "SUB %3, %3, #16\n" \
464 "1:" \
465 "DCACHE [%1+#-64], D0Ar6\n" \
466 "GETL D0Ar6, D1Ar5, [A0StP+#-40]\n" \
467 "GETL D0FrT, D1RtP, [A0StP+#-32]\n" \
468 "GETL D0.5, D1.5, [A0StP+#-24]\n" \
469 "GETL D0.6, D1.6, [A0StP+#-16]\n" \
470 "GETL D0.7, D1.7, [A0StP+#-8]\n" \
471 "SUB A0StP, A0StP, #40\n" \
472 " .section .fixup,\"ax\"\n" \
473 "4:\n" \
474 " ADD %0, %0, #4\n" \
475 "3:\n" \
476 " MOV D0Ar2, TXSTATUS\n" \
477 " MOV D1Ar1, TXSTATUS\n" \
478 " AND D1Ar1, D1Ar1, #0xFFFFF8FF\n" \
479 " MOV TXSTATUS, D1Ar1\n" \
480 FIXUP \
481 " MOVT D0Ar2,#HI(1b)\n" \
482 " JUMP D0Ar2,#LO(1b)\n" \
483 " .previous\n" \
484 " .section __ex_table,\"a\"\n" \
485 " .long 21b,3b\n" \
486 " .long 22b,3b\n" \
487 " .long 23b,3b\n" \
488 " .long 24b,3b\n" \
489 " .long 25b,3b\n" \
490 " .long 26b,3b\n" \
491 " .long 27b,3b\n" \
492 " .long 28b,3b\n" \
493 " .long 29b,3b\n" \
494 " .long 30b,3b\n" \
495 " .long 31b,3b\n" \
496 " .long 32b,3b\n" \
497 " .long 33b,3b\n" \
498 " .long 34b,3b\n" \
499 " .long 35b,3b\n" \
500 " .long 36b,3b\n" \
b884a190
JH		 501 " .long 37b,3b\n" \
502 " .long 38b,3b\n" \
503 " .long 39b,3b\n" \
504 " .long 40b,3b\n" \
505 " .long 41b,3b\n" \
506 " .long 42b,3b\n" \
507 " .long 43b,3b\n" \
508 " .long 44b,3b\n" \
509 " .long 45b,4b\n" \
373cd784
JH		 510 " .previous\n" \
511 : "=r" (to), "=r" (from), "=r" (ret), "=d" (n) \
512 : "0" (to), "1" (from), "2" (ret), "3" (n) \
fd40eee1 513 : "D1Ar1", "D0Ar2", "cc", "memory")
373cd784
JH		 514
515/* rewind 'to' and 'from' pointers when a fault occurs
516 *
517 * Rationale:
518 * A fault always occurs on writing to user buffer. A fault
519 * is at a single address, so we need to rewind by only 4
520 * bytes.
521 * Since we do a complete read from kernel buffer before
522 * writing, we need to rewind it also. The amount to be
523 * rewind equals the number of faulty writes in MSETD
524 * which is: [4 - (LSM_STEP-1)]*4
525 * LSM_STEP is bits 10:8 in TXSTATUS which is already read
526 * and stored in D0Ar2
527 *
528 * NOTE: If a fault occurs at the last operation in M{G,S}ETL
529 * LSM_STEP will be 0. ie: we do 4 writes in our case, if
530 * a fault happens at the 4th write, LSM_STEP will be 0
531 * instead of 4. The code copes with that.
532 *
533 * n is updated by the number of successful writes, which is:
534 * n = n - (LSM_STEP-1)*4
535 */
536#define __asm_copy_to_user_32bit_rapf_loop(to, from, ret, n, id)\
537 __asm_copy_user_32bit_rapf_loop(to, from, ret, n, id, \
538 "LSR D0Ar2, D0Ar2, #8\n" \
fd40eee1 539 "ANDS D0Ar2, D0Ar2, #0x7\n" \
373cd784
JH		 540 "ADDZ D0Ar2, D0Ar2, #4\n" \
541 "SUB D0Ar2, D0Ar2, #1\n" \
542 "MOV D1Ar1, #4\n" \
543 "SUB D0Ar2, D1Ar1, D0Ar2\n" \
544 "LSL D0Ar2, D0Ar2, #2\n" \
545 "LSL D1Ar1, D1Ar1, #2\n" \
546 "SUB D1Ar1, D1Ar1, D0Ar2\n" \
547 "SUB %0, %0, #4\n" \
548 "SUB %1, %1, D0Ar2\n" \
549 "SUB %3, %3, D1Ar1\n")
550
551unsigned long __copy_user(void __user *pdst, const void *psrc,
552 unsigned long n)
553{
554 register char __user *dst asm ("A0.2") = pdst;
555 register const char *src asm ("A1.2") = psrc;
556 unsigned long retn = 0;
557
558 if (n == 0)
559 return 0;
560
561 if ((unsigned long) src & 1) {
562 __asm_copy_to_user_1(dst, src, retn);
563 n--;
fb8ea062
JH		 564 if (retn)
565 return retn + n;
373cd784
JH		 566 }
567 if ((unsigned long) dst & 1) {
568 /* Worst case - byte copy */
569 while (n > 0) {
570 __asm_copy_to_user_1(dst, src, retn);
571 n--;
fb8ea062
JH		 572 if (retn)
573 return retn + n;
373cd784
JH		 574 }
575 }
576 if (((unsigned long) src & 2) && n >= 2) {
577 __asm_copy_to_user_2(dst, src, retn);
578 n -= 2;
fb8ea062
JH		 579 if (retn)
580 return retn + n;
373cd784
JH		 581 }
582 if ((unsigned long) dst & 2) {
583 /* Second worst case - word copy */
584 while (n >= 2) {
585 __asm_copy_to_user_2(dst, src, retn);
586 n -= 2;
fb8ea062
JH		 587 if (retn)
588 return retn + n;
373cd784
JH		 589 }
590 }
591
592#ifdef USE_RAPF
593 /* 64 bit copy loop */
594 if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) {
595 if (n >= RAPF_MIN_BUF_SIZE) {
596 /* copy user using 64 bit rapf copy */
597 __asm_copy_to_user_64bit_rapf_loop(dst, src, retn,
598 n, "64cu");
599 }
600 while (n >= 8) {
601 __asm_copy_to_user_8x64(dst, src, retn);
602 n -= 8;
fb8ea062
JH		 603 if (retn)
604 return retn + n;
373cd784
JH		 605 }
606 }
607 if (n >= RAPF_MIN_BUF_SIZE) {
608 /* copy user using 32 bit rapf copy */
609 __asm_copy_to_user_32bit_rapf_loop(dst, src, retn, n, "32cu");
610 }
611#else
612 /* 64 bit copy loop */
613 if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) {
614 while (n >= 8) {
615 __asm_copy_to_user_8x64(dst, src, retn);
616 n -= 8;
fb8ea062
JH		 617 if (retn)
618 return retn + n;
373cd784
JH		 619 }
620 }
621#endif
622
623 while (n >= 16) {
624 __asm_copy_to_user_16(dst, src, retn);
625 n -= 16;
fb8ea062
JH		 626 if (retn)
627 return retn + n;
373cd784
JH		 628 }
629
630 while (n >= 4) {
631 __asm_copy_to_user_4(dst, src, retn);
632 n -= 4;
fb8ea062
JH		 633 if (retn)
634 return retn + n;
373cd784
JH		 635 }
636
637 switch (n) {
638 case 0:
639 break;
640 case 1:
641 __asm_copy_to_user_1(dst, src, retn);
642 break;
643 case 2:
644 __asm_copy_to_user_2(dst, src, retn);
645 break;
646 case 3:
647 __asm_copy_to_user_3(dst, src, retn);
648 break;
649 }
650
fb8ea062
JH		 651 /*
652 * If we get here, retn correctly reflects the number of failing
653 * bytes.
654 */
373cd784
JH		 655 return retn;
656}
9da3ee9a 657EXPORT_SYMBOL(__copy_user);
373cd784
JH		 658
659#define __asm_copy_from_user_1(to, from, ret) \
660 __asm_copy_user_cont(to, from, ret, \
661 " GETB D1Ar1,[%1++]\n" \
662 "2: SETB [%0++],D1Ar1\n", \
563ddc10 663 "3: ADD %2,%2,#1\n", \
373cd784
JH		 664 " .long 2b,3b\n")
665
666#define __asm_copy_from_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
667 __asm_copy_user_cont(to, from, ret, \
668 " GETW D1Ar1,[%1++]\n" \
669 "2: SETW [%0++],D1Ar1\n" COPY, \
563ddc10 670 "3: ADD %2,%2,#2\n" FIXUP, \
373cd784
JH		 671 " .long 2b,3b\n" TENTRY)
672
673#define __asm_copy_from_user_2(to, from, ret) \
674 __asm_copy_from_user_2x_cont(to, from, ret, "", "", "")
675
676#define __asm_copy_from_user_3(to, from, ret) \
677 __asm_copy_from_user_2x_cont(to, from, ret, \
678 " GETB D1Ar1,[%1++]\n" \
679 "4: SETB [%0++],D1Ar1\n", \
563ddc10 680 "5: ADD %2,%2,#1\n", \
373cd784
JH		 681 " .long 4b,5b\n")
682
683#define __asm_copy_from_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
684 __asm_copy_user_cont(to, from, ret, \
685 " GETD D1Ar1,[%1++]\n" \
686 "2: SETD [%0++],D1Ar1\n" COPY, \
563ddc10 687 "3: ADD %2,%2,#4\n" FIXUP, \
373cd784
JH		 688 " .long 2b,3b\n" TENTRY)
689
690#define __asm_copy_from_user_4(to, from, ret) \
691 __asm_copy_from_user_4x_cont(to, from, ret, "", "", "")
692
373cd784
JH		 693#define __asm_copy_from_user_8x64(to, from, ret) \
694 asm volatile ( \
695 " GETL D0Ar2,D1Ar1,[%1++]\n" \
696 "2: SETL [%0++],D0Ar2,D1Ar1\n" \
697 "1:\n" \
698 " .section .fixup,\"ax\"\n" \
373cd784 699 "3: ADD %2,%2,#8\n" \
373cd784
JH		 700 " MOVT D0Ar2,#HI(1b)\n" \
701 " JUMP D0Ar2,#LO(1b)\n" \
702 " .previous\n" \
703 " .section __ex_table,\"a\"\n" \
704 " .long 2b,3b\n" \
705 " .previous\n" \
706 : "=a" (to), "=r" (from), "=r" (ret) \
707 : "0" (to), "1" (from), "2" (ret) \
708 : "D1Ar1", "D0Ar2", "memory")
709
710/* rewind 'from' pointer when a fault occurs
711 *
712 * Rationale:
713 * A fault occurs while reading from user buffer, which is the
2c0b1df8 714 * source.
373cd784
JH		 715 * Since we don't write to kernel buffer until we read first,
716 * the kernel buffer is at the right state and needn't be
2c0b1df8
JH		 717 * corrected, but the source must be rewound to the beginning of
718 * the block, which is LSM_STEP*8 bytes.
719 * LSM_STEP is bits 10:8 in TXSTATUS which is already read
720 * and stored in D0Ar2
721 *
722 * NOTE: If a fault occurs at the last operation in M{G,S}ETL
723 * LSM_STEP will be 0. ie: we do 4 writes in our case, if
724 * a fault happens at the 4th write, LSM_STEP will be 0
725 * instead of 4. The code copes with that.
373cd784
JH		 726 */
727#define __asm_copy_from_user_64bit_rapf_loop(to, from, ret, n, id) \
728 __asm_copy_user_64bit_rapf_loop(to, from, ret, n, id, \
2c0b1df8
JH		 729 "LSR D0Ar2, D0Ar2, #5\n" \
730 "ANDS D0Ar2, D0Ar2, #0x38\n" \
731 "ADDZ D0Ar2, D0Ar2, #32\n" \
732 "SUB %1, %1, D0Ar2\n")
373cd784
JH		 733
734/* rewind 'from' pointer when a fault occurs
735 *
736 * Rationale:
737 * A fault occurs while reading from user buffer, which is the
2c0b1df8 738 * source.
373cd784
JH		 739 * Since we don't write to kernel buffer until we read first,
740 * the kernel buffer is at the right state and needn't be
2c0b1df8
JH		 741 * corrected, but the source must be rewound to the beginning of
742 * the block, which is LSM_STEP*4 bytes.
743 * LSM_STEP is bits 10:8 in TXSTATUS which is already read
744 * and stored in D0Ar2
745 *
746 * NOTE: If a fault occurs at the last operation in M{G,S}ETL
747 * LSM_STEP will be 0. ie: we do 4 writes in our case, if
748 * a fault happens at the 4th write, LSM_STEP will be 0
749 * instead of 4. The code copes with that.
373cd784
JH		 750 */
751#define __asm_copy_from_user_32bit_rapf_loop(to, from, ret, n, id) \
752 __asm_copy_user_32bit_rapf_loop(to, from, ret, n, id, \
2c0b1df8
JH		 753 "LSR D0Ar2, D0Ar2, #6\n" \
754 "ANDS D0Ar2, D0Ar2, #0x1c\n" \
755 "ADDZ D0Ar2, D0Ar2, #16\n" \
756 "SUB %1, %1, D0Ar2\n")
373cd784
JH		 757
758
563ddc10
JH		 759/*
760 * Copy from user to kernel. The return-value is the number of bytes that were
761 * inaccessible.
762 */
763unsigned long raw_copy_from_user(void *pdst, const void __user *psrc,
764 unsigned long n)
373cd784
JH		 765{
766 register char *dst asm ("A0.2") = pdst;
767 register const char __user *src asm ("A1.2") = psrc;
768 unsigned long retn = 0;
769
770 if (n == 0)
771 return 0;
772
773 if ((unsigned long) src & 1) {
774 __asm_copy_from_user_1(dst, src, retn);
775 n--;
22572119 776 if (retn)
563ddc10 777 return retn + n;
373cd784
JH		 778 }
779 if ((unsigned long) dst & 1) {
780 /* Worst case - byte copy */
781 while (n > 0) {
782 __asm_copy_from_user_1(dst, src, retn);
783 n--;
784 if (retn)
563ddc10 785 return retn + n;
373cd784
JH		 786 }
787 }
788 if (((unsigned long) src & 2) && n >= 2) {
789 __asm_copy_from_user_2(dst, src, retn);
790 n -= 2;
22572119 791 if (retn)
563ddc10 792 return retn + n;
373cd784
JH		 793 }
794 if ((unsigned long) dst & 2) {
795 /* Second worst case - word copy */
796 while (n >= 2) {
797 __asm_copy_from_user_2(dst, src, retn);
798 n -= 2;
799 if (retn)
563ddc10 800 return retn + n;
373cd784
JH		 801 }
802 }
803
373cd784
JH		 804#ifdef USE_RAPF
805 /* 64 bit copy loop */
806 if (!(((unsigned long) src | (unsigned long) dst) & 7)) {
807 if (n >= RAPF_MIN_BUF_SIZE) {
808 /* Copy using fast 64bit rapf */
809 __asm_copy_from_user_64bit_rapf_loop(dst, src, retn,
810 n, "64cuz");
811 }
812 while (n >= 8) {
813 __asm_copy_from_user_8x64(dst, src, retn);
814 n -= 8;
815 if (retn)
563ddc10 816 return retn + n;
373cd784
JH		 817 }
818 }
819
820 if (n >= RAPF_MIN_BUF_SIZE) {
821 /* Copy using fast 32bit rapf */
822 __asm_copy_from_user_32bit_rapf_loop(dst, src, retn,
823 n, "32cuz");
824 }
825#else
826 /* 64 bit copy loop */
827 if (!(((unsigned long) src | (unsigned long) dst) & 7)) {
828 while (n >= 8) {
829 __asm_copy_from_user_8x64(dst, src, retn);
830 n -= 8;
831 if (retn)
563ddc10 832 return retn + n;
373cd784
JH		 833 }
834 }
835#endif
836
837 while (n >= 4) {
838 __asm_copy_from_user_4(dst, src, retn);
839 n -= 4;
840
841 if (retn)
563ddc10 842 return retn + n;
373cd784
JH		 843 }
844
845 /* If we get here, there were no memory read faults. */
846 switch (n) {
847 /* These copies are at least "naturally aligned" (so we don't
848 have to check each byte), due to the src alignment code.
849 The *_3 case *will* get the correct count for retn. */
850 case 0:
851 /* This case deliberately left in (if you have doubts check the
852 generated assembly code). */
853 break;
854 case 1:
855 __asm_copy_from_user_1(dst, src, retn);
856 break;
857 case 2:
858 __asm_copy_from_user_2(dst, src, retn);
859 break;
860 case 3:
861 __asm_copy_from_user_3(dst, src, retn);
862 break;
863 }
864
865 /* If we get here, retn correctly reflects the number of failing
866 bytes. */
867 return retn;
373cd784 868}
563ddc10 869EXPORT_SYMBOL(raw_copy_from_user);
373cd784
JH		 870
871#define __asm_clear_8x64(to, ret) \
872 asm volatile ( \
873 " MOV D0Ar2,#0\n" \
874 " MOV D1Ar1,#0\n" \
875 " SETL [%0],D0Ar2,D1Ar1\n" \
876 "2: SETL [%0++],D0Ar2,D1Ar1\n" \
877 "1:\n" \
878 " .section .fixup,\"ax\"\n" \
879 "3: ADD %1,%1,#8\n" \
880 " MOVT D0Ar2,#HI(1b)\n" \
881 " JUMP D0Ar2,#LO(1b)\n" \
882 " .previous\n" \
883 " .section __ex_table,\"a\"\n" \
884 " .long 2b,3b\n" \
885 " .previous\n" \
886 : "=r" (to), "=r" (ret) \
887 : "0" (to), "1" (ret) \
888 : "D1Ar1", "D0Ar2", "memory")
889
890/* Zero userspace. */
891
892#define __asm_clear(to, ret, CLEAR, FIXUP, TENTRY) \
893 asm volatile ( \
894 " MOV D1Ar1,#0\n" \
895 CLEAR \
896 "1:\n" \
897 " .section .fixup,\"ax\"\n" \
898 FIXUP \
899 " MOVT D1Ar1,#HI(1b)\n" \
900 " JUMP D1Ar1,#LO(1b)\n" \
901 " .previous\n" \
902 " .section __ex_table,\"a\"\n" \
903 TENTRY \
904 " .previous" \
905 : "=r" (to), "=r" (ret) \
906 : "0" (to), "1" (ret) \
907 : "D1Ar1", "memory")
908
909#define __asm_clear_1(to, ret) \
910 __asm_clear(to, ret, \
911 " SETB [%0],D1Ar1\n" \
912 "2: SETB [%0++],D1Ar1\n", \
913 "3: ADD %1,%1,#1\n", \
914 " .long 2b,3b\n")
915
916#define __asm_clear_2(to, ret) \
917 __asm_clear(to, ret, \
918 " SETW [%0],D1Ar1\n" \
919 "2: SETW [%0++],D1Ar1\n", \
920 "3: ADD %1,%1,#2\n", \
921 " .long 2b,3b\n")
922
923#define __asm_clear_3(to, ret) \
924 __asm_clear(to, ret, \
925 "2: SETW [%0++],D1Ar1\n" \
926 " SETB [%0],D1Ar1\n" \
927 "3: SETB [%0++],D1Ar1\n", \
928 "4: ADD %1,%1,#2\n" \
929 "5: ADD %1,%1,#1\n", \
930 " .long 2b,4b\n" \
931 " .long 3b,5b\n")
932
933#define __asm_clear_4x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
934 __asm_clear(to, ret, \
935 " SETD [%0],D1Ar1\n" \
936 "2: SETD [%0++],D1Ar1\n" CLEAR, \
937 "3: ADD %1,%1,#4\n" FIXUP, \
938 " .long 2b,3b\n" TENTRY)
939
940#define __asm_clear_4(to, ret) \
941 __asm_clear_4x_cont(to, ret, "", "", "")
942
943#define __asm_clear_8x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
944 __asm_clear_4x_cont(to, ret, \
945 " SETD [%0],D1Ar1\n" \
946 "4: SETD [%0++],D1Ar1\n" CLEAR, \
947 "5: ADD %1,%1,#4\n" FIXUP, \
948 " .long 4b,5b\n" TENTRY)
949
950#define __asm_clear_8(to, ret) \
951 __asm_clear_8x_cont(to, ret, "", "", "")
952
953#define __asm_clear_12x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
954 __asm_clear_8x_cont(to, ret, \
955 " SETD [%0],D1Ar1\n" \
956 "6: SETD [%0++],D1Ar1\n" CLEAR, \
957 "7: ADD %1,%1,#4\n" FIXUP, \
958 " .long 6b,7b\n" TENTRY)
959
960#define __asm_clear_12(to, ret) \
961 __asm_clear_12x_cont(to, ret, "", "", "")
962
963#define __asm_clear_16x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
964 __asm_clear_12x_cont(to, ret, \
965 " SETD [%0],D1Ar1\n" \
966 "8: SETD [%0++],D1Ar1\n" CLEAR, \
967 "9: ADD %1,%1,#4\n" FIXUP, \
968 " .long 8b,9b\n" TENTRY)
969
970#define __asm_clear_16(to, ret) \
971 __asm_clear_16x_cont(to, ret, "", "", "")
972
973unsigned long __do_clear_user(void __user *pto, unsigned long pn)
974{
975 register char __user *dst asm ("D0Re0") = pto;
976 register unsigned long n asm ("D1Re0") = pn;
977 register unsigned long retn asm ("D0Ar6") = 0;
978
979 if ((unsigned long) dst & 1) {
980 __asm_clear_1(dst, retn);
981 n--;
982 }
983
984 if ((unsigned long) dst & 2) {
985 __asm_clear_2(dst, retn);
986 n -= 2;
987 }
988
989 /* 64 bit copy loop */
990 if (!((__force unsigned long) dst & 7)) {
991 while (n >= 8) {
992 __asm_clear_8x64(dst, retn);
993 n -= 8;
994 }
995 }
996
997 while (n >= 16) {
998 __asm_clear_16(dst, retn);
999 n -= 16;
1000 }
1001
1002 while (n >= 4) {
1003 __asm_clear_4(dst, retn);
1004 n -= 4;
1005 }
1006
1007 switch (n) {
1008 case 0:
1009 break;
1010 case 1:
1011 __asm_clear_1(dst, retn);
1012 break;
1013 case 2:
1014 __asm_clear_2(dst, retn);
1015 break;
1016 case 3:
1017 __asm_clear_3(dst, retn);
1018 break;
1019 }
1020
1021 return retn;
1022}
9da3ee9a 1023EXPORT_SYMBOL(__do_clear_user);
373cd784
JH		 1024
1025unsigned char __get_user_asm_b(const void __user *addr, long *err)
1026{
1027 register unsigned char x asm ("D0Re0") = 0;
1028 asm volatile (
1029 " GETB %0,[%2]\n"
1030 "1:\n"
1031 " GETB %0,[%2]\n"
1032 "2:\n"
1033 " .section .fixup,\"ax\"\n"
1034 "3: MOV D0FrT,%3\n"
1035 " SETD [%1],D0FrT\n"
1036 " MOVT D0FrT,#HI(2b)\n"
1037 " JUMP D0FrT,#LO(2b)\n"
1038 " .previous\n"
1039 " .section __ex_table,\"a\"\n"
1040 " .long 1b,3b\n"
1041 " .previous\n"
1042 : "=r" (x)
1043 : "r" (err), "r" (addr), "P" (-EFAULT)
1044 : "D0FrT");
1045 return x;
1046}
9da3ee9a 1047EXPORT_SYMBOL(__get_user_asm_b);
373cd784
JH		 1048
1049unsigned short __get_user_asm_w(const void __user *addr, long *err)
1050{
1051 register unsigned short x asm ("D0Re0") = 0;
1052 asm volatile (
1053 " GETW %0,[%2]\n"
1054 "1:\n"
1055 " GETW %0,[%2]\n"
1056 "2:\n"
1057 " .section .fixup,\"ax\"\n"
1058 "3: MOV D0FrT,%3\n"
1059 " SETD [%1],D0FrT\n"
1060 " MOVT D0FrT,#HI(2b)\n"
1061 " JUMP D0FrT,#LO(2b)\n"
1062 " .previous\n"
1063 " .section __ex_table,\"a\"\n"
1064 " .long 1b,3b\n"
1065 " .previous\n"
1066 : "=r" (x)
1067 : "r" (err), "r" (addr), "P" (-EFAULT)
1068 : "D0FrT");
1069 return x;
1070}
9da3ee9a 1071EXPORT_SYMBOL(__get_user_asm_w);
373cd784
JH		 1072
1073unsigned int __get_user_asm_d(const void __user *addr, long *err)
1074{
1075 register unsigned int x asm ("D0Re0") = 0;
1076 asm volatile (
1077 " GETD %0,[%2]\n"
1078 "1:\n"
1079 " GETD %0,[%2]\n"
1080 "2:\n"
1081 " .section .fixup,\"ax\"\n"
1082 "3: MOV D0FrT,%3\n"
1083 " SETD [%1],D0FrT\n"
1084 " MOVT D0FrT,#HI(2b)\n"
1085 " JUMP D0FrT,#LO(2b)\n"
1086 " .previous\n"
1087 " .section __ex_table,\"a\"\n"
1088 " .long 1b,3b\n"
1089 " .previous\n"
1090 : "=r" (x)
1091 : "r" (err), "r" (addr), "P" (-EFAULT)
1092 : "D0FrT");
1093 return x;
1094}
9da3ee9a 1095EXPORT_SYMBOL(__get_user_asm_d);
373cd784
JH		 1096
1097long __put_user_asm_b(unsigned int x, void __user *addr)
1098{
1099 register unsigned int err asm ("D0Re0") = 0;
1100 asm volatile (
1101 " MOV %0,#0\n"
1102 " SETB [%2],%1\n"
1103 "1:\n"
1104 " SETB [%2],%1\n"
1105 "2:\n"
1106 ".section .fixup,\"ax\"\n"
1107 "3: MOV %0,%3\n"
1108 " MOVT D0FrT,#HI(2b)\n"
1109 " JUMP D0FrT,#LO(2b)\n"
1110 ".previous\n"
1111 ".section __ex_table,\"a\"\n"
1112 " .long 1b,3b\n"
1113 ".previous"
1114 : "=r"(err)
1115 : "d" (x), "a" (addr), "P"(-EFAULT)
1116 : "D0FrT");
1117 return err;
1118}
9da3ee9a 1119EXPORT_SYMBOL(__put_user_asm_b);
373cd784
JH		 1120
1121long __put_user_asm_w(unsigned int x, void __user *addr)
1122{
1123 register unsigned int err asm ("D0Re0") = 0;
1124 asm volatile (
1125 " MOV %0,#0\n"
1126 " SETW [%2],%1\n"
1127 "1:\n"
1128 " SETW [%2],%1\n"
1129 "2:\n"
1130 ".section .fixup,\"ax\"\n"
1131 "3: MOV %0,%3\n"
1132 " MOVT D0FrT,#HI(2b)\n"
1133 " JUMP D0FrT,#LO(2b)\n"
1134 ".previous\n"
1135 ".section __ex_table,\"a\"\n"
1136 " .long 1b,3b\n"
1137 ".previous"
1138 : "=r"(err)
1139 : "d" (x), "a" (addr), "P"(-EFAULT)
1140 : "D0FrT");
1141 return err;
1142}
9da3ee9a 1143EXPORT_SYMBOL(__put_user_asm_w);
373cd784
JH		 1144
1145long __put_user_asm_d(unsigned int x, void __user *addr)
1146{
1147 register unsigned int err asm ("D0Re0") = 0;
1148 asm volatile (
1149 " MOV %0,#0\n"
1150 " SETD [%2],%1\n"
1151 "1:\n"
1152 " SETD [%2],%1\n"
1153 "2:\n"
1154 ".section .fixup,\"ax\"\n"
1155 "3: MOV %0,%3\n"
1156 " MOVT D0FrT,#HI(2b)\n"
1157 " JUMP D0FrT,#LO(2b)\n"
1158 ".previous\n"
1159 ".section __ex_table,\"a\"\n"
1160 " .long 1b,3b\n"
1161 ".previous"
1162 : "=r"(err)
1163 : "d" (x), "a" (addr), "P"(-EFAULT)
1164 : "D0FrT");
1165 return err;
1166}
9da3ee9a 1167EXPORT_SYMBOL(__put_user_asm_d);
373cd784
JH		 1168
1169long __put_user_asm_l(unsigned long long x, void __user *addr)
1170{
1171 register unsigned int err asm ("D0Re0") = 0;
1172 asm volatile (
1173 " MOV %0,#0\n"
1174 " SETL [%2],%1,%t1\n"
1175 "1:\n"
1176 " SETL [%2],%1,%t1\n"
1177 "2:\n"
1178 ".section .fixup,\"ax\"\n"
1179 "3: MOV %0,%3\n"
1180 " MOVT D0FrT,#HI(2b)\n"
1181 " JUMP D0FrT,#LO(2b)\n"
1182 ".previous\n"
1183 ".section __ex_table,\"a\"\n"
1184 " .long 1b,3b\n"
1185 ".previous"
1186 : "=r"(err)
1187 : "d" (x), "a" (addr), "P"(-EFAULT)
1188 : "D0FrT");
1189 return err;
1190}
9da3ee9a 1191EXPORT_SYMBOL(__put_user_asm_l);
373cd784
JH		 1192
1193long strnlen_user(const char __user *src, long count)
1194{
1195 long res;
1196
1197 if (!access_ok(VERIFY_READ, src, 0))
1198 return 0;
1199
1200 asm volatile (" MOV D0Ar4, %1\n"
1201 " MOV D0Ar6, %2\n"
1202 "0:\n"
1203 " SUBS D0FrT, D0Ar6, #0\n"
1204 " SUB D0Ar6, D0Ar6, #1\n"
1205 " BLE 2f\n"
1206 " GETB D0FrT, [D0Ar4+#1++]\n"
1207 "1:\n"
1208 " TST D0FrT, #255\n"
1209 " BNE 0b\n"
1210 "2:\n"
1211 " SUB %0, %2, D0Ar6\n"
1212 "3:\n"
1213 " .section .fixup,\"ax\"\n"
1214 "4:\n"
1215 " MOV %0, #0\n"
1216 " MOVT D0FrT,#HI(3b)\n"
1217 " JUMP D0FrT,#LO(3b)\n"
1218 " .previous\n"
1219 " .section __ex_table,\"a\"\n"
1220 " .long 1b,4b\n"
1221 " .previous\n"
1222 : "=r" (res)
1223 : "r" (src), "r" (count)
1224 : "D0FrT", "D0Ar4", "D0Ar6", "cc");
1225
1226 return res;
1227}
9da3ee9a 1228EXPORT_SYMBOL(strnlen_user);
373cd784
JH		 1229
1230long __strncpy_from_user(char *dst, const char __user *src, long count)
1231{
1232 long res;
1233
1234 if (count == 0)
1235 return 0;
1236
1237 /*
1238 * Currently, in 2.4.0-test9, most ports use a simple byte-copy loop.
1239 * So do we.
1240 *
1241 * This code is deduced from:
1242 *
1243 * char tmp2;
1244 * long tmp1, tmp3;
1245 * tmp1 = count;
1246 * while ((*dst++ = (tmp2 = *src++)) != 0
1247 * && --tmp1)
1248 * ;
1249 *
1250 * res = count - tmp1;
1251 *
1252 * with tweaks.
1253 */
1254
1255 asm volatile (" MOV %0,%3\n"
1256 "1:\n"
1257 " GETB D0FrT,[%2++]\n"
1258 "2:\n"
1259 " CMP D0FrT,#0\n"
1260 " SETB [%1++],D0FrT\n"
1261 " BEQ 3f\n"
1262 " SUBS %0,%0,#1\n"
1263 " BNZ 1b\n"
1264 "3:\n"
1265 " SUB %0,%3,%0\n"
1266 "4:\n"
1267 " .section .fixup,\"ax\"\n"
1268 "5:\n"
1269 " MOV %0,%7\n"
1270 " MOVT D0FrT,#HI(4b)\n"
1271 " JUMP D0FrT,#LO(4b)\n"
1272 " .previous\n"
1273 " .section __ex_table,\"a\"\n"
1274 " .long 2b,5b\n"
1275 " .previous"
1276 : "=r" (res), "=r" (dst), "=r" (src), "=r" (count)
1277 : "3" (count), "1" (dst), "2" (src), "P" (-EFAULT)
1278 : "D0FrT", "memory", "cc");
1279
1280 return res;
1281}
9da3ee9a 1282EXPORT_SYMBOL(__strncpy_from_user);
Ubuntu Eoan Linux kernel mirror