]>
Commit | Line | Data |
---|---|---|
cf5adce1 DM |
1 | /* NG2memcpy.S: Niagara-2 optimized memcpy. |
2 | * | |
3 | * Copyright (C) 2007 David S. Miller (davem@davemloft.net) | |
4 | */ | |
5 | ||
6 | #ifdef __KERNEL__ | |
7 | #include <asm/visasm.h> | |
8 | #include <asm/asi.h> | |
9 | #define GLOBAL_SPARE %g7 | |
10 | #else | |
11 | #define ASI_PNF 0x82 | |
12 | #define ASI_BLK_P 0xf0 | |
13 | #define ASI_BLK_INIT_QUAD_LDD_P 0xe2 | |
14 | #define FPRS_FEF 0x04 | |
15 | #ifdef MEMCPY_DEBUG | |
16 | #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \ | |
1b62ca7b | 17 | clr %g1; clr %g2; clr %g3; clr %g5; subcc %g0, %g0, %g0; |
cf5adce1 DM |
18 | #define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs |
19 | #else | |
20 | #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs | |
21 | #define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs | |
22 | #endif | |
23 | #define GLOBAL_SPARE %g5 | |
24 | #endif | |
25 | ||
26 | #ifndef STORE_ASI | |
27 | #ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA | |
28 | #define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P | |
29 | #else | |
30 | #define STORE_ASI 0x80 /* ASI_P */ | |
31 | #endif | |
32 | #endif | |
33 | ||
34 | #ifndef EX_LD | |
35 | #define EX_LD(x) x | |
36 | #endif | |
a7c5724b RG |
37 | #ifndef EX_LD_FP |
38 | #define EX_LD_FP(x) x | |
39 | #endif | |
cf5adce1 DM |
40 | |
41 | #ifndef EX_ST | |
42 | #define EX_ST(x) x | |
43 | #endif | |
a7c5724b RG |
44 | #ifndef EX_ST_FP |
45 | #define EX_ST_FP(x) x | |
46 | #endif | |
cf5adce1 DM |
47 | |
48 | #ifndef EX_RETVAL | |
49 | #define EX_RETVAL(x) x | |
50 | #endif | |
51 | ||
52 | #ifndef LOAD | |
53 | #define LOAD(type,addr,dest) type [addr], dest | |
54 | #endif | |
55 | ||
56 | #ifndef LOAD_BLK | |
57 | #define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_P, dest | |
58 | #endif | |
59 | ||
60 | #ifndef STORE | |
61 | #ifndef MEMCPY_DEBUG | |
62 | #define STORE(type,src,addr) type src, [addr] | |
63 | #else | |
64 | #define STORE(type,src,addr) type##a src, [addr] 0x80 | |
65 | #endif | |
66 | #endif | |
67 | ||
68 | #ifndef STORE_BLK | |
69 | #define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P | |
70 | #endif | |
71 | ||
72 | #ifndef STORE_INIT | |
73 | #define STORE_INIT(src,addr) stxa src, [addr] STORE_ASI | |
74 | #endif | |
75 | ||
76 | #ifndef FUNC_NAME | |
77 | #define FUNC_NAME NG2memcpy | |
78 | #endif | |
79 | ||
80 | #ifndef PREAMBLE | |
81 | #define PREAMBLE | |
82 | #endif | |
83 | ||
84 | #ifndef XCC | |
85 | #define XCC xcc | |
86 | #endif | |
87 | ||
88 | #define FREG_FROB(x0, x1, x2, x3, x4, x5, x6, x7, x8) \ | |
89 | faligndata %x0, %x1, %f0; \ | |
90 | faligndata %x1, %x2, %f2; \ | |
91 | faligndata %x2, %x3, %f4; \ | |
92 | faligndata %x3, %x4, %f6; \ | |
93 | faligndata %x4, %x5, %f8; \ | |
94 | faligndata %x5, %x6, %f10; \ | |
95 | faligndata %x6, %x7, %f12; \ | |
96 | faligndata %x7, %x8, %f14; | |
97 | ||
98 | #define FREG_MOVE_1(x0) \ | |
6f1d827f | 99 | fsrc2 %x0, %f0; |
cf5adce1 | 100 | #define FREG_MOVE_2(x0, x1) \ |
6f1d827f DM |
101 | fsrc2 %x0, %f0; \ |
102 | fsrc2 %x1, %f2; | |
cf5adce1 | 103 | #define FREG_MOVE_3(x0, x1, x2) \ |
6f1d827f DM |
104 | fsrc2 %x0, %f0; \ |
105 | fsrc2 %x1, %f2; \ | |
106 | fsrc2 %x2, %f4; | |
cf5adce1 | 107 | #define FREG_MOVE_4(x0, x1, x2, x3) \ |
6f1d827f DM |
108 | fsrc2 %x0, %f0; \ |
109 | fsrc2 %x1, %f2; \ | |
110 | fsrc2 %x2, %f4; \ | |
111 | fsrc2 %x3, %f6; | |
cf5adce1 | 112 | #define FREG_MOVE_5(x0, x1, x2, x3, x4) \ |
6f1d827f DM |
113 | fsrc2 %x0, %f0; \ |
114 | fsrc2 %x1, %f2; \ | |
115 | fsrc2 %x2, %f4; \ | |
116 | fsrc2 %x3, %f6; \ | |
117 | fsrc2 %x4, %f8; | |
cf5adce1 | 118 | #define FREG_MOVE_6(x0, x1, x2, x3, x4, x5) \ |
6f1d827f DM |
119 | fsrc2 %x0, %f0; \ |
120 | fsrc2 %x1, %f2; \ | |
121 | fsrc2 %x2, %f4; \ | |
122 | fsrc2 %x3, %f6; \ | |
123 | fsrc2 %x4, %f8; \ | |
124 | fsrc2 %x5, %f10; | |
cf5adce1 | 125 | #define FREG_MOVE_7(x0, x1, x2, x3, x4, x5, x6) \ |
6f1d827f DM |
126 | fsrc2 %x0, %f0; \ |
127 | fsrc2 %x1, %f2; \ | |
128 | fsrc2 %x2, %f4; \ | |
129 | fsrc2 %x3, %f6; \ | |
130 | fsrc2 %x4, %f8; \ | |
131 | fsrc2 %x5, %f10; \ | |
132 | fsrc2 %x6, %f12; | |
cf5adce1 | 133 | #define FREG_MOVE_8(x0, x1, x2, x3, x4, x5, x6, x7) \ |
6f1d827f DM |
134 | fsrc2 %x0, %f0; \ |
135 | fsrc2 %x1, %f2; \ | |
136 | fsrc2 %x2, %f4; \ | |
137 | fsrc2 %x3, %f6; \ | |
138 | fsrc2 %x4, %f8; \ | |
139 | fsrc2 %x5, %f10; \ | |
140 | fsrc2 %x6, %f12; \ | |
141 | fsrc2 %x7, %f14; | |
cf5adce1 | 142 | #define FREG_LOAD_1(base, x0) \ |
a7c5724b | 143 | EX_LD_FP(LOAD(ldd, base + 0x00, %x0)) |
cf5adce1 | 144 | #define FREG_LOAD_2(base, x0, x1) \ |
a7c5724b RG |
145 | EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ |
146 | EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); | |
cf5adce1 | 147 | #define FREG_LOAD_3(base, x0, x1, x2) \ |
a7c5724b RG |
148 | EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ |
149 | EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ | |
150 | EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); | |
cf5adce1 | 151 | #define FREG_LOAD_4(base, x0, x1, x2, x3) \ |
a7c5724b RG |
152 | EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ |
153 | EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ | |
154 | EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ | |
155 | EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); | |
cf5adce1 | 156 | #define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \ |
a7c5724b RG |
157 | EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ |
158 | EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ | |
159 | EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ | |
160 | EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \ | |
161 | EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); | |
cf5adce1 | 162 | #define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \ |
a7c5724b RG |
163 | EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ |
164 | EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ | |
165 | EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ | |
166 | EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \ | |
167 | EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \ | |
168 | EX_LD_FP(LOAD(ldd, base + 0x28, %x5)); | |
cf5adce1 | 169 | #define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \ |
a7c5724b RG |
170 | EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ |
171 | EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ | |
172 | EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ | |
173 | EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \ | |
174 | EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \ | |
175 | EX_LD_FP(LOAD(ldd, base + 0x28, %x5)); \ | |
176 | EX_LD_FP(LOAD(ldd, base + 0x30, %x6)); | |
cf5adce1 DM |
177 | |
178 | .register %g2,#scratch | |
179 | .register %g3,#scratch | |
180 | ||
181 | .text | |
182 | .align 64 | |
183 | ||
184 | .globl FUNC_NAME | |
185 | .type FUNC_NAME,#function | |
186 | FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ | |
187 | srlx %o2, 31, %g2 | |
188 | cmp %g2, 0 | |
189 | tne %xcc, 5 | |
190 | PREAMBLE | |
1b62ca7b | 191 | mov %o0, %o3 |
cf5adce1 DM |
192 | cmp %o2, 0 |
193 | be,pn %XCC, 85f | |
1b62ca7b | 194 | or %o0, %o1, GLOBAL_SPARE |
cf5adce1 DM |
195 | cmp %o2, 16 |
196 | blu,a,pn %XCC, 80f | |
1b62ca7b | 197 | or GLOBAL_SPARE, %o2, GLOBAL_SPARE |
cf5adce1 DM |
198 | |
199 | /* 2 blocks (128 bytes) is the minimum we can do the block | |
200 | * copy with. We need to ensure that we'll iterate at least | |
201 | * once in the block copy loop. At worst we'll need to align | |
202 | * the destination to a 64-byte boundary which can chew up | |
203 | * to (64 - 1) bytes from the length before we perform the | |
204 | * block copy loop. | |
205 | * | |
206 | * However, the cut-off point, performance wise, is around | |
207 | * 4 64-byte blocks. | |
208 | */ | |
209 | cmp %o2, (4 * 64) | |
210 | blu,pt %XCC, 75f | |
1b62ca7b | 211 | andcc GLOBAL_SPARE, 0x7, %g0 |
cf5adce1 DM |
212 | |
213 | /* %o0: dst | |
214 | * %o1: src | |
215 | * %o2: len (known to be >= 128) | |
216 | * | |
217 | * The block copy loops can use %o4, %g2, %g3 as | |
218 | * temporaries while copying the data. %o5 must | |
219 | * be preserved between VISEntryHalf and VISExitHalf | |
220 | */ | |
221 | ||
222 | LOAD(prefetch, %o1 + 0x000, #one_read) | |
223 | LOAD(prefetch, %o1 + 0x040, #one_read) | |
224 | LOAD(prefetch, %o1 + 0x080, #one_read) | |
225 | ||
226 | /* Align destination on 64-byte boundary. */ | |
227 | andcc %o0, (64 - 1), %o4 | |
228 | be,pt %XCC, 2f | |
229 | sub %o4, 64, %o4 | |
230 | sub %g0, %o4, %o4 ! bytes to align dst | |
231 | sub %o2, %o4, %o2 | |
232 | 1: subcc %o4, 1, %o4 | |
233 | EX_LD(LOAD(ldub, %o1, %g1)) | |
234 | EX_ST(STORE(stb, %g1, %o0)) | |
235 | add %o1, 1, %o1 | |
236 | bne,pt %XCC, 1b | |
237 | add %o0, 1, %o0 | |
238 | ||
239 | 2: | |
240 | /* Clobbers o5/g1/g2/g3/g7/icc/xcc. We must preserve | |
241 | * o5 from here until we hit VISExitHalf. | |
242 | */ | |
243 | VISEntryHalf | |
244 | ||
5aa4ecfd | 245 | membar #Sync |
cf5adce1 DM |
246 | alignaddr %o1, %g0, %g0 |
247 | ||
248 | add %o1, (64 - 1), %o4 | |
249 | andn %o4, (64 - 1), %o4 | |
250 | andn %o2, (64 - 1), %g1 | |
251 | sub %o2, %g1, %o2 | |
252 | ||
253 | and %o1, (64 - 1), %g2 | |
254 | add %o1, %g1, %o1 | |
255 | sub %o0, %o4, %g3 | |
256 | brz,pt %g2, 190f | |
257 | cmp %g2, 32 | |
258 | blu,a 5f | |
259 | cmp %g2, 16 | |
260 | cmp %g2, 48 | |
261 | blu,a 4f | |
262 | cmp %g2, 40 | |
263 | cmp %g2, 56 | |
264 | blu 170f | |
265 | nop | |
266 | ba,a,pt %xcc, 180f | |
267 | ||
268 | 4: /* 32 <= low bits < 48 */ | |
269 | blu 150f | |
270 | nop | |
271 | ba,a,pt %xcc, 160f | |
272 | 5: /* 0 < low bits < 32 */ | |
273 | blu,a 6f | |
274 | cmp %g2, 8 | |
275 | cmp %g2, 24 | |
276 | blu 130f | |
277 | nop | |
278 | ba,a,pt %xcc, 140f | |
279 | 6: /* 0 < low bits < 16 */ | |
280 | bgeu 120f | |
281 | nop | |
282 | /* fall through for 0 < low bits < 8 */ | |
283 | 110: sub %o4, 64, %g2 | |
a7c5724b RG |
284 | EX_LD_FP(LOAD_BLK(%g2, %f0)) |
285 | 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) | |
286 | EX_LD_FP(LOAD_BLK(%o4, %f16)) | |
cf5adce1 | 287 | FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16) |
a7c5724b | 288 | EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) |
cf5adce1 DM |
289 | FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30) |
290 | subcc %g1, 64, %g1 | |
291 | add %o4, 64, %o4 | |
292 | bne,pt %xcc, 1b | |
293 | LOAD(prefetch, %o4 + 64, #one_read) | |
294 | ba,pt %xcc, 195f | |
295 | nop | |
296 | ||
297 | 120: sub %o4, 56, %g2 | |
298 | FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12) | |
a7c5724b RG |
299 | 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) |
300 | EX_LD_FP(LOAD_BLK(%o4, %f16)) | |
cf5adce1 | 301 | FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18) |
a7c5724b | 302 | EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) |
cf5adce1 DM |
303 | FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30) |
304 | subcc %g1, 64, %g1 | |
305 | add %o4, 64, %o4 | |
306 | bne,pt %xcc, 1b | |
307 | LOAD(prefetch, %o4 + 64, #one_read) | |
308 | ba,pt %xcc, 195f | |
309 | nop | |
310 | ||
311 | 130: sub %o4, 48, %g2 | |
312 | FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10) | |
a7c5724b RG |
313 | 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) |
314 | EX_LD_FP(LOAD_BLK(%o4, %f16)) | |
cf5adce1 | 315 | FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20) |
a7c5724b | 316 | EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) |
cf5adce1 DM |
317 | FREG_MOVE_6(f20, f22, f24, f26, f28, f30) |
318 | subcc %g1, 64, %g1 | |
319 | add %o4, 64, %o4 | |
320 | bne,pt %xcc, 1b | |
321 | LOAD(prefetch, %o4 + 64, #one_read) | |
322 | ba,pt %xcc, 195f | |
323 | nop | |
324 | ||
325 | 140: sub %o4, 40, %g2 | |
326 | FREG_LOAD_5(%g2, f0, f2, f4, f6, f8) | |
a7c5724b RG |
327 | 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) |
328 | EX_LD_FP(LOAD_BLK(%o4, %f16)) | |
cf5adce1 | 329 | FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22) |
a7c5724b | 330 | EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) |
cf5adce1 DM |
331 | FREG_MOVE_5(f22, f24, f26, f28, f30) |
332 | subcc %g1, 64, %g1 | |
333 | add %o4, 64, %o4 | |
334 | bne,pt %xcc, 1b | |
335 | LOAD(prefetch, %o4 + 64, #one_read) | |
336 | ba,pt %xcc, 195f | |
337 | nop | |
338 | ||
339 | 150: sub %o4, 32, %g2 | |
340 | FREG_LOAD_4(%g2, f0, f2, f4, f6) | |
a7c5724b RG |
341 | 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) |
342 | EX_LD_FP(LOAD_BLK(%o4, %f16)) | |
cf5adce1 | 343 | FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24) |
a7c5724b | 344 | EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) |
cf5adce1 DM |
345 | FREG_MOVE_4(f24, f26, f28, f30) |
346 | subcc %g1, 64, %g1 | |
347 | add %o4, 64, %o4 | |
348 | bne,pt %xcc, 1b | |
349 | LOAD(prefetch, %o4 + 64, #one_read) | |
350 | ba,pt %xcc, 195f | |
351 | nop | |
352 | ||
353 | 160: sub %o4, 24, %g2 | |
354 | FREG_LOAD_3(%g2, f0, f2, f4) | |
a7c5724b RG |
355 | 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) |
356 | EX_LD_FP(LOAD_BLK(%o4, %f16)) | |
cf5adce1 | 357 | FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26) |
a7c5724b | 358 | EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) |
cf5adce1 DM |
359 | FREG_MOVE_3(f26, f28, f30) |
360 | subcc %g1, 64, %g1 | |
361 | add %o4, 64, %o4 | |
362 | bne,pt %xcc, 1b | |
363 | LOAD(prefetch, %o4 + 64, #one_read) | |
364 | ba,pt %xcc, 195f | |
365 | nop | |
366 | ||
367 | 170: sub %o4, 16, %g2 | |
368 | FREG_LOAD_2(%g2, f0, f2) | |
a7c5724b RG |
369 | 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) |
370 | EX_LD_FP(LOAD_BLK(%o4, %f16)) | |
cf5adce1 | 371 | FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28) |
a7c5724b | 372 | EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) |
cf5adce1 DM |
373 | FREG_MOVE_2(f28, f30) |
374 | subcc %g1, 64, %g1 | |
375 | add %o4, 64, %o4 | |
376 | bne,pt %xcc, 1b | |
377 | LOAD(prefetch, %o4 + 64, #one_read) | |
378 | ba,pt %xcc, 195f | |
379 | nop | |
380 | ||
381 | 180: sub %o4, 8, %g2 | |
382 | FREG_LOAD_1(%g2, f0) | |
a7c5724b RG |
383 | 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) |
384 | EX_LD_FP(LOAD_BLK(%o4, %f16)) | |
cf5adce1 | 385 | FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30) |
a7c5724b | 386 | EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) |
cf5adce1 DM |
387 | FREG_MOVE_1(f30) |
388 | subcc %g1, 64, %g1 | |
389 | add %o4, 64, %o4 | |
390 | bne,pt %xcc, 1b | |
391 | LOAD(prefetch, %o4 + 64, #one_read) | |
392 | ba,pt %xcc, 195f | |
393 | nop | |
394 | ||
395 | 190: | |
a7c5724b | 396 | 1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) |
cf5adce1 | 397 | subcc %g1, 64, %g1 |
a7c5724b RG |
398 | EX_LD_FP(LOAD_BLK(%o4, %f0)) |
399 | EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) | |
cf5adce1 DM |
400 | add %o4, 64, %o4 |
401 | bne,pt %xcc, 1b | |
402 | LOAD(prefetch, %o4 + 64, #one_read) | |
403 | ||
404 | 195: | |
405 | add %o4, %g3, %o0 | |
406 | membar #Sync | |
407 | ||
408 | VISExitHalf | |
409 | ||
410 | /* %o2 contains any final bytes still needed to be copied | |
411 | * over. If anything is left, we copy it one byte at a time. | |
412 | */ | |
413 | brz,pt %o2, 85f | |
1b62ca7b | 414 | sub %o0, %o1, GLOBAL_SPARE |
cf5adce1 DM |
415 | ba,a,pt %XCC, 90f |
416 | ||
417 | .align 64 | |
418 | 75: /* 16 < len <= 64 */ | |
419 | bne,pn %XCC, 75f | |
1b62ca7b | 420 | sub %o0, %o1, GLOBAL_SPARE |
cf5adce1 DM |
421 | |
422 | 72: | |
423 | andn %o2, 0xf, %o4 | |
424 | and %o2, 0xf, %o2 | |
425 | 1: subcc %o4, 0x10, %o4 | |
426 | EX_LD(LOAD(ldx, %o1, %o5)) | |
427 | add %o1, 0x08, %o1 | |
428 | EX_LD(LOAD(ldx, %o1, %g1)) | |
429 | sub %o1, 0x08, %o1 | |
1b62ca7b | 430 | EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE)) |
cf5adce1 | 431 | add %o1, 0x8, %o1 |
1b62ca7b | 432 | EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE)) |
cf5adce1 DM |
433 | bgu,pt %XCC, 1b |
434 | add %o1, 0x8, %o1 | |
435 | 73: andcc %o2, 0x8, %g0 | |
436 | be,pt %XCC, 1f | |
437 | nop | |
438 | sub %o2, 0x8, %o2 | |
439 | EX_LD(LOAD(ldx, %o1, %o5)) | |
1b62ca7b | 440 | EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE)) |
cf5adce1 DM |
441 | add %o1, 0x8, %o1 |
442 | 1: andcc %o2, 0x4, %g0 | |
443 | be,pt %XCC, 1f | |
444 | nop | |
445 | sub %o2, 0x4, %o2 | |
446 | EX_LD(LOAD(lduw, %o1, %o5)) | |
1b62ca7b | 447 | EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE)) |
cf5adce1 DM |
448 | add %o1, 0x4, %o1 |
449 | 1: cmp %o2, 0 | |
450 | be,pt %XCC, 85f | |
451 | nop | |
452 | ba,pt %xcc, 90f | |
453 | nop | |
454 | ||
455 | 75: | |
456 | andcc %o0, 0x7, %g1 | |
457 | sub %g1, 0x8, %g1 | |
458 | be,pn %icc, 2f | |
459 | sub %g0, %g1, %g1 | |
460 | sub %o2, %g1, %o2 | |
461 | ||
462 | 1: subcc %g1, 1, %g1 | |
463 | EX_LD(LOAD(ldub, %o1, %o5)) | |
1b62ca7b | 464 | EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE)) |
cf5adce1 DM |
465 | bgu,pt %icc, 1b |
466 | add %o1, 1, %o1 | |
467 | ||
1b62ca7b | 468 | 2: add %o1, GLOBAL_SPARE, %o0 |
cf5adce1 DM |
469 | andcc %o1, 0x7, %g1 |
470 | bne,pt %icc, 8f | |
471 | sll %g1, 3, %g1 | |
472 | ||
473 | cmp %o2, 16 | |
474 | bgeu,pt %icc, 72b | |
475 | nop | |
476 | ba,a,pt %xcc, 73b | |
477 | ||
1b62ca7b | 478 | 8: mov 64, GLOBAL_SPARE |
cf5adce1 DM |
479 | andn %o1, 0x7, %o1 |
480 | EX_LD(LOAD(ldx, %o1, %g2)) | |
1b62ca7b | 481 | sub GLOBAL_SPARE, %g1, GLOBAL_SPARE |
cf5adce1 DM |
482 | andn %o2, 0x7, %o4 |
483 | sllx %g2, %g1, %g2 | |
484 | 1: add %o1, 0x8, %o1 | |
485 | EX_LD(LOAD(ldx, %o1, %g3)) | |
486 | subcc %o4, 0x8, %o4 | |
1b62ca7b | 487 | srlx %g3, GLOBAL_SPARE, %o5 |
cf5adce1 DM |
488 | or %o5, %g2, %o5 |
489 | EX_ST(STORE(stx, %o5, %o0)) | |
490 | add %o0, 0x8, %o0 | |
491 | bgu,pt %icc, 1b | |
492 | sllx %g3, %g1, %g2 | |
493 | ||
494 | srl %g1, 3, %g1 | |
495 | andcc %o2, 0x7, %o2 | |
496 | be,pn %icc, 85f | |
497 | add %o1, %g1, %o1 | |
498 | ba,pt %xcc, 90f | |
1b62ca7b | 499 | sub %o0, %o1, GLOBAL_SPARE |
cf5adce1 DM |
500 | |
501 | .align 64 | |
502 | 80: /* 0 < len <= 16 */ | |
1b62ca7b | 503 | andcc GLOBAL_SPARE, 0x3, %g0 |
cf5adce1 | 504 | bne,pn %XCC, 90f |
1b62ca7b | 505 | sub %o0, %o1, GLOBAL_SPARE |
cf5adce1 DM |
506 | |
507 | 1: | |
508 | subcc %o2, 4, %o2 | |
509 | EX_LD(LOAD(lduw, %o1, %g1)) | |
1b62ca7b | 510 | EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE)) |
cf5adce1 DM |
511 | bgu,pt %XCC, 1b |
512 | add %o1, 4, %o1 | |
513 | ||
514 | 85: retl | |
1b62ca7b | 515 | mov EX_RETVAL(%o3), %o0 |
cf5adce1 DM |
516 | |
517 | .align 32 | |
518 | 90: | |
519 | subcc %o2, 1, %o2 | |
520 | EX_LD(LOAD(ldub, %o1, %g1)) | |
1b62ca7b | 521 | EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE)) |
cf5adce1 DM |
522 | bgu,pt %XCC, 90b |
523 | add %o1, 1, %o1 | |
524 | retl | |
1b62ca7b | 525 | mov EX_RETVAL(%o3), %o0 |
cf5adce1 DM |
526 | |
527 | .size FUNC_NAME, .-FUNC_NAME |