]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * | |
3 | * Optimized version of the standard copy_page() function | |
4 | * | |
5 | * Inputs: | |
6 | * in0: address of target page | |
7 | * in1: address of source page | |
8 | * Output: | |
9 | * no return value | |
10 | * | |
11 | * Copyright (C) 1999, 2001 Hewlett-Packard Co | |
12 | * Stephane Eranian <eranian@hpl.hp.com> | |
13 | * David Mosberger <davidm@hpl.hp.com> | |
14 | * | |
15 | * 4/06/01 davidm Tuned to make it perform well both for cached and uncached copies. | |
16 | */ | |
17 | #include <asm/asmmacro.h> | |
18 | #include <asm/page.h> | |
e007c533 | 19 | #include <asm/export.h> |
1da177e4 LT |
20 | |
21 | #define PIPE_DEPTH 3 | |
22 | #define EPI p[PIPE_DEPTH-1] | |
23 | ||
24 | #define lcount r16 | |
25 | #define saved_pr r17 | |
26 | #define saved_lc r18 | |
27 | #define saved_pfs r19 | |
28 | #define src1 r20 | |
29 | #define src2 r21 | |
30 | #define tgt1 r22 | |
31 | #define tgt2 r23 | |
32 | #define srcf r24 | |
33 | #define tgtf r25 | |
34 | #define tgt_last r26 | |
35 | ||
36 | #define Nrot ((8*PIPE_DEPTH+7)&~7) | |
37 | ||
38 | GLOBAL_ENTRY(copy_page) | |
39 | .prologue | |
40 | .save ar.pfs, saved_pfs | |
41 | alloc saved_pfs=ar.pfs,3,Nrot-3,0,Nrot | |
42 | ||
43 | .rotr t1[PIPE_DEPTH], t2[PIPE_DEPTH], t3[PIPE_DEPTH], t4[PIPE_DEPTH], \ | |
44 | t5[PIPE_DEPTH], t6[PIPE_DEPTH], t7[PIPE_DEPTH], t8[PIPE_DEPTH] | |
45 | .rotp p[PIPE_DEPTH] | |
46 | ||
47 | .save ar.lc, saved_lc | |
48 | mov saved_lc=ar.lc | |
49 | mov ar.ec=PIPE_DEPTH | |
50 | ||
51 | mov lcount=PAGE_SIZE/64-1 | |
52 | .save pr, saved_pr | |
53 | mov saved_pr=pr | |
54 | mov pr.rot=1<<16 | |
55 | ||
56 | .body | |
57 | ||
58 | mov src1=in1 | |
59 | adds src2=8,in1 | |
60 | mov tgt_last = PAGE_SIZE | |
61 | ;; | |
62 | adds tgt2=8,in0 | |
63 | add srcf=512,in1 | |
64 | mov ar.lc=lcount | |
65 | mov tgt1=in0 | |
66 | add tgtf=512,in0 | |
67 | add tgt_last = tgt_last, in0 | |
68 | ;; | |
69 | 1: | |
70 | (p[0]) ld8 t1[0]=[src1],16 | |
71 | (EPI) st8 [tgt1]=t1[PIPE_DEPTH-1],16 | |
72 | (p[0]) ld8 t2[0]=[src2],16 | |
73 | (EPI) st8 [tgt2]=t2[PIPE_DEPTH-1],16 | |
74 | cmp.ltu p6,p0 = tgtf, tgt_last | |
75 | ;; | |
76 | (p[0]) ld8 t3[0]=[src1],16 | |
77 | (EPI) st8 [tgt1]=t3[PIPE_DEPTH-1],16 | |
78 | (p[0]) ld8 t4[0]=[src2],16 | |
79 | (EPI) st8 [tgt2]=t4[PIPE_DEPTH-1],16 | |
80 | ;; | |
81 | (p[0]) ld8 t5[0]=[src1],16 | |
82 | (EPI) st8 [tgt1]=t5[PIPE_DEPTH-1],16 | |
83 | (p[0]) ld8 t6[0]=[src2],16 | |
84 | (EPI) st8 [tgt2]=t6[PIPE_DEPTH-1],16 | |
85 | ;; | |
86 | (p[0]) ld8 t7[0]=[src1],16 | |
87 | (EPI) st8 [tgt1]=t7[PIPE_DEPTH-1],16 | |
88 | (p[0]) ld8 t8[0]=[src2],16 | |
89 | (EPI) st8 [tgt2]=t8[PIPE_DEPTH-1],16 | |
90 | ||
91 | (p6) lfetch [srcf], 64 | |
92 | (p6) lfetch [tgtf], 64 | |
93 | br.ctop.sptk.few 1b | |
94 | ;; | |
95 | mov pr=saved_pr,0xffffffffffff0000 // restore predicates | |
96 | mov ar.pfs=saved_pfs | |
97 | mov ar.lc=saved_lc | |
98 | br.ret.sptk.many rp | |
99 | END(copy_page) | |
e007c533 | 100 | EXPORT_SYMBOL(copy_page) |