2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * Copyright (C) 2003, 04, 05 Ralf Baechle (ralf@linux-mips.org)
8 #include <linux/init.h>
9 #include <linux/kernel.h>
10 #include <linux/sched.h>
12 #include <linux/module.h>
13 #include <linux/proc_fs.h>
15 #include <asm/cacheops.h>
19 #include <asm/pgtable.h>
20 #include <asm/prefetch.h>
21 #include <asm/system.h>
22 #include <asm/bootinfo.h>
23 #include <asm/mipsregs.h>
24 #include <asm/mmu_context.h>
28 #define half_scache_line_size() (cpu_scache_line_size() >> 1)
29 #define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010)
30 #define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020)
36 * R4000 128 bytes S-cache: 0x58 bytes
37 * R4600 v1.7: 0x5c bytes
38 * R4600 v2.0: 0x60 bytes
39 * With prefetching, 16 byte strides 0xa0 bytes
42 static unsigned int clear_page_array
[0x130 / 4];
44 void clear_page(void * page
) __attribute__((alias("clear_page_array")));
46 EXPORT_SYMBOL(clear_page
);
51 * R4000 128 bytes S-cache: 0x11c bytes
52 * R4600 v1.7: 0x080 bytes
53 * R4600 v2.0: 0x07c bytes
54 * With prefetching, 16 byte strides 0x0b8 bytes
56 static unsigned int copy_page_array
[0x148 / 4];
58 void copy_page(void *to
, void *from
) __attribute__((alias("copy_page_array")));
60 EXPORT_SYMBOL(copy_page
);
63 * This is suboptimal for 32-bit kernels; we assume that R10000 is only used
64 * with 64-bit kernels. The prefetch offsets have been experimentally tuned
67 static int pref_offset_clear __initdata
= 512;
68 static int pref_offset_copy __initdata
= 256;
70 static unsigned int pref_src_mode __initdata
;
71 static unsigned int pref_dst_mode __initdata
;
73 static int load_offset __initdata
;
74 static int store_offset __initdata
;
76 static unsigned int __initdata
*dest
, *epc
;
78 static unsigned int instruction_pending
;
79 static union mips_instruction delayed_mi
;
81 static void __init
emit_instruction(union mips_instruction mi
)
83 if (instruction_pending
)
84 *epc
++ = delayed_mi
.word
;
86 instruction_pending
= 1;
90 static inline void flush_delay_slot_or_nop(void)
92 if (instruction_pending
) {
93 *epc
++ = delayed_mi
.word
;
94 instruction_pending
= 0;
101 static inline unsigned int *label(void)
103 if (instruction_pending
) {
104 *epc
++ = delayed_mi
.word
;
105 instruction_pending
= 0;
111 static inline void build_insn_word(unsigned int word
)
113 union mips_instruction mi
;
117 emit_instruction(mi
);
120 static inline void build_nop(void)
122 build_insn_word(0); /* nop */
125 static inline void build_src_pref(int advance
)
127 if (!(load_offset
& (cpu_dcache_line_size() - 1)) && advance
) {
128 union mips_instruction mi
;
130 mi
.i_format
.opcode
= pref_op
;
131 mi
.i_format
.rs
= 5; /* $a1 */
132 mi
.i_format
.rt
= pref_src_mode
;
133 mi
.i_format
.simmediate
= load_offset
+ advance
;
135 emit_instruction(mi
);
139 static inline void __build_load_reg(int reg
)
141 union mips_instruction mi
;
144 if (cpu_has_64bit_gp_regs
) {
145 mi
.i_format
.opcode
= ld_op
;
148 mi
.i_format
.opcode
= lw_op
;
151 mi
.i_format
.rs
= 5; /* $a1 */
152 mi
.i_format
.rt
= reg
; /* $reg */
153 mi
.i_format
.simmediate
= load_offset
;
155 load_offset
+= width
;
156 emit_instruction(mi
);
159 static inline void build_load_reg(int reg
)
161 if (cpu_has_prefetch
)
162 build_src_pref(pref_offset_copy
);
164 __build_load_reg(reg
);
167 static inline void build_dst_pref(int advance
)
169 if (!(store_offset
& (cpu_dcache_line_size() - 1)) && advance
) {
170 union mips_instruction mi
;
172 mi
.i_format
.opcode
= pref_op
;
173 mi
.i_format
.rs
= 4; /* $a0 */
174 mi
.i_format
.rt
= pref_dst_mode
;
175 mi
.i_format
.simmediate
= store_offset
+ advance
;
177 emit_instruction(mi
);
181 static inline void build_cdex_s(void)
183 union mips_instruction mi
;
185 if ((store_offset
& (cpu_scache_line_size() - 1)))
188 mi
.c_format
.opcode
= cache_op
;
189 mi
.c_format
.rs
= 4; /* $a0 */
190 mi
.c_format
.c_op
= 3; /* Create Dirty Exclusive */
191 mi
.c_format
.cache
= 3; /* Secondary Data Cache */
192 mi
.c_format
.simmediate
= store_offset
;
194 emit_instruction(mi
);
197 static inline void build_cdex_p(void)
199 union mips_instruction mi
;
201 if (store_offset
& (cpu_dcache_line_size() - 1))
204 if (R4600_V1_HIT_CACHEOP_WAR
&& cpu_is_r4600_v1_x()) {
211 if (R4600_V2_HIT_CACHEOP_WAR
&& cpu_is_r4600_v2_x())
212 build_insn_word(0x3c01a000); /* lui $at, 0xa000 */
214 mi
.c_format
.opcode
= cache_op
;
215 mi
.c_format
.rs
= 4; /* $a0 */
216 mi
.c_format
.c_op
= 3; /* Create Dirty Exclusive */
217 mi
.c_format
.cache
= 1; /* Data Cache */
218 mi
.c_format
.simmediate
= store_offset
;
220 emit_instruction(mi
);
223 static void __init
__build_store_reg(int reg
)
225 union mips_instruction mi
;
228 if (cpu_has_64bit_gp_regs
||
229 (cpu_has_64bit_zero_reg
&& reg
== 0)) {
230 mi
.i_format
.opcode
= sd_op
;
233 mi
.i_format
.opcode
= sw_op
;
236 mi
.i_format
.rs
= 4; /* $a0 */
237 mi
.i_format
.rt
= reg
; /* $reg */
238 mi
.i_format
.simmediate
= store_offset
;
240 store_offset
+= width
;
241 emit_instruction(mi
);
244 static inline void build_store_reg(int reg
)
246 if (cpu_has_prefetch
)
248 build_dst_pref(pref_offset_copy
);
250 build_dst_pref(pref_offset_clear
);
251 else if (cpu_has_cache_cdex_s
)
253 else if (cpu_has_cache_cdex_p
)
256 __build_store_reg(reg
);
259 static inline void build_addiu_a2_a0(unsigned long offset
)
261 union mips_instruction mi
;
263 BUG_ON(offset
> 0x7fff);
265 mi
.i_format
.opcode
= cpu_has_64bit_gp_regs
? daddiu_op
: addiu_op
;
266 mi
.i_format
.rs
= 4; /* $a0 */
267 mi
.i_format
.rt
= 6; /* $a2 */
268 mi
.i_format
.simmediate
= offset
;
270 emit_instruction(mi
);
273 static inline void build_addiu_a1(unsigned long offset
)
275 union mips_instruction mi
;
277 BUG_ON(offset
> 0x7fff);
279 mi
.i_format
.opcode
= cpu_has_64bit_gp_regs
? daddiu_op
: addiu_op
;
280 mi
.i_format
.rs
= 5; /* $a1 */
281 mi
.i_format
.rt
= 5; /* $a1 */
282 mi
.i_format
.simmediate
= offset
;
284 load_offset
-= offset
;
286 emit_instruction(mi
);
289 static inline void build_addiu_a0(unsigned long offset
)
291 union mips_instruction mi
;
293 BUG_ON(offset
> 0x7fff);
295 mi
.i_format
.opcode
= cpu_has_64bit_gp_regs
? daddiu_op
: addiu_op
;
296 mi
.i_format
.rs
= 4; /* $a0 */
297 mi
.i_format
.rt
= 4; /* $a0 */
298 mi
.i_format
.simmediate
= offset
;
300 store_offset
-= offset
;
302 emit_instruction(mi
);
305 static inline void build_bne(unsigned int *dest
)
307 union mips_instruction mi
;
309 mi
.i_format
.opcode
= bne_op
;
310 mi
.i_format
.rs
= 6; /* $a2 */
311 mi
.i_format
.rt
= 4; /* $a0 */
312 mi
.i_format
.simmediate
= dest
- epc
- 1;
315 flush_delay_slot_or_nop();
318 static inline void build_jr_ra(void)
320 union mips_instruction mi
;
322 mi
.r_format
.opcode
= spec_op
;
327 mi
.r_format
.func
= jr_op
;
330 flush_delay_slot_or_nop();
333 void __init
build_clear_page(void)
335 unsigned int loop_start
;
337 epc
= (unsigned int *) &clear_page_array
;
338 instruction_pending
= 0;
341 if (cpu_has_prefetch
) {
342 switch (current_cpu_data
.cputype
) {
344 /* TX49 supports only Pref_Load */
345 pref_offset_clear
= 0;
346 pref_offset_copy
= 0;
351 * As a workaround for erratum G105 which make the
352 * PrepareForStore hint unusable we fall back to
353 * StoreRetained on the RM9000. Once it is known which
354 * versions of the RM9000 we'll be able to condition-
360 pref_src_mode
= Pref_LoadStreamed
;
361 pref_dst_mode
= Pref_StoreStreamed
;
365 pref_src_mode
= Pref_LoadStreamed
;
366 pref_dst_mode
= Pref_PrepareForStore
;
371 build_addiu_a2_a0(PAGE_SIZE
- (cpu_has_prefetch
? pref_offset_clear
: 0));
373 if (R4600_V2_HIT_CACHEOP_WAR
&& cpu_is_r4600_v2_x())
374 build_insn_word(0x3c01a000); /* lui $at, 0xa000 */
382 } while (store_offset
< half_scache_line_size());
383 build_addiu_a0(2 * store_offset
);
384 loop_start
= store_offset
;
390 } while ((store_offset
- loop_start
) < half_scache_line_size());
393 if (cpu_has_prefetch
&& pref_offset_clear
) {
394 build_addiu_a2_a0(pref_offset_clear
);
396 loop_start
= store_offset
;
398 __build_store_reg(0);
399 __build_store_reg(0);
400 __build_store_reg(0);
401 __build_store_reg(0);
402 } while ((store_offset
- loop_start
) < half_scache_line_size());
403 build_addiu_a0(2 * store_offset
);
404 loop_start
= store_offset
;
406 __build_store_reg(0);
407 __build_store_reg(0);
408 __build_store_reg(0);
409 __build_store_reg(0);
410 } while ((store_offset
- loop_start
) < half_scache_line_size());
416 BUG_ON(epc
> clear_page_array
+ ARRAY_SIZE(clear_page_array
));
419 void __init
build_copy_page(void)
421 unsigned int loop_start
;
423 epc
= (unsigned int *) ©_page_array
;
424 store_offset
= load_offset
= 0;
425 instruction_pending
= 0;
427 build_addiu_a2_a0(PAGE_SIZE
- (cpu_has_prefetch
? pref_offset_copy
: 0));
429 if (R4600_V2_HIT_CACHEOP_WAR
&& cpu_is_r4600_v2_x())
430 build_insn_word(0x3c01a000); /* lui $at, 0xa000 */
433 loop_start
= store_offset
;
443 } while ((store_offset
- loop_start
) < half_scache_line_size());
444 build_addiu_a0(2 * store_offset
);
445 build_addiu_a1(2 * load_offset
);
446 loop_start
= store_offset
;
456 } while ((store_offset
- loop_start
) < half_scache_line_size());
459 if (cpu_has_prefetch
&& pref_offset_copy
) {
460 build_addiu_a2_a0(pref_offset_copy
);
462 loop_start
= store_offset
;
464 __build_load_reg( 8);
465 __build_load_reg( 9);
466 __build_load_reg(10);
467 __build_load_reg(11);
468 __build_store_reg( 8);
469 __build_store_reg( 9);
470 __build_store_reg(10);
471 __build_store_reg(11);
472 } while ((store_offset
- loop_start
) < half_scache_line_size());
473 build_addiu_a0(2 * store_offset
);
474 build_addiu_a1(2 * load_offset
);
475 loop_start
= store_offset
;
477 __build_load_reg( 8);
478 __build_load_reg( 9);
479 __build_load_reg(10);
480 __build_load_reg(11);
481 __build_store_reg( 8);
482 __build_store_reg( 9);
483 __build_store_reg(10);
484 __build_store_reg(11);
485 } while ((store_offset
- loop_start
) < half_scache_line_size());
491 BUG_ON(epc
> copy_page_array
+ ARRAY_SIZE(copy_page_array
));