[mirror_ubuntu-artful-kernel.git] / arch / mips / mm / pg-r4k.c

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 2003, 04, 05 Ralf Baechle (ralf@linux-mips.org)
 */
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/proc_fs.h>

#include <asm/cacheops.h>
#include <asm/inst.h>
#include <asm/io.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/prefetch.h>
#include <asm/system.h>
#include <asm/bootinfo.h>
#include <asm/mipsregs.h>
#include <asm/mmu_context.h>
#include <asm/cpu.h>
#include <asm/war.h>

#define half_scache_line_size()	(cpu_scache_line_size() >> 1)
#define cpu_is_r4600_v1_x()	((read_c0_prid() & 0xfffffff0) == 0x00002010)
#define cpu_is_r4600_v2_x()	((read_c0_prid() & 0xfffffff0) == 0x00002020)


/*
 * Maximum sizes:
 *
 * R4000 128 bytes S-cache:		0x58 bytes
 * R4600 v1.7:				0x5c bytes
 * R4600 v2.0:				0x60 bytes
 * With prefetching, 16 byte strides	0xa0 bytes
 */

static unsigned int clear_page_array[0x130 / 4];

void clear_page(void * page) __attribute__((alias("clear_page_array")));

EXPORT_SYMBOL(clear_page);

/*
 * Maximum sizes:
 *
 * R4000 128 bytes S-cache:		0x11c bytes
 * R4600 v1.7:				0x080 bytes
 * R4600 v2.0:				0x07c bytes
 * With prefetching, 16 byte strides	0x0b8 bytes
 */
static unsigned int copy_page_array[0x148 / 4];

void copy_page(void *to, void *from) __attribute__((alias("copy_page_array")));

EXPORT_SYMBOL(copy_page);

/*
 * This is suboptimal for 32-bit kernels; we assume that R10000 is only used
 * with 64-bit kernels.  The prefetch offsets have been experimentally tuned
 * an Origin 200.
 */
static int pref_offset_clear __initdata = 512;
static int pref_offset_copy  __initdata = 256;

static unsigned int pref_src_mode __initdata;
static unsigned int pref_dst_mode __initdata;

static int load_offset __initdata;
static int store_offset __initdata;

static unsigned int __initdata *dest, *epc;

static unsigned int instruction_pending;
static union mips_instruction delayed_mi;

static void __init emit_instruction(union mips_instruction mi)
{
	if (instruction_pending)
		*epc++ = delayed_mi.word;

	instruction_pending = 1;
	delayed_mi = mi;
}

static inline void flush_delay_slot_or_nop(void)
{
	if (instruction_pending) {
		*epc++ = delayed_mi.word;
		instruction_pending = 0;
		return;
	}

	*epc++ = 0;
}

static inline unsigned int *label(void)
{
	if (instruction_pending) {
		*epc++ = delayed_mi.word;
		instruction_pending = 0;
	}

	return epc;
}

static inline void build_insn_word(unsigned int word)
{
	union mips_instruction mi;

	mi.word		 = word;

	emit_instruction(mi);
}

static inline void build_nop(void)
{
	build_insn_word(0);			/* nop */
}

static inline void build_src_pref(int advance)
{
	if (!(load_offset & (cpu_dcache_line_size() - 1)) && advance) {
		union mips_instruction mi;

		mi.i_format.opcode     = pref_op;
		mi.i_format.rs         = 5;		/* $a1 */
		mi.i_format.rt         = pref_src_mode;
		mi.i_format.simmediate = load_offset + advance;

		emit_instruction(mi);
	}
}

static inline void __build_load_reg(int reg)
{
	union mips_instruction mi;
	unsigned int width;

	if (cpu_has_64bit_gp_regs) {
		mi.i_format.opcode     = ld_op;
		width = 8;
	} else {
		mi.i_format.opcode     = lw_op;
		width = 4;
	}
	mi.i_format.rs         = 5;		/* $a1 */
	mi.i_format.rt         = reg;		/* $reg */
	mi.i_format.simmediate = load_offset;

	load_offset += width;
	emit_instruction(mi);
}

static inline void build_load_reg(int reg)
{
	if (cpu_has_prefetch)
		build_src_pref(pref_offset_copy);

	__build_load_reg(reg);
}

static inline void build_dst_pref(int advance)
{
	if (!(store_offset & (cpu_dcache_line_size() - 1)) && advance) {
		union mips_instruction mi;

		mi.i_format.opcode     = pref_op;
		mi.i_format.rs         = 4;		/* $a0 */
		mi.i_format.rt         = pref_dst_mode;
		mi.i_format.simmediate = store_offset + advance;

		emit_instruction(mi);
	}
}

static inline void build_cdex_s(void)
{
	union mips_instruction mi;

	if ((store_offset & (cpu_scache_line_size() - 1)))
		return;

	mi.c_format.opcode     = cache_op;
	mi.c_format.rs         = 4;		/* $a0 */
	mi.c_format.c_op       = 3;		/* Create Dirty Exclusive */
	mi.c_format.cache      = 3;		/* Secondary Data Cache */
	mi.c_format.simmediate = store_offset;

	emit_instruction(mi);
}

static inline void build_cdex_p(void)
{
	union mips_instruction mi;

	if (store_offset & (cpu_dcache_line_size() - 1))
		return;

	if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) {
		build_nop();
		build_nop();
		build_nop();
		build_nop();
	}

	if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
		build_insn_word(0x3c01a000);	/* lui     $at, 0xa000  */

	mi.c_format.opcode     = cache_op;
	mi.c_format.rs         = 4;		/* $a0 */
	mi.c_format.c_op       = 3;		/* Create Dirty Exclusive */
	mi.c_format.cache      = 1;		/* Data Cache */
	mi.c_format.simmediate = store_offset;

	emit_instruction(mi);
}

static void __init __build_store_reg(int reg)
{
	union mips_instruction mi;
	unsigned int width;

	if (cpu_has_64bit_gp_regs ||
	    (cpu_has_64bit_zero_reg && reg == 0)) {
		mi.i_format.opcode     = sd_op;
		width = 8;
	} else {
		mi.i_format.opcode     = sw_op;
		width = 4;
	}
	mi.i_format.rs         = 4;		/* $a0 */
	mi.i_format.rt         = reg;		/* $reg */
	mi.i_format.simmediate = store_offset;

	store_offset += width;
	emit_instruction(mi);
}

static inline void build_store_reg(int reg)
{
	if (cpu_has_prefetch)
		if (reg)
			build_dst_pref(pref_offset_copy);
		else
			build_dst_pref(pref_offset_clear);
	else if (cpu_has_cache_cdex_s)
		build_cdex_s();
	else if (cpu_has_cache_cdex_p)
		build_cdex_p();

	__build_store_reg(reg);
}

static inline void build_addiu_a2_a0(unsigned long offset)
{
	union mips_instruction mi;

	BUG_ON(offset > 0x7fff);

	mi.i_format.opcode     = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
	mi.i_format.rs         = 4;		/* $a0 */
	mi.i_format.rt         = 6;		/* $a2 */
	mi.i_format.simmediate = offset;

	emit_instruction(mi);
}

static inline void build_addiu_a2(unsigned long offset)
{
	union mips_instruction mi;

	BUG_ON(offset > 0x7fff);

	mi.i_format.opcode     = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
	mi.i_format.rs         = 6;		/* $a2 */
	mi.i_format.rt         = 6;		/* $a2 */
	mi.i_format.simmediate = offset;

	emit_instruction(mi);
}

static inline void build_addiu_a1(unsigned long offset)
{
	union mips_instruction mi;

	BUG_ON(offset > 0x7fff);

	mi.i_format.opcode     = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
	mi.i_format.rs         = 5;		/* $a1 */
	mi.i_format.rt         = 5;		/* $a1 */
	mi.i_format.simmediate = offset;

	load_offset -= offset;

	emit_instruction(mi);
}

static inline void build_addiu_a0(unsigned long offset)
{
	union mips_instruction mi;

	BUG_ON(offset > 0x7fff);

	mi.i_format.opcode     = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
	mi.i_format.rs         = 4;		/* $a0 */
	mi.i_format.rt         = 4;		/* $a0 */
	mi.i_format.simmediate = offset;

	store_offset -= offset;

	emit_instruction(mi);
}

static inline void build_bne(unsigned int *dest)
{
	union mips_instruction mi;

	mi.i_format.opcode = bne_op;
	mi.i_format.rs     = 6;			/* $a2 */
	mi.i_format.rt     = 4;			/* $a0 */
	mi.i_format.simmediate = dest - epc - 1;

	*epc++ = mi.word;
	flush_delay_slot_or_nop();
}

static inline void build_jr_ra(void)
{
	union mips_instruction mi;

	mi.r_format.opcode = spec_op;
	mi.r_format.rs     = 31;
	mi.r_format.rt     = 0;
	mi.r_format.rd     = 0;
	mi.r_format.re     = 0;
	mi.r_format.func   = jr_op;

	*epc++ = mi.word;
	flush_delay_slot_or_nop();
}

void __init build_clear_page(void)
{
	unsigned int loop_start;
	unsigned long off;

	epc = (unsigned int *) &clear_page_array;
	instruction_pending = 0;
	store_offset = 0;

	if (cpu_has_prefetch) {
		switch (current_cpu_data.cputype) {
		case CPU_TX49XX:
			/* TX49 supports only Pref_Load */
			pref_offset_clear = 0;
			pref_offset_copy = 0;
			break;

		case CPU_RM9000:
			/*
			 * As a workaround for erratum G105 which make the
			 * PrepareForStore hint unusable we fall back to
			 * StoreRetained on the RM9000.  Once it is known which
			 * versions of the RM9000 we'll be able to condition-
			 * alize this.
			 */

		case CPU_R10000:
		case CPU_R12000:
		case CPU_R14000:
			pref_src_mode = Pref_LoadStreamed;
			pref_dst_mode = Pref_StoreStreamed;
			break;

		default:
			pref_src_mode = Pref_LoadStreamed;
			pref_dst_mode = Pref_PrepareForStore;
			break;
		}
	}

        off = PAGE_SIZE - (cpu_has_prefetch ? pref_offset_clear : 0);
	if (off > 0x7fff) {
		build_addiu_a2_a0(off >> 1);
		build_addiu_a2(off >> 1);
	} else
		build_addiu_a2_a0(off);

	if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
		build_insn_word(0x3c01a000);	/* lui     $at, 0xa000  */

dest = label();
	do {
		build_store_reg(0);
		build_store_reg(0);
		build_store_reg(0);
		build_store_reg(0);
	} while (store_offset < half_scache_line_size());
	build_addiu_a0(2 * store_offset);
	loop_start = store_offset;
	do {
		build_store_reg(0);
		build_store_reg(0);
		build_store_reg(0);
		build_store_reg(0);
	} while ((store_offset - loop_start) < half_scache_line_size());
	build_bne(dest);

	if (cpu_has_prefetch && pref_offset_clear) {
		build_addiu_a2_a0(pref_offset_clear);
	dest = label();
		loop_start = store_offset;
		do {
			__build_store_reg(0);
			__build_store_reg(0);
			__build_store_reg(0);
			__build_store_reg(0);
		} while ((store_offset - loop_start) < half_scache_line_size());
		build_addiu_a0(2 * store_offset);
		loop_start = store_offset;
		do {
			__build_store_reg(0);
			__build_store_reg(0);
			__build_store_reg(0);
			__build_store_reg(0);
		} while ((store_offset - loop_start) < half_scache_line_size());
		build_bne(dest);
	}

	build_jr_ra();

	BUG_ON(epc > clear_page_array + ARRAY_SIZE(clear_page_array));
}

void __init build_copy_page(void)
{
	unsigned int loop_start;
	unsigned long off;

	epc = (unsigned int *) &copy_page_array;
	store_offset = load_offset = 0;
	instruction_pending = 0;

	off = PAGE_SIZE - (cpu_has_prefetch ? pref_offset_copy : 0);
	if (off > 0x7fff) {
		build_addiu_a2_a0(off >> 1);
		build_addiu_a2(off >> 1);
	} else
		build_addiu_a2_a0(off);

	if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
		build_insn_word(0x3c01a000);	/* lui     $at, 0xa000  */

dest = label();
	loop_start = store_offset;
	do {
		build_load_reg( 8);
		build_load_reg( 9);
		build_load_reg(10);
		build_load_reg(11);
		build_store_reg( 8);
		build_store_reg( 9);
		build_store_reg(10);
		build_store_reg(11);
	} while ((store_offset - loop_start) < half_scache_line_size());
	build_addiu_a0(2 * store_offset);
	build_addiu_a1(2 * load_offset);
	loop_start = store_offset;
	do {
		build_load_reg( 8);
		build_load_reg( 9);
		build_load_reg(10);
		build_load_reg(11);
		build_store_reg( 8);
		build_store_reg( 9);
		build_store_reg(10);
		build_store_reg(11);
	} while ((store_offset - loop_start) < half_scache_line_size());
	build_bne(dest);

	if (cpu_has_prefetch && pref_offset_copy) {
		build_addiu_a2_a0(pref_offset_copy);
	dest = label();
		loop_start = store_offset;
		do {
			__build_load_reg( 8);
			__build_load_reg( 9);
			__build_load_reg(10);
			__build_load_reg(11);
			__build_store_reg( 8);
			__build_store_reg( 9);
			__build_store_reg(10);
			__build_store_reg(11);
		} while ((store_offset - loop_start) < half_scache_line_size());
		build_addiu_a0(2 * store_offset);
		build_addiu_a1(2 * load_offset);
		loop_start = store_offset;
		do {
			__build_load_reg( 8);
			__build_load_reg( 9);
			__build_load_reg(10);
			__build_load_reg(11);
			__build_store_reg( 8);
			__build_store_reg( 9);
			__build_store_reg(10);
			__build_store_reg(11);
		} while ((store_offset - loop_start) < half_scache_line_size());
		build_bne(dest);
	}

	build_jr_ra();

	BUG_ON(epc > copy_page_array + ARRAY_SIZE(copy_page_array));
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* This file is subject to the terms and conditions of the GNU General Public
	3	* License. See the file "COPYING" in the main directory of this archive
	4	* for more details.
	5	*
	6	* Copyright (C) 2003, 04, 05 Ralf Baechle (ralf@linux-mips.org)
	7	*/
	8	#include <linux/init.h>
	9	#include <linux/kernel.h>
	10	#include <linux/sched.h>
	11	#include <linux/mm.h>
	12	#include <linux/module.h>
	13	#include <linux/proc_fs.h>
	14
	15	#include <asm/cacheops.h>
	16	#include <asm/inst.h>
	17	#include <asm/io.h>
	18	#include <asm/page.h>
	19	#include <asm/pgtable.h>
	20	#include <asm/prefetch.h>
	21	#include <asm/system.h>
	22	#include <asm/bootinfo.h>
	23	#include <asm/mipsregs.h>
	24	#include <asm/mmu_context.h>
	25	#include <asm/cpu.h>
	26	#include <asm/war.h>
	27
330cfe01 TS	28	#define half_scache_line_size() (cpu_scache_line_size() >> 1)
	29	#define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010)
	30	#define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020)
	31
1da177e4 LT	32
	33	/*
	34	* Maximum sizes:
	35	*
	36	* R4000 128 bytes S-cache: 0x58 bytes
	37	* R4600 v1.7: 0x5c bytes
	38	* R4600 v2.0: 0x60 bytes
	39	* With prefetching, 16 byte strides 0xa0 bytes
	40	*/
	41
	42	static unsigned int clear_page_array[0x130 / 4];
	43
	44	void clear_page(void * page) __attribute__((alias("clear_page_array")));
	45
	46	EXPORT_SYMBOL(clear_page);
	47
	48	/*
	49	* Maximum sizes:
	50	*
	51	* R4000 128 bytes S-cache: 0x11c bytes
	52	* R4600 v1.7: 0x080 bytes
	53	* R4600 v2.0: 0x07c bytes
	54	* With prefetching, 16 byte strides 0x0b8 bytes
	55	*/
	56	static unsigned int copy_page_array[0x148 / 4];
	57
	58	void copy_page(void to, void from) __attribute__((alias("copy_page_array")));
	59
	60	EXPORT_SYMBOL(copy_page);
	61
	62	/*
	63	* This is suboptimal for 32-bit kernels; we assume that R10000 is only used
	64	* with 64-bit kernels. The prefetch offsets have been experimentally tuned
	65	* an Origin 200.
	66	*/
	67	static int pref_offset_clear __initdata = 512;
	68	static int pref_offset_copy __initdata = 256;
	69
	70	static unsigned int pref_src_mode __initdata;
	71	static unsigned int pref_dst_mode __initdata;
	72
	73	static int load_offset __initdata;
	74	static int store_offset __initdata;
	75
	76	static unsigned int __initdata dest, epc;
	77
	78	static unsigned int instruction_pending;
	79	static union mips_instruction delayed_mi;
	80
	81	static void __init emit_instruction(union mips_instruction mi)
	82	{
	83	if (instruction_pending)
	84	*epc++ = delayed_mi.word;
	85
	86	instruction_pending = 1;
	87	delayed_mi = mi;
	88	}
	89
	90	static inline void flush_delay_slot_or_nop(void)
	91	{
	92	if (instruction_pending) {
	93	*epc++ = delayed_mi.word;
	94	instruction_pending = 0;
	95	return;
96	}
97
98	*epc++ = 0;
99	}
100
101	static inline unsigned int *label(void)
102	{
103	if (instruction_pending) {
104	*epc++ = delayed_mi.word;
105	instruction_pending = 0;
106	}
107
108	return epc;
109	}
110
111	static inline void build_insn_word(unsigned int word)
112	{
113	union mips_instruction mi;
114
115	mi.word = word;
116
117	emit_instruction(mi);
118	}
119
120	static inline void build_nop(void)
121	{
122	build_insn_word(0); /* nop */
123	}
124
125	static inline void build_src_pref(int advance)
126	{
de862b48	127	if (!(load_offset & (cpu_dcache_line_size() - 1)) && advance) {
1da177e4 LT	128	union mips_instruction mi;
	129
	130	mi.i_format.opcode = pref_op;
	131	mi.i_format.rs = 5; /* $a1 */
	132	mi.i_format.rt = pref_src_mode;
	133	mi.i_format.simmediate = load_offset + advance;
	134
	135	emit_instruction(mi);
	136	}
	137	}
	138
	139	static inline void __build_load_reg(int reg)
	140	{
	141	union mips_instruction mi;
	142	unsigned int width;
	143
	144	if (cpu_has_64bit_gp_regs) {
	145	mi.i_format.opcode = ld_op;
	146	width = 8;
	147	} else {
	148	mi.i_format.opcode = lw_op;
	149	width = 4;
	150	}
	151	mi.i_format.rs = 5; /* $a1 */
	152	mi.i_format.rt = reg; /* $reg */
	153	mi.i_format.simmediate = load_offset;
	154
	155	load_offset += width;
	156	emit_instruction(mi);
	157	}
	158
	159	static inline void build_load_reg(int reg)
	160	{
	161	if (cpu_has_prefetch)
	162	build_src_pref(pref_offset_copy);
	163
	164	__build_load_reg(reg);
	165	}
	166
	167	static inline void build_dst_pref(int advance)
	168	{
de862b48	169	if (!(store_offset & (cpu_dcache_line_size() - 1)) && advance) {
1da177e4 LT	170	union mips_instruction mi;
	171
	172	mi.i_format.opcode = pref_op;
	173	mi.i_format.rs = 4; /* $a0 */
	174	mi.i_format.rt = pref_dst_mode;
	175	mi.i_format.simmediate = store_offset + advance;
	176
	177	emit_instruction(mi);
	178	}
	179	}
	180
	181	static inline void build_cdex_s(void)
	182	{
	183	union mips_instruction mi;
	184
	185	if ((store_offset & (cpu_scache_line_size() - 1)))
	186	return;
	187
	188	mi.c_format.opcode = cache_op;
	189	mi.c_format.rs = 4; /* $a0 */
	190	mi.c_format.c_op = 3; /* Create Dirty Exclusive */
	191	mi.c_format.cache = 3; /* Secondary Data Cache */
	192	mi.c_format.simmediate = store_offset;
	193
	194	emit_instruction(mi);
	195	}
	196
	197	static inline void build_cdex_p(void)
	198	{
	199	union mips_instruction mi;
	200
	201	if (store_offset & (cpu_dcache_line_size() - 1))
	202	return;
	203
330cfe01	204	if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) {
1da177e4 LT	205	build_nop();
	206	build_nop();
	207	build_nop();
	208	build_nop();
	209	}
	210
330cfe01	211	if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
10a3dabd	212	build_insn_word(0x3c01a000); /* lui $at, 0xa000 */
1da177e4 LT	213
	214	mi.c_format.opcode = cache_op;
	215	mi.c_format.rs = 4; /* $a0 */
	216	mi.c_format.c_op = 3; /* Create Dirty Exclusive */
	217	mi.c_format.cache = 1; /* Data Cache */
	218	mi.c_format.simmediate = store_offset;
	219
	220	emit_instruction(mi);
	221	}
	222
	223	static void __init __build_store_reg(int reg)
	224	{
	225	union mips_instruction mi;
	226	unsigned int width;
	227
	228	if (cpu_has_64bit_gp_regs \|\|
	229	(cpu_has_64bit_zero_reg && reg == 0)) {
	230	mi.i_format.opcode = sd_op;
	231	width = 8;
	232	} else {
	233	mi.i_format.opcode = sw_op;
	234	width = 4;
	235	}
	236	mi.i_format.rs = 4; /* $a0 */
	237	mi.i_format.rt = reg; /* $reg */
	238	mi.i_format.simmediate = store_offset;
	239
	240	store_offset += width;
	241	emit_instruction(mi);
	242	}
	243
	244	static inline void build_store_reg(int reg)
	245	{
	246	if (cpu_has_prefetch)
	247	if (reg)
	248	build_dst_pref(pref_offset_copy);
	249	else
	250	build_dst_pref(pref_offset_clear);
	251	else if (cpu_has_cache_cdex_s)
	252	build_cdex_s();
	253	else if (cpu_has_cache_cdex_p)
	254	build_cdex_p();
	255
	256	__build_store_reg(reg);
	257	}
	258
	259	static inline void build_addiu_a2_a0(unsigned long offset)
	260	{
	261	union mips_instruction mi;
	262
	263	BUG_ON(offset > 0x7fff);
	264
	265	mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
	266	mi.i_format.rs = 4; /* $a0 */
	267	mi.i_format.rt = 6; /* $a2 */
	268	mi.i_format.simmediate = offset;
	269
	270	emit_instruction(mi);
	271	}
	272
242954b5 RB	273	static inline void build_addiu_a2(unsigned long offset)
	274	{
	275	union mips_instruction mi;
	276
	277	BUG_ON(offset > 0x7fff);
	278
	279	mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
	280	mi.i_format.rs = 6; /* $a2 */
	281	mi.i_format.rt = 6; /* $a2 */
	282	mi.i_format.simmediate = offset;
	283
	284	emit_instruction(mi);
	285	}
	286
1da177e4 LT	287	static inline void build_addiu_a1(unsigned long offset)
	288	{
	289	union mips_instruction mi;
	290
	291	BUG_ON(offset > 0x7fff);
	292
	293	mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
	294	mi.i_format.rs = 5; /* $a1 */
	295	mi.i_format.rt = 5; /* $a1 */
	296	mi.i_format.simmediate = offset;
	297
	298	load_offset -= offset;
	299
	300	emit_instruction(mi);
	301	}
	302
	303	static inline void build_addiu_a0(unsigned long offset)
	304	{
	305	union mips_instruction mi;
	306
	307	BUG_ON(offset > 0x7fff);
	308
	309	mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
	310	mi.i_format.rs = 4; /* $a0 */
	311	mi.i_format.rt = 4; /* $a0 */
	312	mi.i_format.simmediate = offset;
	313
	314	store_offset -= offset;
	315
	316	emit_instruction(mi);
	317	}
	318
	319	static inline void build_bne(unsigned int *dest)
	320	{
	321	union mips_instruction mi;
	322
	323	mi.i_format.opcode = bne_op;
	324	mi.i_format.rs = 6; /* $a2 */
	325	mi.i_format.rt = 4; /* $a0 */
	326	mi.i_format.simmediate = dest - epc - 1;
	327
	328	*epc++ = mi.word;
	329	flush_delay_slot_or_nop();
	330	}
	331
	332	static inline void build_jr_ra(void)
	333	{
	334	union mips_instruction mi;
	335
	336	mi.r_format.opcode = spec_op;
	337	mi.r_format.rs = 31;
	338	mi.r_format.rt = 0;
	339	mi.r_format.rd = 0;
	340	mi.r_format.re = 0;
	341	mi.r_format.func = jr_op;
	342
	343	*epc++ = mi.word;
	344	flush_delay_slot_or_nop();
	345	}
	346
	347	void __init build_clear_page(void)
	348	{
	349	unsigned int loop_start;
242954b5	350	unsigned long off;
1da177e4 LT	351
	352	epc = (unsigned int *) &clear_page_array;
	353	instruction_pending = 0;
	354	store_offset = 0;
	355
	356	if (cpu_has_prefetch) {
	357	switch (current_cpu_data.cputype) {
de862b48 AN	358	case CPU_TX49XX:
	359	/* TX49 supports only Pref_Load */
	360	pref_offset_clear = 0;
	361	pref_offset_copy = 0;
	362	break;
	363
1da177e4 LT	364	case CPU_RM9000:
	365	/*
	366	* As a workaround for erratum G105 which make the
	367	* PrepareForStore hint unusable we fall back to
	368	* StoreRetained on the RM9000. Once it is known which
	369	* versions of the RM9000 we'll be able to condition-
	370	* alize this.
	371	*/
	372
	373	case CPU_R10000:
	374	case CPU_R12000:
44d921b2	375	case CPU_R14000:
1da177e4 LT	376	pref_src_mode = Pref_LoadStreamed;
	377	pref_dst_mode = Pref_StoreStreamed;
	378	break;
	379
	380	default:
	381	pref_src_mode = Pref_LoadStreamed;
	382	pref_dst_mode = Pref_PrepareForStore;
	383	break;
	384	}
	385	}
	386
242954b5 RB	387	off = PAGE_SIZE - (cpu_has_prefetch ? pref_offset_clear : 0);
	388	if (off > 0x7fff) {
	389	build_addiu_a2_a0(off >> 1);
	390	build_addiu_a2(off >> 1);
	391	} else
	392	build_addiu_a2_a0(off);
1da177e4	393
330cfe01	394	if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
1da177e4 LT	395	build_insn_word(0x3c01a000); /* lui $at, 0xa000 */
	396
	397	dest = label();
	398	do {
	399	build_store_reg(0);
	400	build_store_reg(0);
	401	build_store_reg(0);
	402	build_store_reg(0);
	403	} while (store_offset < half_scache_line_size());
	404	build_addiu_a0(2 * store_offset);
	405	loop_start = store_offset;
	406	do {
	407	build_store_reg(0);
	408	build_store_reg(0);
	409	build_store_reg(0);
	410	build_store_reg(0);
	411	} while ((store_offset - loop_start) < half_scache_line_size());
	412	build_bne(dest);
	413
	414	if (cpu_has_prefetch && pref_offset_clear) {
	415	build_addiu_a2_a0(pref_offset_clear);
	416	dest = label();
	417	loop_start = store_offset;
	418	do {
	419	__build_store_reg(0);
	420	__build_store_reg(0);
	421	__build_store_reg(0);
	422	__build_store_reg(0);
	423	} while ((store_offset - loop_start) < half_scache_line_size());
	424	build_addiu_a0(2 * store_offset);
	425	loop_start = store_offset;
	426	do {
	427	__build_store_reg(0);
	428	__build_store_reg(0);
	429	__build_store_reg(0);
	430	__build_store_reg(0);
	431	} while ((store_offset - loop_start) < half_scache_line_size());
	432	build_bne(dest);
	433	}
	434
	435	build_jr_ra();
	436
1da177e4 LT	437	BUG_ON(epc > clear_page_array + ARRAY_SIZE(clear_page_array));
	438	}
	439
	440	void __init build_copy_page(void)
	441	{
	442	unsigned int loop_start;
242954b5	443	unsigned long off;
1da177e4 LT	444
	445	epc = (unsigned int *) &copy_page_array;
	446	store_offset = load_offset = 0;
	447	instruction_pending = 0;
	448
242954b5 RB	449	off = PAGE_SIZE - (cpu_has_prefetch ? pref_offset_copy : 0);
	450	if (off > 0x7fff) {
	451	build_addiu_a2_a0(off >> 1);
	452	build_addiu_a2(off >> 1);
	453	} else
	454	build_addiu_a2_a0(off);
1da177e4	455
330cfe01	456	if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
1da177e4 LT	457	build_insn_word(0x3c01a000); /* lui $at, 0xa000 */
	458
	459	dest = label();
	460	loop_start = store_offset;
	461	do {
	462	build_load_reg( 8);
	463	build_load_reg( 9);
	464	build_load_reg(10);
	465	build_load_reg(11);
	466	build_store_reg( 8);
	467	build_store_reg( 9);
	468	build_store_reg(10);
	469	build_store_reg(11);
	470	} while ((store_offset - loop_start) < half_scache_line_size());
	471	build_addiu_a0(2 * store_offset);
	472	build_addiu_a1(2 * load_offset);
	473	loop_start = store_offset;
	474	do {
	475	build_load_reg( 8);
	476	build_load_reg( 9);
	477	build_load_reg(10);
	478	build_load_reg(11);
	479	build_store_reg( 8);
	480	build_store_reg( 9);
	481	build_store_reg(10);
	482	build_store_reg(11);
	483	} while ((store_offset - loop_start) < half_scache_line_size());
	484	build_bne(dest);
	485
	486	if (cpu_has_prefetch && pref_offset_copy) {
	487	build_addiu_a2_a0(pref_offset_copy);
	488	dest = label();
	489	loop_start = store_offset;
	490	do {
	491	__build_load_reg( 8);
	492	__build_load_reg( 9);
	493	__build_load_reg(10);
	494	__build_load_reg(11);
	495	__build_store_reg( 8);
	496	__build_store_reg( 9);
	497	__build_store_reg(10);
	498	__build_store_reg(11);
	499	} while ((store_offset - loop_start) < half_scache_line_size());
	500	build_addiu_a0(2 * store_offset);
	501	build_addiu_a1(2 * load_offset);
	502	loop_start = store_offset;
	503	do {
	504	__build_load_reg( 8);
	505	__build_load_reg( 9);
	506	__build_load_reg(10);
	507	__build_load_reg(11);
	508	__build_store_reg( 8);
	509	__build_store_reg( 9);
	510	__build_store_reg(10);
	511	__build_store_reg(11);
	512	} while ((store_offset - loop_start) < half_scache_line_size());
	513	build_bne(dest);
	514	}
	515
	516	build_jr_ra();
	517
1da177e4 LT	518	BUG_ON(epc > copy_page_array + ARRAY_SIZE(copy_page_array));
1da177e4 LT	519	}