[mirror_ubuntu-artful-kernel.git] / arch / mips / kernel / unaligned.c

/*
 * Handle unaligned accesses by emulation.
 *
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle
 * Copyright (C) 1999 Silicon Graphics, Inc.
 *
 * This file contains exception handler for address error exception with the
 * special capability to execute faulting instructions in software.  The
 * handler does not try to handle the case when the program counter points
 * to an address not aligned to a word boundary.
 *
 * Putting data to unaligned addresses is a bad practice even on Intel where
 * only the performance is affected.  Much worse is that such code is non-
 * portable.  Due to several programs that die on MIPS due to alignment
 * problems I decided to implement this handler anyway though I originally
 * didn't intend to do this at all for user code.
 *
 * For now I enable fixing of address errors by default to make life easier.
 * I however intend to disable this somewhen in the future when the alignment
 * problems with user programs have been fixed.  For programmers this is the
 * right way to go.
 *
 * Fixing address errors is a per process option.  The option is inherited
 * across fork(2) and execve(2) calls.  If you really want to use the
 * option in your user programs - I discourage the use of the software
 * emulation strongly - use the following code in your userland stuff:
 *
 * #include <sys/sysmips.h>
 *
 * ...
 * sysmips(MIPS_FIXADE, x);
 * ...
 *
 * The argument x is 0 for disabling software emulation, enabled otherwise.
 *
 * Below a little program to play around with this feature.
 *
 * #include <stdio.h>
 * #include <sys/sysmips.h>
 *
 * struct foo {
 *         unsigned char bar[8];
 * };
 *
 * main(int argc, char *argv[])
 * {
 *         struct foo x = {0, 1, 2, 3, 4, 5, 6, 7};
 *         unsigned int *p = (unsigned int *) (x.bar + 3);
 *         int i;
 *
 *         if (argc > 1)
 *                 sysmips(MIPS_FIXADE, atoi(argv[1]));
 *
 *         printf("*p = %08lx\n", *p);
 *
 *         *p = 0xdeadface;
 *
 *         for(i = 0; i <= 7; i++)
 *         printf("%02x ", x.bar[i]);
 *         printf("\n");
 * }
 *
 * Coprocessor loads are not supported; I think this case is unimportant
 * in the practice.
 *
 * TODO: Handle ndc (attempted store to doubleword in uncached memory)
 *       exception for the R6000.
 *       A store crossing a page boundary might be executed only partially.
 *       Undo the partial store in this case.
 */
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/signal.h>
#include <linux/smp.h>
#include <linux/sched.h>
#include <linux/debugfs.h>
#include <linux/perf_event.h>

#include <asm/asm.h>
#include <asm/branch.h>
#include <asm/byteorder.h>
#include <asm/cop2.h>
#include <asm/inst.h>
#include <asm/uaccess.h>
#include <asm/system.h>

#define STR(x)  __STR(x)
#define __STR(x)  #x

enum {
	UNALIGNED_ACTION_QUIET,
	UNALIGNED_ACTION_SIGNAL,
	UNALIGNED_ACTION_SHOW,
};
#ifdef CONFIG_DEBUG_FS
static u32 unaligned_instructions;
static u32 unaligned_action;
#else
#define unaligned_action UNALIGNED_ACTION_QUIET
#endif
extern void show_registers(struct pt_regs *regs);

static void emulate_load_store_insn(struct pt_regs *regs,
	void __user *addr, unsigned int __user *pc)
{
	union mips_instruction insn;
	unsigned long value;
	unsigned int res;

	perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);

	/*
	 * This load never faults.
	 */
	__get_user(insn.word, pc);

	switch (insn.i_format.opcode) {
	/*
	 * These are instructions that a compiler doesn't generate.  We
	 * can assume therefore that the code is MIPS-aware and
	 * really buggy.  Emulating these instructions would break the
	 * semantics anyway.
	 */
	case ll_op:
	case lld_op:
	case sc_op:
	case scd_op:

	/*
	 * For these instructions the only way to create an address
	 * error is an attempted access to kernel/supervisor address
	 * space.
	 */
	case ldl_op:
	case ldr_op:
	case lwl_op:
	case lwr_op:
	case sdl_op:
	case sdr_op:
	case swl_op:
	case swr_op:
	case lb_op:
	case lbu_op:
	case sb_op:
		goto sigbus;

	/*
	 * The remaining opcodes are the ones that are really of interest.
	 */
	case lh_op:
		if (!access_ok(VERIFY_READ, addr, 2))
			goto sigbus;

		__asm__ __volatile__ (".set\tnoat\n"
#ifdef __BIG_ENDIAN
			"1:\tlb\t%0, 0(%2)\n"
			"2:\tlbu\t$1, 1(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tlb\t%0, 1(%2)\n"
			"2:\tlbu\t$1, 0(%2)\n\t"
#endif
			"sll\t%0, 0x8\n\t"
			"or\t%0, $1\n\t"
			"li\t%1, 0\n"
			"3:\t.set\tat\n\t"
			".section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%1, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=&r" (value), "=r" (res)
			: "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		regs->regs[insn.i_format.rt] = value;
		break;

	case lw_op:
		if (!access_ok(VERIFY_READ, addr, 4))
			goto sigbus;

		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			"1:\tlwl\t%0, (%2)\n"
			"2:\tlwr\t%0, 3(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tlwl\t%0, 3(%2)\n"
			"2:\tlwr\t%0, (%2)\n\t"
#endif
			"li\t%1, 0\n"
			"3:\t.section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%1, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=&r" (value), "=r" (res)
			: "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		regs->regs[insn.i_format.rt] = value;
		break;

	case lhu_op:
		if (!access_ok(VERIFY_READ, addr, 2))
			goto sigbus;

		__asm__ __volatile__ (
			".set\tnoat\n"
#ifdef __BIG_ENDIAN
			"1:\tlbu\t%0, 0(%2)\n"
			"2:\tlbu\t$1, 1(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tlbu\t%0, 1(%2)\n"
			"2:\tlbu\t$1, 0(%2)\n\t"
#endif
			"sll\t%0, 0x8\n\t"
			"or\t%0, $1\n\t"
			"li\t%1, 0\n"
			"3:\t.set\tat\n\t"
			".section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%1, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=&r" (value), "=r" (res)
			: "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		regs->regs[insn.i_format.rt] = value;
		break;

	case lwu_op:
#ifdef CONFIG_64BIT
		/*
		 * A 32-bit kernel might be running on a 64-bit processor.  But
		 * if we're on a 32-bit processor and an i-cache incoherency
		 * or race makes us see a 64-bit instruction here the sdl/sdr
		 * would blow up, so for now we don't handle unaligned 64-bit
		 * instructions on 32-bit kernels.
		 */
		if (!access_ok(VERIFY_READ, addr, 4))
			goto sigbus;

		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			"1:\tlwl\t%0, (%2)\n"
			"2:\tlwr\t%0, 3(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tlwl\t%0, 3(%2)\n"
			"2:\tlwr\t%0, (%2)\n\t"
#endif
			"dsll\t%0, %0, 32\n\t"
			"dsrl\t%0, %0, 32\n\t"
			"li\t%1, 0\n"
			"3:\t.section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%1, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=&r" (value), "=r" (res)
			: "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		regs->regs[insn.i_format.rt] = value;
		break;
#endif /* CONFIG_64BIT */

		/* Cannot handle 64-bit instructions in 32-bit kernel */
		goto sigill;

	case ld_op:
#ifdef CONFIG_64BIT
		/*
		 * A 32-bit kernel might be running on a 64-bit processor.  But
		 * if we're on a 32-bit processor and an i-cache incoherency
		 * or race makes us see a 64-bit instruction here the sdl/sdr
		 * would blow up, so for now we don't handle unaligned 64-bit
		 * instructions on 32-bit kernels.
		 */
		if (!access_ok(VERIFY_READ, addr, 8))
			goto sigbus;

		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			"1:\tldl\t%0, (%2)\n"
			"2:\tldr\t%0, 7(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tldl\t%0, 7(%2)\n"
			"2:\tldr\t%0, (%2)\n\t"
#endif
			"li\t%1, 0\n"
			"3:\t.section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%1, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=&r" (value), "=r" (res)
			: "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		regs->regs[insn.i_format.rt] = value;
		break;
#endif /* CONFIG_64BIT */

		/* Cannot handle 64-bit instructions in 32-bit kernel */
		goto sigill;

	case sh_op:
		if (!access_ok(VERIFY_WRITE, addr, 2))
			goto sigbus;

		value = regs->regs[insn.i_format.rt];
		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			".set\tnoat\n"
			"1:\tsb\t%1, 1(%2)\n\t"
			"srl\t$1, %1, 0x8\n"
			"2:\tsb\t$1, 0(%2)\n\t"
			".set\tat\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			".set\tnoat\n"
			"1:\tsb\t%1, 0(%2)\n\t"
			"srl\t$1,%1, 0x8\n"
			"2:\tsb\t$1, 1(%2)\n\t"
			".set\tat\n\t"
#endif
			"li\t%0, 0\n"
			"3:\n\t"
			".section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%0, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
			: "=r" (res)
			: "r" (value), "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		break;

	case sw_op:
		if (!access_ok(VERIFY_WRITE, addr, 4))
			goto sigbus;

		value = regs->regs[insn.i_format.rt];
		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			"1:\tswl\t%1,(%2)\n"
			"2:\tswr\t%1, 3(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tswl\t%1, 3(%2)\n"
			"2:\tswr\t%1, (%2)\n\t"
#endif
			"li\t%0, 0\n"
			"3:\n\t"
			".section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%0, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
		: "=r" (res)
		: "r" (value), "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		break;

	case sd_op:
#ifdef CONFIG_64BIT
		/*
		 * A 32-bit kernel might be running on a 64-bit processor.  But
		 * if we're on a 32-bit processor and an i-cache incoherency
		 * or race makes us see a 64-bit instruction here the sdl/sdr
		 * would blow up, so for now we don't handle unaligned 64-bit
		 * instructions on 32-bit kernels.
		 */
		if (!access_ok(VERIFY_WRITE, addr, 8))
			goto sigbus;

		value = regs->regs[insn.i_format.rt];
		__asm__ __volatile__ (
#ifdef __BIG_ENDIAN
			"1:\tsdl\t%1,(%2)\n"
			"2:\tsdr\t%1, 7(%2)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
			"1:\tsdl\t%1, 7(%2)\n"
			"2:\tsdr\t%1, (%2)\n\t"
#endif
			"li\t%0, 0\n"
			"3:\n\t"
			".section\t.fixup,\"ax\"\n\t"
			"4:\tli\t%0, %3\n\t"
			"j\t3b\n\t"
			".previous\n\t"
			".section\t__ex_table,\"a\"\n\t"
			STR(PTR)"\t1b, 4b\n\t"
			STR(PTR)"\t2b, 4b\n\t"
			".previous"
		: "=r" (res)
		: "r" (value), "r" (addr), "i" (-EFAULT));
		if (res)
			goto fault;
		compute_return_epc(regs);
		break;
#endif /* CONFIG_64BIT */

		/* Cannot handle 64-bit instructions in 32-bit kernel */
		goto sigill;

	case lwc1_op:
	case ldc1_op:
	case swc1_op:
	case sdc1_op:
		/*
		 * I herewith declare: this does not happen.  So send SIGBUS.
		 */
		goto sigbus;

	/*
	 * COP2 is available to implementor for application specific use.
	 * It's up to applications to register a notifier chain and do
	 * whatever they have to do, including possible sending of signals.
	 */
	case lwc2_op:
		cu2_notifier_call_chain(CU2_LWC2_OP, regs);
		break;

	case ldc2_op:
		cu2_notifier_call_chain(CU2_LDC2_OP, regs);
		break;

	case swc2_op:
		cu2_notifier_call_chain(CU2_SWC2_OP, regs);
		break;

	case sdc2_op:
		cu2_notifier_call_chain(CU2_SDC2_OP, regs);
		break;

	default:
		/*
		 * Pheeee...  We encountered an yet unknown instruction or
		 * cache coherence problem.  Die sucker, die ...
		 */
		goto sigill;
	}

#ifdef CONFIG_DEBUG_FS
	unaligned_instructions++;
#endif

	return;

fault:
	/* Did we have an exception handler installed? */
	if (fixup_exception(regs))
		return;

	die_if_kernel("Unhandled kernel unaligned access", regs);
	force_sig(SIGSEGV, current);

	return;

sigbus:
	die_if_kernel("Unhandled kernel unaligned access", regs);
	force_sig(SIGBUS, current);

	return;

sigill:
	die_if_kernel("Unhandled kernel unaligned access or invalid instruction", regs);
	force_sig(SIGILL, current);
}

asmlinkage void do_ade(struct pt_regs *regs)
{
	unsigned int __user *pc;
	mm_segment_t seg;

	perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS,
			1, regs, regs->cp0_badvaddr);
	/*
	 * Did we catch a fault trying to load an instruction?
	 * Or are we running in MIPS16 mode?
	 */
	if ((regs->cp0_badvaddr == regs->cp0_epc) || (regs->cp0_epc & 0x1))
		goto sigbus;

	pc = (unsigned int __user *) exception_epc(regs);
	if (user_mode(regs) && !test_thread_flag(TIF_FIXADE))
		goto sigbus;
	if (unaligned_action == UNALIGNED_ACTION_SIGNAL)
		goto sigbus;
	else if (unaligned_action == UNALIGNED_ACTION_SHOW)
		show_registers(regs);

	/*
	 * Do branch emulation only if we didn't forward the exception.
	 * This is all so but ugly ...
	 */
	seg = get_fs();
	if (!user_mode(regs))
		set_fs(KERNEL_DS);
	emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc);
	set_fs(seg);

	return;

sigbus:
	die_if_kernel("Kernel unaligned instruction access", regs);
	force_sig(SIGBUS, current);

	/*
	 * XXX On return from the signal handler we should advance the epc
	 */
}

#ifdef CONFIG_DEBUG_FS
extern struct dentry *mips_debugfs_dir;
static int __init debugfs_unaligned(void)
{
	struct dentry *d;

	if (!mips_debugfs_dir)
		return -ENODEV;
	d = debugfs_create_u32("unaligned_instructions", S_IRUGO,
			       mips_debugfs_dir, &unaligned_instructions);
	if (!d)
		return -ENOMEM;
	d = debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR,
			       mips_debugfs_dir, &unaligned_action);
	if (!d)
		return -ENOMEM;
	return 0;
}
__initcall(debugfs_unaligned);
#endif
Commit	Line	Data
1da177e4 LT	1	/*
	2	* Handle unaligned accesses by emulation.
	3	*
	4	* This file is subject to the terms and conditions of the GNU General Public
	5	* License. See the file "COPYING" in the main directory of this archive
	6	* for more details.
	7	*
	8	* Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle
	9	* Copyright (C) 1999 Silicon Graphics, Inc.
	10	*
	11	* This file contains exception handler for address error exception with the
	12	* special capability to execute faulting instructions in software. The
	13	* handler does not try to handle the case when the program counter points
	14	* to an address not aligned to a word boundary.
	15	*
	16	* Putting data to unaligned addresses is a bad practice even on Intel where
	17	* only the performance is affected. Much worse is that such code is non-
	18	* portable. Due to several programs that die on MIPS due to alignment
	19	* problems I decided to implement this handler anyway though I originally
	20	* didn't intend to do this at all for user code.
	21	*
	22	* For now I enable fixing of address errors by default to make life easier.
	23	* I however intend to disable this somewhen in the future when the alignment
	24	* problems with user programs have been fixed. For programmers this is the
	25	* right way to go.
	26	*
	27	* Fixing address errors is a per process option. The option is inherited
	28	* across fork(2) and execve(2) calls. If you really want to use the
	29	* option in your user programs - I discourage the use of the software
	30	* emulation strongly - use the following code in your userland stuff:
	31	*
	32	* #include <sys/sysmips.h>
	33	*
	34	* ...
	35	* sysmips(MIPS_FIXADE, x);
	36	* ...
	37	*
	38	* The argument x is 0 for disabling software emulation, enabled otherwise.
	39	*
	40	* Below a little program to play around with this feature.
	41	*
	42	* #include <stdio.h>
	43	* #include <sys/sysmips.h>
	44	*
	45	* struct foo {
	46	* unsigned char bar[8];
	47	* };
	48	*
	49	* main(int argc, char *argv[])
	50	* {
	51	* struct foo x = {0, 1, 2, 3, 4, 5, 6, 7};
	52	* unsigned int p = (unsigned int ) (x.bar + 3);
	53	* int i;
	54	*
	55	* if (argc > 1)
	56	* sysmips(MIPS_FIXADE, atoi(argv[1]));
	57	*
	58	* printf("p = %08lx\n", p);
	59	*
	60	* *p = 0xdeadface;
	61	*
	62	* for(i = 0; i <= 7; i++)
	63	* printf("%02x ", x.bar[i]);
	64	* printf("\n");
65	* }
66	*
67	* Coprocessor loads are not supported; I think this case is unimportant
68	* in the practice.
69	*
70	* TODO: Handle ndc (attempted store to doubleword in uncached memory)
71	* exception for the R6000.
72	* A store crossing a page boundary might be executed only partially.
73	* Undo the partial store in this case.
74	*/
1da177e4 LT	75	#include <linux/mm.h>
	76	#include <linux/module.h>
	77	#include <linux/signal.h>
	78	#include <linux/smp.h>
e8edc6e0	79	#include <linux/sched.h>
6312e0ee	80	#include <linux/debugfs.h>
7f788d2d DCZ	81	#include <linux/perf_event.h>
7f788d2d DCZ	82
1da177e4 LT	83	#include <asm/asm.h>
	84	#include <asm/branch.h>
	85	#include <asm/byteorder.h>
69f3a7de	86	#include <asm/cop2.h>
1da177e4 LT	87	#include <asm/inst.h>
	88	#include <asm/uaccess.h>
	89	#include <asm/system.h>
	90
	91	#define STR(x) __STR(x)
	92	#define __STR(x) #x
	93
6312e0ee AN	94	enum {
	95	UNALIGNED_ACTION_QUIET,
	96	UNALIGNED_ACTION_SIGNAL,
	97	UNALIGNED_ACTION_SHOW,
	98	};
	99	#ifdef CONFIG_DEBUG_FS
	100	static u32 unaligned_instructions;
	101	static u32 unaligned_action;
	102	#else
	103	#define unaligned_action UNALIGNED_ACTION_QUIET
1da177e4	104	#endif
6312e0ee	105	extern void show_registers(struct pt_regs *regs);
1da177e4	106
7f18f151 RB	107	static void emulate_load_store_insn(struct pt_regs *regs,
7f18f151 RB	108	void __user addr, unsigned int __user pc)
1da177e4 LT	109	{
	110	union mips_instruction insn;
	111	unsigned long value;
	112	unsigned int res;
	113
a8b0ca17	114	perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
7f788d2d	115
1da177e4 LT	116	/*
	117	* This load never faults.
	118	*/
fe00f943	119	__get_user(insn.word, pc);
1da177e4 LT	120
	121	switch (insn.i_format.opcode) {
	122	/*
	123	* These are instructions that a compiler doesn't generate. We
	124	* can assume therefore that the code is MIPS-aware and
	125	* really buggy. Emulating these instructions would break the
	126	* semantics anyway.
	127	*/
	128	case ll_op:
	129	case lld_op:
	130	case sc_op:
	131	case scd_op:
	132
	133	/*
	134	* For these instructions the only way to create an address
	135	* error is an attempted access to kernel/supervisor address
	136	* space.
	137	*/
	138	case ldl_op:
	139	case ldr_op:
	140	case lwl_op:
	141	case lwr_op:
	142	case sdl_op:
	143	case sdr_op:
	144	case swl_op:
	145	case swr_op:
	146	case lb_op:
	147	case lbu_op:
	148	case sb_op:
	149	goto sigbus;
	150
	151	/*
	152	* The remaining opcodes are the ones that are really of interest.
	153	*/
	154	case lh_op:
	155	if (!access_ok(VERIFY_READ, addr, 2))
	156	goto sigbus;
	157
	158	__asm__ __volatile__ (".set\tnoat\n"
	159	#ifdef __BIG_ENDIAN
	160	"1:\tlb\t%0, 0(%2)\n"
	161	"2:\tlbu\t$1, 1(%2)\n\t"
	162	#endif
	163	#ifdef __LITTLE_ENDIAN
	164	"1:\tlb\t%0, 1(%2)\n"
	165	"2:\tlbu\t$1, 0(%2)\n\t"
	166	#endif
	167	"sll\t%0, 0x8\n\t"
	168	"or\t%0, $1\n\t"
	169	"li\t%1, 0\n"
	170	"3:\t.set\tat\n\t"
	171	".section\t.fixup,\"ax\"\n\t"
	172	"4:\tli\t%1, %3\n\t"
	173	"j\t3b\n\t"
	174	".previous\n\t"
	175	".section\t__ex_table,\"a\"\n\t"
	176	STR(PTR)"\t1b, 4b\n\t"
	177	STR(PTR)"\t2b, 4b\n\t"
	178	".previous"
	179	: "=&r" (value), "=r" (res)
	180	: "r" (addr), "i" (-EFAULT));
	181	if (res)
	182	goto fault;
7f18f151 RB	183	compute_return_epc(regs);
7f18f151 RB	184	regs->regs[insn.i_format.rt] = value;
1da177e4 LT	185	break;
	186
	187	case lw_op:
	188	if (!access_ok(VERIFY_READ, addr, 4))
	189	goto sigbus;
	190
	191	__asm__ __volatile__ (
	192	#ifdef __BIG_ENDIAN
	193	"1:\tlwl\t%0, (%2)\n"
	194	"2:\tlwr\t%0, 3(%2)\n\t"
	195	#endif
	196	#ifdef __LITTLE_ENDIAN
	197	"1:\tlwl\t%0, 3(%2)\n"
	198	"2:\tlwr\t%0, (%2)\n\t"
	199	#endif
	200	"li\t%1, 0\n"
	201	"3:\t.section\t.fixup,\"ax\"\n\t"
	202	"4:\tli\t%1, %3\n\t"
	203	"j\t3b\n\t"
	204	".previous\n\t"
	205	".section\t__ex_table,\"a\"\n\t"
	206	STR(PTR)"\t1b, 4b\n\t"
	207	STR(PTR)"\t2b, 4b\n\t"
	208	".previous"
	209	: "=&r" (value), "=r" (res)
	210	: "r" (addr), "i" (-EFAULT));
	211	if (res)
	212	goto fault;
7f18f151 RB	213	compute_return_epc(regs);
7f18f151 RB	214	regs->regs[insn.i_format.rt] = value;
1da177e4 LT	215	break;
	216
	217	case lhu_op:
	218	if (!access_ok(VERIFY_READ, addr, 2))
	219	goto sigbus;
	220
	221	__asm__ __volatile__ (
	222	".set\tnoat\n"
	223	#ifdef __BIG_ENDIAN
	224	"1:\tlbu\t%0, 0(%2)\n"
	225	"2:\tlbu\t$1, 1(%2)\n\t"
	226	#endif
	227	#ifdef __LITTLE_ENDIAN
	228	"1:\tlbu\t%0, 1(%2)\n"
	229	"2:\tlbu\t$1, 0(%2)\n\t"
	230	#endif
	231	"sll\t%0, 0x8\n\t"
	232	"or\t%0, $1\n\t"
	233	"li\t%1, 0\n"
	234	"3:\t.set\tat\n\t"
	235	".section\t.fixup,\"ax\"\n\t"
	236	"4:\tli\t%1, %3\n\t"
	237	"j\t3b\n\t"
	238	".previous\n\t"
	239	".section\t__ex_table,\"a\"\n\t"
	240	STR(PTR)"\t1b, 4b\n\t"
	241	STR(PTR)"\t2b, 4b\n\t"
	242	".previous"
	243	: "=&r" (value), "=r" (res)
	244	: "r" (addr), "i" (-EFAULT));
	245	if (res)
	246	goto fault;
7f18f151 RB	247	compute_return_epc(regs);
7f18f151 RB	248	regs->regs[insn.i_format.rt] = value;
1da177e4 LT	249	break;
	250
	251	case lwu_op:
875d43e7	252	#ifdef CONFIG_64BIT
1da177e4 LT	253	/*
	254	* A 32-bit kernel might be running on a 64-bit processor. But
	255	* if we're on a 32-bit processor and an i-cache incoherency
	256	* or race makes us see a 64-bit instruction here the sdl/sdr
	257	* would blow up, so for now we don't handle unaligned 64-bit
	258	* instructions on 32-bit kernels.
	259	*/
	260	if (!access_ok(VERIFY_READ, addr, 4))
	261	goto sigbus;
	262
	263	__asm__ __volatile__ (
	264	#ifdef __BIG_ENDIAN
	265	"1:\tlwl\t%0, (%2)\n"
	266	"2:\tlwr\t%0, 3(%2)\n\t"
	267	#endif
	268	#ifdef __LITTLE_ENDIAN
	269	"1:\tlwl\t%0, 3(%2)\n"
	270	"2:\tlwr\t%0, (%2)\n\t"
	271	#endif
	272	"dsll\t%0, %0, 32\n\t"
	273	"dsrl\t%0, %0, 32\n\t"
	274	"li\t%1, 0\n"
	275	"3:\t.section\t.fixup,\"ax\"\n\t"
	276	"4:\tli\t%1, %3\n\t"
	277	"j\t3b\n\t"
	278	".previous\n\t"
	279	".section\t__ex_table,\"a\"\n\t"
	280	STR(PTR)"\t1b, 4b\n\t"
	281	STR(PTR)"\t2b, 4b\n\t"
	282	".previous"
	283	: "=&r" (value), "=r" (res)
	284	: "r" (addr), "i" (-EFAULT));
	285	if (res)
	286	goto fault;
7f18f151 RB	287	compute_return_epc(regs);
7f18f151 RB	288	regs->regs[insn.i_format.rt] = value;
1da177e4	289	break;
875d43e7	290	#endif /* CONFIG_64BIT */
1da177e4 LT	291
	292	/* Cannot handle 64-bit instructions in 32-bit kernel */
	293	goto sigill;
	294
	295	case ld_op:
875d43e7	296	#ifdef CONFIG_64BIT
1da177e4 LT	297	/*
	298	* A 32-bit kernel might be running on a 64-bit processor. But
	299	* if we're on a 32-bit processor and an i-cache incoherency
	300	* or race makes us see a 64-bit instruction here the sdl/sdr
	301	* would blow up, so for now we don't handle unaligned 64-bit
	302	* instructions on 32-bit kernels.
	303	*/
	304	if (!access_ok(VERIFY_READ, addr, 8))
	305	goto sigbus;
	306
	307	__asm__ __volatile__ (
	308	#ifdef __BIG_ENDIAN
	309	"1:\tldl\t%0, (%2)\n"
	310	"2:\tldr\t%0, 7(%2)\n\t"
	311	#endif
	312	#ifdef __LITTLE_ENDIAN
	313	"1:\tldl\t%0, 7(%2)\n"
	314	"2:\tldr\t%0, (%2)\n\t"
	315	#endif
	316	"li\t%1, 0\n"
	317	"3:\t.section\t.fixup,\"ax\"\n\t"
	318	"4:\tli\t%1, %3\n\t"
	319	"j\t3b\n\t"
	320	".previous\n\t"
	321	".section\t__ex_table,\"a\"\n\t"
	322	STR(PTR)"\t1b, 4b\n\t"
	323	STR(PTR)"\t2b, 4b\n\t"
	324	".previous"
	325	: "=&r" (value), "=r" (res)
	326	: "r" (addr), "i" (-EFAULT));
	327	if (res)
	328	goto fault;
7f18f151 RB	329	compute_return_epc(regs);
7f18f151 RB	330	regs->regs[insn.i_format.rt] = value;
1da177e4	331	break;
875d43e7	332	#endif /* CONFIG_64BIT */
1da177e4 LT	333
	334	/* Cannot handle 64-bit instructions in 32-bit kernel */
	335	goto sigill;
	336
	337	case sh_op:
	338	if (!access_ok(VERIFY_WRITE, addr, 2))
	339	goto sigbus;
	340
	341	value = regs->regs[insn.i_format.rt];
	342	__asm__ __volatile__ (
	343	#ifdef __BIG_ENDIAN
	344	".set\tnoat\n"
	345	"1:\tsb\t%1, 1(%2)\n\t"
	346	"srl\t$1, %1, 0x8\n"
	347	"2:\tsb\t$1, 0(%2)\n\t"
	348	".set\tat\n\t"
	349	#endif
	350	#ifdef __LITTLE_ENDIAN
	351	".set\tnoat\n"
	352	"1:\tsb\t%1, 0(%2)\n\t"
	353	"srl\t$1,%1, 0x8\n"
	354	"2:\tsb\t$1, 1(%2)\n\t"
	355	".set\tat\n\t"
	356	#endif
	357	"li\t%0, 0\n"
	358	"3:\n\t"
	359	".section\t.fixup,\"ax\"\n\t"
	360	"4:\tli\t%0, %3\n\t"
	361	"j\t3b\n\t"
	362	".previous\n\t"
	363	".section\t__ex_table,\"a\"\n\t"
	364	STR(PTR)"\t1b, 4b\n\t"
	365	STR(PTR)"\t2b, 4b\n\t"
	366	".previous"
	367	: "=r" (res)
	368	: "r" (value), "r" (addr), "i" (-EFAULT));
	369	if (res)
	370	goto fault;
7f18f151	371	compute_return_epc(regs);
1da177e4 LT	372	break;
	373
	374	case sw_op:
	375	if (!access_ok(VERIFY_WRITE, addr, 4))
	376	goto sigbus;
	377
	378	value = regs->regs[insn.i_format.rt];
	379	__asm__ __volatile__ (
	380	#ifdef __BIG_ENDIAN
	381	"1:\tswl\t%1,(%2)\n"
	382	"2:\tswr\t%1, 3(%2)\n\t"
	383	#endif
	384	#ifdef __LITTLE_ENDIAN
	385	"1:\tswl\t%1, 3(%2)\n"
	386	"2:\tswr\t%1, (%2)\n\t"
	387	#endif
	388	"li\t%0, 0\n"
	389	"3:\n\t"
	390	".section\t.fixup,\"ax\"\n\t"
	391	"4:\tli\t%0, %3\n\t"
	392	"j\t3b\n\t"
	393	".previous\n\t"
	394	".section\t__ex_table,\"a\"\n\t"
	395	STR(PTR)"\t1b, 4b\n\t"
	396	STR(PTR)"\t2b, 4b\n\t"
	397	".previous"
	398	: "=r" (res)
	399	: "r" (value), "r" (addr), "i" (-EFAULT));
	400	if (res)
	401	goto fault;
7f18f151	402	compute_return_epc(regs);
1da177e4 LT	403	break;
	404
	405	case sd_op:
875d43e7	406	#ifdef CONFIG_64BIT
1da177e4 LT	407	/*
	408	* A 32-bit kernel might be running on a 64-bit processor. But
	409	* if we're on a 32-bit processor and an i-cache incoherency
	410	* or race makes us see a 64-bit instruction here the sdl/sdr
	411	* would blow up, so for now we don't handle unaligned 64-bit
	412	* instructions on 32-bit kernels.
	413	*/
	414	if (!access_ok(VERIFY_WRITE, addr, 8))
	415	goto sigbus;
	416
	417	value = regs->regs[insn.i_format.rt];
	418	__asm__ __volatile__ (
	419	#ifdef __BIG_ENDIAN
	420	"1:\tsdl\t%1,(%2)\n"
	421	"2:\tsdr\t%1, 7(%2)\n\t"
	422	#endif
	423	#ifdef __LITTLE_ENDIAN
	424	"1:\tsdl\t%1, 7(%2)\n"
	425	"2:\tsdr\t%1, (%2)\n\t"
	426	#endif
	427	"li\t%0, 0\n"
	428	"3:\n\t"
	429	".section\t.fixup,\"ax\"\n\t"
	430	"4:\tli\t%0, %3\n\t"
	431	"j\t3b\n\t"
	432	".previous\n\t"
	433	".section\t__ex_table,\"a\"\n\t"
	434	STR(PTR)"\t1b, 4b\n\t"
	435	STR(PTR)"\t2b, 4b\n\t"
	436	".previous"
	437	: "=r" (res)
	438	: "r" (value), "r" (addr), "i" (-EFAULT));
	439	if (res)
	440	goto fault;
7f18f151	441	compute_return_epc(regs);
1da177e4	442	break;
875d43e7	443	#endif /* CONFIG_64BIT */
1da177e4 LT	444
	445	/* Cannot handle 64-bit instructions in 32-bit kernel */
	446	goto sigill;
	447
	448	case lwc1_op:
	449	case ldc1_op:
	450	case swc1_op:
	451	case sdc1_op:
	452	/*
	453	* I herewith declare: this does not happen. So send SIGBUS.
	454	*/
	455	goto sigbus;
	456
69f3a7de RB	457	/*
	458	* COP2 is available to implementor for application specific use.
	459	* It's up to applications to register a notifier chain and do
	460	* whatever they have to do, including possible sending of signals.
	461	*/
1da177e4	462	case lwc2_op:
69f3a7de RB	463	cu2_notifier_call_chain(CU2_LWC2_OP, regs);
	464	break;
	465
1da177e4	466	case ldc2_op:
69f3a7de RB	467	cu2_notifier_call_chain(CU2_LDC2_OP, regs);
	468	break;
	469
1da177e4	470	case swc2_op:
69f3a7de RB	471	cu2_notifier_call_chain(CU2_SWC2_OP, regs);
	472	break;
	473
1da177e4	474	case sdc2_op:
69f3a7de RB	475	cu2_notifier_call_chain(CU2_SDC2_OP, regs);
	476	break;
	477
1da177e4 LT	478	default:
	479	/*
	480	* Pheeee... We encountered an yet unknown instruction or
	481	* cache coherence problem. Die sucker, die ...
	482	*/
	483	goto sigill;
	484	}
	485
6312e0ee	486	#ifdef CONFIG_DEBUG_FS
1da177e4 LT	487	unaligned_instructions++;
	488	#endif
	489
7f18f151	490	return;
1da177e4 LT	491
	492	fault:
	493	/* Did we have an exception handler installed? */
	494	if (fixup_exception(regs))
7f18f151	495	return;
1da177e4	496
49a89efb	497	die_if_kernel("Unhandled kernel unaligned access", regs);
a6d5ff04	498	force_sig(SIGSEGV, current);
1da177e4	499
7f18f151	500	return;
1da177e4 LT	501
	502	sigbus:
	503	die_if_kernel("Unhandled kernel unaligned access", regs);
a6d5ff04	504	force_sig(SIGBUS, current);
1da177e4	505
7f18f151	506	return;
1da177e4 LT	507
	508	sigill:
	509	die_if_kernel("Unhandled kernel unaligned access or invalid instruction", regs);
a6d5ff04	510	force_sig(SIGILL, current);
1da177e4 LT	511	}
	512
	513	asmlinkage void do_ade(struct pt_regs *regs)
	514	{
fe00f943	515	unsigned int __user *pc;
1da177e4	516	mm_segment_t seg;
1da177e4	517
7f788d2d	518	perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS,
a8b0ca17	519	1, regs, regs->cp0_badvaddr);
1da177e4 LT	520	/*
	521	* Did we catch a fault trying to load an instruction?
	522	* Or are we running in MIPS16 mode?
	523	*/
	524	if ((regs->cp0_badvaddr == regs->cp0_epc) \|\| (regs->cp0_epc & 0x1))
	525	goto sigbus;
	526
fe00f943	527	pc = (unsigned int __user *) exception_epc(regs);
293c5bd1	528	if (user_mode(regs) && !test_thread_flag(TIF_FIXADE))
1da177e4	529	goto sigbus;
6312e0ee AN	530	if (unaligned_action == UNALIGNED_ACTION_SIGNAL)
	531	goto sigbus;
	532	else if (unaligned_action == UNALIGNED_ACTION_SHOW)
	533	show_registers(regs);
1da177e4 LT	534
	535	/*
	536	* Do branch emulation only if we didn't forward the exception.
	537	* This is all so but ugly ...
	538	*/
	539	seg = get_fs();
	540	if (!user_mode(regs))
	541	set_fs(KERNEL_DS);
7f18f151	542	emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc);
1da177e4 LT	543	set_fs(seg);
	544
	545	return;
	546
	547	sigbus:
	548	die_if_kernel("Kernel unaligned instruction access", regs);
	549	force_sig(SIGBUS, current);
	550
	551	/*
	552	* XXX On return from the signal handler we should advance the epc
	553	*/
	554	}
6312e0ee AN	555
	556	#ifdef CONFIG_DEBUG_FS
	557	extern struct dentry *mips_debugfs_dir;
	558	static int __init debugfs_unaligned(void)
	559	{
	560	struct dentry *d;
	561
	562	if (!mips_debugfs_dir)
	563	return -ENODEV;
	564	d = debugfs_create_u32("unaligned_instructions", S_IRUGO,
	565	mips_debugfs_dir, &unaligned_instructions);
b517531c Z	566	if (!d)
b517531c Z	567	return -ENOMEM;
6312e0ee AN	568	d = debugfs_create_u32("unaligned_action", S_IRUGO \| S_IWUSR,
6312e0ee AN	569	mips_debugfs_dir, &unaligned_action);
b517531c Z	570	if (!d)
b517531c Z	571	return -ENOMEM;
6312e0ee AN	572	return 0;
	573	}
	574	__initcall(debugfs_unaligned);
	575	#endif