[qemu.git] / softmmu_template.h

/*
 *  Software MMU support
 *
 * Generate helpers used by TCG for qemu_ld/st ops and code load
 * functions.
 *
 * Included from target op helpers and exec.c.
 *
 *  Copyright (c) 2003 Fabrice Bellard
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
 */
#include "qemu-timer.h"
#include "memory.h"

#define DATA_SIZE (1 << SHIFT)

#if DATA_SIZE == 8
#define SUFFIX q
#define USUFFIX q
#define DATA_TYPE uint64_t
#elif DATA_SIZE == 4
#define SUFFIX l
#define USUFFIX l
#define DATA_TYPE uint32_t
#elif DATA_SIZE == 2
#define SUFFIX w
#define USUFFIX uw
#define DATA_TYPE uint16_t
#elif DATA_SIZE == 1
#define SUFFIX b
#define USUFFIX ub
#define DATA_TYPE uint8_t
#else
#error unsupported data size
#endif

#ifdef SOFTMMU_CODE_ACCESS
#define READ_ACCESS_TYPE 2
#define ADDR_READ addr_code
#else
#define READ_ACCESS_TYPE 0
#define ADDR_READ addr_read
#endif

#ifndef CONFIG_TCG_PASS_AREG0
#define ENV_PARAM
#define ENV_VAR
#define CPU_PREFIX
#define HELPER_PREFIX __
#else
#define ENV_PARAM CPUArchState *env,
#define ENV_VAR env,
#define CPU_PREFIX cpu_
#define HELPER_PREFIX helper_
#endif

static DATA_TYPE glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_PARAM
                                                        target_ulong addr,
                                                        int mmu_idx,
                                                        uintptr_t retaddr);
static inline DATA_TYPE glue(io_read, SUFFIX)(ENV_PARAM
                                              target_phys_addr_t physaddr,
                                              target_ulong addr,
                                              uintptr_t retaddr)
{
    DATA_TYPE res;
    MemoryRegion *mr = iotlb_to_region(physaddr);

    physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
    env->mem_io_pc = retaddr;
    if (mr != &io_mem_ram && mr != &io_mem_rom
        && mr != &io_mem_unassigned
        && mr != &io_mem_notdirty
            && !can_do_io(env)) {
        cpu_io_recompile(env, retaddr);
    }

    env->mem_io_vaddr = addr;
#if SHIFT <= 2
    res = io_mem_read(mr, physaddr, 1 << SHIFT);
#else
#ifdef TARGET_WORDS_BIGENDIAN
    res = io_mem_read(mr, physaddr, 4) << 32;
    res |= io_mem_read(mr, physaddr + 4, 4);
#else
    res = io_mem_read(mr, physaddr, 4);
    res |= io_mem_read(mr, physaddr + 4, 4) << 32;
#endif
#endif /* SHIFT > 2 */
    return res;
}

/* handle all cases except unaligned access which span two pages */
DATA_TYPE
glue(glue(glue(HELPER_PREFIX, ld), SUFFIX), MMUSUFFIX)(ENV_PARAM
                                                       target_ulong addr,
                                                       int mmu_idx)
{
    DATA_TYPE res;
    int index;
    target_ulong tlb_addr;
    target_phys_addr_t ioaddr;
    uintptr_t retaddr;

    /* test if there is match for unaligned or IO access */
    /* XXX: could done more in memory macro in a non portable way */
    index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 redo:
    tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
    if ((addr & TARGET_PAGE_MASK) == (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
        if (tlb_addr & ~TARGET_PAGE_MASK) {
            /* IO access */
            if ((addr & (DATA_SIZE - 1)) != 0)
                goto do_unaligned_access;
            retaddr = GETPC();
            ioaddr = env->iotlb[mmu_idx][index];
            res = glue(io_read, SUFFIX)(ENV_VAR ioaddr, addr, retaddr);
        } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
            /* slow unaligned access (it spans two pages or IO) */
        do_unaligned_access:
            retaddr = GETPC();
#ifdef ALIGNED_ONLY
            do_unaligned_access(ENV_VAR addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
#endif
            res = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_VAR addr,
                                                         mmu_idx, retaddr);
        } else {
            /* unaligned/aligned access in the same page */
            uintptr_t addend;
#ifdef ALIGNED_ONLY
            if ((addr & (DATA_SIZE - 1)) != 0) {
                retaddr = GETPC();
                do_unaligned_access(ENV_VAR addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
            }
#endif
            addend = env->tlb_table[mmu_idx][index].addend;
            res = glue(glue(ld, USUFFIX), _raw)((uint8_t *)(intptr_t)
                                                (addr + addend));
        }
    } else {
        /* the page is not in the TLB : fill it */
        retaddr = GETPC();
#ifdef ALIGNED_ONLY
        if ((addr & (DATA_SIZE - 1)) != 0)
            do_unaligned_access(ENV_VAR addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
#endif
        tlb_fill(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
        goto redo;
    }
    return res;
}

/* handle all unaligned cases */
static DATA_TYPE
glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_PARAM
                                       target_ulong addr,
                                       int mmu_idx,
                                       uintptr_t retaddr)
{
    DATA_TYPE res, res1, res2;
    int index, shift;
    target_phys_addr_t ioaddr;
    target_ulong tlb_addr, addr1, addr2;

    index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 redo:
    tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
    if ((addr & TARGET_PAGE_MASK) == (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
        if (tlb_addr & ~TARGET_PAGE_MASK) {
            /* IO access */
            if ((addr & (DATA_SIZE - 1)) != 0)
                goto do_unaligned_access;
            ioaddr = env->iotlb[mmu_idx][index];
            res = glue(io_read, SUFFIX)(ENV_VAR ioaddr, addr, retaddr);
        } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
        do_unaligned_access:
            /* slow unaligned access (it spans two pages) */
            addr1 = addr & ~(DATA_SIZE - 1);
            addr2 = addr1 + DATA_SIZE;
            res1 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_VAR addr1,
                                                          mmu_idx, retaddr);
            res2 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_VAR addr2,
                                                          mmu_idx, retaddr);
            shift = (addr & (DATA_SIZE - 1)) * 8;
#ifdef TARGET_WORDS_BIGENDIAN
            res = (res1 << shift) | (res2 >> ((DATA_SIZE * 8) - shift));
#else
            res = (res1 >> shift) | (res2 << ((DATA_SIZE * 8) - shift));
#endif
            res = (DATA_TYPE)res;
        } else {
            /* unaligned/aligned access in the same page */
            uintptr_t addend = env->tlb_table[mmu_idx][index].addend;
            res = glue(glue(ld, USUFFIX), _raw)((uint8_t *)(intptr_t)
                                                (addr + addend));
        }
    } else {
        /* the page is not in the TLB : fill it */
        tlb_fill(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
        goto redo;
    }
    return res;
}

#ifndef SOFTMMU_CODE_ACCESS

static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_PARAM
                                                   target_ulong addr,
                                                   DATA_TYPE val,
                                                   int mmu_idx,
                                                   uintptr_t retaddr);

static inline void glue(io_write, SUFFIX)(ENV_PARAM
                                          target_phys_addr_t physaddr,
                                          DATA_TYPE val,
                                          target_ulong addr,
                                          uintptr_t retaddr)
{
    MemoryRegion *mr = iotlb_to_region(physaddr);

    physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
    if (mr != &io_mem_ram && mr != &io_mem_rom
        && mr != &io_mem_unassigned
        && mr != &io_mem_notdirty
            && !can_do_io(env)) {
        cpu_io_recompile(env, retaddr);
    }

    env->mem_io_vaddr = addr;
    env->mem_io_pc = retaddr;
#if SHIFT <= 2
    io_mem_write(mr, physaddr, val, 1 << SHIFT);
#else
#ifdef TARGET_WORDS_BIGENDIAN
    io_mem_write(mr, physaddr, (val >> 32), 4);
    io_mem_write(mr, physaddr + 4, (uint32_t)val, 4);
#else
    io_mem_write(mr, physaddr, (uint32_t)val, 4);
    io_mem_write(mr, physaddr + 4, val >> 32, 4);
#endif
#endif /* SHIFT > 2 */
}

void glue(glue(glue(HELPER_PREFIX, st), SUFFIX), MMUSUFFIX)(ENV_PARAM
                                                            target_ulong addr,
                                                            DATA_TYPE val,
                                                            int mmu_idx)
{
    target_phys_addr_t ioaddr;
    target_ulong tlb_addr;
    uintptr_t retaddr;
    int index;

    index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 redo:
    tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
    if ((addr & TARGET_PAGE_MASK) == (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
        if (tlb_addr & ~TARGET_PAGE_MASK) {
            /* IO access */
            if ((addr & (DATA_SIZE - 1)) != 0)
                goto do_unaligned_access;
            retaddr = GETPC();
            ioaddr = env->iotlb[mmu_idx][index];
            glue(io_write, SUFFIX)(ENV_VAR ioaddr, val, addr, retaddr);
        } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
        do_unaligned_access:
            retaddr = GETPC();
#ifdef ALIGNED_ONLY
            do_unaligned_access(ENV_VAR addr, 1, mmu_idx, retaddr);
#endif
            glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_VAR addr, val,
                                                   mmu_idx, retaddr);
        } else {
            /* aligned/unaligned access in the same page */
            uintptr_t addend;
#ifdef ALIGNED_ONLY
            if ((addr & (DATA_SIZE - 1)) != 0) {
                retaddr = GETPC();
                do_unaligned_access(ENV_VAR addr, 1, mmu_idx, retaddr);
            }
#endif
            addend = env->tlb_table[mmu_idx][index].addend;
            glue(glue(st, SUFFIX), _raw)((uint8_t *)(intptr_t)
                                         (addr + addend), val);
        }
    } else {
        /* the page is not in the TLB : fill it */
        retaddr = GETPC();
#ifdef ALIGNED_ONLY
        if ((addr & (DATA_SIZE - 1)) != 0)
            do_unaligned_access(ENV_VAR addr, 1, mmu_idx, retaddr);
#endif
        tlb_fill(env, addr, 1, mmu_idx, retaddr);
        goto redo;
    }
}

/* handles all unaligned cases */
static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_PARAM
                                                   target_ulong addr,
                                                   DATA_TYPE val,
                                                   int mmu_idx,
                                                   uintptr_t retaddr)
{
    target_phys_addr_t ioaddr;
    target_ulong tlb_addr;
    int index, i;

    index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 redo:
    tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
    if ((addr & TARGET_PAGE_MASK) == (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
        if (tlb_addr & ~TARGET_PAGE_MASK) {
            /* IO access */
            if ((addr & (DATA_SIZE - 1)) != 0)
                goto do_unaligned_access;
            ioaddr = env->iotlb[mmu_idx][index];
            glue(io_write, SUFFIX)(ENV_VAR ioaddr, val, addr, retaddr);
        } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
        do_unaligned_access:
            /* XXX: not efficient, but simple */
            /* Note: relies on the fact that tlb_fill() does not remove the
             * previous page from the TLB cache.  */
            for(i = DATA_SIZE - 1; i >= 0; i--) {
#ifdef TARGET_WORDS_BIGENDIAN
                glue(slow_stb, MMUSUFFIX)(ENV_VAR addr + i,
                                          val >> (((DATA_SIZE - 1) * 8) - (i * 8)),
                                          mmu_idx, retaddr);
#else
                glue(slow_stb, MMUSUFFIX)(ENV_VAR addr + i,
                                          val >> (i * 8),
                                          mmu_idx, retaddr);
#endif
            }
        } else {
            /* aligned/unaligned access in the same page */
            uintptr_t addend = env->tlb_table[mmu_idx][index].addend;
            glue(glue(st, SUFFIX), _raw)((uint8_t *)(intptr_t)
                                         (addr + addend), val);
        }
    } else {
        /* the page is not in the TLB : fill it */
        tlb_fill(env, addr, 1, mmu_idx, retaddr);
        goto redo;
    }
}

#endif /* !defined(SOFTMMU_CODE_ACCESS) */

#undef READ_ACCESS_TYPE
#undef SHIFT
#undef DATA_TYPE
#undef SUFFIX
#undef USUFFIX
#undef DATA_SIZE
#undef ADDR_READ
#undef ENV_PARAM
#undef ENV_VAR
#undef CPU_PREFIX
#undef HELPER_PREFIX
Commit	Line	Data
b92e5a22 FB	1	/*
b92e5a22 FB	2	* Software MMU support
5fafdf24	3	*
efbf29b6 BS	4	* Generate helpers used by TCG for qemu_ld/st ops and code load
	5	* functions.
	6	*
	7	* Included from target op helpers and exec.c.
	8	*
b92e5a22 FB	9	* Copyright (c) 2003 Fabrice Bellard
	10	*
	11	* This library is free software; you can redistribute it and/or
	12	* modify it under the terms of the GNU Lesser General Public
	13	* License as published by the Free Software Foundation; either
	14	* version 2 of the License, or (at your option) any later version.
	15	*
	16	* This library is distributed in the hope that it will be useful,
	17	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* Lesser General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU Lesser General Public
8167ee88	22	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
b92e5a22	23	*/
29e922b6	24	#include "qemu-timer.h"
0e0df1e2	25	#include "memory.h"
29e922b6	26
b92e5a22 FB	27	#define DATA_SIZE (1 << SHIFT)
	28
	29	#if DATA_SIZE == 8
	30	#define SUFFIX q
61382a50	31	#define USUFFIX q
b92e5a22 FB	32	#define DATA_TYPE uint64_t
	33	#elif DATA_SIZE == 4
	34	#define SUFFIX l
61382a50	35	#define USUFFIX l
b92e5a22 FB	36	#define DATA_TYPE uint32_t
	37	#elif DATA_SIZE == 2
	38	#define SUFFIX w
61382a50	39	#define USUFFIX uw
b92e5a22 FB	40	#define DATA_TYPE uint16_t
	41	#elif DATA_SIZE == 1
	42	#define SUFFIX b
61382a50	43	#define USUFFIX ub
b92e5a22 FB	44	#define DATA_TYPE uint8_t
	45	#else
	46	#error unsupported data size
	47	#endif
	48
b769d8fe FB	49	#ifdef SOFTMMU_CODE_ACCESS
b769d8fe FB	50	#define READ_ACCESS_TYPE 2
84b7b8e7	51	#define ADDR_READ addr_code
b769d8fe FB	52	#else
b769d8fe FB	53	#define READ_ACCESS_TYPE 0
84b7b8e7	54	#define ADDR_READ addr_read
b769d8fe FB	55	#endif
b769d8fe FB	56
e141ab52 BS	57	#ifndef CONFIG_TCG_PASS_AREG0
	58	#define ENV_PARAM
	59	#define ENV_VAR
	60	#define CPU_PREFIX
	61	#define HELPER_PREFIX __
	62	#else
	63	#define ENV_PARAM CPUArchState *env,
	64	#define ENV_VAR env,
	65	#define CPU_PREFIX cpu_
	66	#define HELPER_PREFIX helper_
	67	#endif
	68
	69	static DATA_TYPE glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_PARAM
	70	target_ulong addr,
6ebbf390	71	int mmu_idx,
20503968	72	uintptr_t retaddr);
e141ab52 BS	73	static inline DATA_TYPE glue(io_read, SUFFIX)(ENV_PARAM
e141ab52 BS	74	target_phys_addr_t physaddr,
2e70f6ef	75	target_ulong addr,
20503968	76	uintptr_t retaddr)
b92e5a22 FB	77	{
b92e5a22 FB	78	DATA_TYPE res;
37ec01d4 AK	79	MemoryRegion *mr = iotlb_to_region(physaddr);
37ec01d4 AK	80
0f459d16	81	physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
20503968	82	env->mem_io_pc = retaddr;
37ec01d4 AK	83	if (mr != &io_mem_ram && mr != &io_mem_rom
	84	&& mr != &io_mem_unassigned
	85	&& mr != &io_mem_notdirty
2e70f6ef PB	86	&& !can_do_io(env)) {
	87	cpu_io_recompile(env, retaddr);
	88	}
b92e5a22	89
db8886d3	90	env->mem_io_vaddr = addr;
b92e5a22	91	#if SHIFT <= 2
37ec01d4	92	res = io_mem_read(mr, physaddr, 1 << SHIFT);
b92e5a22 FB	93	#else
b92e5a22 FB	94	#ifdef TARGET_WORDS_BIGENDIAN
37ec01d4 AK	95	res = io_mem_read(mr, physaddr, 4) << 32;
37ec01d4 AK	96	res \|= io_mem_read(mr, physaddr + 4, 4);
b92e5a22	97	#else
37ec01d4 AK	98	res = io_mem_read(mr, physaddr, 4);
37ec01d4 AK	99	res \|= io_mem_read(mr, physaddr + 4, 4) << 32;
b92e5a22 FB	100	#endif
	101	#endif /* SHIFT > 2 */
	102	return res;
	103	}
	104
b92e5a22	105	/* handle all cases except unaligned access which span two pages */
e141ab52 BS	106	DATA_TYPE
	107	glue(glue(glue(HELPER_PREFIX, ld), SUFFIX), MMUSUFFIX)(ENV_PARAM
	108	target_ulong addr,
	109	int mmu_idx)
b92e5a22 FB	110	{
b92e5a22 FB	111	DATA_TYPE res;
61382a50	112	int index;
c27004ec	113	target_ulong tlb_addr;
355b1943	114	target_phys_addr_t ioaddr;
20503968	115	uintptr_t retaddr;
3b46e624	116
b92e5a22 FB	117	/* test if there is match for unaligned or IO access */
b92e5a22 FB	118	/* XXX: could done more in memory macro in a non portable way */
b92e5a22 FB	119	index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
b92e5a22 FB	120	redo:
6ebbf390	121	tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
b92e5a22	122	if ((addr & TARGET_PAGE_MASK) == (tlb_addr & (TARGET_PAGE_MASK \| TLB_INVALID_MASK))) {
b92e5a22 FB	123	if (tlb_addr & ~TARGET_PAGE_MASK) {
	124	/* IO access */
	125	if ((addr & (DATA_SIZE - 1)) != 0)
	126	goto do_unaligned_access;
2e70f6ef	127	retaddr = GETPC();
37ec01d4	128	ioaddr = env->iotlb[mmu_idx][index];
e141ab52	129	res = glue(io_read, SUFFIX)(ENV_VAR ioaddr, addr, retaddr);
98699967	130	} else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
b92e5a22 FB	131	/* slow unaligned access (it spans two pages or IO) */
b92e5a22 FB	132	do_unaligned_access:
61382a50	133	retaddr = GETPC();
a64d4718	134	#ifdef ALIGNED_ONLY
e141ab52	135	do_unaligned_access(ENV_VAR addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
a64d4718	136	#endif
e141ab52	137	res = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_VAR addr,
6ebbf390	138	mmu_idx, retaddr);
b92e5a22	139	} else {
a64d4718	140	/* unaligned/aligned access in the same page */
b065927a	141	uintptr_t addend;
a64d4718 FB	142	#ifdef ALIGNED_ONLY
	143	if ((addr & (DATA_SIZE - 1)) != 0) {
	144	retaddr = GETPC();
e141ab52	145	do_unaligned_access(ENV_VAR addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
a64d4718 FB	146	}
a64d4718 FB	147	#endif
0f459d16	148	addend = env->tlb_table[mmu_idx][index].addend;
b065927a SW	149	res = glue(glue(ld, USUFFIX), _raw)((uint8_t *)(intptr_t)
b065927a SW	150	(addr + addend));
b92e5a22 FB	151	}
	152	} else {
	153	/* the page is not in the TLB : fill it */
61382a50	154	retaddr = GETPC();
a64d4718 FB	155	#ifdef ALIGNED_ONLY
a64d4718 FB	156	if ((addr & (DATA_SIZE - 1)) != 0)
e141ab52	157	do_unaligned_access(ENV_VAR addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
a64d4718	158	#endif
bccd9ec5	159	tlb_fill(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
b92e5a22 FB	160	goto redo;
	161	}
	162	return res;
	163	}
	164
	165	/* handle all unaligned cases */
e141ab52 BS	166	static DATA_TYPE
	167	glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_PARAM
	168	target_ulong addr,
	169	int mmu_idx,
20503968	170	uintptr_t retaddr)
b92e5a22 FB	171	{
b92e5a22 FB	172	DATA_TYPE res, res1, res2;
61382a50	173	int index, shift;
355b1943	174	target_phys_addr_t ioaddr;
c27004ec	175	target_ulong tlb_addr, addr1, addr2;
b92e5a22	176
b92e5a22 FB	177	index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
b92e5a22 FB	178	redo:
6ebbf390	179	tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
b92e5a22	180	if ((addr & TARGET_PAGE_MASK) == (tlb_addr & (TARGET_PAGE_MASK \| TLB_INVALID_MASK))) {
b92e5a22 FB	181	if (tlb_addr & ~TARGET_PAGE_MASK) {
	182	/* IO access */
	183	if ((addr & (DATA_SIZE - 1)) != 0)
	184	goto do_unaligned_access;
37ec01d4	185	ioaddr = env->iotlb[mmu_idx][index];
e141ab52	186	res = glue(io_read, SUFFIX)(ENV_VAR ioaddr, addr, retaddr);
98699967	187	} else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
b92e5a22 FB	188	do_unaligned_access:
	189	/* slow unaligned access (it spans two pages) */
	190	addr1 = addr & ~(DATA_SIZE - 1);
	191	addr2 = addr1 + DATA_SIZE;
e141ab52	192	res1 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_VAR addr1,
6ebbf390	193	mmu_idx, retaddr);
e141ab52	194	res2 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_VAR addr2,
6ebbf390	195	mmu_idx, retaddr);
b92e5a22 FB	196	shift = (addr & (DATA_SIZE - 1)) * 8;
	197	#ifdef TARGET_WORDS_BIGENDIAN
	198	res = (res1 << shift) \| (res2 >> ((DATA_SIZE * 8) - shift));
	199	#else
	200	res = (res1 >> shift) \| (res2 << ((DATA_SIZE * 8) - shift));
	201	#endif
6986f88c	202	res = (DATA_TYPE)res;
b92e5a22 FB	203	} else {
b92e5a22 FB	204	/* unaligned/aligned access in the same page */
b065927a SW	205	uintptr_t addend = env->tlb_table[mmu_idx][index].addend;
	206	res = glue(glue(ld, USUFFIX), _raw)((uint8_t *)(intptr_t)
	207	(addr + addend));
b92e5a22 FB	208	}
	209	} else {
	210	/* the page is not in the TLB : fill it */
bccd9ec5	211	tlb_fill(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
b92e5a22 FB	212	goto redo;
	213	}
	214	return res;
	215	}
	216
b769d8fe FB	217	#ifndef SOFTMMU_CODE_ACCESS
b769d8fe FB	218
e141ab52 BS	219	static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_PARAM
e141ab52 BS	220	target_ulong addr,
5fafdf24	221	DATA_TYPE val,
6ebbf390	222	int mmu_idx,
20503968	223	uintptr_t retaddr);
b769d8fe	224
e141ab52 BS	225	static inline void glue(io_write, SUFFIX)(ENV_PARAM
e141ab52 BS	226	target_phys_addr_t physaddr,
b769d8fe	227	DATA_TYPE val,
0f459d16	228	target_ulong addr,
20503968	229	uintptr_t retaddr)
b769d8fe	230	{
37ec01d4 AK	231	MemoryRegion *mr = iotlb_to_region(physaddr);
37ec01d4 AK	232
0f459d16	233	physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
37ec01d4 AK	234	if (mr != &io_mem_ram && mr != &io_mem_rom
	235	&& mr != &io_mem_unassigned
	236	&& mr != &io_mem_notdirty
2e70f6ef PB	237	&& !can_do_io(env)) {
	238	cpu_io_recompile(env, retaddr);
	239	}
b769d8fe	240
2e70f6ef	241	env->mem_io_vaddr = addr;
20503968	242	env->mem_io_pc = retaddr;
b769d8fe	243	#if SHIFT <= 2
37ec01d4	244	io_mem_write(mr, physaddr, val, 1 << SHIFT);
b769d8fe FB	245	#else
b769d8fe FB	246	#ifdef TARGET_WORDS_BIGENDIAN
37ec01d4 AK	247	io_mem_write(mr, physaddr, (val >> 32), 4);
37ec01d4 AK	248	io_mem_write(mr, physaddr + 4, (uint32_t)val, 4);
b769d8fe	249	#else
37ec01d4 AK	250	io_mem_write(mr, physaddr, (uint32_t)val, 4);
37ec01d4 AK	251	io_mem_write(mr, physaddr + 4, val >> 32, 4);
b769d8fe FB	252	#endif
	253	#endif /* SHIFT > 2 */
	254	}
b92e5a22	255
e141ab52 BS	256	void glue(glue(glue(HELPER_PREFIX, st), SUFFIX), MMUSUFFIX)(ENV_PARAM
	257	target_ulong addr,
	258	DATA_TYPE val,
	259	int mmu_idx)
b92e5a22	260	{
355b1943	261	target_phys_addr_t ioaddr;
c27004ec	262	target_ulong tlb_addr;
20503968	263	uintptr_t retaddr;
61382a50	264	int index;
3b46e624	265
b92e5a22 FB	266	index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
b92e5a22 FB	267	redo:
6ebbf390	268	tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
b92e5a22	269	if ((addr & TARGET_PAGE_MASK) == (tlb_addr & (TARGET_PAGE_MASK \| TLB_INVALID_MASK))) {
b92e5a22 FB	270	if (tlb_addr & ~TARGET_PAGE_MASK) {
	271	/* IO access */
	272	if ((addr & (DATA_SIZE - 1)) != 0)
	273	goto do_unaligned_access;
d720b93d	274	retaddr = GETPC();
37ec01d4	275	ioaddr = env->iotlb[mmu_idx][index];
e141ab52	276	glue(io_write, SUFFIX)(ENV_VAR ioaddr, val, addr, retaddr);
98699967	277	} else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
b92e5a22	278	do_unaligned_access:
61382a50	279	retaddr = GETPC();
a64d4718	280	#ifdef ALIGNED_ONLY
e141ab52	281	do_unaligned_access(ENV_VAR addr, 1, mmu_idx, retaddr);
a64d4718	282	#endif
e141ab52	283	glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_VAR addr, val,
6ebbf390	284	mmu_idx, retaddr);
b92e5a22 FB	285	} else {
b92e5a22 FB	286	/* aligned/unaligned access in the same page */
b065927a	287	uintptr_t addend;
a64d4718 FB	288	#ifdef ALIGNED_ONLY
	289	if ((addr & (DATA_SIZE - 1)) != 0) {
	290	retaddr = GETPC();
e141ab52	291	do_unaligned_access(ENV_VAR addr, 1, mmu_idx, retaddr);
a64d4718 FB	292	}
a64d4718 FB	293	#endif
0f459d16	294	addend = env->tlb_table[mmu_idx][index].addend;
b065927a SW	295	glue(glue(st, SUFFIX), _raw)((uint8_t *)(intptr_t)
b065927a SW	296	(addr + addend), val);
b92e5a22 FB	297	}
	298	} else {
	299	/* the page is not in the TLB : fill it */
61382a50	300	retaddr = GETPC();
a64d4718 FB	301	#ifdef ALIGNED_ONLY
a64d4718 FB	302	if ((addr & (DATA_SIZE - 1)) != 0)
e141ab52	303	do_unaligned_access(ENV_VAR addr, 1, mmu_idx, retaddr);
a64d4718	304	#endif
bccd9ec5	305	tlb_fill(env, addr, 1, mmu_idx, retaddr);
b92e5a22 FB	306	goto redo;
	307	}
	308	}
	309
	310	/* handles all unaligned cases */
e141ab52 BS	311	static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_PARAM
e141ab52 BS	312	target_ulong addr,
61382a50	313	DATA_TYPE val,
6ebbf390	314	int mmu_idx,
20503968	315	uintptr_t retaddr)
b92e5a22	316	{
355b1943	317	target_phys_addr_t ioaddr;
c27004ec	318	target_ulong tlb_addr;
61382a50	319	int index, i;
b92e5a22	320
b92e5a22 FB	321	index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
b92e5a22 FB	322	redo:
6ebbf390	323	tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
b92e5a22	324	if ((addr & TARGET_PAGE_MASK) == (tlb_addr & (TARGET_PAGE_MASK \| TLB_INVALID_MASK))) {
b92e5a22 FB	325	if (tlb_addr & ~TARGET_PAGE_MASK) {
	326	/* IO access */
	327	if ((addr & (DATA_SIZE - 1)) != 0)
	328	goto do_unaligned_access;
37ec01d4	329	ioaddr = env->iotlb[mmu_idx][index];
e141ab52	330	glue(io_write, SUFFIX)(ENV_VAR ioaddr, val, addr, retaddr);
98699967	331	} else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
b92e5a22 FB	332	do_unaligned_access:
b92e5a22 FB	333	/* XXX: not efficient, but simple */
6c41b272 AZ	334	/* Note: relies on the fact that tlb_fill() does not remove the
6c41b272 AZ	335	* previous page from the TLB cache. */
7221fa98	336	for(i = DATA_SIZE - 1; i >= 0; i--) {
b92e5a22	337	#ifdef TARGET_WORDS_BIGENDIAN
e141ab52 BS	338	glue(slow_stb, MMUSUFFIX)(ENV_VAR addr + i,
e141ab52 BS	339	val >> (((DATA_SIZE - 1) * 8) - (i * 8)),
6ebbf390	340	mmu_idx, retaddr);
b92e5a22	341	#else
e141ab52 BS	342	glue(slow_stb, MMUSUFFIX)(ENV_VAR addr + i,
e141ab52 BS	343	val >> (i * 8),
6ebbf390	344	mmu_idx, retaddr);
b92e5a22 FB	345	#endif
	346	}
	347	} else {
	348	/* aligned/unaligned access in the same page */
b065927a SW	349	uintptr_t addend = env->tlb_table[mmu_idx][index].addend;
	350	glue(glue(st, SUFFIX), _raw)((uint8_t *)(intptr_t)
	351	(addr + addend), val);
b92e5a22 FB	352	}
	353	} else {
	354	/* the page is not in the TLB : fill it */
bccd9ec5	355	tlb_fill(env, addr, 1, mmu_idx, retaddr);
b92e5a22 FB	356	goto redo;
	357	}
	358	}
	359
b769d8fe FB	360	#endif /* !defined(SOFTMMU_CODE_ACCESS) */
	361
	362	#undef READ_ACCESS_TYPE
b92e5a22 FB	363	#undef SHIFT
	364	#undef DATA_TYPE
	365	#undef SUFFIX
61382a50	366	#undef USUFFIX
b92e5a22	367	#undef DATA_SIZE
84b7b8e7	368	#undef ADDR_READ
e141ab52 BS	369	#undef ENV_PARAM
	370	#undef ENV_VAR
	371	#undef CPU_PREFIX
	372	#undef HELPER_PREFIX