#include "exec/memory.h"
#include "exec/cpu_ldst.h"
#include "exec/cputlb.h"
+#include "exec/tb-flush.h"
#include "exec/memory-internal.h"
#include "exec/ram_addr.h"
#include "tcg/tcg.h"
#include "exec/translate-all.h"
#include "trace.h"
#include "tb-hash.h"
-#include "internal.h"
+#include "internal-common.h"
+#include "internal-target.h"
#ifdef CONFIG_PLUGIN
#include "qemu/plugin-memory.h"
#endif
} while (0)
/* run_on_cpu_data.target_ptr should always be big enough for a
- * target_ulong even on 32 bit builds */
-QEMU_BUILD_BUG_ON(sizeof(target_ulong) > sizeof(run_on_cpu_data));
+ * vaddr even on 32 bit builds
+ */
+QEMU_BUILD_BUG_ON(sizeof(vaddr) > sizeof(run_on_cpu_data));
/* We currently can't handle more than 16 bits in the MMUIDX bitmask.
*/
memset(desc->vtable, -1, sizeof(desc->vtable));
}
-static void tlb_flush_one_mmuidx_locked(CPUArchState *env, int mmu_idx,
+static void tlb_flush_one_mmuidx_locked(CPUState *cpu, int mmu_idx,
int64_t now)
{
- CPUTLBDesc *desc = &env_tlb(env)->d[mmu_idx];
- CPUTLBDescFast *fast = &env_tlb(env)->f[mmu_idx];
+ CPUTLBDesc *desc = &cpu->neg.tlb.d[mmu_idx];
+ CPUTLBDescFast *fast = &cpu->neg.tlb.f[mmu_idx];
tlb_mmu_resize_locked(desc, fast, now);
tlb_mmu_flush_locked(desc, fast);
tlb_mmu_flush_locked(desc, fast);
}
-static inline void tlb_n_used_entries_inc(CPUArchState *env, uintptr_t mmu_idx)
+static inline void tlb_n_used_entries_inc(CPUState *cpu, uintptr_t mmu_idx)
{
- env_tlb(env)->d[mmu_idx].n_used_entries++;
+ cpu->neg.tlb.d[mmu_idx].n_used_entries++;
}
-static inline void tlb_n_used_entries_dec(CPUArchState *env, uintptr_t mmu_idx)
+static inline void tlb_n_used_entries_dec(CPUState *cpu, uintptr_t mmu_idx)
{
- env_tlb(env)->d[mmu_idx].n_used_entries--;
+ cpu->neg.tlb.d[mmu_idx].n_used_entries--;
}
void tlb_init(CPUState *cpu)
{
- CPUArchState *env = cpu->env_ptr;
int64_t now = get_clock_realtime();
int i;
- qemu_spin_init(&env_tlb(env)->c.lock);
+ qemu_spin_init(&cpu->neg.tlb.c.lock);
/* All tlbs are initialized flushed. */
- env_tlb(env)->c.dirty = 0;
+ cpu->neg.tlb.c.dirty = 0;
for (i = 0; i < NB_MMU_MODES; i++) {
- tlb_mmu_init(&env_tlb(env)->d[i], &env_tlb(env)->f[i], now);
+ tlb_mmu_init(&cpu->neg.tlb.d[i], &cpu->neg.tlb.f[i], now);
}
}
void tlb_destroy(CPUState *cpu)
{
- CPUArchState *env = cpu->env_ptr;
int i;
- qemu_spin_destroy(&env_tlb(env)->c.lock);
+ qemu_spin_destroy(&cpu->neg.tlb.c.lock);
for (i = 0; i < NB_MMU_MODES; i++) {
- CPUTLBDesc *desc = &env_tlb(env)->d[i];
- CPUTLBDescFast *fast = &env_tlb(env)->f[i];
+ CPUTLBDesc *desc = &cpu->neg.tlb.d[i];
+ CPUTLBDescFast *fast = &cpu->neg.tlb.f[i];
g_free(fast->table);
g_free(desc->fulltlb);
}
}
-void tlb_flush_counts(size_t *pfull, size_t *ppart, size_t *pelide)
-{
- CPUState *cpu;
- size_t full = 0, part = 0, elide = 0;
-
- CPU_FOREACH(cpu) {
- CPUArchState *env = cpu->env_ptr;
-
- full += qatomic_read(&env_tlb(env)->c.full_flush_count);
- part += qatomic_read(&env_tlb(env)->c.part_flush_count);
- elide += qatomic_read(&env_tlb(env)->c.elide_flush_count);
- }
- *pfull = full;
- *ppart = part;
- *pelide = elide;
-}
-
static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)
{
- CPUArchState *env = cpu->env_ptr;
uint16_t asked = data.host_int;
uint16_t all_dirty, work, to_clean;
int64_t now = get_clock_realtime();
tlb_debug("mmu_idx:0x%04" PRIx16 "\n", asked);
- qemu_spin_lock(&env_tlb(env)->c.lock);
+ qemu_spin_lock(&cpu->neg.tlb.c.lock);
- all_dirty = env_tlb(env)->c.dirty;
+ all_dirty = cpu->neg.tlb.c.dirty;
to_clean = asked & all_dirty;
all_dirty &= ~to_clean;
- env_tlb(env)->c.dirty = all_dirty;
+ cpu->neg.tlb.c.dirty = all_dirty;
for (work = to_clean; work != 0; work &= work - 1) {
int mmu_idx = ctz32(work);
- tlb_flush_one_mmuidx_locked(env, mmu_idx, now);
+ tlb_flush_one_mmuidx_locked(cpu, mmu_idx, now);
}
- qemu_spin_unlock(&env_tlb(env)->c.lock);
+ qemu_spin_unlock(&cpu->neg.tlb.c.lock);
tcg_flush_jmp_cache(cpu);
if (to_clean == ALL_MMUIDX_BITS) {
- qatomic_set(&env_tlb(env)->c.full_flush_count,
- env_tlb(env)->c.full_flush_count + 1);
+ qatomic_set(&cpu->neg.tlb.c.full_flush_count,
+ cpu->neg.tlb.c.full_flush_count + 1);
} else {
- qatomic_set(&env_tlb(env)->c.part_flush_count,
- env_tlb(env)->c.part_flush_count + ctpop16(to_clean));
+ qatomic_set(&cpu->neg.tlb.c.part_flush_count,
+ cpu->neg.tlb.c.part_flush_count + ctpop16(to_clean));
if (to_clean != asked) {
- qatomic_set(&env_tlb(env)->c.elide_flush_count,
- env_tlb(env)->c.elide_flush_count +
- ctpop16(asked & ~to_clean));
+ qatomic_set(&cpu->neg.tlb.c.elide_flush_count,
+ cpu->neg.tlb.c.elide_flush_count +
+ ctpop16(asked & ~to_clean));
}
}
}
}
/* Called with tlb_c.lock held */
-static void tlb_flush_vtlb_page_mask_locked(CPUArchState *env, int mmu_idx,
+static void tlb_flush_vtlb_page_mask_locked(CPUState *cpu, int mmu_idx,
vaddr page,
vaddr mask)
{
- CPUTLBDesc *d = &env_tlb(env)->d[mmu_idx];
+ CPUTLBDesc *d = &cpu->neg.tlb.d[mmu_idx];
int k;
- assert_cpu_is_self(env_cpu(env));
+ assert_cpu_is_self(cpu);
for (k = 0; k < CPU_VTLB_SIZE; k++) {
if (tlb_flush_entry_mask_locked(&d->vtable[k], page, mask)) {
- tlb_n_used_entries_dec(env, mmu_idx);
+ tlb_n_used_entries_dec(cpu, mmu_idx);
}
}
}
-static inline void tlb_flush_vtlb_page_locked(CPUArchState *env, int mmu_idx,
+static inline void tlb_flush_vtlb_page_locked(CPUState *cpu, int mmu_idx,
vaddr page)
{
- tlb_flush_vtlb_page_mask_locked(env, mmu_idx, page, -1);
+ tlb_flush_vtlb_page_mask_locked(cpu, mmu_idx, page, -1);
}
-static void tlb_flush_page_locked(CPUArchState *env, int midx, vaddr page)
+static void tlb_flush_page_locked(CPUState *cpu, int midx, vaddr page)
{
- vaddr lp_addr = env_tlb(env)->d[midx].large_page_addr;
- vaddr lp_mask = env_tlb(env)->d[midx].large_page_mask;
+ vaddr lp_addr = cpu->neg.tlb.d[midx].large_page_addr;
+ vaddr lp_mask = cpu->neg.tlb.d[midx].large_page_mask;
/* Check if we need to flush due to large pages. */
if ((page & lp_mask) == lp_addr) {
- tlb_debug("forcing full flush midx %d (%"
- VADDR_PRIx "/%" VADDR_PRIx ")\n",
+ tlb_debug("forcing full flush midx %d (%016"
+ VADDR_PRIx "/%016" VADDR_PRIx ")\n",
midx, lp_addr, lp_mask);
- tlb_flush_one_mmuidx_locked(env, midx, get_clock_realtime());
+ tlb_flush_one_mmuidx_locked(cpu, midx, get_clock_realtime());
} else {
- if (tlb_flush_entry_locked(tlb_entry(env, midx, page), page)) {
- tlb_n_used_entries_dec(env, midx);
+ if (tlb_flush_entry_locked(tlb_entry(cpu, midx, page), page)) {
+ tlb_n_used_entries_dec(cpu, midx);
}
- tlb_flush_vtlb_page_locked(env, midx, page);
+ tlb_flush_vtlb_page_locked(cpu, midx, page);
}
}
vaddr addr,
uint16_t idxmap)
{
- CPUArchState *env = cpu->env_ptr;
int mmu_idx;
assert_cpu_is_self(cpu);
- tlb_debug("page addr: %" VADDR_PRIx " mmu_map:0x%x\n", addr, idxmap);
+ tlb_debug("page addr: %016" VADDR_PRIx " mmu_map:0x%x\n", addr, idxmap);
- qemu_spin_lock(&env_tlb(env)->c.lock);
+ qemu_spin_lock(&cpu->neg.tlb.c.lock);
for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
if ((idxmap >> mmu_idx) & 1) {
- tlb_flush_page_locked(env, mmu_idx, addr);
+ tlb_flush_page_locked(cpu, mmu_idx, addr);
}
}
- qemu_spin_unlock(&env_tlb(env)->c.lock);
+ qemu_spin_unlock(&cpu->neg.tlb.c.lock);
/*
* Discard jump cache entries for any tb which might potentially
void tlb_flush_page_by_mmuidx(CPUState *cpu, vaddr addr, uint16_t idxmap)
{
- tlb_debug("addr: %" VADDR_PRIx " mmu_idx:%" PRIx16 "\n", addr, idxmap);
+ tlb_debug("addr: %016" VADDR_PRIx " mmu_idx:%" PRIx16 "\n", addr, idxmap);
/* This should already be page aligned */
addr &= TARGET_PAGE_MASK;
void tlb_flush_page_by_mmuidx_all_cpus(CPUState *src_cpu, vaddr addr,
uint16_t idxmap)
{
- tlb_debug("addr: %" VADDR_PRIx " mmu_idx:%"PRIx16"\n", addr, idxmap);
+ tlb_debug("addr: %016" VADDR_PRIx " mmu_idx:%"PRIx16"\n", addr, idxmap);
/* This should already be page aligned */
addr &= TARGET_PAGE_MASK;
vaddr addr,
uint16_t idxmap)
{
- tlb_debug("addr: %" VADDR_PRIx " mmu_idx:%"PRIx16"\n", addr, idxmap);
+ tlb_debug("addr: %016" VADDR_PRIx " mmu_idx:%"PRIx16"\n", addr, idxmap);
/* This should already be page aligned */
addr &= TARGET_PAGE_MASK;
tlb_flush_page_by_mmuidx_all_cpus_synced(src, addr, ALL_MMUIDX_BITS);
}
-static void tlb_flush_range_locked(CPUArchState *env, int midx,
+static void tlb_flush_range_locked(CPUState *cpu, int midx,
vaddr addr, vaddr len,
unsigned bits)
{
- CPUTLBDesc *d = &env_tlb(env)->d[midx];
- CPUTLBDescFast *f = &env_tlb(env)->f[midx];
+ CPUTLBDesc *d = &cpu->neg.tlb.d[midx];
+ CPUTLBDescFast *f = &cpu->neg.tlb.f[midx];
vaddr mask = MAKE_64BIT_MASK(0, bits);
/*
*/
if (mask < f->mask || len > f->mask) {
tlb_debug("forcing full flush midx %d ("
- "%" VADDR_PRIx "/%" VADDR_PRIx "+%" VADDR_PRIx ")\n",
+ "%016" VADDR_PRIx "/%016" VADDR_PRIx "+%016" VADDR_PRIx ")\n",
midx, addr, mask, len);
- tlb_flush_one_mmuidx_locked(env, midx, get_clock_realtime());
+ tlb_flush_one_mmuidx_locked(cpu, midx, get_clock_realtime());
return;
}
*/
if (((addr + len - 1) & d->large_page_mask) == d->large_page_addr) {
tlb_debug("forcing full flush midx %d ("
- "%" VADDR_PRIx "/%" VADDR_PRIx ")\n",
+ "%016" VADDR_PRIx "/%016" VADDR_PRIx ")\n",
midx, d->large_page_addr, d->large_page_mask);
- tlb_flush_one_mmuidx_locked(env, midx, get_clock_realtime());
+ tlb_flush_one_mmuidx_locked(cpu, midx, get_clock_realtime());
return;
}
for (vaddr i = 0; i < len; i += TARGET_PAGE_SIZE) {
vaddr page = addr + i;
- CPUTLBEntry *entry = tlb_entry(env, midx, page);
+ CPUTLBEntry *entry = tlb_entry(cpu, midx, page);
if (tlb_flush_entry_mask_locked(entry, page, mask)) {
- tlb_n_used_entries_dec(env, midx);
+ tlb_n_used_entries_dec(cpu, midx);
}
- tlb_flush_vtlb_page_mask_locked(env, midx, page, mask);
+ tlb_flush_vtlb_page_mask_locked(cpu, midx, page, mask);
}
}
static void tlb_flush_range_by_mmuidx_async_0(CPUState *cpu,
TLBFlushRangeData d)
{
- CPUArchState *env = cpu->env_ptr;
int mmu_idx;
assert_cpu_is_self(cpu);
- tlb_debug("range: %" VADDR_PRIx "/%u+%" VADDR_PRIx " mmu_map:0x%x\n",
+ tlb_debug("range: %016" VADDR_PRIx "/%u+%016" VADDR_PRIx " mmu_map:0x%x\n",
d.addr, d.bits, d.len, d.idxmap);
- qemu_spin_lock(&env_tlb(env)->c.lock);
+ qemu_spin_lock(&cpu->neg.tlb.c.lock);
for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
if ((d.idxmap >> mmu_idx) & 1) {
- tlb_flush_range_locked(env, mmu_idx, d.addr, d.len, d.bits);
+ tlb_flush_range_locked(cpu, mmu_idx, d.addr, d.len, d.bits);
}
}
- qemu_spin_unlock(&env_tlb(env)->c.lock);
+ qemu_spin_unlock(&cpu->neg.tlb.c.lock);
/*
* If the length is larger than the jump cache size, then it will take
*/
void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
{
- CPUArchState *env;
-
int mmu_idx;
- env = cpu->env_ptr;
- qemu_spin_lock(&env_tlb(env)->c.lock);
+ qemu_spin_lock(&cpu->neg.tlb.c.lock);
for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
unsigned int i;
- unsigned int n = tlb_n_entries(&env_tlb(env)->f[mmu_idx]);
+ unsigned int n = tlb_n_entries(&cpu->neg.tlb.f[mmu_idx]);
for (i = 0; i < n; i++) {
- tlb_reset_dirty_range_locked(&env_tlb(env)->f[mmu_idx].table[i],
+ tlb_reset_dirty_range_locked(&cpu->neg.tlb.f[mmu_idx].table[i],
start1, length);
}
for (i = 0; i < CPU_VTLB_SIZE; i++) {
- tlb_reset_dirty_range_locked(&env_tlb(env)->d[mmu_idx].vtable[i],
+ tlb_reset_dirty_range_locked(&cpu->neg.tlb.d[mmu_idx].vtable[i],
start1, length);
}
}
- qemu_spin_unlock(&env_tlb(env)->c.lock);
+ qemu_spin_unlock(&cpu->neg.tlb.c.lock);
}
/* Called with tlb_c.lock held */
so that it is no longer dirty */
void tlb_set_dirty(CPUState *cpu, vaddr addr)
{
- CPUArchState *env = cpu->env_ptr;
int mmu_idx;
assert_cpu_is_self(cpu);
addr &= TARGET_PAGE_MASK;
- qemu_spin_lock(&env_tlb(env)->c.lock);
+ qemu_spin_lock(&cpu->neg.tlb.c.lock);
for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
- tlb_set_dirty1_locked(tlb_entry(env, mmu_idx, addr), addr);
+ tlb_set_dirty1_locked(tlb_entry(cpu, mmu_idx, addr), addr);
}
for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
int k;
for (k = 0; k < CPU_VTLB_SIZE; k++) {
- tlb_set_dirty1_locked(&env_tlb(env)->d[mmu_idx].vtable[k], addr);
+ tlb_set_dirty1_locked(&cpu->neg.tlb.d[mmu_idx].vtable[k], addr);
}
}
- qemu_spin_unlock(&env_tlb(env)->c.lock);
+ qemu_spin_unlock(&cpu->neg.tlb.c.lock);
}
/* Our TLB does not support large pages, so remember the area covered by
large pages and trigger a full TLB flush if these are invalidated. */
-static void tlb_add_large_page(CPUArchState *env, int mmu_idx,
+static void tlb_add_large_page(CPUState *cpu, int mmu_idx,
vaddr addr, uint64_t size)
{
- vaddr lp_addr = env_tlb(env)->d[mmu_idx].large_page_addr;
+ vaddr lp_addr = cpu->neg.tlb.d[mmu_idx].large_page_addr;
vaddr lp_mask = ~(size - 1);
if (lp_addr == (vaddr)-1) {
/* Extend the existing region to include the new page.
This is a compromise between unnecessary flushes and
the cost of maintaining a full variable size TLB. */
- lp_mask &= env_tlb(env)->d[mmu_idx].large_page_mask;
+ lp_mask &= cpu->neg.tlb.d[mmu_idx].large_page_mask;
while (((lp_addr ^ addr) & lp_mask) != 0) {
lp_mask <<= 1;
}
}
- env_tlb(env)->d[mmu_idx].large_page_addr = lp_addr & lp_mask;
- env_tlb(env)->d[mmu_idx].large_page_mask = lp_mask;
+ cpu->neg.tlb.d[mmu_idx].large_page_addr = lp_addr & lp_mask;
+ cpu->neg.tlb.d[mmu_idx].large_page_mask = lp_mask;
+}
+
+static inline void tlb_set_compare(CPUTLBEntryFull *full, CPUTLBEntry *ent,
+ vaddr address, int flags,
+ MMUAccessType access_type, bool enable)
+{
+ if (enable) {
+ address |= flags & TLB_FLAGS_MASK;
+ flags &= TLB_SLOW_FLAGS_MASK;
+ if (flags) {
+ address |= TLB_FORCE_SLOW;
+ }
+ } else {
+ address = -1;
+ flags = 0;
+ }
+ ent->addr_idx[access_type] = address;
+ full->slow_flags[access_type] = flags;
}
/*
void tlb_set_page_full(CPUState *cpu, int mmu_idx,
vaddr addr, CPUTLBEntryFull *full)
{
- CPUArchState *env = cpu->env_ptr;
- CPUTLB *tlb = env_tlb(env);
+ CPUTLB *tlb = &cpu->neg.tlb;
CPUTLBDesc *desc = &tlb->d[mmu_idx];
MemoryRegionSection *section;
- unsigned int index;
- vaddr address;
- vaddr write_address;
+ unsigned int index, read_flags, write_flags;
uintptr_t addend;
CPUTLBEntry *te, tn;
hwaddr iotlb, xlat, sz, paddr_page;
sz = TARGET_PAGE_SIZE;
} else {
sz = (hwaddr)1 << full->lg_page_size;
- tlb_add_large_page(env, mmu_idx, addr, sz);
+ tlb_add_large_page(cpu, mmu_idx, addr, sz);
}
addr_page = addr & TARGET_PAGE_MASK;
paddr_page = full->phys_addr & TARGET_PAGE_MASK;
&xlat, &sz, full->attrs, &prot);
assert(sz >= TARGET_PAGE_SIZE);
- tlb_debug("vaddr=%" VADDR_PRIx " paddr=0x" HWADDR_FMT_plx
+ tlb_debug("vaddr=%016" VADDR_PRIx " paddr=0x" HWADDR_FMT_plx
" prot=%x idx=%d\n",
addr, full->phys_addr, prot, mmu_idx);
- address = addr_page;
+ read_flags = 0;
if (full->lg_page_size < TARGET_PAGE_BITS) {
/* Repeat the MMU check and TLB fill on every access. */
- address |= TLB_INVALID_MASK;
+ read_flags |= TLB_INVALID_MASK;
}
if (full->attrs.byte_swap) {
- address |= TLB_BSWAP;
+ read_flags |= TLB_BSWAP;
}
is_ram = memory_region_is_ram(section->mr);
addend = 0;
}
- write_address = address;
+ write_flags = read_flags;
if (is_ram) {
iotlb = memory_region_get_ram_addr(section->mr) + xlat;
+ assert(!(iotlb & ~TARGET_PAGE_MASK));
/*
* Computing is_clean is expensive; avoid all that unless
* the page is actually writable.
*/
if (prot & PAGE_WRITE) {
if (section->readonly) {
- write_address |= TLB_DISCARD_WRITE;
+ write_flags |= TLB_DISCARD_WRITE;
} else if (cpu_physical_memory_is_clean(iotlb)) {
- write_address |= TLB_NOTDIRTY;
+ write_flags |= TLB_NOTDIRTY;
}
}
} else {
* Reads to romd devices go through the ram_ptr found above,
* but of course reads to I/O must go through MMIO.
*/
- write_address |= TLB_MMIO;
+ write_flags |= TLB_MMIO;
if (!is_romd) {
- address = write_address;
+ read_flags = write_flags;
}
}
wp_flags = cpu_watchpoint_address_matches(cpu, addr_page,
TARGET_PAGE_SIZE);
- index = tlb_index(env, mmu_idx, addr_page);
- te = tlb_entry(env, mmu_idx, addr_page);
+ index = tlb_index(cpu, mmu_idx, addr_page);
+ te = tlb_entry(cpu, mmu_idx, addr_page);
/*
* Hold the TLB lock for the rest of the function. We could acquire/release
tlb->c.dirty |= 1 << mmu_idx;
/* Make sure there's no cached translation for the new page. */
- tlb_flush_vtlb_page_locked(env, mmu_idx, addr_page);
+ tlb_flush_vtlb_page_locked(cpu, mmu_idx, addr_page);
/*
* Only evict the old entry to the victim tlb if it's for a
/* Evict the old entry into the victim tlb. */
copy_tlb_helper_locked(tv, te);
desc->vfulltlb[vidx] = desc->fulltlb[index];
- tlb_n_used_entries_dec(env, mmu_idx);
+ tlb_n_used_entries_dec(cpu, mmu_idx);
}
/* refill the tlb */
/*
- * At this point iotlb contains a physical section number in the lower
- * TARGET_PAGE_BITS, and either
- * + the ram_addr_t of the page base of the target RAM (RAM)
- * + the offset within section->mr of the page base (I/O, ROMD)
- * We subtract the vaddr_page (which is page aligned and thus won't
+ * When memory region is ram, iotlb contains a TARGET_PAGE_BITS
+ * aligned ram_addr_t of the page base of the target RAM.
+ * Otherwise, iotlb contains
+ * - a physical section number in the lower TARGET_PAGE_BITS
+ * - the offset within section->mr of the page base (I/O, ROMD) with the
+ * TARGET_PAGE_BITS masked off.
+ * We subtract addr_page (which is page aligned and thus won't
* disturb the low bits) to give an offset which can be added to the
* (non-page-aligned) vaddr of the eventual memory access to get
* the MemoryRegion offset for the access. Note that the vaddr we
* subtract here is that of the page base, and not the same as the
- * vaddr we add back in io_readx()/io_writex()/get_page_addr_code().
+ * vaddr we add back in io_prepare()/get_page_addr_code().
*/
desc->fulltlb[index] = *full;
- desc->fulltlb[index].xlat_section = iotlb - addr_page;
- desc->fulltlb[index].phys_addr = paddr_page;
+ full = &desc->fulltlb[index];
+ full->xlat_section = iotlb - addr_page;
+ full->phys_addr = paddr_page;
/* Now calculate the new entry */
tn.addend = addend - addr_page;
- if (prot & PAGE_READ) {
- tn.addr_read = address;
- if (wp_flags & BP_MEM_READ) {
- tn.addr_read |= TLB_WATCHPOINT;
- }
- } else {
- tn.addr_read = -1;
- }
- if (prot & PAGE_EXEC) {
- tn.addr_code = address;
- } else {
- tn.addr_code = -1;
+ tlb_set_compare(full, &tn, addr_page, read_flags,
+ MMU_INST_FETCH, prot & PAGE_EXEC);
+
+ if (wp_flags & BP_MEM_READ) {
+ read_flags |= TLB_WATCHPOINT;
}
+ tlb_set_compare(full, &tn, addr_page, read_flags,
+ MMU_DATA_LOAD, prot & PAGE_READ);
- tn.addr_write = -1;
- if (prot & PAGE_WRITE) {
- tn.addr_write = write_address;
- if (prot & PAGE_WRITE_INV) {
- tn.addr_write |= TLB_INVALID_MASK;
- }
- if (wp_flags & BP_MEM_WRITE) {
- tn.addr_write |= TLB_WATCHPOINT;
- }
+ if (prot & PAGE_WRITE_INV) {
+ write_flags |= TLB_INVALID_MASK;
}
+ if (wp_flags & BP_MEM_WRITE) {
+ write_flags |= TLB_WATCHPOINT;
+ }
+ tlb_set_compare(full, &tn, addr_page, write_flags,
+ MMU_DATA_STORE, prot & PAGE_WRITE);
copy_tlb_helper_locked(te, &tn);
- tlb_n_used_entries_inc(env, mmu_idx);
+ tlb_n_used_entries_inc(cpu, mmu_idx);
qemu_spin_unlock(&tlb->c.lock);
}
mmu_idx, retaddr);
}
-static inline void cpu_transaction_failed(CPUState *cpu, hwaddr physaddr,
- vaddr addr, unsigned size,
- MMUAccessType access_type,
- int mmu_idx, MemTxAttrs attrs,
- MemTxResult response,
- uintptr_t retaddr)
+static MemoryRegionSection *
+io_prepare(hwaddr *out_offset, CPUState *cpu, hwaddr xlat,
+ MemTxAttrs attrs, vaddr addr, uintptr_t retaddr)
{
- CPUClass *cc = CPU_GET_CLASS(cpu);
-
- if (!cpu->ignore_memory_transaction_failures &&
- cc->tcg_ops->do_transaction_failed) {
- cc->tcg_ops->do_transaction_failed(cpu, physaddr, addr, size,
- access_type, mmu_idx, attrs,
- response, retaddr);
- }
-}
-
-static uint64_t io_readx(CPUArchState *env, CPUTLBEntryFull *full,
- int mmu_idx, vaddr addr, uintptr_t retaddr,
- MMUAccessType access_type, MemOp op)
-{
- CPUState *cpu = env_cpu(env);
- hwaddr mr_offset;
MemoryRegionSection *section;
- MemoryRegion *mr;
- uint64_t val;
- MemTxResult r;
+ hwaddr mr_offset;
- section = iotlb_to_section(cpu, full->xlat_section, full->attrs);
- mr = section->mr;
- mr_offset = (full->xlat_section & TARGET_PAGE_MASK) + addr;
+ section = iotlb_to_section(cpu, xlat, attrs);
+ mr_offset = (xlat & TARGET_PAGE_MASK) + addr;
cpu->mem_io_pc = retaddr;
- if (!cpu->can_do_io) {
+ if (!cpu->neg.can_do_io) {
cpu_io_recompile(cpu, retaddr);
}
- {
- QEMU_IOTHREAD_LOCK_GUARD();
- r = memory_region_dispatch_read(mr, mr_offset, &val, op, full->attrs);
- }
-
- if (r != MEMTX_OK) {
- hwaddr physaddr = mr_offset +
- section->offset_within_address_space -
- section->offset_within_region;
-
- cpu_transaction_failed(cpu, physaddr, addr, memop_size(op), access_type,
- mmu_idx, full->attrs, r, retaddr);
- }
- return val;
-}
-
-/*
- * Save a potentially trashed CPUTLBEntryFull for later lookup by plugin.
- * This is read by tlb_plugin_lookup if the fulltlb entry doesn't match
- * because of the side effect of io_writex changing memory layout.
- */
-static void save_iotlb_data(CPUState *cs, MemoryRegionSection *section,
- hwaddr mr_offset)
-{
-#ifdef CONFIG_PLUGIN
- SavedIOTLB *saved = &cs->saved_iotlb;
- saved->section = section;
- saved->mr_offset = mr_offset;
-#endif
+ *out_offset = mr_offset;
+ return section;
}
-static void io_writex(CPUArchState *env, CPUTLBEntryFull *full,
- int mmu_idx, uint64_t val, vaddr addr,
- uintptr_t retaddr, MemOp op)
+static void io_failed(CPUState *cpu, CPUTLBEntryFull *full, vaddr addr,
+ unsigned size, MMUAccessType access_type, int mmu_idx,
+ MemTxResult response, uintptr_t retaddr)
{
- CPUState *cpu = env_cpu(env);
- hwaddr mr_offset;
- MemoryRegionSection *section;
- MemoryRegion *mr;
- MemTxResult r;
-
- section = iotlb_to_section(cpu, full->xlat_section, full->attrs);
- mr = section->mr;
- mr_offset = (full->xlat_section & TARGET_PAGE_MASK) + addr;
- if (!cpu->can_do_io) {
- cpu_io_recompile(cpu, retaddr);
- }
- cpu->mem_io_pc = retaddr;
-
- /*
- * The memory_region_dispatch may trigger a flush/resize
- * so for plugins we save the iotlb_data just in case.
- */
- save_iotlb_data(cpu, section, mr_offset);
-
- {
- QEMU_IOTHREAD_LOCK_GUARD();
- r = memory_region_dispatch_write(mr, mr_offset, val, op, full->attrs);
- }
-
- if (r != MEMTX_OK) {
- hwaddr physaddr = mr_offset +
- section->offset_within_address_space -
- section->offset_within_region;
+ if (!cpu->ignore_memory_transaction_failures
+ && cpu->cc->tcg_ops->do_transaction_failed) {
+ hwaddr physaddr = full->phys_addr | (addr & ~TARGET_PAGE_MASK);
- cpu_transaction_failed(cpu, physaddr, addr, memop_size(op),
- MMU_DATA_STORE, mmu_idx, full->attrs, r,
- retaddr);
+ cpu->cc->tcg_ops->do_transaction_failed(cpu, physaddr, addr, size,
+ access_type, mmu_idx,
+ full->attrs, response, retaddr);
}
}
/* Return true if ADDR is present in the victim tlb, and has been copied
back to the main tlb. */
-static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
+static bool victim_tlb_hit(CPUState *cpu, size_t mmu_idx, size_t index,
MMUAccessType access_type, vaddr page)
{
size_t vidx;
- assert_cpu_is_self(env_cpu(env));
+ assert_cpu_is_self(cpu);
for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) {
- CPUTLBEntry *vtlb = &env_tlb(env)->d[mmu_idx].vtable[vidx];
+ CPUTLBEntry *vtlb = &cpu->neg.tlb.d[mmu_idx].vtable[vidx];
uint64_t cmp = tlb_read_idx(vtlb, access_type);
if (cmp == page) {
/* Found entry in victim tlb, swap tlb and iotlb. */
- CPUTLBEntry tmptlb, *tlb = &env_tlb(env)->f[mmu_idx].table[index];
+ CPUTLBEntry tmptlb, *tlb = &cpu->neg.tlb.f[mmu_idx].table[index];
- qemu_spin_lock(&env_tlb(env)->c.lock);
+ qemu_spin_lock(&cpu->neg.tlb.c.lock);
copy_tlb_helper_locked(&tmptlb, tlb);
copy_tlb_helper_locked(tlb, vtlb);
copy_tlb_helper_locked(vtlb, &tmptlb);
- qemu_spin_unlock(&env_tlb(env)->c.lock);
+ qemu_spin_unlock(&cpu->neg.tlb.c.lock);
- CPUTLBEntryFull *f1 = &env_tlb(env)->d[mmu_idx].fulltlb[index];
- CPUTLBEntryFull *f2 = &env_tlb(env)->d[mmu_idx].vfulltlb[vidx];
+ CPUTLBEntryFull *f1 = &cpu->neg.tlb.d[mmu_idx].fulltlb[index];
+ CPUTLBEntryFull *f2 = &cpu->neg.tlb.d[mmu_idx].vfulltlb[vidx];
CPUTLBEntryFull tmpf;
tmpf = *f1; *f1 = *f2; *f2 = tmpf;
return true;
}
}
-static int probe_access_internal(CPUArchState *env, vaddr addr,
+static int probe_access_internal(CPUState *cpu, vaddr addr,
int fault_size, MMUAccessType access_type,
int mmu_idx, bool nonfault,
void **phost, CPUTLBEntryFull **pfull,
- uintptr_t retaddr)
+ uintptr_t retaddr, bool check_mem_cbs)
{
- uintptr_t index = tlb_index(env, mmu_idx, addr);
- CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
+ uintptr_t index = tlb_index(cpu, mmu_idx, addr);
+ CPUTLBEntry *entry = tlb_entry(cpu, mmu_idx, addr);
uint64_t tlb_addr = tlb_read_idx(entry, access_type);
vaddr page_addr = addr & TARGET_PAGE_MASK;
- int flags = TLB_FLAGS_MASK;
+ int flags = TLB_FLAGS_MASK & ~TLB_FORCE_SLOW;
+ bool force_mmio = check_mem_cbs && cpu_plugin_mem_cbs_enabled(cpu);
+ CPUTLBEntryFull *full;
if (!tlb_hit_page(tlb_addr, page_addr)) {
- if (!victim_tlb_hit(env, mmu_idx, index, access_type, page_addr)) {
- CPUState *cs = env_cpu(env);
-
- if (!cs->cc->tcg_ops->tlb_fill(cs, addr, fault_size, access_type,
- mmu_idx, nonfault, retaddr)) {
+ if (!victim_tlb_hit(cpu, mmu_idx, index, access_type, page_addr)) {
+ if (!cpu->cc->tcg_ops->tlb_fill(cpu, addr, fault_size, access_type,
+ mmu_idx, nonfault, retaddr)) {
/* Non-faulting page table read failed. */
*phost = NULL;
*pfull = NULL;
}
/* TLB resize via tlb_fill may have moved the entry. */
- index = tlb_index(env, mmu_idx, addr);
- entry = tlb_entry(env, mmu_idx, addr);
+ index = tlb_index(cpu, mmu_idx, addr);
+ entry = tlb_entry(cpu, mmu_idx, addr);
/*
* With PAGE_WRITE_INV, we set TLB_INVALID_MASK immediately,
}
flags &= tlb_addr;
- *pfull = &env_tlb(env)->d[mmu_idx].fulltlb[index];
+ *pfull = full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index];
+ flags |= full->slow_flags[access_type];
/* Fold all "mmio-like" bits into TLB_MMIO. This is not RAM. */
- if (unlikely(flags & ~(TLB_WATCHPOINT | TLB_NOTDIRTY))) {
+ if (unlikely(flags & ~(TLB_WATCHPOINT | TLB_NOTDIRTY))
+ ||
+ (access_type != MMU_INST_FETCH && force_mmio)) {
*phost = NULL;
return TLB_MMIO;
}
bool nonfault, void **phost, CPUTLBEntryFull **pfull,
uintptr_t retaddr)
{
- int flags = probe_access_internal(env, addr, size, access_type, mmu_idx,
- nonfault, phost, pfull, retaddr);
+ int flags = probe_access_internal(env_cpu(env), addr, size, access_type,
+ mmu_idx, nonfault, phost, pfull, retaddr,
+ true);
/* Handle clean RAM pages. */
if (unlikely(flags & TLB_NOTDIRTY)) {
- notdirty_write(env_cpu(env), addr, 1, *pfull, retaddr);
+ int dirtysize = size == 0 ? 1 : size;
+ notdirty_write(env_cpu(env), addr, dirtysize, *pfull, retaddr);
+ flags &= ~TLB_NOTDIRTY;
+ }
+
+ return flags;
+}
+
+int probe_access_full_mmu(CPUArchState *env, vaddr addr, int size,
+ MMUAccessType access_type, int mmu_idx,
+ void **phost, CPUTLBEntryFull **pfull)
+{
+ void *discard_phost;
+ CPUTLBEntryFull *discard_tlb;
+
+ /* privately handle users that don't need full results */
+ phost = phost ? phost : &discard_phost;
+ pfull = pfull ? pfull : &discard_tlb;
+
+ int flags = probe_access_internal(env_cpu(env), addr, size, access_type,
+ mmu_idx, true, phost, pfull, 0, false);
+
+ /* Handle clean RAM pages. */
+ if (unlikely(flags & TLB_NOTDIRTY)) {
+ int dirtysize = size == 0 ? 1 : size;
+ notdirty_write(env_cpu(env), addr, dirtysize, *pfull, 0);
flags &= ~TLB_NOTDIRTY;
}
g_assert(-(addr | TARGET_PAGE_MASK) >= size);
- flags = probe_access_internal(env, addr, size, access_type, mmu_idx,
- nonfault, phost, &full, retaddr);
+ flags = probe_access_internal(env_cpu(env), addr, size, access_type,
+ mmu_idx, nonfault, phost, &full, retaddr,
+ true);
/* Handle clean RAM pages. */
if (unlikely(flags & TLB_NOTDIRTY)) {
- notdirty_write(env_cpu(env), addr, 1, full, retaddr);
+ int dirtysize = size == 0 ? 1 : size;
+ notdirty_write(env_cpu(env), addr, dirtysize, full, retaddr);
flags &= ~TLB_NOTDIRTY;
}
g_assert(-(addr | TARGET_PAGE_MASK) >= size);
- flags = probe_access_internal(env, addr, size, access_type, mmu_idx,
- false, &host, &full, retaddr);
+ flags = probe_access_internal(env_cpu(env), addr, size, access_type,
+ mmu_idx, false, &host, &full, retaddr,
+ true);
/* Per the interface, size == 0 merely faults the access. */
if (size == 0) {
/* Handle clean RAM pages. */
if (flags & TLB_NOTDIRTY) {
- notdirty_write(env_cpu(env), addr, 1, full, retaddr);
+ notdirty_write(env_cpu(env), addr, size, full, retaddr);
}
}
void *host;
int flags;
- flags = probe_access_internal(env, addr, 0, access_type,
- mmu_idx, true, &host, &full, 0);
+ flags = probe_access_internal(env_cpu(env), addr, 0, access_type,
+ mmu_idx, true, &host, &full, 0, false);
/* No combination of flags are expected by the caller. */
return flags ? NULL : host;
CPUTLBEntryFull *full;
void *p;
- (void)probe_access_internal(env, addr, 1, MMU_INST_FETCH,
- cpu_mmu_index(env, true), false, &p, &full, 0);
+ (void)probe_access_internal(env_cpu(env), addr, 1, MMU_INST_FETCH,
+ cpu_mmu_index(env, true), false,
+ &p, &full, 0, false);
if (p == NULL) {
return -1;
}
* in the softmmu lookup code (or helper). We don't handle re-fills or
* checking the victim table. This is purely informational.
*
- * This almost never fails as the memory access being instrumented
- * should have just filled the TLB. The one corner case is io_writex
- * which can cause TLB flushes and potential resizing of the TLBs
- * losing the information we need. In those cases we need to recover
- * data from a copy of the CPUTLBEntryFull. As long as this always occurs
- * from the same thread (which a mem callback will be) this is safe.
+ * The one corner case is i/o write, which can cause changes to the
+ * address space. Those changes, and the corresponding tlb flush,
+ * should be delayed until the next TB, so even then this ought not fail.
+ * But check, Just in Case.
*/
-
bool tlb_plugin_lookup(CPUState *cpu, vaddr addr, int mmu_idx,
bool is_store, struct qemu_plugin_hwaddr *data)
{
- CPUArchState *env = cpu->env_ptr;
- CPUTLBEntry *tlbe = tlb_entry(env, mmu_idx, addr);
- uintptr_t index = tlb_index(env, mmu_idx, addr);
- uint64_t tlb_addr = is_store ? tlb_addr_write(tlbe) : tlbe->addr_read;
-
- if (likely(tlb_hit(tlb_addr, addr))) {
- /* We must have an iotlb entry for MMIO */
- if (tlb_addr & TLB_MMIO) {
- CPUTLBEntryFull *full;
- full = &env_tlb(env)->d[mmu_idx].fulltlb[index];
- data->is_io = true;
- data->v.io.section =
- iotlb_to_section(cpu, full->xlat_section, full->attrs);
- data->v.io.offset = (full->xlat_section & TARGET_PAGE_MASK) + addr;
- } else {
- data->is_io = false;
- data->v.ram.hostaddr = (void *)((uintptr_t)addr + tlbe->addend);
- }
- return true;
- } else {
- SavedIOTLB *saved = &cpu->saved_iotlb;
+ CPUTLBEntry *tlbe = tlb_entry(cpu, mmu_idx, addr);
+ uintptr_t index = tlb_index(cpu, mmu_idx, addr);
+ MMUAccessType access_type = is_store ? MMU_DATA_STORE : MMU_DATA_LOAD;
+ uint64_t tlb_addr = tlb_read_idx(tlbe, access_type);
+ CPUTLBEntryFull *full;
+
+ if (unlikely(!tlb_hit(tlb_addr, addr))) {
+ return false;
+ }
+
+ full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index];
+ data->phys_addr = full->phys_addr | (addr & ~TARGET_PAGE_MASK);
+
+ /* We must have an iotlb entry for MMIO */
+ if (tlb_addr & TLB_MMIO) {
+ MemoryRegionSection *section =
+ iotlb_to_section(cpu, full->xlat_section & ~TARGET_PAGE_MASK,
+ full->attrs);
data->is_io = true;
- data->v.io.section = saved->section;
- data->v.io.offset = saved->mr_offset;
- return true;
+ data->mr = section->mr;
+ } else {
+ data->is_io = false;
+ data->mr = NULL;
}
+ return true;
}
-
#endif
/*
/**
* mmu_lookup1: translate one page
- * @env: cpu context
+ * @cpu: generic cpu state
* @data: lookup parameters
* @mmu_idx: virtual address context
* @access_type: load/store/code
* tlb_fill will longjmp out. Return true if the softmmu tlb for
* @mmu_idx may have resized.
*/
-static bool mmu_lookup1(CPUArchState *env, MMULookupPageData *data,
+static bool mmu_lookup1(CPUState *cpu, MMULookupPageData *data,
int mmu_idx, MMUAccessType access_type, uintptr_t ra)
{
vaddr addr = data->addr;
- uintptr_t index = tlb_index(env, mmu_idx, addr);
- CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
+ uintptr_t index = tlb_index(cpu, mmu_idx, addr);
+ CPUTLBEntry *entry = tlb_entry(cpu, mmu_idx, addr);
uint64_t tlb_addr = tlb_read_idx(entry, access_type);
bool maybe_resized = false;
+ CPUTLBEntryFull *full;
+ int flags;
/* If the TLB entry is for a different page, reload and try again. */
if (!tlb_hit(tlb_addr, addr)) {
- if (!victim_tlb_hit(env, mmu_idx, index, access_type,
+ if (!victim_tlb_hit(cpu, mmu_idx, index, access_type,
addr & TARGET_PAGE_MASK)) {
- tlb_fill(env_cpu(env), addr, data->size, access_type, mmu_idx, ra);
+ tlb_fill(cpu, addr, data->size, access_type, mmu_idx, ra);
maybe_resized = true;
- index = tlb_index(env, mmu_idx, addr);
- entry = tlb_entry(env, mmu_idx, addr);
+ index = tlb_index(cpu, mmu_idx, addr);
+ entry = tlb_entry(cpu, mmu_idx, addr);
}
tlb_addr = tlb_read_idx(entry, access_type) & ~TLB_INVALID_MASK;
}
- data->flags = tlb_addr & TLB_FLAGS_MASK;
- data->full = &env_tlb(env)->d[mmu_idx].fulltlb[index];
+ full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index];
+ flags = tlb_addr & (TLB_FLAGS_MASK & ~TLB_FORCE_SLOW);
+ flags |= full->slow_flags[access_type];
+
+ data->full = full;
+ data->flags = flags;
/* Compute haddr speculatively; depending on flags it might be invalid. */
data->haddr = (void *)((uintptr_t)addr + entry->addend);
/**
* mmu_watch_or_dirty
- * @env: cpu context
+ * @cpu: generic cpu state
* @data: lookup parameters
* @access_type: load/store/code
* @ra: return address into tcg generated code, or 0
* Trigger watchpoints for @data.addr:@data.size;
* record writes to protected clean pages.
*/
-static void mmu_watch_or_dirty(CPUArchState *env, MMULookupPageData *data,
+static void mmu_watch_or_dirty(CPUState *cpu, MMULookupPageData *data,
MMUAccessType access_type, uintptr_t ra)
{
CPUTLBEntryFull *full = data->full;
/* On watchpoint hit, this will longjmp out. */
if (flags & TLB_WATCHPOINT) {
int wp = access_type == MMU_DATA_STORE ? BP_MEM_WRITE : BP_MEM_READ;
- cpu_check_watchpoint(env_cpu(env), addr, size, full->attrs, wp, ra);
+ cpu_check_watchpoint(cpu, addr, size, full->attrs, wp, ra);
flags &= ~TLB_WATCHPOINT;
}
/* Note that notdirty is only set for writes. */
if (flags & TLB_NOTDIRTY) {
- notdirty_write(env_cpu(env), addr, size, full, ra);
+ notdirty_write(cpu, addr, size, full, ra);
flags &= ~TLB_NOTDIRTY;
}
data->flags = flags;
/**
* mmu_lookup: translate page(s)
- * @env: cpu context
+ * @cpu: generic cpu state
* @addr: virtual address
* @oi: combined mmu_idx and MemOp
* @ra: return address into tcg generated code, or 0
* Resolve the translation for the page(s) beginning at @addr, for MemOp.size
* bytes. Return true if the lookup crosses a page boundary.
*/
-static bool mmu_lookup(CPUArchState *env, vaddr addr, MemOpIdx oi,
+static bool mmu_lookup(CPUState *cpu, vaddr addr, MemOpIdx oi,
uintptr_t ra, MMUAccessType type, MMULookupLocals *l)
{
unsigned a_bits;
/* Handle CPU specific unaligned behaviour */
a_bits = get_alignment_bits(l->memop);
if (addr & ((1 << a_bits) - 1)) {
- cpu_unaligned_access(env_cpu(env), addr, type, l->mmu_idx, ra);
+ cpu_unaligned_access(cpu, addr, type, l->mmu_idx, ra);
}
l->page[0].addr = addr;
crosspage = (addr ^ l->page[1].addr) & TARGET_PAGE_MASK;
if (likely(!crosspage)) {
- mmu_lookup1(env, &l->page[0], l->mmu_idx, type, ra);
+ mmu_lookup1(cpu, &l->page[0], l->mmu_idx, type, ra);
flags = l->page[0].flags;
if (unlikely(flags & (TLB_WATCHPOINT | TLB_NOTDIRTY))) {
- mmu_watch_or_dirty(env, &l->page[0], type, ra);
+ mmu_watch_or_dirty(cpu, &l->page[0], type, ra);
}
if (unlikely(flags & TLB_BSWAP)) {
l->memop ^= MO_BSWAP;
* Lookup both pages, recognizing exceptions from either. If the
* second lookup potentially resized, refresh first CPUTLBEntryFull.
*/
- mmu_lookup1(env, &l->page[0], l->mmu_idx, type, ra);
- if (mmu_lookup1(env, &l->page[1], l->mmu_idx, type, ra)) {
- uintptr_t index = tlb_index(env, l->mmu_idx, addr);
- l->page[0].full = &env_tlb(env)->d[l->mmu_idx].fulltlb[index];
+ mmu_lookup1(cpu, &l->page[0], l->mmu_idx, type, ra);
+ if (mmu_lookup1(cpu, &l->page[1], l->mmu_idx, type, ra)) {
+ uintptr_t index = tlb_index(cpu, l->mmu_idx, addr);
+ l->page[0].full = &cpu->neg.tlb.d[l->mmu_idx].fulltlb[index];
}
flags = l->page[0].flags | l->page[1].flags;
if (unlikely(flags & (TLB_WATCHPOINT | TLB_NOTDIRTY))) {
- mmu_watch_or_dirty(env, &l->page[0], type, ra);
- mmu_watch_or_dirty(env, &l->page[1], type, ra);
+ mmu_watch_or_dirty(cpu, &l->page[0], type, ra);
+ mmu_watch_or_dirty(cpu, &l->page[1], type, ra);
}
/*
* Probe for an atomic operation. Do not allow unaligned operations,
* or io operations to proceed. Return the host address.
*/
-static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
- MemOpIdx oi, int size, uintptr_t retaddr)
+static void *atomic_mmu_lookup(CPUState *cpu, vaddr addr, MemOpIdx oi,
+ int size, uintptr_t retaddr)
{
uintptr_t mmu_idx = get_mmuidx(oi);
MemOp mop = get_memop(oi);
int a_bits = get_alignment_bits(mop);
uintptr_t index;
CPUTLBEntry *tlbe;
- target_ulong tlb_addr;
+ vaddr tlb_addr;
void *hostaddr;
CPUTLBEntryFull *full;
/* Enforce guest required alignment. */
if (unlikely(a_bits > 0 && (addr & ((1 << a_bits) - 1)))) {
/* ??? Maybe indicate atomic op to cpu_unaligned_access */
- cpu_unaligned_access(env_cpu(env), addr, MMU_DATA_STORE,
+ cpu_unaligned_access(cpu, addr, MMU_DATA_STORE,
mmu_idx, retaddr);
}
goto stop_the_world;
}
- index = tlb_index(env, mmu_idx, addr);
- tlbe = tlb_entry(env, mmu_idx, addr);
+ index = tlb_index(cpu, mmu_idx, addr);
+ tlbe = tlb_entry(cpu, mmu_idx, addr);
/* Check TLB entry and enforce page permissions. */
tlb_addr = tlb_addr_write(tlbe);
if (!tlb_hit(tlb_addr, addr)) {
- if (!victim_tlb_hit(env, mmu_idx, index, MMU_DATA_STORE,
+ if (!victim_tlb_hit(cpu, mmu_idx, index, MMU_DATA_STORE,
addr & TARGET_PAGE_MASK)) {
- tlb_fill(env_cpu(env), addr, size,
+ tlb_fill(cpu, addr, size,
MMU_DATA_STORE, mmu_idx, retaddr);
- index = tlb_index(env, mmu_idx, addr);
- tlbe = tlb_entry(env, mmu_idx, addr);
+ index = tlb_index(cpu, mmu_idx, addr);
+ tlbe = tlb_entry(cpu, mmu_idx, addr);
}
tlb_addr = tlb_addr_write(tlbe) & ~TLB_INVALID_MASK;
}
* but addr_read will only be -1 if PAGE_READ was unset.
*/
if (unlikely(tlbe->addr_read == -1)) {
- tlb_fill(env_cpu(env), addr, size, MMU_DATA_LOAD, mmu_idx, retaddr);
+ tlb_fill(cpu, addr, size, MMU_DATA_LOAD, mmu_idx, retaddr);
/*
* Since we don't support reads and writes to different
* addresses, and we do have the proper page loaded for
*/
goto stop_the_world;
}
- /* Collect TLB_WATCHPOINT for read. */
+ /* Collect tlb flags for read. */
tlb_addr |= tlbe->addr_read;
/* Notice an IO access or a needs-MMU-lookup access */
}
hostaddr = (void *)((uintptr_t)addr + tlbe->addend);
- full = &env_tlb(env)->d[mmu_idx].fulltlb[index];
+ full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index];
if (unlikely(tlb_addr & TLB_NOTDIRTY)) {
- notdirty_write(env_cpu(env), addr, size, full, retaddr);
+ notdirty_write(cpu, addr, size, full, retaddr);
}
- if (unlikely(tlb_addr & TLB_WATCHPOINT)) {
- cpu_check_watchpoint(env_cpu(env), addr, size, full->attrs,
- BP_MEM_READ | BP_MEM_WRITE, retaddr);
+ if (unlikely(tlb_addr & TLB_FORCE_SLOW)) {
+ int wp_flags = 0;
+
+ if (full->slow_flags[MMU_DATA_STORE] & TLB_WATCHPOINT) {
+ wp_flags |= BP_MEM_WRITE;
+ }
+ if (full->slow_flags[MMU_DATA_LOAD] & TLB_WATCHPOINT) {
+ wp_flags |= BP_MEM_READ;
+ }
+ if (wp_flags) {
+ cpu_check_watchpoint(cpu, addr, size,
+ full->attrs, wp_flags, retaddr);
+ }
}
return hostaddr;
stop_the_world:
- cpu_loop_exit_atomic(env_cpu(env), retaddr);
+ cpu_loop_exit_atomic(cpu, retaddr);
}
/*
/**
* do_ld_mmio_beN:
- * @env: cpu context
- * @p: translation parameters
+ * @cpu: generic cpu state
+ * @full: page parameters
* @ret_be: accumulated data
+ * @addr: virtual address
+ * @size: number of bytes
* @mmu_idx: virtual address context
* @ra: return address into tcg generated code, or 0
+ * Context: BQL held
*
- * Load @p->size bytes from @p->addr, which is memory-mapped i/o.
+ * Load @size bytes from @addr, which is memory-mapped i/o.
* The bytes are concatenated in big-endian order with @ret_be.
*/
-static uint64_t do_ld_mmio_beN(CPUArchState *env, MMULookupPageData *p,
- uint64_t ret_be, int mmu_idx,
- MMUAccessType type, uintptr_t ra)
+static uint64_t int_ld_mmio_beN(CPUState *cpu, CPUTLBEntryFull *full,
+ uint64_t ret_be, vaddr addr, int size,
+ int mmu_idx, MMUAccessType type, uintptr_t ra,
+ MemoryRegion *mr, hwaddr mr_offset)
{
- CPUTLBEntryFull *full = p->full;
- vaddr addr = p->addr;
- int i, size = p->size;
+ do {
+ MemOp this_mop;
+ unsigned this_size;
+ uint64_t val;
+ MemTxResult r;
+
+ /* Read aligned pieces up to 8 bytes. */
+ this_mop = ctz32(size | (int)addr | 8);
+ this_size = 1 << this_mop;
+ this_mop |= MO_BE;
+
+ r = memory_region_dispatch_read(mr, mr_offset, &val,
+ this_mop, full->attrs);
+ if (unlikely(r != MEMTX_OK)) {
+ io_failed(cpu, full, addr, this_size, type, mmu_idx, r, ra);
+ }
+ if (this_size == 8) {
+ return val;
+ }
+
+ ret_be = (ret_be << (this_size * 8)) | val;
+ addr += this_size;
+ mr_offset += this_size;
+ size -= this_size;
+ } while (size);
- QEMU_IOTHREAD_LOCK_GUARD();
- for (i = 0; i < size; i++) {
- uint8_t x = io_readx(env, full, mmu_idx, addr + i, ra, type, MO_UB);
- ret_be = (ret_be << 8) | x;
- }
return ret_be;
}
+static uint64_t do_ld_mmio_beN(CPUState *cpu, CPUTLBEntryFull *full,
+ uint64_t ret_be, vaddr addr, int size,
+ int mmu_idx, MMUAccessType type, uintptr_t ra)
+{
+ MemoryRegionSection *section;
+ MemoryRegion *mr;
+ hwaddr mr_offset;
+ MemTxAttrs attrs;
+ uint64_t ret;
+
+ tcg_debug_assert(size > 0 && size <= 8);
+
+ attrs = full->attrs;
+ section = io_prepare(&mr_offset, cpu, full->xlat_section, attrs, addr, ra);
+ mr = section->mr;
+
+ bql_lock();
+ ret = int_ld_mmio_beN(cpu, full, ret_be, addr, size, mmu_idx,
+ type, ra, mr, mr_offset);
+ bql_unlock();
+
+ return ret;
+}
+
+static Int128 do_ld16_mmio_beN(CPUState *cpu, CPUTLBEntryFull *full,
+ uint64_t ret_be, vaddr addr, int size,
+ int mmu_idx, uintptr_t ra)
+{
+ MemoryRegionSection *section;
+ MemoryRegion *mr;
+ hwaddr mr_offset;
+ MemTxAttrs attrs;
+ uint64_t a, b;
+
+ tcg_debug_assert(size > 8 && size <= 16);
+
+ attrs = full->attrs;
+ section = io_prepare(&mr_offset, cpu, full->xlat_section, attrs, addr, ra);
+ mr = section->mr;
+
+ bql_lock();
+ a = int_ld_mmio_beN(cpu, full, ret_be, addr, size - 8, mmu_idx,
+ MMU_DATA_LOAD, ra, mr, mr_offset);
+ b = int_ld_mmio_beN(cpu, full, ret_be, addr + size - 8, 8, mmu_idx,
+ MMU_DATA_LOAD, ra, mr, mr_offset + size - 8);
+ bql_unlock();
+
+ return int128_make128(b, a);
+}
+
/**
* do_ld_bytes_beN
* @p: translation parameters
* As do_ld_bytes_beN, but with one atomic load.
* Eight aligned bytes are guaranteed to cover the load.
*/
-static uint64_t do_ld_whole_be8(CPUArchState *env, uintptr_t ra,
+static uint64_t do_ld_whole_be8(CPUState *cpu, uintptr_t ra,
MMULookupPageData *p, uint64_t ret_be)
{
int o = p->addr & 7;
- uint64_t x = load_atomic8_or_exit(env, ra, p->haddr - o);
+ uint64_t x = load_atomic8_or_exit(cpu, ra, p->haddr - o);
x = cpu_to_be64(x);
x <<= o * 8;
* As do_ld_bytes_beN, but with one atomic load.
* 16 aligned bytes are guaranteed to cover the load.
*/
-static Int128 do_ld_whole_be16(CPUArchState *env, uintptr_t ra,
+static Int128 do_ld_whole_be16(CPUState *cpu, uintptr_t ra,
MMULookupPageData *p, uint64_t ret_be)
{
int o = p->addr & 15;
- Int128 x, y = load_atomic16_or_exit(env, ra, p->haddr - o);
+ Int128 x, y = load_atomic16_or_exit(cpu, ra, p->haddr - o);
int size = p->size;
if (!HOST_BIG_ENDIAN) {
/*
* Wrapper for the above.
*/
-static uint64_t do_ld_beN(CPUArchState *env, MMULookupPageData *p,
+static uint64_t do_ld_beN(CPUState *cpu, MMULookupPageData *p,
uint64_t ret_be, int mmu_idx, MMUAccessType type,
MemOp mop, uintptr_t ra)
{
unsigned tmp, half_size;
if (unlikely(p->flags & TLB_MMIO)) {
- return do_ld_mmio_beN(env, p, ret_be, mmu_idx, type, ra);
+ return do_ld_mmio_beN(cpu, p->full, ret_be, p->addr, p->size,
+ mmu_idx, type, ra);
}
/*
if (!HAVE_al8_fast && p->size < 4) {
return do_ld_whole_be4(p, ret_be);
} else {
- return do_ld_whole_be8(env, ra, p, ret_be);
+ return do_ld_whole_be8(cpu, ra, p, ret_be);
}
}
/* fall through */
/*
* Wrapper for the above, for 8 < size < 16.
*/
-static Int128 do_ld16_beN(CPUArchState *env, MMULookupPageData *p,
+static Int128 do_ld16_beN(CPUState *cpu, MMULookupPageData *p,
uint64_t a, int mmu_idx, MemOp mop, uintptr_t ra)
{
int size = p->size;
MemOp atom;
if (unlikely(p->flags & TLB_MMIO)) {
- p->size = size - 8;
- a = do_ld_mmio_beN(env, p, a, mmu_idx, MMU_DATA_LOAD, ra);
- p->addr += p->size;
- p->size = 8;
- b = do_ld_mmio_beN(env, p, 0, mmu_idx, MMU_DATA_LOAD, ra);
- return int128_make128(b, a);
+ return do_ld16_mmio_beN(cpu, p->full, a, p->addr, size, mmu_idx, ra);
}
/*
case MO_ATOM_WITHIN16_PAIR:
/* Since size > 8, this is the half that must be atomic. */
- return do_ld_whole_be16(env, ra, p, a);
+ return do_ld_whole_be16(cpu, ra, p, a);
case MO_ATOM_IFALIGN_PAIR:
/*
return int128_make128(b, a);
}
-static uint8_t do_ld_1(CPUArchState *env, MMULookupPageData *p, int mmu_idx,
+static uint8_t do_ld_1(CPUState *cpu, MMULookupPageData *p, int mmu_idx,
MMUAccessType type, uintptr_t ra)
{
if (unlikely(p->flags & TLB_MMIO)) {
- return io_readx(env, p->full, mmu_idx, p->addr, ra, type, MO_UB);
+ return do_ld_mmio_beN(cpu, p->full, 0, p->addr, 1, mmu_idx, type, ra);
} else {
return *(uint8_t *)p->haddr;
}
}
-static uint16_t do_ld_2(CPUArchState *env, MMULookupPageData *p, int mmu_idx,
+static uint16_t do_ld_2(CPUState *cpu, MMULookupPageData *p, int mmu_idx,
MMUAccessType type, MemOp memop, uintptr_t ra)
{
- uint64_t ret;
+ uint16_t ret;
if (unlikely(p->flags & TLB_MMIO)) {
- return io_readx(env, p->full, mmu_idx, p->addr, ra, type, memop);
- }
-
- /* Perform the load host endian, then swap if necessary. */
- ret = load_atom_2(env, ra, p->haddr, memop);
- if (memop & MO_BSWAP) {
- ret = bswap16(ret);
+ ret = do_ld_mmio_beN(cpu, p->full, 0, p->addr, 2, mmu_idx, type, ra);
+ if ((memop & MO_BSWAP) == MO_LE) {
+ ret = bswap16(ret);
+ }
+ } else {
+ /* Perform the load host endian, then swap if necessary. */
+ ret = load_atom_2(cpu, ra, p->haddr, memop);
+ if (memop & MO_BSWAP) {
+ ret = bswap16(ret);
+ }
}
return ret;
}
-static uint32_t do_ld_4(CPUArchState *env, MMULookupPageData *p, int mmu_idx,
+static uint32_t do_ld_4(CPUState *cpu, MMULookupPageData *p, int mmu_idx,
MMUAccessType type, MemOp memop, uintptr_t ra)
{
uint32_t ret;
if (unlikely(p->flags & TLB_MMIO)) {
- return io_readx(env, p->full, mmu_idx, p->addr, ra, type, memop);
- }
-
- /* Perform the load host endian. */
- ret = load_atom_4(env, ra, p->haddr, memop);
- if (memop & MO_BSWAP) {
- ret = bswap32(ret);
+ ret = do_ld_mmio_beN(cpu, p->full, 0, p->addr, 4, mmu_idx, type, ra);
+ if ((memop & MO_BSWAP) == MO_LE) {
+ ret = bswap32(ret);
+ }
+ } else {
+ /* Perform the load host endian. */
+ ret = load_atom_4(cpu, ra, p->haddr, memop);
+ if (memop & MO_BSWAP) {
+ ret = bswap32(ret);
+ }
}
return ret;
}
-static uint64_t do_ld_8(CPUArchState *env, MMULookupPageData *p, int mmu_idx,
+static uint64_t do_ld_8(CPUState *cpu, MMULookupPageData *p, int mmu_idx,
MMUAccessType type, MemOp memop, uintptr_t ra)
{
uint64_t ret;
if (unlikely(p->flags & TLB_MMIO)) {
- return io_readx(env, p->full, mmu_idx, p->addr, ra, type, memop);
- }
-
- /* Perform the load host endian. */
- ret = load_atom_8(env, ra, p->haddr, memop);
- if (memop & MO_BSWAP) {
- ret = bswap64(ret);
+ ret = do_ld_mmio_beN(cpu, p->full, 0, p->addr, 8, mmu_idx, type, ra);
+ if ((memop & MO_BSWAP) == MO_LE) {
+ ret = bswap64(ret);
+ }
+ } else {
+ /* Perform the load host endian. */
+ ret = load_atom_8(cpu, ra, p->haddr, memop);
+ if (memop & MO_BSWAP) {
+ ret = bswap64(ret);
+ }
}
return ret;
}
-static uint8_t do_ld1_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi,
+static uint8_t do_ld1_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
uintptr_t ra, MMUAccessType access_type)
{
MMULookupLocals l;
bool crosspage;
- crosspage = mmu_lookup(env, addr, oi, ra, access_type, &l);
+ cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
+ crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
tcg_debug_assert(!crosspage);
- return do_ld_1(env, &l.page[0], l.mmu_idx, access_type, ra);
+ return do_ld_1(cpu, &l.page[0], l.mmu_idx, access_type, ra);
}
-tcg_target_ulong helper_ldub_mmu(CPUArchState *env, uint64_t addr,
- MemOpIdx oi, uintptr_t retaddr)
-{
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_8);
- return do_ld1_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD);
-}
-
-static uint16_t do_ld2_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi,
+static uint16_t do_ld2_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
uintptr_t ra, MMUAccessType access_type)
{
MMULookupLocals l;
uint16_t ret;
uint8_t a, b;
- crosspage = mmu_lookup(env, addr, oi, ra, access_type, &l);
+ cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
+ crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
if (likely(!crosspage)) {
- return do_ld_2(env, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
+ return do_ld_2(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
}
- a = do_ld_1(env, &l.page[0], l.mmu_idx, access_type, ra);
- b = do_ld_1(env, &l.page[1], l.mmu_idx, access_type, ra);
+ a = do_ld_1(cpu, &l.page[0], l.mmu_idx, access_type, ra);
+ b = do_ld_1(cpu, &l.page[1], l.mmu_idx, access_type, ra);
if ((l.memop & MO_BSWAP) == MO_LE) {
ret = a | (b << 8);
return ret;
}
-tcg_target_ulong helper_lduw_mmu(CPUArchState *env, uint64_t addr,
- MemOpIdx oi, uintptr_t retaddr)
-{
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_16);
- return do_ld2_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD);
-}
-
-static uint32_t do_ld4_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi,
+static uint32_t do_ld4_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
uintptr_t ra, MMUAccessType access_type)
{
MMULookupLocals l;
bool crosspage;
uint32_t ret;
- crosspage = mmu_lookup(env, addr, oi, ra, access_type, &l);
+ cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
+ crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
if (likely(!crosspage)) {
- return do_ld_4(env, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
+ return do_ld_4(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
}
- ret = do_ld_beN(env, &l.page[0], 0, l.mmu_idx, access_type, l.memop, ra);
- ret = do_ld_beN(env, &l.page[1], ret, l.mmu_idx, access_type, l.memop, ra);
+ ret = do_ld_beN(cpu, &l.page[0], 0, l.mmu_idx, access_type, l.memop, ra);
+ ret = do_ld_beN(cpu, &l.page[1], ret, l.mmu_idx, access_type, l.memop, ra);
if ((l.memop & MO_BSWAP) == MO_LE) {
ret = bswap32(ret);
}
return ret;
}
-tcg_target_ulong helper_ldul_mmu(CPUArchState *env, uint64_t addr,
- MemOpIdx oi, uintptr_t retaddr)
-{
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_32);
- return do_ld4_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD);
-}
-
-static uint64_t do_ld8_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi,
+static uint64_t do_ld8_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
uintptr_t ra, MMUAccessType access_type)
{
MMULookupLocals l;
bool crosspage;
uint64_t ret;
- crosspage = mmu_lookup(env, addr, oi, ra, access_type, &l);
+ cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
+ crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
if (likely(!crosspage)) {
- return do_ld_8(env, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
+ return do_ld_8(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
}
- ret = do_ld_beN(env, &l.page[0], 0, l.mmu_idx, access_type, l.memop, ra);
- ret = do_ld_beN(env, &l.page[1], ret, l.mmu_idx, access_type, l.memop, ra);
+ ret = do_ld_beN(cpu, &l.page[0], 0, l.mmu_idx, access_type, l.memop, ra);
+ ret = do_ld_beN(cpu, &l.page[1], ret, l.mmu_idx, access_type, l.memop, ra);
if ((l.memop & MO_BSWAP) == MO_LE) {
ret = bswap64(ret);
}
return ret;
}
-uint64_t helper_ldq_mmu(CPUArchState *env, uint64_t addr,
- MemOpIdx oi, uintptr_t retaddr)
-{
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_64);
- return do_ld8_mmu(env, addr, oi, retaddr, MMU_DATA_LOAD);
-}
-
-/*
- * Provide signed versions of the load routines as well. We can of course
- * avoid this for 64-bit data, or for 32-bit data on 32-bit host.
- */
-
-tcg_target_ulong helper_ldsb_mmu(CPUArchState *env, uint64_t addr,
- MemOpIdx oi, uintptr_t retaddr)
-{
- return (int8_t)helper_ldub_mmu(env, addr, oi, retaddr);
-}
-
-tcg_target_ulong helper_ldsw_mmu(CPUArchState *env, uint64_t addr,
- MemOpIdx oi, uintptr_t retaddr)
-{
- return (int16_t)helper_lduw_mmu(env, addr, oi, retaddr);
-}
-
-tcg_target_ulong helper_ldsl_mmu(CPUArchState *env, uint64_t addr,
- MemOpIdx oi, uintptr_t retaddr)
-{
- return (int32_t)helper_ldul_mmu(env, addr, oi, retaddr);
-}
-
-static Int128 do_ld16_mmu(CPUArchState *env, vaddr addr,
+static Int128 do_ld16_mmu(CPUState *cpu, vaddr addr,
MemOpIdx oi, uintptr_t ra)
{
MMULookupLocals l;
Int128 ret;
int first;
- crosspage = mmu_lookup(env, addr, oi, ra, MMU_DATA_LOAD, &l);
+ cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
+ crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_LOAD, &l);
if (likely(!crosspage)) {
- /* Perform the load host endian. */
if (unlikely(l.page[0].flags & TLB_MMIO)) {
- QEMU_IOTHREAD_LOCK_GUARD();
- a = io_readx(env, l.page[0].full, l.mmu_idx, addr,
- ra, MMU_DATA_LOAD, MO_64);
- b = io_readx(env, l.page[0].full, l.mmu_idx, addr + 8,
- ra, MMU_DATA_LOAD, MO_64);
- ret = int128_make128(HOST_BIG_ENDIAN ? b : a,
- HOST_BIG_ENDIAN ? a : b);
+ ret = do_ld16_mmio_beN(cpu, l.page[0].full, 0, addr, 16,
+ l.mmu_idx, ra);
+ if ((l.memop & MO_BSWAP) == MO_LE) {
+ ret = bswap128(ret);
+ }
} else {
- ret = load_atom_16(env, ra, l.page[0].haddr, l.memop);
- }
- if (l.memop & MO_BSWAP) {
- ret = bswap128(ret);
+ /* Perform the load host endian. */
+ ret = load_atom_16(cpu, ra, l.page[0].haddr, l.memop);
+ if (l.memop & MO_BSWAP) {
+ ret = bswap128(ret);
+ }
}
return ret;
}
if (first == 8) {
MemOp mop8 = (l.memop & ~MO_SIZE) | MO_64;
- a = do_ld_8(env, &l.page[0], l.mmu_idx, MMU_DATA_LOAD, mop8, ra);
- b = do_ld_8(env, &l.page[1], l.mmu_idx, MMU_DATA_LOAD, mop8, ra);
+ a = do_ld_8(cpu, &l.page[0], l.mmu_idx, MMU_DATA_LOAD, mop8, ra);
+ b = do_ld_8(cpu, &l.page[1], l.mmu_idx, MMU_DATA_LOAD, mop8, ra);
if ((mop8 & MO_BSWAP) == MO_LE) {
ret = int128_make128(a, b);
} else {
}
if (first < 8) {
- a = do_ld_beN(env, &l.page[0], 0, l.mmu_idx,
+ a = do_ld_beN(cpu, &l.page[0], 0, l.mmu_idx,
MMU_DATA_LOAD, l.memop, ra);
- ret = do_ld16_beN(env, &l.page[1], a, l.mmu_idx, l.memop, ra);
+ ret = do_ld16_beN(cpu, &l.page[1], a, l.mmu_idx, l.memop, ra);
} else {
- ret = do_ld16_beN(env, &l.page[0], 0, l.mmu_idx, l.memop, ra);
+ ret = do_ld16_beN(cpu, &l.page[0], 0, l.mmu_idx, l.memop, ra);
b = int128_getlo(ret);
ret = int128_lshift(ret, l.page[1].size * 8);
a = int128_gethi(ret);
- b = do_ld_beN(env, &l.page[1], b, l.mmu_idx,
+ b = do_ld_beN(cpu, &l.page[1], b, l.mmu_idx,
MMU_DATA_LOAD, l.memop, ra);
ret = int128_make128(b, a);
}
return ret;
}
-Int128 helper_ld16_mmu(CPUArchState *env, uint64_t addr,
- uint32_t oi, uintptr_t retaddr)
-{
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_128);
- return do_ld16_mmu(env, addr, oi, retaddr);
-}
-
-Int128 helper_ld_i128(CPUArchState *env, uint64_t addr, uint32_t oi)
-{
- return helper_ld16_mmu(env, addr, oi, GETPC());
-}
-
/*
- * Load helpers for cpu_ldst.h.
+ * Store Helpers
*/
-static void plugin_load_cb(CPUArchState *env, abi_ptr addr, MemOpIdx oi)
+/**
+ * do_st_mmio_leN:
+ * @cpu: generic cpu state
+ * @full: page parameters
+ * @val_le: data to store
+ * @addr: virtual address
+ * @size: number of bytes
+ * @mmu_idx: virtual address context
+ * @ra: return address into tcg generated code, or 0
+ * Context: BQL held
+ *
+ * Store @size bytes at @addr, which is memory-mapped i/o.
+ * The bytes to store are extracted in little-endian order from @val_le;
+ * return the bytes of @val_le beyond @p->size that have not been stored.
+ */
+static uint64_t int_st_mmio_leN(CPUState *cpu, CPUTLBEntryFull *full,
+ uint64_t val_le, vaddr addr, int size,
+ int mmu_idx, uintptr_t ra,
+ MemoryRegion *mr, hwaddr mr_offset)
{
- qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
-}
+ do {
+ MemOp this_mop;
+ unsigned this_size;
+ MemTxResult r;
+
+ /* Store aligned pieces up to 8 bytes. */
+ this_mop = ctz32(size | (int)addr | 8);
+ this_size = 1 << this_mop;
+ this_mop |= MO_LE;
+
+ r = memory_region_dispatch_write(mr, mr_offset, val_le,
+ this_mop, full->attrs);
+ if (unlikely(r != MEMTX_OK)) {
+ io_failed(cpu, full, addr, this_size, MMU_DATA_STORE,
+ mmu_idx, r, ra);
+ }
+ if (this_size == 8) {
+ return 0;
+ }
-uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra)
-{
- uint8_t ret;
+ val_le >>= this_size * 8;
+ addr += this_size;
+ mr_offset += this_size;
+ size -= this_size;
+ } while (size);
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_UB);
- ret = do_ld1_mmu(env, addr, oi, ra, MMU_DATA_LOAD);
- plugin_load_cb(env, addr, oi);
- return ret;
+ return val_le;
}
-uint16_t cpu_ldw_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
+static uint64_t do_st_mmio_leN(CPUState *cpu, CPUTLBEntryFull *full,
+ uint64_t val_le, vaddr addr, int size,
+ int mmu_idx, uintptr_t ra)
{
- uint16_t ret;
-
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_16);
- ret = do_ld2_mmu(env, addr, oi, ra, MMU_DATA_LOAD);
- plugin_load_cb(env, addr, oi);
- return ret;
-}
+ MemoryRegionSection *section;
+ hwaddr mr_offset;
+ MemoryRegion *mr;
+ MemTxAttrs attrs;
+ uint64_t ret;
-uint32_t cpu_ldl_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
-{
- uint32_t ret;
+ tcg_debug_assert(size > 0 && size <= 8);
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_32);
- ret = do_ld4_mmu(env, addr, oi, ra, MMU_DATA_LOAD);
- plugin_load_cb(env, addr, oi);
- return ret;
-}
+ attrs = full->attrs;
+ section = io_prepare(&mr_offset, cpu, full->xlat_section, attrs, addr, ra);
+ mr = section->mr;
-uint64_t cpu_ldq_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
-{
- uint64_t ret;
+ bql_lock();
+ ret = int_st_mmio_leN(cpu, full, val_le, addr, size, mmu_idx,
+ ra, mr, mr_offset);
+ bql_unlock();
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_64);
- ret = do_ld8_mmu(env, addr, oi, ra, MMU_DATA_LOAD);
- plugin_load_cb(env, addr, oi);
return ret;
}
-Int128 cpu_ld16_mmu(CPUArchState *env, abi_ptr addr,
- MemOpIdx oi, uintptr_t ra)
+static uint64_t do_st16_mmio_leN(CPUState *cpu, CPUTLBEntryFull *full,
+ Int128 val_le, vaddr addr, int size,
+ int mmu_idx, uintptr_t ra)
{
- Int128 ret;
+ MemoryRegionSection *section;
+ MemoryRegion *mr;
+ hwaddr mr_offset;
+ MemTxAttrs attrs;
+ uint64_t ret;
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_128);
- ret = do_ld16_mmu(env, addr, oi, ra);
- plugin_load_cb(env, addr, oi);
- return ret;
-}
+ tcg_debug_assert(size > 8 && size <= 16);
-/*
- * Store Helpers
- */
+ attrs = full->attrs;
+ section = io_prepare(&mr_offset, cpu, full->xlat_section, attrs, addr, ra);
+ mr = section->mr;
-/**
- * do_st_mmio_leN:
- * @env: cpu context
- * @p: translation parameters
- * @val_le: data to store
- * @mmu_idx: virtual address context
- * @ra: return address into tcg generated code, or 0
- *
- * Store @p->size bytes at @p->addr, which is memory-mapped i/o.
- * The bytes to store are extracted in little-endian order from @val_le;
- * return the bytes of @val_le beyond @p->size that have not been stored.
- */
-static uint64_t do_st_mmio_leN(CPUArchState *env, MMULookupPageData *p,
- uint64_t val_le, int mmu_idx, uintptr_t ra)
-{
- CPUTLBEntryFull *full = p->full;
- vaddr addr = p->addr;
- int i, size = p->size;
+ bql_lock();
+ int_st_mmio_leN(cpu, full, int128_getlo(val_le), addr, 8,
+ mmu_idx, ra, mr, mr_offset);
+ ret = int_st_mmio_leN(cpu, full, int128_gethi(val_le), addr + 8,
+ size - 8, mmu_idx, ra, mr, mr_offset + 8);
+ bql_unlock();
- QEMU_IOTHREAD_LOCK_GUARD();
- for (i = 0; i < size; i++, val_le >>= 8) {
- io_writex(env, full, mmu_idx, val_le, addr + i, ra, MO_UB);
- }
- return val_le;
+ return ret;
}
/*
* Wrapper for the above.
*/
-static uint64_t do_st_leN(CPUArchState *env, MMULookupPageData *p,
+static uint64_t do_st_leN(CPUState *cpu, MMULookupPageData *p,
uint64_t val_le, int mmu_idx,
MemOp mop, uintptr_t ra)
{
unsigned tmp, half_size;
if (unlikely(p->flags & TLB_MMIO)) {
- return do_st_mmio_leN(env, p, val_le, mmu_idx, ra);
+ return do_st_mmio_leN(cpu, p->full, val_le, p->addr,
+ p->size, mmu_idx, ra);
} else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
return val_le >> (p->size * 8);
}
} else if (HAVE_al8) {
return store_whole_le8(p->haddr, p->size, val_le);
} else {
- cpu_loop_exit_atomic(env_cpu(env), ra);
+ cpu_loop_exit_atomic(cpu, ra);
}
}
/* fall through */
/*
* Wrapper for the above, for 8 < size < 16.
*/
-static uint64_t do_st16_leN(CPUArchState *env, MMULookupPageData *p,
+static uint64_t do_st16_leN(CPUState *cpu, MMULookupPageData *p,
Int128 val_le, int mmu_idx,
MemOp mop, uintptr_t ra)
{
MemOp atom;
if (unlikely(p->flags & TLB_MMIO)) {
- p->size = 8;
- do_st_mmio_leN(env, p, int128_getlo(val_le), mmu_idx, ra);
- p->size = size - 8;
- p->addr += 8;
- return do_st_mmio_leN(env, p, int128_gethi(val_le), mmu_idx, ra);
+ return do_st16_mmio_leN(cpu, p->full, val_le, p->addr,
+ size, mmu_idx, ra);
} else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
return int128_gethi(val_le) >> ((size - 8) * 8);
}
case MO_ATOM_WITHIN16_PAIR:
/* Since size > 8, this is the half that must be atomic. */
- if (!HAVE_ATOMIC128_RW) {
- cpu_loop_exit_atomic(env_cpu(env), ra);
+ if (!HAVE_CMPXCHG128) {
+ cpu_loop_exit_atomic(cpu, ra);
}
return store_whole_le16(p->haddr, p->size, val_le);
}
}
-static void do_st_1(CPUArchState *env, MMULookupPageData *p, uint8_t val,
+static void do_st_1(CPUState *cpu, MMULookupPageData *p, uint8_t val,
int mmu_idx, uintptr_t ra)
{
if (unlikely(p->flags & TLB_MMIO)) {
- io_writex(env, p->full, mmu_idx, val, p->addr, ra, MO_UB);
+ do_st_mmio_leN(cpu, p->full, val, p->addr, 1, mmu_idx, ra);
} else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
/* nothing */
} else {
}
}
-static void do_st_2(CPUArchState *env, MMULookupPageData *p, uint16_t val,
+static void do_st_2(CPUState *cpu, MMULookupPageData *p, uint16_t val,
int mmu_idx, MemOp memop, uintptr_t ra)
{
if (unlikely(p->flags & TLB_MMIO)) {
- io_writex(env, p->full, mmu_idx, val, p->addr, ra, memop);
+ if ((memop & MO_BSWAP) != MO_LE) {
+ val = bswap16(val);
+ }
+ do_st_mmio_leN(cpu, p->full, val, p->addr, 2, mmu_idx, ra);
} else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
/* nothing */
} else {
if (memop & MO_BSWAP) {
val = bswap16(val);
}
- store_atom_2(env, ra, p->haddr, memop, val);
+ store_atom_2(cpu, ra, p->haddr, memop, val);
}
}
-static void do_st_4(CPUArchState *env, MMULookupPageData *p, uint32_t val,
+static void do_st_4(CPUState *cpu, MMULookupPageData *p, uint32_t val,
int mmu_idx, MemOp memop, uintptr_t ra)
{
if (unlikely(p->flags & TLB_MMIO)) {
- io_writex(env, p->full, mmu_idx, val, p->addr, ra, memop);
+ if ((memop & MO_BSWAP) != MO_LE) {
+ val = bswap32(val);
+ }
+ do_st_mmio_leN(cpu, p->full, val, p->addr, 4, mmu_idx, ra);
} else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
/* nothing */
} else {
if (memop & MO_BSWAP) {
val = bswap32(val);
}
- store_atom_4(env, ra, p->haddr, memop, val);
+ store_atom_4(cpu, ra, p->haddr, memop, val);
}
}
-static void do_st_8(CPUArchState *env, MMULookupPageData *p, uint64_t val,
+static void do_st_8(CPUState *cpu, MMULookupPageData *p, uint64_t val,
int mmu_idx, MemOp memop, uintptr_t ra)
{
if (unlikely(p->flags & TLB_MMIO)) {
- io_writex(env, p->full, mmu_idx, val, p->addr, ra, memop);
+ if ((memop & MO_BSWAP) != MO_LE) {
+ val = bswap64(val);
+ }
+ do_st_mmio_leN(cpu, p->full, val, p->addr, 8, mmu_idx, ra);
} else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
/* nothing */
} else {
if (memop & MO_BSWAP) {
val = bswap64(val);
}
- store_atom_8(env, ra, p->haddr, memop, val);
+ store_atom_8(cpu, ra, p->haddr, memop, val);
}
}
-void helper_stb_mmu(CPUArchState *env, uint64_t addr, uint32_t val,
- MemOpIdx oi, uintptr_t ra)
+static void do_st1_mmu(CPUState *cpu, vaddr addr, uint8_t val,
+ MemOpIdx oi, uintptr_t ra)
{
MMULookupLocals l;
bool crosspage;
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_8);
- crosspage = mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE, &l);
+ cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
+ crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
tcg_debug_assert(!crosspage);
- do_st_1(env, &l.page[0], val, l.mmu_idx, ra);
+ do_st_1(cpu, &l.page[0], val, l.mmu_idx, ra);
}
-static void do_st2_mmu(CPUArchState *env, vaddr addr, uint16_t val,
+static void do_st2_mmu(CPUState *cpu, vaddr addr, uint16_t val,
MemOpIdx oi, uintptr_t ra)
{
MMULookupLocals l;
bool crosspage;
uint8_t a, b;
- crosspage = mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE, &l);
+ cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
+ crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
if (likely(!crosspage)) {
- do_st_2(env, &l.page[0], val, l.mmu_idx, l.memop, ra);
+ do_st_2(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
return;
}
} else {
b = val, a = val >> 8;
}
- do_st_1(env, &l.page[0], a, l.mmu_idx, ra);
- do_st_1(env, &l.page[1], b, l.mmu_idx, ra);
+ do_st_1(cpu, &l.page[0], a, l.mmu_idx, ra);
+ do_st_1(cpu, &l.page[1], b, l.mmu_idx, ra);
}
-void helper_stw_mmu(CPUArchState *env, uint64_t addr, uint32_t val,
- MemOpIdx oi, uintptr_t retaddr)
-{
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_16);
- do_st2_mmu(env, addr, val, oi, retaddr);
-}
-
-static void do_st4_mmu(CPUArchState *env, vaddr addr, uint32_t val,
+static void do_st4_mmu(CPUState *cpu, vaddr addr, uint32_t val,
MemOpIdx oi, uintptr_t ra)
{
MMULookupLocals l;
bool crosspage;
- crosspage = mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE, &l);
+ cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
+ crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
if (likely(!crosspage)) {
- do_st_4(env, &l.page[0], val, l.mmu_idx, l.memop, ra);
+ do_st_4(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
return;
}
if ((l.memop & MO_BSWAP) != MO_LE) {
val = bswap32(val);
}
- val = do_st_leN(env, &l.page[0], val, l.mmu_idx, l.memop, ra);
- (void) do_st_leN(env, &l.page[1], val, l.mmu_idx, l.memop, ra);
+ val = do_st_leN(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
+ (void) do_st_leN(cpu, &l.page[1], val, l.mmu_idx, l.memop, ra);
}
-void helper_stl_mmu(CPUArchState *env, uint64_t addr, uint32_t val,
- MemOpIdx oi, uintptr_t retaddr)
-{
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_32);
- do_st4_mmu(env, addr, val, oi, retaddr);
-}
-
-static void do_st8_mmu(CPUArchState *env, vaddr addr, uint64_t val,
+static void do_st8_mmu(CPUState *cpu, vaddr addr, uint64_t val,
MemOpIdx oi, uintptr_t ra)
{
MMULookupLocals l;
bool crosspage;
- crosspage = mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE, &l);
+ cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
+ crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
if (likely(!crosspage)) {
- do_st_8(env, &l.page[0], val, l.mmu_idx, l.memop, ra);
+ do_st_8(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
return;
}
if ((l.memop & MO_BSWAP) != MO_LE) {
val = bswap64(val);
}
- val = do_st_leN(env, &l.page[0], val, l.mmu_idx, l.memop, ra);
- (void) do_st_leN(env, &l.page[1], val, l.mmu_idx, l.memop, ra);
+ val = do_st_leN(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
+ (void) do_st_leN(cpu, &l.page[1], val, l.mmu_idx, l.memop, ra);
}
-void helper_stq_mmu(CPUArchState *env, uint64_t addr, uint64_t val,
- MemOpIdx oi, uintptr_t retaddr)
-{
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_64);
- do_st8_mmu(env, addr, val, oi, retaddr);
-}
-
-static void do_st16_mmu(CPUArchState *env, vaddr addr, Int128 val,
+static void do_st16_mmu(CPUState *cpu, vaddr addr, Int128 val,
MemOpIdx oi, uintptr_t ra)
{
MMULookupLocals l;
uint64_t a, b;
int first;
- crosspage = mmu_lookup(env, addr, oi, ra, MMU_DATA_STORE, &l);
+ cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
+ crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
if (likely(!crosspage)) {
- /* Swap to host endian if necessary, then store. */
- if (l.memop & MO_BSWAP) {
- val = bswap128(val);
- }
if (unlikely(l.page[0].flags & TLB_MMIO)) {
- QEMU_IOTHREAD_LOCK_GUARD();
- if (HOST_BIG_ENDIAN) {
- b = int128_getlo(val), a = int128_gethi(val);
- } else {
- a = int128_getlo(val), b = int128_gethi(val);
+ if ((l.memop & MO_BSWAP) != MO_LE) {
+ val = bswap128(val);
}
- io_writex(env, l.page[0].full, l.mmu_idx, a, addr, ra, MO_64);
- io_writex(env, l.page[0].full, l.mmu_idx, b, addr + 8, ra, MO_64);
+ do_st16_mmio_leN(cpu, l.page[0].full, val, addr, 16, l.mmu_idx, ra);
} else if (unlikely(l.page[0].flags & TLB_DISCARD_WRITE)) {
/* nothing */
} else {
- store_atom_16(env, ra, l.page[0].haddr, l.memop, val);
+ /* Swap to host endian if necessary, then store. */
+ if (l.memop & MO_BSWAP) {
+ val = bswap128(val);
+ }
+ store_atom_16(cpu, ra, l.page[0].haddr, l.memop, val);
}
return;
}
} else {
a = int128_getlo(val), b = int128_gethi(val);
}
- do_st_8(env, &l.page[0], a, l.mmu_idx, mop8, ra);
- do_st_8(env, &l.page[1], b, l.mmu_idx, mop8, ra);
+ do_st_8(cpu, &l.page[0], a, l.mmu_idx, mop8, ra);
+ do_st_8(cpu, &l.page[1], b, l.mmu_idx, mop8, ra);
return;
}
val = bswap128(val);
}
if (first < 8) {
- do_st_leN(env, &l.page[0], int128_getlo(val), l.mmu_idx, l.memop, ra);
+ do_st_leN(cpu, &l.page[0], int128_getlo(val), l.mmu_idx, l.memop, ra);
val = int128_urshift(val, first * 8);
- do_st16_leN(env, &l.page[1], val, l.mmu_idx, l.memop, ra);
+ do_st16_leN(cpu, &l.page[1], val, l.mmu_idx, l.memop, ra);
} else {
- b = do_st16_leN(env, &l.page[0], val, l.mmu_idx, l.memop, ra);
- do_st_leN(env, &l.page[1], b, l.mmu_idx, l.memop, ra);
+ b = do_st16_leN(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
+ do_st_leN(cpu, &l.page[1], b, l.mmu_idx, l.memop, ra);
}
}
-void helper_st16_mmu(CPUArchState *env, uint64_t addr, Int128 val,
- MemOpIdx oi, uintptr_t retaddr)
-{
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_128);
- do_st16_mmu(env, addr, val, oi, retaddr);
-}
-
-void helper_st_i128(CPUArchState *env, uint64_t addr, Int128 val, MemOpIdx oi)
-{
- helper_st16_mmu(env, addr, val, oi, GETPC());
-}
-
-/*
- * Store Helpers for cpu_ldst.h
- */
-
-static void plugin_store_cb(CPUArchState *env, abi_ptr addr, MemOpIdx oi)
-{
- qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
-}
-
-void cpu_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,
- MemOpIdx oi, uintptr_t retaddr)
-{
- helper_stb_mmu(env, addr, val, oi, retaddr);
- plugin_store_cb(env, addr, oi);
-}
-
-void cpu_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
- MemOpIdx oi, uintptr_t retaddr)
-{
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_16);
- do_st2_mmu(env, addr, val, oi, retaddr);
- plugin_store_cb(env, addr, oi);
-}
-
-void cpu_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
- MemOpIdx oi, uintptr_t retaddr)
-{
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_32);
- do_st4_mmu(env, addr, val, oi, retaddr);
- plugin_store_cb(env, addr, oi);
-}
-
-void cpu_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
- MemOpIdx oi, uintptr_t retaddr)
-{
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_64);
- do_st8_mmu(env, addr, val, oi, retaddr);
- plugin_store_cb(env, addr, oi);
-}
-
-void cpu_st16_mmu(CPUArchState *env, target_ulong addr, Int128 val,
- MemOpIdx oi, uintptr_t retaddr)
-{
- tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_128);
- do_st16_mmu(env, addr, val, oi, retaddr);
- plugin_store_cb(env, addr, oi);
-}
-
#include "ldst_common.c.inc"
/*
#include "atomic_template.h"
#endif
-#if defined(CONFIG_ATOMIC128) || defined(CONFIG_CMPXCHG128)
+#if defined(CONFIG_ATOMIC128) || HAVE_CMPXCHG128
#define DATA_SIZE 16
#include "atomic_template.h"
#endif
uint32_t cpu_ldub_code(CPUArchState *env, abi_ptr addr)
{
MemOpIdx oi = make_memop_idx(MO_UB, cpu_mmu_index(env, true));
- return do_ld1_mmu(env, addr, oi, 0, MMU_INST_FETCH);
+ return do_ld1_mmu(env_cpu(env), addr, oi, 0, MMU_INST_FETCH);
}
uint32_t cpu_lduw_code(CPUArchState *env, abi_ptr addr)
{
MemOpIdx oi = make_memop_idx(MO_TEUW, cpu_mmu_index(env, true));
- return do_ld2_mmu(env, addr, oi, 0, MMU_INST_FETCH);
+ return do_ld2_mmu(env_cpu(env), addr, oi, 0, MMU_INST_FETCH);
}
uint32_t cpu_ldl_code(CPUArchState *env, abi_ptr addr)
{
MemOpIdx oi = make_memop_idx(MO_TEUL, cpu_mmu_index(env, true));
- return do_ld4_mmu(env, addr, oi, 0, MMU_INST_FETCH);
+ return do_ld4_mmu(env_cpu(env), addr, oi, 0, MMU_INST_FETCH);
}
uint64_t cpu_ldq_code(CPUArchState *env, abi_ptr addr)
{
MemOpIdx oi = make_memop_idx(MO_TEUQ, cpu_mmu_index(env, true));
- return do_ld8_mmu(env, addr, oi, 0, MMU_INST_FETCH);
+ return do_ld8_mmu(env_cpu(env), addr, oi, 0, MMU_INST_FETCH);
}
uint8_t cpu_ldb_code_mmu(CPUArchState *env, abi_ptr addr,
MemOpIdx oi, uintptr_t retaddr)
{
- return do_ld1_mmu(env, addr, oi, retaddr, MMU_INST_FETCH);
+ return do_ld1_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
}
uint16_t cpu_ldw_code_mmu(CPUArchState *env, abi_ptr addr,
MemOpIdx oi, uintptr_t retaddr)
{
- return do_ld2_mmu(env, addr, oi, retaddr, MMU_INST_FETCH);
+ return do_ld2_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
}
uint32_t cpu_ldl_code_mmu(CPUArchState *env, abi_ptr addr,
MemOpIdx oi, uintptr_t retaddr)
{
- return do_ld4_mmu(env, addr, oi, retaddr, MMU_INST_FETCH);
+ return do_ld4_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
}
uint64_t cpu_ldq_code_mmu(CPUArchState *env, abi_ptr addr,
MemOpIdx oi, uintptr_t retaddr)
{
- return do_ld8_mmu(env, addr, oi, retaddr, MMU_INST_FETCH);
+ return do_ld8_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
}
projects / mirror_qemu.git / blobdiff