X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=memory.c;h=75ca281e978efad550afbe1e1223a6412152b310;hb=15d23fb96656b1ae31bf4305b2108230c29298c6;hp=4c3dc492624ae9e592af8473c662f5d0f6e60fda;hpb=a2d335214a53b0cc6f7a4a2009d842bea1958085;p=qemu.git diff --git a/memory.c b/memory.c index 4c3dc4926..75ca281e9 100644 --- a/memory.c +++ b/memory.c @@ -13,23 +13,25 @@ * GNU GPL, version 2 or (at your option) any later version. */ -#include "memory.h" -#include "exec-memory.h" -#include "ioport.h" -#include "bitops.h" -#include "kvm.h" +#include "exec/memory.h" +#include "exec/address-spaces.h" +#include "exec/ioport.h" +#include "qemu/bitops.h" +#include "sysemu/kvm.h" #include -#define WANT_EXEC_OBSOLETE -#include "exec-obsolete.h" +#include "exec/memory-internal.h" -unsigned memory_region_transaction_depth = 0; -static bool memory_region_update_pending = false; +static unsigned memory_region_transaction_depth; +static bool memory_region_update_pending; static bool global_dirty_log = false; static QTAILQ_HEAD(memory_listeners, MemoryListener) memory_listeners = QTAILQ_HEAD_INITIALIZER(memory_listeners); +static QTAILQ_HEAD(, AddressSpace) address_spaces + = QTAILQ_HEAD_INITIALIZER(address_spaces); + typedef struct AddrRange AddrRange; /* @@ -98,13 +100,17 @@ static bool memory_listener_match(MemoryListener *listener, switch (_direction) { \ case Forward: \ QTAILQ_FOREACH(_listener, &memory_listeners, link) { \ - _listener->_callback(_listener, ##_args); \ + if (_listener->_callback) { \ + _listener->_callback(_listener, ##_args); \ + } \ } \ break; \ case Reverse: \ QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \ memory_listeners, link) { \ - _listener->_callback(_listener, ##_args); \ + if (_listener->_callback) { \ + _listener->_callback(_listener, ##_args); \ + } \ } \ break; \ default: \ @@ -119,7 +125,8 @@ static bool memory_listener_match(MemoryListener *listener, switch (_direction) { \ case Forward: \ QTAILQ_FOREACH(_listener, &memory_listeners, link) { \ - if (memory_listener_match(_listener, _section)) { \ + if (_listener->_callback \ + && memory_listener_match(_listener, _section)) { \ _listener->_callback(_listener, _section, ##_args); \ } \ } \ @@ -127,7 +134,8 @@ static bool memory_listener_match(MemoryListener *listener, case Reverse: \ QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \ memory_listeners, link) { \ - if (memory_listener_match(_listener, _section)) { \ + if (_listener->_callback \ + && memory_listener_match(_listener, _section)) { \ _listener->_callback(_listener, _section, ##_args); \ } \ } \ @@ -140,7 +148,7 @@ static bool memory_listener_match(MemoryListener *listener, #define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback) \ MEMORY_LISTENER_CALL(callback, dir, (&(MemoryRegionSection) { \ .mr = (fr)->mr, \ - .address_space = (as)->root, \ + .address_space = (as), \ .offset_within_region = (fr)->offset_in_region, \ .size = int128_get64((fr)->addr.size), \ .offset_within_address_space = int128_get64((fr)->addr.start), \ @@ -156,7 +164,7 @@ struct MemoryRegionIoeventfd { AddrRange addr; bool match_data; uint64_t data; - int fd; + EventNotifier *e; }; static bool memory_region_ioeventfd_before(MemoryRegionIoeventfd a, @@ -181,9 +189,9 @@ static bool memory_region_ioeventfd_before(MemoryRegionIoeventfd a, return false; } } - if (a.fd < b.fd) { + if (a.e < b.e) { return true; - } else if (a.fd > b.fd) { + } else if (a.e > b.e) { return false; } return false; @@ -202,7 +210,7 @@ typedef struct FlatView FlatView; /* Range of memory in the global map. Addresses are absolute. */ struct FlatRange { MemoryRegion *mr; - target_phys_addr_t offset_in_region; + hwaddr offset_in_region; AddrRange addr; uint8_t dirty_log_mask; bool readable; @@ -218,17 +226,8 @@ struct FlatView { unsigned nr_allocated; }; -typedef struct AddressSpace AddressSpace; typedef struct AddressSpaceOps AddressSpaceOps; -/* A system address space - I/O, memory, etc. */ -struct AddressSpace { - MemoryRegion *root; - FlatView current_map; - int ioeventfd_nb; - MemoryRegionIoeventfd *ioeventfds; -}; - #define FOR_EACH_FLAT_RANGE(var, view) \ for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var) @@ -302,7 +301,7 @@ static void flatview_simplify(FlatView *view) } static void memory_region_read_accessor(void *opaque, - target_phys_addr_t addr, + hwaddr addr, uint64_t *value, unsigned size, unsigned shift, @@ -311,12 +310,15 @@ static void memory_region_read_accessor(void *opaque, MemoryRegion *mr = opaque; uint64_t tmp; + if (mr->flush_coalesced_mmio) { + qemu_flush_coalesced_mmio_buffer(); + } tmp = mr->ops->read(mr->opaque, addr, size); *value |= (tmp & mask) << shift; } static void memory_region_write_accessor(void *opaque, - target_phys_addr_t addr, + hwaddr addr, uint64_t *value, unsigned size, unsigned shift, @@ -325,17 +327,20 @@ static void memory_region_write_accessor(void *opaque, MemoryRegion *mr = opaque; uint64_t tmp; + if (mr->flush_coalesced_mmio) { + qemu_flush_coalesced_mmio_buffer(); + } tmp = (*value >> shift) & mask; mr->ops->write(mr->opaque, addr, tmp, size); } -static void access_with_adjusted_size(target_phys_addr_t addr, +static void access_with_adjusted_size(hwaddr addr, uint64_t *value, unsigned size, unsigned access_size_min, unsigned access_size_max, void (*access)(void *opaque, - target_phys_addr_t addr, + hwaddr addr, uint64_t *value, unsigned size, unsigned shift, @@ -360,8 +365,6 @@ static void access_with_adjusted_size(target_phys_addr_t addr, } } -static AddressSpace address_space_memory; - static const MemoryRegionPortio *find_portio(MemoryRegion *mr, uint64_t offset, unsigned width, bool write) { @@ -426,7 +429,7 @@ static void memory_region_iorange_write(IORange *iorange, if (mrp) { mrp->write(mr->opaque, offset, data); } else if (width == 2) { - mrp = find_portio(mr, offset - mrio->offset, 1, false); + mrp = find_portio(mr, offset - mrio->offset, 1, true); assert(mrp); mrp->write(mr->opaque, offset, data & 0xff); mrp->write(mr->opaque, offset + 1, data >> 8); @@ -450,18 +453,17 @@ const IORangeOps memory_region_iorange_ops = { .destructor = memory_region_iorange_destructor, }; -static AddressSpace address_space_io; - static AddressSpace *memory_region_to_address_space(MemoryRegion *mr) { + AddressSpace *as; + while (mr->parent) { mr = mr->parent; } - if (mr == address_space_memory.root) { - return &address_space_memory; - } - if (mr == address_space_io.root) { - return &address_space_io; + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + if (mr == as->root) { + return as; + } } abort(); } @@ -477,7 +479,7 @@ static void render_memory_region(FlatView *view, { MemoryRegion *subregion; unsigned i; - target_phys_addr_t offset_in_region; + hwaddr offset_in_region; Int128 remain; Int128 now; FlatRange fr; @@ -538,12 +540,12 @@ static void render_memory_region(FlatView *view, offset_in_region += int128_get64(now); int128_subfrom(&remain, now); } - if (int128_eq(base, view->ranges[i].addr.start)) { - now = int128_min(remain, view->ranges[i].addr.size); - int128_addto(&base, now); - offset_in_region += int128_get64(now); - int128_subfrom(&remain, now); - } + now = int128_sub(int128_min(int128_add(base, remain), + addrrange_end(view->ranges[i].addr)), + base); + int128_addto(&base, now); + offset_in_region += int128_get64(now); + int128_subfrom(&remain, now); } if (int128_nz(remain)) { fr.mr = mr; @@ -563,8 +565,10 @@ static FlatView generate_memory_topology(MemoryRegion *mr) flatview_init(&view); - render_memory_region(&view, mr, int128_zero(), - addrrange_make(int128_zero(), int128_2_64()), false); + if (mr) { + render_memory_region(&view, mr, int128_zero(), + addrrange_make(int128_zero(), int128_2_64()), false); + } flatview_simplify(&view); return view; @@ -592,12 +596,12 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, fds_new[inew]))) { fd = &fds_old[iold]; section = (MemoryRegionSection) { - .address_space = as->root, + .address_space = as, .offset_within_address_space = int128_get64(fd->addr.start), .size = int128_get64(fd->addr.size), }; MEMORY_LISTENER_CALL(eventfd_del, Forward, §ion, - fd->match_data, fd->data, fd->fd); + fd->match_data, fd->data, fd->e); ++iold; } else if (inew < fds_new_nb && (iold == fds_old_nb @@ -605,12 +609,12 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, fds_old[iold]))) { fd = &fds_new[inew]; section = (MemoryRegionSection) { - .address_space = as->root, + .address_space = as, .offset_within_address_space = int128_get64(fd->addr.start), .size = int128_get64(fd->addr.size), }; MEMORY_LISTENER_CALL(eventfd_add, Reverse, §ion, - fd->match_data, fd->data, fd->fd); + fd->match_data, fd->data, fd->e); ++inew; } else { ++iold; @@ -627,7 +631,7 @@ static void address_space_update_ioeventfds(AddressSpace *as) AddrRange tmp; unsigned i; - FOR_EACH_FLAT_RANGE(fr, &as->current_map) { + FOR_EACH_FLAT_RANGE(fr, as->current_map) { for (i = 0; i < fr->mr->ioeventfd_nb; ++i) { tmp = addrrange_shift(fr->mr->ioeventfds[i].addr, int128_sub(fr->addr.start, @@ -715,53 +719,38 @@ static void address_space_update_topology_pass(AddressSpace *as, static void address_space_update_topology(AddressSpace *as) { - FlatView old_view = as->current_map; + FlatView old_view = *as->current_map; FlatView new_view = generate_memory_topology(as->root); address_space_update_topology_pass(as, old_view, new_view, false); address_space_update_topology_pass(as, old_view, new_view, true); - as->current_map = new_view; + *as->current_map = new_view; flatview_destroy(&old_view); address_space_update_ioeventfds(as); } -static void memory_region_update_topology(MemoryRegion *mr) -{ - if (memory_region_transaction_depth) { - memory_region_update_pending |= !mr || mr->enabled; - return; - } - - if (mr && !mr->enabled) { - return; - } - - MEMORY_LISTENER_CALL_GLOBAL(begin, Forward); - - if (address_space_memory.root) { - address_space_update_topology(&address_space_memory); - } - if (address_space_io.root) { - address_space_update_topology(&address_space_io); - } - - MEMORY_LISTENER_CALL_GLOBAL(commit, Forward); - - memory_region_update_pending = false; -} - void memory_region_transaction_begin(void) { + qemu_flush_coalesced_mmio_buffer(); ++memory_region_transaction_depth; } void memory_region_transaction_commit(void) { + AddressSpace *as; + assert(memory_region_transaction_depth); --memory_region_transaction_depth; if (!memory_region_transaction_depth && memory_region_update_pending) { - memory_region_update_topology(NULL); + memory_region_update_pending = false; + MEMORY_LISTENER_CALL_GLOBAL(begin, Forward); + + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + address_space_update_topology(as); + } + + MEMORY_LISTENER_CALL_GLOBAL(commit, Forward); } } @@ -781,13 +770,11 @@ static void memory_region_destructor_ram_from_ptr(MemoryRegion *mr) static void memory_region_destructor_iomem(MemoryRegion *mr) { - cpu_unregister_io_memory(mr->ram_addr); } static void memory_region_destructor_rom_device(MemoryRegion *mr) { qemu_ram_free(mr->ram_addr & TARGET_PAGE_MASK); - cpu_unregister_io_memory(mr->ram_addr & ~TARGET_PAGE_MASK); } static bool memory_region_wrong_endianness(MemoryRegion *mr) @@ -828,10 +815,11 @@ void memory_region_init(MemoryRegion *mr, mr->dirty_log_mask = 0; mr->ioeventfd_nb = 0; mr->ioeventfds = NULL; + mr->flush_coalesced_mmio = false; } static bool memory_region_access_valid(MemoryRegion *mr, - target_phys_addr_t addr, + hwaddr addr, unsigned size, bool is_write) { @@ -857,7 +845,7 @@ static bool memory_region_access_valid(MemoryRegion *mr, } static uint64_t memory_region_dispatch_read1(MemoryRegion *mr, - target_phys_addr_t addr, + hwaddr addr, unsigned size) { uint64_t data = 0; @@ -867,7 +855,7 @@ static uint64_t memory_region_dispatch_read1(MemoryRegion *mr, } if (!mr->ops->read) { - return mr->ops->old_mmio.read[bitops_ffsl(size)](mr->opaque, addr); + return mr->ops->old_mmio.read[ctz32(size)](mr->opaque, addr); } /* FIXME: support unaligned access */ @@ -898,7 +886,7 @@ static void adjust_endianness(MemoryRegion *mr, uint64_t *data, unsigned size) } static uint64_t memory_region_dispatch_read(MemoryRegion *mr, - target_phys_addr_t addr, + hwaddr addr, unsigned size) { uint64_t ret; @@ -909,7 +897,7 @@ static uint64_t memory_region_dispatch_read(MemoryRegion *mr, } static void memory_region_dispatch_write(MemoryRegion *mr, - target_phys_addr_t addr, + hwaddr addr, uint64_t data, unsigned size) { @@ -920,7 +908,7 @@ static void memory_region_dispatch_write(MemoryRegion *mr, adjust_endianness(mr, &data, size); if (!mr->ops->write) { - mr->ops->old_mmio.write[bitops_ffsl(size)](mr->opaque, addr, data); + mr->ops->old_mmio.write[ctz32(size)](mr->opaque, addr, data); return; } @@ -942,7 +930,7 @@ void memory_region_init_io(MemoryRegion *mr, mr->opaque = opaque; mr->terminates = true; mr->destructor = memory_region_destructor_iomem; - mr->ram_addr = cpu_register_io_memory(mr); + mr->ram_addr = ~(ram_addr_t)0; } void memory_region_init_ram(MemoryRegion *mr, @@ -971,7 +959,7 @@ void memory_region_init_ram_ptr(MemoryRegion *mr, void memory_region_init_alias(MemoryRegion *mr, const char *name, MemoryRegion *orig, - target_phys_addr_t offset, + hwaddr offset, uint64_t size) { memory_region_init(mr, name, size); @@ -992,10 +980,9 @@ void memory_region_init_rom_device(MemoryRegion *mr, mr->rom_device = true; mr->destructor = memory_region_destructor_rom_device; mr->ram_addr = qemu_ram_alloc(size, mr); - mr->ram_addr |= cpu_register_io_memory(mr); } -static uint64_t invalid_read(void *opaque, target_phys_addr_t addr, +static uint64_t invalid_read(void *opaque, hwaddr addr, unsigned size) { MemoryRegion *mr = opaque; @@ -1007,7 +994,7 @@ static uint64_t invalid_read(void *opaque, target_phys_addr_t addr, return -1U; } -static void invalid_write(void *opaque, target_phys_addr_t addr, uint64_t data, +static void invalid_write(void *opaque, hwaddr addr, uint64_t data, unsigned size) { MemoryRegion *mr = opaque; @@ -1034,6 +1021,7 @@ void memory_region_init_reservation(MemoryRegion *mr, void memory_region_destroy(MemoryRegion *mr) { assert(QTAILQ_EMPTY(&mr->subregions)); + assert(memory_region_transaction_depth == 0); mr->destructor(mr); memory_region_clear_coalescing(mr); g_free((char *)mr->name); @@ -1072,33 +1060,53 @@ void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client) { uint8_t mask = 1 << client; + memory_region_transaction_begin(); mr->dirty_log_mask = (mr->dirty_log_mask & ~mask) | (log * mask); - memory_region_update_topology(mr); + memory_region_update_pending |= mr->enabled; + memory_region_transaction_commit(); } -bool memory_region_get_dirty(MemoryRegion *mr, target_phys_addr_t addr, - target_phys_addr_t size, unsigned client) +bool memory_region_get_dirty(MemoryRegion *mr, hwaddr addr, + hwaddr size, unsigned client) { assert(mr->terminates); return cpu_physical_memory_get_dirty(mr->ram_addr + addr, size, 1 << client); } -void memory_region_set_dirty(MemoryRegion *mr, target_phys_addr_t addr, - target_phys_addr_t size) +void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr, + hwaddr size) { assert(mr->terminates); return cpu_physical_memory_set_dirty_range(mr->ram_addr + addr, size, -1); } +bool memory_region_test_and_clear_dirty(MemoryRegion *mr, hwaddr addr, + hwaddr size, unsigned client) +{ + bool ret; + assert(mr->terminates); + ret = cpu_physical_memory_get_dirty(mr->ram_addr + addr, size, + 1 << client); + if (ret) { + cpu_physical_memory_reset_dirty(mr->ram_addr + addr, + mr->ram_addr + addr + size, + 1 << client); + } + return ret; +} + + void memory_region_sync_dirty_bitmap(MemoryRegion *mr) { + AddressSpace *as; FlatRange *fr; - FOR_EACH_FLAT_RANGE(fr, &address_space_memory.current_map) { - if (fr->mr == mr) { - MEMORY_LISTENER_UPDATE_REGION(fr, &address_space_memory, - Forward, log_sync); + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + FOR_EACH_FLAT_RANGE(fr, as->current_map) { + if (fr->mr == mr) { + MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, log_sync); + } } } } @@ -1106,21 +1114,25 @@ void memory_region_sync_dirty_bitmap(MemoryRegion *mr) void memory_region_set_readonly(MemoryRegion *mr, bool readonly) { if (mr->readonly != readonly) { + memory_region_transaction_begin(); mr->readonly = readonly; - memory_region_update_topology(mr); + memory_region_update_pending |= mr->enabled; + memory_region_transaction_commit(); } } void memory_region_rom_device_set_readable(MemoryRegion *mr, bool readable) { if (mr->readable != readable) { + memory_region_transaction_begin(); mr->readable = readable; - memory_region_update_topology(mr); + memory_region_update_pending |= mr->enabled; + memory_region_transaction_commit(); } } -void memory_region_reset_dirty(MemoryRegion *mr, target_phys_addr_t addr, - target_phys_addr_t size, unsigned client) +void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr, + hwaddr size, unsigned client) { assert(mr->terminates); cpu_physical_memory_reset_dirty(mr->ram_addr + addr, @@ -1139,16 +1151,24 @@ void *memory_region_get_ram_ptr(MemoryRegion *mr) return qemu_get_ram_ptr(mr->ram_addr & TARGET_PAGE_MASK); } -static void memory_region_update_coalesced_range(MemoryRegion *mr) +static void memory_region_update_coalesced_range_as(MemoryRegion *mr, AddressSpace *as) { FlatRange *fr; CoalescedMemoryRange *cmr; AddrRange tmp; + MemoryRegionSection section; - FOR_EACH_FLAT_RANGE(fr, &address_space_memory.current_map) { + FOR_EACH_FLAT_RANGE(fr, as->current_map) { if (fr->mr == mr) { - qemu_unregister_coalesced_mmio(int128_get64(fr->addr.start), - int128_get64(fr->addr.size)); + section = (MemoryRegionSection) { + .address_space = as, + .offset_within_address_space = int128_get64(fr->addr.start), + .size = int128_get64(fr->addr.size), + }; + + MEMORY_LISTENER_CALL(coalesced_mmio_del, Reverse, §ion, + int128_get64(fr->addr.start), + int128_get64(fr->addr.size)); QTAILQ_FOREACH(cmr, &mr->coalesced, link) { tmp = addrrange_shift(cmr->addr, int128_sub(fr->addr.start, @@ -1157,13 +1177,23 @@ static void memory_region_update_coalesced_range(MemoryRegion *mr) continue; } tmp = addrrange_intersection(tmp, fr->addr); - qemu_register_coalesced_mmio(int128_get64(tmp.start), - int128_get64(tmp.size)); + MEMORY_LISTENER_CALL(coalesced_mmio_add, Forward, §ion, + int128_get64(tmp.start), + int128_get64(tmp.size)); } } } } +static void memory_region_update_coalesced_range(MemoryRegion *mr) +{ + AddressSpace *as; + + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + memory_region_update_coalesced_range_as(mr, as); + } +} + void memory_region_set_coalescing(MemoryRegion *mr) { memory_region_clear_coalescing(mr); @@ -1171,7 +1201,7 @@ void memory_region_set_coalescing(MemoryRegion *mr) } void memory_region_add_coalescing(MemoryRegion *mr, - target_phys_addr_t offset, + hwaddr offset, uint64_t size) { CoalescedMemoryRange *cmr = g_malloc(sizeof(*cmr)); @@ -1179,12 +1209,16 @@ void memory_region_add_coalescing(MemoryRegion *mr, cmr->addr = addrrange_make(int128_make64(offset), int128_make64(size)); QTAILQ_INSERT_TAIL(&mr->coalesced, cmr, link); memory_region_update_coalesced_range(mr); + memory_region_set_flush_coalesced(mr); } void memory_region_clear_coalescing(MemoryRegion *mr) { CoalescedMemoryRange *cmr; + qemu_flush_coalesced_mmio_buffer(); + mr->flush_coalesced_mmio = false; + while (!QTAILQ_EMPTY(&mr->coalesced)) { cmr = QTAILQ_FIRST(&mr->coalesced); QTAILQ_REMOVE(&mr->coalesced, cmr, link); @@ -1193,22 +1227,37 @@ void memory_region_clear_coalescing(MemoryRegion *mr) memory_region_update_coalesced_range(mr); } +void memory_region_set_flush_coalesced(MemoryRegion *mr) +{ + mr->flush_coalesced_mmio = true; +} + +void memory_region_clear_flush_coalesced(MemoryRegion *mr) +{ + qemu_flush_coalesced_mmio_buffer(); + if (QTAILQ_EMPTY(&mr->coalesced)) { + mr->flush_coalesced_mmio = false; + } +} + void memory_region_add_eventfd(MemoryRegion *mr, - target_phys_addr_t addr, + hwaddr addr, unsigned size, bool match_data, uint64_t data, - int fd) + EventNotifier *e) { MemoryRegionIoeventfd mrfd = { .addr.start = int128_make64(addr), .addr.size = int128_make64(size), .match_data = match_data, .data = data, - .fd = fd, + .e = e, }; unsigned i; + adjust_endianness(mr, &mrfd.data, size); + memory_region_transaction_begin(); for (i = 0; i < mr->ioeventfd_nb; ++i) { if (memory_region_ioeventfd_before(mrfd, mr->ioeventfds[i])) { break; @@ -1220,25 +1269,28 @@ void memory_region_add_eventfd(MemoryRegion *mr, memmove(&mr->ioeventfds[i+1], &mr->ioeventfds[i], sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb-1 - i)); mr->ioeventfds[i] = mrfd; - memory_region_update_topology(mr); + memory_region_update_pending |= mr->enabled; + memory_region_transaction_commit(); } void memory_region_del_eventfd(MemoryRegion *mr, - target_phys_addr_t addr, + hwaddr addr, unsigned size, bool match_data, uint64_t data, - int fd) + EventNotifier *e) { MemoryRegionIoeventfd mrfd = { .addr.start = int128_make64(addr), .addr.size = int128_make64(size), .match_data = match_data, .data = data, - .fd = fd, + .e = e, }; unsigned i; + adjust_endianness(mr, &mrfd.data, size); + memory_region_transaction_begin(); for (i = 0; i < mr->ioeventfd_nb; ++i) { if (memory_region_ioeventfd_equal(mrfd, mr->ioeventfds[i])) { break; @@ -1250,15 +1302,18 @@ void memory_region_del_eventfd(MemoryRegion *mr, --mr->ioeventfd_nb; mr->ioeventfds = g_realloc(mr->ioeventfds, sizeof(*mr->ioeventfds)*mr->ioeventfd_nb + 1); - memory_region_update_topology(mr); + memory_region_update_pending |= mr->enabled; + memory_region_transaction_commit(); } static void memory_region_add_subregion_common(MemoryRegion *mr, - target_phys_addr_t offset, + hwaddr offset, MemoryRegion *subregion) { MemoryRegion *other; + memory_region_transaction_begin(); + assert(!subregion->parent); subregion->parent = mr; subregion->addr = offset; @@ -1266,7 +1321,7 @@ static void memory_region_add_subregion_common(MemoryRegion *mr, if (subregion->may_overlap || other->may_overlap) { continue; } - if (int128_gt(int128_make64(offset), + if (int128_ge(int128_make64(offset), int128_add(int128_make64(other->addr), other->size)) || int128_le(int128_add(int128_make64(offset), subregion->size), int128_make64(other->addr))) { @@ -1291,12 +1346,13 @@ static void memory_region_add_subregion_common(MemoryRegion *mr, } QTAILQ_INSERT_TAIL(&mr->subregions, subregion, subregions_link); done: - memory_region_update_topology(mr); + memory_region_update_pending |= mr->enabled && subregion->enabled; + memory_region_transaction_commit(); } void memory_region_add_subregion(MemoryRegion *mr, - target_phys_addr_t offset, + hwaddr offset, MemoryRegion *subregion) { subregion->may_overlap = false; @@ -1305,7 +1361,7 @@ void memory_region_add_subregion(MemoryRegion *mr, } void memory_region_add_subregion_overlap(MemoryRegion *mr, - target_phys_addr_t offset, + hwaddr offset, MemoryRegion *subregion, unsigned priority) { @@ -1317,10 +1373,12 @@ void memory_region_add_subregion_overlap(MemoryRegion *mr, void memory_region_del_subregion(MemoryRegion *mr, MemoryRegion *subregion) { + memory_region_transaction_begin(); assert(subregion->parent == mr); subregion->parent = NULL; QTAILQ_REMOVE(&mr->subregions, subregion, subregions_link); - memory_region_update_topology(mr); + memory_region_update_pending |= mr->enabled && subregion->enabled; + memory_region_transaction_commit(); } void memory_region_set_enabled(MemoryRegion *mr, bool enabled) @@ -1328,11 +1386,13 @@ void memory_region_set_enabled(MemoryRegion *mr, bool enabled) if (enabled == mr->enabled) { return; } + memory_region_transaction_begin(); mr->enabled = enabled; - memory_region_update_topology(NULL); + memory_region_update_pending = true; + memory_region_transaction_commit(); } -void memory_region_set_address(MemoryRegion *mr, target_phys_addr_t addr) +void memory_region_set_address(MemoryRegion *mr, hwaddr addr) { MemoryRegion *parent = mr->parent; unsigned priority = mr->priority; @@ -1353,18 +1413,18 @@ void memory_region_set_address(MemoryRegion *mr, target_phys_addr_t addr) memory_region_transaction_commit(); } -void memory_region_set_alias_offset(MemoryRegion *mr, target_phys_addr_t offset) +void memory_region_set_alias_offset(MemoryRegion *mr, hwaddr offset) { - target_phys_addr_t old_offset = mr->alias_offset; - assert(mr->alias); - mr->alias_offset = offset; - if (offset == old_offset || !mr->parent) { + if (offset == mr->alias_offset) { return; } - memory_region_update_topology(mr); + memory_region_transaction_begin(); + mr->alias_offset = offset; + memory_region_update_pending |= mr->enabled; + memory_region_transaction_commit(); } ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr) @@ -1387,12 +1447,12 @@ static int cmp_flatrange_addr(const void *addr_, const void *fr_) static FlatRange *address_space_lookup(AddressSpace *as, AddrRange addr) { - return bsearch(&addr, as->current_map.ranges, as->current_map.nr, + return bsearch(&addr, as->current_map->ranges, as->current_map->nr, sizeof(FlatRange), cmp_flatrange_addr); } MemoryRegionSection memory_region_find(MemoryRegion *address_space, - target_phys_addr_t addr, uint64_t size) + hwaddr addr, uint64_t size) { AddressSpace *as = memory_region_to_address_space(address_space); AddrRange range = addrrange_make(int128_make64(addr), @@ -1404,7 +1464,7 @@ MemoryRegionSection memory_region_find(MemoryRegion *address_space, return ret; } - while (fr > as->current_map.ranges + while (fr > as->current_map->ranges && addrrange_intersects(fr[-1].addr, range)) { --fr; } @@ -1425,7 +1485,7 @@ void memory_global_sync_dirty_bitmap(MemoryRegion *address_space) AddressSpace *as = memory_region_to_address_space(address_space); FlatRange *fr; - FOR_EACH_FLAT_RANGE(fr, &as->current_map) { + FOR_EACH_FLAT_RANGE(fr, as->current_map) { MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, log_sync); } } @@ -1447,25 +1507,36 @@ static void listener_add_address_space(MemoryListener *listener, { FlatRange *fr; + if (listener->address_space_filter + && listener->address_space_filter != as) { + return; + } + if (global_dirty_log) { - listener->log_global_start(listener); + if (listener->log_global_start) { + listener->log_global_start(listener); + } } - FOR_EACH_FLAT_RANGE(fr, &as->current_map) { + + FOR_EACH_FLAT_RANGE(fr, as->current_map) { MemoryRegionSection section = { .mr = fr->mr, - .address_space = as->root, + .address_space = as, .offset_within_region = fr->offset_in_region, .size = int128_get64(fr->addr.size), .offset_within_address_space = int128_get64(fr->addr.start), .readonly = fr->readonly, }; - listener->region_add(listener, §ion); + if (listener->region_add) { + listener->region_add(listener, §ion); + } } } -void memory_listener_register(MemoryListener *listener, MemoryRegion *filter) +void memory_listener_register(MemoryListener *listener, AddressSpace *filter) { MemoryListener *other = NULL; + AddressSpace *as; listener->address_space_filter = filter; if (QTAILQ_EMPTY(&memory_listeners) @@ -1480,8 +1551,10 @@ void memory_listener_register(MemoryListener *listener, MemoryRegion *filter) } QTAILQ_INSERT_BEFORE(other, listener, link); } - listener_add_address_space(listener, &address_space_memory); - listener_add_address_space(listener, &address_space_io); + + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + listener_add_address_space(listener, as); + } } void memory_listener_unregister(MemoryListener *listener) @@ -1489,27 +1562,39 @@ void memory_listener_unregister(MemoryListener *listener) QTAILQ_REMOVE(&memory_listeners, listener, link); } -void set_system_memory_map(MemoryRegion *mr) +void address_space_init(AddressSpace *as, MemoryRegion *root) { - address_space_memory.root = mr; - memory_region_update_topology(NULL); + memory_region_transaction_begin(); + as->root = root; + as->current_map = g_new(FlatView, 1); + flatview_init(as->current_map); + QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link); + as->name = NULL; + memory_region_transaction_commit(); + address_space_init_dispatch(as); } -void set_system_io_map(MemoryRegion *mr) +void address_space_destroy(AddressSpace *as) { - address_space_io.root = mr; - memory_region_update_topology(NULL); + /* Flush out anything from MemoryListeners listening in on this */ + memory_region_transaction_begin(); + as->root = NULL; + memory_region_transaction_commit(); + QTAILQ_REMOVE(&address_spaces, as, address_spaces_link); + address_space_destroy_dispatch(as); + flatview_destroy(as->current_map); + g_free(as->current_map); } -uint64_t io_mem_read(int io_index, target_phys_addr_t addr, unsigned size) +uint64_t io_mem_read(MemoryRegion *mr, hwaddr addr, unsigned size) { - return memory_region_dispatch_read(io_mem_region[io_index], addr, size); + return memory_region_dispatch_read(mr, addr, size); } -void io_mem_write(int io_index, target_phys_addr_t addr, +void io_mem_write(MemoryRegion *mr, hwaddr addr, uint64_t val, unsigned size) { - memory_region_dispatch_write(io_mem_region[io_index], addr, val, size); + memory_region_dispatch_write(mr, addr, val, size); } typedef struct MemoryRegionList MemoryRegionList; @@ -1524,7 +1609,7 @@ typedef QTAILQ_HEAD(queue, MemoryRegionList) MemoryRegionListHead; static void mtree_print_mr(fprintf_function mon_printf, void *f, const MemoryRegion *mr, unsigned int level, - target_phys_addr_t base, + hwaddr base, MemoryRegionListHead *alias_print_queue) { MemoryRegionList *new_ml, *ml, *next_ml; @@ -1532,7 +1617,7 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f, const MemoryRegion *submr; unsigned int i; - if (!mr) { + if (!mr || !mr->enabled) { return; } @@ -1562,7 +1647,7 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f, "-" TARGET_FMT_plx "\n", base + mr->addr, base + mr->addr - + (target_phys_addr_t)int128_get64(mr->size) - 1, + + (hwaddr)int128_get64(mr->size) - 1, mr->priority, mr->readable ? 'R' : '-', !mr->readonly && !(mr->rom_device && mr->readable) ? 'W' @@ -1571,13 +1656,13 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f, mr->alias->name, mr->alias_offset, mr->alias_offset - + (target_phys_addr_t)int128_get64(mr->size) - 1); + + (hwaddr)int128_get64(mr->size) - 1); } else { mon_printf(f, TARGET_FMT_plx "-" TARGET_FMT_plx " (prio %d, %c%c): %s\n", base + mr->addr, base + mr->addr - + (target_phys_addr_t)int128_get64(mr->size) - 1, + + (hwaddr)int128_get64(mr->size) - 1, mr->priority, mr->readable ? 'R' : '-', !mr->readonly && !(mr->rom_device && mr->readable) ? 'W' @@ -1618,12 +1703,19 @@ void mtree_info(fprintf_function mon_printf, void *f) { MemoryRegionListHead ml_head; MemoryRegionList *ml, *ml2; + AddressSpace *as; QTAILQ_INIT(&ml_head); - mon_printf(f, "memory\n"); - mtree_print_mr(mon_printf, f, address_space_memory.root, 0, 0, &ml_head); + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + if (!as->name) { + continue; + } + mon_printf(f, "%s\n", as->name); + mtree_print_mr(mon_printf, f, as->root, 0, 0, &ml_head); + } + mon_printf(f, "aliases\n"); /* print aliased regions */ QTAILQ_FOREACH(ml, &ml_head, queue) { if (!ml->printed) { @@ -1635,11 +1727,4 @@ void mtree_info(fprintf_function mon_printf, void *f) QTAILQ_FOREACH_SAFE(ml, &ml_head, queue, ml2) { g_free(ml); } - - if (address_space_io.root && - !QTAILQ_EMPTY(&address_space_io.root->subregions)) { - QTAILQ_INIT(&ml_head); - mon_printf(f, "I/O\n"); - mtree_print_mr(mon_printf, f, address_space_io.root, 0, 0, &ml_head); - } }