X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=memory.c;h=741985385ca9de6b4d8c3d37f362b6e14e2ca7f5;hb=24c35a504e8b09e697d0268bbefb2a329b901611;hp=5ab21129a011f3672a7223086a4e076c9bc90f56;hpb=cd9244e48a6643752fccecd344d85d1c4de5cbad;p=qemu.git diff --git a/memory.c b/memory.c index 5ab21129a..741985385 100644 --- a/memory.c +++ b/memory.c @@ -9,6 +9,8 @@ * This work is licensed under the terms of the GNU GPL, version 2. See * the COPYING file in the top-level directory. * + * Contributions after 2012-01-13 are licensed under the terms of the + * GNU GPL, version 2 or (at your option) any later version. */ #include "memory.h" @@ -18,15 +20,17 @@ #include "kvm.h" #include -#define WANT_EXEC_OBSOLETE -#include "exec-obsolete.h" +#include "memory-internal.h" -unsigned memory_region_transaction_depth = 0; -static bool memory_region_update_pending = false; +static unsigned memory_region_transaction_depth; +static bool memory_region_update_pending; static bool global_dirty_log = false; -static QLIST_HEAD(, MemoryListener) memory_listeners - = QLIST_HEAD_INITIALIZER(memory_listeners); +static QTAILQ_HEAD(memory_listeners, MemoryListener) memory_listeners + = QTAILQ_HEAD_INITIALIZER(memory_listeners); + +static QTAILQ_HEAD(, AddressSpace) address_spaces + = QTAILQ_HEAD_INITIALIZER(address_spaces); typedef struct AddrRange AddrRange; @@ -80,6 +84,77 @@ static AddrRange addrrange_intersection(AddrRange r1, AddrRange r2) return addrrange_make(start, int128_sub(end, start)); } +enum ListenerDirection { Forward, Reverse }; + +static bool memory_listener_match(MemoryListener *listener, + MemoryRegionSection *section) +{ + return !listener->address_space_filter + || listener->address_space_filter == section->address_space; +} + +#define MEMORY_LISTENER_CALL_GLOBAL(_callback, _direction, _args...) \ + do { \ + MemoryListener *_listener; \ + \ + switch (_direction) { \ + case Forward: \ + QTAILQ_FOREACH(_listener, &memory_listeners, link) { \ + if (_listener->_callback) { \ + _listener->_callback(_listener, ##_args); \ + } \ + } \ + break; \ + case Reverse: \ + QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \ + memory_listeners, link) { \ + if (_listener->_callback) { \ + _listener->_callback(_listener, ##_args); \ + } \ + } \ + break; \ + default: \ + abort(); \ + } \ + } while (0) + +#define MEMORY_LISTENER_CALL(_callback, _direction, _section, _args...) \ + do { \ + MemoryListener *_listener; \ + \ + switch (_direction) { \ + case Forward: \ + QTAILQ_FOREACH(_listener, &memory_listeners, link) { \ + if (_listener->_callback \ + && memory_listener_match(_listener, _section)) { \ + _listener->_callback(_listener, _section, ##_args); \ + } \ + } \ + break; \ + case Reverse: \ + QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \ + memory_listeners, link) { \ + if (_listener->_callback \ + && memory_listener_match(_listener, _section)) { \ + _listener->_callback(_listener, _section, ##_args); \ + } \ + } \ + break; \ + default: \ + abort(); \ + } \ + } while (0) + +#define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback) \ + MEMORY_LISTENER_CALL(callback, dir, (&(MemoryRegionSection) { \ + .mr = (fr)->mr, \ + .address_space = (as), \ + .offset_within_region = (fr)->offset_in_region, \ + .size = int128_get64((fr)->addr.size), \ + .offset_within_address_space = int128_get64((fr)->addr.start), \ + .readonly = (fr)->readonly, \ + })) + struct CoalescedMemoryRange { AddrRange addr; QTAILQ_ENTRY(CoalescedMemoryRange) link; @@ -89,7 +164,7 @@ struct MemoryRegionIoeventfd { AddrRange addr; bool match_data; uint64_t data; - int fd; + EventNotifier *e; }; static bool memory_region_ioeventfd_before(MemoryRegionIoeventfd a, @@ -114,9 +189,9 @@ static bool memory_region_ioeventfd_before(MemoryRegionIoeventfd a, return false; } } - if (a.fd < b.fd) { + if (a.e < b.e) { return true; - } else if (a.fd > b.fd) { + } else if (a.e > b.e) { return false; } return false; @@ -135,7 +210,7 @@ typedef struct FlatView FlatView; /* Range of memory in the global map. Addresses are absolute. */ struct FlatRange { MemoryRegion *mr; - target_phys_addr_t offset_in_region; + hwaddr offset_in_region; AddrRange addr; uint8_t dirty_log_mask; bool readable; @@ -151,27 +226,8 @@ struct FlatView { unsigned nr_allocated; }; -typedef struct AddressSpace AddressSpace; typedef struct AddressSpaceOps AddressSpaceOps; -/* A system address space - I/O, memory, etc. */ -struct AddressSpace { - const AddressSpaceOps *ops; - MemoryRegion *root; - FlatView current_map; - int ioeventfd_nb; - MemoryRegionIoeventfd *ioeventfds; -}; - -struct AddressSpaceOps { - void (*range_add)(AddressSpace *as, FlatRange *fr); - void (*range_del)(AddressSpace *as, FlatRange *fr); - void (*log_start)(AddressSpace *as, FlatRange *fr); - void (*log_stop)(AddressSpace *as, FlatRange *fr); - void (*ioeventfd_add)(AddressSpace *as, MemoryRegionIoeventfd *fd); - void (*ioeventfd_del)(AddressSpace *as, MemoryRegionIoeventfd *fd); -}; - #define FOR_EACH_FLAT_RANGE(var, view) \ for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var) @@ -245,7 +301,7 @@ static void flatview_simplify(FlatView *view) } static void memory_region_read_accessor(void *opaque, - target_phys_addr_t addr, + hwaddr addr, uint64_t *value, unsigned size, unsigned shift, @@ -254,12 +310,15 @@ static void memory_region_read_accessor(void *opaque, MemoryRegion *mr = opaque; uint64_t tmp; + if (mr->flush_coalesced_mmio) { + qemu_flush_coalesced_mmio_buffer(); + } tmp = mr->ops->read(mr->opaque, addr, size); *value |= (tmp & mask) << shift; } static void memory_region_write_accessor(void *opaque, - target_phys_addr_t addr, + hwaddr addr, uint64_t *value, unsigned size, unsigned shift, @@ -268,17 +327,20 @@ static void memory_region_write_accessor(void *opaque, MemoryRegion *mr = opaque; uint64_t tmp; + if (mr->flush_coalesced_mmio) { + qemu_flush_coalesced_mmio_buffer(); + } tmp = (*value >> shift) & mask; mr->ops->write(mr->opaque, addr, tmp, size); } -static void access_with_adjusted_size(target_phys_addr_t addr, +static void access_with_adjusted_size(hwaddr addr, uint64_t *value, unsigned size, unsigned access_size_min, unsigned access_size_max, void (*access)(void *opaque, - target_phys_addr_t addr, + hwaddr addr, uint64_t *value, unsigned size, unsigned shift, @@ -303,75 +365,6 @@ static void access_with_adjusted_size(target_phys_addr_t addr, } } -static void as_memory_range_add(AddressSpace *as, FlatRange *fr) -{ - MemoryRegionSection section = { - .mr = fr->mr, - .offset_within_address_space = int128_get64(fr->addr.start), - .offset_within_region = fr->offset_in_region, - .size = int128_get64(fr->addr.size), - }; - - cpu_register_physical_memory_log(§ion, fr->readable, fr->readonly); -} - -static void as_memory_range_del(AddressSpace *as, FlatRange *fr) -{ - MemoryRegionSection section = { - .mr = &io_mem_unassigned, - .offset_within_address_space = int128_get64(fr->addr.start), - .offset_within_region = int128_get64(fr->addr.start), - .size = int128_get64(fr->addr.size), - }; - - cpu_register_physical_memory_log(§ion, true, false); -} - -static void as_memory_log_start(AddressSpace *as, FlatRange *fr) -{ -} - -static void as_memory_log_stop(AddressSpace *as, FlatRange *fr) -{ -} - -static void as_memory_ioeventfd_add(AddressSpace *as, MemoryRegionIoeventfd *fd) -{ - int r; - - assert(fd->match_data && int128_get64(fd->addr.size) == 4); - - r = kvm_set_ioeventfd_mmio_long(fd->fd, int128_get64(fd->addr.start), - fd->data, true); - if (r < 0) { - abort(); - } -} - -static void as_memory_ioeventfd_del(AddressSpace *as, MemoryRegionIoeventfd *fd) -{ - int r; - - r = kvm_set_ioeventfd_mmio_long(fd->fd, int128_get64(fd->addr.start), - fd->data, false); - if (r < 0) { - abort(); - } -} - -static const AddressSpaceOps address_space_ops_memory = { - .range_add = as_memory_range_add, - .range_del = as_memory_range_del, - .log_start = as_memory_log_start, - .log_stop = as_memory_log_stop, - .ioeventfd_add = as_memory_ioeventfd_add, - .ioeventfd_del = as_memory_ioeventfd_del, -}; - -static AddressSpace address_space_memory = { - .ops = &address_space_ops_memory, -}; - static const MemoryRegionPortio *find_portio(MemoryRegion *mr, uint64_t offset, unsigned width, bool write) { @@ -392,24 +385,28 @@ static void memory_region_iorange_read(IORange *iorange, unsigned width, uint64_t *data) { - MemoryRegion *mr = container_of(iorange, MemoryRegion, iorange); + MemoryRegionIORange *mrio + = container_of(iorange, MemoryRegionIORange, iorange); + MemoryRegion *mr = mrio->mr; + offset += mrio->offset; if (mr->ops->old_portio) { - const MemoryRegionPortio *mrp = find_portio(mr, offset, width, false); + const MemoryRegionPortio *mrp = find_portio(mr, offset - mrio->offset, + width, false); *data = ((uint64_t)1 << (width * 8)) - 1; if (mrp) { - *data = mrp->read(mr->opaque, offset + mr->offset); + *data = mrp->read(mr->opaque, offset); } else if (width == 2) { - mrp = find_portio(mr, offset, 1, false); + mrp = find_portio(mr, offset - mrio->offset, 1, false); assert(mrp); - *data = mrp->read(mr->opaque, offset + mr->offset) | - (mrp->read(mr->opaque, offset + mr->offset + 1) << 8); + *data = mrp->read(mr->opaque, offset) | + (mrp->read(mr->opaque, offset + 1) << 8); } return; } *data = 0; - access_with_adjusted_size(offset + mr->offset, data, width, + access_with_adjusted_size(offset, data, width, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_read_accessor, mr); @@ -420,90 +417,53 @@ static void memory_region_iorange_write(IORange *iorange, unsigned width, uint64_t data) { - MemoryRegion *mr = container_of(iorange, MemoryRegion, iorange); + MemoryRegionIORange *mrio + = container_of(iorange, MemoryRegionIORange, iorange); + MemoryRegion *mr = mrio->mr; + offset += mrio->offset; if (mr->ops->old_portio) { - const MemoryRegionPortio *mrp = find_portio(mr, offset, width, true); + const MemoryRegionPortio *mrp = find_portio(mr, offset - mrio->offset, + width, true); if (mrp) { - mrp->write(mr->opaque, offset + mr->offset, data); + mrp->write(mr->opaque, offset, data); } else if (width == 2) { - mrp = find_portio(mr, offset, 1, false); + mrp = find_portio(mr, offset - mrio->offset, 1, true); assert(mrp); - mrp->write(mr->opaque, offset + mr->offset, data & 0xff); - mrp->write(mr->opaque, offset + mr->offset + 1, data >> 8); + mrp->write(mr->opaque, offset, data & 0xff); + mrp->write(mr->opaque, offset + 1, data >> 8); } return; } - access_with_adjusted_size(offset + mr->offset, &data, width, + access_with_adjusted_size(offset, &data, width, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_write_accessor, mr); } -static const IORangeOps memory_region_iorange_ops = { - .read = memory_region_iorange_read, - .write = memory_region_iorange_write, -}; - -static void as_io_range_add(AddressSpace *as, FlatRange *fr) -{ - iorange_init(&fr->mr->iorange, &memory_region_iorange_ops, - int128_get64(fr->addr.start), int128_get64(fr->addr.size)); - ioport_register(&fr->mr->iorange); -} - -static void as_io_range_del(AddressSpace *as, FlatRange *fr) -{ - isa_unassign_ioport(int128_get64(fr->addr.start), - int128_get64(fr->addr.size)); -} - -static void as_io_ioeventfd_add(AddressSpace *as, MemoryRegionIoeventfd *fd) -{ - int r; - - assert(fd->match_data && int128_get64(fd->addr.size) == 2); - - r = kvm_set_ioeventfd_pio_word(fd->fd, int128_get64(fd->addr.start), - fd->data, true); - if (r < 0) { - abort(); - } -} - -static void as_io_ioeventfd_del(AddressSpace *as, MemoryRegionIoeventfd *fd) +static void memory_region_iorange_destructor(IORange *iorange) { - int r; - - r = kvm_set_ioeventfd_pio_word(fd->fd, int128_get64(fd->addr.start), - fd->data, false); - if (r < 0) { - abort(); - } + g_free(container_of(iorange, MemoryRegionIORange, iorange)); } -static const AddressSpaceOps address_space_ops_io = { - .range_add = as_io_range_add, - .range_del = as_io_range_del, - .ioeventfd_add = as_io_ioeventfd_add, - .ioeventfd_del = as_io_ioeventfd_del, -}; - -static AddressSpace address_space_io = { - .ops = &address_space_ops_io, +const IORangeOps memory_region_iorange_ops = { + .read = memory_region_iorange_read, + .write = memory_region_iorange_write, + .destructor = memory_region_iorange_destructor, }; static AddressSpace *memory_region_to_address_space(MemoryRegion *mr) { + AddressSpace *as; + while (mr->parent) { mr = mr->parent; } - if (mr == address_space_memory.root) { - return &address_space_memory; - } - if (mr == address_space_io.root) { - return &address_space_io; + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + if (mr == as->root) { + return as; + } } abort(); } @@ -519,7 +479,7 @@ static void render_memory_region(FlatView *view, { MemoryRegion *subregion; unsigned i; - target_phys_addr_t offset_in_region; + hwaddr offset_in_region; Int128 remain; Int128 now; FlatRange fr; @@ -580,12 +540,12 @@ static void render_memory_region(FlatView *view, offset_in_region += int128_get64(now); int128_subfrom(&remain, now); } - if (int128_eq(base, view->ranges[i].addr.start)) { - now = int128_min(remain, view->ranges[i].addr.size); - int128_addto(&base, now); - offset_in_region += int128_get64(now); - int128_subfrom(&remain, now); - } + now = int128_sub(int128_min(int128_add(base, remain), + addrrange_end(view->ranges[i].addr)), + base); + int128_addto(&base, now); + offset_in_region += int128_get64(now); + int128_subfrom(&remain, now); } if (int128_nz(remain)) { fr.mr = mr; @@ -605,8 +565,10 @@ static FlatView generate_memory_topology(MemoryRegion *mr) flatview_init(&view); - render_memory_region(&view, mr, int128_zero(), - addrrange_make(int128_zero(), int128_2_64()), false); + if (mr) { + render_memory_region(&view, mr, int128_zero(), + addrrange_make(int128_zero(), int128_2_64()), false); + } flatview_simplify(&view); return view; @@ -619,6 +581,8 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, unsigned fds_old_nb) { unsigned iold, inew; + MemoryRegionIoeventfd *fd; + MemoryRegionSection section; /* Generate a symmetric difference of the old and new fd sets, adding * and deleting as necessary. @@ -630,13 +594,27 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, && (inew == fds_new_nb || memory_region_ioeventfd_before(fds_old[iold], fds_new[inew]))) { - as->ops->ioeventfd_del(as, &fds_old[iold]); + fd = &fds_old[iold]; + section = (MemoryRegionSection) { + .address_space = as, + .offset_within_address_space = int128_get64(fd->addr.start), + .size = int128_get64(fd->addr.size), + }; + MEMORY_LISTENER_CALL(eventfd_del, Forward, §ion, + fd->match_data, fd->data, fd->e); ++iold; } else if (inew < fds_new_nb && (iold == fds_old_nb || memory_region_ioeventfd_before(fds_new[inew], fds_old[iold]))) { - as->ops->ioeventfd_add(as, &fds_new[inew]); + fd = &fds_new[inew]; + section = (MemoryRegionSection) { + .address_space = as, + .offset_within_address_space = int128_get64(fd->addr.start), + .size = int128_get64(fd->addr.size), + }; + MEMORY_LISTENER_CALL(eventfd_add, Reverse, §ion, + fd->match_data, fd->data, fd->e); ++inew; } else { ++iold; @@ -653,7 +631,7 @@ static void address_space_update_ioeventfds(AddressSpace *as) AddrRange tmp; unsigned i; - FOR_EACH_FLAT_RANGE(fr, &as->current_map) { + FOR_EACH_FLAT_RANGE(fr, as->current_map) { for (i = 0; i < fr->mr->ioeventfd_nb; ++i) { tmp = addrrange_shift(fr->mr->ioeventfds[i].addr, int128_sub(fr->addr.start, @@ -676,32 +654,6 @@ static void address_space_update_ioeventfds(AddressSpace *as) as->ioeventfd_nb = ioeventfd_nb; } -typedef void ListenerCallback(MemoryListener *listener, - MemoryRegionSection *mrs); - -/* Want "void (&MemoryListener::*callback)(const MemoryRegionSection& s)" */ -static void memory_listener_update_region(FlatRange *fr, AddressSpace *as, - size_t callback_offset) -{ - MemoryRegionSection section = { - .mr = fr->mr, - .address_space = as->root, - .offset_within_region = fr->offset_in_region, - .size = int128_get64(fr->addr.size), - .offset_within_address_space = int128_get64(fr->addr.start), - }; - MemoryListener *listener; - - QLIST_FOREACH(listener, &memory_listeners, link) { - ListenerCallback *callback - = *(ListenerCallback **)((void *)listener + callback_offset); - callback(listener, §ion); - } -} - -#define MEMORY_LISTENER_UPDATE_REGION(fr, as, callback) \ - memory_listener_update_region(fr, as, offsetof(MemoryListener, callback)) - static void address_space_update_topology_pass(AddressSpace *as, FlatView old_view, FlatView new_view, @@ -734,8 +686,7 @@ static void address_space_update_topology_pass(AddressSpace *as, /* In old, but (not in new, or in new but attributes changed). */ if (!adding) { - MEMORY_LISTENER_UPDATE_REGION(frold, as, region_del); - as->ops->range_del(as, frold); + MEMORY_LISTENER_UPDATE_REGION(frold, as, Reverse, region_del); } ++iold; @@ -743,12 +694,11 @@ static void address_space_update_topology_pass(AddressSpace *as, /* In both (logging may have changed) */ if (adding) { + MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_nop); if (frold->dirty_log_mask && !frnew->dirty_log_mask) { - MEMORY_LISTENER_UPDATE_REGION(frnew, as, log_stop); - as->ops->log_stop(as, frnew); + MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop); } else if (frnew->dirty_log_mask && !frold->dirty_log_mask) { - as->ops->log_start(as, frnew); - MEMORY_LISTENER_UPDATE_REGION(frnew, as, log_start); + MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, log_start); } } @@ -758,8 +708,7 @@ static void address_space_update_topology_pass(AddressSpace *as, /* In new */ if (adding) { - as->ops->range_add(as, frnew); - MEMORY_LISTENER_UPDATE_REGION(frnew, as, region_add); + MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_add); } ++inew; @@ -770,49 +719,38 @@ static void address_space_update_topology_pass(AddressSpace *as, static void address_space_update_topology(AddressSpace *as) { - FlatView old_view = as->current_map; + FlatView old_view = *as->current_map; FlatView new_view = generate_memory_topology(as->root); address_space_update_topology_pass(as, old_view, new_view, false); address_space_update_topology_pass(as, old_view, new_view, true); - as->current_map = new_view; + *as->current_map = new_view; flatview_destroy(&old_view); address_space_update_ioeventfds(as); } -static void memory_region_update_topology(MemoryRegion *mr) -{ - if (memory_region_transaction_depth) { - memory_region_update_pending |= !mr || mr->enabled; - return; - } - - if (mr && !mr->enabled) { - return; - } - - if (address_space_memory.root) { - address_space_update_topology(&address_space_memory); - } - if (address_space_io.root) { - address_space_update_topology(&address_space_io); - } - - memory_region_update_pending = false; -} - void memory_region_transaction_begin(void) { + qemu_flush_coalesced_mmio_buffer(); ++memory_region_transaction_depth; } void memory_region_transaction_commit(void) { + AddressSpace *as; + assert(memory_region_transaction_depth); --memory_region_transaction_depth; if (!memory_region_transaction_depth && memory_region_update_pending) { - memory_region_update_topology(NULL); + memory_region_update_pending = false; + MEMORY_LISTENER_CALL_GLOBAL(begin, Forward); + + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + address_space_update_topology(as); + } + + MEMORY_LISTENER_CALL_GLOBAL(commit, Forward); } } @@ -832,13 +770,11 @@ static void memory_region_destructor_ram_from_ptr(MemoryRegion *mr) static void memory_region_destructor_iomem(MemoryRegion *mr) { - cpu_unregister_io_memory(mr->ram_addr); } static void memory_region_destructor_rom_device(MemoryRegion *mr) { qemu_ram_free(mr->ram_addr & TARGET_PAGE_MASK); - cpu_unregister_io_memory(mr->ram_addr & ~TARGET_PAGE_MASK); } static bool memory_region_wrong_endianness(MemoryRegion *mr) @@ -861,7 +797,6 @@ void memory_region_init(MemoryRegion *mr, mr->size = int128_2_64(); } mr->addr = 0; - mr->offset = 0; mr->subpage = false; mr->enabled = true; mr->terminates = false; @@ -880,10 +815,11 @@ void memory_region_init(MemoryRegion *mr, mr->dirty_log_mask = 0; mr->ioeventfd_nb = 0; mr->ioeventfds = NULL; + mr->flush_coalesced_mmio = false; } static bool memory_region_access_valid(MemoryRegion *mr, - target_phys_addr_t addr, + hwaddr addr, unsigned size, bool is_write) { @@ -909,7 +845,7 @@ static bool memory_region_access_valid(MemoryRegion *mr, } static uint64_t memory_region_dispatch_read1(MemoryRegion *mr, - target_phys_addr_t addr, + hwaddr addr, unsigned size) { uint64_t data = 0; @@ -923,7 +859,7 @@ static uint64_t memory_region_dispatch_read1(MemoryRegion *mr, } /* FIXME: support unaligned access */ - access_with_adjusted_size(addr + mr->offset, &data, size, + access_with_adjusted_size(addr, &data, size, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_read_accessor, mr); @@ -950,7 +886,7 @@ static void adjust_endianness(MemoryRegion *mr, uint64_t *data, unsigned size) } static uint64_t memory_region_dispatch_read(MemoryRegion *mr, - target_phys_addr_t addr, + hwaddr addr, unsigned size) { uint64_t ret; @@ -961,7 +897,7 @@ static uint64_t memory_region_dispatch_read(MemoryRegion *mr, } static void memory_region_dispatch_write(MemoryRegion *mr, - target_phys_addr_t addr, + hwaddr addr, uint64_t data, unsigned size) { @@ -977,7 +913,7 @@ static void memory_region_dispatch_write(MemoryRegion *mr, } /* FIXME: support unaligned access */ - access_with_adjusted_size(addr + mr->offset, &data, size, + access_with_adjusted_size(addr, &data, size, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_write_accessor, mr); @@ -994,7 +930,7 @@ void memory_region_init_io(MemoryRegion *mr, mr->opaque = opaque; mr->terminates = true; mr->destructor = memory_region_destructor_iomem; - mr->ram_addr = cpu_register_io_memory(mr); + mr->ram_addr = ~(ram_addr_t)0; } void memory_region_init_ram(MemoryRegion *mr, @@ -1023,7 +959,7 @@ void memory_region_init_ram_ptr(MemoryRegion *mr, void memory_region_init_alias(MemoryRegion *mr, const char *name, MemoryRegion *orig, - target_phys_addr_t offset, + hwaddr offset, uint64_t size) { memory_region_init(mr, name, size); @@ -1044,12 +980,48 @@ void memory_region_init_rom_device(MemoryRegion *mr, mr->rom_device = true; mr->destructor = memory_region_destructor_rom_device; mr->ram_addr = qemu_ram_alloc(size, mr); - mr->ram_addr |= cpu_register_io_memory(mr); +} + +static uint64_t invalid_read(void *opaque, hwaddr addr, + unsigned size) +{ + MemoryRegion *mr = opaque; + + if (!mr->warning_printed) { + fprintf(stderr, "Invalid read from memory region %s\n", mr->name); + mr->warning_printed = true; + } + return -1U; +} + +static void invalid_write(void *opaque, hwaddr addr, uint64_t data, + unsigned size) +{ + MemoryRegion *mr = opaque; + + if (!mr->warning_printed) { + fprintf(stderr, "Invalid write to memory region %s\n", mr->name); + mr->warning_printed = true; + } +} + +static const MemoryRegionOps reservation_ops = { + .read = invalid_read, + .write = invalid_write, + .endianness = DEVICE_NATIVE_ENDIAN, +}; + +void memory_region_init_reservation(MemoryRegion *mr, + const char *name, + uint64_t size) +{ + memory_region_init_io(mr, &reservation_ops, mr, name, size); } void memory_region_destroy(MemoryRegion *mr) { assert(QTAILQ_EMPTY(&mr->subregions)); + assert(memory_region_transaction_depth == 0); mr->destructor(mr); memory_region_clear_coalescing(mr); g_free((char *)mr->name); @@ -1084,39 +1056,41 @@ bool memory_region_is_rom(MemoryRegion *mr) return mr->ram && mr->readonly; } -void memory_region_set_offset(MemoryRegion *mr, target_phys_addr_t offset) -{ - mr->offset = offset; -} - void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client) { uint8_t mask = 1 << client; + memory_region_transaction_begin(); mr->dirty_log_mask = (mr->dirty_log_mask & ~mask) | (log * mask); - memory_region_update_topology(mr); + memory_region_update_pending |= mr->enabled; + memory_region_transaction_commit(); } -bool memory_region_get_dirty(MemoryRegion *mr, target_phys_addr_t addr, - unsigned client) +bool memory_region_get_dirty(MemoryRegion *mr, hwaddr addr, + hwaddr size, unsigned client) { assert(mr->terminates); - return cpu_physical_memory_get_dirty(mr->ram_addr + addr, 1 << client); + return cpu_physical_memory_get_dirty(mr->ram_addr + addr, size, + 1 << client); } -void memory_region_set_dirty(MemoryRegion *mr, target_phys_addr_t addr) +void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr, + hwaddr size) { assert(mr->terminates); - return cpu_physical_memory_set_dirty(mr->ram_addr + addr); + return cpu_physical_memory_set_dirty_range(mr->ram_addr + addr, size, -1); } void memory_region_sync_dirty_bitmap(MemoryRegion *mr) { + AddressSpace *as; FlatRange *fr; - FOR_EACH_FLAT_RANGE(fr, &address_space_memory.current_map) { - if (fr->mr == mr) { - MEMORY_LISTENER_UPDATE_REGION(fr, &address_space_memory, log_sync); + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + FOR_EACH_FLAT_RANGE(fr, as->current_map) { + if (fr->mr == mr) { + MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, log_sync); + } } } } @@ -1124,21 +1098,25 @@ void memory_region_sync_dirty_bitmap(MemoryRegion *mr) void memory_region_set_readonly(MemoryRegion *mr, bool readonly) { if (mr->readonly != readonly) { + memory_region_transaction_begin(); mr->readonly = readonly; - memory_region_update_topology(mr); + memory_region_update_pending |= mr->enabled; + memory_region_transaction_commit(); } } void memory_region_rom_device_set_readable(MemoryRegion *mr, bool readable) { if (mr->readable != readable) { + memory_region_transaction_begin(); mr->readable = readable; - memory_region_update_topology(mr); + memory_region_update_pending |= mr->enabled; + memory_region_transaction_commit(); } } -void memory_region_reset_dirty(MemoryRegion *mr, target_phys_addr_t addr, - target_phys_addr_t size, unsigned client) +void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr, + hwaddr size, unsigned client) { assert(mr->terminates); cpu_physical_memory_reset_dirty(mr->ram_addr + addr, @@ -1157,16 +1135,24 @@ void *memory_region_get_ram_ptr(MemoryRegion *mr) return qemu_get_ram_ptr(mr->ram_addr & TARGET_PAGE_MASK); } -static void memory_region_update_coalesced_range(MemoryRegion *mr) +static void memory_region_update_coalesced_range_as(MemoryRegion *mr, AddressSpace *as) { FlatRange *fr; CoalescedMemoryRange *cmr; AddrRange tmp; + MemoryRegionSection section; - FOR_EACH_FLAT_RANGE(fr, &address_space_memory.current_map) { + FOR_EACH_FLAT_RANGE(fr, as->current_map) { if (fr->mr == mr) { - qemu_unregister_coalesced_mmio(int128_get64(fr->addr.start), - int128_get64(fr->addr.size)); + section = (MemoryRegionSection) { + .address_space = as, + .offset_within_address_space = int128_get64(fr->addr.start), + .size = int128_get64(fr->addr.size), + }; + + MEMORY_LISTENER_CALL(coalesced_mmio_del, Reverse, §ion, + int128_get64(fr->addr.start), + int128_get64(fr->addr.size)); QTAILQ_FOREACH(cmr, &mr->coalesced, link) { tmp = addrrange_shift(cmr->addr, int128_sub(fr->addr.start, @@ -1175,13 +1161,23 @@ static void memory_region_update_coalesced_range(MemoryRegion *mr) continue; } tmp = addrrange_intersection(tmp, fr->addr); - qemu_register_coalesced_mmio(int128_get64(tmp.start), - int128_get64(tmp.size)); + MEMORY_LISTENER_CALL(coalesced_mmio_add, Forward, §ion, + int128_get64(tmp.start), + int128_get64(tmp.size)); } } } } +static void memory_region_update_coalesced_range(MemoryRegion *mr) +{ + AddressSpace *as; + + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + memory_region_update_coalesced_range_as(mr, as); + } +} + void memory_region_set_coalescing(MemoryRegion *mr) { memory_region_clear_coalescing(mr); @@ -1189,7 +1185,7 @@ void memory_region_set_coalescing(MemoryRegion *mr) } void memory_region_add_coalescing(MemoryRegion *mr, - target_phys_addr_t offset, + hwaddr offset, uint64_t size) { CoalescedMemoryRange *cmr = g_malloc(sizeof(*cmr)); @@ -1197,12 +1193,16 @@ void memory_region_add_coalescing(MemoryRegion *mr, cmr->addr = addrrange_make(int128_make64(offset), int128_make64(size)); QTAILQ_INSERT_TAIL(&mr->coalesced, cmr, link); memory_region_update_coalesced_range(mr); + memory_region_set_flush_coalesced(mr); } void memory_region_clear_coalescing(MemoryRegion *mr) { CoalescedMemoryRange *cmr; + qemu_flush_coalesced_mmio_buffer(); + mr->flush_coalesced_mmio = false; + while (!QTAILQ_EMPTY(&mr->coalesced)) { cmr = QTAILQ_FIRST(&mr->coalesced); QTAILQ_REMOVE(&mr->coalesced, cmr, link); @@ -1211,22 +1211,37 @@ void memory_region_clear_coalescing(MemoryRegion *mr) memory_region_update_coalesced_range(mr); } +void memory_region_set_flush_coalesced(MemoryRegion *mr) +{ + mr->flush_coalesced_mmio = true; +} + +void memory_region_clear_flush_coalesced(MemoryRegion *mr) +{ + qemu_flush_coalesced_mmio_buffer(); + if (QTAILQ_EMPTY(&mr->coalesced)) { + mr->flush_coalesced_mmio = false; + } +} + void memory_region_add_eventfd(MemoryRegion *mr, - target_phys_addr_t addr, + hwaddr addr, unsigned size, bool match_data, uint64_t data, - int fd) + EventNotifier *e) { MemoryRegionIoeventfd mrfd = { .addr.start = int128_make64(addr), .addr.size = int128_make64(size), .match_data = match_data, .data = data, - .fd = fd, + .e = e, }; unsigned i; + adjust_endianness(mr, &mrfd.data, size); + memory_region_transaction_begin(); for (i = 0; i < mr->ioeventfd_nb; ++i) { if (memory_region_ioeventfd_before(mrfd, mr->ioeventfds[i])) { break; @@ -1238,25 +1253,28 @@ void memory_region_add_eventfd(MemoryRegion *mr, memmove(&mr->ioeventfds[i+1], &mr->ioeventfds[i], sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb-1 - i)); mr->ioeventfds[i] = mrfd; - memory_region_update_topology(mr); + memory_region_update_pending |= mr->enabled; + memory_region_transaction_commit(); } void memory_region_del_eventfd(MemoryRegion *mr, - target_phys_addr_t addr, + hwaddr addr, unsigned size, bool match_data, uint64_t data, - int fd) + EventNotifier *e) { MemoryRegionIoeventfd mrfd = { .addr.start = int128_make64(addr), .addr.size = int128_make64(size), .match_data = match_data, .data = data, - .fd = fd, + .e = e, }; unsigned i; + adjust_endianness(mr, &mrfd.data, size); + memory_region_transaction_begin(); for (i = 0; i < mr->ioeventfd_nb; ++i) { if (memory_region_ioeventfd_equal(mrfd, mr->ioeventfds[i])) { break; @@ -1268,15 +1286,18 @@ void memory_region_del_eventfd(MemoryRegion *mr, --mr->ioeventfd_nb; mr->ioeventfds = g_realloc(mr->ioeventfds, sizeof(*mr->ioeventfds)*mr->ioeventfd_nb + 1); - memory_region_update_topology(mr); + memory_region_update_pending |= mr->enabled; + memory_region_transaction_commit(); } static void memory_region_add_subregion_common(MemoryRegion *mr, - target_phys_addr_t offset, + hwaddr offset, MemoryRegion *subregion) { MemoryRegion *other; + memory_region_transaction_begin(); + assert(!subregion->parent); subregion->parent = mr; subregion->addr = offset; @@ -1309,12 +1330,13 @@ static void memory_region_add_subregion_common(MemoryRegion *mr, } QTAILQ_INSERT_TAIL(&mr->subregions, subregion, subregions_link); done: - memory_region_update_topology(mr); + memory_region_update_pending |= mr->enabled && subregion->enabled; + memory_region_transaction_commit(); } void memory_region_add_subregion(MemoryRegion *mr, - target_phys_addr_t offset, + hwaddr offset, MemoryRegion *subregion) { subregion->may_overlap = false; @@ -1323,7 +1345,7 @@ void memory_region_add_subregion(MemoryRegion *mr, } void memory_region_add_subregion_overlap(MemoryRegion *mr, - target_phys_addr_t offset, + hwaddr offset, MemoryRegion *subregion, unsigned priority) { @@ -1335,10 +1357,12 @@ void memory_region_add_subregion_overlap(MemoryRegion *mr, void memory_region_del_subregion(MemoryRegion *mr, MemoryRegion *subregion) { + memory_region_transaction_begin(); assert(subregion->parent == mr); subregion->parent = NULL; QTAILQ_REMOVE(&mr->subregions, subregion, subregions_link); - memory_region_update_topology(mr); + memory_region_update_pending |= mr->enabled && subregion->enabled; + memory_region_transaction_commit(); } void memory_region_set_enabled(MemoryRegion *mr, bool enabled) @@ -1346,11 +1370,13 @@ void memory_region_set_enabled(MemoryRegion *mr, bool enabled) if (enabled == mr->enabled) { return; } + memory_region_transaction_begin(); mr->enabled = enabled; - memory_region_update_topology(NULL); + memory_region_update_pending = true; + memory_region_transaction_commit(); } -void memory_region_set_address(MemoryRegion *mr, target_phys_addr_t addr) +void memory_region_set_address(MemoryRegion *mr, hwaddr addr) { MemoryRegion *parent = mr->parent; unsigned priority = mr->priority; @@ -1371,18 +1397,18 @@ void memory_region_set_address(MemoryRegion *mr, target_phys_addr_t addr) memory_region_transaction_commit(); } -void memory_region_set_alias_offset(MemoryRegion *mr, target_phys_addr_t offset) +void memory_region_set_alias_offset(MemoryRegion *mr, hwaddr offset) { - target_phys_addr_t old_offset = mr->alias_offset; - assert(mr->alias); - mr->alias_offset = offset; - if (offset == old_offset || !mr->parent) { + if (offset == mr->alias_offset) { return; } - memory_region_update_topology(mr); + memory_region_transaction_begin(); + mr->alias_offset = offset; + memory_region_update_pending |= mr->enabled; + memory_region_transaction_commit(); } ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr) @@ -1405,12 +1431,12 @@ static int cmp_flatrange_addr(const void *addr_, const void *fr_) static FlatRange *address_space_lookup(AddressSpace *as, AddrRange addr) { - return bsearch(&addr, as->current_map.ranges, as->current_map.nr, + return bsearch(&addr, as->current_map->ranges, as->current_map->nr, sizeof(FlatRange), cmp_flatrange_addr); } MemoryRegionSection memory_region_find(MemoryRegion *address_space, - target_phys_addr_t addr, uint64_t size) + hwaddr addr, uint64_t size) { AddressSpace *as = memory_region_to_address_space(address_space); AddrRange range = addrrange_make(int128_make64(addr), @@ -1422,7 +1448,7 @@ MemoryRegionSection memory_region_find(MemoryRegion *address_space, return ret; } - while (fr > as->current_map.ranges + while (fr > as->current_map->ranges && addrrange_intersects(fr[-1].addr, range)) { --fr; } @@ -1434,6 +1460,7 @@ MemoryRegionSection memory_region_find(MemoryRegion *address_space, fr->addr.start)); ret.size = int128_get64(range.size); ret.offset_within_address_space = int128_get64(range.start); + ret.readonly = fr->readonly; return ret; } @@ -1442,31 +1469,21 @@ void memory_global_sync_dirty_bitmap(MemoryRegion *address_space) AddressSpace *as = memory_region_to_address_space(address_space); FlatRange *fr; - FOR_EACH_FLAT_RANGE(fr, &as->current_map) { - MEMORY_LISTENER_UPDATE_REGION(fr, as, log_sync); + FOR_EACH_FLAT_RANGE(fr, as->current_map) { + MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, log_sync); } } void memory_global_dirty_log_start(void) { - MemoryListener *listener; - - cpu_physical_memory_set_dirty_tracking(1); global_dirty_log = true; - QLIST_FOREACH(listener, &memory_listeners, link) { - listener->log_global_start(listener); - } + MEMORY_LISTENER_CALL_GLOBAL(log_global_start, Forward); } void memory_global_dirty_log_stop(void) { - MemoryListener *listener; - global_dirty_log = false; - QLIST_FOREACH(listener, &memory_listeners, link) { - listener->log_global_stop(listener); - } - cpu_physical_memory_set_dirty_tracking(0); + MEMORY_LISTENER_CALL_GLOBAL(log_global_stop, Reverse); } static void listener_add_address_space(MemoryListener *listener, @@ -1474,54 +1491,94 @@ static void listener_add_address_space(MemoryListener *listener, { FlatRange *fr; + if (listener->address_space_filter + && listener->address_space_filter != as) { + return; + } + if (global_dirty_log) { - listener->log_global_start(listener); + if (listener->log_global_start) { + listener->log_global_start(listener); + } } - FOR_EACH_FLAT_RANGE(fr, &as->current_map) { + + FOR_EACH_FLAT_RANGE(fr, as->current_map) { MemoryRegionSection section = { .mr = fr->mr, - .address_space = as->root, + .address_space = as, .offset_within_region = fr->offset_in_region, .size = int128_get64(fr->addr.size), .offset_within_address_space = int128_get64(fr->addr.start), + .readonly = fr->readonly, }; - listener->region_add(listener, §ion); + if (listener->region_add) { + listener->region_add(listener, §ion); + } } } -void memory_listener_register(MemoryListener *listener) +void memory_listener_register(MemoryListener *listener, AddressSpace *filter) { - QLIST_INSERT_HEAD(&memory_listeners, listener, link); - listener_add_address_space(listener, &address_space_memory); - listener_add_address_space(listener, &address_space_io); + MemoryListener *other = NULL; + AddressSpace *as; + + listener->address_space_filter = filter; + if (QTAILQ_EMPTY(&memory_listeners) + || listener->priority >= QTAILQ_LAST(&memory_listeners, + memory_listeners)->priority) { + QTAILQ_INSERT_TAIL(&memory_listeners, listener, link); + } else { + QTAILQ_FOREACH(other, &memory_listeners, link) { + if (listener->priority < other->priority) { + break; + } + } + QTAILQ_INSERT_BEFORE(other, listener, link); + } + + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + listener_add_address_space(listener, as); + } } void memory_listener_unregister(MemoryListener *listener) { - QLIST_REMOVE(listener, link); + QTAILQ_REMOVE(&memory_listeners, listener, link); } -void set_system_memory_map(MemoryRegion *mr) +void address_space_init(AddressSpace *as, MemoryRegion *root) { - address_space_memory.root = mr; - memory_region_update_topology(NULL); + memory_region_transaction_begin(); + as->root = root; + as->current_map = g_new(FlatView, 1); + flatview_init(as->current_map); + QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link); + as->name = NULL; + memory_region_transaction_commit(); + address_space_init_dispatch(as); } -void set_system_io_map(MemoryRegion *mr) +void address_space_destroy(AddressSpace *as) { - address_space_io.root = mr; - memory_region_update_topology(NULL); + /* Flush out anything from MemoryListeners listening in on this */ + memory_region_transaction_begin(); + as->root = NULL; + memory_region_transaction_commit(); + QTAILQ_REMOVE(&address_spaces, as, address_spaces_link); + address_space_destroy_dispatch(as); + flatview_destroy(as->current_map); + g_free(as->current_map); } -uint64_t io_mem_read(int io_index, target_phys_addr_t addr, unsigned size) +uint64_t io_mem_read(MemoryRegion *mr, hwaddr addr, unsigned size) { - return memory_region_dispatch_read(io_mem_region[io_index], addr, size); + return memory_region_dispatch_read(mr, addr, size); } -void io_mem_write(int io_index, target_phys_addr_t addr, +void io_mem_write(MemoryRegion *mr, hwaddr addr, uint64_t val, unsigned size) { - memory_region_dispatch_write(io_mem_region[io_index], addr, val, size); + memory_region_dispatch_write(mr, addr, val, size); } typedef struct MemoryRegionList MemoryRegionList; @@ -1536,7 +1593,7 @@ typedef QTAILQ_HEAD(queue, MemoryRegionList) MemoryRegionListHead; static void mtree_print_mr(fprintf_function mon_printf, void *f, const MemoryRegion *mr, unsigned int level, - target_phys_addr_t base, + hwaddr base, MemoryRegionListHead *alias_print_queue) { MemoryRegionList *new_ml, *ml, *next_ml; @@ -1544,7 +1601,7 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f, const MemoryRegion *submr; unsigned int i; - if (!mr) { + if (!mr || !mr->enabled) { return; } @@ -1569,23 +1626,31 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f, ml->printed = false; QTAILQ_INSERT_TAIL(alias_print_queue, ml, queue); } - mon_printf(f, TARGET_FMT_plx "-" TARGET_FMT_plx " (prio %d): alias %s @%s " - TARGET_FMT_plx "-" TARGET_FMT_plx "\n", + mon_printf(f, TARGET_FMT_plx "-" TARGET_FMT_plx + " (prio %d, %c%c): alias %s @%s " TARGET_FMT_plx + "-" TARGET_FMT_plx "\n", base + mr->addr, base + mr->addr - + (target_phys_addr_t)int128_get64(mr->size) - 1, + + (hwaddr)int128_get64(mr->size) - 1, mr->priority, + mr->readable ? 'R' : '-', + !mr->readonly && !(mr->rom_device && mr->readable) ? 'W' + : '-', mr->name, mr->alias->name, mr->alias_offset, mr->alias_offset - + (target_phys_addr_t)int128_get64(mr->size) - 1); + + (hwaddr)int128_get64(mr->size) - 1); } else { - mon_printf(f, TARGET_FMT_plx "-" TARGET_FMT_plx " (prio %d): %s\n", + mon_printf(f, + TARGET_FMT_plx "-" TARGET_FMT_plx " (prio %d, %c%c): %s\n", base + mr->addr, base + mr->addr - + (target_phys_addr_t)int128_get64(mr->size) - 1, + + (hwaddr)int128_get64(mr->size) - 1, mr->priority, + mr->readable ? 'R' : '-', + !mr->readonly && !(mr->rom_device && mr->readable) ? 'W' + : '-', mr->name); } @@ -1622,12 +1687,19 @@ void mtree_info(fprintf_function mon_printf, void *f) { MemoryRegionListHead ml_head; MemoryRegionList *ml, *ml2; + AddressSpace *as; QTAILQ_INIT(&ml_head); - mon_printf(f, "memory\n"); - mtree_print_mr(mon_printf, f, address_space_memory.root, 0, 0, &ml_head); + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + if (!as->name) { + continue; + } + mon_printf(f, "%s\n", as->name); + mtree_print_mr(mon_printf, f, as->root, 0, 0, &ml_head); + } + mon_printf(f, "aliases\n"); /* print aliased regions */ QTAILQ_FOREACH(ml, &ml_head, queue) { if (!ml->printed) { @@ -1639,11 +1711,4 @@ void mtree_info(fprintf_function mon_printf, void *f) QTAILQ_FOREACH_SAFE(ml, &ml_head, queue, ml2) { g_free(ml); } - - if (address_space_io.root && - !QTAILQ_EMPTY(&address_space_io.root->subregions)) { - QTAILQ_INIT(&ml_head); - mon_printf(f, "I/O\n"); - mtree_print_mr(mon_printf, f, address_space_io.root, 0, 0, &ml_head); - } }