#include "qemu/accel.h"
#include "hw/boards.h"
#include "migration/vmstate.h"
+#include "exec/address-spaces.h"
//#define DEBUG_UNASSIGNED
static unsigned memory_region_transaction_depth;
static bool memory_region_update_pending;
static bool ioeventfd_update_pending;
-bool global_dirty_log;
+unsigned int global_dirty_tracking;
static QTAILQ_HEAD(, MemoryListener) memory_listeners
= QTAILQ_HEAD_INITIALIZER(memory_listeners);
static bool memory_region_big_endian(MemoryRegion *mr)
{
-#ifdef TARGET_WORDS_BIGENDIAN
+#if TARGET_BIG_ENDIAN
return mr->ops->endianness != DEVICE_LITTLE_ENDIAN;
#else
return mr->ops->endianness == DEVICE_BIG_ENDIAN;
unsigned access_size;
unsigned i;
MemTxResult r = MEMTX_OK;
+ bool reentrancy_guard_applied = false;
if (!access_size_min) {
access_size_min = 1;
access_size_max = 4;
}
+ /* Do not allow more than one simultaneous access to a device's IO Regions */
+ if (mr->dev && !mr->disable_reentrancy_guard &&
+ !mr->ram_device && !mr->ram && !mr->rom_device && !mr->readonly) {
+ if (mr->dev->mem_reentrancy_guard.engaged_in_io) {
+ warn_report_once("Blocked re-entrant IO on MemoryRegion: "
+ "%s at addr: 0x%" HWADDR_PRIX,
+ memory_region_name(mr), addr);
+ return MEMTX_ACCESS_ERROR;
+ }
+ mr->dev->mem_reentrancy_guard.engaged_in_io = true;
+ reentrancy_guard_applied = true;
+ }
+
/* FIXME: support unaligned access? */
access_size = MAX(MIN(size, access_size_max), access_size_min);
access_mask = MAKE_64BIT_MASK(0, access_size * 8);
access_mask, attrs);
}
}
+ if (mr->dev && reentrancy_guard_applied) {
+ mr->dev->mem_reentrancy_guard.engaged_in_io = false;
+ }
return r;
}
}
mr->name = g_strdup(name);
mr->owner = owner;
+ mr->dev = (DeviceState *) object_dynamic_cast(mr->owner, TYPE_DEVICE);
mr->ram_block = NULL;
if (name) {
unsigned size)
{
#ifdef DEBUG_UNASSIGNED
- printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
+ printf("Unassigned mem read " HWADDR_FMT_plx "\n", addr);
#endif
return 0;
}
uint64_t val, unsigned size)
{
#ifdef DEBUG_UNASSIGNED
- printf("Unassigned mem write " TARGET_FMT_plx " = 0x%"PRIx64"\n", addr, val);
+ printf("Unassigned mem write " HWADDR_FMT_plx " = 0x%"PRIx64"\n", addr, val);
#endif
}
unsigned size = memop_size(op);
MemTxResult r;
+ if (mr->alias) {
+ return memory_region_dispatch_read(mr->alias,
+ mr->alias_offset + addr,
+ pval, op, attrs);
+ }
if (!memory_region_access_valid(mr, addr, size, false, attrs)) {
*pval = unassigned_mem_read(mr, addr, size);
return MEMTX_DECODE_ERROR;
{
unsigned size = memop_size(op);
+ if (mr->alias) {
+ return memory_region_dispatch_write(mr->alias,
+ mr->alias_offset + addr,
+ data, op, attrs);
+ }
if (!memory_region_access_valid(mr, addr, size, true, attrs)) {
unassigned_mem_write(mr, addr, data, size);
return MEMTX_DECODE_ERROR;
uint64_t align,
uint32_t ram_flags,
const char *path,
- bool readonly,
+ ram_addr_t offset,
Error **errp)
{
Error *err = NULL;
memory_region_init(mr, owner, name, size);
mr->ram = true;
- mr->readonly = readonly;
+ mr->readonly = !!(ram_flags & RAM_READONLY);
mr->terminates = true;
mr->destructor = memory_region_destructor_ram;
mr->align = align;
mr->ram_block = qemu_ram_alloc_from_file(size, mr, ram_flags, path,
- readonly, &err);
+ offset, &err);
if (err) {
mr->size = int128_zero();
object_unparent(OBJECT(mr));
Error *err = NULL;
memory_region_init(mr, owner, name, size);
mr->ram = true;
+ mr->readonly = !!(ram_flags & RAM_READONLY);
mr->terminates = true;
mr->destructor = memory_region_destructor_ram;
mr->ram_block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, offset,
- false, &err);
+ &err);
if (err) {
mr->size = int128_zero();
object_unparent(OBJECT(mr));
uint8_t mask = mr->dirty_log_mask;
RAMBlock *rb = mr->ram_block;
- if (global_dirty_log && ((rb && qemu_ram_is_migratable(rb)) ||
+ if (global_dirty_tracking && ((rb && qemu_ram_is_migratable(rb)) ||
memory_region_is_iommu(mr))) {
mask |= (1 << DIRTY_MEMORY_MIGRATION);
}
}
}
+void memory_region_unmap_iommu_notifier_range(IOMMUNotifier *notifier)
+{
+ IOMMUTLBEvent event;
+
+ event.type = IOMMU_NOTIFIER_UNMAP;
+ event.entry.target_as = &address_space_memory;
+ event.entry.iova = notifier->start;
+ event.entry.perm = IOMMU_NONE;
+ event.entry.addr_mask = notifier->end - notifier->start;
+
+ memory_region_notify_iommu_one(notifier, &event);
+}
+
void memory_region_notify_iommu(IOMMUMemoryRegion *iommu_mr,
int iommu_idx,
IOMMUTLBEvent event)
return rdmc->replay_populated(rdm, section, replay_fn, opaque);
}
+void ram_discard_manager_replay_discarded(const RamDiscardManager *rdm,
+ MemoryRegionSection *section,
+ ReplayRamDiscard replay_fn,
+ void *opaque)
+{
+ RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_GET_CLASS(rdm);
+
+ g_assert(rdmc->replay_discarded);
+ rdmc->replay_discarded(rdm, section, replay_fn, opaque);
+}
+
void ram_discard_manager_register_listener(RamDiscardManager *rdm,
RamDiscardListener *rdl,
MemoryRegionSection *section)
rdmc->unregister_listener(rdm, rdl);
}
+/* Called with rcu_read_lock held. */
+bool memory_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr,
+ ram_addr_t *ram_addr, bool *read_only,
+ bool *mr_has_discard_manager)
+{
+ MemoryRegion *mr;
+ hwaddr xlat;
+ hwaddr len = iotlb->addr_mask + 1;
+ bool writable = iotlb->perm & IOMMU_WO;
+
+ if (mr_has_discard_manager) {
+ *mr_has_discard_manager = false;
+ }
+ /*
+ * The IOMMU TLB entry we have just covers translation through
+ * this IOMMU to its immediate target. We need to translate
+ * it the rest of the way through to memory.
+ */
+ mr = address_space_translate(&address_space_memory, iotlb->translated_addr,
+ &xlat, &len, writable, MEMTXATTRS_UNSPECIFIED);
+ if (!memory_region_is_ram(mr)) {
+ error_report("iommu map to non memory area %" HWADDR_PRIx "", xlat);
+ return false;
+ } else if (memory_region_has_ram_discard_manager(mr)) {
+ RamDiscardManager *rdm = memory_region_get_ram_discard_manager(mr);
+ MemoryRegionSection tmp = {
+ .mr = mr,
+ .offset_within_region = xlat,
+ .size = int128_make64(len),
+ };
+ if (mr_has_discard_manager) {
+ *mr_has_discard_manager = true;
+ }
+ /*
+ * Malicious VMs can map memory into the IOMMU, which is expected
+ * to remain discarded. vfio will pin all pages, populating memory.
+ * Disallow that. vmstate priorities make sure any RamDiscardManager
+ * were already restored before IOMMUs are restored.
+ */
+ if (!ram_discard_manager_is_populated(rdm, &tmp)) {
+ error_report("iommu map to discarded memory (e.g., unplugged via"
+ " virtio-mem): %" HWADDR_PRIx "",
+ iotlb->translated_addr);
+ return false;
+ }
+ }
+
+ /*
+ * Translation truncates length to the IOMMU page size,
+ * check that it did not truncate too much.
+ */
+ if (len & iotlb->addr_mask) {
+ error_report("iommu has granularity incompatible with target AS");
+ return false;
+ }
+
+ if (vaddr) {
+ *vaddr = memory_region_get_ram_ptr(mr) + xlat;
+ }
+
+ if (ram_addr) {
+ *ram_addr = memory_region_get_ram_addr(mr) + xlat;
+ }
+
+ if (read_only) {
+ *read_only = !writable || mr->readonly;
+ }
+
+ return true;
+}
+
void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client)
{
uint8_t mask = 1 << client;
* If memory region `mr' is NULL, do global sync. Otherwise, sync
* dirty bitmap for the specified memory region.
*/
-static void memory_region_sync_dirty_bitmap(MemoryRegion *mr)
+static void memory_region_sync_dirty_bitmap(MemoryRegion *mr, bool last_stage)
{
MemoryListener *listener;
AddressSpace *as;
* is to do a global sync, because we are not capable to
* sync in a finer granularity.
*/
- listener->log_sync_global(listener);
+ listener->log_sync_global(listener, last_stage);
trace_memory_region_sync_dirty(mr ? mr->name : "(all)", listener->name, 1);
}
}
{
DirtyBitmapSnapshot *snapshot;
assert(mr->ram_block);
- memory_region_sync_dirty_bitmap(mr);
+ memory_region_sync_dirty_bitmap(mr, false);
snapshot = cpu_physical_memory_snapshot_and_clear_dirty(mr, addr, size, client);
memory_global_after_dirty_log_sync();
return snapshot;
int memory_region_get_fd(MemoryRegion *mr)
{
- int fd;
-
RCU_READ_LOCK_GUARD();
while (mr->alias) {
mr = mr->alias;
}
- fd = mr->ram_block->fd;
-
- return fd;
+ return mr->ram_block->fd;
}
void *memory_region_get_ram_ptr(MemoryRegion *mr)
{
- void *ptr;
uint64_t offset = 0;
RCU_READ_LOCK_GUARD();
mr = mr->alias;
}
assert(mr->ram_block);
- ptr = qemu_map_ram_ptr(mr->ram_block, offset);
-
- return ptr;
+ return qemu_map_ram_ptr(mr->ram_block, offset);
}
MemoryRegion *memory_region_from_host(void *ptr, ram_addr_t *offset)
hwaddr offset,
MemoryRegion *subregion)
{
+ MemoryRegion *alias;
+
assert(!subregion->container);
subregion->container = mr;
+ for (alias = subregion->alias; alias; alias = alias->alias) {
+ alias->mapped_via_alias++;
+ }
subregion->addr = offset;
memory_region_update_container_subregions(subregion);
}
void memory_region_del_subregion(MemoryRegion *mr,
MemoryRegion *subregion)
{
+ MemoryRegion *alias;
+
memory_region_transaction_begin();
assert(subregion->container == mr);
subregion->container = NULL;
+ for (alias = subregion->alias; alias; alias = alias->alias) {
+ alias->mapped_via_alias--;
+ assert(alias->mapped_via_alias >= 0);
+ }
QTAILQ_REMOVE(&mr->subregions, subregion, subregions_link);
memory_region_unref(subregion);
memory_region_update_pending |= mr->enabled && subregion->enabled;
memory_region_transaction_begin();
memory_region_ref(mr);
memory_region_del_subregion(container, mr);
- mr->container = container;
- memory_region_update_container_subregions(mr);
+ memory_region_add_subregion_common(container, mr->addr, mr);
memory_region_unref(mr);
memory_region_transaction_commit();
}
bool memory_region_is_mapped(MemoryRegion *mr)
{
- return mr->container ? true : false;
+ return !!mr->container || mr->mapped_via_alias;
}
/* Same as memory_region_find, but it does not add a reference to the
return mr && mr != container;
}
-void memory_global_dirty_log_sync(void)
+void memory_global_dirty_log_sync(bool last_stage)
{
- memory_region_sync_dirty_bitmap(NULL);
+ memory_region_sync_dirty_bitmap(NULL, last_stage);
}
void memory_global_after_dirty_log_sync(void)
MEMORY_LISTENER_CALL_GLOBAL(log_global_after_sync, Forward);
}
+/*
+ * Dirty track stop flags that are postponed due to VM being stopped. Should
+ * only be used within vmstate_change hook.
+ */
+static unsigned int postponed_stop_flags;
static VMChangeStateEntry *vmstate_change;
+static void memory_global_dirty_log_stop_postponed_run(void);
-void memory_global_dirty_log_start(void)
+void memory_global_dirty_log_start(unsigned int flags)
{
+ unsigned int old_flags;
+
+ assert(flags && !(flags & (~GLOBAL_DIRTY_MASK)));
+
if (vmstate_change) {
- qemu_del_vm_change_state_handler(vmstate_change);
- vmstate_change = NULL;
+ /* If there is postponed stop(), operate on it first */
+ postponed_stop_flags &= ~flags;
+ memory_global_dirty_log_stop_postponed_run();
}
- global_dirty_log = true;
+ flags &= ~global_dirty_tracking;
+ if (!flags) {
+ return;
+ }
- MEMORY_LISTENER_CALL_GLOBAL(log_global_start, Forward);
+ old_flags = global_dirty_tracking;
+ global_dirty_tracking |= flags;
+ trace_global_dirty_changed(global_dirty_tracking);
- /* Refresh DIRTY_MEMORY_MIGRATION bit. */
- memory_region_transaction_begin();
- memory_region_update_pending = true;
- memory_region_transaction_commit();
+ if (!old_flags) {
+ MEMORY_LISTENER_CALL_GLOBAL(log_global_start, Forward);
+ memory_region_transaction_begin();
+ memory_region_update_pending = true;
+ memory_region_transaction_commit();
+ }
}
-static void memory_global_dirty_log_do_stop(void)
+static void memory_global_dirty_log_do_stop(unsigned int flags)
{
- global_dirty_log = false;
+ assert(flags && !(flags & (~GLOBAL_DIRTY_MASK)));
+ assert((global_dirty_tracking & flags) == flags);
+ global_dirty_tracking &= ~flags;
- /* Refresh DIRTY_MEMORY_MIGRATION bit. */
- memory_region_transaction_begin();
- memory_region_update_pending = true;
- memory_region_transaction_commit();
+ trace_global_dirty_changed(global_dirty_tracking);
- MEMORY_LISTENER_CALL_GLOBAL(log_global_stop, Reverse);
+ if (!global_dirty_tracking) {
+ memory_region_transaction_begin();
+ memory_region_update_pending = true;
+ memory_region_transaction_commit();
+ MEMORY_LISTENER_CALL_GLOBAL(log_global_stop, Reverse);
+ }
+}
+
+/*
+ * Execute the postponed dirty log stop operations if there is, then reset
+ * everything (including the flags and the vmstate change hook).
+ */
+static void memory_global_dirty_log_stop_postponed_run(void)
+{
+ /* This must be called with the vmstate handler registered */
+ assert(vmstate_change);
+
+ /* Note: postponed_stop_flags can be cleared in log start routine */
+ if (postponed_stop_flags) {
+ memory_global_dirty_log_do_stop(postponed_stop_flags);
+ postponed_stop_flags = 0;
+ }
+
+ qemu_del_vm_change_state_handler(vmstate_change);
+ vmstate_change = NULL;
}
static void memory_vm_change_state_handler(void *opaque, bool running,
RunState state)
{
if (running) {
- memory_global_dirty_log_do_stop();
-
- if (vmstate_change) {
- qemu_del_vm_change_state_handler(vmstate_change);
- vmstate_change = NULL;
- }
+ memory_global_dirty_log_stop_postponed_run();
}
}
-void memory_global_dirty_log_stop(void)
+void memory_global_dirty_log_stop(unsigned int flags)
{
if (!runstate_is_running()) {
+ /* Postpone the dirty log stop, e.g., to when VM starts again */
if (vmstate_change) {
- return;
+ /* Batch with previous postponed flags */
+ postponed_stop_flags |= flags;
+ } else {
+ postponed_stop_flags = flags;
+ vmstate_change = qemu_add_vm_change_state_handler(
+ memory_vm_change_state_handler, NULL);
}
- vmstate_change = qemu_add_vm_change_state_handler(
- memory_vm_change_state_handler, NULL);
return;
}
- memory_global_dirty_log_do_stop();
+ memory_global_dirty_log_do_stop(flags);
}
static void listener_add_address_space(MemoryListener *listener,
if (listener->begin) {
listener->begin(listener);
}
- if (global_dirty_log) {
+ if (global_dirty_tracking) {
if (listener->log_global_start) {
listener->log_global_start(listener);
}
for (i = 0; i < level; i++) {
qemu_printf(MTREE_INDENT);
}
- qemu_printf(TARGET_FMT_plx "-" TARGET_FMT_plx
- " (prio %d, %s%s): alias %s @%s " TARGET_FMT_plx
- "-" TARGET_FMT_plx "%s",
+ qemu_printf(HWADDR_FMT_plx "-" HWADDR_FMT_plx
+ " (prio %d, %s%s): alias %s @%s " HWADDR_FMT_plx
+ "-" HWADDR_FMT_plx "%s",
cur_start, cur_end,
mr->priority,
mr->nonvolatile ? "nv-" : "",
for (i = 0; i < level; i++) {
qemu_printf(MTREE_INDENT);
}
- qemu_printf(TARGET_FMT_plx "-" TARGET_FMT_plx
+ qemu_printf(HWADDR_FMT_plx "-" HWADDR_FMT_plx
" (prio %d, %s%s): %s%s",
cur_start, cur_end,
mr->priority,
while (n--) {
mr = range->mr;
if (range->offset_in_region) {
- qemu_printf(MTREE_INDENT TARGET_FMT_plx "-" TARGET_FMT_plx
- " (prio %d, %s%s): %s @" TARGET_FMT_plx,
+ qemu_printf(MTREE_INDENT HWADDR_FMT_plx "-" HWADDR_FMT_plx
+ " (prio %d, %s%s): %s @" HWADDR_FMT_plx,
int128_get64(range->addr.start),
int128_get64(range->addr.start)
+ MR_SIZE(range->addr.size),
memory_region_name(mr),
range->offset_in_region);
} else {
- qemu_printf(MTREE_INDENT TARGET_FMT_plx "-" TARGET_FMT_plx
+ qemu_printf(MTREE_INDENT HWADDR_FMT_plx "-" HWADDR_FMT_plx
" (prio %d, %s%s): %s",
int128_get64(range->addr.start),
int128_get64(range->addr.start)
return true;
}
-void mtree_info(bool flatview, bool dispatch_tree, bool owner, bool disabled)
+static void mtree_info_flatview(bool dispatch_tree, bool owner)
{
- MemoryRegionListHead ml_head;
- MemoryRegionList *ml, *ml2;
+ struct FlatViewInfo fvi = {
+ .counter = 0,
+ .dispatch_tree = dispatch_tree,
+ .owner = owner,
+ };
AddressSpace *as;
+ FlatView *view;
+ GArray *fv_address_spaces;
+ GHashTable *views = g_hash_table_new(g_direct_hash, g_direct_equal);
+ AccelClass *ac = ACCEL_GET_CLASS(current_accel());
- if (flatview) {
- FlatView *view;
- struct FlatViewInfo fvi = {
- .counter = 0,
- .dispatch_tree = dispatch_tree,
- .owner = owner,
- };
- GArray *fv_address_spaces;
- GHashTable *views = g_hash_table_new(g_direct_hash, g_direct_equal);
- AccelClass *ac = ACCEL_GET_CLASS(current_accel());
+ if (ac->has_memory) {
+ fvi.ac = ac;
+ }
+
+ /* Gather all FVs in one table */
+ QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
+ view = address_space_get_flatview(as);
- if (ac->has_memory) {
- fvi.ac = ac;
+ fv_address_spaces = g_hash_table_lookup(views, view);
+ if (!fv_address_spaces) {
+ fv_address_spaces = g_array_new(false, false, sizeof(as));
+ g_hash_table_insert(views, view, fv_address_spaces);
}
- /* Gather all FVs in one table */
- QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
- view = address_space_get_flatview(as);
+ g_array_append_val(fv_address_spaces, as);
+ }
- fv_address_spaces = g_hash_table_lookup(views, view);
- if (!fv_address_spaces) {
- fv_address_spaces = g_array_new(false, false, sizeof(as));
- g_hash_table_insert(views, view, fv_address_spaces);
- }
+ /* Print */
+ g_hash_table_foreach(views, mtree_print_flatview, &fvi);
- g_array_append_val(fv_address_spaces, as);
- }
+ /* Free */
+ g_hash_table_foreach_remove(views, mtree_info_flatview_free, 0);
+ g_hash_table_unref(views);
+}
- /* Print */
- g_hash_table_foreach(views, mtree_print_flatview, &fvi);
+struct AddressSpaceInfo {
+ MemoryRegionListHead *ml_head;
+ bool owner;
+ bool disabled;
+};
- /* Free */
- g_hash_table_foreach_remove(views, mtree_info_flatview_free, 0);
- g_hash_table_unref(views);
+/* Returns negative value if a < b; zero if a = b; positive value if a > b. */
+static gint address_space_compare_name(gconstpointer a, gconstpointer b)
+{
+ const AddressSpace *as_a = a;
+ const AddressSpace *as_b = b;
- return;
- }
+ return g_strcmp0(as_a->name, as_b->name);
+}
+
+static void mtree_print_as_name(gpointer data, gpointer user_data)
+{
+ AddressSpace *as = data;
+
+ qemu_printf("address-space: %s\n", as->name);
+}
+
+static void mtree_print_as(gpointer key, gpointer value, gpointer user_data)
+{
+ MemoryRegion *mr = key;
+ GSList *as_same_root_mr_list = value;
+ struct AddressSpaceInfo *asi = user_data;
+
+ g_slist_foreach(as_same_root_mr_list, mtree_print_as_name, NULL);
+ mtree_print_mr(mr, 1, 0, asi->ml_head, asi->owner, asi->disabled);
+ qemu_printf("\n");
+}
+
+static gboolean mtree_info_as_free(gpointer key, gpointer value,
+ gpointer user_data)
+{
+ GSList *as_same_root_mr_list = value;
+
+ g_slist_free(as_same_root_mr_list);
+
+ return true;
+}
+
+static void mtree_info_as(bool dispatch_tree, bool owner, bool disabled)
+{
+ MemoryRegionListHead ml_head;
+ MemoryRegionList *ml, *ml2;
+ AddressSpace *as;
+ GHashTable *views = g_hash_table_new(g_direct_hash, g_direct_equal);
+ GSList *as_same_root_mr_list;
+ struct AddressSpaceInfo asi = {
+ .ml_head = &ml_head,
+ .owner = owner,
+ .disabled = disabled,
+ };
QTAILQ_INIT(&ml_head);
QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
- qemu_printf("address-space: %s\n", as->name);
- mtree_print_mr(as->root, 1, 0, &ml_head, owner, disabled);
- qemu_printf("\n");
+ /* Create hashtable, key=AS root MR, value = list of AS */
+ as_same_root_mr_list = g_hash_table_lookup(views, as->root);
+ as_same_root_mr_list = g_slist_insert_sorted(as_same_root_mr_list, as,
+ address_space_compare_name);
+ g_hash_table_insert(views, as->root, as_same_root_mr_list);
}
+ /* print address spaces */
+ g_hash_table_foreach(views, mtree_print_as, &asi);
+ g_hash_table_foreach_remove(views, mtree_info_as_free, 0);
+ g_hash_table_unref(views);
+
/* print aliased regions */
QTAILQ_FOREACH(ml, &ml_head, mrqueue) {
qemu_printf("memory-region: %s\n", memory_region_name(ml->mr));
}
}
+void mtree_info(bool flatview, bool dispatch_tree, bool owner, bool disabled)
+{
+ if (flatview) {
+ mtree_info_flatview(dispatch_tree, owner);
+ } else {
+ mtree_info_as(dispatch_tree, owner, disabled);
+ }
+}
+
void memory_region_init_ram(MemoryRegion *mr,
Object *owner,
const char *name,
projects / mirror_qemu.git / blobdiff