2 * Physical memory management
4 * Copyright 2011 Red Hat, Inc. and/or its affiliates
7 * Avi Kivity <avi@redhat.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
15 #include "exec-memory.h"
21 #define WANT_EXEC_OBSOLETE
22 #include "exec-obsolete.h"
24 unsigned memory_region_transaction_depth
= 0;
25 static bool memory_region_update_pending
= false;
27 typedef struct AddrRange AddrRange
;
30 * Note using signed integers limits us to physical addresses at most
31 * 63 bits wide. They are needed for negative offsetting in aliases
32 * (large MemoryRegion::alias_offset).
39 static AddrRange
addrrange_make(Int128 start
, Int128 size
)
41 return (AddrRange
) { start
, size
};
44 static bool addrrange_equal(AddrRange r1
, AddrRange r2
)
46 return int128_eq(r1
.start
, r2
.start
) && int128_eq(r1
.size
, r2
.size
);
49 static Int128
addrrange_end(AddrRange r
)
51 return int128_add(r
.start
, r
.size
);
54 static AddrRange
addrrange_shift(AddrRange range
, Int128 delta
)
56 int128_addto(&range
.start
, delta
);
60 static bool addrrange_contains(AddrRange range
, Int128 addr
)
62 return int128_ge(addr
, range
.start
)
63 && int128_lt(addr
, addrrange_end(range
));
66 static bool addrrange_intersects(AddrRange r1
, AddrRange r2
)
68 return addrrange_contains(r1
, r2
.start
)
69 || addrrange_contains(r2
, r1
.start
);
72 static AddrRange
addrrange_intersection(AddrRange r1
, AddrRange r2
)
74 Int128 start
= int128_max(r1
.start
, r2
.start
);
75 Int128 end
= int128_min(addrrange_end(r1
), addrrange_end(r2
));
76 return addrrange_make(start
, int128_sub(end
, start
));
79 struct CoalescedMemoryRange
{
81 QTAILQ_ENTRY(CoalescedMemoryRange
) link
;
84 struct MemoryRegionIoeventfd
{
91 static bool memory_region_ioeventfd_before(MemoryRegionIoeventfd a
,
92 MemoryRegionIoeventfd b
)
94 if (int128_lt(a
.addr
.start
, b
.addr
.start
)) {
96 } else if (int128_gt(a
.addr
.start
, b
.addr
.start
)) {
98 } else if (int128_lt(a
.addr
.size
, b
.addr
.size
)) {
100 } else if (int128_gt(a
.addr
.size
, b
.addr
.size
)) {
102 } else if (a
.match_data
< b
.match_data
) {
104 } else if (a
.match_data
> b
.match_data
) {
106 } else if (a
.match_data
) {
107 if (a
.data
< b
.data
) {
109 } else if (a
.data
> b
.data
) {
115 } else if (a
.fd
> b
.fd
) {
121 static bool memory_region_ioeventfd_equal(MemoryRegionIoeventfd a
,
122 MemoryRegionIoeventfd b
)
124 return !memory_region_ioeventfd_before(a
, b
)
125 && !memory_region_ioeventfd_before(b
, a
);
128 typedef struct FlatRange FlatRange
;
129 typedef struct FlatView FlatView
;
131 /* Range of memory in the global map. Addresses are absolute. */
134 target_phys_addr_t offset_in_region
;
136 uint8_t dirty_log_mask
;
141 /* Flattened global view of current active memory hierarchy. Kept in sorted
147 unsigned nr_allocated
;
150 typedef struct AddressSpace AddressSpace
;
151 typedef struct AddressSpaceOps AddressSpaceOps
;
153 /* A system address space - I/O, memory, etc. */
154 struct AddressSpace
{
155 const AddressSpaceOps
*ops
;
157 FlatView current_map
;
159 MemoryRegionIoeventfd
*ioeventfds
;
162 struct AddressSpaceOps
{
163 void (*range_add
)(AddressSpace
*as
, FlatRange
*fr
);
164 void (*range_del
)(AddressSpace
*as
, FlatRange
*fr
);
165 void (*log_start
)(AddressSpace
*as
, FlatRange
*fr
);
166 void (*log_stop
)(AddressSpace
*as
, FlatRange
*fr
);
167 void (*ioeventfd_add
)(AddressSpace
*as
, MemoryRegionIoeventfd
*fd
);
168 void (*ioeventfd_del
)(AddressSpace
*as
, MemoryRegionIoeventfd
*fd
);
171 #define FOR_EACH_FLAT_RANGE(var, view) \
172 for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var)
174 static bool flatrange_equal(FlatRange
*a
, FlatRange
*b
)
176 return a
->mr
== b
->mr
177 && addrrange_equal(a
->addr
, b
->addr
)
178 && a
->offset_in_region
== b
->offset_in_region
179 && a
->readable
== b
->readable
180 && a
->readonly
== b
->readonly
;
183 static void flatview_init(FlatView
*view
)
187 view
->nr_allocated
= 0;
190 /* Insert a range into a given position. Caller is responsible for maintaining
193 static void flatview_insert(FlatView
*view
, unsigned pos
, FlatRange
*range
)
195 if (view
->nr
== view
->nr_allocated
) {
196 view
->nr_allocated
= MAX(2 * view
->nr
, 10);
197 view
->ranges
= g_realloc(view
->ranges
,
198 view
->nr_allocated
* sizeof(*view
->ranges
));
200 memmove(view
->ranges
+ pos
+ 1, view
->ranges
+ pos
,
201 (view
->nr
- pos
) * sizeof(FlatRange
));
202 view
->ranges
[pos
] = *range
;
206 static void flatview_destroy(FlatView
*view
)
208 g_free(view
->ranges
);
211 static bool can_merge(FlatRange
*r1
, FlatRange
*r2
)
213 return int128_eq(addrrange_end(r1
->addr
), r2
->addr
.start
)
215 && int128_eq(int128_add(int128_make64(r1
->offset_in_region
),
217 int128_make64(r2
->offset_in_region
))
218 && r1
->dirty_log_mask
== r2
->dirty_log_mask
219 && r1
->readable
== r2
->readable
220 && r1
->readonly
== r2
->readonly
;
223 /* Attempt to simplify a view by merging ajacent ranges */
224 static void flatview_simplify(FlatView
*view
)
229 while (i
< view
->nr
) {
232 && can_merge(&view
->ranges
[j
-1], &view
->ranges
[j
])) {
233 int128_addto(&view
->ranges
[i
].addr
.size
, view
->ranges
[j
].addr
.size
);
237 memmove(&view
->ranges
[i
], &view
->ranges
[j
],
238 (view
->nr
- j
) * sizeof(view
->ranges
[j
]));
243 static void memory_region_read_accessor(void *opaque
,
244 target_phys_addr_t addr
,
250 MemoryRegion
*mr
= opaque
;
253 tmp
= mr
->ops
->read(mr
->opaque
, addr
, size
);
254 *value
|= (tmp
& mask
) << shift
;
257 static void memory_region_write_accessor(void *opaque
,
258 target_phys_addr_t addr
,
264 MemoryRegion
*mr
= opaque
;
267 tmp
= (*value
>> shift
) & mask
;
268 mr
->ops
->write(mr
->opaque
, addr
, tmp
, size
);
271 static void access_with_adjusted_size(target_phys_addr_t addr
,
274 unsigned access_size_min
,
275 unsigned access_size_max
,
276 void (*access
)(void *opaque
,
277 target_phys_addr_t addr
,
284 uint64_t access_mask
;
285 unsigned access_size
;
288 if (!access_size_min
) {
291 if (!access_size_max
) {
294 access_size
= MAX(MIN(size
, access_size_max
), access_size_min
);
295 access_mask
= -1ULL >> (64 - access_size
* 8);
296 for (i
= 0; i
< size
; i
+= access_size
) {
297 /* FIXME: big-endian support */
298 access(opaque
, addr
+ i
, value
, access_size
, i
* 8, access_mask
);
302 static void memory_region_prepare_ram_addr(MemoryRegion
*mr
);
304 static void as_memory_range_add(AddressSpace
*as
, FlatRange
*fr
)
306 ram_addr_t phys_offset
, region_offset
;
308 memory_region_prepare_ram_addr(fr
->mr
);
310 phys_offset
= fr
->mr
->ram_addr
;
311 region_offset
= fr
->offset_in_region
;
312 /* cpu_register_physical_memory_log() wants region_offset for
313 * mmio, but prefers offseting phys_offset for RAM. Humour it.
315 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
) {
316 phys_offset
+= region_offset
;
321 phys_offset
&= ~TARGET_PAGE_MASK
& ~IO_MEM_ROMD
;
325 phys_offset
|= IO_MEM_ROM
;
328 cpu_register_physical_memory_log(int128_get64(fr
->addr
.start
),
329 int128_get64(fr
->addr
.size
),
335 static void as_memory_range_del(AddressSpace
*as
, FlatRange
*fr
)
337 if (fr
->dirty_log_mask
) {
338 Int128 end
= addrrange_end(fr
->addr
);
339 cpu_physical_sync_dirty_bitmap(int128_get64(fr
->addr
.start
),
342 cpu_register_physical_memory(int128_get64(fr
->addr
.start
),
343 int128_get64(fr
->addr
.size
),
347 static void as_memory_log_start(AddressSpace
*as
, FlatRange
*fr
)
349 cpu_physical_log_start(int128_get64(fr
->addr
.start
),
350 int128_get64(fr
->addr
.size
));
353 static void as_memory_log_stop(AddressSpace
*as
, FlatRange
*fr
)
355 cpu_physical_log_stop(int128_get64(fr
->addr
.start
),
356 int128_get64(fr
->addr
.size
));
359 static void as_memory_ioeventfd_add(AddressSpace
*as
, MemoryRegionIoeventfd
*fd
)
363 assert(fd
->match_data
&& int128_get64(fd
->addr
.size
) == 4);
365 r
= kvm_set_ioeventfd_mmio_long(fd
->fd
, int128_get64(fd
->addr
.start
),
372 static void as_memory_ioeventfd_del(AddressSpace
*as
, MemoryRegionIoeventfd
*fd
)
376 r
= kvm_set_ioeventfd_mmio_long(fd
->fd
, int128_get64(fd
->addr
.start
),
383 static const AddressSpaceOps address_space_ops_memory
= {
384 .range_add
= as_memory_range_add
,
385 .range_del
= as_memory_range_del
,
386 .log_start
= as_memory_log_start
,
387 .log_stop
= as_memory_log_stop
,
388 .ioeventfd_add
= as_memory_ioeventfd_add
,
389 .ioeventfd_del
= as_memory_ioeventfd_del
,
392 static AddressSpace address_space_memory
= {
393 .ops
= &address_space_ops_memory
,
396 static const MemoryRegionPortio
*find_portio(MemoryRegion
*mr
, uint64_t offset
,
397 unsigned width
, bool write
)
399 const MemoryRegionPortio
*mrp
;
401 for (mrp
= mr
->ops
->old_portio
; mrp
->size
; ++mrp
) {
402 if (offset
>= mrp
->offset
&& offset
< mrp
->offset
+ mrp
->len
403 && width
== mrp
->size
404 && (write
? (bool)mrp
->write
: (bool)mrp
->read
)) {
411 static void memory_region_iorange_read(IORange
*iorange
,
416 MemoryRegion
*mr
= container_of(iorange
, MemoryRegion
, iorange
);
418 if (mr
->ops
->old_portio
) {
419 const MemoryRegionPortio
*mrp
= find_portio(mr
, offset
, width
, false);
421 *data
= ((uint64_t)1 << (width
* 8)) - 1;
423 *data
= mrp
->read(mr
->opaque
, offset
+ mr
->offset
);
424 } else if (width
== 2) {
425 mrp
= find_portio(mr
, offset
, 1, false);
427 *data
= mrp
->read(mr
->opaque
, offset
+ mr
->offset
) |
428 (mrp
->read(mr
->opaque
, offset
+ mr
->offset
+ 1) << 8);
433 access_with_adjusted_size(offset
+ mr
->offset
, data
, width
,
434 mr
->ops
->impl
.min_access_size
,
435 mr
->ops
->impl
.max_access_size
,
436 memory_region_read_accessor
, mr
);
439 static void memory_region_iorange_write(IORange
*iorange
,
444 MemoryRegion
*mr
= container_of(iorange
, MemoryRegion
, iorange
);
446 if (mr
->ops
->old_portio
) {
447 const MemoryRegionPortio
*mrp
= find_portio(mr
, offset
, width
, true);
450 mrp
->write(mr
->opaque
, offset
+ mr
->offset
, data
);
451 } else if (width
== 2) {
452 mrp
= find_portio(mr
, offset
, 1, false);
454 mrp
->write(mr
->opaque
, offset
+ mr
->offset
, data
& 0xff);
455 mrp
->write(mr
->opaque
, offset
+ mr
->offset
+ 1, data
>> 8);
459 access_with_adjusted_size(offset
+ mr
->offset
, &data
, width
,
460 mr
->ops
->impl
.min_access_size
,
461 mr
->ops
->impl
.max_access_size
,
462 memory_region_write_accessor
, mr
);
465 static const IORangeOps memory_region_iorange_ops
= {
466 .read
= memory_region_iorange_read
,
467 .write
= memory_region_iorange_write
,
470 static void as_io_range_add(AddressSpace
*as
, FlatRange
*fr
)
472 iorange_init(&fr
->mr
->iorange
, &memory_region_iorange_ops
,
473 int128_get64(fr
->addr
.start
), int128_get64(fr
->addr
.size
));
474 ioport_register(&fr
->mr
->iorange
);
477 static void as_io_range_del(AddressSpace
*as
, FlatRange
*fr
)
479 isa_unassign_ioport(int128_get64(fr
->addr
.start
),
480 int128_get64(fr
->addr
.size
));
483 static void as_io_ioeventfd_add(AddressSpace
*as
, MemoryRegionIoeventfd
*fd
)
487 assert(fd
->match_data
&& int128_get64(fd
->addr
.size
) == 2);
489 r
= kvm_set_ioeventfd_pio_word(fd
->fd
, int128_get64(fd
->addr
.start
),
496 static void as_io_ioeventfd_del(AddressSpace
*as
, MemoryRegionIoeventfd
*fd
)
500 r
= kvm_set_ioeventfd_pio_word(fd
->fd
, int128_get64(fd
->addr
.start
),
507 static const AddressSpaceOps address_space_ops_io
= {
508 .range_add
= as_io_range_add
,
509 .range_del
= as_io_range_del
,
510 .ioeventfd_add
= as_io_ioeventfd_add
,
511 .ioeventfd_del
= as_io_ioeventfd_del
,
514 static AddressSpace address_space_io
= {
515 .ops
= &address_space_ops_io
,
518 static AddressSpace
*memory_region_to_address_space(MemoryRegion
*mr
)
523 if (mr
== address_space_memory
.root
) {
524 return &address_space_memory
;
526 if (mr
== address_space_io
.root
) {
527 return &address_space_io
;
532 /* Render a memory region into the global view. Ranges in @view obscure
535 static void render_memory_region(FlatView
*view
,
541 MemoryRegion
*subregion
;
543 target_phys_addr_t offset_in_region
;
553 int128_addto(&base
, int128_make64(mr
->addr
));
554 readonly
|= mr
->readonly
;
556 tmp
= addrrange_make(base
, mr
->size
);
558 if (!addrrange_intersects(tmp
, clip
)) {
562 clip
= addrrange_intersection(tmp
, clip
);
565 int128_subfrom(&base
, int128_make64(mr
->alias
->addr
));
566 int128_subfrom(&base
, int128_make64(mr
->alias_offset
));
567 render_memory_region(view
, mr
->alias
, base
, clip
, readonly
);
571 /* Render subregions in priority order. */
572 QTAILQ_FOREACH(subregion
, &mr
->subregions
, subregions_link
) {
573 render_memory_region(view
, subregion
, base
, clip
, readonly
);
576 if (!mr
->terminates
) {
580 offset_in_region
= int128_get64(int128_sub(clip
.start
, base
));
584 /* Render the region itself into any gaps left by the current view. */
585 for (i
= 0; i
< view
->nr
&& int128_nz(remain
); ++i
) {
586 if (int128_ge(base
, addrrange_end(view
->ranges
[i
].addr
))) {
589 if (int128_lt(base
, view
->ranges
[i
].addr
.start
)) {
590 now
= int128_min(remain
,
591 int128_sub(view
->ranges
[i
].addr
.start
, base
));
593 fr
.offset_in_region
= offset_in_region
;
594 fr
.addr
= addrrange_make(base
, now
);
595 fr
.dirty_log_mask
= mr
->dirty_log_mask
;
596 fr
.readable
= mr
->readable
;
597 fr
.readonly
= readonly
;
598 flatview_insert(view
, i
, &fr
);
600 int128_addto(&base
, now
);
601 offset_in_region
+= int128_get64(now
);
602 int128_subfrom(&remain
, now
);
604 if (int128_eq(base
, view
->ranges
[i
].addr
.start
)) {
605 now
= int128_min(remain
, view
->ranges
[i
].addr
.size
);
606 int128_addto(&base
, now
);
607 offset_in_region
+= int128_get64(now
);
608 int128_subfrom(&remain
, now
);
611 if (int128_nz(remain
)) {
613 fr
.offset_in_region
= offset_in_region
;
614 fr
.addr
= addrrange_make(base
, remain
);
615 fr
.dirty_log_mask
= mr
->dirty_log_mask
;
616 fr
.readable
= mr
->readable
;
617 fr
.readonly
= readonly
;
618 flatview_insert(view
, i
, &fr
);
622 /* Render a memory topology into a list of disjoint absolute ranges. */
623 static FlatView
generate_memory_topology(MemoryRegion
*mr
)
627 flatview_init(&view
);
629 render_memory_region(&view
, mr
, int128_zero(),
630 addrrange_make(int128_zero(), int128_2_64()), false);
631 flatview_simplify(&view
);
636 static void address_space_add_del_ioeventfds(AddressSpace
*as
,
637 MemoryRegionIoeventfd
*fds_new
,
639 MemoryRegionIoeventfd
*fds_old
,
644 /* Generate a symmetric difference of the old and new fd sets, adding
645 * and deleting as necessary.
649 while (iold
< fds_old_nb
|| inew
< fds_new_nb
) {
650 if (iold
< fds_old_nb
651 && (inew
== fds_new_nb
652 || memory_region_ioeventfd_before(fds_old
[iold
],
654 as
->ops
->ioeventfd_del(as
, &fds_old
[iold
]);
656 } else if (inew
< fds_new_nb
657 && (iold
== fds_old_nb
658 || memory_region_ioeventfd_before(fds_new
[inew
],
660 as
->ops
->ioeventfd_add(as
, &fds_new
[inew
]);
669 static void address_space_update_ioeventfds(AddressSpace
*as
)
672 unsigned ioeventfd_nb
= 0;
673 MemoryRegionIoeventfd
*ioeventfds
= NULL
;
677 FOR_EACH_FLAT_RANGE(fr
, &as
->current_map
) {
678 for (i
= 0; i
< fr
->mr
->ioeventfd_nb
; ++i
) {
679 tmp
= addrrange_shift(fr
->mr
->ioeventfds
[i
].addr
,
680 int128_sub(fr
->addr
.start
,
681 int128_make64(fr
->offset_in_region
)));
682 if (addrrange_intersects(fr
->addr
, tmp
)) {
684 ioeventfds
= g_realloc(ioeventfds
,
685 ioeventfd_nb
* sizeof(*ioeventfds
));
686 ioeventfds
[ioeventfd_nb
-1] = fr
->mr
->ioeventfds
[i
];
687 ioeventfds
[ioeventfd_nb
-1].addr
= tmp
;
692 address_space_add_del_ioeventfds(as
, ioeventfds
, ioeventfd_nb
,
693 as
->ioeventfds
, as
->ioeventfd_nb
);
695 g_free(as
->ioeventfds
);
696 as
->ioeventfds
= ioeventfds
;
697 as
->ioeventfd_nb
= ioeventfd_nb
;
700 static void address_space_update_topology_pass(AddressSpace
*as
,
706 FlatRange
*frold
, *frnew
;
708 /* Generate a symmetric difference of the old and new memory maps.
709 * Kill ranges in the old map, and instantiate ranges in the new map.
712 while (iold
< old_view
.nr
|| inew
< new_view
.nr
) {
713 if (iold
< old_view
.nr
) {
714 frold
= &old_view
.ranges
[iold
];
718 if (inew
< new_view
.nr
) {
719 frnew
= &new_view
.ranges
[inew
];
726 || int128_lt(frold
->addr
.start
, frnew
->addr
.start
)
727 || (int128_eq(frold
->addr
.start
, frnew
->addr
.start
)
728 && !flatrange_equal(frold
, frnew
)))) {
729 /* In old, but (not in new, or in new but attributes changed). */
732 as
->ops
->range_del(as
, frold
);
736 } else if (frold
&& frnew
&& flatrange_equal(frold
, frnew
)) {
737 /* In both (logging may have changed) */
740 if (frold
->dirty_log_mask
&& !frnew
->dirty_log_mask
) {
741 as
->ops
->log_stop(as
, frnew
);
742 } else if (frnew
->dirty_log_mask
&& !frold
->dirty_log_mask
) {
743 as
->ops
->log_start(as
, frnew
);
753 as
->ops
->range_add(as
, frnew
);
762 static void address_space_update_topology(AddressSpace
*as
)
764 FlatView old_view
= as
->current_map
;
765 FlatView new_view
= generate_memory_topology(as
->root
);
767 address_space_update_topology_pass(as
, old_view
, new_view
, false);
768 address_space_update_topology_pass(as
, old_view
, new_view
, true);
770 as
->current_map
= new_view
;
771 flatview_destroy(&old_view
);
772 address_space_update_ioeventfds(as
);
775 static void memory_region_update_topology(MemoryRegion
*mr
)
777 if (memory_region_transaction_depth
) {
778 memory_region_update_pending
|= !mr
|| mr
->enabled
;
782 if (mr
&& !mr
->enabled
) {
786 if (address_space_memory
.root
) {
787 address_space_update_topology(&address_space_memory
);
789 if (address_space_io
.root
) {
790 address_space_update_topology(&address_space_io
);
793 memory_region_update_pending
= false;
796 void memory_region_transaction_begin(void)
798 ++memory_region_transaction_depth
;
801 void memory_region_transaction_commit(void)
803 assert(memory_region_transaction_depth
);
804 --memory_region_transaction_depth
;
805 if (!memory_region_transaction_depth
&& memory_region_update_pending
) {
806 memory_region_update_topology(NULL
);
810 static void memory_region_destructor_none(MemoryRegion
*mr
)
814 static void memory_region_destructor_ram(MemoryRegion
*mr
)
816 qemu_ram_free(mr
->ram_addr
);
819 static void memory_region_destructor_ram_from_ptr(MemoryRegion
*mr
)
821 qemu_ram_free_from_ptr(mr
->ram_addr
);
824 static void memory_region_destructor_iomem(MemoryRegion
*mr
)
826 cpu_unregister_io_memory(mr
->ram_addr
);
829 static void memory_region_destructor_rom_device(MemoryRegion
*mr
)
831 qemu_ram_free(mr
->ram_addr
& TARGET_PAGE_MASK
);
832 cpu_unregister_io_memory(mr
->ram_addr
& ~(TARGET_PAGE_MASK
| IO_MEM_ROMD
));
835 void memory_region_init(MemoryRegion
*mr
,
841 mr
->size
= int128_make64(size
);
842 if (size
== UINT64_MAX
) {
843 mr
->size
= int128_2_64();
848 mr
->terminates
= false;
851 mr
->readonly
= false;
852 mr
->destructor
= memory_region_destructor_none
;
854 mr
->may_overlap
= false;
856 QTAILQ_INIT(&mr
->subregions
);
857 memset(&mr
->subregions_link
, 0, sizeof mr
->subregions_link
);
858 QTAILQ_INIT(&mr
->coalesced
);
859 mr
->name
= g_strdup(name
);
860 mr
->dirty_log_mask
= 0;
861 mr
->ioeventfd_nb
= 0;
862 mr
->ioeventfds
= NULL
;
865 static bool memory_region_access_valid(MemoryRegion
*mr
,
866 target_phys_addr_t addr
,
870 if (mr
->ops
->valid
.accepts
871 && !mr
->ops
->valid
.accepts(mr
->opaque
, addr
, size
, is_write
)) {
875 if (!mr
->ops
->valid
.unaligned
&& (addr
& (size
- 1))) {
879 /* Treat zero as compatibility all valid */
880 if (!mr
->ops
->valid
.max_access_size
) {
884 if (size
> mr
->ops
->valid
.max_access_size
885 || size
< mr
->ops
->valid
.min_access_size
) {
891 static uint32_t memory_region_read_thunk_n(void *_mr
,
892 target_phys_addr_t addr
,
895 MemoryRegion
*mr
= _mr
;
898 if (!memory_region_access_valid(mr
, addr
, size
, false)) {
899 return -1U; /* FIXME: better signalling */
902 if (!mr
->ops
->read
) {
903 return mr
->ops
->old_mmio
.read
[bitops_ffsl(size
)](mr
->opaque
, addr
);
906 /* FIXME: support unaligned access */
907 access_with_adjusted_size(addr
+ mr
->offset
, &data
, size
,
908 mr
->ops
->impl
.min_access_size
,
909 mr
->ops
->impl
.max_access_size
,
910 memory_region_read_accessor
, mr
);
915 static void memory_region_write_thunk_n(void *_mr
,
916 target_phys_addr_t addr
,
920 MemoryRegion
*mr
= _mr
;
922 if (!memory_region_access_valid(mr
, addr
, size
, true)) {
923 return; /* FIXME: better signalling */
926 if (!mr
->ops
->write
) {
927 mr
->ops
->old_mmio
.write
[bitops_ffsl(size
)](mr
->opaque
, addr
, data
);
931 /* FIXME: support unaligned access */
932 access_with_adjusted_size(addr
+ mr
->offset
, &data
, size
,
933 mr
->ops
->impl
.min_access_size
,
934 mr
->ops
->impl
.max_access_size
,
935 memory_region_write_accessor
, mr
);
938 static uint32_t memory_region_read_thunk_b(void *mr
, target_phys_addr_t addr
)
940 return memory_region_read_thunk_n(mr
, addr
, 1);
943 static uint32_t memory_region_read_thunk_w(void *mr
, target_phys_addr_t addr
)
945 return memory_region_read_thunk_n(mr
, addr
, 2);
948 static uint32_t memory_region_read_thunk_l(void *mr
, target_phys_addr_t addr
)
950 return memory_region_read_thunk_n(mr
, addr
, 4);
953 static void memory_region_write_thunk_b(void *mr
, target_phys_addr_t addr
,
956 memory_region_write_thunk_n(mr
, addr
, 1, data
);
959 static void memory_region_write_thunk_w(void *mr
, target_phys_addr_t addr
,
962 memory_region_write_thunk_n(mr
, addr
, 2, data
);
965 static void memory_region_write_thunk_l(void *mr
, target_phys_addr_t addr
,
968 memory_region_write_thunk_n(mr
, addr
, 4, data
);
971 static CPUReadMemoryFunc
* const memory_region_read_thunk
[] = {
972 memory_region_read_thunk_b
,
973 memory_region_read_thunk_w
,
974 memory_region_read_thunk_l
,
977 static CPUWriteMemoryFunc
* const memory_region_write_thunk
[] = {
978 memory_region_write_thunk_b
,
979 memory_region_write_thunk_w
,
980 memory_region_write_thunk_l
,
983 static void memory_region_prepare_ram_addr(MemoryRegion
*mr
)
985 if (mr
->backend_registered
) {
989 mr
->destructor
= memory_region_destructor_iomem
;
990 mr
->ram_addr
= cpu_register_io_memory(memory_region_read_thunk
,
991 memory_region_write_thunk
,
993 mr
->ops
->endianness
);
994 mr
->backend_registered
= true;
997 void memory_region_init_io(MemoryRegion
*mr
,
998 const MemoryRegionOps
*ops
,
1003 memory_region_init(mr
, name
, size
);
1005 mr
->opaque
= opaque
;
1006 mr
->terminates
= true;
1007 mr
->backend_registered
= false;
1010 void memory_region_init_ram(MemoryRegion
*mr
,
1015 memory_region_init(mr
, name
, size
);
1017 mr
->terminates
= true;
1018 mr
->destructor
= memory_region_destructor_ram
;
1019 mr
->ram_addr
= qemu_ram_alloc(dev
, name
, size
, mr
);
1020 mr
->backend_registered
= true;
1023 void memory_region_init_ram_ptr(MemoryRegion
*mr
,
1029 memory_region_init(mr
, name
, size
);
1031 mr
->terminates
= true;
1032 mr
->destructor
= memory_region_destructor_ram_from_ptr
;
1033 mr
->ram_addr
= qemu_ram_alloc_from_ptr(dev
, name
, size
, ptr
, mr
);
1034 mr
->backend_registered
= true;
1037 void memory_region_init_alias(MemoryRegion
*mr
,
1040 target_phys_addr_t offset
,
1043 memory_region_init(mr
, name
, size
);
1045 mr
->alias_offset
= offset
;
1048 void memory_region_init_rom_device(MemoryRegion
*mr
,
1049 const MemoryRegionOps
*ops
,
1055 memory_region_init(mr
, name
, size
);
1057 mr
->opaque
= opaque
;
1058 mr
->terminates
= true;
1059 mr
->destructor
= memory_region_destructor_rom_device
;
1060 mr
->ram_addr
= qemu_ram_alloc(dev
, name
, size
, mr
);
1061 mr
->ram_addr
|= cpu_register_io_memory(memory_region_read_thunk
,
1062 memory_region_write_thunk
,
1064 mr
->ops
->endianness
);
1065 mr
->ram_addr
|= IO_MEM_ROMD
;
1066 mr
->backend_registered
= true;
1069 void memory_region_destroy(MemoryRegion
*mr
)
1071 assert(QTAILQ_EMPTY(&mr
->subregions
));
1073 memory_region_clear_coalescing(mr
);
1074 g_free((char *)mr
->name
);
1075 g_free(mr
->ioeventfds
);
1078 uint64_t memory_region_size(MemoryRegion
*mr
)
1080 if (int128_eq(mr
->size
, int128_2_64())) {
1083 return int128_get64(mr
->size
);
1086 bool memory_region_is_ram(MemoryRegion
*mr
)
1091 bool memory_region_is_logging(MemoryRegion
*mr
)
1093 return mr
->dirty_log_mask
;
1096 bool memory_region_is_rom(MemoryRegion
*mr
)
1098 return mr
->ram
&& mr
->readonly
;
1101 void memory_region_set_offset(MemoryRegion
*mr
, target_phys_addr_t offset
)
1103 mr
->offset
= offset
;
1106 void memory_region_set_log(MemoryRegion
*mr
, bool log
, unsigned client
)
1108 uint8_t mask
= 1 << client
;
1110 mr
->dirty_log_mask
= (mr
->dirty_log_mask
& ~mask
) | (log
* mask
);
1111 memory_region_update_topology(mr
);
1114 bool memory_region_get_dirty(MemoryRegion
*mr
, target_phys_addr_t addr
,
1117 assert(mr
->terminates
);
1118 return cpu_physical_memory_get_dirty(mr
->ram_addr
+ addr
, 1 << client
);
1121 void memory_region_set_dirty(MemoryRegion
*mr
, target_phys_addr_t addr
)
1123 assert(mr
->terminates
);
1124 return cpu_physical_memory_set_dirty(mr
->ram_addr
+ addr
);
1127 void memory_region_sync_dirty_bitmap(MemoryRegion
*mr
)
1131 FOR_EACH_FLAT_RANGE(fr
, &address_space_memory
.current_map
) {
1133 cpu_physical_sync_dirty_bitmap(int128_get64(fr
->addr
.start
),
1134 int128_get64(addrrange_end(fr
->addr
)));
1139 void memory_region_set_readonly(MemoryRegion
*mr
, bool readonly
)
1141 if (mr
->readonly
!= readonly
) {
1142 mr
->readonly
= readonly
;
1143 memory_region_update_topology(mr
);
1147 void memory_region_rom_device_set_readable(MemoryRegion
*mr
, bool readable
)
1149 if (mr
->readable
!= readable
) {
1150 mr
->readable
= readable
;
1151 memory_region_update_topology(mr
);
1155 void memory_region_reset_dirty(MemoryRegion
*mr
, target_phys_addr_t addr
,
1156 target_phys_addr_t size
, unsigned client
)
1158 assert(mr
->terminates
);
1159 cpu_physical_memory_reset_dirty(mr
->ram_addr
+ addr
,
1160 mr
->ram_addr
+ addr
+ size
,
1164 void *memory_region_get_ram_ptr(MemoryRegion
*mr
)
1167 return memory_region_get_ram_ptr(mr
->alias
) + mr
->alias_offset
;
1170 assert(mr
->terminates
);
1172 return qemu_get_ram_ptr(mr
->ram_addr
& TARGET_PAGE_MASK
);
1175 static void memory_region_update_coalesced_range(MemoryRegion
*mr
)
1178 CoalescedMemoryRange
*cmr
;
1181 FOR_EACH_FLAT_RANGE(fr
, &address_space_memory
.current_map
) {
1183 qemu_unregister_coalesced_mmio(int128_get64(fr
->addr
.start
),
1184 int128_get64(fr
->addr
.size
));
1185 QTAILQ_FOREACH(cmr
, &mr
->coalesced
, link
) {
1186 tmp
= addrrange_shift(cmr
->addr
,
1187 int128_sub(fr
->addr
.start
,
1188 int128_make64(fr
->offset_in_region
)));
1189 if (!addrrange_intersects(tmp
, fr
->addr
)) {
1192 tmp
= addrrange_intersection(tmp
, fr
->addr
);
1193 qemu_register_coalesced_mmio(int128_get64(tmp
.start
),
1194 int128_get64(tmp
.size
));
1200 void memory_region_set_coalescing(MemoryRegion
*mr
)
1202 memory_region_clear_coalescing(mr
);
1203 memory_region_add_coalescing(mr
, 0, int128_get64(mr
->size
));
1206 void memory_region_add_coalescing(MemoryRegion
*mr
,
1207 target_phys_addr_t offset
,
1210 CoalescedMemoryRange
*cmr
= g_malloc(sizeof(*cmr
));
1212 cmr
->addr
= addrrange_make(int128_make64(offset
), int128_make64(size
));
1213 QTAILQ_INSERT_TAIL(&mr
->coalesced
, cmr
, link
);
1214 memory_region_update_coalesced_range(mr
);
1217 void memory_region_clear_coalescing(MemoryRegion
*mr
)
1219 CoalescedMemoryRange
*cmr
;
1221 while (!QTAILQ_EMPTY(&mr
->coalesced
)) {
1222 cmr
= QTAILQ_FIRST(&mr
->coalesced
);
1223 QTAILQ_REMOVE(&mr
->coalesced
, cmr
, link
);
1226 memory_region_update_coalesced_range(mr
);
1229 void memory_region_add_eventfd(MemoryRegion
*mr
,
1230 target_phys_addr_t addr
,
1236 MemoryRegionIoeventfd mrfd
= {
1237 .addr
.start
= int128_make64(addr
),
1238 .addr
.size
= int128_make64(size
),
1239 .match_data
= match_data
,
1245 for (i
= 0; i
< mr
->ioeventfd_nb
; ++i
) {
1246 if (memory_region_ioeventfd_before(mrfd
, mr
->ioeventfds
[i
])) {
1251 mr
->ioeventfds
= g_realloc(mr
->ioeventfds
,
1252 sizeof(*mr
->ioeventfds
) * mr
->ioeventfd_nb
);
1253 memmove(&mr
->ioeventfds
[i
+1], &mr
->ioeventfds
[i
],
1254 sizeof(*mr
->ioeventfds
) * (mr
->ioeventfd_nb
-1 - i
));
1255 mr
->ioeventfds
[i
] = mrfd
;
1256 memory_region_update_topology(mr
);
1259 void memory_region_del_eventfd(MemoryRegion
*mr
,
1260 target_phys_addr_t addr
,
1266 MemoryRegionIoeventfd mrfd
= {
1267 .addr
.start
= int128_make64(addr
),
1268 .addr
.size
= int128_make64(size
),
1269 .match_data
= match_data
,
1275 for (i
= 0; i
< mr
->ioeventfd_nb
; ++i
) {
1276 if (memory_region_ioeventfd_equal(mrfd
, mr
->ioeventfds
[i
])) {
1280 assert(i
!= mr
->ioeventfd_nb
);
1281 memmove(&mr
->ioeventfds
[i
], &mr
->ioeventfds
[i
+1],
1282 sizeof(*mr
->ioeventfds
) * (mr
->ioeventfd_nb
- (i
+1)));
1284 mr
->ioeventfds
= g_realloc(mr
->ioeventfds
,
1285 sizeof(*mr
->ioeventfds
)*mr
->ioeventfd_nb
+ 1);
1286 memory_region_update_topology(mr
);
1289 static void memory_region_add_subregion_common(MemoryRegion
*mr
,
1290 target_phys_addr_t offset
,
1291 MemoryRegion
*subregion
)
1293 MemoryRegion
*other
;
1295 assert(!subregion
->parent
);
1296 subregion
->parent
= mr
;
1297 subregion
->addr
= offset
;
1298 QTAILQ_FOREACH(other
, &mr
->subregions
, subregions_link
) {
1299 if (subregion
->may_overlap
|| other
->may_overlap
) {
1302 if (int128_gt(int128_make64(offset
),
1303 int128_add(int128_make64(other
->addr
), other
->size
))
1304 || int128_le(int128_add(int128_make64(offset
), subregion
->size
),
1305 int128_make64(other
->addr
))) {
1309 printf("warning: subregion collision %llx/%llx (%s) "
1310 "vs %llx/%llx (%s)\n",
1311 (unsigned long long)offset
,
1312 (unsigned long long)int128_get64(subregion
->size
),
1314 (unsigned long long)other
->addr
,
1315 (unsigned long long)int128_get64(other
->size
),
1319 QTAILQ_FOREACH(other
, &mr
->subregions
, subregions_link
) {
1320 if (subregion
->priority
>= other
->priority
) {
1321 QTAILQ_INSERT_BEFORE(other
, subregion
, subregions_link
);
1325 QTAILQ_INSERT_TAIL(&mr
->subregions
, subregion
, subregions_link
);
1327 memory_region_update_topology(mr
);
1331 void memory_region_add_subregion(MemoryRegion
*mr
,
1332 target_phys_addr_t offset
,
1333 MemoryRegion
*subregion
)
1335 subregion
->may_overlap
= false;
1336 subregion
->priority
= 0;
1337 memory_region_add_subregion_common(mr
, offset
, subregion
);
1340 void memory_region_add_subregion_overlap(MemoryRegion
*mr
,
1341 target_phys_addr_t offset
,
1342 MemoryRegion
*subregion
,
1345 subregion
->may_overlap
= true;
1346 subregion
->priority
= priority
;
1347 memory_region_add_subregion_common(mr
, offset
, subregion
);
1350 void memory_region_del_subregion(MemoryRegion
*mr
,
1351 MemoryRegion
*subregion
)
1353 assert(subregion
->parent
== mr
);
1354 subregion
->parent
= NULL
;
1355 QTAILQ_REMOVE(&mr
->subregions
, subregion
, subregions_link
);
1356 memory_region_update_topology(mr
);
1359 void memory_region_set_enabled(MemoryRegion
*mr
, bool enabled
)
1361 if (enabled
== mr
->enabled
) {
1364 mr
->enabled
= enabled
;
1365 memory_region_update_topology(NULL
);
1368 void memory_region_set_address(MemoryRegion
*mr
, target_phys_addr_t addr
)
1370 MemoryRegion
*parent
= mr
->parent
;
1371 unsigned priority
= mr
->priority
;
1372 bool may_overlap
= mr
->may_overlap
;
1374 if (addr
== mr
->addr
|| !parent
) {
1379 memory_region_transaction_begin();
1380 memory_region_del_subregion(parent
, mr
);
1382 memory_region_add_subregion_overlap(parent
, addr
, mr
, priority
);
1384 memory_region_add_subregion(parent
, addr
, mr
);
1386 memory_region_transaction_commit();
1389 void memory_region_set_alias_offset(MemoryRegion
*mr
, target_phys_addr_t offset
)
1391 target_phys_addr_t old_offset
= mr
->alias_offset
;
1394 mr
->alias_offset
= offset
;
1396 if (offset
== old_offset
|| !mr
->parent
) {
1400 memory_region_update_topology(mr
);
1403 static int cmp_flatrange_addr(const void *addr_
, const void *fr_
)
1405 const AddrRange
*addr
= addr_
;
1406 const FlatRange
*fr
= fr_
;
1408 if (int128_le(addrrange_end(*addr
), fr
->addr
.start
)) {
1410 } else if (int128_ge(addr
->start
, addrrange_end(fr
->addr
))) {
1416 static FlatRange
*address_space_lookup(AddressSpace
*as
, AddrRange addr
)
1418 return bsearch(&addr
, as
->current_map
.ranges
, as
->current_map
.nr
,
1419 sizeof(FlatRange
), cmp_flatrange_addr
);
1422 MemoryRegionSection
memory_region_find(MemoryRegion
*address_space
,
1423 target_phys_addr_t addr
, uint64_t size
)
1425 AddressSpace
*as
= memory_region_to_address_space(address_space
);
1426 AddrRange range
= addrrange_make(int128_make64(addr
),
1427 int128_make64(size
));
1428 FlatRange
*fr
= address_space_lookup(as
, range
);
1429 MemoryRegionSection ret
= { .mr
= NULL
, .size
= 0 };
1435 while (fr
> as
->current_map
.ranges
1436 && addrrange_intersects(fr
[-1].addr
, range
)) {
1441 range
= addrrange_intersection(range
, fr
->addr
);
1442 ret
.offset_within_region
= fr
->offset_in_region
;
1443 ret
.offset_within_region
+= int128_get64(int128_sub(range
.start
,
1445 ret
.size
= int128_get64(range
.size
);
1446 ret
.offset_within_address_space
= int128_get64(range
.start
);
1451 void set_system_memory_map(MemoryRegion
*mr
)
1453 address_space_memory
.root
= mr
;
1454 memory_region_update_topology(NULL
);
1457 void set_system_io_map(MemoryRegion
*mr
)
1459 address_space_io
.root
= mr
;
1460 memory_region_update_topology(NULL
);
1463 typedef struct MemoryRegionList MemoryRegionList
;
1465 struct MemoryRegionList
{
1466 const MemoryRegion
*mr
;
1468 QTAILQ_ENTRY(MemoryRegionList
) queue
;
1471 typedef QTAILQ_HEAD(queue
, MemoryRegionList
) MemoryRegionListHead
;
1473 static void mtree_print_mr(fprintf_function mon_printf
, void *f
,
1474 const MemoryRegion
*mr
, unsigned int level
,
1475 target_phys_addr_t base
,
1476 MemoryRegionListHead
*alias_print_queue
)
1478 MemoryRegionList
*new_ml
, *ml
, *next_ml
;
1479 MemoryRegionListHead submr_print_queue
;
1480 const MemoryRegion
*submr
;
1487 for (i
= 0; i
< level
; i
++) {
1492 MemoryRegionList
*ml
;
1495 /* check if the alias is already in the queue */
1496 QTAILQ_FOREACH(ml
, alias_print_queue
, queue
) {
1497 if (ml
->mr
== mr
->alias
&& !ml
->printed
) {
1503 ml
= g_new(MemoryRegionList
, 1);
1505 ml
->printed
= false;
1506 QTAILQ_INSERT_TAIL(alias_print_queue
, ml
, queue
);
1508 mon_printf(f
, TARGET_FMT_plx
"-" TARGET_FMT_plx
" (prio %d): alias %s @%s "
1509 TARGET_FMT_plx
"-" TARGET_FMT_plx
"\n",
1512 + (target_phys_addr_t
)int128_get64(mr
->size
) - 1,
1518 + (target_phys_addr_t
)int128_get64(mr
->size
) - 1);
1520 mon_printf(f
, TARGET_FMT_plx
"-" TARGET_FMT_plx
" (prio %d): %s\n",
1523 + (target_phys_addr_t
)int128_get64(mr
->size
) - 1,
1528 QTAILQ_INIT(&submr_print_queue
);
1530 QTAILQ_FOREACH(submr
, &mr
->subregions
, subregions_link
) {
1531 new_ml
= g_new(MemoryRegionList
, 1);
1533 QTAILQ_FOREACH(ml
, &submr_print_queue
, queue
) {
1534 if (new_ml
->mr
->addr
< ml
->mr
->addr
||
1535 (new_ml
->mr
->addr
== ml
->mr
->addr
&&
1536 new_ml
->mr
->priority
> ml
->mr
->priority
)) {
1537 QTAILQ_INSERT_BEFORE(ml
, new_ml
, queue
);
1543 QTAILQ_INSERT_TAIL(&submr_print_queue
, new_ml
, queue
);
1547 QTAILQ_FOREACH(ml
, &submr_print_queue
, queue
) {
1548 mtree_print_mr(mon_printf
, f
, ml
->mr
, level
+ 1, base
+ mr
->addr
,
1552 QTAILQ_FOREACH_SAFE(ml
, &submr_print_queue
, queue
, next_ml
) {
1557 void mtree_info(fprintf_function mon_printf
, void *f
)
1559 MemoryRegionListHead ml_head
;
1560 MemoryRegionList
*ml
, *ml2
;
1562 QTAILQ_INIT(&ml_head
);
1564 mon_printf(f
, "memory\n");
1565 mtree_print_mr(mon_printf
, f
, address_space_memory
.root
, 0, 0, &ml_head
);
1567 /* print aliased regions */
1568 QTAILQ_FOREACH(ml
, &ml_head
, queue
) {
1570 mon_printf(f
, "%s\n", ml
->mr
->name
);
1571 mtree_print_mr(mon_printf
, f
, ml
->mr
, 0, 0, &ml_head
);
1575 QTAILQ_FOREACH_SAFE(ml
, &ml_head
, queue
, ml2
) {
1579 if (address_space_io
.root
&&
1580 !QTAILQ_EMPTY(&address_space_io
.root
->subregions
)) {
1581 QTAILQ_INIT(&ml_head
);
1582 mon_printf(f
, "I/O\n");
1583 mtree_print_mr(mon_printf
, f
, address_space_io
.root
, 0, 0, &ml_head
);