]> git.proxmox.com Git - mirror_qemu.git/blob - memory.c
projects / mirror_qemu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
memory: Create FlatView directly
[mirror_qemu.git] / memory.c
1 /*
2 * Physical memory management
3 *
4 * Copyright 2011 Red Hat, Inc. and/or its affiliates
5 *
6 * Authors:
7 * Avi Kivity <avi@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 * Contributions after 2012-01-13 are licensed under the terms of the
13 * GNU GPL, version 2 or (at your option) any later version.
14 */
15
16 #include "qemu/osdep.h"
17 #include "qapi/error.h"
18 #include "qemu-common.h"
19 #include "cpu.h"
20 #include "exec/memory.h"
21 #include "exec/address-spaces.h"
22 #include "exec/ioport.h"
23 #include "qapi/visitor.h"
24 #include "qemu/bitops.h"
25 #include "qemu/error-report.h"
26 #include "qom/object.h"
27 #include "trace-root.h"
28
29 #include "exec/memory-internal.h"
30 #include "exec/ram_addr.h"
31 #include "sysemu/kvm.h"
32 #include "sysemu/sysemu.h"
33 #include "hw/misc/mmio_interface.h"
34 #include "hw/qdev-properties.h"
35 #include "migration/vmstate.h"
36
37 //#define DEBUG_UNASSIGNED
38
39 static unsigned memory_region_transaction_depth;
40 static bool memory_region_update_pending;
41 static bool ioeventfd_update_pending;
42 static bool global_dirty_log = false;
43
44 static QTAILQ_HEAD(memory_listeners, MemoryListener) memory_listeners
45 = QTAILQ_HEAD_INITIALIZER(memory_listeners);
46
47 static QTAILQ_HEAD(, AddressSpace) address_spaces
48 = QTAILQ_HEAD_INITIALIZER(address_spaces);
49
50 static GHashTable *flat_views;
51
52 typedef struct AddrRange AddrRange;
53
54 /*
55 * Note that signed integers are needed for negative offsetting in aliases
56 * (large MemoryRegion::alias_offset).
57 */
58 struct AddrRange {
59 Int128 start;
60 Int128 size;
61 };
62
63 static AddrRange addrrange_make(Int128 start, Int128 size)
64 {
65 return (AddrRange) { start, size };
66 }
67
68 static bool addrrange_equal(AddrRange r1, AddrRange r2)
69 {
70 return int128_eq(r1.start, r2.start) && int128_eq(r1.size, r2.size);
71 }
72
73 static Int128 addrrange_end(AddrRange r)
74 {
75 return int128_add(r.start, r.size);
76 }
77
78 static AddrRange addrrange_shift(AddrRange range, Int128 delta)
79 {
80 int128_addto(&range.start, delta);
81 return range;
82 }
83
84 static bool addrrange_contains(AddrRange range, Int128 addr)
85 {
86 return int128_ge(addr, range.start)
87 && int128_lt(addr, addrrange_end(range));
88 }
89
90 static bool addrrange_intersects(AddrRange r1, AddrRange r2)
91 {
92 return addrrange_contains(r1, r2.start)
93 || addrrange_contains(r2, r1.start);
94 }
95
96 static AddrRange addrrange_intersection(AddrRange r1, AddrRange r2)
97 {
98 Int128 start = int128_max(r1.start, r2.start);
99 Int128 end = int128_min(addrrange_end(r1), addrrange_end(r2));
100 return addrrange_make(start, int128_sub(end, start));
101 }
102
103 enum ListenerDirection { Forward, Reverse };
104
105 #define MEMORY_LISTENER_CALL_GLOBAL(_callback, _direction, _args...) \
106 do { \
107 MemoryListener *_listener; \
108 \
109 switch (_direction) { \
110 case Forward: \
111 QTAILQ_FOREACH(_listener, &memory_listeners, link) { \
112 if (_listener->_callback) { \
113 _listener->_callback(_listener, ##_args); \
114 } \
115 } \
116 break; \
117 case Reverse: \
118 QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \
119 memory_listeners, link) { \
120 if (_listener->_callback) { \
121 _listener->_callback(_listener, ##_args); \
122 } \
123 } \
124 break; \
125 default: \
126 abort(); \
127 } \
128 } while (0)
129
130 #define MEMORY_LISTENER_CALL(_as, _callback, _direction, _section, _args...) \
131 do { \
132 MemoryListener *_listener; \
133 struct memory_listeners_as *list = &(_as)->listeners; \
134 \
135 switch (_direction) { \
136 case Forward: \
137 QTAILQ_FOREACH(_listener, list, link_as) { \
138 if (_listener->_callback) { \
139 _listener->_callback(_listener, _section, ##_args); \
140 } \
141 } \
142 break; \
143 case Reverse: \
144 QTAILQ_FOREACH_REVERSE(_listener, list, memory_listeners_as, \
145 link_as) { \
146 if (_listener->_callback) { \
147 _listener->_callback(_listener, _section, ##_args); \
148 } \
149 } \
150 break; \
151 default: \
152 abort(); \
153 } \
154 } while (0)
155
156 /* No need to ref/unref .mr, the FlatRange keeps it alive. */
157 #define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback, _args...) \
158 do { \
159 MemoryRegionSection mrs = section_from_flat_range(fr, \
160 address_space_to_flatview(as)); \
161 MEMORY_LISTENER_CALL(as, callback, dir, &mrs, ##_args); \
162 } while(0)
163
164 struct CoalescedMemoryRange {
165 AddrRange addr;
166 QTAILQ_ENTRY(CoalescedMemoryRange) link;
167 };
168
169 struct MemoryRegionIoeventfd {
170 AddrRange addr;
171 bool match_data;
172 uint64_t data;
173 EventNotifier *e;
174 };
175
176 static bool memory_region_ioeventfd_before(MemoryRegionIoeventfd a,
177 MemoryRegionIoeventfd b)
178 {
179 if (int128_lt(a.addr.start, b.addr.start)) {
180 return true;
181 } else if (int128_gt(a.addr.start, b.addr.start)) {
182 return false;
183 } else if (int128_lt(a.addr.size, b.addr.size)) {
184 return true;
185 } else if (int128_gt(a.addr.size, b.addr.size)) {
186 return false;
187 } else if (a.match_data < b.match_data) {
188 return true;
189 } else if (a.match_data > b.match_data) {
190 return false;
191 } else if (a.match_data) {
192 if (a.data < b.data) {
193 return true;
194 } else if (a.data > b.data) {
195 return false;
196 }
197 }
198 if (a.e < b.e) {
199 return true;
200 } else if (a.e > b.e) {
201 return false;
202 }
203 return false;
204 }
205
206 static bool memory_region_ioeventfd_equal(MemoryRegionIoeventfd a,
207 MemoryRegionIoeventfd b)
208 {
209 return !memory_region_ioeventfd_before(a, b)
210 && !memory_region_ioeventfd_before(b, a);
211 }
212
213 typedef struct FlatRange FlatRange;
214
215 /* Range of memory in the global map. Addresses are absolute. */
216 struct FlatRange {
217 MemoryRegion *mr;
218 hwaddr offset_in_region;
219 AddrRange addr;
220 uint8_t dirty_log_mask;
221 bool romd_mode;
222 bool readonly;
223 };
224
225 /* Flattened global view of current active memory hierarchy. Kept in sorted
226 * order.
227 */
228 struct FlatView {
229 struct rcu_head rcu;
230 unsigned ref;
231 FlatRange *ranges;
232 unsigned nr;
233 unsigned nr_allocated;
234 struct AddressSpaceDispatch *dispatch;
235 MemoryRegion *root;
236 };
237
238 typedef struct AddressSpaceOps AddressSpaceOps;
239
240 #define FOR_EACH_FLAT_RANGE(var, view) \
241 for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var)
242
243 static inline MemoryRegionSection
244 section_from_flat_range(FlatRange *fr, FlatView *fv)
245 {
246 return (MemoryRegionSection) {
247 .mr = fr->mr,
248 .fv = fv,
249 .offset_within_region = fr->offset_in_region,
250 .size = fr->addr.size,
251 .offset_within_address_space = int128_get64(fr->addr.start),
252 .readonly = fr->readonly,
253 };
254 }
255
256 static bool flatrange_equal(FlatRange *a, FlatRange *b)
257 {
258 return a->mr == b->mr
259 && addrrange_equal(a->addr, b->addr)
260 && a->offset_in_region == b->offset_in_region
261 && a->romd_mode == b->romd_mode
262 && a->readonly == b->readonly;
263 }
264
265 static FlatView *flatview_new(MemoryRegion *mr_root)
266 {
267 FlatView *view;
268
269 view = g_new0(FlatView, 1);
270 view->ref = 1;
271 view->root = mr_root;
272 memory_region_ref(mr_root);
273
274 return view;
275 }
276
277 /* Insert a range into a given position. Caller is responsible for maintaining
278 * sorting order.
279 */
280 static void flatview_insert(FlatView *view, unsigned pos, FlatRange *range)
281 {
282 if (view->nr == view->nr_allocated) {
283 view->nr_allocated = MAX(2 * view->nr, 10);
284 view->ranges = g_realloc(view->ranges,
285 view->nr_allocated * sizeof(*view->ranges));
286 }
287 memmove(view->ranges + pos + 1, view->ranges + pos,
288 (view->nr - pos) * sizeof(FlatRange));
289 view->ranges[pos] = *range;
290 memory_region_ref(range->mr);
291 ++view->nr;
292 }
293
294 static void flatview_destroy(FlatView *view)
295 {
296 int i;
297
298 if (view->dispatch) {
299 address_space_dispatch_free(view->dispatch);
300 }
301 for (i = 0; i < view->nr; i++) {
302 memory_region_unref(view->ranges[i].mr);
303 }
304 g_free(view->ranges);
305 memory_region_unref(view->root);
306 g_free(view);
307 }
308
309 static bool flatview_ref(FlatView *view)
310 {
311 return atomic_fetch_inc_nonzero(&view->ref) > 0;
312 }
313
314 static void flatview_unref(FlatView *view)
315 {
316 if (atomic_fetch_dec(&view->ref) == 1) {
317 call_rcu(view, flatview_destroy, rcu);
318 }
319 }
320
321 FlatView *address_space_to_flatview(AddressSpace *as)
322 {
323 return atomic_rcu_read(&as->current_map);
324 }
325
326 AddressSpaceDispatch *flatview_to_dispatch(FlatView *fv)
327 {
328 return fv->dispatch;
329 }
330
331 AddressSpaceDispatch *address_space_to_dispatch(AddressSpace *as)
332 {
333 return flatview_to_dispatch(address_space_to_flatview(as));
334 }
335
336 static bool can_merge(FlatRange *r1, FlatRange *r2)
337 {
338 return int128_eq(addrrange_end(r1->addr), r2->addr.start)
339 && r1->mr == r2->mr
340 && int128_eq(int128_add(int128_make64(r1->offset_in_region),
341 r1->addr.size),
342 int128_make64(r2->offset_in_region))
343 && r1->dirty_log_mask == r2->dirty_log_mask
344 && r1->romd_mode == r2->romd_mode
345 && r1->readonly == r2->readonly;
346 }
347
348 /* Attempt to simplify a view by merging adjacent ranges */
349 static void flatview_simplify(FlatView *view)
350 {
351 unsigned i, j;
352
353 i = 0;
354 while (i < view->nr) {
355 j = i + 1;
356 while (j < view->nr
357 && can_merge(&view->ranges[j-1], &view->ranges[j])) {
358 int128_addto(&view->ranges[i].addr.size, view->ranges[j].addr.size);
359 ++j;
360 }
361 ++i;
362 memmove(&view->ranges[i], &view->ranges[j],
363 (view->nr - j) * sizeof(view->ranges[j]));
364 view->nr -= j - i;
365 }
366 }
367
368 static bool memory_region_big_endian(MemoryRegion *mr)
369 {
370 #ifdef TARGET_WORDS_BIGENDIAN
371 return mr->ops->endianness != DEVICE_LITTLE_ENDIAN;
372 #else
373 return mr->ops->endianness == DEVICE_BIG_ENDIAN;
374 #endif
375 }
376
377 static bool memory_region_wrong_endianness(MemoryRegion *mr)
378 {
379 #ifdef TARGET_WORDS_BIGENDIAN
380 return mr->ops->endianness == DEVICE_LITTLE_ENDIAN;
381 #else
382 return mr->ops->endianness == DEVICE_BIG_ENDIAN;
383 #endif
384 }
385
386 static void adjust_endianness(MemoryRegion *mr, uint64_t *data, unsigned size)
387 {
388 if (memory_region_wrong_endianness(mr)) {
389 switch (size) {
390 case 1:
391 break;
392 case 2:
393 *data = bswap16(*data);
394 break;
395 case 4:
396 *data = bswap32(*data);
397 break;
398 case 8:
399 *data = bswap64(*data);
400 break;
401 default:
402 abort();
403 }
404 }
405 }
406
407 static hwaddr memory_region_to_absolute_addr(MemoryRegion *mr, hwaddr offset)
408 {
409 MemoryRegion *root;
410 hwaddr abs_addr = offset;
411
412 abs_addr += mr->addr;
413 for (root = mr; root->container; ) {
414 root = root->container;
415 abs_addr += root->addr;
416 }
417
418 return abs_addr;
419 }
420
421 static int get_cpu_index(void)
422 {
423 if (current_cpu) {
424 return current_cpu->cpu_index;
425 }
426 return -1;
427 }
428
429 static MemTxResult memory_region_oldmmio_read_accessor(MemoryRegion *mr,
430 hwaddr addr,
431 uint64_t *value,
432 unsigned size,
433 unsigned shift,
434 uint64_t mask,
435 MemTxAttrs attrs)
436 {
437 uint64_t tmp;
438
439 tmp = mr->ops->old_mmio.read[ctz32(size)](mr->opaque, addr);
440 if (mr->subpage) {
441 trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);
442 } else if (mr == &io_mem_notdirty) {
443 /* Accesses to code which has previously been translated into a TB show
444 * up in the MMIO path, as accesses to the io_mem_notdirty
445 * MemoryRegion. */
446 trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);
447 } else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {
448 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
449 trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);
450 }
451 *value |= (tmp & mask) << shift;
452 return MEMTX_OK;
453 }
454
455 static MemTxResult memory_region_read_accessor(MemoryRegion *mr,
456 hwaddr addr,
457 uint64_t *value,
458 unsigned size,
459 unsigned shift,
460 uint64_t mask,
461 MemTxAttrs attrs)
462 {
463 uint64_t tmp;
464
465 tmp = mr->ops->read(mr->opaque, addr, size);
466 if (mr->subpage) {
467 trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);
468 } else if (mr == &io_mem_notdirty) {
469 /* Accesses to code which has previously been translated into a TB show
470 * up in the MMIO path, as accesses to the io_mem_notdirty
471 * MemoryRegion. */
472 trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);
473 } else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {
474 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
475 trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);
476 }
477 *value |= (tmp & mask) << shift;
478 return MEMTX_OK;
479 }
480
481 static MemTxResult memory_region_read_with_attrs_accessor(MemoryRegion *mr,
482 hwaddr addr,
483 uint64_t *value,
484 unsigned size,
485 unsigned shift,
486 uint64_t mask,
487 MemTxAttrs attrs)
488 {
489 uint64_t tmp = 0;
490 MemTxResult r;
491
492 r = mr->ops->read_with_attrs(mr->opaque, addr, &tmp, size, attrs);
493 if (mr->subpage) {
494 trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);
495 } else if (mr == &io_mem_notdirty) {
496 /* Accesses to code which has previously been translated into a TB show
497 * up in the MMIO path, as accesses to the io_mem_notdirty
498 * MemoryRegion. */
499 trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);
500 } else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {
501 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
502 trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);
503 }
504 *value |= (tmp & mask) << shift;
505 return r;
506 }
507
508 static MemTxResult memory_region_oldmmio_write_accessor(MemoryRegion *mr,
509 hwaddr addr,
510 uint64_t *value,
511 unsigned size,
512 unsigned shift,
513 uint64_t mask,
514 MemTxAttrs attrs)
515 {
516 uint64_t tmp;
517
518 tmp = (*value >> shift) & mask;
519 if (mr->subpage) {
520 trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);
521 } else if (mr == &io_mem_notdirty) {
522 /* Accesses to code which has previously been translated into a TB show
523 * up in the MMIO path, as accesses to the io_mem_notdirty
524 * MemoryRegion. */
525 trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);
526 } else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {
527 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
528 trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);
529 }
530 mr->ops->old_mmio.write[ctz32(size)](mr->opaque, addr, tmp);
531 return MEMTX_OK;
532 }
533
534 static MemTxResult memory_region_write_accessor(MemoryRegion *mr,
535 hwaddr addr,
536 uint64_t *value,
537 unsigned size,
538 unsigned shift,
539 uint64_t mask,
540 MemTxAttrs attrs)
541 {
542 uint64_t tmp;
543
544 tmp = (*value >> shift) & mask;
545 if (mr->subpage) {
546 trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);
547 } else if (mr == &io_mem_notdirty) {
548 /* Accesses to code which has previously been translated into a TB show
549 * up in the MMIO path, as accesses to the io_mem_notdirty
550 * MemoryRegion. */
551 trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);
552 } else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {
553 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
554 trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);
555 }
556 mr->ops->write(mr->opaque, addr, tmp, size);
557 return MEMTX_OK;
558 }
559
560 static MemTxResult memory_region_write_with_attrs_accessor(MemoryRegion *mr,
561 hwaddr addr,
562 uint64_t *value,
563 unsigned size,
564 unsigned shift,
565 uint64_t mask,
566 MemTxAttrs attrs)
567 {
568 uint64_t tmp;
569
570 tmp = (*value >> shift) & mask;
571 if (mr->subpage) {
572 trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);
573 } else if (mr == &io_mem_notdirty) {
574 /* Accesses to code which has previously been translated into a TB show
575 * up in the MMIO path, as accesses to the io_mem_notdirty
576 * MemoryRegion. */
577 trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);
578 } else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {
579 hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
580 trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);
581 }
582 return mr->ops->write_with_attrs(mr->opaque, addr, tmp, size, attrs);
583 }
584
585 static MemTxResult access_with_adjusted_size(hwaddr addr,
586 uint64_t *value,
587 unsigned size,
588 unsigned access_size_min,
589 unsigned access_size_max,
590 MemTxResult (*access_fn)
591 (MemoryRegion *mr,
592 hwaddr addr,
593 uint64_t *value,
594 unsigned size,
595 unsigned shift,
596 uint64_t mask,
597 MemTxAttrs attrs),
598 MemoryRegion *mr,
599 MemTxAttrs attrs)
600 {
601 uint64_t access_mask;
602 unsigned access_size;
603 unsigned i;
604 MemTxResult r = MEMTX_OK;
605
606 if (!access_size_min) {
607 access_size_min = 1;
608 }
609 if (!access_size_max) {
610 access_size_max = 4;
611 }
612
613 /* FIXME: support unaligned access? */
614 access_size = MAX(MIN(size, access_size_max), access_size_min);
615 access_mask = -1ULL >> (64 - access_size * 8);
616 if (memory_region_big_endian(mr)) {
617 for (i = 0; i < size; i += access_size) {
618 r |= access_fn(mr, addr + i, value, access_size,
619 (size - access_size - i) * 8, access_mask, attrs);
620 }
621 } else {
622 for (i = 0; i < size; i += access_size) {
623 r |= access_fn(mr, addr + i, value, access_size, i * 8,
624 access_mask, attrs);
625 }
626 }
627 return r;
628 }
629
630 static AddressSpace *memory_region_to_address_space(MemoryRegion *mr)
631 {
632 AddressSpace *as;
633
634 while (mr->container) {
635 mr = mr->container;
636 }
637 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
638 if (mr == as->root) {
639 return as;
640 }
641 }
642 return NULL;
643 }
644
645 /* Render a memory region into the global view. Ranges in @view obscure
646 * ranges in @mr.
647 */
648 static void render_memory_region(FlatView *view,
649 MemoryRegion *mr,
650 Int128 base,
651 AddrRange clip,
652 bool readonly)
653 {
654 MemoryRegion *subregion;
655 unsigned i;
656 hwaddr offset_in_region;
657 Int128 remain;
658 Int128 now;
659 FlatRange fr;
660 AddrRange tmp;
661
662 if (!mr->enabled) {
663 return;
664 }
665
666 int128_addto(&base, int128_make64(mr->addr));
667 readonly |= mr->readonly;
668
669 tmp = addrrange_make(base, mr->size);
670
671 if (!addrrange_intersects(tmp, clip)) {
672 return;
673 }
674
675 clip = addrrange_intersection(tmp, clip);
676
677 if (mr->alias) {
678 int128_subfrom(&base, int128_make64(mr->alias->addr));
679 int128_subfrom(&base, int128_make64(mr->alias_offset));
680 render_memory_region(view, mr->alias, base, clip, readonly);
681 return;
682 }
683
684 /* Render subregions in priority order. */
685 QTAILQ_FOREACH(subregion, &mr->subregions, subregions_link) {
686 render_memory_region(view, subregion, base, clip, readonly);
687 }
688
689 if (!mr->terminates) {
690 return;
691 }
692
693 offset_in_region = int128_get64(int128_sub(clip.start, base));
694 base = clip.start;
695 remain = clip.size;
696
697 fr.mr = mr;
698 fr.dirty_log_mask = memory_region_get_dirty_log_mask(mr);
699 fr.romd_mode = mr->romd_mode;
700 fr.readonly = readonly;
701
702 /* Render the region itself into any gaps left by the current view. */
703 for (i = 0; i < view->nr && int128_nz(remain); ++i) {
704 if (int128_ge(base, addrrange_end(view->ranges[i].addr))) {
705 continue;
706 }
707 if (int128_lt(base, view->ranges[i].addr.start)) {
708 now = int128_min(remain,
709 int128_sub(view->ranges[i].addr.start, base));
710 fr.offset_in_region = offset_in_region;
711 fr.addr = addrrange_make(base, now);
712 flatview_insert(view, i, &fr);
713 ++i;
714 int128_addto(&base, now);
715 offset_in_region += int128_get64(now);
716 int128_subfrom(&remain, now);
717 }
718 now = int128_sub(int128_min(int128_add(base, remain),
719 addrrange_end(view->ranges[i].addr)),
720 base);
721 int128_addto(&base, now);
722 offset_in_region += int128_get64(now);
723 int128_subfrom(&remain, now);
724 }
725 if (int128_nz(remain)) {
726 fr.offset_in_region = offset_in_region;
727 fr.addr = addrrange_make(base, remain);
728 flatview_insert(view, i, &fr);
729 }
730 }
731
732 static MemoryRegion *memory_region_get_flatview_root(MemoryRegion *mr)
733 {
734 while (mr->alias && !mr->alias_offset &&
735 int128_ge(mr->size, mr->alias->size)) {
736 /* The alias is included in its entirety. Use it as
737 * the "real" root, so that we can share more FlatViews.
738 */
739 mr = mr->alias;
740 }
741
742 return mr;
743 }
744
745 /* Render a memory topology into a list of disjoint absolute ranges. */
746 static FlatView *generate_memory_topology(MemoryRegion *mr)
747 {
748 int i;
749 FlatView *view;
750
751 view = flatview_new(mr);
752
753 if (mr) {
754 render_memory_region(view, mr, int128_zero(),
755 addrrange_make(int128_zero(), int128_2_64()), false);
756 }
757 flatview_simplify(view);
758
759 view->dispatch = address_space_dispatch_new(view);
760 for (i = 0; i < view->nr; i++) {
761 MemoryRegionSection mrs =
762 section_from_flat_range(&view->ranges[i], view);
763 flatview_add_to_dispatch(view, &mrs);
764 }
765 address_space_dispatch_compact(view->dispatch);
766 g_hash_table_replace(flat_views, mr, view);
767
768 return view;
769 }
770
771 static void address_space_add_del_ioeventfds(AddressSpace *as,
772 MemoryRegionIoeventfd *fds_new,
773 unsigned fds_new_nb,
774 MemoryRegionIoeventfd *fds_old,
775 unsigned fds_old_nb)
776 {
777 unsigned iold, inew;
778 MemoryRegionIoeventfd *fd;
779 MemoryRegionSection section;
780
781 /* Generate a symmetric difference of the old and new fd sets, adding
782 * and deleting as necessary.
783 */
784
785 iold = inew = 0;
786 while (iold < fds_old_nb || inew < fds_new_nb) {
787 if (iold < fds_old_nb
788 && (inew == fds_new_nb
789 || memory_region_ioeventfd_before(fds_old[iold],
790 fds_new[inew]))) {
791 fd = &fds_old[iold];
792 section = (MemoryRegionSection) {
793 .fv = address_space_to_flatview(as),
794 .offset_within_address_space = int128_get64(fd->addr.start),
795 .size = fd->addr.size,
796 };
797 MEMORY_LISTENER_CALL(as, eventfd_del, Forward, &section,
798 fd->match_data, fd->data, fd->e);
799 ++iold;
800 } else if (inew < fds_new_nb
801 && (iold == fds_old_nb
802 || memory_region_ioeventfd_before(fds_new[inew],
803 fds_old[iold]))) {
804 fd = &fds_new[inew];
805 section = (MemoryRegionSection) {
806 .fv = address_space_to_flatview(as),
807 .offset_within_address_space = int128_get64(fd->addr.start),
808 .size = fd->addr.size,
809 };
810 MEMORY_LISTENER_CALL(as, eventfd_add, Reverse, &section,
811 fd->match_data, fd->data, fd->e);
812 ++inew;
813 } else {
814 ++iold;
815 ++inew;
816 }
817 }
818 }
819
820 static FlatView *address_space_get_flatview(AddressSpace *as)
821 {
822 FlatView *view;
823
824 rcu_read_lock();
825 do {
826 view = address_space_to_flatview(as);
827 /* If somebody has replaced as->current_map concurrently,
828 * flatview_ref returns false.
829 */
830 } while (!flatview_ref(view));
831 rcu_read_unlock();
832 return view;
833 }
834
835 static void address_space_update_ioeventfds(AddressSpace *as)
836 {
837 FlatView *view;
838 FlatRange *fr;
839 unsigned ioeventfd_nb = 0;
840 MemoryRegionIoeventfd *ioeventfds = NULL;
841 AddrRange tmp;
842 unsigned i;
843
844 view = address_space_get_flatview(as);
845 FOR_EACH_FLAT_RANGE(fr, view) {
846 for (i = 0; i < fr->mr->ioeventfd_nb; ++i) {
847 tmp = addrrange_shift(fr->mr->ioeventfds[i].addr,
848 int128_sub(fr->addr.start,
849 int128_make64(fr->offset_in_region)));
850 if (addrrange_intersects(fr->addr, tmp)) {
851 ++ioeventfd_nb;
852 ioeventfds = g_realloc(ioeventfds,
853 ioeventfd_nb * sizeof(*ioeventfds));
854 ioeventfds[ioeventfd_nb-1] = fr->mr->ioeventfds[i];
855 ioeventfds[ioeventfd_nb-1].addr = tmp;
856 }
857 }
858 }
859
860 address_space_add_del_ioeventfds(as, ioeventfds, ioeventfd_nb,
861 as->ioeventfds, as->ioeventfd_nb);
862
863 g_free(as->ioeventfds);
864 as->ioeventfds = ioeventfds;
865 as->ioeventfd_nb = ioeventfd_nb;
866 flatview_unref(view);
867 }
868
869 static void address_space_update_topology_pass(AddressSpace *as,
870 const FlatView *old_view,
871 const FlatView *new_view,
872 bool adding)
873 {
874 unsigned iold, inew;
875 FlatRange *frold, *frnew;
876
877 /* Generate a symmetric difference of the old and new memory maps.
878 * Kill ranges in the old map, and instantiate ranges in the new map.
879 */
880 iold = inew = 0;
881 while (iold < old_view->nr || inew < new_view->nr) {
882 if (iold < old_view->nr) {
883 frold = &old_view->ranges[iold];
884 } else {
885 frold = NULL;
886 }
887 if (inew < new_view->nr) {
888 frnew = &new_view->ranges[inew];
889 } else {
890 frnew = NULL;
891 }
892
893 if (frold
894 && (!frnew
895 || int128_lt(frold->addr.start, frnew->addr.start)
896 || (int128_eq(frold->addr.start, frnew->addr.start)
897 && !flatrange_equal(frold, frnew)))) {
898 /* In old but not in new, or in both but attributes changed. */
899
900 if (!adding) {
901 MEMORY_LISTENER_UPDATE_REGION(frold, as, Reverse, region_del);
902 }
903
904 ++iold;
905 } else if (frold && frnew && flatrange_equal(frold, frnew)) {
906 /* In both and unchanged (except logging may have changed) */
907
908 if (adding) {
909 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_nop);
910 if (frnew->dirty_log_mask & ~frold->dirty_log_mask) {
911 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, log_start,
912 frold->dirty_log_mask,
913 frnew->dirty_log_mask);
914 }
915 if (frold->dirty_log_mask & ~frnew->dirty_log_mask) {
916 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop,
917 frold->dirty_log_mask,
918 frnew->dirty_log_mask);
919 }
920 }
921
922 ++iold;
923 ++inew;
924 } else {
925 /* In new */
926
927 if (adding) {
928 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_add);
929 }
930
931 ++inew;
932 }
933 }
934 }
935
936 static void flatviews_init(void)
937 {
938 if (flat_views) {
939 return;
940 }
941
942 flat_views = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL,
943 (GDestroyNotify) flatview_unref);
944 }
945
946 static void flatviews_reset(void)
947 {
948 AddressSpace *as;
949
950 if (flat_views) {
951 g_hash_table_unref(flat_views);
952 flat_views = NULL;
953 }
954 flatviews_init();
955
956 /* Render unique FVs */
957 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
958 MemoryRegion *physmr = memory_region_get_flatview_root(as->root);
959
960 if (g_hash_table_lookup(flat_views, physmr)) {
961 continue;
962 }
963
964 generate_memory_topology(physmr);
965 }
966 }
967
968 static void address_space_set_flatview(AddressSpace *as)
969 {
970 FlatView *old_view = address_space_to_flatview(as);
971 MemoryRegion *physmr = memory_region_get_flatview_root(as->root);
972 FlatView *new_view = g_hash_table_lookup(flat_views, physmr);
973
974 assert(new_view);
975
976 if (old_view == new_view) {
977 return;
978 }
979
980 if (old_view) {
981 flatview_ref(old_view);
982 }
983
984 flatview_ref(new_view);
985
986 if (!QTAILQ_EMPTY(&as->listeners)) {
987 FlatView tmpview = { .nr = 0 }, *old_view2 = old_view;
988
989 if (!old_view2) {
990 old_view2 = &tmpview;
991 }
992 address_space_update_topology_pass(as, old_view2, new_view, false);
993 address_space_update_topology_pass(as, old_view2, new_view, true);
994 }
995
996 /* Writes are protected by the BQL. */
997 atomic_rcu_set(&as->current_map, new_view);
998 if (old_view) {
999 flatview_unref(old_view);
1000 }
1001
1002 /* Note that all the old MemoryRegions are still alive up to this
1003 * point. This relieves most MemoryListeners from the need to
1004 * ref/unref the MemoryRegions they get---unless they use them
1005 * outside the iothread mutex, in which case precise reference
1006 * counting is necessary.
1007 */
1008 if (old_view) {
1009 flatview_unref(old_view);
1010 }
1011 }
1012
1013 static void address_space_update_topology(AddressSpace *as)
1014 {
1015 MemoryRegion *physmr = memory_region_get_flatview_root(as->root);
1016
1017 flatviews_init();
1018 if (!g_hash_table_lookup(flat_views, physmr)) {
1019 generate_memory_topology(physmr);
1020 }
1021 address_space_set_flatview(as);
1022 }
1023
1024 void memory_region_transaction_begin(void)
1025 {
1026 qemu_flush_coalesced_mmio_buffer();
1027 ++memory_region_transaction_depth;
1028 }
1029
1030 void memory_region_transaction_commit(void)
1031 {
1032 AddressSpace *as;
1033
1034 assert(memory_region_transaction_depth);
1035 assert(qemu_mutex_iothread_locked());
1036
1037 --memory_region_transaction_depth;
1038 if (!memory_region_transaction_depth) {
1039 if (memory_region_update_pending) {
1040 flatviews_reset();
1041
1042 MEMORY_LISTENER_CALL_GLOBAL(begin, Forward);
1043
1044 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
1045 address_space_set_flatview(as);
1046 address_space_update_ioeventfds(as);
1047 }
1048 memory_region_update_pending = false;
1049 MEMORY_LISTENER_CALL_GLOBAL(commit, Forward);
1050 } else if (ioeventfd_update_pending) {
1051 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
1052 address_space_update_ioeventfds(as);
1053 }
1054 ioeventfd_update_pending = false;
1055 }
1056 }
1057 }
1058
1059 static void memory_region_destructor_none(MemoryRegion *mr)
1060 {
1061 }
1062
1063 static void memory_region_destructor_ram(MemoryRegion *mr)
1064 {
1065 qemu_ram_free(mr->ram_block);
1066 }
1067
1068 static bool memory_region_need_escape(char c)
1069 {
1070 return c == '/' || c == '[' || c == '\\' || c == ']';
1071 }
1072
1073 static char *memory_region_escape_name(const char *name)
1074 {
1075 const char *p;
1076 char *escaped, *q;
1077 uint8_t c;
1078 size_t bytes = 0;
1079
1080 for (p = name; *p; p++) {
1081 bytes += memory_region_need_escape(*p) ? 4 : 1;
1082 }
1083 if (bytes == p - name) {
1084 return g_memdup(name, bytes + 1);
1085 }
1086
1087 escaped = g_malloc(bytes + 1);
1088 for (p = name, q = escaped; *p; p++) {
1089 c = *p;
1090 if (unlikely(memory_region_need_escape(c))) {
1091 *q++ = '\\';
1092 *q++ = 'x';
1093 *q++ = "0123456789abcdef"[c >> 4];
1094 c = "0123456789abcdef"[c & 15];
1095 }
1096 *q++ = c;
1097 }
1098 *q = 0;
1099 return escaped;
1100 }
1101
1102 static void memory_region_do_init(MemoryRegion *mr,
1103 Object *owner,
1104 const char *name,
1105 uint64_t size)
1106 {
1107 mr->size = int128_make64(size);
1108 if (size == UINT64_MAX) {
1109 mr->size = int128_2_64();
1110 }
1111 mr->name = g_strdup(name);
1112 mr->owner = owner;
1113 mr->ram_block = NULL;
1114
1115 if (name) {
1116 char *escaped_name = memory_region_escape_name(name);
1117 char *name_array = g_strdup_printf("%s[*]", escaped_name);
1118
1119 if (!owner) {
1120 owner = container_get(qdev_get_machine(), "/unattached");
1121 }
1122
1123 object_property_add_child(owner, name_array, OBJECT(mr), &error_abort);
1124 object_unref(OBJECT(mr));
1125 g_free(name_array);
1126 g_free(escaped_name);
1127 }
1128 }
1129
1130 void memory_region_init(MemoryRegion *mr,
1131 Object *owner,
1132 const char *name,
1133 uint64_t size)
1134 {
1135 object_initialize(mr, sizeof(*mr), TYPE_MEMORY_REGION);
1136 memory_region_do_init(mr, owner, name, size);
1137 }
1138
1139 static void memory_region_get_addr(Object *obj, Visitor *v, const char *name,
1140 void *opaque, Error **errp)
1141 {
1142 MemoryRegion *mr = MEMORY_REGION(obj);
1143 uint64_t value = mr->addr;
1144
1145 visit_type_uint64(v, name, &value, errp);
1146 }
1147
1148 static void memory_region_get_container(Object *obj, Visitor *v,
1149 const char *name, void *opaque,
1150 Error **errp)
1151 {
1152 MemoryRegion *mr = MEMORY_REGION(obj);
1153 gchar *path = (gchar *)"";
1154
1155 if (mr->container) {
1156 path = object_get_canonical_path(OBJECT(mr->container));
1157 }
1158 visit_type_str(v, name, &path, errp);
1159 if (mr->container) {
1160 g_free(path);
1161 }
1162 }
1163
1164 static Object *memory_region_resolve_container(Object *obj, void *opaque,
1165 const char *part)
1166 {
1167 MemoryRegion *mr = MEMORY_REGION(obj);
1168
1169 return OBJECT(mr->container);
1170 }
1171
1172 static void memory_region_get_priority(Object *obj, Visitor *v,
1173 const char *name, void *opaque,
1174 Error **errp)
1175 {
1176 MemoryRegion *mr = MEMORY_REGION(obj);
1177 int32_t value = mr->priority;
1178
1179 visit_type_int32(v, name, &value, errp);
1180 }
1181
1182 static void memory_region_get_size(Object *obj, Visitor *v, const char *name,
1183 void *opaque, Error **errp)
1184 {
1185 MemoryRegion *mr = MEMORY_REGION(obj);
1186 uint64_t value = memory_region_size(mr);
1187
1188 visit_type_uint64(v, name, &value, errp);
1189 }
1190
1191 static void memory_region_initfn(Object *obj)
1192 {
1193 MemoryRegion *mr = MEMORY_REGION(obj);
1194 ObjectProperty *op;
1195
1196 mr->ops = &unassigned_mem_ops;
1197 mr->enabled = true;
1198 mr->romd_mode = true;
1199 mr->global_locking = true;
1200 mr->destructor = memory_region_destructor_none;
1201 QTAILQ_INIT(&mr->subregions);
1202 QTAILQ_INIT(&mr->coalesced);
1203
1204 op = object_property_add(OBJECT(mr), "container",
1205 "link<" TYPE_MEMORY_REGION ">",
1206 memory_region_get_container,
1207 NULL, /* memory_region_set_container */
1208 NULL, NULL, &error_abort);
1209 op->resolve = memory_region_resolve_container;
1210
1211 object_property_add(OBJECT(mr), "addr", "uint64",
1212 memory_region_get_addr,
1213 NULL, /* memory_region_set_addr */
1214 NULL, NULL, &error_abort);
1215 object_property_add(OBJECT(mr), "priority", "uint32",
1216 memory_region_get_priority,
1217 NULL, /* memory_region_set_priority */
1218 NULL, NULL, &error_abort);
1219 object_property_add(OBJECT(mr), "size", "uint64",
1220 memory_region_get_size,
1221 NULL, /* memory_region_set_size, */
1222 NULL, NULL, &error_abort);
1223 }
1224
1225 static void iommu_memory_region_initfn(Object *obj)
1226 {
1227 MemoryRegion *mr = MEMORY_REGION(obj);
1228
1229 mr->is_iommu = true;
1230 }
1231
1232 static uint64_t unassigned_mem_read(void *opaque, hwaddr addr,
1233 unsigned size)
1234 {
1235 #ifdef DEBUG_UNASSIGNED
1236 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
1237 #endif
1238 if (current_cpu != NULL) {
1239 cpu_unassigned_access(current_cpu, addr, false, false, 0, size);
1240 }
1241 return 0;
1242 }
1243
1244 static void unassigned_mem_write(void *opaque, hwaddr addr,
1245 uint64_t val, unsigned size)
1246 {
1247 #ifdef DEBUG_UNASSIGNED
1248 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%"PRIx64"\n", addr, val);
1249 #endif
1250 if (current_cpu != NULL) {
1251 cpu_unassigned_access(current_cpu, addr, true, false, 0, size);
1252 }
1253 }
1254
1255 static bool unassigned_mem_accepts(void *opaque, hwaddr addr,
1256 unsigned size, bool is_write)
1257 {
1258 return false;
1259 }
1260
1261 const MemoryRegionOps unassigned_mem_ops = {
1262 .valid.accepts = unassigned_mem_accepts,
1263 .endianness = DEVICE_NATIVE_ENDIAN,
1264 };
1265
1266 static uint64_t memory_region_ram_device_read(void *opaque,
1267 hwaddr addr, unsigned size)
1268 {
1269 MemoryRegion *mr = opaque;
1270 uint64_t data = (uint64_t)~0;
1271
1272 switch (size) {
1273 case 1:
1274 data = *(uint8_t *)(mr->ram_block->host + addr);
1275 break;
1276 case 2:
1277 data = *(uint16_t *)(mr->ram_block->host + addr);
1278 break;
1279 case 4:
1280 data = *(uint32_t *)(mr->ram_block->host + addr);
1281 break;
1282 case 8:
1283 data = *(uint64_t *)(mr->ram_block->host + addr);
1284 break;
1285 }
1286
1287 trace_memory_region_ram_device_read(get_cpu_index(), mr, addr, data, size);
1288
1289 return data;
1290 }
1291
1292 static void memory_region_ram_device_write(void *opaque, hwaddr addr,
1293 uint64_t data, unsigned size)
1294 {
1295 MemoryRegion *mr = opaque;
1296
1297 trace_memory_region_ram_device_write(get_cpu_index(), mr, addr, data, size);
1298
1299 switch (size) {
1300 case 1:
1301 *(uint8_t *)(mr->ram_block->host + addr) = (uint8_t)data;
1302 break;
1303 case 2:
1304 *(uint16_t *)(mr->ram_block->host + addr) = (uint16_t)data;
1305 break;
1306 case 4:
1307 *(uint32_t *)(mr->ram_block->host + addr) = (uint32_t)data;
1308 break;
1309 case 8:
1310 *(uint64_t *)(mr->ram_block->host + addr) = data;
1311 break;
1312 }
1313 }
1314
1315 static const MemoryRegionOps ram_device_mem_ops = {
1316 .read = memory_region_ram_device_read,
1317 .write = memory_region_ram_device_write,
1318 .endianness = DEVICE_HOST_ENDIAN,
1319 .valid = {
1320 .min_access_size = 1,
1321 .max_access_size = 8,
1322 .unaligned = true,
1323 },
1324 .impl = {
1325 .min_access_size = 1,
1326 .max_access_size = 8,
1327 .unaligned = true,
1328 },
1329 };
1330
1331 bool memory_region_access_valid(MemoryRegion *mr,
1332 hwaddr addr,
1333 unsigned size,
1334 bool is_write)
1335 {
1336 int access_size_min, access_size_max;
1337 int access_size, i;
1338
1339 if (!mr->ops->valid.unaligned && (addr & (size - 1))) {
1340 return false;
1341 }
1342
1343 if (!mr->ops->valid.accepts) {
1344 return true;
1345 }
1346
1347 access_size_min = mr->ops->valid.min_access_size;
1348 if (!mr->ops->valid.min_access_size) {
1349 access_size_min = 1;
1350 }
1351
1352 access_size_max = mr->ops->valid.max_access_size;
1353 if (!mr->ops->valid.max_access_size) {
1354 access_size_max = 4;
1355 }
1356
1357 access_size = MAX(MIN(size, access_size_max), access_size_min);
1358 for (i = 0; i < size; i += access_size) {
1359 if (!mr->ops->valid.accepts(mr->opaque, addr + i, access_size,
1360 is_write)) {
1361 return false;
1362 }
1363 }
1364
1365 return true;
1366 }
1367
1368 static MemTxResult memory_region_dispatch_read1(MemoryRegion *mr,
1369 hwaddr addr,
1370 uint64_t *pval,
1371 unsigned size,
1372 MemTxAttrs attrs)
1373 {
1374 *pval = 0;
1375
1376 if (mr->ops->read) {
1377 return access_with_adjusted_size(addr, pval, size,
1378 mr->ops->impl.min_access_size,
1379 mr->ops->impl.max_access_size,
1380 memory_region_read_accessor,
1381 mr, attrs);
1382 } else if (mr->ops->read_with_attrs) {
1383 return access_with_adjusted_size(addr, pval, size,
1384 mr->ops->impl.min_access_size,
1385 mr->ops->impl.max_access_size,
1386 memory_region_read_with_attrs_accessor,
1387 mr, attrs);
1388 } else {
1389 return access_with_adjusted_size(addr, pval, size, 1, 4,
1390 memory_region_oldmmio_read_accessor,
1391 mr, attrs);
1392 }
1393 }
1394
1395 MemTxResult memory_region_dispatch_read(MemoryRegion *mr,
1396 hwaddr addr,
1397 uint64_t *pval,
1398 unsigned size,
1399 MemTxAttrs attrs)
1400 {
1401 MemTxResult r;
1402
1403 if (!memory_region_access_valid(mr, addr, size, false)) {
1404 *pval = unassigned_mem_read(mr, addr, size);
1405 return MEMTX_DECODE_ERROR;
1406 }
1407
1408 r = memory_region_dispatch_read1(mr, addr, pval, size, attrs);
1409 adjust_endianness(mr, pval, size);
1410 return r;
1411 }
1412
1413 /* Return true if an eventfd was signalled */
1414 static bool memory_region_dispatch_write_eventfds(MemoryRegion *mr,
1415 hwaddr addr,
1416 uint64_t data,
1417 unsigned size,
1418 MemTxAttrs attrs)
1419 {
1420 MemoryRegionIoeventfd ioeventfd = {
1421 .addr = addrrange_make(int128_make64(addr), int128_make64(size)),
1422 .data = data,
1423 };
1424 unsigned i;
1425
1426 for (i = 0; i < mr->ioeventfd_nb; i++) {
1427 ioeventfd.match_data = mr->ioeventfds[i].match_data;
1428 ioeventfd.e = mr->ioeventfds[i].e;
1429
1430 if (memory_region_ioeventfd_equal(ioeventfd, mr->ioeventfds[i])) {
1431 event_notifier_set(ioeventfd.e);
1432 return true;
1433 }
1434 }
1435
1436 return false;
1437 }
1438
1439 MemTxResult memory_region_dispatch_write(MemoryRegion *mr,
1440 hwaddr addr,
1441 uint64_t data,
1442 unsigned size,
1443 MemTxAttrs attrs)
1444 {
1445 if (!memory_region_access_valid(mr, addr, size, true)) {
1446 unassigned_mem_write(mr, addr, data, size);
1447 return MEMTX_DECODE_ERROR;
1448 }
1449
1450 adjust_endianness(mr, &data, size);
1451
1452 if ((!kvm_eventfds_enabled()) &&
1453 memory_region_dispatch_write_eventfds(mr, addr, data, size, attrs)) {
1454 return MEMTX_OK;
1455 }
1456
1457 if (mr->ops->write) {
1458 return access_with_adjusted_size(addr, &data, size,
1459 mr->ops->impl.min_access_size,
1460 mr->ops->impl.max_access_size,
1461 memory_region_write_accessor, mr,
1462 attrs);
1463 } else if (mr->ops->write_with_attrs) {
1464 return
1465 access_with_adjusted_size(addr, &data, size,
1466 mr->ops->impl.min_access_size,
1467 mr->ops->impl.max_access_size,
1468 memory_region_write_with_attrs_accessor,
1469 mr, attrs);
1470 } else {
1471 return access_with_adjusted_size(addr, &data, size, 1, 4,
1472 memory_region_oldmmio_write_accessor,
1473 mr, attrs);
1474 }
1475 }
1476
1477 void memory_region_init_io(MemoryRegion *mr,
1478 Object *owner,
1479 const MemoryRegionOps *ops,
1480 void *opaque,
1481 const char *name,
1482 uint64_t size)
1483 {
1484 memory_region_init(mr, owner, name, size);
1485 mr->ops = ops ? ops : &unassigned_mem_ops;
1486 mr->opaque = opaque;
1487 mr->terminates = true;
1488 }
1489
1490 void memory_region_init_ram_nomigrate(MemoryRegion *mr,
1491 Object *owner,
1492 const char *name,
1493 uint64_t size,
1494 Error **errp)
1495 {
1496 memory_region_init(mr, owner, name, size);
1497 mr->ram = true;
1498 mr->terminates = true;
1499 mr->destructor = memory_region_destructor_ram;
1500 mr->ram_block = qemu_ram_alloc(size, mr, errp);
1501 mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
1502 }
1503
1504 void memory_region_init_resizeable_ram(MemoryRegion *mr,
1505 Object *owner,
1506 const char *name,
1507 uint64_t size,
1508 uint64_t max_size,
1509 void (*resized)(const char*,
1510 uint64_t length,
1511 void *host),
1512 Error **errp)
1513 {
1514 memory_region_init(mr, owner, name, size);
1515 mr->ram = true;
1516 mr->terminates = true;
1517 mr->destructor = memory_region_destructor_ram;
1518 mr->ram_block = qemu_ram_alloc_resizeable(size, max_size, resized,
1519 mr, errp);
1520 mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
1521 }
1522
1523 #ifdef __linux__
1524 void memory_region_init_ram_from_file(MemoryRegion *mr,
1525 struct Object *owner,
1526 const char *name,
1527 uint64_t size,
1528 bool share,
1529 const char *path,
1530 Error **errp)
1531 {
1532 memory_region_init(mr, owner, name, size);
1533 mr->ram = true;
1534 mr->terminates = true;
1535 mr->destructor = memory_region_destructor_ram;
1536 mr->ram_block = qemu_ram_alloc_from_file(size, mr, share, path, errp);
1537 mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
1538 }
1539
1540 void memory_region_init_ram_from_fd(MemoryRegion *mr,
1541 struct Object *owner,
1542 const char *name,
1543 uint64_t size,
1544 bool share,
1545 int fd,
1546 Error **errp)
1547 {
1548 memory_region_init(mr, owner, name, size);
1549 mr->ram = true;
1550 mr->terminates = true;
1551 mr->destructor = memory_region_destructor_ram;
1552 mr->ram_block = qemu_ram_alloc_from_fd(size, mr, share, fd, errp);
1553 mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
1554 }
1555 #endif
1556
1557 void memory_region_init_ram_ptr(MemoryRegion *mr,
1558 Object *owner,
1559 const char *name,
1560 uint64_t size,
1561 void *ptr)
1562 {
1563 memory_region_init(mr, owner, name, size);
1564 mr->ram = true;
1565 mr->terminates = true;
1566 mr->destructor = memory_region_destructor_ram;
1567 mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
1568
1569 /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL. */
1570 assert(ptr != NULL);
1571 mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_fatal);
1572 }
1573
1574 void memory_region_init_ram_device_ptr(MemoryRegion *mr,
1575 Object *owner,
1576 const char *name,
1577 uint64_t size,
1578 void *ptr)
1579 {
1580 memory_region_init_ram_ptr(mr, owner, name, size, ptr);
1581 mr->ram_device = true;
1582 mr->ops = &ram_device_mem_ops;
1583 mr->opaque = mr;
1584 }
1585
1586 void memory_region_init_alias(MemoryRegion *mr,
1587 Object *owner,
1588 const char *name,
1589 MemoryRegion *orig,
1590 hwaddr offset,
1591 uint64_t size)
1592 {
1593 memory_region_init(mr, owner, name, size);
1594 mr->alias = orig;
1595 mr->alias_offset = offset;
1596 }
1597
1598 void memory_region_init_rom_nomigrate(MemoryRegion *mr,
1599 struct Object *owner,
1600 const char *name,
1601 uint64_t size,
1602 Error **errp)
1603 {
1604 memory_region_init(mr, owner, name, size);
1605 mr->ram = true;
1606 mr->readonly = true;
1607 mr->terminates = true;
1608 mr->destructor = memory_region_destructor_ram;
1609 mr->ram_block = qemu_ram_alloc(size, mr, errp);
1610 mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
1611 }
1612
1613 void memory_region_init_rom_device_nomigrate(MemoryRegion *mr,
1614 Object *owner,
1615 const MemoryRegionOps *ops,
1616 void *opaque,
1617 const char *name,
1618 uint64_t size,
1619 Error **errp)
1620 {
1621 assert(ops);
1622 memory_region_init(mr, owner, name, size);
1623 mr->ops = ops;
1624 mr->opaque = opaque;
1625 mr->terminates = true;
1626 mr->rom_device = true;
1627 mr->destructor = memory_region_destructor_ram;
1628 mr->ram_block = qemu_ram_alloc(size, mr, errp);
1629 }
1630
1631 void memory_region_init_iommu(void *_iommu_mr,
1632 size_t instance_size,
1633 const char *mrtypename,
1634 Object *owner,
1635 const char *name,
1636 uint64_t size)
1637 {
1638 struct IOMMUMemoryRegion *iommu_mr;
1639 struct MemoryRegion *mr;
1640
1641 object_initialize(_iommu_mr, instance_size, mrtypename);
1642 mr = MEMORY_REGION(_iommu_mr);
1643 memory_region_do_init(mr, owner, name, size);
1644 iommu_mr = IOMMU_MEMORY_REGION(mr);
1645 mr->terminates = true; /* then re-forwards */
1646 QLIST_INIT(&iommu_mr->iommu_notify);
1647 iommu_mr->iommu_notify_flags = IOMMU_NOTIFIER_NONE;
1648 }
1649
1650 static void memory_region_finalize(Object *obj)
1651 {
1652 MemoryRegion *mr = MEMORY_REGION(obj);
1653
1654 assert(!mr->container);
1655
1656 /* We know the region is not visible in any address space (it
1657 * does not have a container and cannot be a root either because
1658 * it has no references, so we can blindly clear mr->enabled.
1659 * memory_region_set_enabled instead could trigger a transaction
1660 * and cause an infinite loop.
1661 */
1662 mr->enabled = false;
1663 memory_region_transaction_begin();
1664 while (!QTAILQ_EMPTY(&mr->subregions)) {
1665 MemoryRegion *subregion = QTAILQ_FIRST(&mr->subregions);
1666 memory_region_del_subregion(mr, subregion);
1667 }
1668 memory_region_transaction_commit();
1669
1670 mr->destructor(mr);
1671 memory_region_clear_coalescing(mr);
1672 g_free((char *)mr->name);
1673 g_free(mr->ioeventfds);
1674 }
1675
1676 Object *memory_region_owner(MemoryRegion *mr)
1677 {
1678 Object *obj = OBJECT(mr);
1679 return obj->parent;
1680 }
1681
1682 void memory_region_ref(MemoryRegion *mr)
1683 {
1684 /* MMIO callbacks most likely will access data that belongs
1685 * to the owner, hence the need to ref/unref the owner whenever
1686 * the memory region is in use.
1687 *
1688 * The memory region is a child of its owner. As long as the
1689 * owner doesn't call unparent itself on the memory region,
1690 * ref-ing the owner will also keep the memory region alive.
1691 * Memory regions without an owner are supposed to never go away;
1692 * we do not ref/unref them because it slows down DMA sensibly.
1693 */
1694 if (mr && mr->owner) {
1695 object_ref(mr->owner);
1696 }
1697 }
1698
1699 void memory_region_unref(MemoryRegion *mr)
1700 {
1701 if (mr && mr->owner) {
1702 object_unref(mr->owner);
1703 }
1704 }
1705
1706 uint64_t memory_region_size(MemoryRegion *mr)
1707 {
1708 if (int128_eq(mr->size, int128_2_64())) {
1709 return UINT64_MAX;
1710 }
1711 return int128_get64(mr->size);
1712 }
1713
1714 const char *memory_region_name(const MemoryRegion *mr)
1715 {
1716 if (!mr->name) {
1717 ((MemoryRegion *)mr)->name =
1718 object_get_canonical_path_component(OBJECT(mr));
1719 }
1720 return mr->name;
1721 }
1722
1723 bool memory_region_is_ram_device(MemoryRegion *mr)
1724 {
1725 return mr->ram_device;
1726 }
1727
1728 uint8_t memory_region_get_dirty_log_mask(MemoryRegion *mr)
1729 {
1730 uint8_t mask = mr->dirty_log_mask;
1731 if (global_dirty_log && mr->ram_block) {
1732 mask |= (1 << DIRTY_MEMORY_MIGRATION);
1733 }
1734 return mask;
1735 }
1736
1737 bool memory_region_is_logging(MemoryRegion *mr, uint8_t client)
1738 {
1739 return memory_region_get_dirty_log_mask(mr) & (1 << client);
1740 }
1741
1742 static void memory_region_update_iommu_notify_flags(IOMMUMemoryRegion *iommu_mr)
1743 {
1744 IOMMUNotifierFlag flags = IOMMU_NOTIFIER_NONE;
1745 IOMMUNotifier *iommu_notifier;
1746 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr);
1747
1748 IOMMU_NOTIFIER_FOREACH(iommu_notifier, iommu_mr) {
1749 flags |= iommu_notifier->notifier_flags;
1750 }
1751
1752 if (flags != iommu_mr->iommu_notify_flags && imrc->notify_flag_changed) {
1753 imrc->notify_flag_changed(iommu_mr,
1754 iommu_mr->iommu_notify_flags,
1755 flags);
1756 }
1757
1758 iommu_mr->iommu_notify_flags = flags;
1759 }
1760
1761 void memory_region_register_iommu_notifier(MemoryRegion *mr,
1762 IOMMUNotifier *n)
1763 {
1764 IOMMUMemoryRegion *iommu_mr;
1765
1766 if (mr->alias) {
1767 memory_region_register_iommu_notifier(mr->alias, n);
1768 return;
1769 }
1770
1771 /* We need to register for at least one bitfield */
1772 iommu_mr = IOMMU_MEMORY_REGION(mr);
1773 assert(n->notifier_flags != IOMMU_NOTIFIER_NONE);
1774 assert(n->start <= n->end);
1775 QLIST_INSERT_HEAD(&iommu_mr->iommu_notify, n, node);
1776 memory_region_update_iommu_notify_flags(iommu_mr);
1777 }
1778
1779 uint64_t memory_region_iommu_get_min_page_size(IOMMUMemoryRegion *iommu_mr)
1780 {
1781 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr);
1782
1783 if (imrc->get_min_page_size) {
1784 return imrc->get_min_page_size(iommu_mr);
1785 }
1786 return TARGET_PAGE_SIZE;
1787 }
1788
1789 void memory_region_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n)
1790 {
1791 MemoryRegion *mr = MEMORY_REGION(iommu_mr);
1792 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr);
1793 hwaddr addr, granularity;
1794 IOMMUTLBEntry iotlb;
1795
1796 /* If the IOMMU has its own replay callback, override */
1797 if (imrc->replay) {
1798 imrc->replay(iommu_mr, n);
1799 return;
1800 }
1801
1802 granularity = memory_region_iommu_get_min_page_size(iommu_mr);
1803
1804 for (addr = 0; addr < memory_region_size(mr); addr += granularity) {
1805 iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE);
1806 if (iotlb.perm != IOMMU_NONE) {
1807 n->notify(n, &iotlb);
1808 }
1809
1810 /* if (2^64 - MR size) < granularity, it's possible to get an
1811 * infinite loop here. This should catch such a wraparound */
1812 if ((addr + granularity) < addr) {
1813 break;
1814 }
1815 }
1816 }
1817
1818 void memory_region_iommu_replay_all(IOMMUMemoryRegion *iommu_mr)
1819 {
1820 IOMMUNotifier *notifier;
1821
1822 IOMMU_NOTIFIER_FOREACH(notifier, iommu_mr) {
1823 memory_region_iommu_replay(iommu_mr, notifier);
1824 }
1825 }
1826
1827 void memory_region_unregister_iommu_notifier(MemoryRegion *mr,
1828 IOMMUNotifier *n)
1829 {
1830 IOMMUMemoryRegion *iommu_mr;
1831
1832 if (mr->alias) {
1833 memory_region_unregister_iommu_notifier(mr->alias, n);
1834 return;
1835 }
1836 QLIST_REMOVE(n, node);
1837 iommu_mr = IOMMU_MEMORY_REGION(mr);
1838 memory_region_update_iommu_notify_flags(iommu_mr);
1839 }
1840
1841 void memory_region_notify_one(IOMMUNotifier *notifier,
1842 IOMMUTLBEntry *entry)
1843 {
1844 IOMMUNotifierFlag request_flags;
1845
1846 /*
1847 * Skip the notification if the notification does not overlap
1848 * with registered range.
1849 */
1850 if (notifier->start > entry->iova + entry->addr_mask + 1 ||
1851 notifier->end < entry->iova) {
1852 return;
1853 }
1854
1855 if (entry->perm & IOMMU_RW) {
1856 request_flags = IOMMU_NOTIFIER_MAP;
1857 } else {
1858 request_flags = IOMMU_NOTIFIER_UNMAP;
1859 }
1860
1861 if (notifier->notifier_flags & request_flags) {
1862 notifier->notify(notifier, entry);
1863 }
1864 }
1865
1866 void memory_region_notify_iommu(IOMMUMemoryRegion *iommu_mr,
1867 IOMMUTLBEntry entry)
1868 {
1869 IOMMUNotifier *iommu_notifier;
1870
1871 assert(memory_region_is_iommu(MEMORY_REGION(iommu_mr)));
1872
1873 IOMMU_NOTIFIER_FOREACH(iommu_notifier, iommu_mr) {
1874 memory_region_notify_one(iommu_notifier, &entry);
1875 }
1876 }
1877
1878 void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client)
1879 {
1880 uint8_t mask = 1 << client;
1881 uint8_t old_logging;
1882
1883 assert(client == DIRTY_MEMORY_VGA);
1884 old_logging = mr->vga_logging_count;
1885 mr->vga_logging_count += log ? 1 : -1;
1886 if (!!old_logging == !!mr->vga_logging_count) {
1887 return;
1888 }
1889
1890 memory_region_transaction_begin();
1891 mr->dirty_log_mask = (mr->dirty_log_mask & ~mask) | (log * mask);
1892 memory_region_update_pending |= mr->enabled;
1893 memory_region_transaction_commit();
1894 }
1895
1896 bool memory_region_get_dirty(MemoryRegion *mr, hwaddr addr,
1897 hwaddr size, unsigned client)
1898 {
1899 assert(mr->ram_block);
1900 return cpu_physical_memory_get_dirty(memory_region_get_ram_addr(mr) + addr,
1901 size, client);
1902 }
1903
1904 void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr,
1905 hwaddr size)
1906 {
1907 assert(mr->ram_block);
1908 cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr,
1909 size,
1910 memory_region_get_dirty_log_mask(mr));
1911 }
1912
1913 bool memory_region_test_and_clear_dirty(MemoryRegion *mr, hwaddr addr,
1914 hwaddr size, unsigned client)
1915 {
1916 assert(mr->ram_block);
1917 return cpu_physical_memory_test_and_clear_dirty(
1918 memory_region_get_ram_addr(mr) + addr, size, client);
1919 }
1920
1921 DirtyBitmapSnapshot *memory_region_snapshot_and_clear_dirty(MemoryRegion *mr,
1922 hwaddr addr,
1923 hwaddr size,
1924 unsigned client)
1925 {
1926 assert(mr->ram_block);
1927 return cpu_physical_memory_snapshot_and_clear_dirty(
1928 memory_region_get_ram_addr(mr) + addr, size, client);
1929 }
1930
1931 bool memory_region_snapshot_get_dirty(MemoryRegion *mr, DirtyBitmapSnapshot *snap,
1932 hwaddr addr, hwaddr size)
1933 {
1934 assert(mr->ram_block);
1935 return cpu_physical_memory_snapshot_get_dirty(snap,
1936 memory_region_get_ram_addr(mr) + addr, size);
1937 }
1938
1939 void memory_region_sync_dirty_bitmap(MemoryRegion *mr)
1940 {
1941 MemoryListener *listener;
1942 AddressSpace *as;
1943 FlatView *view;
1944 FlatRange *fr;
1945
1946 /* If the same address space has multiple log_sync listeners, we
1947 * visit that address space's FlatView multiple times. But because
1948 * log_sync listeners are rare, it's still cheaper than walking each
1949 * address space once.
1950 */
1951 QTAILQ_FOREACH(listener, &memory_listeners, link) {
1952 if (!listener->log_sync) {
1953 continue;
1954 }
1955 as = listener->address_space;
1956 view = address_space_get_flatview(as);
1957 FOR_EACH_FLAT_RANGE(fr, view) {
1958 if (fr->mr == mr) {
1959 MemoryRegionSection mrs = section_from_flat_range(fr, view);
1960 listener->log_sync(listener, &mrs);
1961 }
1962 }
1963 flatview_unref(view);
1964 }
1965 }
1966
1967 void memory_region_set_readonly(MemoryRegion *mr, bool readonly)
1968 {
1969 if (mr->readonly != readonly) {
1970 memory_region_transaction_begin();
1971 mr->readonly = readonly;
1972 memory_region_update_pending |= mr->enabled;
1973 memory_region_transaction_commit();
1974 }
1975 }
1976
1977 void memory_region_rom_device_set_romd(MemoryRegion *mr, bool romd_mode)
1978 {
1979 if (mr->romd_mode != romd_mode) {
1980 memory_region_transaction_begin();
1981 mr->romd_mode = romd_mode;
1982 memory_region_update_pending |= mr->enabled;
1983 memory_region_transaction_commit();
1984 }
1985 }
1986
1987 void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr,
1988 hwaddr size, unsigned client)
1989 {
1990 assert(mr->ram_block);
1991 cpu_physical_memory_test_and_clear_dirty(
1992 memory_region_get_ram_addr(mr) + addr, size, client);
1993 }
1994
1995 int memory_region_get_fd(MemoryRegion *mr)
1996 {
1997 int fd;
1998
1999 rcu_read_lock();
2000 while (mr->alias) {
2001 mr = mr->alias;
2002 }
2003 fd = mr->ram_block->fd;
2004 rcu_read_unlock();
2005
2006 return fd;
2007 }
2008
2009 void *memory_region_get_ram_ptr(MemoryRegion *mr)
2010 {
2011 void *ptr;
2012 uint64_t offset = 0;
2013
2014 rcu_read_lock();
2015 while (mr->alias) {
2016 offset += mr->alias_offset;
2017 mr = mr->alias;
2018 }
2019 assert(mr->ram_block);
2020 ptr = qemu_map_ram_ptr(mr->ram_block, offset);
2021 rcu_read_unlock();
2022
2023 return ptr;
2024 }
2025
2026 MemoryRegion *memory_region_from_host(void *ptr, ram_addr_t *offset)
2027 {
2028 RAMBlock *block;
2029
2030 block = qemu_ram_block_from_host(ptr, false, offset);
2031 if (!block) {
2032 return NULL;
2033 }
2034
2035 return block->mr;
2036 }
2037
2038 ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr)
2039 {
2040 return mr->ram_block ? mr->ram_block->offset : RAM_ADDR_INVALID;
2041 }
2042
2043 void memory_region_ram_resize(MemoryRegion *mr, ram_addr_t newsize, Error **errp)
2044 {
2045 assert(mr->ram_block);
2046
2047 qemu_ram_resize(mr->ram_block, newsize, errp);
2048 }
2049
2050 static void memory_region_update_coalesced_range_as(MemoryRegion *mr, AddressSpace *as)
2051 {
2052 FlatView *view;
2053 FlatRange *fr;
2054 CoalescedMemoryRange *cmr;
2055 AddrRange tmp;
2056 MemoryRegionSection section;
2057
2058 view = address_space_get_flatview(as);
2059 FOR_EACH_FLAT_RANGE(fr, view) {
2060 if (fr->mr == mr) {
2061 section = (MemoryRegionSection) {
2062 .fv = view,
2063 .offset_within_address_space = int128_get64(fr->addr.start),
2064 .size = fr->addr.size,
2065 };
2066
2067 MEMORY_LISTENER_CALL(as, coalesced_mmio_del, Reverse, &section,
2068 int128_get64(fr->addr.start),
2069 int128_get64(fr->addr.size));
2070 QTAILQ_FOREACH(cmr, &mr->coalesced, link) {
2071 tmp = addrrange_shift(cmr->addr,
2072 int128_sub(fr->addr.start,
2073 int128_make64(fr->offset_in_region)));
2074 if (!addrrange_intersects(tmp, fr->addr)) {
2075 continue;
2076 }
2077 tmp = addrrange_intersection(tmp, fr->addr);
2078 MEMORY_LISTENER_CALL(as, coalesced_mmio_add, Forward, &section,
2079 int128_get64(tmp.start),
2080 int128_get64(tmp.size));
2081 }
2082 }
2083 }
2084 flatview_unref(view);
2085 }
2086
2087 static void memory_region_update_coalesced_range(MemoryRegion *mr)
2088 {
2089 AddressSpace *as;
2090
2091 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
2092 memory_region_update_coalesced_range_as(mr, as);
2093 }
2094 }
2095
2096 void memory_region_set_coalescing(MemoryRegion *mr)
2097 {
2098 memory_region_clear_coalescing(mr);
2099 memory_region_add_coalescing(mr, 0, int128_get64(mr->size));
2100 }
2101
2102 void memory_region_add_coalescing(MemoryRegion *mr,
2103 hwaddr offset,
2104 uint64_t size)
2105 {
2106 CoalescedMemoryRange *cmr = g_malloc(sizeof(*cmr));
2107
2108 cmr->addr = addrrange_make(int128_make64(offset), int128_make64(size));
2109 QTAILQ_INSERT_TAIL(&mr->coalesced, cmr, link);
2110 memory_region_update_coalesced_range(mr);
2111 memory_region_set_flush_coalesced(mr);
2112 }
2113
2114 void memory_region_clear_coalescing(MemoryRegion *mr)
2115 {
2116 CoalescedMemoryRange *cmr;
2117 bool updated = false;
2118
2119 qemu_flush_coalesced_mmio_buffer();
2120 mr->flush_coalesced_mmio = false;
2121
2122 while (!QTAILQ_EMPTY(&mr->coalesced)) {
2123 cmr = QTAILQ_FIRST(&mr->coalesced);
2124 QTAILQ_REMOVE(&mr->coalesced, cmr, link);
2125 g_free(cmr);
2126 updated = true;
2127 }
2128
2129 if (updated) {
2130 memory_region_update_coalesced_range(mr);
2131 }
2132 }
2133
2134 void memory_region_set_flush_coalesced(MemoryRegion *mr)
2135 {
2136 mr->flush_coalesced_mmio = true;
2137 }
2138
2139 void memory_region_clear_flush_coalesced(MemoryRegion *mr)
2140 {
2141 qemu_flush_coalesced_mmio_buffer();
2142 if (QTAILQ_EMPTY(&mr->coalesced)) {
2143 mr->flush_coalesced_mmio = false;
2144 }
2145 }
2146
2147 void memory_region_set_global_locking(MemoryRegion *mr)
2148 {
2149 mr->global_locking = true;
2150 }
2151
2152 void memory_region_clear_global_locking(MemoryRegion *mr)
2153 {
2154 mr->global_locking = false;
2155 }
2156
2157 static bool userspace_eventfd_warning;
2158
2159 void memory_region_add_eventfd(MemoryRegion *mr,
2160 hwaddr addr,
2161 unsigned size,
2162 bool match_data,
2163 uint64_t data,
2164 EventNotifier *e)
2165 {
2166 MemoryRegionIoeventfd mrfd = {
2167 .addr.start = int128_make64(addr),
2168 .addr.size = int128_make64(size),
2169 .match_data = match_data,
2170 .data = data,
2171 .e = e,
2172 };
2173 unsigned i;
2174
2175 if (kvm_enabled() && (!(kvm_eventfds_enabled() ||
2176 userspace_eventfd_warning))) {
2177 userspace_eventfd_warning = true;
2178 error_report("Using eventfd without MMIO binding in KVM. "
2179 "Suboptimal performance expected");
2180 }
2181
2182 if (size) {
2183 adjust_endianness(mr, &mrfd.data, size);
2184 }
2185 memory_region_transaction_begin();
2186 for (i = 0; i < mr->ioeventfd_nb; ++i) {
2187 if (memory_region_ioeventfd_before(mrfd, mr->ioeventfds[i])) {
2188 break;
2189 }
2190 }
2191 ++mr->ioeventfd_nb;
2192 mr->ioeventfds = g_realloc(mr->ioeventfds,
2193 sizeof(*mr->ioeventfds) * mr->ioeventfd_nb);
2194 memmove(&mr->ioeventfds[i+1], &mr->ioeventfds[i],
2195 sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb-1 - i));
2196 mr->ioeventfds[i] = mrfd;
2197 ioeventfd_update_pending |= mr->enabled;
2198 memory_region_transaction_commit();
2199 }
2200
2201 void memory_region_del_eventfd(MemoryRegion *mr,
2202 hwaddr addr,
2203 unsigned size,
2204 bool match_data,
2205 uint64_t data,
2206 EventNotifier *e)
2207 {
2208 MemoryRegionIoeventfd mrfd = {
2209 .addr.start = int128_make64(addr),
2210 .addr.size = int128_make64(size),
2211 .match_data = match_data,
2212 .data = data,
2213 .e = e,
2214 };
2215 unsigned i;
2216
2217 if (size) {
2218 adjust_endianness(mr, &mrfd.data, size);
2219 }
2220 memory_region_transaction_begin();
2221 for (i = 0; i < mr->ioeventfd_nb; ++i) {
2222 if (memory_region_ioeventfd_equal(mrfd, mr->ioeventfds[i])) {
2223 break;
2224 }
2225 }
2226 assert(i != mr->ioeventfd_nb);
2227 memmove(&mr->ioeventfds[i], &mr->ioeventfds[i+1],
2228 sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb - (i+1)));
2229 --mr->ioeventfd_nb;
2230 mr->ioeventfds = g_realloc(mr->ioeventfds,
2231 sizeof(*mr->ioeventfds)*mr->ioeventfd_nb + 1);
2232 ioeventfd_update_pending |= mr->enabled;
2233 memory_region_transaction_commit();
2234 }
2235
2236 static void memory_region_update_container_subregions(MemoryRegion *subregion)
2237 {
2238 MemoryRegion *mr = subregion->container;
2239 MemoryRegion *other;
2240
2241 memory_region_transaction_begin();
2242
2243 memory_region_ref(subregion);
2244 QTAILQ_FOREACH(other, &mr->subregions, subregions_link) {
2245 if (subregion->priority >= other->priority) {
2246 QTAILQ_INSERT_BEFORE(other, subregion, subregions_link);
2247 goto done;
2248 }
2249 }
2250 QTAILQ_INSERT_TAIL(&mr->subregions, subregion, subregions_link);
2251 done:
2252 memory_region_update_pending |= mr->enabled && subregion->enabled;
2253 memory_region_transaction_commit();
2254 }
2255
2256 static void memory_region_add_subregion_common(MemoryRegion *mr,
2257 hwaddr offset,
2258 MemoryRegion *subregion)
2259 {
2260 assert(!subregion->container);
2261 subregion->container = mr;
2262 subregion->addr = offset;
2263 memory_region_update_container_subregions(subregion);
2264 }
2265
2266 void memory_region_add_subregion(MemoryRegion *mr,
2267 hwaddr offset,
2268 MemoryRegion *subregion)
2269 {
2270 subregion->priority = 0;
2271 memory_region_add_subregion_common(mr, offset, subregion);
2272 }
2273
2274 void memory_region_add_subregion_overlap(MemoryRegion *mr,
2275 hwaddr offset,
2276 MemoryRegion *subregion,
2277 int priority)
2278 {
2279 subregion->priority = priority;
2280 memory_region_add_subregion_common(mr, offset, subregion);
2281 }
2282
2283 void memory_region_del_subregion(MemoryRegion *mr,
2284 MemoryRegion *subregion)
2285 {
2286 memory_region_transaction_begin();
2287 assert(subregion->container == mr);
2288 subregion->container = NULL;
2289 QTAILQ_REMOVE(&mr->subregions, subregion, subregions_link);
2290 memory_region_unref(subregion);
2291 memory_region_update_pending |= mr->enabled && subregion->enabled;
2292 memory_region_transaction_commit();
2293 }
2294
2295 void memory_region_set_enabled(MemoryRegion *mr, bool enabled)
2296 {
2297 if (enabled == mr->enabled) {
2298 return;
2299 }
2300 memory_region_transaction_begin();
2301 mr->enabled = enabled;
2302 memory_region_update_pending = true;
2303 memory_region_transaction_commit();
2304 }
2305
2306 void memory_region_set_size(MemoryRegion *mr, uint64_t size)
2307 {
2308 Int128 s = int128_make64(size);
2309
2310 if (size == UINT64_MAX) {
2311 s = int128_2_64();
2312 }
2313 if (int128_eq(s, mr->size)) {
2314 return;
2315 }
2316 memory_region_transaction_begin();
2317 mr->size = s;
2318 memory_region_update_pending = true;
2319 memory_region_transaction_commit();
2320 }
2321
2322 static void memory_region_readd_subregion(MemoryRegion *mr)
2323 {
2324 MemoryRegion *container = mr->container;
2325
2326 if (container) {
2327 memory_region_transaction_begin();
2328 memory_region_ref(mr);
2329 memory_region_del_subregion(container, mr);
2330 mr->container = container;
2331 memory_region_update_container_subregions(mr);
2332 memory_region_unref(mr);
2333 memory_region_transaction_commit();
2334 }
2335 }
2336
2337 void memory_region_set_address(MemoryRegion *mr, hwaddr addr)
2338 {
2339 if (addr != mr->addr) {
2340 mr->addr = addr;
2341 memory_region_readd_subregion(mr);
2342 }
2343 }
2344
2345 void memory_region_set_alias_offset(MemoryRegion *mr, hwaddr offset)
2346 {
2347 assert(mr->alias);
2348
2349 if (offset == mr->alias_offset) {
2350 return;
2351 }
2352
2353 memory_region_transaction_begin();
2354 mr->alias_offset = offset;
2355 memory_region_update_pending |= mr->enabled;
2356 memory_region_transaction_commit();
2357 }
2358
2359 uint64_t memory_region_get_alignment(const MemoryRegion *mr)
2360 {
2361 return mr->align;
2362 }
2363
2364 static int cmp_flatrange_addr(const void *addr_, const void *fr_)
2365 {
2366 const AddrRange *addr = addr_;
2367 const FlatRange *fr = fr_;
2368
2369 if (int128_le(addrrange_end(*addr), fr->addr.start)) {
2370 return -1;
2371 } else if (int128_ge(addr->start, addrrange_end(fr->addr))) {
2372 return 1;
2373 }
2374 return 0;
2375 }
2376
2377 static FlatRange *flatview_lookup(FlatView *view, AddrRange addr)
2378 {
2379 return bsearch(&addr, view->ranges, view->nr,
2380 sizeof(FlatRange), cmp_flatrange_addr);
2381 }
2382
2383 bool memory_region_is_mapped(MemoryRegion *mr)
2384 {
2385 return mr->container ? true : false;
2386 }
2387
2388 /* Same as memory_region_find, but it does not add a reference to the
2389 * returned region. It must be called from an RCU critical section.
2390 */
2391 static MemoryRegionSection memory_region_find_rcu(MemoryRegion *mr,
2392 hwaddr addr, uint64_t size)
2393 {
2394 MemoryRegionSection ret = { .mr = NULL };
2395 MemoryRegion *root;
2396 AddressSpace *as;
2397 AddrRange range;
2398 FlatView *view;
2399 FlatRange *fr;
2400
2401 addr += mr->addr;
2402 for (root = mr; root->container; ) {
2403 root = root->container;
2404 addr += root->addr;
2405 }
2406
2407 as = memory_region_to_address_space(root);
2408 if (!as) {
2409 return ret;
2410 }
2411 range = addrrange_make(int128_make64(addr), int128_make64(size));
2412
2413 view = address_space_to_flatview(as);
2414 fr = flatview_lookup(view, range);
2415 if (!fr) {
2416 return ret;
2417 }
2418
2419 while (fr > view->ranges && addrrange_intersects(fr[-1].addr, range)) {
2420 --fr;
2421 }
2422
2423 ret.mr = fr->mr;
2424 ret.fv = view;
2425 range = addrrange_intersection(range, fr->addr);
2426 ret.offset_within_region = fr->offset_in_region;
2427 ret.offset_within_region += int128_get64(int128_sub(range.start,
2428 fr->addr.start));
2429 ret.size = range.size;
2430 ret.offset_within_address_space = int128_get64(range.start);
2431 ret.readonly = fr->readonly;
2432 return ret;
2433 }
2434
2435 MemoryRegionSection memory_region_find(MemoryRegion *mr,
2436 hwaddr addr, uint64_t size)
2437 {
2438 MemoryRegionSection ret;
2439 rcu_read_lock();
2440 ret = memory_region_find_rcu(mr, addr, size);
2441 if (ret.mr) {
2442 memory_region_ref(ret.mr);
2443 }
2444 rcu_read_unlock();
2445 return ret;
2446 }
2447
2448 bool memory_region_present(MemoryRegion *container, hwaddr addr)
2449 {
2450 MemoryRegion *mr;
2451
2452 rcu_read_lock();
2453 mr = memory_region_find_rcu(container, addr, 1).mr;
2454 rcu_read_unlock();
2455 return mr && mr != container;
2456 }
2457
2458 void memory_global_dirty_log_sync(void)
2459 {
2460 MemoryListener *listener;
2461 AddressSpace *as;
2462 FlatView *view;
2463 FlatRange *fr;
2464
2465 QTAILQ_FOREACH(listener, &memory_listeners, link) {
2466 if (!listener->log_sync) {
2467 continue;
2468 }
2469 as = listener->address_space;
2470 view = address_space_get_flatview(as);
2471 FOR_EACH_FLAT_RANGE(fr, view) {
2472 if (fr->dirty_log_mask) {
2473 MemoryRegionSection mrs = section_from_flat_range(fr, view);
2474
2475 listener->log_sync(listener, &mrs);
2476 }
2477 }
2478 flatview_unref(view);
2479 }
2480 }
2481
2482 static VMChangeStateEntry *vmstate_change;
2483
2484 void memory_global_dirty_log_start(void)
2485 {
2486 if (vmstate_change) {
2487 qemu_del_vm_change_state_handler(vmstate_change);
2488 vmstate_change = NULL;
2489 }
2490
2491 global_dirty_log = true;
2492
2493 MEMORY_LISTENER_CALL_GLOBAL(log_global_start, Forward);
2494
2495 /* Refresh DIRTY_LOG_MIGRATION bit. */
2496 memory_region_transaction_begin();
2497 memory_region_update_pending = true;
2498 memory_region_transaction_commit();
2499 }
2500
2501 static void memory_global_dirty_log_do_stop(void)
2502 {
2503 global_dirty_log = false;
2504
2505 /* Refresh DIRTY_LOG_MIGRATION bit. */
2506 memory_region_transaction_begin();
2507 memory_region_update_pending = true;
2508 memory_region_transaction_commit();
2509
2510 MEMORY_LISTENER_CALL_GLOBAL(log_global_stop, Reverse);
2511 }
2512
2513 static void memory_vm_change_state_handler(void *opaque, int running,
2514 RunState state)
2515 {
2516 if (running) {
2517 memory_global_dirty_log_do_stop();
2518
2519 if (vmstate_change) {
2520 qemu_del_vm_change_state_handler(vmstate_change);
2521 vmstate_change = NULL;
2522 }
2523 }
2524 }
2525
2526 void memory_global_dirty_log_stop(void)
2527 {
2528 if (!runstate_is_running()) {
2529 if (vmstate_change) {
2530 return;
2531 }
2532 vmstate_change = qemu_add_vm_change_state_handler(
2533 memory_vm_change_state_handler, NULL);
2534 return;
2535 }
2536
2537 memory_global_dirty_log_do_stop();
2538 }
2539
2540 static void listener_add_address_space(MemoryListener *listener,
2541 AddressSpace *as)
2542 {
2543 FlatView *view;
2544 FlatRange *fr;
2545
2546 if (listener->begin) {
2547 listener->begin(listener);
2548 }
2549 if (global_dirty_log) {
2550 if (listener->log_global_start) {
2551 listener->log_global_start(listener);
2552 }
2553 }
2554
2555 view = address_space_get_flatview(as);
2556 FOR_EACH_FLAT_RANGE(fr, view) {
2557 MemoryRegionSection section = {
2558 .mr = fr->mr,
2559 .fv = view,
2560 .offset_within_region = fr->offset_in_region,
2561 .size = fr->addr.size,
2562 .offset_within_address_space = int128_get64(fr->addr.start),
2563 .readonly = fr->readonly,
2564 };
2565 if (fr->dirty_log_mask && listener->log_start) {
2566 listener->log_start(listener, &section, 0, fr->dirty_log_mask);
2567 }
2568 if (listener->region_add) {
2569 listener->region_add(listener, &section);
2570 }
2571 }
2572 if (listener->commit) {
2573 listener->commit(listener);
2574 }
2575 flatview_unref(view);
2576 }
2577
2578 void memory_listener_register(MemoryListener *listener, AddressSpace *as)
2579 {
2580 MemoryListener *other = NULL;
2581
2582 listener->address_space = as;
2583 if (QTAILQ_EMPTY(&memory_listeners)
2584 || listener->priority >= QTAILQ_LAST(&memory_listeners,
2585 memory_listeners)->priority) {
2586 QTAILQ_INSERT_TAIL(&memory_listeners, listener, link);
2587 } else {
2588 QTAILQ_FOREACH(other, &memory_listeners, link) {
2589 if (listener->priority < other->priority) {
2590 break;
2591 }
2592 }
2593 QTAILQ_INSERT_BEFORE(other, listener, link);
2594 }
2595
2596 if (QTAILQ_EMPTY(&as->listeners)
2597 || listener->priority >= QTAILQ_LAST(&as->listeners,
2598 memory_listeners)->priority) {
2599 QTAILQ_INSERT_TAIL(&as->listeners, listener, link_as);
2600 } else {
2601 QTAILQ_FOREACH(other, &as->listeners, link_as) {
2602 if (listener->priority < other->priority) {
2603 break;
2604 }
2605 }
2606 QTAILQ_INSERT_BEFORE(other, listener, link_as);
2607 }
2608
2609 listener_add_address_space(listener, as);
2610 }
2611
2612 void memory_listener_unregister(MemoryListener *listener)
2613 {
2614 if (!listener->address_space) {
2615 return;
2616 }
2617
2618 QTAILQ_REMOVE(&memory_listeners, listener, link);
2619 QTAILQ_REMOVE(&listener->address_space->listeners, listener, link_as);
2620 listener->address_space = NULL;
2621 }
2622
2623 bool memory_region_request_mmio_ptr(MemoryRegion *mr, hwaddr addr)
2624 {
2625 void *host;
2626 unsigned size = 0;
2627 unsigned offset = 0;
2628 Object *new_interface;
2629
2630 if (!mr || !mr->ops->request_ptr) {
2631 return false;
2632 }
2633
2634 /*
2635 * Avoid an update if the request_ptr call
2636 * memory_region_invalidate_mmio_ptr which seems to be likely when we use
2637 * a cache.
2638 */
2639 memory_region_transaction_begin();
2640
2641 host = mr->ops->request_ptr(mr->opaque, addr - mr->addr, &size, &offset);
2642
2643 if (!host || !size) {
2644 memory_region_transaction_commit();
2645 return false;
2646 }
2647
2648 new_interface = object_new("mmio_interface");
2649 qdev_prop_set_uint64(DEVICE(new_interface), "start", offset);
2650 qdev_prop_set_uint64(DEVICE(new_interface), "end", offset + size - 1);
2651 qdev_prop_set_bit(DEVICE(new_interface), "ro", true);
2652 qdev_prop_set_ptr(DEVICE(new_interface), "host_ptr", host);
2653 qdev_prop_set_ptr(DEVICE(new_interface), "subregion", mr);
2654 object_property_set_bool(OBJECT(new_interface), true, "realized", NULL);
2655
2656 memory_region_transaction_commit();
2657 return true;
2658 }
2659
2660 typedef struct MMIOPtrInvalidate {
2661 MemoryRegion *mr;
2662 hwaddr offset;
2663 unsigned size;
2664 int busy;
2665 int allocated;
2666 } MMIOPtrInvalidate;
2667
2668 #define MAX_MMIO_INVALIDATE 10
2669 static MMIOPtrInvalidate mmio_ptr_invalidate_list[MAX_MMIO_INVALIDATE];
2670
2671 static void memory_region_do_invalidate_mmio_ptr(CPUState *cpu,
2672 run_on_cpu_data data)
2673 {
2674 MMIOPtrInvalidate *invalidate_data = (MMIOPtrInvalidate *)data.host_ptr;
2675 MemoryRegion *mr = invalidate_data->mr;
2676 hwaddr offset = invalidate_data->offset;
2677 unsigned size = invalidate_data->size;
2678 MemoryRegionSection section = memory_region_find(mr, offset, size);
2679
2680 qemu_mutex_lock_iothread();
2681
2682 /* Reset dirty so this doesn't happen later. */
2683 cpu_physical_memory_test_and_clear_dirty(offset, size, 1);
2684
2685 if (section.mr != mr) {
2686 /* memory_region_find add a ref on section.mr */
2687 memory_region_unref(section.mr);
2688 if (MMIO_INTERFACE(section.mr->owner)) {
2689 /* We found the interface just drop it. */
2690 object_property_set_bool(section.mr->owner, false, "realized",
2691 NULL);
2692 object_unref(section.mr->owner);
2693 object_unparent(section.mr->owner);
2694 }
2695 }
2696
2697 qemu_mutex_unlock_iothread();
2698
2699 if (invalidate_data->allocated) {
2700 g_free(invalidate_data);
2701 } else {
2702 invalidate_data->busy = 0;
2703 }
2704 }
2705
2706 void memory_region_invalidate_mmio_ptr(MemoryRegion *mr, hwaddr offset,
2707 unsigned size)
2708 {
2709 size_t i;
2710 MMIOPtrInvalidate *invalidate_data = NULL;
2711
2712 for (i = 0; i < MAX_MMIO_INVALIDATE; i++) {
2713 if (atomic_cmpxchg(&(mmio_ptr_invalidate_list[i].busy), 0, 1) == 0) {
2714 invalidate_data = &mmio_ptr_invalidate_list[i];
2715 break;
2716 }
2717 }
2718
2719 if (!invalidate_data) {
2720 invalidate_data = g_malloc0(sizeof(MMIOPtrInvalidate));
2721 invalidate_data->allocated = 1;
2722 }
2723
2724 invalidate_data->mr = mr;
2725 invalidate_data->offset = offset;
2726 invalidate_data->size = size;
2727
2728 async_safe_run_on_cpu(first_cpu, memory_region_do_invalidate_mmio_ptr,
2729 RUN_ON_CPU_HOST_PTR(invalidate_data));
2730 }
2731
2732 void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name)
2733 {
2734 memory_region_ref(root);
2735 as->root = root;
2736 as->current_map = NULL;
2737 as->ioeventfd_nb = 0;
2738 as->ioeventfds = NULL;
2739 QTAILQ_INIT(&as->listeners);
2740 QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link);
2741 as->name = g_strdup(name ? name : "anonymous");
2742 address_space_update_topology(as);
2743 address_space_update_ioeventfds(as);
2744 }
2745
2746 static void do_address_space_destroy(AddressSpace *as)
2747 {
2748 assert(QTAILQ_EMPTY(&as->listeners));
2749
2750 flatview_unref(as->current_map);
2751 g_free(as->name);
2752 g_free(as->ioeventfds);
2753 memory_region_unref(as->root);
2754 }
2755
2756 void address_space_destroy(AddressSpace *as)
2757 {
2758 MemoryRegion *root = as->root;
2759
2760 /* Flush out anything from MemoryListeners listening in on this */
2761 memory_region_transaction_begin();
2762 as->root = NULL;
2763 memory_region_transaction_commit();
2764 QTAILQ_REMOVE(&address_spaces, as, address_spaces_link);
2765
2766 /* At this point, as->dispatch and as->current_map are dummy
2767 * entries that the guest should never use. Wait for the old
2768 * values to expire before freeing the data.
2769 */
2770 as->root = root;
2771 call_rcu(as, do_address_space_destroy, rcu);
2772 }
2773
2774 static const char *memory_region_type(MemoryRegion *mr)
2775 {
2776 if (memory_region_is_ram_device(mr)) {
2777 return "ramd";
2778 } else if (memory_region_is_romd(mr)) {
2779 return "romd";
2780 } else if (memory_region_is_rom(mr)) {
2781 return "rom";
2782 } else if (memory_region_is_ram(mr)) {
2783 return "ram";
2784 } else {
2785 return "i/o";
2786 }
2787 }
2788
2789 typedef struct MemoryRegionList MemoryRegionList;
2790
2791 struct MemoryRegionList {
2792 const MemoryRegion *mr;
2793 QTAILQ_ENTRY(MemoryRegionList) mrqueue;
2794 };
2795
2796 typedef QTAILQ_HEAD(mrqueue, MemoryRegionList) MemoryRegionListHead;
2797
2798 #define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \
2799 int128_sub((size), int128_one())) : 0)
2800 #define MTREE_INDENT " "
2801
2802 static void mtree_print_mr(fprintf_function mon_printf, void *f,
2803 const MemoryRegion *mr, unsigned int level,
2804 hwaddr base,
2805 MemoryRegionListHead *alias_print_queue)
2806 {
2807 MemoryRegionList *new_ml, *ml, *next_ml;
2808 MemoryRegionListHead submr_print_queue;
2809 const MemoryRegion *submr;
2810 unsigned int i;
2811 hwaddr cur_start, cur_end;
2812
2813 if (!mr) {
2814 return;
2815 }
2816
2817 for (i = 0; i < level; i++) {
2818 mon_printf(f, MTREE_INDENT);
2819 }
2820
2821 cur_start = base + mr->addr;
2822 cur_end = cur_start + MR_SIZE(mr->size);
2823
2824 /*
2825 * Try to detect overflow of memory region. This should never
2826 * happen normally. When it happens, we dump something to warn the
2827 * user who is observing this.
2828 */
2829 if (cur_start < base || cur_end < cur_start) {
2830 mon_printf(f, "[DETECTED OVERFLOW!] ");
2831 }
2832
2833 if (mr->alias) {
2834 MemoryRegionList *ml;
2835 bool found = false;
2836
2837 /* check if the alias is already in the queue */
2838 QTAILQ_FOREACH(ml, alias_print_queue, mrqueue) {
2839 if (ml->mr == mr->alias) {
2840 found = true;
2841 }
2842 }
2843
2844 if (!found) {
2845 ml = g_new(MemoryRegionList, 1);
2846 ml->mr = mr->alias;
2847 QTAILQ_INSERT_TAIL(alias_print_queue, ml, mrqueue);
2848 }
2849 mon_printf(f, TARGET_FMT_plx "-" TARGET_FMT_plx
2850 " (prio %d, %s): alias %s @%s " TARGET_FMT_plx
2851 "-" TARGET_FMT_plx "%s\n",
2852 cur_start, cur_end,
2853 mr->priority,
2854 memory_region_type((MemoryRegion *)mr),
2855 memory_region_name(mr),
2856 memory_region_name(mr->alias),
2857 mr->alias_offset,
2858 mr->alias_offset + MR_SIZE(mr->size),
2859 mr->enabled ? "" : " [disabled]");
2860 } else {
2861 mon_printf(f,
2862 TARGET_FMT_plx "-" TARGET_FMT_plx " (prio %d, %s): %s%s\n",
2863 cur_start, cur_end,
2864 mr->priority,
2865 memory_region_type((MemoryRegion *)mr),
2866 memory_region_name(mr),
2867 mr->enabled ? "" : " [disabled]");
2868 }
2869
2870 QTAILQ_INIT(&submr_print_queue);
2871
2872 QTAILQ_FOREACH(submr, &mr->subregions, subregions_link) {
2873 new_ml = g_new(MemoryRegionList, 1);
2874 new_ml->mr = submr;
2875 QTAILQ_FOREACH(ml, &submr_print_queue, mrqueue) {
2876 if (new_ml->mr->addr < ml->mr->addr ||
2877 (new_ml->mr->addr == ml->mr->addr &&
2878 new_ml->mr->priority > ml->mr->priority)) {
2879 QTAILQ_INSERT_BEFORE(ml, new_ml, mrqueue);
2880 new_ml = NULL;
2881 break;
2882 }
2883 }
2884 if (new_ml) {
2885 QTAILQ_INSERT_TAIL(&submr_print_queue, new_ml, mrqueue);
2886 }
2887 }
2888
2889 QTAILQ_FOREACH(ml, &submr_print_queue, mrqueue) {
2890 mtree_print_mr(mon_printf, f, ml->mr, level + 1, cur_start,
2891 alias_print_queue);
2892 }
2893
2894 QTAILQ_FOREACH_SAFE(ml, &submr_print_queue, mrqueue, next_ml) {
2895 g_free(ml);
2896 }
2897 }
2898
2899 struct FlatViewInfo {
2900 fprintf_function mon_printf;
2901 void *f;
2902 int counter;
2903 bool dispatch_tree;
2904 };
2905
2906 static void mtree_print_flatview(gpointer key, gpointer value,
2907 gpointer user_data)
2908 {
2909 FlatView *view = key;
2910 GArray *fv_address_spaces = value;
2911 struct FlatViewInfo *fvi = user_data;
2912 fprintf_function p = fvi->mon_printf;
2913 void *f = fvi->f;
2914 FlatRange *range = &view->ranges[0];
2915 MemoryRegion *mr;
2916 int n = view->nr;
2917 int i;
2918 AddressSpace *as;
2919
2920 p(f, "FlatView #%d\n", fvi->counter);
2921 ++fvi->counter;
2922
2923 for (i = 0; i < fv_address_spaces->len; ++i) {
2924 as = g_array_index(fv_address_spaces, AddressSpace*, i);
2925 p(f, " AS \"%s\", root: %s", as->name, memory_region_name(as->root));
2926 if (as->root->alias) {
2927 p(f, ", alias %s", memory_region_name(as->root->alias));
2928 }
2929 p(f, "\n");
2930 }
2931
2932 p(f, " Root memory region: %s\n",
2933 view->root ? memory_region_name(view->root) : "(none)");
2934
2935 if (n <= 0) {
2936 p(f, MTREE_INDENT "No rendered FlatView\n\n");
2937 return;
2938 }
2939
2940 while (n--) {
2941 mr = range->mr;
2942 if (range->offset_in_region) {
2943 p(f, MTREE_INDENT TARGET_FMT_plx "-"
2944 TARGET_FMT_plx " (prio %d, %s): %s @" TARGET_FMT_plx "\n",
2945 int128_get64(range->addr.start),
2946 int128_get64(range->addr.start) + MR_SIZE(range->addr.size),
2947 mr->priority,
2948 range->readonly ? "rom" : memory_region_type(mr),
2949 memory_region_name(mr),
2950 range->offset_in_region);
2951 } else {
2952 p(f, MTREE_INDENT TARGET_FMT_plx "-"
2953 TARGET_FMT_plx " (prio %d, %s): %s\n",
2954 int128_get64(range->addr.start),
2955 int128_get64(range->addr.start) + MR_SIZE(range->addr.size),
2956 mr->priority,
2957 range->readonly ? "rom" : memory_region_type(mr),
2958 memory_region_name(mr));
2959 }
2960 range++;
2961 }
2962
2963 #if !defined(CONFIG_USER_ONLY)
2964 if (fvi->dispatch_tree && view->root) {
2965 mtree_print_dispatch(p, f, view->dispatch, view->root);
2966 }
2967 #endif
2968
2969 p(f, "\n");
2970 }
2971
2972 static gboolean mtree_info_flatview_free(gpointer key, gpointer value,
2973 gpointer user_data)
2974 {
2975 FlatView *view = key;
2976 GArray *fv_address_spaces = value;
2977
2978 g_array_unref(fv_address_spaces);
2979 flatview_unref(view);
2980
2981 return true;
2982 }
2983
2984 void mtree_info(fprintf_function mon_printf, void *f, bool flatview,
2985 bool dispatch_tree)
2986 {
2987 MemoryRegionListHead ml_head;
2988 MemoryRegionList *ml, *ml2;
2989 AddressSpace *as;
2990
2991 if (flatview) {
2992 FlatView *view;
2993 struct FlatViewInfo fvi = {
2994 .mon_printf = mon_printf,
2995 .f = f,
2996 .counter = 0,
2997 .dispatch_tree = dispatch_tree
2998 };
2999 GArray *fv_address_spaces;
3000 GHashTable *views = g_hash_table_new(g_direct_hash, g_direct_equal);
3001
3002 /* Gather all FVs in one table */
3003 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
3004 view = address_space_get_flatview(as);
3005
3006 fv_address_spaces = g_hash_table_lookup(views, view);
3007 if (!fv_address_spaces) {
3008 fv_address_spaces = g_array_new(false, false, sizeof(as));
3009 g_hash_table_insert(views, view, fv_address_spaces);
3010 }
3011
3012 g_array_append_val(fv_address_spaces, as);
3013 }
3014
3015 /* Print */
3016 g_hash_table_foreach(views, mtree_print_flatview, &fvi);
3017
3018 /* Free */
3019 g_hash_table_foreach_remove(views, mtree_info_flatview_free, 0);
3020 g_hash_table_unref(views);
3021
3022 return;
3023 }
3024
3025 QTAILQ_INIT(&ml_head);
3026
3027 QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
3028 mon_printf(f, "address-space: %s\n", as->name);
3029 mtree_print_mr(mon_printf, f, as->root, 1, 0, &ml_head);
3030 mon_printf(f, "\n");
3031 }
3032
3033 /* print aliased regions */
3034 QTAILQ_FOREACH(ml, &ml_head, mrqueue) {
3035 mon_printf(f, "memory-region: %s\n", memory_region_name(ml->mr));
3036 mtree_print_mr(mon_printf, f, ml->mr, 1, 0, &ml_head);
3037 mon_printf(f, "\n");
3038 }
3039
3040 QTAILQ_FOREACH_SAFE(ml, &ml_head, mrqueue, ml2) {
3041 g_free(ml);
3042 }
3043 }
3044
3045 void memory_region_init_ram(MemoryRegion *mr,
3046 struct Object *owner,
3047 const char *name,
3048 uint64_t size,
3049 Error **errp)
3050 {
3051 DeviceState *owner_dev;
3052 Error *err = NULL;
3053
3054 memory_region_init_ram_nomigrate(mr, owner, name, size, &err);
3055 if (err) {
3056 error_propagate(errp, err);
3057 return;
3058 }
3059 /* This will assert if owner is neither NULL nor a DeviceState.
3060 * We only want the owner here for the purposes of defining a
3061 * unique name for migration. TODO: Ideally we should implement
3062 * a naming scheme for Objects which are not DeviceStates, in
3063 * which case we can relax this restriction.
3064 */
3065 owner_dev = DEVICE(owner);
3066 vmstate_register_ram(mr, owner_dev);
3067 }
3068
3069 void memory_region_init_rom(MemoryRegion *mr,
3070 struct Object *owner,
3071 const char *name,
3072 uint64_t size,
3073 Error **errp)
3074 {
3075 DeviceState *owner_dev;
3076 Error *err = NULL;
3077
3078 memory_region_init_rom_nomigrate(mr, owner, name, size, &err);
3079 if (err) {
3080 error_propagate(errp, err);
3081 return;
3082 }
3083 /* This will assert if owner is neither NULL nor a DeviceState.
3084 * We only want the owner here for the purposes of defining a
3085 * unique name for migration. TODO: Ideally we should implement
3086 * a naming scheme for Objects which are not DeviceStates, in
3087 * which case we can relax this restriction.
3088 */
3089 owner_dev = DEVICE(owner);
3090 vmstate_register_ram(mr, owner_dev);
3091 }
3092
3093 void memory_region_init_rom_device(MemoryRegion *mr,
3094 struct Object *owner,
3095 const MemoryRegionOps *ops,
3096 void *opaque,
3097 const char *name,
3098 uint64_t size,
3099 Error **errp)
3100 {
3101 DeviceState *owner_dev;
3102 Error *err = NULL;
3103
3104 memory_region_init_rom_device_nomigrate(mr, owner, ops, opaque,
3105 name, size, &err);
3106 if (err) {
3107 error_propagate(errp, err);
3108 return;
3109 }
3110 /* This will assert if owner is neither NULL nor a DeviceState.
3111 * We only want the owner here for the purposes of defining a
3112 * unique name for migration. TODO: Ideally we should implement
3113 * a naming scheme for Objects which are not DeviceStates, in
3114 * which case we can relax this restriction.
3115 */
3116 owner_dev = DEVICE(owner);
3117 vmstate_register_ram(mr, owner_dev);
3118 }
3119
3120 static const TypeInfo memory_region_info = {
3121 .parent = TYPE_OBJECT,
3122 .name = TYPE_MEMORY_REGION,
3123 .instance_size = sizeof(MemoryRegion),
3124 .instance_init = memory_region_initfn,
3125 .instance_finalize = memory_region_finalize,
3126 };
3127
3128 static const TypeInfo iommu_memory_region_info = {
3129 .parent = TYPE_MEMORY_REGION,
3130 .name = TYPE_IOMMU_MEMORY_REGION,
3131 .class_size = sizeof(IOMMUMemoryRegionClass),
3132 .instance_size = sizeof(IOMMUMemoryRegion),
3133 .instance_init = iommu_memory_region_initfn,
3134 .abstract = true,
3135 };
3136
3137 static void memory_register_types(void)
3138 {
3139 type_register_static(&memory_region_info);
3140 type_register_static(&iommu_memory_region_info);
3141 }
3142
3143 type_init(memory_register_types)
QEMU mirror