memory.c

   1 /*
   2  * Physical memory management
   3  *
   4  * Copyright 2011 Red Hat, Inc. and/or its affiliates
   5  *
   6  * Authors:
   7  *  Avi Kivity <avi@redhat.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  */
  13
  14 #include "memory.h"
  15 #include "exec-memory.h"
  16 #include "ioport.h"
  17 #include "bitops.h"
  18 #include "kvm.h"
  19 #include <assert.h>
  20
  21 unsigned memory_region_transaction_depth = 0;
  22
  23 typedef struct AddrRange AddrRange;
  24
  25 struct AddrRange {
  26     uint64_t start;
  27     uint64_t size;
  28 };
  29
  30 static AddrRange addrrange_make(uint64_t start, uint64_t size)
  31 {
  32     return (AddrRange) { start, size };
  33 }
  34
  35 static bool addrrange_equal(AddrRange r1, AddrRange r2)
  36 {
  37     return r1.start == r2.start && r1.size == r2.size;
  38 }
  39
  40 static uint64_t addrrange_end(AddrRange r)
  41 {
  42     return r.start + r.size;
  43 }
  44
  45 static AddrRange addrrange_shift(AddrRange range, int64_t delta)
  46 {
  47     range.start += delta;
  48     return range;
  49 }
  50
  51 static bool addrrange_intersects(AddrRange r1, AddrRange r2)
  52 {
  53     return (r1.start >= r2.start && r1.start < r2.start + r2.size)
  54         || (r2.start >= r1.start && r2.start < r1.start + r1.size);
  55 }
  56
  57 static AddrRange addrrange_intersection(AddrRange r1, AddrRange r2)
  58 {
  59     uint64_t start = MAX(r1.start, r2.start);
  60     /* off-by-one arithmetic to prevent overflow */
  61     uint64_t end = MIN(addrrange_end(r1) - 1, addrrange_end(r2) - 1);
  62     return addrrange_make(start, end - start + 1);
  63 }
  64
  65 struct CoalescedMemoryRange {
  66     AddrRange addr;
  67     QTAILQ_ENTRY(CoalescedMemoryRange) link;
  68 };
  69
  70 struct MemoryRegionIoeventfd {
  71     AddrRange addr;
  72     bool match_data;
  73     uint64_t data;
  74     int fd;
  75 };
  76
  77 static bool memory_region_ioeventfd_before(MemoryRegionIoeventfd a,
  78                                            MemoryRegionIoeventfd b)
  79 {
  80     if (a.addr.start < b.addr.start) {
  81         return true;
  82     } else if (a.addr.start > b.addr.start) {
  83         return false;
  84     } else if (a.addr.size < b.addr.size) {
  85         return true;
  86     } else if (a.addr.size > b.addr.size) {
  87         return false;
  88     } else if (a.match_data < b.match_data) {
  89         return true;
  90     } else  if (a.match_data > b.match_data) {
  91         return false;
  92     } else if (a.match_data) {
  93         if (a.data < b.data) {
  94             return true;
  95         } else if (a.data > b.data) {
  96             return false;
  97         }
  98     }
  99     if (a.fd < b.fd) {
 100         return true;
 101     } else if (a.fd > b.fd) {
 102         return false;
 103     }
 104     return false;
 105 }
 106
 107 static bool memory_region_ioeventfd_equal(MemoryRegionIoeventfd a,
 108                                           MemoryRegionIoeventfd b)
 109 {
 110     return !memory_region_ioeventfd_before(a, b)
 111         && !memory_region_ioeventfd_before(b, a);
 112 }
 113
 114 typedef struct FlatRange FlatRange;
 115 typedef struct FlatView FlatView;
 116
 117 /* Range of memory in the global map.  Addresses are absolute. */
 118 struct FlatRange {
 119     MemoryRegion *mr;
 120     target_phys_addr_t offset_in_region;
 121     AddrRange addr;
 122     uint8_t dirty_log_mask;
 123 };
 124
 125 /* Flattened global view of current active memory hierarchy.  Kept in sorted
 126  * order.
 127  */
 128 struct FlatView {
 129     FlatRange *ranges;
 130     unsigned nr;
 131     unsigned nr_allocated;
 132 };
 133
 134 typedef struct AddressSpace AddressSpace;
 135 typedef struct AddressSpaceOps AddressSpaceOps;
 136
 137 /* A system address space - I/O, memory, etc. */
 138 struct AddressSpace {
 139     const AddressSpaceOps *ops;
 140     MemoryRegion *root;
 141     FlatView current_map;
 142     int ioeventfd_nb;
 143     MemoryRegionIoeventfd *ioeventfds;
 144 };
 145
 146 struct AddressSpaceOps {
 147     void (*range_add)(AddressSpace *as, FlatRange *fr);
 148     void (*range_del)(AddressSpace *as, FlatRange *fr);
 149     void (*log_start)(AddressSpace *as, FlatRange *fr);
 150     void (*log_stop)(AddressSpace *as, FlatRange *fr);
 151     void (*ioeventfd_add)(AddressSpace *as, MemoryRegionIoeventfd *fd);
 152     void (*ioeventfd_del)(AddressSpace *as, MemoryRegionIoeventfd *fd);
 153 };
 154
 155 #define FOR_EACH_FLAT_RANGE(var, view)          \
 156     for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var)
 157
 158 static bool flatrange_equal(FlatRange *a, FlatRange *b)
 159 {
 160     return a->mr == b->mr
 161         && addrrange_equal(a->addr, b->addr)
 162         && a->offset_in_region == b->offset_in_region;
 163 }
 164
 165 static void flatview_init(FlatView *view)
 166 {
 167     view->ranges = NULL;
 168     view->nr = 0;
 169     view->nr_allocated = 0;
 170 }
 171
 172 /* Insert a range into a given position.  Caller is responsible for maintaining
 173  * sorting order.
 174  */
 175 static void flatview_insert(FlatView *view, unsigned pos, FlatRange *range)
 176 {
 177     if (view->nr == view->nr_allocated) {
 178         view->nr_allocated = MAX(2 * view->nr, 10);
 179         view->ranges = qemu_realloc(view->ranges,
 180                                     view->nr_allocated * sizeof(*view->ranges));
 181     }
 182     memmove(view->ranges + pos + 1, view->ranges + pos,
 183             (view->nr - pos) * sizeof(FlatRange));
 184     view->ranges[pos] = *range;
 185     ++view->nr;
 186 }
 187
 188 static void flatview_destroy(FlatView *view)
 189 {
 190     qemu_free(view->ranges);
 191 }
 192
 193 static bool can_merge(FlatRange *r1, FlatRange *r2)
 194 {
 195     return addrrange_end(r1->addr) == r2->addr.start
 196         && r1->mr == r2->mr
 197         && r1->offset_in_region + r1->addr.size == r2->offset_in_region
 198         && r1->dirty_log_mask == r2->dirty_log_mask;
 199 }
 200
 201 /* Attempt to simplify a view by merging ajacent ranges */
 202 static void flatview_simplify(FlatView *view)
 203 {
 204     unsigned i, j;
 205
 206     i = 0;
 207     while (i < view->nr) {
 208         j = i + 1;
 209         while (j < view->nr
 210                && can_merge(&view->ranges[j-1], &view->ranges[j])) {
 211             view->ranges[i].addr.size += view->ranges[j].addr.size;
 212             ++j;
 213         }
 214         ++i;
 215         memmove(&view->ranges[i], &view->ranges[j],
 216                 (view->nr - j) * sizeof(view->ranges[j]));
 217         view->nr -= j - i;
 218     }
 219 }
 220
 221 static void memory_region_prepare_ram_addr(MemoryRegion *mr);
 222
 223 static void as_memory_range_add(AddressSpace *as, FlatRange *fr)
 224 {
 225     ram_addr_t phys_offset, region_offset;
 226
 227     memory_region_prepare_ram_addr(fr->mr);
 228
 229     phys_offset = fr->mr->ram_addr;
 230     region_offset = fr->offset_in_region;
 231     /* cpu_register_physical_memory_log() wants region_offset for
 232      * mmio, but prefers offseting phys_offset for RAM.  Humour it.
 233      */
 234     if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
 235         phys_offset += region_offset;
 236         region_offset = 0;
 237     }
 238
 239     cpu_register_physical_memory_log(fr->addr.start,
 240                                      fr->addr.size,
 241                                      phys_offset,
 242                                      region_offset,
 243                                      fr->dirty_log_mask);
 244 }
 245
 246 static void as_memory_range_del(AddressSpace *as, FlatRange *fr)
 247 {
 248     if (fr->dirty_log_mask) {
 249         cpu_physical_sync_dirty_bitmap(fr->addr.start,
 250                                        fr->addr.start + fr->addr.size);
 251     }
 252     cpu_register_physical_memory(fr->addr.start, fr->addr.size,
 253                                  IO_MEM_UNASSIGNED);
 254 }
 255
 256 static void as_memory_log_start(AddressSpace *as, FlatRange *fr)
 257 {
 258     cpu_physical_log_start(fr->addr.start, fr->addr.size);
 259 }
 260
 261 static void as_memory_log_stop(AddressSpace *as, FlatRange *fr)
 262 {
 263     cpu_physical_log_stop(fr->addr.start, fr->addr.size);
 264 }
 265
 266 static void as_memory_ioeventfd_add(AddressSpace *as, MemoryRegionIoeventfd *fd)
 267 {
 268     int r;
 269
 270     assert(fd->match_data && fd->addr.size == 4);
 271
 272     r = kvm_set_ioeventfd_mmio_long(fd->fd, fd->addr.start, fd->data, true);
 273     if (r < 0) {
 274         abort();
 275     }
 276 }
 277
 278 static void as_memory_ioeventfd_del(AddressSpace *as, MemoryRegionIoeventfd *fd)
 279 {
 280     int r;
 281
 282     r = kvm_set_ioeventfd_mmio_long(fd->fd, fd->addr.start, fd->data, false);
 283     if (r < 0) {
 284         abort();
 285     }
 286 }
 287
 288 static const AddressSpaceOps address_space_ops_memory = {
 289     .range_add = as_memory_range_add,
 290     .range_del = as_memory_range_del,
 291     .log_start = as_memory_log_start,
 292     .log_stop = as_memory_log_stop,
 293     .ioeventfd_add = as_memory_ioeventfd_add,
 294     .ioeventfd_del = as_memory_ioeventfd_del,
 295 };
 296
 297 static AddressSpace address_space_memory = {
 298     .ops = &address_space_ops_memory,
 299 };
 300
 301 static const MemoryRegionPortio *find_portio(MemoryRegion *mr, uint64_t offset,
 302                                              unsigned width, bool write)
 303 {
 304     const MemoryRegionPortio *mrp;
 305
 306     for (mrp = mr->ops->old_portio; mrp->size; ++mrp) {
 307         if (offset >= mrp->offset && offset < mrp->offset + mrp->len
 308             && width == mrp->size
 309             && (write ? (bool)mrp->write : (bool)mrp->read)) {
 310             return mrp;
 311         }
 312     }
 313     return NULL;
 314 }
 315
 316 static void memory_region_iorange_read(IORange *iorange,
 317                                        uint64_t offset,
 318                                        unsigned width,
 319                                        uint64_t *data)
 320 {
 321     MemoryRegion *mr = container_of(iorange, MemoryRegion, iorange);
 322
 323     if (mr->ops->old_portio) {
 324         const MemoryRegionPortio *mrp = find_portio(mr, offset, width, false);
 325
 326         *data = ((uint64_t)1 << (width * 8)) - 1;
 327         if (mrp) {
 328             *data = mrp->read(mr->opaque, offset - mrp->offset);
 329         }
 330         return;
 331     }
 332     *data = mr->ops->read(mr->opaque, offset, width);
 333 }
 334
 335 static void memory_region_iorange_write(IORange *iorange,
 336                                         uint64_t offset,
 337                                         unsigned width,
 338                                         uint64_t data)
 339 {
 340     MemoryRegion *mr = container_of(iorange, MemoryRegion, iorange);
 341
 342     if (mr->ops->old_portio) {
 343         const MemoryRegionPortio *mrp = find_portio(mr, offset, width, true);
 344
 345         if (mrp) {
 346             mrp->write(mr->opaque, offset - mrp->offset, data);
 347         }
 348         return;
 349     }
 350     mr->ops->write(mr->opaque, offset, data, width);
 351 }
 352
 353 static const IORangeOps memory_region_iorange_ops = {
 354     .read = memory_region_iorange_read,
 355     .write = memory_region_iorange_write,
 356 };
 357
 358 static void as_io_range_add(AddressSpace *as, FlatRange *fr)
 359 {
 360     iorange_init(&fr->mr->iorange, &memory_region_iorange_ops,
 361                  fr->addr.start,fr->addr.size);
 362     ioport_register(&fr->mr->iorange);
 363 }
 364
 365 static void as_io_range_del(AddressSpace *as, FlatRange *fr)
 366 {
 367     isa_unassign_ioport(fr->addr.start, fr->addr.size);
 368 }
 369
 370 static void as_io_ioeventfd_add(AddressSpace *as, MemoryRegionIoeventfd *fd)
 371 {
 372     int r;
 373
 374     assert(fd->match_data && fd->addr.size == 2);
 375
 376     r = kvm_set_ioeventfd_pio_word(fd->fd, fd->addr.start, fd->data, true);
 377     if (r < 0) {
 378         abort();
 379     }
 380 }
 381
 382 static void as_io_ioeventfd_del(AddressSpace *as, MemoryRegionIoeventfd *fd)
 383 {
 384     int r;
 385
 386     r = kvm_set_ioeventfd_pio_word(fd->fd, fd->addr.start, fd->data, false);
 387     if (r < 0) {
 388         abort();
 389     }
 390 }
 391
 392 static const AddressSpaceOps address_space_ops_io = {
 393     .range_add = as_io_range_add,
 394     .range_del = as_io_range_del,
 395     .ioeventfd_add = as_io_ioeventfd_add,
 396     .ioeventfd_del = as_io_ioeventfd_del,
 397 };
 398
 399 static AddressSpace address_space_io = {
 400     .ops = &address_space_ops_io,
 401 };
 402
 403 /* Render a memory region into the global view.  Ranges in @view obscure
 404  * ranges in @mr.
 405  */
 406 static void render_memory_region(FlatView *view,
 407                                  MemoryRegion *mr,
 408                                  target_phys_addr_t base,
 409                                  AddrRange clip)
 410 {
 411     MemoryRegion *subregion;
 412     unsigned i;
 413     target_phys_addr_t offset_in_region;
 414     uint64_t remain;
 415     uint64_t now;
 416     FlatRange fr;
 417     AddrRange tmp;
 418
 419     base += mr->addr;
 420
 421     tmp = addrrange_make(base, mr->size);
 422
 423     if (!addrrange_intersects(tmp, clip)) {
 424         return;
 425     }
 426
 427     clip = addrrange_intersection(tmp, clip);
 428
 429     if (mr->alias) {
 430         base -= mr->alias->addr;
 431         base -= mr->alias_offset;
 432         render_memory_region(view, mr->alias, base, clip);
 433         return;
 434     }
 435
 436     /* Render subregions in priority order. */
 437     QTAILQ_FOREACH(subregion, &mr->subregions, subregions_link) {
 438         render_memory_region(view, subregion, base, clip);
 439     }
 440
 441     if (!mr->terminates) {
 442         return;
 443     }
 444
 445     offset_in_region = clip.start - base;
 446     base = clip.start;
 447     remain = clip.size;
 448
 449     /* Render the region itself into any gaps left by the current view. */
 450     for (i = 0; i < view->nr && remain; ++i) {
 451         if (base >= addrrange_end(view->ranges[i].addr)) {
 452             continue;
 453         }
 454         if (base < view->ranges[i].addr.start) {
 455             now = MIN(remain, view->ranges[i].addr.start - base);
 456             fr.mr = mr;
 457             fr.offset_in_region = offset_in_region;
 458             fr.addr = addrrange_make(base, now);
 459             fr.dirty_log_mask = mr->dirty_log_mask;
 460             flatview_insert(view, i, &fr);
 461             ++i;
 462             base += now;
 463             offset_in_region += now;
 464             remain -= now;
 465         }
 466         if (base == view->ranges[i].addr.start) {
 467             now = MIN(remain, view->ranges[i].addr.size);
 468             base += now;
 469             offset_in_region += now;
 470             remain -= now;
 471         }
 472     }
 473     if (remain) {
 474         fr.mr = mr;
 475         fr.offset_in_region = offset_in_region;
 476         fr.addr = addrrange_make(base, remain);
 477         fr.dirty_log_mask = mr->dirty_log_mask;
 478         flatview_insert(view, i, &fr);
 479     }
 480 }
 481
 482 /* Render a memory topology into a list of disjoint absolute ranges. */
 483 static FlatView generate_memory_topology(MemoryRegion *mr)
 484 {
 485     FlatView view;
 486
 487     flatview_init(&view);
 488
 489     render_memory_region(&view, mr, 0, addrrange_make(0, UINT64_MAX));
 490     flatview_simplify(&view);
 491
 492     return view;
 493 }
 494
 495 static void address_space_add_del_ioeventfds(AddressSpace *as,
 496                                              MemoryRegionIoeventfd *fds_new,
 497                                              unsigned fds_new_nb,
 498                                              MemoryRegionIoeventfd *fds_old,
 499                                              unsigned fds_old_nb)
 500 {
 501     unsigned iold, inew;
 502
 503     /* Generate a symmetric difference of the old and new fd sets, adding
 504      * and deleting as necessary.
 505      */
 506
 507     iold = inew = 0;
 508     while (iold < fds_old_nb || inew < fds_new_nb) {
 509         if (iold < fds_old_nb
 510             && (inew == fds_new_nb
 511                 || memory_region_ioeventfd_before(fds_old[iold],
 512                                                   fds_new[inew]))) {
 513             as->ops->ioeventfd_del(as, &fds_old[iold]);
 514             ++iold;
 515         } else if (inew < fds_new_nb
 516                    && (iold == fds_old_nb
 517                        || memory_region_ioeventfd_before(fds_new[inew],
 518                                                          fds_old[iold]))) {
 519             as->ops->ioeventfd_add(as, &fds_new[inew]);
 520             ++inew;
 521         } else {
 522             ++iold;
 523             ++inew;
 524         }
 525     }
 526 }
 527
 528 static void address_space_update_ioeventfds(AddressSpace *as)
 529 {
 530     FlatRange *fr;
 531     unsigned ioeventfd_nb = 0;
 532     MemoryRegionIoeventfd *ioeventfds = NULL;
 533     AddrRange tmp;
 534     unsigned i;
 535
 536     FOR_EACH_FLAT_RANGE(fr, &as->current_map) {
 537         for (i = 0; i < fr->mr->ioeventfd_nb; ++i) {
 538             tmp = addrrange_shift(fr->mr->ioeventfds[i].addr,
 539                                   fr->addr.start - fr->offset_in_region);
 540             if (addrrange_intersects(fr->addr, tmp)) {
 541                 ++ioeventfd_nb;
 542                 ioeventfds = qemu_realloc(ioeventfds,
 543                                           ioeventfd_nb * sizeof(*ioeventfds));
 544                 ioeventfds[ioeventfd_nb-1] = fr->mr->ioeventfds[i];
 545                 ioeventfds[ioeventfd_nb-1].addr = tmp;
 546             }
 547         }
 548     }
 549
 550     address_space_add_del_ioeventfds(as, ioeventfds, ioeventfd_nb,
 551                                      as->ioeventfds, as->ioeventfd_nb);
 552
 553     qemu_free(as->ioeventfds);
 554     as->ioeventfds = ioeventfds;
 555     as->ioeventfd_nb = ioeventfd_nb;
 556 }
 557
 558 static void address_space_update_topology_pass(AddressSpace *as,
 559                                                FlatView old_view,
 560                                                FlatView new_view,
 561                                                bool adding)
 562 {
 563     unsigned iold, inew;
 564     FlatRange *frold, *frnew;
 565
 566     /* Generate a symmetric difference of the old and new memory maps.
 567      * Kill ranges in the old map, and instantiate ranges in the new map.
 568      */
 569     iold = inew = 0;
 570     while (iold < old_view.nr || inew < new_view.nr) {
 571         if (iold < old_view.nr) {
 572             frold = &old_view.ranges[iold];
 573         } else {
 574             frold = NULL;
 575         }
 576         if (inew < new_view.nr) {
 577             frnew = &new_view.ranges[inew];
 578         } else {
 579             frnew = NULL;
 580         }
 581
 582         if (frold
 583             && (!frnew
 584                 || frold->addr.start < frnew->addr.start
 585                 || (frold->addr.start == frnew->addr.start
 586                     && !flatrange_equal(frold, frnew)))) {
 587             /* In old, but (not in new, or in new but attributes changed). */
 588
 589             if (!adding) {
 590                 as->ops->range_del(as, frold);
 591             }
 592
 593             ++iold;
 594         } else if (frold && frnew && flatrange_equal(frold, frnew)) {
 595             /* In both (logging may have changed) */
 596
 597             if (adding) {
 598                 if (frold->dirty_log_mask && !frnew->dirty_log_mask) {
 599                     as->ops->log_stop(as, frnew);
 600                 } else if (frnew->dirty_log_mask && !frold->dirty_log_mask) {
 601                     as->ops->log_start(as, frnew);
 602                 }
 603             }
 604
 605             ++iold;
 606             ++inew;
 607         } else {
 608             /* In new */
 609
 610             if (adding) {
 611                 as->ops->range_add(as, frnew);
 612             }
 613
 614             ++inew;
 615         }
 616     }
 617 }
 618
 619
 620 static void address_space_update_topology(AddressSpace *as)
 621 {
 622     FlatView old_view = as->current_map;
 623     FlatView new_view = generate_memory_topology(as->root);
 624
 625     address_space_update_topology_pass(as, old_view, new_view, false);
 626     address_space_update_topology_pass(as, old_view, new_view, true);
 627
 628     as->current_map = new_view;
 629     flatview_destroy(&old_view);
 630     address_space_update_ioeventfds(as);
 631 }
 632
 633 static void memory_region_update_topology(void)
 634 {
 635     if (memory_region_transaction_depth) {
 636         return;
 637     }
 638
 639     if (address_space_memory.root) {
 640         address_space_update_topology(&address_space_memory);
 641     }
 642     if (address_space_io.root) {
 643         address_space_update_topology(&address_space_io);
 644     }
 645 }
 646
 647 void memory_region_transaction_begin(void)
 648 {
 649     ++memory_region_transaction_depth;
 650 }
 651
 652 void memory_region_transaction_commit(void)
 653 {
 654     assert(memory_region_transaction_depth);
 655     --memory_region_transaction_depth;
 656     memory_region_update_topology();
 657 }
 658
 659 void memory_region_init(MemoryRegion *mr,
 660                         const char *name,
 661                         uint64_t size)
 662 {
 663     mr->ops = NULL;
 664     mr->parent = NULL;
 665     mr->size = size;
 666     mr->addr = 0;
 667     mr->offset = 0;
 668     mr->terminates = false;
 669     mr->priority = 0;
 670     mr->may_overlap = false;
 671     mr->alias = NULL;
 672     QTAILQ_INIT(&mr->subregions);
 673     memset(&mr->subregions_link, 0, sizeof mr->subregions_link);
 674     QTAILQ_INIT(&mr->coalesced);
 675     mr->name = qemu_strdup(name);
 676     mr->dirty_log_mask = 0;
 677     mr->ioeventfd_nb = 0;
 678     mr->ioeventfds = NULL;
 679 }
 680
 681 static bool memory_region_access_valid(MemoryRegion *mr,
 682                                        target_phys_addr_t addr,
 683                                        unsigned size)
 684 {
 685     if (!mr->ops->valid.unaligned && (addr & (size - 1))) {
 686         return false;
 687     }
 688
 689     /* Treat zero as compatibility all valid */
 690     if (!mr->ops->valid.max_access_size) {
 691         return true;
 692     }
 693
 694     if (size > mr->ops->valid.max_access_size
 695         || size < mr->ops->valid.min_access_size) {
 696         return false;
 697     }
 698     return true;
 699 }
 700
 701 static uint32_t memory_region_read_thunk_n(void *_mr,
 702                                            target_phys_addr_t addr,
 703                                            unsigned size)
 704 {
 705     MemoryRegion *mr = _mr;
 706     unsigned access_size, access_size_min, access_size_max;
 707     uint64_t access_mask;
 708     uint32_t data = 0, tmp;
 709     unsigned i;
 710
 711     if (!memory_region_access_valid(mr, addr, size)) {
 712         return -1U; /* FIXME: better signalling */
 713     }
 714
 715     if (!mr->ops->read) {
 716         return mr->ops->old_mmio.read[bitops_ffsl(size)](mr->opaque, addr);
 717     }
 718
 719     /* FIXME: support unaligned access */
 720
 721     access_size_min = mr->ops->impl.min_access_size;
 722     if (!access_size_min) {
 723         access_size_min = 1;
 724     }
 725     access_size_max = mr->ops->impl.max_access_size;
 726     if (!access_size_max) {
 727         access_size_max = 4;
 728     }
 729     access_size = MAX(MIN(size, access_size_max), access_size_min);
 730     access_mask = -1ULL >> (64 - access_size * 8);
 731     addr += mr->offset;
 732     for (i = 0; i < size; i += access_size) {
 733         /* FIXME: big-endian support */
 734         tmp = mr->ops->read(mr->opaque, addr + i, access_size);
 735         data |= (tmp & access_mask) << (i * 8);
 736     }
 737
 738     return data;
 739 }
 740
 741 static void memory_region_write_thunk_n(void *_mr,
 742                                         target_phys_addr_t addr,
 743                                         unsigned size,
 744                                         uint64_t data)
 745 {
 746     MemoryRegion *mr = _mr;
 747     unsigned access_size, access_size_min, access_size_max;
 748     uint64_t access_mask;
 749     unsigned i;
 750
 751     if (!memory_region_access_valid(mr, addr, size)) {
 752         return; /* FIXME: better signalling */
 753     }
 754
 755     if (!mr->ops->write) {
 756         mr->ops->old_mmio.write[bitops_ffsl(size)](mr->opaque, addr, data);
 757         return;
 758     }
 759
 760     /* FIXME: support unaligned access */
 761
 762     access_size_min = mr->ops->impl.min_access_size;
 763     if (!access_size_min) {
 764         access_size_min = 1;
 765     }
 766     access_size_max = mr->ops->impl.max_access_size;
 767     if (!access_size_max) {
 768         access_size_max = 4;
 769     }
 770     access_size = MAX(MIN(size, access_size_max), access_size_min);
 771     access_mask = -1ULL >> (64 - access_size * 8);
 772     addr += mr->offset;
 773     for (i = 0; i < size; i += access_size) {
 774         /* FIXME: big-endian support */
 775         mr->ops->write(mr->opaque, addr + i, (data >> (i * 8)) & access_mask,
 776                        access_size);
 777     }
 778 }
 779
 780 static uint32_t memory_region_read_thunk_b(void *mr, target_phys_addr_t addr)
 781 {
 782     return memory_region_read_thunk_n(mr, addr, 1);
 783 }
 784
 785 static uint32_t memory_region_read_thunk_w(void *mr, target_phys_addr_t addr)
 786 {
 787     return memory_region_read_thunk_n(mr, addr, 2);
 788 }
 789
 790 static uint32_t memory_region_read_thunk_l(void *mr, target_phys_addr_t addr)
 791 {
 792     return memory_region_read_thunk_n(mr, addr, 4);
 793 }
 794
 795 static void memory_region_write_thunk_b(void *mr, target_phys_addr_t addr,
 796                                         uint32_t data)
 797 {
 798     memory_region_write_thunk_n(mr, addr, 1, data);
 799 }
 800
 801 static void memory_region_write_thunk_w(void *mr, target_phys_addr_t addr,
 802                                         uint32_t data)
 803 {
 804     memory_region_write_thunk_n(mr, addr, 2, data);
 805 }
 806
 807 static void memory_region_write_thunk_l(void *mr, target_phys_addr_t addr,
 808                                         uint32_t data)
 809 {
 810     memory_region_write_thunk_n(mr, addr, 4, data);
 811 }
 812
 813 static CPUReadMemoryFunc * const memory_region_read_thunk[] = {
 814     memory_region_read_thunk_b,
 815     memory_region_read_thunk_w,
 816     memory_region_read_thunk_l,
 817 };
 818
 819 static CPUWriteMemoryFunc * const memory_region_write_thunk[] = {
 820     memory_region_write_thunk_b,
 821     memory_region_write_thunk_w,
 822     memory_region_write_thunk_l,
 823 };
 824
 825 static void memory_region_prepare_ram_addr(MemoryRegion *mr)
 826 {
 827     if (mr->backend_registered) {
 828         return;
 829     }
 830
 831     mr->ram_addr = cpu_register_io_memory(memory_region_read_thunk,
 832                                           memory_region_write_thunk,
 833                                           mr,
 834                                           mr->ops->endianness);
 835     mr->backend_registered = true;
 836 }
 837
 838 void memory_region_init_io(MemoryRegion *mr,
 839                            const MemoryRegionOps *ops,
 840                            void *opaque,
 841                            const char *name,
 842                            uint64_t size)
 843 {
 844     memory_region_init(mr, name, size);
 845     mr->ops = ops;
 846     mr->opaque = opaque;
 847     mr->terminates = true;
 848     mr->backend_registered = false;
 849 }
 850
 851 void memory_region_init_ram(MemoryRegion *mr,
 852                             DeviceState *dev,
 853                             const char *name,
 854                             uint64_t size)
 855 {
 856     memory_region_init(mr, name, size);
 857     mr->terminates = true;
 858     mr->ram_addr = qemu_ram_alloc(dev, name, size);
 859     mr->backend_registered = true;
 860 }
 861
 862 void memory_region_init_ram_ptr(MemoryRegion *mr,
 863                                 DeviceState *dev,
 864                                 const char *name,
 865                                 uint64_t size,
 866                                 void *ptr)
 867 {
 868     memory_region_init(mr, name, size);
 869     mr->terminates = true;
 870     mr->ram_addr = qemu_ram_alloc_from_ptr(dev, name, size, ptr);
 871     mr->backend_registered = true;
 872 }
 873
 874 void memory_region_init_alias(MemoryRegion *mr,
 875                               const char *name,
 876                               MemoryRegion *orig,
 877                               target_phys_addr_t offset,
 878                               uint64_t size)
 879 {
 880     memory_region_init(mr, name, size);
 881     mr->alias = orig;
 882     mr->alias_offset = offset;
 883 }
 884
 885 void memory_region_destroy(MemoryRegion *mr)
 886 {
 887     assert(QTAILQ_EMPTY(&mr->subregions));
 888     memory_region_clear_coalescing(mr);
 889     qemu_free((char *)mr->name);
 890     qemu_free(mr->ioeventfds);
 891 }
 892
 893 uint64_t memory_region_size(MemoryRegion *mr)
 894 {
 895     return mr->size;
 896 }
 897
 898 void memory_region_set_offset(MemoryRegion *mr, target_phys_addr_t offset)
 899 {
 900     mr->offset = offset;
 901 }
 902
 903 void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client)
 904 {
 905     uint8_t mask = 1 << client;
 906
 907     mr->dirty_log_mask = (mr->dirty_log_mask & ~mask) | (log * mask);
 908     memory_region_update_topology();
 909 }
 910
 911 bool memory_region_get_dirty(MemoryRegion *mr, target_phys_addr_t addr,
 912                              unsigned client)
 913 {
 914     assert(mr->terminates);
 915     return cpu_physical_memory_get_dirty(mr->ram_addr + addr, 1 << client);
 916 }
 917
 918 void memory_region_set_dirty(MemoryRegion *mr, target_phys_addr_t addr)
 919 {
 920     assert(mr->terminates);
 921     return cpu_physical_memory_set_dirty(mr->ram_addr + addr);
 922 }
 923
 924 void memory_region_sync_dirty_bitmap(MemoryRegion *mr)
 925 {
 926     FlatRange *fr;
 927
 928     FOR_EACH_FLAT_RANGE(fr, &address_space_memory.current_map) {
 929         if (fr->mr == mr) {
 930             cpu_physical_sync_dirty_bitmap(fr->addr.start,
 931                                            fr->addr.start + fr->addr.size);
 932         }
 933     }
 934 }
 935
 936 void memory_region_set_readonly(MemoryRegion *mr, bool readonly)
 937 {
 938     /* FIXME */
 939 }
 940
 941 void memory_region_reset_dirty(MemoryRegion *mr, target_phys_addr_t addr,
 942                                target_phys_addr_t size, unsigned client)
 943 {
 944     assert(mr->terminates);
 945     cpu_physical_memory_reset_dirty(mr->ram_addr + addr,
 946                                     mr->ram_addr + addr + size,
 947                                     1 << client);
 948 }
 949
 950 void *memory_region_get_ram_ptr(MemoryRegion *mr)
 951 {
 952     if (mr->alias) {
 953         return memory_region_get_ram_ptr(mr->alias) + mr->alias_offset;
 954     }
 955
 956     assert(mr->terminates);
 957
 958     return qemu_get_ram_ptr(mr->ram_addr);
 959 }
 960
 961 static void memory_region_update_coalesced_range(MemoryRegion *mr)
 962 {
 963     FlatRange *fr;
 964     CoalescedMemoryRange *cmr;
 965     AddrRange tmp;
 966
 967     FOR_EACH_FLAT_RANGE(fr, &address_space_memory.current_map) {
 968         if (fr->mr == mr) {
 969             qemu_unregister_coalesced_mmio(fr->addr.start, fr->addr.size);
 970             QTAILQ_FOREACH(cmr, &mr->coalesced, link) {
 971                 tmp = addrrange_shift(cmr->addr,
 972                                       fr->addr.start - fr->offset_in_region);
 973                 if (!addrrange_intersects(tmp, fr->addr)) {
 974                     continue;
 975                 }
 976                 tmp = addrrange_intersection(tmp, fr->addr);
 977                 qemu_register_coalesced_mmio(tmp.start, tmp.size);
 978             }
 979         }
 980     }
 981 }
 982
 983 void memory_region_set_coalescing(MemoryRegion *mr)
 984 {
 985     memory_region_clear_coalescing(mr);
 986     memory_region_add_coalescing(mr, 0, mr->size);
 987 }
 988
 989 void memory_region_add_coalescing(MemoryRegion *mr,
 990                                   target_phys_addr_t offset,
 991                                   uint64_t size)
 992 {
 993     CoalescedMemoryRange *cmr = qemu_malloc(sizeof(*cmr));
 994
 995     cmr->addr = addrrange_make(offset, size);
 996     QTAILQ_INSERT_TAIL(&mr->coalesced, cmr, link);
 997     memory_region_update_coalesced_range(mr);
 998 }
 999
1000 void memory_region_clear_coalescing(MemoryRegion *mr)
1001 {
1002     CoalescedMemoryRange *cmr;
1003
1004     while (!QTAILQ_EMPTY(&mr->coalesced)) {
1005         cmr = QTAILQ_FIRST(&mr->coalesced);
1006         QTAILQ_REMOVE(&mr->coalesced, cmr, link);
1007         qemu_free(cmr);
1008     }
1009     memory_region_update_coalesced_range(mr);
1010 }
1011
1012 void memory_region_add_eventfd(MemoryRegion *mr,
1013                                target_phys_addr_t addr,
1014                                unsigned size,
1015                                bool match_data,
1016                                uint64_t data,
1017                                int fd)
1018 {
1019     MemoryRegionIoeventfd mrfd = {
1020         .addr.start = addr,
1021         .addr.size = size,
1022         .match_data = match_data,
1023         .data = data,
1024         .fd = fd,
1025     };
1026     unsigned i;
1027
1028     for (i = 0; i < mr->ioeventfd_nb; ++i) {
1029         if (memory_region_ioeventfd_before(mrfd, mr->ioeventfds[i])) {
1030             break;
1031         }
1032     }
1033     ++mr->ioeventfd_nb;
1034     mr->ioeventfds = qemu_realloc(mr->ioeventfds,
1035                                   sizeof(*mr->ioeventfds) * mr->ioeventfd_nb);
1036     memmove(&mr->ioeventfds[i+1], &mr->ioeventfds[i],
1037             sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb-1 - i));
1038     mr->ioeventfds[i] = mrfd;
1039     memory_region_update_topology();
1040 }
1041
1042 void memory_region_del_eventfd(MemoryRegion *mr,
1043                                target_phys_addr_t addr,
1044                                unsigned size,
1045                                bool match_data,
1046                                uint64_t data,
1047                                int fd)
1048 {
1049     MemoryRegionIoeventfd mrfd = {
1050         .addr.start = addr,
1051         .addr.size = size,
1052         .match_data = match_data,
1053         .data = data,
1054         .fd = fd,
1055     };
1056     unsigned i;
1057
1058     for (i = 0; i < mr->ioeventfd_nb; ++i) {
1059         if (memory_region_ioeventfd_equal(mrfd, mr->ioeventfds[i])) {
1060             break;
1061         }
1062     }
1063     assert(i != mr->ioeventfd_nb);
1064     memmove(&mr->ioeventfds[i], &mr->ioeventfds[i+1],
1065             sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb - (i+1)));
1066     --mr->ioeventfd_nb;
1067     mr->ioeventfds = qemu_realloc(mr->ioeventfds,
1068                                   sizeof(*mr->ioeventfds)*mr->ioeventfd_nb + 1);
1069     memory_region_update_topology();
1070 }
1071
1072 static void memory_region_add_subregion_common(MemoryRegion *mr,
1073                                                target_phys_addr_t offset,
1074                                                MemoryRegion *subregion)
1075 {
1076     MemoryRegion *other;
1077
1078     assert(!subregion->parent);
1079     subregion->parent = mr;
1080     subregion->addr = offset;
1081     QTAILQ_FOREACH(other, &mr->subregions, subregions_link) {
1082         if (subregion->may_overlap || other->may_overlap) {
1083             continue;
1084         }
1085         if (offset >= other->offset + other->size
1086             || offset + subregion->size <= other->offset) {
1087             continue;
1088         }
1089         printf("warning: subregion collision %llx/%llx vs %llx/%llx\n",
1090                (unsigned long long)offset,
1091                (unsigned long long)subregion->size,
1092                (unsigned long long)other->offset,
1093                (unsigned long long)other->size);
1094     }
1095     QTAILQ_FOREACH(other, &mr->subregions, subregions_link) {
1096         if (subregion->priority >= other->priority) {
1097             QTAILQ_INSERT_BEFORE(other, subregion, subregions_link);
1098             goto done;
1099         }
1100     }
1101     QTAILQ_INSERT_TAIL(&mr->subregions, subregion, subregions_link);
1102 done:
1103     memory_region_update_topology();
1104 }
1105
1106
1107 void memory_region_add_subregion(MemoryRegion *mr,
1108                                  target_phys_addr_t offset,
1109                                  MemoryRegion *subregion)
1110 {
1111     subregion->may_overlap = false;
1112     subregion->priority = 0;
1113     memory_region_add_subregion_common(mr, offset, subregion);
1114 }
1115
1116 void memory_region_add_subregion_overlap(MemoryRegion *mr,
1117                                          target_phys_addr_t offset,
1118                                          MemoryRegion *subregion,
1119                                          unsigned priority)
1120 {
1121     subregion->may_overlap = true;
1122     subregion->priority = priority;
1123     memory_region_add_subregion_common(mr, offset, subregion);
1124 }
1125
1126 void memory_region_del_subregion(MemoryRegion *mr,
1127                                  MemoryRegion *subregion)
1128 {
1129     assert(subregion->parent == mr);
1130     subregion->parent = NULL;
1131     QTAILQ_REMOVE(&mr->subregions, subregion, subregions_link);
1132     memory_region_update_topology();
1133 }
1134
1135 void set_system_memory_map(MemoryRegion *mr)
1136 {
1137     address_space_memory.root = mr;
1138     memory_region_update_topology();
1139 }
1140
1141 void set_system_io_map(MemoryRegion *mr)
1142 {
1143     address_space_io.root = mr;
1144     memory_region_update_topology();
1145 }