exec.c

   1 /*
   2  *  Virtual page mapping
   3  *
   4  *  Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19 #include "config.h"
  20 #ifdef _WIN32
  21 #include <windows.h>
  22 #else
  23 #include <sys/types.h>
  24 #include <sys/mman.h>
  25 #endif
  26
  27 #include "qemu-common.h"
  28 #include "cpu.h"
  29 #include "tcg.h"
  30 #include "hw/hw.h"
  31 #include "hw/qdev.h"
  32 #include "qemu/osdep.h"
  33 #include "sysemu/kvm.h"
  34 #include "sysemu/sysemu.h"
  35 #include "hw/xen/xen.h"
  36 #include "qemu/timer.h"
  37 #include "qemu/config-file.h"
  38 #include "exec/memory.h"
  39 #include "sysemu/dma.h"
  40 #include "exec/address-spaces.h"
  41 #if defined(CONFIG_USER_ONLY)
  42 #include <qemu.h>
  43 #else /* !CONFIG_USER_ONLY */
  44 #include "sysemu/xen-mapcache.h"
  45 #include "trace.h"
  46 #endif
  47 #include "exec/cpu-all.h"
  48
  49 #include "exec/cputlb.h"
  50 #include "translate-all.h"
  51
  52 #include "exec/memory-internal.h"
  53
  54 //#define DEBUG_SUBPAGE
  55
  56 #if !defined(CONFIG_USER_ONLY)
  57 static int in_migration;
  58
  59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
  60
  61 static MemoryRegion *system_memory;
  62 static MemoryRegion *system_io;
  63
  64 AddressSpace address_space_io;
  65 AddressSpace address_space_memory;
  66
  67 MemoryRegion io_mem_rom, io_mem_notdirty;
  68 static MemoryRegion io_mem_unassigned;
  69
  70 #endif
  71
  72 CPUState *first_cpu;
  73 /* current CPU in the current thread. It is only valid inside
  74    cpu_exec() */
  75 DEFINE_TLS(CPUState *, current_cpu);
  76 /* 0 = Do not count executed instructions.
  77    1 = Precise instruction counting.
  78    2 = Adaptive rate instruction counting.  */
  79 int use_icount;
  80
  81 #if !defined(CONFIG_USER_ONLY)
  82
  83 typedef struct PhysPageEntry PhysPageEntry;
  84
  85 struct PhysPageEntry {
  86     uint16_t is_leaf : 1;
  87      /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
  88     uint16_t ptr : 15;
  89 };
  90
  91 typedef PhysPageEntry Node[L2_SIZE];
  92
  93 struct AddressSpaceDispatch {
  94     /* This is a multi-level map on the physical address space.
  95      * The bottom level has pointers to MemoryRegionSections.
  96      */
  97     PhysPageEntry phys_map;
  98     Node *nodes;
  99     MemoryRegionSection *sections;
 100     AddressSpace *as;
 101 };
 102
 103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
 104 typedef struct subpage_t {
 105     MemoryRegion iomem;
 106     AddressSpace *as;
 107     hwaddr base;
 108     uint16_t sub_section[TARGET_PAGE_SIZE];
 109 } subpage_t;
 110
 111 #define PHYS_SECTION_UNASSIGNED 0
 112 #define PHYS_SECTION_NOTDIRTY 1
 113 #define PHYS_SECTION_ROM 2
 114 #define PHYS_SECTION_WATCH 3
 115
 116 typedef struct PhysPageMap {
 117     unsigned sections_nb;
 118     unsigned sections_nb_alloc;
 119     unsigned nodes_nb;
 120     unsigned nodes_nb_alloc;
 121     Node *nodes;
 122     MemoryRegionSection *sections;
 123 } PhysPageMap;
 124
 125 static PhysPageMap *prev_map;
 126 static PhysPageMap next_map;
 127
 128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
 129
 130 static void io_mem_init(void);
 131 static void memory_map_init(void);
 132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
 133
 134 static MemoryRegion io_mem_watch;
 135 #endif
 136
 137 #if !defined(CONFIG_USER_ONLY)
 138
 139 static void phys_map_node_reserve(unsigned nodes)
 140 {
 141     if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
 142         next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
 143                                             16);
 144         next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
 145                                       next_map.nodes_nb + nodes);
 146         next_map.nodes = g_renew(Node, next_map.nodes,
 147                                  next_map.nodes_nb_alloc);
 148     }
 149 }
 150
 151 static uint16_t phys_map_node_alloc(void)
 152 {
 153     unsigned i;
 154     uint16_t ret;
 155
 156     ret = next_map.nodes_nb++;
 157     assert(ret != PHYS_MAP_NODE_NIL);
 158     assert(ret != next_map.nodes_nb_alloc);
 159     for (i = 0; i < L2_SIZE; ++i) {
 160         next_map.nodes[ret][i].is_leaf = 0;
 161         next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
 162     }
 163     return ret;
 164 }
 165
 166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
 167                                 hwaddr *nb, uint16_t leaf,
 168                                 int level)
 169 {
 170     PhysPageEntry *p;
 171     int i;
 172     hwaddr step = (hwaddr)1 << (level * L2_BITS);
 173
 174     if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
 175         lp->ptr = phys_map_node_alloc();
 176         p = next_map.nodes[lp->ptr];
 177         if (level == 0) {
 178             for (i = 0; i < L2_SIZE; i++) {
 179                 p[i].is_leaf = 1;
 180                 p[i].ptr = PHYS_SECTION_UNASSIGNED;
 181             }
 182         }
 183     } else {
 184         p = next_map.nodes[lp->ptr];
 185     }
 186     lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
 187
 188     while (*nb && lp < &p[L2_SIZE]) {
 189         if ((*index & (step - 1)) == 0 && *nb >= step) {
 190             lp->is_leaf = true;
 191             lp->ptr = leaf;
 192             *index += step;
 193             *nb -= step;
 194         } else {
 195             phys_page_set_level(lp, index, nb, leaf, level - 1);
 196         }
 197         ++lp;
 198     }
 199 }
 200
 201 static void phys_page_set(AddressSpaceDispatch *d,
 202                           hwaddr index, hwaddr nb,
 203                           uint16_t leaf)
 204 {
 205     /* Wildly overreserve - it doesn't matter much. */
 206     phys_map_node_reserve(3 * P_L2_LEVELS);
 207
 208     phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
 209 }
 210
 211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
 212                                            Node *nodes, MemoryRegionSection *sections)
 213 {
 214     PhysPageEntry *p;
 215     int i;
 216
 217     for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
 218         if (lp.ptr == PHYS_MAP_NODE_NIL) {
 219             return &sections[PHYS_SECTION_UNASSIGNED];
 220         }
 221         p = nodes[lp.ptr];
 222         lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
 223     }
 224     return &sections[lp.ptr];
 225 }
 226
 227 bool memory_region_is_unassigned(MemoryRegion *mr)
 228 {
 229     return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
 230         && mr != &io_mem_watch;
 231 }
 232
 233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
 234                                                         hwaddr addr,
 235                                                         bool resolve_subpage)
 236 {
 237     MemoryRegionSection *section;
 238     subpage_t *subpage;
 239
 240     section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
 241                              d->nodes, d->sections);
 242     if (resolve_subpage && section->mr->subpage) {
 243         subpage = container_of(section->mr, subpage_t, iomem);
 244         section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
 245     }
 246     return section;
 247 }
 248
 249 static MemoryRegionSection *
 250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
 251                                  hwaddr *plen, bool resolve_subpage)
 252 {
 253     MemoryRegionSection *section;
 254     Int128 diff;
 255
 256     section = address_space_lookup_region(d, addr, resolve_subpage);
 257     /* Compute offset within MemoryRegionSection */
 258     addr -= section->offset_within_address_space;
 259
 260     /* Compute offset within MemoryRegion */
 261     *xlat = addr + section->offset_within_region;
 262
 263     diff = int128_sub(section->mr->size, int128_make64(addr));
 264     *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
 265     return section;
 266 }
 267
 268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
 269                                       hwaddr *xlat, hwaddr *plen,
 270                                       bool is_write)
 271 {
 272     IOMMUTLBEntry iotlb;
 273     MemoryRegionSection *section;
 274     MemoryRegion *mr;
 275     hwaddr len = *plen;
 276
 277     for (;;) {
 278         section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
 279         mr = section->mr;
 280
 281         if (!mr->iommu_ops) {
 282             break;
 283         }
 284
 285         iotlb = mr->iommu_ops->translate(mr, addr);
 286         addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
 287                 | (addr & iotlb.addr_mask));
 288         len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
 289         if (!(iotlb.perm & (1 << is_write))) {
 290             mr = &io_mem_unassigned;
 291             break;
 292         }
 293
 294         as = iotlb.target_as;
 295     }
 296
 297     *plen = len;
 298     *xlat = addr;
 299     return mr;
 300 }
 301
 302 MemoryRegionSection *
 303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
 304                                   hwaddr *plen)
 305 {
 306     MemoryRegionSection *section;
 307     section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
 308
 309     assert(!section->mr->iommu_ops);
 310     return section;
 311 }
 312 #endif
 313
 314 void cpu_exec_init_all(void)
 315 {
 316 #if !defined(CONFIG_USER_ONLY)
 317     qemu_mutex_init(&ram_list.mutex);
 318     memory_map_init();
 319     io_mem_init();
 320 #endif
 321 }
 322
 323 #if !defined(CONFIG_USER_ONLY)
 324
 325 static int cpu_common_post_load(void *opaque, int version_id)
 326 {
 327     CPUState *cpu = opaque;
 328
 329     /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
 330        version_id is increased. */
 331     cpu->interrupt_request &= ~0x01;
 332     tlb_flush(cpu->env_ptr, 1);
 333
 334     return 0;
 335 }
 336
 337 const VMStateDescription vmstate_cpu_common = {
 338     .name = "cpu_common",
 339     .version_id = 1,
 340     .minimum_version_id = 1,
 341     .minimum_version_id_old = 1,
 342     .post_load = cpu_common_post_load,
 343     .fields      = (VMStateField []) {
 344         VMSTATE_UINT32(halted, CPUState),
 345         VMSTATE_UINT32(interrupt_request, CPUState),
 346         VMSTATE_END_OF_LIST()
 347     }
 348 };
 349
 350 #endif
 351
 352 CPUState *qemu_get_cpu(int index)
 353 {
 354     CPUState *cpu = first_cpu;
 355
 356     while (cpu) {
 357         if (cpu->cpu_index == index) {
 358             break;
 359         }
 360         cpu = cpu->next_cpu;
 361     }
 362
 363     return cpu;
 364 }
 365
 366 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
 367 {
 368     CPUState *cpu;
 369
 370     cpu = first_cpu;
 371     while (cpu) {
 372         func(cpu, data);
 373         cpu = cpu->next_cpu;
 374     }
 375 }
 376
 377 void cpu_exec_init(CPUArchState *env)
 378 {
 379     CPUState *cpu = ENV_GET_CPU(env);
 380     CPUClass *cc = CPU_GET_CLASS(cpu);
 381     CPUState **pcpu;
 382     int cpu_index;
 383
 384 #if defined(CONFIG_USER_ONLY)
 385     cpu_list_lock();
 386 #endif
 387     cpu->next_cpu = NULL;
 388     pcpu = &first_cpu;
 389     cpu_index = 0;
 390     while (*pcpu != NULL) {
 391         pcpu = &(*pcpu)->next_cpu;
 392         cpu_index++;
 393     }
 394     cpu->cpu_index = cpu_index;
 395     cpu->numa_node = 0;
 396     QTAILQ_INIT(&env->breakpoints);
 397     QTAILQ_INIT(&env->watchpoints);
 398 #ifndef CONFIG_USER_ONLY
 399     cpu->thread_id = qemu_get_thread_id();
 400 #endif
 401     *pcpu = cpu;
 402 #if defined(CONFIG_USER_ONLY)
 403     cpu_list_unlock();
 404 #endif
 405     vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
 406 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
 407     register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
 408                     cpu_save, cpu_load, env);
 409     assert(cc->vmsd == NULL);
 410 #endif
 411     if (cc->vmsd != NULL) {
 412         vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
 413     }
 414 }
 415
 416 #if defined(TARGET_HAS_ICE)
 417 #if defined(CONFIG_USER_ONLY)
 418 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
 419 {
 420     tb_invalidate_phys_page_range(pc, pc + 1, 0);
 421 }
 422 #else
 423 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
 424 {
 425     tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
 426             (pc & ~TARGET_PAGE_MASK));
 427 }
 428 #endif
 429 #endif /* TARGET_HAS_ICE */
 430
 431 #if defined(CONFIG_USER_ONLY)
 432 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 433
 434 {
 435 }
 436
 437 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 438                           int flags, CPUWatchpoint **watchpoint)
 439 {
 440     return -ENOSYS;
 441 }
 442 #else
 443 /* Add a watchpoint.  */
 444 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 445                           int flags, CPUWatchpoint **watchpoint)
 446 {
 447     target_ulong len_mask = ~(len - 1);
 448     CPUWatchpoint *wp;
 449
 450     /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
 451     if ((len & (len - 1)) || (addr & ~len_mask) ||
 452             len == 0 || len > TARGET_PAGE_SIZE) {
 453         fprintf(stderr, "qemu: tried to set invalid watchpoint at "
 454                 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
 455         return -EINVAL;
 456     }
 457     wp = g_malloc(sizeof(*wp));
 458
 459     wp->vaddr = addr;
 460     wp->len_mask = len_mask;
 461     wp->flags = flags;
 462
 463     /* keep all GDB-injected watchpoints in front */
 464     if (flags & BP_GDB)
 465         QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
 466     else
 467         QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
 468
 469     tlb_flush_page(env, addr);
 470
 471     if (watchpoint)
 472         *watchpoint = wp;
 473     return 0;
 474 }
 475
 476 /* Remove a specific watchpoint.  */
 477 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
 478                           int flags)
 479 {
 480     target_ulong len_mask = ~(len - 1);
 481     CPUWatchpoint *wp;
 482
 483     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 484         if (addr == wp->vaddr && len_mask == wp->len_mask
 485                 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
 486             cpu_watchpoint_remove_by_ref(env, wp);
 487             return 0;
 488         }
 489     }
 490     return -ENOENT;
 491 }
 492
 493 /* Remove a specific watchpoint by reference.  */
 494 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
 495 {
 496     QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
 497
 498     tlb_flush_page(env, watchpoint->vaddr);
 499
 500     g_free(watchpoint);
 501 }
 502
 503 /* Remove all matching watchpoints.  */
 504 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 505 {
 506     CPUWatchpoint *wp, *next;
 507
 508     QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
 509         if (wp->flags & mask)
 510             cpu_watchpoint_remove_by_ref(env, wp);
 511     }
 512 }
 513 #endif
 514
 515 /* Add a breakpoint.  */
 516 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
 517                           CPUBreakpoint **breakpoint)
 518 {
 519 #if defined(TARGET_HAS_ICE)
 520     CPUBreakpoint *bp;
 521
 522     bp = g_malloc(sizeof(*bp));
 523
 524     bp->pc = pc;
 525     bp->flags = flags;
 526
 527     /* keep all GDB-injected breakpoints in front */
 528     if (flags & BP_GDB)
 529         QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
 530     else
 531         QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
 532
 533     breakpoint_invalidate(env, pc);
 534
 535     if (breakpoint)
 536         *breakpoint = bp;
 537     return 0;
 538 #else
 539     return -ENOSYS;
 540 #endif
 541 }
 542
 543 /* Remove a specific breakpoint.  */
 544 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
 545 {
 546 #if defined(TARGET_HAS_ICE)
 547     CPUBreakpoint *bp;
 548
 549     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
 550         if (bp->pc == pc && bp->flags == flags) {
 551             cpu_breakpoint_remove_by_ref(env, bp);
 552             return 0;
 553         }
 554     }
 555     return -ENOENT;
 556 #else
 557     return -ENOSYS;
 558 #endif
 559 }
 560
 561 /* Remove a specific breakpoint by reference.  */
 562 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
 563 {
 564 #if defined(TARGET_HAS_ICE)
 565     QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
 566
 567     breakpoint_invalidate(env, breakpoint->pc);
 568
 569     g_free(breakpoint);
 570 #endif
 571 }
 572
 573 /* Remove all matching breakpoints. */
 574 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
 575 {
 576 #if defined(TARGET_HAS_ICE)
 577     CPUBreakpoint *bp, *next;
 578
 579     QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
 580         if (bp->flags & mask)
 581             cpu_breakpoint_remove_by_ref(env, bp);
 582     }
 583 #endif
 584 }
 585
 586 /* enable or disable single step mode. EXCP_DEBUG is returned by the
 587    CPU loop after each instruction */
 588 void cpu_single_step(CPUArchState *env, int enabled)
 589 {
 590 #if defined(TARGET_HAS_ICE)
 591     if (env->singlestep_enabled != enabled) {
 592         env->singlestep_enabled = enabled;
 593         if (kvm_enabled())
 594             kvm_update_guest_debug(env, 0);
 595         else {
 596             /* must flush all the translated code to avoid inconsistencies */
 597             /* XXX: only flush what is necessary */
 598             tb_flush(env);
 599         }
 600     }
 601 #endif
 602 }
 603
 604 void cpu_abort(CPUArchState *env, const char *fmt, ...)
 605 {
 606     CPUState *cpu = ENV_GET_CPU(env);
 607     va_list ap;
 608     va_list ap2;
 609
 610     va_start(ap, fmt);
 611     va_copy(ap2, ap);
 612     fprintf(stderr, "qemu: fatal: ");
 613     vfprintf(stderr, fmt, ap);
 614     fprintf(stderr, "\n");
 615     cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 616     if (qemu_log_enabled()) {
 617         qemu_log("qemu: fatal: ");
 618         qemu_log_vprintf(fmt, ap2);
 619         qemu_log("\n");
 620         log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 621         qemu_log_flush();
 622         qemu_log_close();
 623     }
 624     va_end(ap2);
 625     va_end(ap);
 626 #if defined(CONFIG_USER_ONLY)
 627     {
 628         struct sigaction act;
 629         sigfillset(&act.sa_mask);
 630         act.sa_handler = SIG_DFL;
 631         sigaction(SIGABRT, &act, NULL);
 632     }
 633 #endif
 634     abort();
 635 }
 636
 637 CPUArchState *cpu_copy(CPUArchState *env)
 638 {
 639     CPUArchState *new_env = cpu_init(env->cpu_model_str);
 640 #if defined(TARGET_HAS_ICE)
 641     CPUBreakpoint *bp;
 642     CPUWatchpoint *wp;
 643 #endif
 644
 645     memcpy(new_env, env, sizeof(CPUArchState));
 646
 647     /* Clone all break/watchpoints.
 648        Note: Once we support ptrace with hw-debug register access, make sure
 649        BP_CPU break/watchpoints are handled correctly on clone. */
 650     QTAILQ_INIT(&env->breakpoints);
 651     QTAILQ_INIT(&env->watchpoints);
 652 #if defined(TARGET_HAS_ICE)
 653     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
 654         cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
 655     }
 656     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 657         cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
 658                               wp->flags, NULL);
 659     }
 660 #endif
 661
 662     return new_env;
 663 }
 664
 665 #if !defined(CONFIG_USER_ONLY)
 666 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
 667                                       uintptr_t length)
 668 {
 669     uintptr_t start1;
 670
 671     /* we modify the TLB cache so that the dirty bit will be set again
 672        when accessing the range */
 673     start1 = (uintptr_t)qemu_safe_ram_ptr(start);
 674     /* Check that we don't span multiple blocks - this breaks the
 675        address comparisons below.  */
 676     if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
 677             != (end - 1) - start) {
 678         abort();
 679     }
 680     cpu_tlb_reset_dirty_all(start1, length);
 681
 682 }
 683
 684 /* Note: start and end must be within the same ram block.  */
 685 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
 686                                      int dirty_flags)
 687 {
 688     uintptr_t length;
 689
 690     start &= TARGET_PAGE_MASK;
 691     end = TARGET_PAGE_ALIGN(end);
 692
 693     length = end - start;
 694     if (length == 0)
 695         return;
 696     cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
 697
 698     if (tcg_enabled()) {
 699         tlb_reset_dirty_range_all(start, end, length);
 700     }
 701 }
 702
 703 static int cpu_physical_memory_set_dirty_tracking(int enable)
 704 {
 705     int ret = 0;
 706     in_migration = enable;
 707     return ret;
 708 }
 709
 710 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
 711                                        MemoryRegionSection *section,
 712                                        target_ulong vaddr,
 713                                        hwaddr paddr, hwaddr xlat,
 714                                        int prot,
 715                                        target_ulong *address)
 716 {
 717     hwaddr iotlb;
 718     CPUWatchpoint *wp;
 719
 720     if (memory_region_is_ram(section->mr)) {
 721         /* Normal RAM.  */
 722         iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
 723             + xlat;
 724         if (!section->readonly) {
 725             iotlb |= PHYS_SECTION_NOTDIRTY;
 726         } else {
 727             iotlb |= PHYS_SECTION_ROM;
 728         }
 729     } else {
 730         iotlb = section - address_space_memory.dispatch->sections;
 731         iotlb += xlat;
 732     }
 733
 734     /* Make accesses to pages with watchpoints go via the
 735        watchpoint trap routines.  */
 736     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 737         if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
 738             /* Avoid trapping reads of pages with a write breakpoint. */
 739             if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
 740                 iotlb = PHYS_SECTION_WATCH + paddr;
 741                 *address |= TLB_MMIO;
 742                 break;
 743             }
 744         }
 745     }
 746
 747     return iotlb;
 748 }
 749 #endif /* defined(CONFIG_USER_ONLY) */
 750
 751 #if !defined(CONFIG_USER_ONLY)
 752
 753 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
 754                              uint16_t section);
 755 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
 756
 757 static uint16_t phys_section_add(MemoryRegionSection *section)
 758 {
 759     /* The physical section number is ORed with a page-aligned
 760      * pointer to produce the iotlb entries.  Thus it should
 761      * never overflow into the page-aligned value.
 762      */
 763     assert(next_map.sections_nb < TARGET_PAGE_SIZE);
 764
 765     if (next_map.sections_nb == next_map.sections_nb_alloc) {
 766         next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
 767                                          16);
 768         next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
 769                                     next_map.sections_nb_alloc);
 770     }
 771     next_map.sections[next_map.sections_nb] = *section;
 772     memory_region_ref(section->mr);
 773     return next_map.sections_nb++;
 774 }
 775
 776 static void phys_section_destroy(MemoryRegion *mr)
 777 {
 778     memory_region_unref(mr);
 779
 780     if (mr->subpage) {
 781         subpage_t *subpage = container_of(mr, subpage_t, iomem);
 782         memory_region_destroy(&subpage->iomem);
 783         g_free(subpage);
 784     }
 785 }
 786
 787 static void phys_sections_free(PhysPageMap *map)
 788 {
 789     while (map->sections_nb > 0) {
 790         MemoryRegionSection *section = &map->sections[--map->sections_nb];
 791         phys_section_destroy(section->mr);
 792     }
 793     g_free(map->sections);
 794     g_free(map->nodes);
 795     g_free(map);
 796 }
 797
 798 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
 799 {
 800     subpage_t *subpage;
 801     hwaddr base = section->offset_within_address_space
 802         & TARGET_PAGE_MASK;
 803     MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
 804                                                    next_map.nodes, next_map.sections);
 805     MemoryRegionSection subsection = {
 806         .offset_within_address_space = base,
 807         .size = int128_make64(TARGET_PAGE_SIZE),
 808     };
 809     hwaddr start, end;
 810
 811     assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
 812
 813     if (!(existing->mr->subpage)) {
 814         subpage = subpage_init(d->as, base);
 815         subsection.mr = &subpage->iomem;
 816         phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
 817                       phys_section_add(&subsection));
 818     } else {
 819         subpage = container_of(existing->mr, subpage_t, iomem);
 820     }
 821     start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
 822     end = start + int128_get64(section->size) - 1;
 823     subpage_register(subpage, start, end, phys_section_add(section));
 824 }
 825
 826
 827 static void register_multipage(AddressSpaceDispatch *d,
 828                                MemoryRegionSection *section)
 829 {
 830     hwaddr start_addr = section->offset_within_address_space;
 831     uint16_t section_index = phys_section_add(section);
 832     uint64_t num_pages = int128_get64(int128_rshift(section->size,
 833                                                     TARGET_PAGE_BITS));
 834
 835     assert(num_pages);
 836     phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
 837 }
 838
 839 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
 840 {
 841     AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
 842     AddressSpaceDispatch *d = as->next_dispatch;
 843     MemoryRegionSection now = *section, remain = *section;
 844     Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
 845
 846     if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
 847         uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
 848                        - now.offset_within_address_space;
 849
 850         now.size = int128_min(int128_make64(left), now.size);
 851         register_subpage(d, &now);
 852     } else {
 853         now.size = int128_zero();
 854     }
 855     while (int128_ne(remain.size, now.size)) {
 856         remain.size = int128_sub(remain.size, now.size);
 857         remain.offset_within_address_space += int128_get64(now.size);
 858         remain.offset_within_region += int128_get64(now.size);
 859         now = remain;
 860         if (int128_lt(remain.size, page_size)) {
 861             register_subpage(d, &now);
 862         } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
 863             now.size = page_size;
 864             register_subpage(d, &now);
 865         } else {
 866             now.size = int128_and(now.size, int128_neg(page_size));
 867             register_multipage(d, &now);
 868         }
 869     }
 870 }
 871
 872 void qemu_flush_coalesced_mmio_buffer(void)
 873 {
 874     if (kvm_enabled())
 875         kvm_flush_coalesced_mmio_buffer();
 876 }
 877
 878 void qemu_mutex_lock_ramlist(void)
 879 {
 880     qemu_mutex_lock(&ram_list.mutex);
 881 }
 882
 883 void qemu_mutex_unlock_ramlist(void)
 884 {
 885     qemu_mutex_unlock(&ram_list.mutex);
 886 }
 887
 888 #if defined(__linux__) && !defined(TARGET_S390X)
 889
 890 #include <sys/vfs.h>
 891
 892 #define HUGETLBFS_MAGIC       0x958458f6
 893
 894 static long gethugepagesize(const char *path)
 895 {
 896     struct statfs fs;
 897     int ret;
 898
 899     do {
 900         ret = statfs(path, &fs);
 901     } while (ret != 0 && errno == EINTR);
 902
 903     if (ret != 0) {
 904         perror(path);
 905         return 0;
 906     }
 907
 908     if (fs.f_type != HUGETLBFS_MAGIC)
 909         fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
 910
 911     return fs.f_bsize;
 912 }
 913
 914 static void *file_ram_alloc(RAMBlock *block,
 915                             ram_addr_t memory,
 916                             const char *path)
 917 {
 918     char *filename;
 919     char *sanitized_name;
 920     char *c;
 921     void *area;
 922     int fd;
 923 #ifdef MAP_POPULATE
 924     int flags;
 925 #endif
 926     unsigned long hpagesize;
 927
 928     hpagesize = gethugepagesize(path);
 929     if (!hpagesize) {
 930         return NULL;
 931     }
 932
 933     if (memory < hpagesize) {
 934         return NULL;
 935     }
 936
 937     if (kvm_enabled() && !kvm_has_sync_mmu()) {
 938         fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
 939         return NULL;
 940     }
 941
 942     /* Make name safe to use with mkstemp by replacing '/' with '_'. */
 943     sanitized_name = g_strdup(block->mr->name);
 944     for (c = sanitized_name; *c != '\0'; c++) {
 945         if (*c == '/')
 946             *c = '_';
 947     }
 948
 949     filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
 950                                sanitized_name);
 951     g_free(sanitized_name);
 952
 953     fd = mkstemp(filename);
 954     if (fd < 0) {
 955         perror("unable to create backing store for hugepages");
 956         g_free(filename);
 957         return NULL;
 958     }
 959     unlink(filename);
 960     g_free(filename);
 961
 962     memory = (memory+hpagesize-1) & ~(hpagesize-1);
 963
 964     /*
 965      * ftruncate is not supported by hugetlbfs in older
 966      * hosts, so don't bother bailing out on errors.
 967      * If anything goes wrong with it under other filesystems,
 968      * mmap will fail.
 969      */
 970     if (ftruncate(fd, memory))
 971         perror("ftruncate");
 972
 973 #ifdef MAP_POPULATE
 974     /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
 975      * MAP_PRIVATE is requested.  For mem_prealloc we mmap as MAP_SHARED
 976      * to sidestep this quirk.
 977      */
 978     flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
 979     area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
 980 #else
 981     area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
 982 #endif
 983     if (area == MAP_FAILED) {
 984         perror("file_ram_alloc: can't mmap RAM pages");
 985         close(fd);
 986         return (NULL);
 987     }
 988     block->fd = fd;
 989     return area;
 990 }
 991 #endif
 992
 993 static ram_addr_t find_ram_offset(ram_addr_t size)
 994 {
 995     RAMBlock *block, *next_block;
 996     ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
 997
 998     assert(size != 0); /* it would hand out same offset multiple times */
 999
1000     if (QTAILQ_EMPTY(&ram_list.blocks))
1001         return 0;
1002
1003     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1004         ram_addr_t end, next = RAM_ADDR_MAX;
1005
1006         end = block->offset + block->length;
1007
1008         QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1009             if (next_block->offset >= end) {
1010                 next = MIN(next, next_block->offset);
1011             }
1012         }
1013         if (next - end >= size && next - end < mingap) {
1014             offset = end;
1015             mingap = next - end;
1016         }
1017     }
1018
1019     if (offset == RAM_ADDR_MAX) {
1020         fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1021                 (uint64_t)size);
1022         abort();
1023     }
1024
1025     return offset;
1026 }
1027
1028 ram_addr_t last_ram_offset(void)
1029 {
1030     RAMBlock *block;
1031     ram_addr_t last = 0;
1032
1033     QTAILQ_FOREACH(block, &ram_list.blocks, next)
1034         last = MAX(last, block->offset + block->length);
1035
1036     return last;
1037 }
1038
1039 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1040 {
1041     int ret;
1042
1043     /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1044     if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1045                            "dump-guest-core", true)) {
1046         ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1047         if (ret) {
1048             perror("qemu_madvise");
1049             fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1050                             "but dump_guest_core=off specified\n");
1051         }
1052     }
1053 }
1054
1055 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1056 {
1057     RAMBlock *new_block, *block;
1058
1059     new_block = NULL;
1060     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1061         if (block->offset == addr) {
1062             new_block = block;
1063             break;
1064         }
1065     }
1066     assert(new_block);
1067     assert(!new_block->idstr[0]);
1068
1069     if (dev) {
1070         char *id = qdev_get_dev_path(dev);
1071         if (id) {
1072             snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1073             g_free(id);
1074         }
1075     }
1076     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1077
1078     /* This assumes the iothread lock is taken here too.  */
1079     qemu_mutex_lock_ramlist();
1080     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1081         if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1082             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1083                     new_block->idstr);
1084             abort();
1085         }
1086     }
1087     qemu_mutex_unlock_ramlist();
1088 }
1089
1090 static int memory_try_enable_merging(void *addr, size_t len)
1091 {
1092     if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1093         /* disabled by the user */
1094         return 0;
1095     }
1096
1097     return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1098 }
1099
1100 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1101                                    MemoryRegion *mr)
1102 {
1103     RAMBlock *block, *new_block;
1104
1105     size = TARGET_PAGE_ALIGN(size);
1106     new_block = g_malloc0(sizeof(*new_block));
1107
1108     /* This assumes the iothread lock is taken here too.  */
1109     qemu_mutex_lock_ramlist();
1110     new_block->mr = mr;
1111     new_block->offset = find_ram_offset(size);
1112     if (host) {
1113         new_block->host = host;
1114         new_block->flags |= RAM_PREALLOC_MASK;
1115     } else {
1116         if (mem_path) {
1117 #if defined (__linux__) && !defined(TARGET_S390X)
1118             new_block->host = file_ram_alloc(new_block, size, mem_path);
1119             if (!new_block->host) {
1120                 new_block->host = qemu_anon_ram_alloc(size);
1121                 memory_try_enable_merging(new_block->host, size);
1122             }
1123 #else
1124             fprintf(stderr, "-mem-path option unsupported\n");
1125             exit(1);
1126 #endif
1127         } else {
1128             if (xen_enabled()) {
1129                 xen_ram_alloc(new_block->offset, size, mr);
1130             } else if (kvm_enabled()) {
1131                 /* some s390/kvm configurations have special constraints */
1132                 new_block->host = kvm_ram_alloc(size);
1133             } else {
1134                 new_block->host = qemu_anon_ram_alloc(size);
1135             }
1136             memory_try_enable_merging(new_block->host, size);
1137         }
1138     }
1139     new_block->length = size;
1140
1141     /* Keep the list sorted from biggest to smallest block.  */
1142     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1143         if (block->length < new_block->length) {
1144             break;
1145         }
1146     }
1147     if (block) {
1148         QTAILQ_INSERT_BEFORE(block, new_block, next);
1149     } else {
1150         QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1151     }
1152     ram_list.mru_block = NULL;
1153
1154     ram_list.version++;
1155     qemu_mutex_unlock_ramlist();
1156
1157     ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1158                                        last_ram_offset() >> TARGET_PAGE_BITS);
1159     memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1160            0, size >> TARGET_PAGE_BITS);
1161     cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1162
1163     qemu_ram_setup_dump(new_block->host, size);
1164     qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1165
1166     if (kvm_enabled())
1167         kvm_setup_guest_memory(new_block->host, size);
1168
1169     return new_block->offset;
1170 }
1171
1172 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1173 {
1174     return qemu_ram_alloc_from_ptr(size, NULL, mr);
1175 }
1176
1177 void qemu_ram_free_from_ptr(ram_addr_t addr)
1178 {
1179     RAMBlock *block;
1180
1181     /* This assumes the iothread lock is taken here too.  */
1182     qemu_mutex_lock_ramlist();
1183     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1184         if (addr == block->offset) {
1185             QTAILQ_REMOVE(&ram_list.blocks, block, next);
1186             ram_list.mru_block = NULL;
1187             ram_list.version++;
1188             g_free(block);
1189             break;
1190         }
1191     }
1192     qemu_mutex_unlock_ramlist();
1193 }
1194
1195 void qemu_ram_free(ram_addr_t addr)
1196 {
1197     RAMBlock *block;
1198
1199     /* This assumes the iothread lock is taken here too.  */
1200     qemu_mutex_lock_ramlist();
1201     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1202         if (addr == block->offset) {
1203             QTAILQ_REMOVE(&ram_list.blocks, block, next);
1204             ram_list.mru_block = NULL;
1205             ram_list.version++;
1206             if (block->flags & RAM_PREALLOC_MASK) {
1207                 ;
1208             } else if (mem_path) {
1209 #if defined (__linux__) && !defined(TARGET_S390X)
1210                 if (block->fd) {
1211                     munmap(block->host, block->length);
1212                     close(block->fd);
1213                 } else {
1214                     qemu_anon_ram_free(block->host, block->length);
1215                 }
1216 #else
1217                 abort();
1218 #endif
1219             } else {
1220                 if (xen_enabled()) {
1221                     xen_invalidate_map_cache_entry(block->host);
1222                 } else {
1223                     qemu_anon_ram_free(block->host, block->length);
1224                 }
1225             }
1226             g_free(block);
1227             break;
1228         }
1229     }
1230     qemu_mutex_unlock_ramlist();
1231
1232 }
1233
1234 #ifndef _WIN32
1235 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1236 {
1237     RAMBlock *block;
1238     ram_addr_t offset;
1239     int flags;
1240     void *area, *vaddr;
1241
1242     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1243         offset = addr - block->offset;
1244         if (offset < block->length) {
1245             vaddr = block->host + offset;
1246             if (block->flags & RAM_PREALLOC_MASK) {
1247                 ;
1248             } else {
1249                 flags = MAP_FIXED;
1250                 munmap(vaddr, length);
1251                 if (mem_path) {
1252 #if defined(__linux__) && !defined(TARGET_S390X)
1253                     if (block->fd) {
1254 #ifdef MAP_POPULATE
1255                         flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1256                             MAP_PRIVATE;
1257 #else
1258                         flags |= MAP_PRIVATE;
1259 #endif
1260                         area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1261                                     flags, block->fd, offset);
1262                     } else {
1263                         flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1264                         area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1265                                     flags, -1, 0);
1266                     }
1267 #else
1268                     abort();
1269 #endif
1270                 } else {
1271 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1272                     flags |= MAP_SHARED | MAP_ANONYMOUS;
1273                     area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1274                                 flags, -1, 0);
1275 #else
1276                     flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1277                     area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1278                                 flags, -1, 0);
1279 #endif
1280                 }
1281                 if (area != vaddr) {
1282                     fprintf(stderr, "Could not remap addr: "
1283                             RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1284                             length, addr);
1285                     exit(1);
1286                 }
1287                 memory_try_enable_merging(vaddr, length);
1288                 qemu_ram_setup_dump(vaddr, length);
1289             }
1290             return;
1291         }
1292     }
1293 }
1294 #endif /* !_WIN32 */
1295
1296 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1297 {
1298     RAMBlock *block;
1299
1300     /* The list is protected by the iothread lock here.  */
1301     block = ram_list.mru_block;
1302     if (block && addr - block->offset < block->length) {
1303         goto found;
1304     }
1305     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1306         if (addr - block->offset < block->length) {
1307             goto found;
1308         }
1309     }
1310
1311     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1312     abort();
1313
1314 found:
1315     ram_list.mru_block = block;
1316     return block;
1317 }
1318
1319 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1320    With the exception of the softmmu code in this file, this should
1321    only be used for local memory (e.g. video ram) that the device owns,
1322    and knows it isn't going to access beyond the end of the block.
1323
1324    It should not be used for general purpose DMA.
1325    Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1326  */
1327 void *qemu_get_ram_ptr(ram_addr_t addr)
1328 {
1329     RAMBlock *block = qemu_get_ram_block(addr);
1330
1331     if (xen_enabled()) {
1332         /* We need to check if the requested address is in the RAM
1333          * because we don't want to map the entire memory in QEMU.
1334          * In that case just map until the end of the page.
1335          */
1336         if (block->offset == 0) {
1337             return xen_map_cache(addr, 0, 0);
1338         } else if (block->host == NULL) {
1339             block->host =
1340                 xen_map_cache(block->offset, block->length, 1);
1341         }
1342     }
1343     return block->host + (addr - block->offset);
1344 }
1345
1346 /* Return a host pointer to ram allocated with qemu_ram_alloc.  Same as
1347  * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1348  *
1349  * ??? Is this still necessary?
1350  */
1351 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1352 {
1353     RAMBlock *block;
1354
1355     /* The list is protected by the iothread lock here.  */
1356     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1357         if (addr - block->offset < block->length) {
1358             if (xen_enabled()) {
1359                 /* We need to check if the requested address is in the RAM
1360                  * because we don't want to map the entire memory in QEMU.
1361                  * In that case just map until the end of the page.
1362                  */
1363                 if (block->offset == 0) {
1364                     return xen_map_cache(addr, 0, 0);
1365                 } else if (block->host == NULL) {
1366                     block->host =
1367                         xen_map_cache(block->offset, block->length, 1);
1368                 }
1369             }
1370             return block->host + (addr - block->offset);
1371         }
1372     }
1373
1374     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1375     abort();
1376
1377     return NULL;
1378 }
1379
1380 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1381  * but takes a size argument */
1382 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1383 {
1384     if (*size == 0) {
1385         return NULL;
1386     }
1387     if (xen_enabled()) {
1388         return xen_map_cache(addr, *size, 1);
1389     } else {
1390         RAMBlock *block;
1391
1392         QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1393             if (addr - block->offset < block->length) {
1394                 if (addr - block->offset + *size > block->length)
1395                     *size = block->length - addr + block->offset;
1396                 return block->host + (addr - block->offset);
1397             }
1398         }
1399
1400         fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1401         abort();
1402     }
1403 }
1404
1405 /* Some of the softmmu routines need to translate from a host pointer
1406    (typically a TLB entry) back to a ram offset.  */
1407 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1408 {
1409     RAMBlock *block;
1410     uint8_t *host = ptr;
1411
1412     if (xen_enabled()) {
1413         *ram_addr = xen_ram_addr_from_mapcache(ptr);
1414         return qemu_get_ram_block(*ram_addr)->mr;
1415     }
1416
1417     block = ram_list.mru_block;
1418     if (block && block->host && host - block->host < block->length) {
1419         goto found;
1420     }
1421
1422     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1423         /* This case append when the block is not mapped. */
1424         if (block->host == NULL) {
1425             continue;
1426         }
1427         if (host - block->host < block->length) {
1428             goto found;
1429         }
1430     }
1431
1432     return NULL;
1433
1434 found:
1435     *ram_addr = block->offset + (host - block->host);
1436     return block->mr;
1437 }
1438
1439 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1440                                uint64_t val, unsigned size)
1441 {
1442     int dirty_flags;
1443     dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1444     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1445         tb_invalidate_phys_page_fast(ram_addr, size);
1446         dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1447     }
1448     switch (size) {
1449     case 1:
1450         stb_p(qemu_get_ram_ptr(ram_addr), val);
1451         break;
1452     case 2:
1453         stw_p(qemu_get_ram_ptr(ram_addr), val);
1454         break;
1455     case 4:
1456         stl_p(qemu_get_ram_ptr(ram_addr), val);
1457         break;
1458     default:
1459         abort();
1460     }
1461     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1462     cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1463     /* we remove the notdirty callback only if the code has been
1464        flushed */
1465     if (dirty_flags == 0xff) {
1466         CPUArchState *env = current_cpu->env_ptr;
1467         tlb_set_dirty(env, env->mem_io_vaddr);
1468     }
1469 }
1470
1471 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1472                                  unsigned size, bool is_write)
1473 {
1474     return is_write;
1475 }
1476
1477 static const MemoryRegionOps notdirty_mem_ops = {
1478     .write = notdirty_mem_write,
1479     .valid.accepts = notdirty_mem_accepts,
1480     .endianness = DEVICE_NATIVE_ENDIAN,
1481 };
1482
1483 /* Generate a debug exception if a watchpoint has been hit.  */
1484 static void check_watchpoint(int offset, int len_mask, int flags)
1485 {
1486     CPUArchState *env = current_cpu->env_ptr;
1487     target_ulong pc, cs_base;
1488     target_ulong vaddr;
1489     CPUWatchpoint *wp;
1490     int cpu_flags;
1491
1492     if (env->watchpoint_hit) {
1493         /* We re-entered the check after replacing the TB. Now raise
1494          * the debug interrupt so that is will trigger after the
1495          * current instruction. */
1496         cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1497         return;
1498     }
1499     vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1500     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1501         if ((vaddr == (wp->vaddr & len_mask) ||
1502              (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1503             wp->flags |= BP_WATCHPOINT_HIT;
1504             if (!env->watchpoint_hit) {
1505                 env->watchpoint_hit = wp;
1506                 tb_check_watchpoint(env);
1507                 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1508                     env->exception_index = EXCP_DEBUG;
1509                     cpu_loop_exit(env);
1510                 } else {
1511                     cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1512                     tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1513                     cpu_resume_from_signal(env, NULL);
1514                 }
1515             }
1516         } else {
1517             wp->flags &= ~BP_WATCHPOINT_HIT;
1518         }
1519     }
1520 }
1521
1522 /* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
1523    so these check for a hit then pass through to the normal out-of-line
1524    phys routines.  */
1525 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1526                                unsigned size)
1527 {
1528     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1529     switch (size) {
1530     case 1: return ldub_phys(addr);
1531     case 2: return lduw_phys(addr);
1532     case 4: return ldl_phys(addr);
1533     default: abort();
1534     }
1535 }
1536
1537 static void watch_mem_write(void *opaque, hwaddr addr,
1538                             uint64_t val, unsigned size)
1539 {
1540     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1541     switch (size) {
1542     case 1:
1543         stb_phys(addr, val);
1544         break;
1545     case 2:
1546         stw_phys(addr, val);
1547         break;
1548     case 4:
1549         stl_phys(addr, val);
1550         break;
1551     default: abort();
1552     }
1553 }
1554
1555 static const MemoryRegionOps watch_mem_ops = {
1556     .read = watch_mem_read,
1557     .write = watch_mem_write,
1558     .endianness = DEVICE_NATIVE_ENDIAN,
1559 };
1560
1561 static uint64_t subpage_read(void *opaque, hwaddr addr,
1562                              unsigned len)
1563 {
1564     subpage_t *subpage = opaque;
1565     uint8_t buf[4];
1566
1567 #if defined(DEBUG_SUBPAGE)
1568     printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1569            subpage, len, addr);
1570 #endif
1571     address_space_read(subpage->as, addr + subpage->base, buf, len);
1572     switch (len) {
1573     case 1:
1574         return ldub_p(buf);
1575     case 2:
1576         return lduw_p(buf);
1577     case 4:
1578         return ldl_p(buf);
1579     default:
1580         abort();
1581     }
1582 }
1583
1584 static void subpage_write(void *opaque, hwaddr addr,
1585                           uint64_t value, unsigned len)
1586 {
1587     subpage_t *subpage = opaque;
1588     uint8_t buf[4];
1589
1590 #if defined(DEBUG_SUBPAGE)
1591     printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1592            " value %"PRIx64"\n",
1593            __func__, subpage, len, addr, value);
1594 #endif
1595     switch (len) {
1596     case 1:
1597         stb_p(buf, value);
1598         break;
1599     case 2:
1600         stw_p(buf, value);
1601         break;
1602     case 4:
1603         stl_p(buf, value);
1604         break;
1605     default:
1606         abort();
1607     }
1608     address_space_write(subpage->as, addr + subpage->base, buf, len);
1609 }
1610
1611 static bool subpage_accepts(void *opaque, hwaddr addr,
1612                             unsigned size, bool is_write)
1613 {
1614     subpage_t *subpage = opaque;
1615 #if defined(DEBUG_SUBPAGE)
1616     printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1617            __func__, subpage, is_write ? 'w' : 'r', len, addr);
1618 #endif
1619
1620     return address_space_access_valid(subpage->as, addr + subpage->base,
1621                                       size, is_write);
1622 }
1623
1624 static const MemoryRegionOps subpage_ops = {
1625     .read = subpage_read,
1626     .write = subpage_write,
1627     .valid.accepts = subpage_accepts,
1628     .endianness = DEVICE_NATIVE_ENDIAN,
1629 };
1630
1631 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1632                              uint16_t section)
1633 {
1634     int idx, eidx;
1635
1636     if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1637         return -1;
1638     idx = SUBPAGE_IDX(start);
1639     eidx = SUBPAGE_IDX(end);
1640 #if defined(DEBUG_SUBPAGE)
1641     printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1642            mmio, start, end, idx, eidx, memory);
1643 #endif
1644     for (; idx <= eidx; idx++) {
1645         mmio->sub_section[idx] = section;
1646     }
1647
1648     return 0;
1649 }
1650
1651 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1652 {
1653     subpage_t *mmio;
1654
1655     mmio = g_malloc0(sizeof(subpage_t));
1656
1657     mmio->as = as;
1658     mmio->base = base;
1659     memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1660                           "subpage", TARGET_PAGE_SIZE);
1661     mmio->iomem.subpage = true;
1662 #if defined(DEBUG_SUBPAGE)
1663     printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1664            mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1665 #endif
1666     subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1667
1668     return mmio;
1669 }
1670
1671 static uint16_t dummy_section(MemoryRegion *mr)
1672 {
1673     MemoryRegionSection section = {
1674         .mr = mr,
1675         .offset_within_address_space = 0,
1676         .offset_within_region = 0,
1677         .size = int128_2_64(),
1678     };
1679
1680     return phys_section_add(&section);
1681 }
1682
1683 MemoryRegion *iotlb_to_region(hwaddr index)
1684 {
1685     return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1686 }
1687
1688 static void io_mem_init(void)
1689 {
1690     memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1691     memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1692                           "unassigned", UINT64_MAX);
1693     memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1694                           "notdirty", UINT64_MAX);
1695     memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1696                           "watch", UINT64_MAX);
1697 }
1698
1699 static void mem_begin(MemoryListener *listener)
1700 {
1701     AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1702     AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1703
1704     d->phys_map  = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1705     d->as = as;
1706     as->next_dispatch = d;
1707 }
1708
1709 static void mem_commit(MemoryListener *listener)
1710 {
1711     AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1712     AddressSpaceDispatch *cur = as->dispatch;
1713     AddressSpaceDispatch *next = as->next_dispatch;
1714
1715     next->nodes = next_map.nodes;
1716     next->sections = next_map.sections;
1717
1718     as->dispatch = next;
1719     g_free(cur);
1720 }
1721
1722 static void core_begin(MemoryListener *listener)
1723 {
1724     uint16_t n;
1725
1726     prev_map = g_new(PhysPageMap, 1);
1727     *prev_map = next_map;
1728
1729     memset(&next_map, 0, sizeof(next_map));
1730     n = dummy_section(&io_mem_unassigned);
1731     assert(n == PHYS_SECTION_UNASSIGNED);
1732     n = dummy_section(&io_mem_notdirty);
1733     assert(n == PHYS_SECTION_NOTDIRTY);
1734     n = dummy_section(&io_mem_rom);
1735     assert(n == PHYS_SECTION_ROM);
1736     n = dummy_section(&io_mem_watch);
1737     assert(n == PHYS_SECTION_WATCH);
1738 }
1739
1740 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1741  * All AddressSpaceDispatch instances have switched to the next map.
1742  */
1743 static void core_commit(MemoryListener *listener)
1744 {
1745     phys_sections_free(prev_map);
1746 }
1747
1748 static void tcg_commit(MemoryListener *listener)
1749 {
1750     CPUState *cpu;
1751
1752     /* since each CPU stores ram addresses in its TLB cache, we must
1753        reset the modified entries */
1754     /* XXX: slow ! */
1755     for (cpu = first_cpu; cpu != NULL; cpu = cpu->next_cpu) {
1756         CPUArchState *env = cpu->env_ptr;
1757
1758         tlb_flush(env, 1);
1759     }
1760 }
1761
1762 static void core_log_global_start(MemoryListener *listener)
1763 {
1764     cpu_physical_memory_set_dirty_tracking(1);
1765 }
1766
1767 static void core_log_global_stop(MemoryListener *listener)
1768 {
1769     cpu_physical_memory_set_dirty_tracking(0);
1770 }
1771
1772 static MemoryListener core_memory_listener = {
1773     .begin = core_begin,
1774     .commit = core_commit,
1775     .log_global_start = core_log_global_start,
1776     .log_global_stop = core_log_global_stop,
1777     .priority = 1,
1778 };
1779
1780 static MemoryListener tcg_memory_listener = {
1781     .commit = tcg_commit,
1782 };
1783
1784 void address_space_init_dispatch(AddressSpace *as)
1785 {
1786     as->dispatch = NULL;
1787     as->dispatch_listener = (MemoryListener) {
1788         .begin = mem_begin,
1789         .commit = mem_commit,
1790         .region_add = mem_add,
1791         .region_nop = mem_add,
1792         .priority = 0,
1793     };
1794     memory_listener_register(&as->dispatch_listener, as);
1795 }
1796
1797 void address_space_destroy_dispatch(AddressSpace *as)
1798 {
1799     AddressSpaceDispatch *d = as->dispatch;
1800
1801     memory_listener_unregister(&as->dispatch_listener);
1802     g_free(d);
1803     as->dispatch = NULL;
1804 }
1805
1806 static void memory_map_init(void)
1807 {
1808     system_memory = g_malloc(sizeof(*system_memory));
1809     memory_region_init(system_memory, NULL, "system", INT64_MAX);
1810     address_space_init(&address_space_memory, system_memory, "memory");
1811
1812     system_io = g_malloc(sizeof(*system_io));
1813     memory_region_init(system_io, NULL, "io", 65536);
1814     address_space_init(&address_space_io, system_io, "I/O");
1815
1816     memory_listener_register(&core_memory_listener, &address_space_memory);
1817     memory_listener_register(&tcg_memory_listener, &address_space_memory);
1818 }
1819
1820 MemoryRegion *get_system_memory(void)
1821 {
1822     return system_memory;
1823 }
1824
1825 MemoryRegion *get_system_io(void)
1826 {
1827     return system_io;
1828 }
1829
1830 #endif /* !defined(CONFIG_USER_ONLY) */
1831
1832 /* physical memory access (slow version, mainly for debug) */
1833 #if defined(CONFIG_USER_ONLY)
1834 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1835                         uint8_t *buf, int len, int is_write)
1836 {
1837     int l, flags;
1838     target_ulong page;
1839     void * p;
1840
1841     while (len > 0) {
1842         page = addr & TARGET_PAGE_MASK;
1843         l = (page + TARGET_PAGE_SIZE) - addr;
1844         if (l > len)
1845             l = len;
1846         flags = page_get_flags(page);
1847         if (!(flags & PAGE_VALID))
1848             return -1;
1849         if (is_write) {
1850             if (!(flags & PAGE_WRITE))
1851                 return -1;
1852             /* XXX: this code should not depend on lock_user */
1853             if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1854                 return -1;
1855             memcpy(p, buf, l);
1856             unlock_user(p, addr, l);
1857         } else {
1858             if (!(flags & PAGE_READ))
1859                 return -1;
1860             /* XXX: this code should not depend on lock_user */
1861             if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1862                 return -1;
1863             memcpy(buf, p, l);
1864             unlock_user(p, addr, 0);
1865         }
1866         len -= l;
1867         buf += l;
1868         addr += l;
1869     }
1870     return 0;
1871 }
1872
1873 #else
1874
1875 static void invalidate_and_set_dirty(hwaddr addr,
1876                                      hwaddr length)
1877 {
1878     if (!cpu_physical_memory_is_dirty(addr)) {
1879         /* invalidate code */
1880         tb_invalidate_phys_page_range(addr, addr + length, 0);
1881         /* set dirty bit */
1882         cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1883     }
1884     xen_modified_memory(addr, length);
1885 }
1886
1887 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1888 {
1889     if (memory_region_is_ram(mr)) {
1890         return !(is_write && mr->readonly);
1891     }
1892     if (memory_region_is_romd(mr)) {
1893         return !is_write;
1894     }
1895
1896     return false;
1897 }
1898
1899 static inline int memory_access_size(MemoryRegion *mr, int l, hwaddr addr)
1900 {
1901     if (l >= 4 && (((addr & 3) == 0 || mr->ops->impl.unaligned))) {
1902         return 4;
1903     }
1904     if (l >= 2 && (((addr & 1) == 0) || mr->ops->impl.unaligned)) {
1905         return 2;
1906     }
1907     return 1;
1908 }
1909
1910 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1911                       int len, bool is_write)
1912 {
1913     hwaddr l;
1914     uint8_t *ptr;
1915     uint64_t val;
1916     hwaddr addr1;
1917     MemoryRegion *mr;
1918     bool error = false;
1919
1920     while (len > 0) {
1921         l = len;
1922         mr = address_space_translate(as, addr, &addr1, &l, is_write);
1923
1924         if (is_write) {
1925             if (!memory_access_is_direct(mr, is_write)) {
1926                 l = memory_access_size(mr, l, addr1);
1927                 /* XXX: could force current_cpu to NULL to avoid
1928                    potential bugs */
1929                 if (l == 4) {
1930                     /* 32 bit write access */
1931                     val = ldl_p(buf);
1932                     error |= io_mem_write(mr, addr1, val, 4);
1933                 } else if (l == 2) {
1934                     /* 16 bit write access */
1935                     val = lduw_p(buf);
1936                     error |= io_mem_write(mr, addr1, val, 2);
1937                 } else {
1938                     /* 8 bit write access */
1939                     val = ldub_p(buf);
1940                     error |= io_mem_write(mr, addr1, val, 1);
1941                 }
1942             } else {
1943                 addr1 += memory_region_get_ram_addr(mr);
1944                 /* RAM case */
1945                 ptr = qemu_get_ram_ptr(addr1);
1946                 memcpy(ptr, buf, l);
1947                 invalidate_and_set_dirty(addr1, l);
1948             }
1949         } else {
1950             if (!memory_access_is_direct(mr, is_write)) {
1951                 /* I/O case */
1952                 l = memory_access_size(mr, l, addr1);
1953                 if (l == 4) {
1954                     /* 32 bit read access */
1955                     error |= io_mem_read(mr, addr1, &val, 4);
1956                     stl_p(buf, val);
1957                 } else if (l == 2) {
1958                     /* 16 bit read access */
1959                     error |= io_mem_read(mr, addr1, &val, 2);
1960                     stw_p(buf, val);
1961                 } else {
1962                     /* 8 bit read access */
1963                     error |= io_mem_read(mr, addr1, &val, 1);
1964                     stb_p(buf, val);
1965                 }
1966             } else {
1967                 /* RAM case */
1968                 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1969                 memcpy(buf, ptr, l);
1970             }
1971         }
1972         len -= l;
1973         buf += l;
1974         addr += l;
1975     }
1976
1977     return error;
1978 }
1979
1980 bool address_space_write(AddressSpace *as, hwaddr addr,
1981                          const uint8_t *buf, int len)
1982 {
1983     return address_space_rw(as, addr, (uint8_t *)buf, len, true);
1984 }
1985
1986 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1987 {
1988     return address_space_rw(as, addr, buf, len, false);
1989 }
1990
1991
1992 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1993                             int len, int is_write)
1994 {
1995     address_space_rw(&address_space_memory, addr, buf, len, is_write);
1996 }
1997
1998 /* used for ROM loading : can write in RAM and ROM */
1999 void cpu_physical_memory_write_rom(hwaddr addr,
2000                                    const uint8_t *buf, int len)
2001 {
2002     hwaddr l;
2003     uint8_t *ptr;
2004     hwaddr addr1;
2005     MemoryRegion *mr;
2006
2007     while (len > 0) {
2008         l = len;
2009         mr = address_space_translate(&address_space_memory,
2010                                      addr, &addr1, &l, true);
2011
2012         if (!(memory_region_is_ram(mr) ||
2013               memory_region_is_romd(mr))) {
2014             /* do nothing */
2015         } else {
2016             addr1 += memory_region_get_ram_addr(mr);
2017             /* ROM/RAM case */
2018             ptr = qemu_get_ram_ptr(addr1);
2019             memcpy(ptr, buf, l);
2020             invalidate_and_set_dirty(addr1, l);
2021         }
2022         len -= l;
2023         buf += l;
2024         addr += l;
2025     }
2026 }
2027
2028 typedef struct {
2029     MemoryRegion *mr;
2030     void *buffer;
2031     hwaddr addr;
2032     hwaddr len;
2033 } BounceBuffer;
2034
2035 static BounceBuffer bounce;
2036
2037 typedef struct MapClient {
2038     void *opaque;
2039     void (*callback)(void *opaque);
2040     QLIST_ENTRY(MapClient) link;
2041 } MapClient;
2042
2043 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2044     = QLIST_HEAD_INITIALIZER(map_client_list);
2045
2046 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2047 {
2048     MapClient *client = g_malloc(sizeof(*client));
2049
2050     client->opaque = opaque;
2051     client->callback = callback;
2052     QLIST_INSERT_HEAD(&map_client_list, client, link);
2053     return client;
2054 }
2055
2056 static void cpu_unregister_map_client(void *_client)
2057 {
2058     MapClient *client = (MapClient *)_client;
2059
2060     QLIST_REMOVE(client, link);
2061     g_free(client);
2062 }
2063
2064 static void cpu_notify_map_clients(void)
2065 {
2066     MapClient *client;
2067
2068     while (!QLIST_EMPTY(&map_client_list)) {
2069         client = QLIST_FIRST(&map_client_list);
2070         client->callback(client->opaque);
2071         cpu_unregister_map_client(client);
2072     }
2073 }
2074
2075 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2076 {
2077     MemoryRegion *mr;
2078     hwaddr l, xlat;
2079
2080     while (len > 0) {
2081         l = len;
2082         mr = address_space_translate(as, addr, &xlat, &l, is_write);
2083         if (!memory_access_is_direct(mr, is_write)) {
2084             l = memory_access_size(mr, l, addr);
2085             if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2086                 return false;
2087             }
2088         }
2089
2090         len -= l;
2091         addr += l;
2092     }
2093     return true;
2094 }
2095
2096 /* Map a physical memory region into a host virtual address.
2097  * May map a subset of the requested range, given by and returned in *plen.
2098  * May return NULL if resources needed to perform the mapping are exhausted.
2099  * Use only for reads OR writes - not for read-modify-write operations.
2100  * Use cpu_register_map_client() to know when retrying the map operation is
2101  * likely to succeed.
2102  */
2103 void *address_space_map(AddressSpace *as,
2104                         hwaddr addr,
2105                         hwaddr *plen,
2106                         bool is_write)
2107 {
2108     hwaddr len = *plen;
2109     hwaddr done = 0;
2110     hwaddr l, xlat, base;
2111     MemoryRegion *mr, *this_mr;
2112     ram_addr_t raddr;
2113
2114     if (len == 0) {
2115         return NULL;
2116     }
2117
2118     l = len;
2119     mr = address_space_translate(as, addr, &xlat, &l, is_write);
2120     if (!memory_access_is_direct(mr, is_write)) {
2121         if (bounce.buffer) {
2122             return NULL;
2123         }
2124         bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2125         bounce.addr = addr;
2126         bounce.len = l;
2127
2128         memory_region_ref(mr);
2129         bounce.mr = mr;
2130         if (!is_write) {
2131             address_space_read(as, addr, bounce.buffer, l);
2132         }
2133
2134         *plen = l;
2135         return bounce.buffer;
2136     }
2137
2138     base = xlat;
2139     raddr = memory_region_get_ram_addr(mr);
2140
2141     for (;;) {
2142         len -= l;
2143         addr += l;
2144         done += l;
2145         if (len == 0) {
2146             break;
2147         }
2148
2149         l = len;
2150         this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2151         if (this_mr != mr || xlat != base + done) {
2152             break;
2153         }
2154     }
2155
2156     memory_region_ref(mr);
2157     *plen = done;
2158     return qemu_ram_ptr_length(raddr + base, plen);
2159 }
2160
2161 /* Unmaps a memory region previously mapped by address_space_map().
2162  * Will also mark the memory as dirty if is_write == 1.  access_len gives
2163  * the amount of memory that was actually read or written by the caller.
2164  */
2165 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2166                          int is_write, hwaddr access_len)
2167 {
2168     if (buffer != bounce.buffer) {
2169         MemoryRegion *mr;
2170         ram_addr_t addr1;
2171
2172         mr = qemu_ram_addr_from_host(buffer, &addr1);
2173         assert(mr != NULL);
2174         if (is_write) {
2175             while (access_len) {
2176                 unsigned l;
2177                 l = TARGET_PAGE_SIZE;
2178                 if (l > access_len)
2179                     l = access_len;
2180                 invalidate_and_set_dirty(addr1, l);
2181                 addr1 += l;
2182                 access_len -= l;
2183             }
2184         }
2185         if (xen_enabled()) {
2186             xen_invalidate_map_cache_entry(buffer);
2187         }
2188         memory_region_unref(mr);
2189         return;
2190     }
2191     if (is_write) {
2192         address_space_write(as, bounce.addr, bounce.buffer, access_len);
2193     }
2194     qemu_vfree(bounce.buffer);
2195     bounce.buffer = NULL;
2196     memory_region_unref(bounce.mr);
2197     cpu_notify_map_clients();
2198 }
2199
2200 void *cpu_physical_memory_map(hwaddr addr,
2201                               hwaddr *plen,
2202                               int is_write)
2203 {
2204     return address_space_map(&address_space_memory, addr, plen, is_write);
2205 }
2206
2207 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2208                                int is_write, hwaddr access_len)
2209 {
2210     return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2211 }
2212
2213 /* warning: addr must be aligned */
2214 static inline uint32_t ldl_phys_internal(hwaddr addr,
2215                                          enum device_endian endian)
2216 {
2217     uint8_t *ptr;
2218     uint64_t val;
2219     MemoryRegion *mr;
2220     hwaddr l = 4;
2221     hwaddr addr1;
2222
2223     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2224                                  false);
2225     if (l < 4 || !memory_access_is_direct(mr, false)) {
2226         /* I/O case */
2227         io_mem_read(mr, addr1, &val, 4);
2228 #if defined(TARGET_WORDS_BIGENDIAN)
2229         if (endian == DEVICE_LITTLE_ENDIAN) {
2230             val = bswap32(val);
2231         }
2232 #else
2233         if (endian == DEVICE_BIG_ENDIAN) {
2234             val = bswap32(val);
2235         }
2236 #endif
2237     } else {
2238         /* RAM case */
2239         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2240                                 & TARGET_PAGE_MASK)
2241                                + addr1);
2242         switch (endian) {
2243         case DEVICE_LITTLE_ENDIAN:
2244             val = ldl_le_p(ptr);
2245             break;
2246         case DEVICE_BIG_ENDIAN:
2247             val = ldl_be_p(ptr);
2248             break;
2249         default:
2250             val = ldl_p(ptr);
2251             break;
2252         }
2253     }
2254     return val;
2255 }
2256
2257 uint32_t ldl_phys(hwaddr addr)
2258 {
2259     return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2260 }
2261
2262 uint32_t ldl_le_phys(hwaddr addr)
2263 {
2264     return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2265 }
2266
2267 uint32_t ldl_be_phys(hwaddr addr)
2268 {
2269     return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2270 }
2271
2272 /* warning: addr must be aligned */
2273 static inline uint64_t ldq_phys_internal(hwaddr addr,
2274                                          enum device_endian endian)
2275 {
2276     uint8_t *ptr;
2277     uint64_t val;
2278     MemoryRegion *mr;
2279     hwaddr l = 8;
2280     hwaddr addr1;
2281
2282     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2283                                  false);
2284     if (l < 8 || !memory_access_is_direct(mr, false)) {
2285         /* I/O case */
2286         io_mem_read(mr, addr1, &val, 8);
2287 #if defined(TARGET_WORDS_BIGENDIAN)
2288         if (endian == DEVICE_LITTLE_ENDIAN) {
2289             val = bswap64(val);
2290         }
2291 #else
2292         if (endian == DEVICE_BIG_ENDIAN) {
2293             val = bswap64(val);
2294         }
2295 #endif
2296     } else {
2297         /* RAM case */
2298         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2299                                 & TARGET_PAGE_MASK)
2300                                + addr1);
2301         switch (endian) {
2302         case DEVICE_LITTLE_ENDIAN:
2303             val = ldq_le_p(ptr);
2304             break;
2305         case DEVICE_BIG_ENDIAN:
2306             val = ldq_be_p(ptr);
2307             break;
2308         default:
2309             val = ldq_p(ptr);
2310             break;
2311         }
2312     }
2313     return val;
2314 }
2315
2316 uint64_t ldq_phys(hwaddr addr)
2317 {
2318     return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2319 }
2320
2321 uint64_t ldq_le_phys(hwaddr addr)
2322 {
2323     return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2324 }
2325
2326 uint64_t ldq_be_phys(hwaddr addr)
2327 {
2328     return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2329 }
2330
2331 /* XXX: optimize */
2332 uint32_t ldub_phys(hwaddr addr)
2333 {
2334     uint8_t val;
2335     cpu_physical_memory_read(addr, &val, 1);
2336     return val;
2337 }
2338
2339 /* warning: addr must be aligned */
2340 static inline uint32_t lduw_phys_internal(hwaddr addr,
2341                                           enum device_endian endian)
2342 {
2343     uint8_t *ptr;
2344     uint64_t val;
2345     MemoryRegion *mr;
2346     hwaddr l = 2;
2347     hwaddr addr1;
2348
2349     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2350                                  false);
2351     if (l < 2 || !memory_access_is_direct(mr, false)) {
2352         /* I/O case */
2353         io_mem_read(mr, addr1, &val, 2);
2354 #if defined(TARGET_WORDS_BIGENDIAN)
2355         if (endian == DEVICE_LITTLE_ENDIAN) {
2356             val = bswap16(val);
2357         }
2358 #else
2359         if (endian == DEVICE_BIG_ENDIAN) {
2360             val = bswap16(val);
2361         }
2362 #endif
2363     } else {
2364         /* RAM case */
2365         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2366                                 & TARGET_PAGE_MASK)
2367                                + addr1);
2368         switch (endian) {
2369         case DEVICE_LITTLE_ENDIAN:
2370             val = lduw_le_p(ptr);
2371             break;
2372         case DEVICE_BIG_ENDIAN:
2373             val = lduw_be_p(ptr);
2374             break;
2375         default:
2376             val = lduw_p(ptr);
2377             break;
2378         }
2379     }
2380     return val;
2381 }
2382
2383 uint32_t lduw_phys(hwaddr addr)
2384 {
2385     return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2386 }
2387
2388 uint32_t lduw_le_phys(hwaddr addr)
2389 {
2390     return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2391 }
2392
2393 uint32_t lduw_be_phys(hwaddr addr)
2394 {
2395     return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2396 }
2397
2398 /* warning: addr must be aligned. The ram page is not masked as dirty
2399    and the code inside is not invalidated. It is useful if the dirty
2400    bits are used to track modified PTEs */
2401 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2402 {
2403     uint8_t *ptr;
2404     MemoryRegion *mr;
2405     hwaddr l = 4;
2406     hwaddr addr1;
2407
2408     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2409                                  true);
2410     if (l < 4 || !memory_access_is_direct(mr, true)) {
2411         io_mem_write(mr, addr1, val, 4);
2412     } else {
2413         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2414         ptr = qemu_get_ram_ptr(addr1);
2415         stl_p(ptr, val);
2416
2417         if (unlikely(in_migration)) {
2418             if (!cpu_physical_memory_is_dirty(addr1)) {
2419                 /* invalidate code */
2420                 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2421                 /* set dirty bit */
2422                 cpu_physical_memory_set_dirty_flags(
2423                     addr1, (0xff & ~CODE_DIRTY_FLAG));
2424             }
2425         }
2426     }
2427 }
2428
2429 /* warning: addr must be aligned */
2430 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2431                                      enum device_endian endian)
2432 {
2433     uint8_t *ptr;
2434     MemoryRegion *mr;
2435     hwaddr l = 4;
2436     hwaddr addr1;
2437
2438     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2439                                  true);
2440     if (l < 4 || !memory_access_is_direct(mr, true)) {
2441 #if defined(TARGET_WORDS_BIGENDIAN)
2442         if (endian == DEVICE_LITTLE_ENDIAN) {
2443             val = bswap32(val);
2444         }
2445 #else
2446         if (endian == DEVICE_BIG_ENDIAN) {
2447             val = bswap32(val);
2448         }
2449 #endif
2450         io_mem_write(mr, addr1, val, 4);
2451     } else {
2452         /* RAM case */
2453         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2454         ptr = qemu_get_ram_ptr(addr1);
2455         switch (endian) {
2456         case DEVICE_LITTLE_ENDIAN:
2457             stl_le_p(ptr, val);
2458             break;
2459         case DEVICE_BIG_ENDIAN:
2460             stl_be_p(ptr, val);
2461             break;
2462         default:
2463             stl_p(ptr, val);
2464             break;
2465         }
2466         invalidate_and_set_dirty(addr1, 4);
2467     }
2468 }
2469
2470 void stl_phys(hwaddr addr, uint32_t val)
2471 {
2472     stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2473 }
2474
2475 void stl_le_phys(hwaddr addr, uint32_t val)
2476 {
2477     stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2478 }
2479
2480 void stl_be_phys(hwaddr addr, uint32_t val)
2481 {
2482     stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2483 }
2484
2485 /* XXX: optimize */
2486 void stb_phys(hwaddr addr, uint32_t val)
2487 {
2488     uint8_t v = val;
2489     cpu_physical_memory_write(addr, &v, 1);
2490 }
2491
2492 /* warning: addr must be aligned */
2493 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2494                                      enum device_endian endian)
2495 {
2496     uint8_t *ptr;
2497     MemoryRegion *mr;
2498     hwaddr l = 2;
2499     hwaddr addr1;
2500
2501     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2502                                  true);
2503     if (l < 2 || !memory_access_is_direct(mr, true)) {
2504 #if defined(TARGET_WORDS_BIGENDIAN)
2505         if (endian == DEVICE_LITTLE_ENDIAN) {
2506             val = bswap16(val);
2507         }
2508 #else
2509         if (endian == DEVICE_BIG_ENDIAN) {
2510             val = bswap16(val);
2511         }
2512 #endif
2513         io_mem_write(mr, addr1, val, 2);
2514     } else {
2515         /* RAM case */
2516         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2517         ptr = qemu_get_ram_ptr(addr1);
2518         switch (endian) {
2519         case DEVICE_LITTLE_ENDIAN:
2520             stw_le_p(ptr, val);
2521             break;
2522         case DEVICE_BIG_ENDIAN:
2523             stw_be_p(ptr, val);
2524             break;
2525         default:
2526             stw_p(ptr, val);
2527             break;
2528         }
2529         invalidate_and_set_dirty(addr1, 2);
2530     }
2531 }
2532
2533 void stw_phys(hwaddr addr, uint32_t val)
2534 {
2535     stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2536 }
2537
2538 void stw_le_phys(hwaddr addr, uint32_t val)
2539 {
2540     stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2541 }
2542
2543 void stw_be_phys(hwaddr addr, uint32_t val)
2544 {
2545     stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2546 }
2547
2548 /* XXX: optimize */
2549 void stq_phys(hwaddr addr, uint64_t val)
2550 {
2551     val = tswap64(val);
2552     cpu_physical_memory_write(addr, &val, 8);
2553 }
2554
2555 void stq_le_phys(hwaddr addr, uint64_t val)
2556 {
2557     val = cpu_to_le64(val);
2558     cpu_physical_memory_write(addr, &val, 8);
2559 }
2560
2561 void stq_be_phys(hwaddr addr, uint64_t val)
2562 {
2563     val = cpu_to_be64(val);
2564     cpu_physical_memory_write(addr, &val, 8);
2565 }
2566
2567 /* virtual memory access for debug (includes writing to ROM) */
2568 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2569                         uint8_t *buf, int len, int is_write)
2570 {
2571     int l;
2572     hwaddr phys_addr;
2573     target_ulong page;
2574
2575     while (len > 0) {
2576         page = addr & TARGET_PAGE_MASK;
2577         phys_addr = cpu_get_phys_page_debug(env, page);
2578         /* if no physical page mapped, return an error */
2579         if (phys_addr == -1)
2580             return -1;
2581         l = (page + TARGET_PAGE_SIZE) - addr;
2582         if (l > len)
2583             l = len;
2584         phys_addr += (addr & ~TARGET_PAGE_MASK);
2585         if (is_write)
2586             cpu_physical_memory_write_rom(phys_addr, buf, l);
2587         else
2588             cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2589         len -= l;
2590         buf += l;
2591         addr += l;
2592     }
2593     return 0;
2594 }
2595 #endif
2596
2597 #if !defined(CONFIG_USER_ONLY)
2598
2599 /*
2600  * A helper function for the _utterly broken_ virtio device model to find out if
2601  * it's running on a big endian machine. Don't do this at home kids!
2602  */
2603 bool virtio_is_big_endian(void);
2604 bool virtio_is_big_endian(void)
2605 {
2606 #if defined(TARGET_WORDS_BIGENDIAN)
2607     return true;
2608 #else
2609     return false;
2610 #endif
2611 }
2612
2613 #endif
2614
2615 #ifndef CONFIG_USER_ONLY
2616 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2617 {
2618     MemoryRegion*mr;
2619     hwaddr l = 1;
2620
2621     mr = address_space_translate(&address_space_memory,
2622                                  phys_addr, &phys_addr, &l, false);
2623
2624     return !(memory_region_is_ram(mr) ||
2625              memory_region_is_romd(mr));
2626 }
2627
2628 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2629 {
2630     RAMBlock *block;
2631
2632     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2633         func(block->host, block->offset, block->length, opaque);
2634     }
2635 }
2636 #endif