exec.c

   1 /*
   2  *  Virtual page mapping
   3  *
   4  *  Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19 #include "config.h"
  20 #ifdef _WIN32
  21 #include <windows.h>
  22 #else
  23 #include <sys/types.h>
  24 #include <sys/mman.h>
  25 #endif
  26
  27 #include "qemu-common.h"
  28 #include "cpu.h"
  29 #include "tcg.h"
  30 #include "hw/hw.h"
  31 #include "hw/qdev.h"
  32 #include "qemu/osdep.h"
  33 #include "sysemu/kvm.h"
  34 #include "hw/xen/xen.h"
  35 #include "qemu/timer.h"
  36 #include "qemu/config-file.h"
  37 #include "exec/memory.h"
  38 #include "sysemu/dma.h"
  39 #include "exec/address-spaces.h"
  40 #if defined(CONFIG_USER_ONLY)
  41 #include <qemu.h>
  42 #else /* !CONFIG_USER_ONLY */
  43 #include "sysemu/xen-mapcache.h"
  44 #include "trace.h"
  45 #endif
  46 #include "exec/cpu-all.h"
  47
  48 #include "exec/cputlb.h"
  49 #include "translate-all.h"
  50
  51 #include "exec/memory-internal.h"
  52
  53 //#define DEBUG_SUBPAGE
  54
  55 #if !defined(CONFIG_USER_ONLY)
  56 int phys_ram_fd;
  57 static int in_migration;
  58
  59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
  60
  61 static MemoryRegion *system_memory;
  62 static MemoryRegion *system_io;
  63
  64 AddressSpace address_space_io;
  65 AddressSpace address_space_memory;
  66
  67 MemoryRegion io_mem_rom, io_mem_notdirty;
  68 static MemoryRegion io_mem_unassigned;
  69
  70 #endif
  71
  72 CPUArchState *first_cpu;
  73 /* current CPU in the current thread. It is only valid inside
  74    cpu_exec() */
  75 DEFINE_TLS(CPUState *, current_cpu);
  76 /* 0 = Do not count executed instructions.
  77    1 = Precise instruction counting.
  78    2 = Adaptive rate instruction counting.  */
  79 int use_icount;
  80
  81 #if !defined(CONFIG_USER_ONLY)
  82
  83 typedef struct PhysPageEntry PhysPageEntry;
  84
  85 struct PhysPageEntry {
  86     uint16_t is_leaf : 1;
  87      /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
  88     uint16_t ptr : 15;
  89 };
  90
  91 typedef PhysPageEntry Node[L2_SIZE];
  92
  93 struct AddressSpaceDispatch {
  94     /* This is a multi-level map on the physical address space.
  95      * The bottom level has pointers to MemoryRegionSections.
  96      */
  97     PhysPageEntry phys_map;
  98     Node *nodes;
  99     MemoryRegionSection *sections;
 100     AddressSpace *as;
 101 };
 102
 103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
 104 typedef struct subpage_t {
 105     MemoryRegion iomem;
 106     AddressSpace *as;
 107     hwaddr base;
 108     uint16_t sub_section[TARGET_PAGE_SIZE];
 109 } subpage_t;
 110
 111 #define PHYS_SECTION_UNASSIGNED 0
 112 #define PHYS_SECTION_NOTDIRTY 1
 113 #define PHYS_SECTION_ROM 2
 114 #define PHYS_SECTION_WATCH 3
 115
 116 typedef struct PhysPageMap {
 117     unsigned sections_nb;
 118     unsigned sections_nb_alloc;
 119     unsigned nodes_nb;
 120     unsigned nodes_nb_alloc;
 121     Node *nodes;
 122     MemoryRegionSection *sections;
 123 } PhysPageMap;
 124
 125 static PhysPageMap *prev_map;
 126 static PhysPageMap next_map;
 127
 128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
 129
 130 static void io_mem_init(void);
 131 static void memory_map_init(void);
 132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
 133
 134 static MemoryRegion io_mem_watch;
 135 #endif
 136
 137 #if !defined(CONFIG_USER_ONLY)
 138
 139 static void phys_map_node_reserve(unsigned nodes)
 140 {
 141     if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
 142         next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
 143                                             16);
 144         next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
 145                                       next_map.nodes_nb + nodes);
 146         next_map.nodes = g_renew(Node, next_map.nodes,
 147                                  next_map.nodes_nb_alloc);
 148     }
 149 }
 150
 151 static uint16_t phys_map_node_alloc(void)
 152 {
 153     unsigned i;
 154     uint16_t ret;
 155
 156     ret = next_map.nodes_nb++;
 157     assert(ret != PHYS_MAP_NODE_NIL);
 158     assert(ret != next_map.nodes_nb_alloc);
 159     for (i = 0; i < L2_SIZE; ++i) {
 160         next_map.nodes[ret][i].is_leaf = 0;
 161         next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
 162     }
 163     return ret;
 164 }
 165
 166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
 167                                 hwaddr *nb, uint16_t leaf,
 168                                 int level)
 169 {
 170     PhysPageEntry *p;
 171     int i;
 172     hwaddr step = (hwaddr)1 << (level * L2_BITS);
 173
 174     if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
 175         lp->ptr = phys_map_node_alloc();
 176         p = next_map.nodes[lp->ptr];
 177         if (level == 0) {
 178             for (i = 0; i < L2_SIZE; i++) {
 179                 p[i].is_leaf = 1;
 180                 p[i].ptr = PHYS_SECTION_UNASSIGNED;
 181             }
 182         }
 183     } else {
 184         p = next_map.nodes[lp->ptr];
 185     }
 186     lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
 187
 188     while (*nb && lp < &p[L2_SIZE]) {
 189         if ((*index & (step - 1)) == 0 && *nb >= step) {
 190             lp->is_leaf = true;
 191             lp->ptr = leaf;
 192             *index += step;
 193             *nb -= step;
 194         } else {
 195             phys_page_set_level(lp, index, nb, leaf, level - 1);
 196         }
 197         ++lp;
 198     }
 199 }
 200
 201 static void phys_page_set(AddressSpaceDispatch *d,
 202                           hwaddr index, hwaddr nb,
 203                           uint16_t leaf)
 204 {
 205     /* Wildly overreserve - it doesn't matter much. */
 206     phys_map_node_reserve(3 * P_L2_LEVELS);
 207
 208     phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
 209 }
 210
 211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
 212                                            Node *nodes, MemoryRegionSection *sections)
 213 {
 214     PhysPageEntry *p;
 215     int i;
 216
 217     for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
 218         if (lp.ptr == PHYS_MAP_NODE_NIL) {
 219             return &sections[PHYS_SECTION_UNASSIGNED];
 220         }
 221         p = nodes[lp.ptr];
 222         lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
 223     }
 224     return &sections[lp.ptr];
 225 }
 226
 227 bool memory_region_is_unassigned(MemoryRegion *mr)
 228 {
 229     return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
 230         && mr != &io_mem_watch;
 231 }
 232
 233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
 234                                                         hwaddr addr,
 235                                                         bool resolve_subpage)
 236 {
 237     MemoryRegionSection *section;
 238     subpage_t *subpage;
 239
 240     section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
 241                              d->nodes, d->sections);
 242     if (resolve_subpage && section->mr->subpage) {
 243         subpage = container_of(section->mr, subpage_t, iomem);
 244         section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
 245     }
 246     return section;
 247 }
 248
 249 static MemoryRegionSection *
 250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
 251                                  hwaddr *plen, bool resolve_subpage)
 252 {
 253     MemoryRegionSection *section;
 254     Int128 diff;
 255
 256     section = address_space_lookup_region(d, addr, resolve_subpage);
 257     /* Compute offset within MemoryRegionSection */
 258     addr -= section->offset_within_address_space;
 259
 260     /* Compute offset within MemoryRegion */
 261     *xlat = addr + section->offset_within_region;
 262
 263     diff = int128_sub(section->mr->size, int128_make64(addr));
 264     *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
 265     return section;
 266 }
 267
 268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
 269                                       hwaddr *xlat, hwaddr *plen,
 270                                       bool is_write)
 271 {
 272     IOMMUTLBEntry iotlb;
 273     MemoryRegionSection *section;
 274     MemoryRegion *mr;
 275     hwaddr len = *plen;
 276
 277     for (;;) {
 278         section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
 279         mr = section->mr;
 280
 281         if (!mr->iommu_ops) {
 282             break;
 283         }
 284
 285         iotlb = mr->iommu_ops->translate(mr, addr);
 286         addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
 287                 | (addr & iotlb.addr_mask));
 288         len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
 289         if (!(iotlb.perm & (1 << is_write))) {
 290             mr = &io_mem_unassigned;
 291             break;
 292         }
 293
 294         as = iotlb.target_as;
 295     }
 296
 297     *plen = len;
 298     *xlat = addr;
 299     return mr;
 300 }
 301
 302 MemoryRegionSection *
 303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
 304                                   hwaddr *plen)
 305 {
 306     MemoryRegionSection *section;
 307     section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
 308
 309     assert(!section->mr->iommu_ops);
 310     return section;
 311 }
 312 #endif
 313
 314 void cpu_exec_init_all(void)
 315 {
 316 #if !defined(CONFIG_USER_ONLY)
 317     qemu_mutex_init(&ram_list.mutex);
 318     memory_map_init();
 319     io_mem_init();
 320 #endif
 321 }
 322
 323 #if !defined(CONFIG_USER_ONLY)
 324
 325 static int cpu_common_post_load(void *opaque, int version_id)
 326 {
 327     CPUState *cpu = opaque;
 328
 329     /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
 330        version_id is increased. */
 331     cpu->interrupt_request &= ~0x01;
 332     tlb_flush(cpu->env_ptr, 1);
 333
 334     return 0;
 335 }
 336
 337 const VMStateDescription vmstate_cpu_common = {
 338     .name = "cpu_common",
 339     .version_id = 1,
 340     .minimum_version_id = 1,
 341     .minimum_version_id_old = 1,
 342     .post_load = cpu_common_post_load,
 343     .fields      = (VMStateField []) {
 344         VMSTATE_UINT32(halted, CPUState),
 345         VMSTATE_UINT32(interrupt_request, CPUState),
 346         VMSTATE_END_OF_LIST()
 347     }
 348 };
 349
 350 #endif
 351
 352 CPUState *qemu_get_cpu(int index)
 353 {
 354     CPUArchState *env = first_cpu;
 355     CPUState *cpu = NULL;
 356
 357     while (env) {
 358         cpu = ENV_GET_CPU(env);
 359         if (cpu->cpu_index == index) {
 360             break;
 361         }
 362         env = env->next_cpu;
 363     }
 364
 365     return env ? cpu : NULL;
 366 }
 367
 368 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
 369 {
 370     CPUArchState *env = first_cpu;
 371
 372     while (env) {
 373         func(ENV_GET_CPU(env), data);
 374         env = env->next_cpu;
 375     }
 376 }
 377
 378 void cpu_exec_init(CPUArchState *env)
 379 {
 380     CPUState *cpu = ENV_GET_CPU(env);
 381     CPUClass *cc = CPU_GET_CLASS(cpu);
 382     CPUArchState **penv;
 383     int cpu_index;
 384
 385 #if defined(CONFIG_USER_ONLY)
 386     cpu_list_lock();
 387 #endif
 388     env->next_cpu = NULL;
 389     penv = &first_cpu;
 390     cpu_index = 0;
 391     while (*penv != NULL) {
 392         penv = &(*penv)->next_cpu;
 393         cpu_index++;
 394     }
 395     cpu->cpu_index = cpu_index;
 396     cpu->numa_node = 0;
 397     QTAILQ_INIT(&env->breakpoints);
 398     QTAILQ_INIT(&env->watchpoints);
 399 #ifndef CONFIG_USER_ONLY
 400     cpu->thread_id = qemu_get_thread_id();
 401 #endif
 402     *penv = env;
 403 #if defined(CONFIG_USER_ONLY)
 404     cpu_list_unlock();
 405 #endif
 406     vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
 407 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
 408     register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
 409                     cpu_save, cpu_load, env);
 410     assert(cc->vmsd == NULL);
 411 #endif
 412     if (cc->vmsd != NULL) {
 413         vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
 414     }
 415 }
 416
 417 #if defined(TARGET_HAS_ICE)
 418 #if defined(CONFIG_USER_ONLY)
 419 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
 420 {
 421     tb_invalidate_phys_page_range(pc, pc + 1, 0);
 422 }
 423 #else
 424 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
 425 {
 426     tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
 427             (pc & ~TARGET_PAGE_MASK));
 428 }
 429 #endif
 430 #endif /* TARGET_HAS_ICE */
 431
 432 #if defined(CONFIG_USER_ONLY)
 433 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 434
 435 {
 436 }
 437
 438 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 439                           int flags, CPUWatchpoint **watchpoint)
 440 {
 441     return -ENOSYS;
 442 }
 443 #else
 444 /* Add a watchpoint.  */
 445 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 446                           int flags, CPUWatchpoint **watchpoint)
 447 {
 448     target_ulong len_mask = ~(len - 1);
 449     CPUWatchpoint *wp;
 450
 451     /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
 452     if ((len & (len - 1)) || (addr & ~len_mask) ||
 453             len == 0 || len > TARGET_PAGE_SIZE) {
 454         fprintf(stderr, "qemu: tried to set invalid watchpoint at "
 455                 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
 456         return -EINVAL;
 457     }
 458     wp = g_malloc(sizeof(*wp));
 459
 460     wp->vaddr = addr;
 461     wp->len_mask = len_mask;
 462     wp->flags = flags;
 463
 464     /* keep all GDB-injected watchpoints in front */
 465     if (flags & BP_GDB)
 466         QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
 467     else
 468         QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
 469
 470     tlb_flush_page(env, addr);
 471
 472     if (watchpoint)
 473         *watchpoint = wp;
 474     return 0;
 475 }
 476
 477 /* Remove a specific watchpoint.  */
 478 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
 479                           int flags)
 480 {
 481     target_ulong len_mask = ~(len - 1);
 482     CPUWatchpoint *wp;
 483
 484     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 485         if (addr == wp->vaddr && len_mask == wp->len_mask
 486                 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
 487             cpu_watchpoint_remove_by_ref(env, wp);
 488             return 0;
 489         }
 490     }
 491     return -ENOENT;
 492 }
 493
 494 /* Remove a specific watchpoint by reference.  */
 495 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
 496 {
 497     QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
 498
 499     tlb_flush_page(env, watchpoint->vaddr);
 500
 501     g_free(watchpoint);
 502 }
 503
 504 /* Remove all matching watchpoints.  */
 505 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 506 {
 507     CPUWatchpoint *wp, *next;
 508
 509     QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
 510         if (wp->flags & mask)
 511             cpu_watchpoint_remove_by_ref(env, wp);
 512     }
 513 }
 514 #endif
 515
 516 /* Add a breakpoint.  */
 517 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
 518                           CPUBreakpoint **breakpoint)
 519 {
 520 #if defined(TARGET_HAS_ICE)
 521     CPUBreakpoint *bp;
 522
 523     bp = g_malloc(sizeof(*bp));
 524
 525     bp->pc = pc;
 526     bp->flags = flags;
 527
 528     /* keep all GDB-injected breakpoints in front */
 529     if (flags & BP_GDB)
 530         QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
 531     else
 532         QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
 533
 534     breakpoint_invalidate(env, pc);
 535
 536     if (breakpoint)
 537         *breakpoint = bp;
 538     return 0;
 539 #else
 540     return -ENOSYS;
 541 #endif
 542 }
 543
 544 /* Remove a specific breakpoint.  */
 545 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
 546 {
 547 #if defined(TARGET_HAS_ICE)
 548     CPUBreakpoint *bp;
 549
 550     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
 551         if (bp->pc == pc && bp->flags == flags) {
 552             cpu_breakpoint_remove_by_ref(env, bp);
 553             return 0;
 554         }
 555     }
 556     return -ENOENT;
 557 #else
 558     return -ENOSYS;
 559 #endif
 560 }
 561
 562 /* Remove a specific breakpoint by reference.  */
 563 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
 564 {
 565 #if defined(TARGET_HAS_ICE)
 566     QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
 567
 568     breakpoint_invalidate(env, breakpoint->pc);
 569
 570     g_free(breakpoint);
 571 #endif
 572 }
 573
 574 /* Remove all matching breakpoints. */
 575 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
 576 {
 577 #if defined(TARGET_HAS_ICE)
 578     CPUBreakpoint *bp, *next;
 579
 580     QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
 581         if (bp->flags & mask)
 582             cpu_breakpoint_remove_by_ref(env, bp);
 583     }
 584 #endif
 585 }
 586
 587 /* enable or disable single step mode. EXCP_DEBUG is returned by the
 588    CPU loop after each instruction */
 589 void cpu_single_step(CPUArchState *env, int enabled)
 590 {
 591 #if defined(TARGET_HAS_ICE)
 592     if (env->singlestep_enabled != enabled) {
 593         env->singlestep_enabled = enabled;
 594         if (kvm_enabled())
 595             kvm_update_guest_debug(env, 0);
 596         else {
 597             /* must flush all the translated code to avoid inconsistencies */
 598             /* XXX: only flush what is necessary */
 599             tb_flush(env);
 600         }
 601     }
 602 #endif
 603 }
 604
 605 void cpu_abort(CPUArchState *env, const char *fmt, ...)
 606 {
 607     CPUState *cpu = ENV_GET_CPU(env);
 608     va_list ap;
 609     va_list ap2;
 610
 611     va_start(ap, fmt);
 612     va_copy(ap2, ap);
 613     fprintf(stderr, "qemu: fatal: ");
 614     vfprintf(stderr, fmt, ap);
 615     fprintf(stderr, "\n");
 616     cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 617     if (qemu_log_enabled()) {
 618         qemu_log("qemu: fatal: ");
 619         qemu_log_vprintf(fmt, ap2);
 620         qemu_log("\n");
 621         log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 622         qemu_log_flush();
 623         qemu_log_close();
 624     }
 625     va_end(ap2);
 626     va_end(ap);
 627 #if defined(CONFIG_USER_ONLY)
 628     {
 629         struct sigaction act;
 630         sigfillset(&act.sa_mask);
 631         act.sa_handler = SIG_DFL;
 632         sigaction(SIGABRT, &act, NULL);
 633     }
 634 #endif
 635     abort();
 636 }
 637
 638 CPUArchState *cpu_copy(CPUArchState *env)
 639 {
 640     CPUArchState *new_env = cpu_init(env->cpu_model_str);
 641     CPUArchState *next_cpu = new_env->next_cpu;
 642 #if defined(TARGET_HAS_ICE)
 643     CPUBreakpoint *bp;
 644     CPUWatchpoint *wp;
 645 #endif
 646
 647     memcpy(new_env, env, sizeof(CPUArchState));
 648
 649     /* Preserve chaining. */
 650     new_env->next_cpu = next_cpu;
 651
 652     /* Clone all break/watchpoints.
 653        Note: Once we support ptrace with hw-debug register access, make sure
 654        BP_CPU break/watchpoints are handled correctly on clone. */
 655     QTAILQ_INIT(&env->breakpoints);
 656     QTAILQ_INIT(&env->watchpoints);
 657 #if defined(TARGET_HAS_ICE)
 658     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
 659         cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
 660     }
 661     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 662         cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
 663                               wp->flags, NULL);
 664     }
 665 #endif
 666
 667     return new_env;
 668 }
 669
 670 #if !defined(CONFIG_USER_ONLY)
 671 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
 672                                       uintptr_t length)
 673 {
 674     uintptr_t start1;
 675
 676     /* we modify the TLB cache so that the dirty bit will be set again
 677        when accessing the range */
 678     start1 = (uintptr_t)qemu_safe_ram_ptr(start);
 679     /* Check that we don't span multiple blocks - this breaks the
 680        address comparisons below.  */
 681     if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
 682             != (end - 1) - start) {
 683         abort();
 684     }
 685     cpu_tlb_reset_dirty_all(start1, length);
 686
 687 }
 688
 689 /* Note: start and end must be within the same ram block.  */
 690 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
 691                                      int dirty_flags)
 692 {
 693     uintptr_t length;
 694
 695     start &= TARGET_PAGE_MASK;
 696     end = TARGET_PAGE_ALIGN(end);
 697
 698     length = end - start;
 699     if (length == 0)
 700         return;
 701     cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
 702
 703     if (tcg_enabled()) {
 704         tlb_reset_dirty_range_all(start, end, length);
 705     }
 706 }
 707
 708 static int cpu_physical_memory_set_dirty_tracking(int enable)
 709 {
 710     int ret = 0;
 711     in_migration = enable;
 712     return ret;
 713 }
 714
 715 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
 716                                        MemoryRegionSection *section,
 717                                        target_ulong vaddr,
 718                                        hwaddr paddr, hwaddr xlat,
 719                                        int prot,
 720                                        target_ulong *address)
 721 {
 722     hwaddr iotlb;
 723     CPUWatchpoint *wp;
 724
 725     if (memory_region_is_ram(section->mr)) {
 726         /* Normal RAM.  */
 727         iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
 728             + xlat;
 729         if (!section->readonly) {
 730             iotlb |= PHYS_SECTION_NOTDIRTY;
 731         } else {
 732             iotlb |= PHYS_SECTION_ROM;
 733         }
 734     } else {
 735         iotlb = section - address_space_memory.dispatch->sections;
 736         iotlb += xlat;
 737     }
 738
 739     /* Make accesses to pages with watchpoints go via the
 740        watchpoint trap routines.  */
 741     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 742         if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
 743             /* Avoid trapping reads of pages with a write breakpoint. */
 744             if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
 745                 iotlb = PHYS_SECTION_WATCH + paddr;
 746                 *address |= TLB_MMIO;
 747                 break;
 748             }
 749         }
 750     }
 751
 752     return iotlb;
 753 }
 754 #endif /* defined(CONFIG_USER_ONLY) */
 755
 756 #if !defined(CONFIG_USER_ONLY)
 757
 758 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
 759                              uint16_t section);
 760 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
 761
 762 static uint16_t phys_section_add(MemoryRegionSection *section)
 763 {
 764     /* The physical section number is ORed with a page-aligned
 765      * pointer to produce the iotlb entries.  Thus it should
 766      * never overflow into the page-aligned value.
 767      */
 768     assert(next_map.sections_nb < TARGET_PAGE_SIZE);
 769
 770     if (next_map.sections_nb == next_map.sections_nb_alloc) {
 771         next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
 772                                          16);
 773         next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
 774                                     next_map.sections_nb_alloc);
 775     }
 776     next_map.sections[next_map.sections_nb] = *section;
 777     memory_region_ref(section->mr);
 778     return next_map.sections_nb++;
 779 }
 780
 781 static void phys_section_destroy(MemoryRegion *mr)
 782 {
 783     memory_region_unref(mr);
 784
 785     if (mr->subpage) {
 786         subpage_t *subpage = container_of(mr, subpage_t, iomem);
 787         memory_region_destroy(&subpage->iomem);
 788         g_free(subpage);
 789     }
 790 }
 791
 792 static void phys_sections_free(PhysPageMap *map)
 793 {
 794     while (map->sections_nb > 0) {
 795         MemoryRegionSection *section = &map->sections[--map->sections_nb];
 796         phys_section_destroy(section->mr);
 797     }
 798     g_free(map->sections);
 799     g_free(map->nodes);
 800     g_free(map);
 801 }
 802
 803 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
 804 {
 805     subpage_t *subpage;
 806     hwaddr base = section->offset_within_address_space
 807         & TARGET_PAGE_MASK;
 808     MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
 809                                                    next_map.nodes, next_map.sections);
 810     MemoryRegionSection subsection = {
 811         .offset_within_address_space = base,
 812         .size = int128_make64(TARGET_PAGE_SIZE),
 813     };
 814     hwaddr start, end;
 815
 816     assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
 817
 818     if (!(existing->mr->subpage)) {
 819         subpage = subpage_init(d->as, base);
 820         subsection.mr = &subpage->iomem;
 821         phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
 822                       phys_section_add(&subsection));
 823     } else {
 824         subpage = container_of(existing->mr, subpage_t, iomem);
 825     }
 826     start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
 827     end = start + int128_get64(section->size) - 1;
 828     subpage_register(subpage, start, end, phys_section_add(section));
 829 }
 830
 831
 832 static void register_multipage(AddressSpaceDispatch *d,
 833                                MemoryRegionSection *section)
 834 {
 835     hwaddr start_addr = section->offset_within_address_space;
 836     uint16_t section_index = phys_section_add(section);
 837     uint64_t num_pages = int128_get64(int128_rshift(section->size,
 838                                                     TARGET_PAGE_BITS));
 839
 840     assert(num_pages);
 841     phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
 842 }
 843
 844 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
 845 {
 846     AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
 847     AddressSpaceDispatch *d = as->next_dispatch;
 848     MemoryRegionSection now = *section, remain = *section;
 849     Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
 850
 851     if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
 852         uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
 853                        - now.offset_within_address_space;
 854
 855         now.size = int128_min(int128_make64(left), now.size);
 856         register_subpage(d, &now);
 857     } else {
 858         now.size = int128_zero();
 859     }
 860     while (int128_ne(remain.size, now.size)) {
 861         remain.size = int128_sub(remain.size, now.size);
 862         remain.offset_within_address_space += int128_get64(now.size);
 863         remain.offset_within_region += int128_get64(now.size);
 864         now = remain;
 865         if (int128_lt(remain.size, page_size)) {
 866             register_subpage(d, &now);
 867         } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
 868             now.size = page_size;
 869             register_subpage(d, &now);
 870         } else {
 871             now.size = int128_and(now.size, int128_neg(page_size));
 872             register_multipage(d, &now);
 873         }
 874     }
 875 }
 876
 877 void qemu_flush_coalesced_mmio_buffer(void)
 878 {
 879     if (kvm_enabled())
 880         kvm_flush_coalesced_mmio_buffer();
 881 }
 882
 883 void qemu_mutex_lock_ramlist(void)
 884 {
 885     qemu_mutex_lock(&ram_list.mutex);
 886 }
 887
 888 void qemu_mutex_unlock_ramlist(void)
 889 {
 890     qemu_mutex_unlock(&ram_list.mutex);
 891 }
 892
 893 #if defined(__linux__) && !defined(TARGET_S390X)
 894
 895 #include <sys/vfs.h>
 896
 897 #define HUGETLBFS_MAGIC       0x958458f6
 898
 899 static long gethugepagesize(const char *path)
 900 {
 901     struct statfs fs;
 902     int ret;
 903
 904     do {
 905         ret = statfs(path, &fs);
 906     } while (ret != 0 && errno == EINTR);
 907
 908     if (ret != 0) {
 909         perror(path);
 910         return 0;
 911     }
 912
 913     if (fs.f_type != HUGETLBFS_MAGIC)
 914         fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
 915
 916     return fs.f_bsize;
 917 }
 918
 919 static void *file_ram_alloc(RAMBlock *block,
 920                             ram_addr_t memory,
 921                             const char *path)
 922 {
 923     char *filename;
 924     char *sanitized_name;
 925     char *c;
 926     void *area;
 927     int fd;
 928 #ifdef MAP_POPULATE
 929     int flags;
 930 #endif
 931     unsigned long hpagesize;
 932
 933     hpagesize = gethugepagesize(path);
 934     if (!hpagesize) {
 935         return NULL;
 936     }
 937
 938     if (memory < hpagesize) {
 939         return NULL;
 940     }
 941
 942     if (kvm_enabled() && !kvm_has_sync_mmu()) {
 943         fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
 944         return NULL;
 945     }
 946
 947     /* Make name safe to use with mkstemp by replacing '/' with '_'. */
 948     sanitized_name = g_strdup(block->mr->name);
 949     for (c = sanitized_name; *c != '\0'; c++) {
 950         if (*c == '/')
 951             *c = '_';
 952     }
 953
 954     filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
 955                                sanitized_name);
 956     g_free(sanitized_name);
 957
 958     fd = mkstemp(filename);
 959     if (fd < 0) {
 960         perror("unable to create backing store for hugepages");
 961         g_free(filename);
 962         return NULL;
 963     }
 964     unlink(filename);
 965     g_free(filename);
 966
 967     memory = (memory+hpagesize-1) & ~(hpagesize-1);
 968
 969     /*
 970      * ftruncate is not supported by hugetlbfs in older
 971      * hosts, so don't bother bailing out on errors.
 972      * If anything goes wrong with it under other filesystems,
 973      * mmap will fail.
 974      */
 975     if (ftruncate(fd, memory))
 976         perror("ftruncate");
 977
 978 #ifdef MAP_POPULATE
 979     /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
 980      * MAP_PRIVATE is requested.  For mem_prealloc we mmap as MAP_SHARED
 981      * to sidestep this quirk.
 982      */
 983     flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
 984     area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
 985 #else
 986     area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
 987 #endif
 988     if (area == MAP_FAILED) {
 989         perror("file_ram_alloc: can't mmap RAM pages");
 990         close(fd);
 991         return (NULL);
 992     }
 993     block->fd = fd;
 994     return area;
 995 }
 996 #endif
 997
 998 static ram_addr_t find_ram_offset(ram_addr_t size)
 999 {
1000     RAMBlock *block, *next_block;
1001     ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1002
1003     assert(size != 0); /* it would hand out same offset multiple times */
1004
1005     if (QTAILQ_EMPTY(&ram_list.blocks))
1006         return 0;
1007
1008     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1009         ram_addr_t end, next = RAM_ADDR_MAX;
1010
1011         end = block->offset + block->length;
1012
1013         QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1014             if (next_block->offset >= end) {
1015                 next = MIN(next, next_block->offset);
1016             }
1017         }
1018         if (next - end >= size && next - end < mingap) {
1019             offset = end;
1020             mingap = next - end;
1021         }
1022     }
1023
1024     if (offset == RAM_ADDR_MAX) {
1025         fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1026                 (uint64_t)size);
1027         abort();
1028     }
1029
1030     return offset;
1031 }
1032
1033 ram_addr_t last_ram_offset(void)
1034 {
1035     RAMBlock *block;
1036     ram_addr_t last = 0;
1037
1038     QTAILQ_FOREACH(block, &ram_list.blocks, next)
1039         last = MAX(last, block->offset + block->length);
1040
1041     return last;
1042 }
1043
1044 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1045 {
1046     int ret;
1047     QemuOpts *machine_opts;
1048
1049     /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1050     machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1051     if (machine_opts &&
1052         !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
1053         ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1054         if (ret) {
1055             perror("qemu_madvise");
1056             fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1057                             "but dump_guest_core=off specified\n");
1058         }
1059     }
1060 }
1061
1062 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1063 {
1064     RAMBlock *new_block, *block;
1065
1066     new_block = NULL;
1067     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1068         if (block->offset == addr) {
1069             new_block = block;
1070             break;
1071         }
1072     }
1073     assert(new_block);
1074     assert(!new_block->idstr[0]);
1075
1076     if (dev) {
1077         char *id = qdev_get_dev_path(dev);
1078         if (id) {
1079             snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1080             g_free(id);
1081         }
1082     }
1083     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1084
1085     /* This assumes the iothread lock is taken here too.  */
1086     qemu_mutex_lock_ramlist();
1087     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1088         if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1089             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1090                     new_block->idstr);
1091             abort();
1092         }
1093     }
1094     qemu_mutex_unlock_ramlist();
1095 }
1096
1097 static int memory_try_enable_merging(void *addr, size_t len)
1098 {
1099     QemuOpts *opts;
1100
1101     opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1102     if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
1103         /* disabled by the user */
1104         return 0;
1105     }
1106
1107     return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1108 }
1109
1110 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1111                                    MemoryRegion *mr)
1112 {
1113     RAMBlock *block, *new_block;
1114
1115     size = TARGET_PAGE_ALIGN(size);
1116     new_block = g_malloc0(sizeof(*new_block));
1117
1118     /* This assumes the iothread lock is taken here too.  */
1119     qemu_mutex_lock_ramlist();
1120     new_block->mr = mr;
1121     new_block->offset = find_ram_offset(size);
1122     if (host) {
1123         new_block->host = host;
1124         new_block->flags |= RAM_PREALLOC_MASK;
1125     } else {
1126         if (mem_path) {
1127 #if defined (__linux__) && !defined(TARGET_S390X)
1128             new_block->host = file_ram_alloc(new_block, size, mem_path);
1129             if (!new_block->host) {
1130                 new_block->host = qemu_anon_ram_alloc(size);
1131                 memory_try_enable_merging(new_block->host, size);
1132             }
1133 #else
1134             fprintf(stderr, "-mem-path option unsupported\n");
1135             exit(1);
1136 #endif
1137         } else {
1138             if (xen_enabled()) {
1139                 xen_ram_alloc(new_block->offset, size, mr);
1140             } else if (kvm_enabled()) {
1141                 /* some s390/kvm configurations have special constraints */
1142                 new_block->host = kvm_ram_alloc(size);
1143             } else {
1144                 new_block->host = qemu_anon_ram_alloc(size);
1145             }
1146             memory_try_enable_merging(new_block->host, size);
1147         }
1148     }
1149     new_block->length = size;
1150
1151     /* Keep the list sorted from biggest to smallest block.  */
1152     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1153         if (block->length < new_block->length) {
1154             break;
1155         }
1156     }
1157     if (block) {
1158         QTAILQ_INSERT_BEFORE(block, new_block, next);
1159     } else {
1160         QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1161     }
1162     ram_list.mru_block = NULL;
1163
1164     ram_list.version++;
1165     qemu_mutex_unlock_ramlist();
1166
1167     ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1168                                        last_ram_offset() >> TARGET_PAGE_BITS);
1169     memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1170            0, size >> TARGET_PAGE_BITS);
1171     cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1172
1173     qemu_ram_setup_dump(new_block->host, size);
1174     qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1175
1176     if (kvm_enabled())
1177         kvm_setup_guest_memory(new_block->host, size);
1178
1179     return new_block->offset;
1180 }
1181
1182 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1183 {
1184     return qemu_ram_alloc_from_ptr(size, NULL, mr);
1185 }
1186
1187 void qemu_ram_free_from_ptr(ram_addr_t addr)
1188 {
1189     RAMBlock *block;
1190
1191     /* This assumes the iothread lock is taken here too.  */
1192     qemu_mutex_lock_ramlist();
1193     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1194         if (addr == block->offset) {
1195             QTAILQ_REMOVE(&ram_list.blocks, block, next);
1196             ram_list.mru_block = NULL;
1197             ram_list.version++;
1198             g_free(block);
1199             break;
1200         }
1201     }
1202     qemu_mutex_unlock_ramlist();
1203 }
1204
1205 void qemu_ram_free(ram_addr_t addr)
1206 {
1207     RAMBlock *block;
1208
1209     /* This assumes the iothread lock is taken here too.  */
1210     qemu_mutex_lock_ramlist();
1211     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1212         if (addr == block->offset) {
1213             QTAILQ_REMOVE(&ram_list.blocks, block, next);
1214             ram_list.mru_block = NULL;
1215             ram_list.version++;
1216             if (block->flags & RAM_PREALLOC_MASK) {
1217                 ;
1218             } else if (mem_path) {
1219 #if defined (__linux__) && !defined(TARGET_S390X)
1220                 if (block->fd) {
1221                     munmap(block->host, block->length);
1222                     close(block->fd);
1223                 } else {
1224                     qemu_anon_ram_free(block->host, block->length);
1225                 }
1226 #else
1227                 abort();
1228 #endif
1229             } else {
1230                 if (xen_enabled()) {
1231                     xen_invalidate_map_cache_entry(block->host);
1232                 } else {
1233                     qemu_anon_ram_free(block->host, block->length);
1234                 }
1235             }
1236             g_free(block);
1237             break;
1238         }
1239     }
1240     qemu_mutex_unlock_ramlist();
1241
1242 }
1243
1244 #ifndef _WIN32
1245 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1246 {
1247     RAMBlock *block;
1248     ram_addr_t offset;
1249     int flags;
1250     void *area, *vaddr;
1251
1252     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1253         offset = addr - block->offset;
1254         if (offset < block->length) {
1255             vaddr = block->host + offset;
1256             if (block->flags & RAM_PREALLOC_MASK) {
1257                 ;
1258             } else {
1259                 flags = MAP_FIXED;
1260                 munmap(vaddr, length);
1261                 if (mem_path) {
1262 #if defined(__linux__) && !defined(TARGET_S390X)
1263                     if (block->fd) {
1264 #ifdef MAP_POPULATE
1265                         flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1266                             MAP_PRIVATE;
1267 #else
1268                         flags |= MAP_PRIVATE;
1269 #endif
1270                         area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1271                                     flags, block->fd, offset);
1272                     } else {
1273                         flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1274                         area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1275                                     flags, -1, 0);
1276                     }
1277 #else
1278                     abort();
1279 #endif
1280                 } else {
1281 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1282                     flags |= MAP_SHARED | MAP_ANONYMOUS;
1283                     area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1284                                 flags, -1, 0);
1285 #else
1286                     flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1287                     area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1288                                 flags, -1, 0);
1289 #endif
1290                 }
1291                 if (area != vaddr) {
1292                     fprintf(stderr, "Could not remap addr: "
1293                             RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1294                             length, addr);
1295                     exit(1);
1296                 }
1297                 memory_try_enable_merging(vaddr, length);
1298                 qemu_ram_setup_dump(vaddr, length);
1299             }
1300             return;
1301         }
1302     }
1303 }
1304 #endif /* !_WIN32 */
1305
1306 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1307 {
1308     RAMBlock *block;
1309
1310     /* The list is protected by the iothread lock here.  */
1311     block = ram_list.mru_block;
1312     if (block && addr - block->offset < block->length) {
1313         goto found;
1314     }
1315     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1316         if (addr - block->offset < block->length) {
1317             goto found;
1318         }
1319     }
1320
1321     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1322     abort();
1323
1324 found:
1325     ram_list.mru_block = block;
1326     return block;
1327 }
1328
1329 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1330    With the exception of the softmmu code in this file, this should
1331    only be used for local memory (e.g. video ram) that the device owns,
1332    and knows it isn't going to access beyond the end of the block.
1333
1334    It should not be used for general purpose DMA.
1335    Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1336  */
1337 void *qemu_get_ram_ptr(ram_addr_t addr)
1338 {
1339     RAMBlock *block = qemu_get_ram_block(addr);
1340
1341     if (xen_enabled()) {
1342         /* We need to check if the requested address is in the RAM
1343          * because we don't want to map the entire memory in QEMU.
1344          * In that case just map until the end of the page.
1345          */
1346         if (block->offset == 0) {
1347             return xen_map_cache(addr, 0, 0);
1348         } else if (block->host == NULL) {
1349             block->host =
1350                 xen_map_cache(block->offset, block->length, 1);
1351         }
1352     }
1353     return block->host + (addr - block->offset);
1354 }
1355
1356 /* Return a host pointer to ram allocated with qemu_ram_alloc.  Same as
1357  * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1358  *
1359  * ??? Is this still necessary?
1360  */
1361 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1362 {
1363     RAMBlock *block;
1364
1365     /* The list is protected by the iothread lock here.  */
1366     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1367         if (addr - block->offset < block->length) {
1368             if (xen_enabled()) {
1369                 /* We need to check if the requested address is in the RAM
1370                  * because we don't want to map the entire memory in QEMU.
1371                  * In that case just map until the end of the page.
1372                  */
1373                 if (block->offset == 0) {
1374                     return xen_map_cache(addr, 0, 0);
1375                 } else if (block->host == NULL) {
1376                     block->host =
1377                         xen_map_cache(block->offset, block->length, 1);
1378                 }
1379             }
1380             return block->host + (addr - block->offset);
1381         }
1382     }
1383
1384     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1385     abort();
1386
1387     return NULL;
1388 }
1389
1390 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1391  * but takes a size argument */
1392 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1393 {
1394     if (*size == 0) {
1395         return NULL;
1396     }
1397     if (xen_enabled()) {
1398         return xen_map_cache(addr, *size, 1);
1399     } else {
1400         RAMBlock *block;
1401
1402         QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1403             if (addr - block->offset < block->length) {
1404                 if (addr - block->offset + *size > block->length)
1405                     *size = block->length - addr + block->offset;
1406                 return block->host + (addr - block->offset);
1407             }
1408         }
1409
1410         fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1411         abort();
1412     }
1413 }
1414
1415 /* Some of the softmmu routines need to translate from a host pointer
1416    (typically a TLB entry) back to a ram offset.  */
1417 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1418 {
1419     RAMBlock *block;
1420     uint8_t *host = ptr;
1421
1422     if (xen_enabled()) {
1423         *ram_addr = xen_ram_addr_from_mapcache(ptr);
1424         return qemu_get_ram_block(*ram_addr)->mr;
1425     }
1426
1427     block = ram_list.mru_block;
1428     if (block && block->host && host - block->host < block->length) {
1429         goto found;
1430     }
1431
1432     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1433         /* This case append when the block is not mapped. */
1434         if (block->host == NULL) {
1435             continue;
1436         }
1437         if (host - block->host < block->length) {
1438             goto found;
1439         }
1440     }
1441
1442     return NULL;
1443
1444 found:
1445     *ram_addr = block->offset + (host - block->host);
1446     return block->mr;
1447 }
1448
1449 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1450                                uint64_t val, unsigned size)
1451 {
1452     int dirty_flags;
1453     dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1454     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1455         tb_invalidate_phys_page_fast(ram_addr, size);
1456         dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1457     }
1458     switch (size) {
1459     case 1:
1460         stb_p(qemu_get_ram_ptr(ram_addr), val);
1461         break;
1462     case 2:
1463         stw_p(qemu_get_ram_ptr(ram_addr), val);
1464         break;
1465     case 4:
1466         stl_p(qemu_get_ram_ptr(ram_addr), val);
1467         break;
1468     default:
1469         abort();
1470     }
1471     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1472     cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1473     /* we remove the notdirty callback only if the code has been
1474        flushed */
1475     if (dirty_flags == 0xff) {
1476         CPUArchState *env = current_cpu->env_ptr;
1477         tlb_set_dirty(env, env->mem_io_vaddr);
1478     }
1479 }
1480
1481 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1482                                  unsigned size, bool is_write)
1483 {
1484     return is_write;
1485 }
1486
1487 static const MemoryRegionOps notdirty_mem_ops = {
1488     .write = notdirty_mem_write,
1489     .valid.accepts = notdirty_mem_accepts,
1490     .endianness = DEVICE_NATIVE_ENDIAN,
1491 };
1492
1493 /* Generate a debug exception if a watchpoint has been hit.  */
1494 static void check_watchpoint(int offset, int len_mask, int flags)
1495 {
1496     CPUArchState *env = current_cpu->env_ptr;
1497     target_ulong pc, cs_base;
1498     target_ulong vaddr;
1499     CPUWatchpoint *wp;
1500     int cpu_flags;
1501
1502     if (env->watchpoint_hit) {
1503         /* We re-entered the check after replacing the TB. Now raise
1504          * the debug interrupt so that is will trigger after the
1505          * current instruction. */
1506         cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1507         return;
1508     }
1509     vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1510     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1511         if ((vaddr == (wp->vaddr & len_mask) ||
1512              (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1513             wp->flags |= BP_WATCHPOINT_HIT;
1514             if (!env->watchpoint_hit) {
1515                 env->watchpoint_hit = wp;
1516                 tb_check_watchpoint(env);
1517                 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1518                     env->exception_index = EXCP_DEBUG;
1519                     cpu_loop_exit(env);
1520                 } else {
1521                     cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1522                     tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1523                     cpu_resume_from_signal(env, NULL);
1524                 }
1525             }
1526         } else {
1527             wp->flags &= ~BP_WATCHPOINT_HIT;
1528         }
1529     }
1530 }
1531
1532 /* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
1533    so these check for a hit then pass through to the normal out-of-line
1534    phys routines.  */
1535 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1536                                unsigned size)
1537 {
1538     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1539     switch (size) {
1540     case 1: return ldub_phys(addr);
1541     case 2: return lduw_phys(addr);
1542     case 4: return ldl_phys(addr);
1543     default: abort();
1544     }
1545 }
1546
1547 static void watch_mem_write(void *opaque, hwaddr addr,
1548                             uint64_t val, unsigned size)
1549 {
1550     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1551     switch (size) {
1552     case 1:
1553         stb_phys(addr, val);
1554         break;
1555     case 2:
1556         stw_phys(addr, val);
1557         break;
1558     case 4:
1559         stl_phys(addr, val);
1560         break;
1561     default: abort();
1562     }
1563 }
1564
1565 static const MemoryRegionOps watch_mem_ops = {
1566     .read = watch_mem_read,
1567     .write = watch_mem_write,
1568     .endianness = DEVICE_NATIVE_ENDIAN,
1569 };
1570
1571 static uint64_t subpage_read(void *opaque, hwaddr addr,
1572                              unsigned len)
1573 {
1574     subpage_t *subpage = opaque;
1575     uint8_t buf[4];
1576
1577 #if defined(DEBUG_SUBPAGE)
1578     printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1579            subpage, len, addr);
1580 #endif
1581     address_space_read(subpage->as, addr + subpage->base, buf, len);
1582     switch (len) {
1583     case 1:
1584         return ldub_p(buf);
1585     case 2:
1586         return lduw_p(buf);
1587     case 4:
1588         return ldl_p(buf);
1589     default:
1590         abort();
1591     }
1592 }
1593
1594 static void subpage_write(void *opaque, hwaddr addr,
1595                           uint64_t value, unsigned len)
1596 {
1597     subpage_t *subpage = opaque;
1598     uint8_t buf[4];
1599
1600 #if defined(DEBUG_SUBPAGE)
1601     printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1602            " value %"PRIx64"\n",
1603            __func__, subpage, len, addr, value);
1604 #endif
1605     switch (len) {
1606     case 1:
1607         stb_p(buf, value);
1608         break;
1609     case 2:
1610         stw_p(buf, value);
1611         break;
1612     case 4:
1613         stl_p(buf, value);
1614         break;
1615     default:
1616         abort();
1617     }
1618     address_space_write(subpage->as, addr + subpage->base, buf, len);
1619 }
1620
1621 static bool subpage_accepts(void *opaque, hwaddr addr,
1622                             unsigned size, bool is_write)
1623 {
1624     subpage_t *subpage = opaque;
1625 #if defined(DEBUG_SUBPAGE)
1626     printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1627            __func__, subpage, is_write ? 'w' : 'r', len, addr);
1628 #endif
1629
1630     return address_space_access_valid(subpage->as, addr + subpage->base,
1631                                       size, is_write);
1632 }
1633
1634 static const MemoryRegionOps subpage_ops = {
1635     .read = subpage_read,
1636     .write = subpage_write,
1637     .valid.accepts = subpage_accepts,
1638     .endianness = DEVICE_NATIVE_ENDIAN,
1639 };
1640
1641 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1642                              uint16_t section)
1643 {
1644     int idx, eidx;
1645
1646     if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1647         return -1;
1648     idx = SUBPAGE_IDX(start);
1649     eidx = SUBPAGE_IDX(end);
1650 #if defined(DEBUG_SUBPAGE)
1651     printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1652            mmio, start, end, idx, eidx, memory);
1653 #endif
1654     for (; idx <= eidx; idx++) {
1655         mmio->sub_section[idx] = section;
1656     }
1657
1658     return 0;
1659 }
1660
1661 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1662 {
1663     subpage_t *mmio;
1664
1665     mmio = g_malloc0(sizeof(subpage_t));
1666
1667     mmio->as = as;
1668     mmio->base = base;
1669     memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1670                           "subpage", TARGET_PAGE_SIZE);
1671     mmio->iomem.subpage = true;
1672 #if defined(DEBUG_SUBPAGE)
1673     printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1674            mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1675 #endif
1676     subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1677
1678     return mmio;
1679 }
1680
1681 static uint16_t dummy_section(MemoryRegion *mr)
1682 {
1683     MemoryRegionSection section = {
1684         .mr = mr,
1685         .offset_within_address_space = 0,
1686         .offset_within_region = 0,
1687         .size = int128_2_64(),
1688     };
1689
1690     return phys_section_add(&section);
1691 }
1692
1693 MemoryRegion *iotlb_to_region(hwaddr index)
1694 {
1695     return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1696 }
1697
1698 static void io_mem_init(void)
1699 {
1700     memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1701     memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1702                           "unassigned", UINT64_MAX);
1703     memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1704                           "notdirty", UINT64_MAX);
1705     memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1706                           "watch", UINT64_MAX);
1707 }
1708
1709 static void mem_begin(MemoryListener *listener)
1710 {
1711     AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1712     AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1713
1714     d->phys_map  = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1715     d->as = as;
1716     as->next_dispatch = d;
1717 }
1718
1719 static void mem_commit(MemoryListener *listener)
1720 {
1721     AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1722     AddressSpaceDispatch *cur = as->dispatch;
1723     AddressSpaceDispatch *next = as->next_dispatch;
1724
1725     next->nodes = next_map.nodes;
1726     next->sections = next_map.sections;
1727
1728     as->dispatch = next;
1729     g_free(cur);
1730 }
1731
1732 static void core_begin(MemoryListener *listener)
1733 {
1734     uint16_t n;
1735
1736     prev_map = g_new(PhysPageMap, 1);
1737     *prev_map = next_map;
1738
1739     memset(&next_map, 0, sizeof(next_map));
1740     n = dummy_section(&io_mem_unassigned);
1741     assert(n == PHYS_SECTION_UNASSIGNED);
1742     n = dummy_section(&io_mem_notdirty);
1743     assert(n == PHYS_SECTION_NOTDIRTY);
1744     n = dummy_section(&io_mem_rom);
1745     assert(n == PHYS_SECTION_ROM);
1746     n = dummy_section(&io_mem_watch);
1747     assert(n == PHYS_SECTION_WATCH);
1748 }
1749
1750 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1751  * All AddressSpaceDispatch instances have switched to the next map.
1752  */
1753 static void core_commit(MemoryListener *listener)
1754 {
1755     phys_sections_free(prev_map);
1756 }
1757
1758 static void tcg_commit(MemoryListener *listener)
1759 {
1760     CPUArchState *env;
1761
1762     /* since each CPU stores ram addresses in its TLB cache, we must
1763        reset the modified entries */
1764     /* XXX: slow ! */
1765     for(env = first_cpu; env != NULL; env = env->next_cpu) {
1766         tlb_flush(env, 1);
1767     }
1768 }
1769
1770 static void core_log_global_start(MemoryListener *listener)
1771 {
1772     cpu_physical_memory_set_dirty_tracking(1);
1773 }
1774
1775 static void core_log_global_stop(MemoryListener *listener)
1776 {
1777     cpu_physical_memory_set_dirty_tracking(0);
1778 }
1779
1780 static MemoryListener core_memory_listener = {
1781     .begin = core_begin,
1782     .commit = core_commit,
1783     .log_global_start = core_log_global_start,
1784     .log_global_stop = core_log_global_stop,
1785     .priority = 1,
1786 };
1787
1788 static MemoryListener tcg_memory_listener = {
1789     .commit = tcg_commit,
1790 };
1791
1792 void address_space_init_dispatch(AddressSpace *as)
1793 {
1794     as->dispatch = NULL;
1795     as->dispatch_listener = (MemoryListener) {
1796         .begin = mem_begin,
1797         .commit = mem_commit,
1798         .region_add = mem_add,
1799         .region_nop = mem_add,
1800         .priority = 0,
1801     };
1802     memory_listener_register(&as->dispatch_listener, as);
1803 }
1804
1805 void address_space_destroy_dispatch(AddressSpace *as)
1806 {
1807     AddressSpaceDispatch *d = as->dispatch;
1808
1809     memory_listener_unregister(&as->dispatch_listener);
1810     g_free(d);
1811     as->dispatch = NULL;
1812 }
1813
1814 static void memory_map_init(void)
1815 {
1816     system_memory = g_malloc(sizeof(*system_memory));
1817     memory_region_init(system_memory, NULL, "system", INT64_MAX);
1818     address_space_init(&address_space_memory, system_memory, "memory");
1819
1820     system_io = g_malloc(sizeof(*system_io));
1821     memory_region_init(system_io, NULL, "io", 65536);
1822     address_space_init(&address_space_io, system_io, "I/O");
1823
1824     memory_listener_register(&core_memory_listener, &address_space_memory);
1825     memory_listener_register(&tcg_memory_listener, &address_space_memory);
1826 }
1827
1828 MemoryRegion *get_system_memory(void)
1829 {
1830     return system_memory;
1831 }
1832
1833 MemoryRegion *get_system_io(void)
1834 {
1835     return system_io;
1836 }
1837
1838 #endif /* !defined(CONFIG_USER_ONLY) */
1839
1840 /* physical memory access (slow version, mainly for debug) */
1841 #if defined(CONFIG_USER_ONLY)
1842 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1843                         uint8_t *buf, int len, int is_write)
1844 {
1845     int l, flags;
1846     target_ulong page;
1847     void * p;
1848
1849     while (len > 0) {
1850         page = addr & TARGET_PAGE_MASK;
1851         l = (page + TARGET_PAGE_SIZE) - addr;
1852         if (l > len)
1853             l = len;
1854         flags = page_get_flags(page);
1855         if (!(flags & PAGE_VALID))
1856             return -1;
1857         if (is_write) {
1858             if (!(flags & PAGE_WRITE))
1859                 return -1;
1860             /* XXX: this code should not depend on lock_user */
1861             if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1862                 return -1;
1863             memcpy(p, buf, l);
1864             unlock_user(p, addr, l);
1865         } else {
1866             if (!(flags & PAGE_READ))
1867                 return -1;
1868             /* XXX: this code should not depend on lock_user */
1869             if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1870                 return -1;
1871             memcpy(buf, p, l);
1872             unlock_user(p, addr, 0);
1873         }
1874         len -= l;
1875         buf += l;
1876         addr += l;
1877     }
1878     return 0;
1879 }
1880
1881 #else
1882
1883 static void invalidate_and_set_dirty(hwaddr addr,
1884                                      hwaddr length)
1885 {
1886     if (!cpu_physical_memory_is_dirty(addr)) {
1887         /* invalidate code */
1888         tb_invalidate_phys_page_range(addr, addr + length, 0);
1889         /* set dirty bit */
1890         cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1891     }
1892     xen_modified_memory(addr, length);
1893 }
1894
1895 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1896 {
1897     if (memory_region_is_ram(mr)) {
1898         return !(is_write && mr->readonly);
1899     }
1900     if (memory_region_is_romd(mr)) {
1901         return !is_write;
1902     }
1903
1904     return false;
1905 }
1906
1907 static inline int memory_access_size(MemoryRegion *mr, int l, hwaddr addr)
1908 {
1909     if (l >= 4 && (((addr & 3) == 0 || mr->ops->impl.unaligned))) {
1910         return 4;
1911     }
1912     if (l >= 2 && (((addr & 1) == 0) || mr->ops->impl.unaligned)) {
1913         return 2;
1914     }
1915     return 1;
1916 }
1917
1918 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1919                       int len, bool is_write)
1920 {
1921     hwaddr l;
1922     uint8_t *ptr;
1923     uint64_t val;
1924     hwaddr addr1;
1925     MemoryRegion *mr;
1926     bool error = false;
1927
1928     while (len > 0) {
1929         l = len;
1930         mr = address_space_translate(as, addr, &addr1, &l, is_write);
1931
1932         if (is_write) {
1933             if (!memory_access_is_direct(mr, is_write)) {
1934                 l = memory_access_size(mr, l, addr1);
1935                 /* XXX: could force current_cpu to NULL to avoid
1936                    potential bugs */
1937                 if (l == 4) {
1938                     /* 32 bit write access */
1939                     val = ldl_p(buf);
1940                     error |= io_mem_write(mr, addr1, val, 4);
1941                 } else if (l == 2) {
1942                     /* 16 bit write access */
1943                     val = lduw_p(buf);
1944                     error |= io_mem_write(mr, addr1, val, 2);
1945                 } else {
1946                     /* 8 bit write access */
1947                     val = ldub_p(buf);
1948                     error |= io_mem_write(mr, addr1, val, 1);
1949                 }
1950             } else {
1951                 addr1 += memory_region_get_ram_addr(mr);
1952                 /* RAM case */
1953                 ptr = qemu_get_ram_ptr(addr1);
1954                 memcpy(ptr, buf, l);
1955                 invalidate_and_set_dirty(addr1, l);
1956             }
1957         } else {
1958             if (!memory_access_is_direct(mr, is_write)) {
1959                 /* I/O case */
1960                 l = memory_access_size(mr, l, addr1);
1961                 if (l == 4) {
1962                     /* 32 bit read access */
1963                     error |= io_mem_read(mr, addr1, &val, 4);
1964                     stl_p(buf, val);
1965                 } else if (l == 2) {
1966                     /* 16 bit read access */
1967                     error |= io_mem_read(mr, addr1, &val, 2);
1968                     stw_p(buf, val);
1969                 } else {
1970                     /* 8 bit read access */
1971                     error |= io_mem_read(mr, addr1, &val, 1);
1972                     stb_p(buf, val);
1973                 }
1974             } else {
1975                 /* RAM case */
1976                 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1977                 memcpy(buf, ptr, l);
1978             }
1979         }
1980         len -= l;
1981         buf += l;
1982         addr += l;
1983     }
1984
1985     return error;
1986 }
1987
1988 bool address_space_write(AddressSpace *as, hwaddr addr,
1989                          const uint8_t *buf, int len)
1990 {
1991     return address_space_rw(as, addr, (uint8_t *)buf, len, true);
1992 }
1993
1994 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1995 {
1996     return address_space_rw(as, addr, buf, len, false);
1997 }
1998
1999
2000 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2001                             int len, int is_write)
2002 {
2003     address_space_rw(&address_space_memory, addr, buf, len, is_write);
2004 }
2005
2006 /* used for ROM loading : can write in RAM and ROM */
2007 void cpu_physical_memory_write_rom(hwaddr addr,
2008                                    const uint8_t *buf, int len)
2009 {
2010     hwaddr l;
2011     uint8_t *ptr;
2012     hwaddr addr1;
2013     MemoryRegion *mr;
2014
2015     while (len > 0) {
2016         l = len;
2017         mr = address_space_translate(&address_space_memory,
2018                                      addr, &addr1, &l, true);
2019
2020         if (!(memory_region_is_ram(mr) ||
2021               memory_region_is_romd(mr))) {
2022             /* do nothing */
2023         } else {
2024             addr1 += memory_region_get_ram_addr(mr);
2025             /* ROM/RAM case */
2026             ptr = qemu_get_ram_ptr(addr1);
2027             memcpy(ptr, buf, l);
2028             invalidate_and_set_dirty(addr1, l);
2029         }
2030         len -= l;
2031         buf += l;
2032         addr += l;
2033     }
2034 }
2035
2036 typedef struct {
2037     MemoryRegion *mr;
2038     void *buffer;
2039     hwaddr addr;
2040     hwaddr len;
2041 } BounceBuffer;
2042
2043 static BounceBuffer bounce;
2044
2045 typedef struct MapClient {
2046     void *opaque;
2047     void (*callback)(void *opaque);
2048     QLIST_ENTRY(MapClient) link;
2049 } MapClient;
2050
2051 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2052     = QLIST_HEAD_INITIALIZER(map_client_list);
2053
2054 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2055 {
2056     MapClient *client = g_malloc(sizeof(*client));
2057
2058     client->opaque = opaque;
2059     client->callback = callback;
2060     QLIST_INSERT_HEAD(&map_client_list, client, link);
2061     return client;
2062 }
2063
2064 static void cpu_unregister_map_client(void *_client)
2065 {
2066     MapClient *client = (MapClient *)_client;
2067
2068     QLIST_REMOVE(client, link);
2069     g_free(client);
2070 }
2071
2072 static void cpu_notify_map_clients(void)
2073 {
2074     MapClient *client;
2075
2076     while (!QLIST_EMPTY(&map_client_list)) {
2077         client = QLIST_FIRST(&map_client_list);
2078         client->callback(client->opaque);
2079         cpu_unregister_map_client(client);
2080     }
2081 }
2082
2083 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2084 {
2085     MemoryRegion *mr;
2086     hwaddr l, xlat;
2087
2088     while (len > 0) {
2089         l = len;
2090         mr = address_space_translate(as, addr, &xlat, &l, is_write);
2091         if (!memory_access_is_direct(mr, is_write)) {
2092             l = memory_access_size(mr, l, addr);
2093             if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2094                 return false;
2095             }
2096         }
2097
2098         len -= l;
2099         addr += l;
2100     }
2101     return true;
2102 }
2103
2104 /* Map a physical memory region into a host virtual address.
2105  * May map a subset of the requested range, given by and returned in *plen.
2106  * May return NULL if resources needed to perform the mapping are exhausted.
2107  * Use only for reads OR writes - not for read-modify-write operations.
2108  * Use cpu_register_map_client() to know when retrying the map operation is
2109  * likely to succeed.
2110  */
2111 void *address_space_map(AddressSpace *as,
2112                         hwaddr addr,
2113                         hwaddr *plen,
2114                         bool is_write)
2115 {
2116     hwaddr len = *plen;
2117     hwaddr done = 0;
2118     hwaddr l, xlat, base;
2119     MemoryRegion *mr, *this_mr;
2120     ram_addr_t raddr;
2121
2122     if (len == 0) {
2123         return NULL;
2124     }
2125
2126     l = len;
2127     mr = address_space_translate(as, addr, &xlat, &l, is_write);
2128     if (!memory_access_is_direct(mr, is_write)) {
2129         if (bounce.buffer) {
2130             return NULL;
2131         }
2132         bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2133         bounce.addr = addr;
2134         bounce.len = l;
2135
2136         memory_region_ref(mr);
2137         bounce.mr = mr;
2138         if (!is_write) {
2139             address_space_read(as, addr, bounce.buffer, l);
2140         }
2141
2142         *plen = l;
2143         return bounce.buffer;
2144     }
2145
2146     base = xlat;
2147     raddr = memory_region_get_ram_addr(mr);
2148
2149     for (;;) {
2150         len -= l;
2151         addr += l;
2152         done += l;
2153         if (len == 0) {
2154             break;
2155         }
2156
2157         l = len;
2158         this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2159         if (this_mr != mr || xlat != base + done) {
2160             break;
2161         }
2162     }
2163
2164     memory_region_ref(mr);
2165     *plen = done;
2166     return qemu_ram_ptr_length(raddr + base, plen);
2167 }
2168
2169 /* Unmaps a memory region previously mapped by address_space_map().
2170  * Will also mark the memory as dirty if is_write == 1.  access_len gives
2171  * the amount of memory that was actually read or written by the caller.
2172  */
2173 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2174                          int is_write, hwaddr access_len)
2175 {
2176     if (buffer != bounce.buffer) {
2177         MemoryRegion *mr;
2178         ram_addr_t addr1;
2179
2180         mr = qemu_ram_addr_from_host(buffer, &addr1);
2181         assert(mr != NULL);
2182         if (is_write) {
2183             while (access_len) {
2184                 unsigned l;
2185                 l = TARGET_PAGE_SIZE;
2186                 if (l > access_len)
2187                     l = access_len;
2188                 invalidate_and_set_dirty(addr1, l);
2189                 addr1 += l;
2190                 access_len -= l;
2191             }
2192         }
2193         if (xen_enabled()) {
2194             xen_invalidate_map_cache_entry(buffer);
2195         }
2196         memory_region_unref(mr);
2197         return;
2198     }
2199     if (is_write) {
2200         address_space_write(as, bounce.addr, bounce.buffer, access_len);
2201     }
2202     qemu_vfree(bounce.buffer);
2203     bounce.buffer = NULL;
2204     memory_region_unref(bounce.mr);
2205     cpu_notify_map_clients();
2206 }
2207
2208 void *cpu_physical_memory_map(hwaddr addr,
2209                               hwaddr *plen,
2210                               int is_write)
2211 {
2212     return address_space_map(&address_space_memory, addr, plen, is_write);
2213 }
2214
2215 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2216                                int is_write, hwaddr access_len)
2217 {
2218     return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2219 }
2220
2221 /* warning: addr must be aligned */
2222 static inline uint32_t ldl_phys_internal(hwaddr addr,
2223                                          enum device_endian endian)
2224 {
2225     uint8_t *ptr;
2226     uint64_t val;
2227     MemoryRegion *mr;
2228     hwaddr l = 4;
2229     hwaddr addr1;
2230
2231     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2232                                  false);
2233     if (l < 4 || !memory_access_is_direct(mr, false)) {
2234         /* I/O case */
2235         io_mem_read(mr, addr1, &val, 4);
2236 #if defined(TARGET_WORDS_BIGENDIAN)
2237         if (endian == DEVICE_LITTLE_ENDIAN) {
2238             val = bswap32(val);
2239         }
2240 #else
2241         if (endian == DEVICE_BIG_ENDIAN) {
2242             val = bswap32(val);
2243         }
2244 #endif
2245     } else {
2246         /* RAM case */
2247         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2248                                 & TARGET_PAGE_MASK)
2249                                + addr1);
2250         switch (endian) {
2251         case DEVICE_LITTLE_ENDIAN:
2252             val = ldl_le_p(ptr);
2253             break;
2254         case DEVICE_BIG_ENDIAN:
2255             val = ldl_be_p(ptr);
2256             break;
2257         default:
2258             val = ldl_p(ptr);
2259             break;
2260         }
2261     }
2262     return val;
2263 }
2264
2265 uint32_t ldl_phys(hwaddr addr)
2266 {
2267     return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2268 }
2269
2270 uint32_t ldl_le_phys(hwaddr addr)
2271 {
2272     return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2273 }
2274
2275 uint32_t ldl_be_phys(hwaddr addr)
2276 {
2277     return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2278 }
2279
2280 /* warning: addr must be aligned */
2281 static inline uint64_t ldq_phys_internal(hwaddr addr,
2282                                          enum device_endian endian)
2283 {
2284     uint8_t *ptr;
2285     uint64_t val;
2286     MemoryRegion *mr;
2287     hwaddr l = 8;
2288     hwaddr addr1;
2289
2290     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2291                                  false);
2292     if (l < 8 || !memory_access_is_direct(mr, false)) {
2293         /* I/O case */
2294         io_mem_read(mr, addr1, &val, 8);
2295 #if defined(TARGET_WORDS_BIGENDIAN)
2296         if (endian == DEVICE_LITTLE_ENDIAN) {
2297             val = bswap64(val);
2298         }
2299 #else
2300         if (endian == DEVICE_BIG_ENDIAN) {
2301             val = bswap64(val);
2302         }
2303 #endif
2304     } else {
2305         /* RAM case */
2306         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2307                                 & TARGET_PAGE_MASK)
2308                                + addr1);
2309         switch (endian) {
2310         case DEVICE_LITTLE_ENDIAN:
2311             val = ldq_le_p(ptr);
2312             break;
2313         case DEVICE_BIG_ENDIAN:
2314             val = ldq_be_p(ptr);
2315             break;
2316         default:
2317             val = ldq_p(ptr);
2318             break;
2319         }
2320     }
2321     return val;
2322 }
2323
2324 uint64_t ldq_phys(hwaddr addr)
2325 {
2326     return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2327 }
2328
2329 uint64_t ldq_le_phys(hwaddr addr)
2330 {
2331     return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2332 }
2333
2334 uint64_t ldq_be_phys(hwaddr addr)
2335 {
2336     return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2337 }
2338
2339 /* XXX: optimize */
2340 uint32_t ldub_phys(hwaddr addr)
2341 {
2342     uint8_t val;
2343     cpu_physical_memory_read(addr, &val, 1);
2344     return val;
2345 }
2346
2347 /* warning: addr must be aligned */
2348 static inline uint32_t lduw_phys_internal(hwaddr addr,
2349                                           enum device_endian endian)
2350 {
2351     uint8_t *ptr;
2352     uint64_t val;
2353     MemoryRegion *mr;
2354     hwaddr l = 2;
2355     hwaddr addr1;
2356
2357     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2358                                  false);
2359     if (l < 2 || !memory_access_is_direct(mr, false)) {
2360         /* I/O case */
2361         io_mem_read(mr, addr1, &val, 2);
2362 #if defined(TARGET_WORDS_BIGENDIAN)
2363         if (endian == DEVICE_LITTLE_ENDIAN) {
2364             val = bswap16(val);
2365         }
2366 #else
2367         if (endian == DEVICE_BIG_ENDIAN) {
2368             val = bswap16(val);
2369         }
2370 #endif
2371     } else {
2372         /* RAM case */
2373         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2374                                 & TARGET_PAGE_MASK)
2375                                + addr1);
2376         switch (endian) {
2377         case DEVICE_LITTLE_ENDIAN:
2378             val = lduw_le_p(ptr);
2379             break;
2380         case DEVICE_BIG_ENDIAN:
2381             val = lduw_be_p(ptr);
2382             break;
2383         default:
2384             val = lduw_p(ptr);
2385             break;
2386         }
2387     }
2388     return val;
2389 }
2390
2391 uint32_t lduw_phys(hwaddr addr)
2392 {
2393     return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2394 }
2395
2396 uint32_t lduw_le_phys(hwaddr addr)
2397 {
2398     return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2399 }
2400
2401 uint32_t lduw_be_phys(hwaddr addr)
2402 {
2403     return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2404 }
2405
2406 /* warning: addr must be aligned. The ram page is not masked as dirty
2407    and the code inside is not invalidated. It is useful if the dirty
2408    bits are used to track modified PTEs */
2409 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2410 {
2411     uint8_t *ptr;
2412     MemoryRegion *mr;
2413     hwaddr l = 4;
2414     hwaddr addr1;
2415
2416     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2417                                  true);
2418     if (l < 4 || !memory_access_is_direct(mr, true)) {
2419         io_mem_write(mr, addr1, val, 4);
2420     } else {
2421         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2422         ptr = qemu_get_ram_ptr(addr1);
2423         stl_p(ptr, val);
2424
2425         if (unlikely(in_migration)) {
2426             if (!cpu_physical_memory_is_dirty(addr1)) {
2427                 /* invalidate code */
2428                 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2429                 /* set dirty bit */
2430                 cpu_physical_memory_set_dirty_flags(
2431                     addr1, (0xff & ~CODE_DIRTY_FLAG));
2432             }
2433         }
2434     }
2435 }
2436
2437 /* warning: addr must be aligned */
2438 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2439                                      enum device_endian endian)
2440 {
2441     uint8_t *ptr;
2442     MemoryRegion *mr;
2443     hwaddr l = 4;
2444     hwaddr addr1;
2445
2446     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2447                                  true);
2448     if (l < 4 || !memory_access_is_direct(mr, true)) {
2449 #if defined(TARGET_WORDS_BIGENDIAN)
2450         if (endian == DEVICE_LITTLE_ENDIAN) {
2451             val = bswap32(val);
2452         }
2453 #else
2454         if (endian == DEVICE_BIG_ENDIAN) {
2455             val = bswap32(val);
2456         }
2457 #endif
2458         io_mem_write(mr, addr1, val, 4);
2459     } else {
2460         /* RAM case */
2461         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2462         ptr = qemu_get_ram_ptr(addr1);
2463         switch (endian) {
2464         case DEVICE_LITTLE_ENDIAN:
2465             stl_le_p(ptr, val);
2466             break;
2467         case DEVICE_BIG_ENDIAN:
2468             stl_be_p(ptr, val);
2469             break;
2470         default:
2471             stl_p(ptr, val);
2472             break;
2473         }
2474         invalidate_and_set_dirty(addr1, 4);
2475     }
2476 }
2477
2478 void stl_phys(hwaddr addr, uint32_t val)
2479 {
2480     stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2481 }
2482
2483 void stl_le_phys(hwaddr addr, uint32_t val)
2484 {
2485     stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2486 }
2487
2488 void stl_be_phys(hwaddr addr, uint32_t val)
2489 {
2490     stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2491 }
2492
2493 /* XXX: optimize */
2494 void stb_phys(hwaddr addr, uint32_t val)
2495 {
2496     uint8_t v = val;
2497     cpu_physical_memory_write(addr, &v, 1);
2498 }
2499
2500 /* warning: addr must be aligned */
2501 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2502                                      enum device_endian endian)
2503 {
2504     uint8_t *ptr;
2505     MemoryRegion *mr;
2506     hwaddr l = 2;
2507     hwaddr addr1;
2508
2509     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2510                                  true);
2511     if (l < 2 || !memory_access_is_direct(mr, true)) {
2512 #if defined(TARGET_WORDS_BIGENDIAN)
2513         if (endian == DEVICE_LITTLE_ENDIAN) {
2514             val = bswap16(val);
2515         }
2516 #else
2517         if (endian == DEVICE_BIG_ENDIAN) {
2518             val = bswap16(val);
2519         }
2520 #endif
2521         io_mem_write(mr, addr1, val, 2);
2522     } else {
2523         /* RAM case */
2524         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2525         ptr = qemu_get_ram_ptr(addr1);
2526         switch (endian) {
2527         case DEVICE_LITTLE_ENDIAN:
2528             stw_le_p(ptr, val);
2529             break;
2530         case DEVICE_BIG_ENDIAN:
2531             stw_be_p(ptr, val);
2532             break;
2533         default:
2534             stw_p(ptr, val);
2535             break;
2536         }
2537         invalidate_and_set_dirty(addr1, 2);
2538     }
2539 }
2540
2541 void stw_phys(hwaddr addr, uint32_t val)
2542 {
2543     stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2544 }
2545
2546 void stw_le_phys(hwaddr addr, uint32_t val)
2547 {
2548     stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2549 }
2550
2551 void stw_be_phys(hwaddr addr, uint32_t val)
2552 {
2553     stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2554 }
2555
2556 /* XXX: optimize */
2557 void stq_phys(hwaddr addr, uint64_t val)
2558 {
2559     val = tswap64(val);
2560     cpu_physical_memory_write(addr, &val, 8);
2561 }
2562
2563 void stq_le_phys(hwaddr addr, uint64_t val)
2564 {
2565     val = cpu_to_le64(val);
2566     cpu_physical_memory_write(addr, &val, 8);
2567 }
2568
2569 void stq_be_phys(hwaddr addr, uint64_t val)
2570 {
2571     val = cpu_to_be64(val);
2572     cpu_physical_memory_write(addr, &val, 8);
2573 }
2574
2575 /* virtual memory access for debug (includes writing to ROM) */
2576 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2577                         uint8_t *buf, int len, int is_write)
2578 {
2579     int l;
2580     hwaddr phys_addr;
2581     target_ulong page;
2582
2583     while (len > 0) {
2584         page = addr & TARGET_PAGE_MASK;
2585         phys_addr = cpu_get_phys_page_debug(env, page);
2586         /* if no physical page mapped, return an error */
2587         if (phys_addr == -1)
2588             return -1;
2589         l = (page + TARGET_PAGE_SIZE) - addr;
2590         if (l > len)
2591             l = len;
2592         phys_addr += (addr & ~TARGET_PAGE_MASK);
2593         if (is_write)
2594             cpu_physical_memory_write_rom(phys_addr, buf, l);
2595         else
2596             cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2597         len -= l;
2598         buf += l;
2599         addr += l;
2600     }
2601     return 0;
2602 }
2603 #endif
2604
2605 #if !defined(CONFIG_USER_ONLY)
2606
2607 /*
2608  * A helper function for the _utterly broken_ virtio device model to find out if
2609  * it's running on a big endian machine. Don't do this at home kids!
2610  */
2611 bool virtio_is_big_endian(void);
2612 bool virtio_is_big_endian(void)
2613 {
2614 #if defined(TARGET_WORDS_BIGENDIAN)
2615     return true;
2616 #else
2617     return false;
2618 #endif
2619 }
2620
2621 #endif
2622
2623 #ifndef CONFIG_USER_ONLY
2624 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2625 {
2626     MemoryRegion*mr;
2627     hwaddr l = 1;
2628
2629     mr = address_space_translate(&address_space_memory,
2630                                  phys_addr, &phys_addr, &l, false);
2631
2632     return !(memory_region_is_ram(mr) ||
2633              memory_region_is_romd(mr));
2634 }
2635
2636 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2637 {
2638     RAMBlock *block;
2639
2640     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2641         func(block->host, block->offset, block->length, opaque);
2642     }
2643 }
2644 #endif