exec.c

   1 /*
   2  *  Virtual page mapping
   3  *
   4  *  Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19 #include "config.h"
  20 #ifdef _WIN32
  21 #include <windows.h>
  22 #else
  23 #include <sys/types.h>
  24 #include <sys/mman.h>
  25 #endif
  26
  27 #include "qemu-common.h"
  28 #include "cpu.h"
  29 #include "tcg.h"
  30 #include "hw/hw.h"
  31 #include "hw/qdev.h"
  32 #include "qemu/osdep.h"
  33 #include "sysemu/kvm.h"
  34 #include "hw/xen/xen.h"
  35 #include "qemu/timer.h"
  36 #include "qemu/config-file.h"
  37 #include "exec/memory.h"
  38 #include "sysemu/dma.h"
  39 #include "exec/address-spaces.h"
  40 #if defined(CONFIG_USER_ONLY)
  41 #include <qemu.h>
  42 #else /* !CONFIG_USER_ONLY */
  43 #include "sysemu/xen-mapcache.h"
  44 #include "trace.h"
  45 #endif
  46 #include "exec/cpu-all.h"
  47
  48 #include "exec/cputlb.h"
  49 #include "translate-all.h"
  50
  51 #include "exec/memory-internal.h"
  52
  53 //#define DEBUG_SUBPAGE
  54
  55 #if !defined(CONFIG_USER_ONLY)
  56 int phys_ram_fd;
  57 static int in_migration;
  58
  59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
  60
  61 static MemoryRegion *system_memory;
  62 static MemoryRegion *system_io;
  63
  64 AddressSpace address_space_io;
  65 AddressSpace address_space_memory;
  66
  67 MemoryRegion io_mem_rom, io_mem_notdirty;
  68 static MemoryRegion io_mem_unassigned;
  69
  70 #endif
  71
  72 CPUArchState *first_cpu;
  73 /* current CPU in the current thread. It is only valid inside
  74    cpu_exec() */
  75 DEFINE_TLS(CPUArchState *,cpu_single_env);
  76 /* 0 = Do not count executed instructions.
  77    1 = Precise instruction counting.
  78    2 = Adaptive rate instruction counting.  */
  79 int use_icount;
  80
  81 #if !defined(CONFIG_USER_ONLY)
  82
  83 typedef struct PhysPageEntry PhysPageEntry;
  84
  85 struct PhysPageEntry {
  86     uint16_t is_leaf : 1;
  87      /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
  88     uint16_t ptr : 15;
  89 };
  90
  91 struct AddressSpaceDispatch {
  92     /* This is a multi-level map on the physical address space.
  93      * The bottom level has pointers to MemoryRegionSections.
  94      */
  95     PhysPageEntry phys_map;
  96     MemoryListener listener;
  97     AddressSpace *as;
  98 };
  99
 100 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
 101 typedef struct subpage_t {
 102     MemoryRegion iomem;
 103     AddressSpace *as;
 104     hwaddr base;
 105     uint16_t sub_section[TARGET_PAGE_SIZE];
 106 } subpage_t;
 107
 108 #define PHYS_SECTION_UNASSIGNED 0
 109 #define PHYS_SECTION_NOTDIRTY 1
 110 #define PHYS_SECTION_ROM 2
 111 #define PHYS_SECTION_WATCH 3
 112
 113 typedef PhysPageEntry Node[L2_SIZE];
 114
 115 typedef struct PhysPageMap {
 116     unsigned sections_nb;
 117     unsigned sections_nb_alloc;
 118     unsigned nodes_nb;
 119     unsigned nodes_nb_alloc;
 120     Node *nodes;
 121     MemoryRegionSection *sections;
 122 } PhysPageMap;
 123
 124 static PhysPageMap cur_map;
 125 static PhysPageMap next_map;
 126
 127 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
 128
 129 static void io_mem_init(void);
 130 static void memory_map_init(void);
 131 static void *qemu_safe_ram_ptr(ram_addr_t addr);
 132
 133 static MemoryRegion io_mem_watch;
 134 #endif
 135
 136 #if !defined(CONFIG_USER_ONLY)
 137
 138 static void phys_map_node_reserve(unsigned nodes)
 139 {
 140     if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
 141         next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
 142                                             16);
 143         next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
 144                                       next_map.nodes_nb + nodes);
 145         next_map.nodes = g_renew(Node, next_map.nodes,
 146                                  next_map.nodes_nb_alloc);
 147     }
 148 }
 149
 150 static uint16_t phys_map_node_alloc(void)
 151 {
 152     unsigned i;
 153     uint16_t ret;
 154
 155     ret = next_map.nodes_nb++;
 156     assert(ret != PHYS_MAP_NODE_NIL);
 157     assert(ret != next_map.nodes_nb_alloc);
 158     for (i = 0; i < L2_SIZE; ++i) {
 159         next_map.nodes[ret][i].is_leaf = 0;
 160         next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
 161     }
 162     return ret;
 163 }
 164
 165 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
 166                                 hwaddr *nb, uint16_t leaf,
 167                                 int level)
 168 {
 169     PhysPageEntry *p;
 170     int i;
 171     hwaddr step = (hwaddr)1 << (level * L2_BITS);
 172
 173     if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
 174         lp->ptr = phys_map_node_alloc();
 175         p = next_map.nodes[lp->ptr];
 176         if (level == 0) {
 177             for (i = 0; i < L2_SIZE; i++) {
 178                 p[i].is_leaf = 1;
 179                 p[i].ptr = PHYS_SECTION_UNASSIGNED;
 180             }
 181         }
 182     } else {
 183         p = next_map.nodes[lp->ptr];
 184     }
 185     lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
 186
 187     while (*nb && lp < &p[L2_SIZE]) {
 188         if ((*index & (step - 1)) == 0 && *nb >= step) {
 189             lp->is_leaf = true;
 190             lp->ptr = leaf;
 191             *index += step;
 192             *nb -= step;
 193         } else {
 194             phys_page_set_level(lp, index, nb, leaf, level - 1);
 195         }
 196         ++lp;
 197     }
 198 }
 199
 200 static void phys_page_set(AddressSpaceDispatch *d,
 201                           hwaddr index, hwaddr nb,
 202                           uint16_t leaf)
 203 {
 204     /* Wildly overreserve - it doesn't matter much. */
 205     phys_map_node_reserve(3 * P_L2_LEVELS);
 206
 207     phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
 208 }
 209
 210 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
 211                                            Node *nodes, MemoryRegionSection *sections)
 212 {
 213     PhysPageEntry *p;
 214     int i;
 215
 216     for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
 217         if (lp.ptr == PHYS_MAP_NODE_NIL) {
 218             return &sections[PHYS_SECTION_UNASSIGNED];
 219         }
 220         p = nodes[lp.ptr];
 221         lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
 222     }
 223     return &sections[lp.ptr];
 224 }
 225
 226 bool memory_region_is_unassigned(MemoryRegion *mr)
 227 {
 228     return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
 229         && mr != &io_mem_watch;
 230 }
 231
 232 static MemoryRegionSection *address_space_lookup_region(AddressSpace *as,
 233                                                         hwaddr addr,
 234                                                         bool resolve_subpage)
 235 {
 236     MemoryRegionSection *section;
 237     subpage_t *subpage;
 238
 239     section = phys_page_find(as->dispatch->phys_map, addr >> TARGET_PAGE_BITS,
 240                              cur_map.nodes, cur_map.sections);
 241     if (resolve_subpage && section->mr->subpage) {
 242         subpage = container_of(section->mr, subpage_t, iomem);
 243         section = &cur_map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
 244     }
 245     return section;
 246 }
 247
 248 static MemoryRegionSection *
 249 address_space_translate_internal(AddressSpace *as, hwaddr addr, hwaddr *xlat,
 250                                  hwaddr *plen, bool resolve_subpage)
 251 {
 252     MemoryRegionSection *section;
 253     Int128 diff;
 254
 255     section = address_space_lookup_region(as, addr, resolve_subpage);
 256     /* Compute offset within MemoryRegionSection */
 257     addr -= section->offset_within_address_space;
 258
 259     /* Compute offset within MemoryRegion */
 260     *xlat = addr + section->offset_within_region;
 261
 262     diff = int128_sub(section->mr->size, int128_make64(addr));
 263     *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
 264     return section;
 265 }
 266
 267 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
 268                                       hwaddr *xlat, hwaddr *plen,
 269                                       bool is_write)
 270 {
 271     IOMMUTLBEntry iotlb;
 272     MemoryRegionSection *section;
 273     MemoryRegion *mr;
 274     hwaddr len = *plen;
 275
 276     for (;;) {
 277         section = address_space_translate_internal(as, addr, &addr, plen, true);
 278         mr = section->mr;
 279
 280         if (!mr->iommu_ops) {
 281             break;
 282         }
 283
 284         iotlb = mr->iommu_ops->translate(mr, addr);
 285         addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
 286                 | (addr & iotlb.addr_mask));
 287         len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
 288         if (!(iotlb.perm & (1 << is_write))) {
 289             mr = &io_mem_unassigned;
 290             break;
 291         }
 292
 293         as = iotlb.target_as;
 294     }
 295
 296     *plen = len;
 297     *xlat = addr;
 298     return mr;
 299 }
 300
 301 MemoryRegionSection *
 302 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
 303                                   hwaddr *plen)
 304 {
 305     MemoryRegionSection *section;
 306     section = address_space_translate_internal(as, addr, xlat, plen, false);
 307
 308     assert(!section->mr->iommu_ops);
 309     return section;
 310 }
 311 #endif
 312
 313 void cpu_exec_init_all(void)
 314 {
 315 #if !defined(CONFIG_USER_ONLY)
 316     qemu_mutex_init(&ram_list.mutex);
 317     memory_map_init();
 318     io_mem_init();
 319 #endif
 320 }
 321
 322 #if !defined(CONFIG_USER_ONLY)
 323
 324 static int cpu_common_post_load(void *opaque, int version_id)
 325 {
 326     CPUState *cpu = opaque;
 327
 328     /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
 329        version_id is increased. */
 330     cpu->interrupt_request &= ~0x01;
 331     tlb_flush(cpu->env_ptr, 1);
 332
 333     return 0;
 334 }
 335
 336 const VMStateDescription vmstate_cpu_common = {
 337     .name = "cpu_common",
 338     .version_id = 1,
 339     .minimum_version_id = 1,
 340     .minimum_version_id_old = 1,
 341     .post_load = cpu_common_post_load,
 342     .fields      = (VMStateField []) {
 343         VMSTATE_UINT32(halted, CPUState),
 344         VMSTATE_UINT32(interrupt_request, CPUState),
 345         VMSTATE_END_OF_LIST()
 346     }
 347 };
 348
 349 #endif
 350
 351 CPUState *qemu_get_cpu(int index)
 352 {
 353     CPUArchState *env = first_cpu;
 354     CPUState *cpu = NULL;
 355
 356     while (env) {
 357         cpu = ENV_GET_CPU(env);
 358         if (cpu->cpu_index == index) {
 359             break;
 360         }
 361         env = env->next_cpu;
 362     }
 363
 364     return env ? cpu : NULL;
 365 }
 366
 367 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
 368 {
 369     CPUArchState *env = first_cpu;
 370
 371     while (env) {
 372         func(ENV_GET_CPU(env), data);
 373         env = env->next_cpu;
 374     }
 375 }
 376
 377 void cpu_exec_init(CPUArchState *env)
 378 {
 379     CPUState *cpu = ENV_GET_CPU(env);
 380     CPUClass *cc = CPU_GET_CLASS(cpu);
 381     CPUArchState **penv;
 382     int cpu_index;
 383
 384 #if defined(CONFIG_USER_ONLY)
 385     cpu_list_lock();
 386 #endif
 387     env->next_cpu = NULL;
 388     penv = &first_cpu;
 389     cpu_index = 0;
 390     while (*penv != NULL) {
 391         penv = &(*penv)->next_cpu;
 392         cpu_index++;
 393     }
 394     cpu->cpu_index = cpu_index;
 395     cpu->numa_node = 0;
 396     QTAILQ_INIT(&env->breakpoints);
 397     QTAILQ_INIT(&env->watchpoints);
 398 #ifndef CONFIG_USER_ONLY
 399     cpu->thread_id = qemu_get_thread_id();
 400 #endif
 401     *penv = env;
 402 #if defined(CONFIG_USER_ONLY)
 403     cpu_list_unlock();
 404 #endif
 405     vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
 406 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
 407     register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
 408                     cpu_save, cpu_load, env);
 409     assert(cc->vmsd == NULL);
 410 #endif
 411     if (cc->vmsd != NULL) {
 412         vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
 413     }
 414 }
 415
 416 #if defined(TARGET_HAS_ICE)
 417 #if defined(CONFIG_USER_ONLY)
 418 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
 419 {
 420     tb_invalidate_phys_page_range(pc, pc + 1, 0);
 421 }
 422 #else
 423 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
 424 {
 425     tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
 426             (pc & ~TARGET_PAGE_MASK));
 427 }
 428 #endif
 429 #endif /* TARGET_HAS_ICE */
 430
 431 #if defined(CONFIG_USER_ONLY)
 432 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 433
 434 {
 435 }
 436
 437 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 438                           int flags, CPUWatchpoint **watchpoint)
 439 {
 440     return -ENOSYS;
 441 }
 442 #else
 443 /* Add a watchpoint.  */
 444 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 445                           int flags, CPUWatchpoint **watchpoint)
 446 {
 447     target_ulong len_mask = ~(len - 1);
 448     CPUWatchpoint *wp;
 449
 450     /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
 451     if ((len & (len - 1)) || (addr & ~len_mask) ||
 452             len == 0 || len > TARGET_PAGE_SIZE) {
 453         fprintf(stderr, "qemu: tried to set invalid watchpoint at "
 454                 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
 455         return -EINVAL;
 456     }
 457     wp = g_malloc(sizeof(*wp));
 458
 459     wp->vaddr = addr;
 460     wp->len_mask = len_mask;
 461     wp->flags = flags;
 462
 463     /* keep all GDB-injected watchpoints in front */
 464     if (flags & BP_GDB)
 465         QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
 466     else
 467         QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
 468
 469     tlb_flush_page(env, addr);
 470
 471     if (watchpoint)
 472         *watchpoint = wp;
 473     return 0;
 474 }
 475
 476 /* Remove a specific watchpoint.  */
 477 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
 478                           int flags)
 479 {
 480     target_ulong len_mask = ~(len - 1);
 481     CPUWatchpoint *wp;
 482
 483     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 484         if (addr == wp->vaddr && len_mask == wp->len_mask
 485                 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
 486             cpu_watchpoint_remove_by_ref(env, wp);
 487             return 0;
 488         }
 489     }
 490     return -ENOENT;
 491 }
 492
 493 /* Remove a specific watchpoint by reference.  */
 494 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
 495 {
 496     QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
 497
 498     tlb_flush_page(env, watchpoint->vaddr);
 499
 500     g_free(watchpoint);
 501 }
 502
 503 /* Remove all matching watchpoints.  */
 504 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 505 {
 506     CPUWatchpoint *wp, *next;
 507
 508     QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
 509         if (wp->flags & mask)
 510             cpu_watchpoint_remove_by_ref(env, wp);
 511     }
 512 }
 513 #endif
 514
 515 /* Add a breakpoint.  */
 516 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
 517                           CPUBreakpoint **breakpoint)
 518 {
 519 #if defined(TARGET_HAS_ICE)
 520     CPUBreakpoint *bp;
 521
 522     bp = g_malloc(sizeof(*bp));
 523
 524     bp->pc = pc;
 525     bp->flags = flags;
 526
 527     /* keep all GDB-injected breakpoints in front */
 528     if (flags & BP_GDB)
 529         QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
 530     else
 531         QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
 532
 533     breakpoint_invalidate(env, pc);
 534
 535     if (breakpoint)
 536         *breakpoint = bp;
 537     return 0;
 538 #else
 539     return -ENOSYS;
 540 #endif
 541 }
 542
 543 /* Remove a specific breakpoint.  */
 544 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
 545 {
 546 #if defined(TARGET_HAS_ICE)
 547     CPUBreakpoint *bp;
 548
 549     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
 550         if (bp->pc == pc && bp->flags == flags) {
 551             cpu_breakpoint_remove_by_ref(env, bp);
 552             return 0;
 553         }
 554     }
 555     return -ENOENT;
 556 #else
 557     return -ENOSYS;
 558 #endif
 559 }
 560
 561 /* Remove a specific breakpoint by reference.  */
 562 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
 563 {
 564 #if defined(TARGET_HAS_ICE)
 565     QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
 566
 567     breakpoint_invalidate(env, breakpoint->pc);
 568
 569     g_free(breakpoint);
 570 #endif
 571 }
 572
 573 /* Remove all matching breakpoints. */
 574 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
 575 {
 576 #if defined(TARGET_HAS_ICE)
 577     CPUBreakpoint *bp, *next;
 578
 579     QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
 580         if (bp->flags & mask)
 581             cpu_breakpoint_remove_by_ref(env, bp);
 582     }
 583 #endif
 584 }
 585
 586 /* enable or disable single step mode. EXCP_DEBUG is returned by the
 587    CPU loop after each instruction */
 588 void cpu_single_step(CPUArchState *env, int enabled)
 589 {
 590 #if defined(TARGET_HAS_ICE)
 591     if (env->singlestep_enabled != enabled) {
 592         env->singlestep_enabled = enabled;
 593         if (kvm_enabled())
 594             kvm_update_guest_debug(env, 0);
 595         else {
 596             /* must flush all the translated code to avoid inconsistencies */
 597             /* XXX: only flush what is necessary */
 598             tb_flush(env);
 599         }
 600     }
 601 #endif
 602 }
 603
 604 void cpu_abort(CPUArchState *env, const char *fmt, ...)
 605 {
 606     CPUState *cpu = ENV_GET_CPU(env);
 607     va_list ap;
 608     va_list ap2;
 609
 610     va_start(ap, fmt);
 611     va_copy(ap2, ap);
 612     fprintf(stderr, "qemu: fatal: ");
 613     vfprintf(stderr, fmt, ap);
 614     fprintf(stderr, "\n");
 615     cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 616     if (qemu_log_enabled()) {
 617         qemu_log("qemu: fatal: ");
 618         qemu_log_vprintf(fmt, ap2);
 619         qemu_log("\n");
 620         log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 621         qemu_log_flush();
 622         qemu_log_close();
 623     }
 624     va_end(ap2);
 625     va_end(ap);
 626 #if defined(CONFIG_USER_ONLY)
 627     {
 628         struct sigaction act;
 629         sigfillset(&act.sa_mask);
 630         act.sa_handler = SIG_DFL;
 631         sigaction(SIGABRT, &act, NULL);
 632     }
 633 #endif
 634     abort();
 635 }
 636
 637 CPUArchState *cpu_copy(CPUArchState *env)
 638 {
 639     CPUArchState *new_env = cpu_init(env->cpu_model_str);
 640     CPUArchState *next_cpu = new_env->next_cpu;
 641 #if defined(TARGET_HAS_ICE)
 642     CPUBreakpoint *bp;
 643     CPUWatchpoint *wp;
 644 #endif
 645
 646     memcpy(new_env, env, sizeof(CPUArchState));
 647
 648     /* Preserve chaining. */
 649     new_env->next_cpu = next_cpu;
 650
 651     /* Clone all break/watchpoints.
 652        Note: Once we support ptrace with hw-debug register access, make sure
 653        BP_CPU break/watchpoints are handled correctly on clone. */
 654     QTAILQ_INIT(&env->breakpoints);
 655     QTAILQ_INIT(&env->watchpoints);
 656 #if defined(TARGET_HAS_ICE)
 657     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
 658         cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
 659     }
 660     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 661         cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
 662                               wp->flags, NULL);
 663     }
 664 #endif
 665
 666     return new_env;
 667 }
 668
 669 #if !defined(CONFIG_USER_ONLY)
 670 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
 671                                       uintptr_t length)
 672 {
 673     uintptr_t start1;
 674
 675     /* we modify the TLB cache so that the dirty bit will be set again
 676        when accessing the range */
 677     start1 = (uintptr_t)qemu_safe_ram_ptr(start);
 678     /* Check that we don't span multiple blocks - this breaks the
 679        address comparisons below.  */
 680     if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
 681             != (end - 1) - start) {
 682         abort();
 683     }
 684     cpu_tlb_reset_dirty_all(start1, length);
 685
 686 }
 687
 688 /* Note: start and end must be within the same ram block.  */
 689 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
 690                                      int dirty_flags)
 691 {
 692     uintptr_t length;
 693
 694     start &= TARGET_PAGE_MASK;
 695     end = TARGET_PAGE_ALIGN(end);
 696
 697     length = end - start;
 698     if (length == 0)
 699         return;
 700     cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
 701
 702     if (tcg_enabled()) {
 703         tlb_reset_dirty_range_all(start, end, length);
 704     }
 705 }
 706
 707 static int cpu_physical_memory_set_dirty_tracking(int enable)
 708 {
 709     int ret = 0;
 710     in_migration = enable;
 711     return ret;
 712 }
 713
 714 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
 715                                        MemoryRegionSection *section,
 716                                        target_ulong vaddr,
 717                                        hwaddr paddr, hwaddr xlat,
 718                                        int prot,
 719                                        target_ulong *address)
 720 {
 721     hwaddr iotlb;
 722     CPUWatchpoint *wp;
 723
 724     if (memory_region_is_ram(section->mr)) {
 725         /* Normal RAM.  */
 726         iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
 727             + xlat;
 728         if (!section->readonly) {
 729             iotlb |= PHYS_SECTION_NOTDIRTY;
 730         } else {
 731             iotlb |= PHYS_SECTION_ROM;
 732         }
 733     } else {
 734         iotlb = section - cur_map.sections;
 735         iotlb += xlat;
 736     }
 737
 738     /* Make accesses to pages with watchpoints go via the
 739        watchpoint trap routines.  */
 740     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 741         if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
 742             /* Avoid trapping reads of pages with a write breakpoint. */
 743             if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
 744                 iotlb = PHYS_SECTION_WATCH + paddr;
 745                 *address |= TLB_MMIO;
 746                 break;
 747             }
 748         }
 749     }
 750
 751     return iotlb;
 752 }
 753 #endif /* defined(CONFIG_USER_ONLY) */
 754
 755 #if !defined(CONFIG_USER_ONLY)
 756
 757 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
 758                              uint16_t section);
 759 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
 760
 761 static uint16_t phys_section_add(MemoryRegionSection *section)
 762 {
 763     /* The physical section number is ORed with a page-aligned
 764      * pointer to produce the iotlb entries.  Thus it should
 765      * never overflow into the page-aligned value.
 766      */
 767     assert(next_map.sections_nb < TARGET_PAGE_SIZE);
 768
 769     if (next_map.sections_nb == next_map.sections_nb_alloc) {
 770         next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
 771                                          16);
 772         next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
 773                                     next_map.sections_nb_alloc);
 774     }
 775     next_map.sections[next_map.sections_nb] = *section;
 776     memory_region_ref(section->mr);
 777     return next_map.sections_nb++;
 778 }
 779
 780 static void phys_section_destroy(MemoryRegion *mr)
 781 {
 782     memory_region_unref(mr);
 783
 784     if (mr->subpage) {
 785         subpage_t *subpage = container_of(mr, subpage_t, iomem);
 786         memory_region_destroy(&subpage->iomem);
 787         g_free(subpage);
 788     }
 789 }
 790
 791 static void phys_sections_clear(PhysPageMap *map)
 792 {
 793     while (map->sections_nb > 0) {
 794         MemoryRegionSection *section = &map->sections[--map->sections_nb];
 795         phys_section_destroy(section->mr);
 796     }
 797     g_free(map->sections);
 798     g_free(map->nodes);
 799 }
 800
 801 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
 802 {
 803     subpage_t *subpage;
 804     hwaddr base = section->offset_within_address_space
 805         & TARGET_PAGE_MASK;
 806     MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
 807                                                    next_map.nodes, next_map.sections);
 808     MemoryRegionSection subsection = {
 809         .offset_within_address_space = base,
 810         .size = int128_make64(TARGET_PAGE_SIZE),
 811     };
 812     hwaddr start, end;
 813
 814     assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
 815
 816     if (!(existing->mr->subpage)) {
 817         subpage = subpage_init(d->as, base);
 818         subsection.mr = &subpage->iomem;
 819         phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
 820                       phys_section_add(&subsection));
 821     } else {
 822         subpage = container_of(existing->mr, subpage_t, iomem);
 823     }
 824     start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
 825     end = start + int128_get64(section->size) - 1;
 826     subpage_register(subpage, start, end, phys_section_add(section));
 827 }
 828
 829
 830 static void register_multipage(AddressSpaceDispatch *d,
 831                                MemoryRegionSection *section)
 832 {
 833     hwaddr start_addr = section->offset_within_address_space;
 834     uint16_t section_index = phys_section_add(section);
 835     uint64_t num_pages = int128_get64(int128_rshift(section->size,
 836                                                     TARGET_PAGE_BITS));
 837
 838     assert(num_pages);
 839     phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
 840 }
 841
 842 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
 843 {
 844     AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
 845     MemoryRegionSection now = *section, remain = *section;
 846     Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
 847
 848     if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
 849         uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
 850                        - now.offset_within_address_space;
 851
 852         now.size = int128_min(int128_make64(left), now.size);
 853         register_subpage(d, &now);
 854     } else {
 855         now.size = int128_zero();
 856     }
 857     while (int128_ne(remain.size, now.size)) {
 858         remain.size = int128_sub(remain.size, now.size);
 859         remain.offset_within_address_space += int128_get64(now.size);
 860         remain.offset_within_region += int128_get64(now.size);
 861         now = remain;
 862         if (int128_lt(remain.size, page_size)) {
 863             register_subpage(d, &now);
 864         } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
 865             now.size = page_size;
 866             register_subpage(d, &now);
 867         } else {
 868             now.size = int128_and(now.size, int128_neg(page_size));
 869             register_multipage(d, &now);
 870         }
 871     }
 872 }
 873
 874 void qemu_flush_coalesced_mmio_buffer(void)
 875 {
 876     if (kvm_enabled())
 877         kvm_flush_coalesced_mmio_buffer();
 878 }
 879
 880 void qemu_mutex_lock_ramlist(void)
 881 {
 882     qemu_mutex_lock(&ram_list.mutex);
 883 }
 884
 885 void qemu_mutex_unlock_ramlist(void)
 886 {
 887     qemu_mutex_unlock(&ram_list.mutex);
 888 }
 889
 890 #if defined(__linux__) && !defined(TARGET_S390X)
 891
 892 #include <sys/vfs.h>
 893
 894 #define HUGETLBFS_MAGIC       0x958458f6
 895
 896 static long gethugepagesize(const char *path)
 897 {
 898     struct statfs fs;
 899     int ret;
 900
 901     do {
 902         ret = statfs(path, &fs);
 903     } while (ret != 0 && errno == EINTR);
 904
 905     if (ret != 0) {
 906         perror(path);
 907         return 0;
 908     }
 909
 910     if (fs.f_type != HUGETLBFS_MAGIC)
 911         fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
 912
 913     return fs.f_bsize;
 914 }
 915
 916 static void *file_ram_alloc(RAMBlock *block,
 917                             ram_addr_t memory,
 918                             const char *path)
 919 {
 920     char *filename;
 921     char *sanitized_name;
 922     char *c;
 923     void *area;
 924     int fd;
 925 #ifdef MAP_POPULATE
 926     int flags;
 927 #endif
 928     unsigned long hpagesize;
 929
 930     hpagesize = gethugepagesize(path);
 931     if (!hpagesize) {
 932         return NULL;
 933     }
 934
 935     if (memory < hpagesize) {
 936         return NULL;
 937     }
 938
 939     if (kvm_enabled() && !kvm_has_sync_mmu()) {
 940         fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
 941         return NULL;
 942     }
 943
 944     /* Make name safe to use with mkstemp by replacing '/' with '_'. */
 945     sanitized_name = g_strdup(block->mr->name);
 946     for (c = sanitized_name; *c != '\0'; c++) {
 947         if (*c == '/')
 948             *c = '_';
 949     }
 950
 951     filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
 952                                sanitized_name);
 953     g_free(sanitized_name);
 954
 955     fd = mkstemp(filename);
 956     if (fd < 0) {
 957         perror("unable to create backing store for hugepages");
 958         g_free(filename);
 959         return NULL;
 960     }
 961     unlink(filename);
 962     g_free(filename);
 963
 964     memory = (memory+hpagesize-1) & ~(hpagesize-1);
 965
 966     /*
 967      * ftruncate is not supported by hugetlbfs in older
 968      * hosts, so don't bother bailing out on errors.
 969      * If anything goes wrong with it under other filesystems,
 970      * mmap will fail.
 971      */
 972     if (ftruncate(fd, memory))
 973         perror("ftruncate");
 974
 975 #ifdef MAP_POPULATE
 976     /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
 977      * MAP_PRIVATE is requested.  For mem_prealloc we mmap as MAP_SHARED
 978      * to sidestep this quirk.
 979      */
 980     flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
 981     area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
 982 #else
 983     area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
 984 #endif
 985     if (area == MAP_FAILED) {
 986         perror("file_ram_alloc: can't mmap RAM pages");
 987         close(fd);
 988         return (NULL);
 989     }
 990     block->fd = fd;
 991     return area;
 992 }
 993 #endif
 994
 995 static ram_addr_t find_ram_offset(ram_addr_t size)
 996 {
 997     RAMBlock *block, *next_block;
 998     ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
 999
1000     assert(size != 0); /* it would hand out same offset multiple times */
1001
1002     if (QTAILQ_EMPTY(&ram_list.blocks))
1003         return 0;
1004
1005     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1006         ram_addr_t end, next = RAM_ADDR_MAX;
1007
1008         end = block->offset + block->length;
1009
1010         QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1011             if (next_block->offset >= end) {
1012                 next = MIN(next, next_block->offset);
1013             }
1014         }
1015         if (next - end >= size && next - end < mingap) {
1016             offset = end;
1017             mingap = next - end;
1018         }
1019     }
1020
1021     if (offset == RAM_ADDR_MAX) {
1022         fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1023                 (uint64_t)size);
1024         abort();
1025     }
1026
1027     return offset;
1028 }
1029
1030 ram_addr_t last_ram_offset(void)
1031 {
1032     RAMBlock *block;
1033     ram_addr_t last = 0;
1034
1035     QTAILQ_FOREACH(block, &ram_list.blocks, next)
1036         last = MAX(last, block->offset + block->length);
1037
1038     return last;
1039 }
1040
1041 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1042 {
1043     int ret;
1044     QemuOpts *machine_opts;
1045
1046     /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1047     machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1048     if (machine_opts &&
1049         !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
1050         ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1051         if (ret) {
1052             perror("qemu_madvise");
1053             fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1054                             "but dump_guest_core=off specified\n");
1055         }
1056     }
1057 }
1058
1059 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1060 {
1061     RAMBlock *new_block, *block;
1062
1063     new_block = NULL;
1064     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1065         if (block->offset == addr) {
1066             new_block = block;
1067             break;
1068         }
1069     }
1070     assert(new_block);
1071     assert(!new_block->idstr[0]);
1072
1073     if (dev) {
1074         char *id = qdev_get_dev_path(dev);
1075         if (id) {
1076             snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1077             g_free(id);
1078         }
1079     }
1080     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1081
1082     /* This assumes the iothread lock is taken here too.  */
1083     qemu_mutex_lock_ramlist();
1084     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1085         if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1086             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1087                     new_block->idstr);
1088             abort();
1089         }
1090     }
1091     qemu_mutex_unlock_ramlist();
1092 }
1093
1094 static int memory_try_enable_merging(void *addr, size_t len)
1095 {
1096     QemuOpts *opts;
1097
1098     opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1099     if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
1100         /* disabled by the user */
1101         return 0;
1102     }
1103
1104     return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1105 }
1106
1107 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1108                                    MemoryRegion *mr)
1109 {
1110     RAMBlock *block, *new_block;
1111
1112     size = TARGET_PAGE_ALIGN(size);
1113     new_block = g_malloc0(sizeof(*new_block));
1114
1115     /* This assumes the iothread lock is taken here too.  */
1116     qemu_mutex_lock_ramlist();
1117     new_block->mr = mr;
1118     new_block->offset = find_ram_offset(size);
1119     if (host) {
1120         new_block->host = host;
1121         new_block->flags |= RAM_PREALLOC_MASK;
1122     } else {
1123         if (mem_path) {
1124 #if defined (__linux__) && !defined(TARGET_S390X)
1125             new_block->host = file_ram_alloc(new_block, size, mem_path);
1126             if (!new_block->host) {
1127                 new_block->host = qemu_anon_ram_alloc(size);
1128                 memory_try_enable_merging(new_block->host, size);
1129             }
1130 #else
1131             fprintf(stderr, "-mem-path option unsupported\n");
1132             exit(1);
1133 #endif
1134         } else {
1135             if (xen_enabled()) {
1136                 xen_ram_alloc(new_block->offset, size, mr);
1137             } else if (kvm_enabled()) {
1138                 /* some s390/kvm configurations have special constraints */
1139                 new_block->host = kvm_ram_alloc(size);
1140             } else {
1141                 new_block->host = qemu_anon_ram_alloc(size);
1142             }
1143             memory_try_enable_merging(new_block->host, size);
1144         }
1145     }
1146     new_block->length = size;
1147
1148     /* Keep the list sorted from biggest to smallest block.  */
1149     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1150         if (block->length < new_block->length) {
1151             break;
1152         }
1153     }
1154     if (block) {
1155         QTAILQ_INSERT_BEFORE(block, new_block, next);
1156     } else {
1157         QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1158     }
1159     ram_list.mru_block = NULL;
1160
1161     ram_list.version++;
1162     qemu_mutex_unlock_ramlist();
1163
1164     ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1165                                        last_ram_offset() >> TARGET_PAGE_BITS);
1166     memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1167            0, size >> TARGET_PAGE_BITS);
1168     cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1169
1170     qemu_ram_setup_dump(new_block->host, size);
1171     qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1172
1173     if (kvm_enabled())
1174         kvm_setup_guest_memory(new_block->host, size);
1175
1176     return new_block->offset;
1177 }
1178
1179 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1180 {
1181     return qemu_ram_alloc_from_ptr(size, NULL, mr);
1182 }
1183
1184 void qemu_ram_free_from_ptr(ram_addr_t addr)
1185 {
1186     RAMBlock *block;
1187
1188     /* This assumes the iothread lock is taken here too.  */
1189     qemu_mutex_lock_ramlist();
1190     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1191         if (addr == block->offset) {
1192             QTAILQ_REMOVE(&ram_list.blocks, block, next);
1193             ram_list.mru_block = NULL;
1194             ram_list.version++;
1195             g_free(block);
1196             break;
1197         }
1198     }
1199     qemu_mutex_unlock_ramlist();
1200 }
1201
1202 void qemu_ram_free(ram_addr_t addr)
1203 {
1204     RAMBlock *block;
1205
1206     /* This assumes the iothread lock is taken here too.  */
1207     qemu_mutex_lock_ramlist();
1208     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1209         if (addr == block->offset) {
1210             QTAILQ_REMOVE(&ram_list.blocks, block, next);
1211             ram_list.mru_block = NULL;
1212             ram_list.version++;
1213             if (block->flags & RAM_PREALLOC_MASK) {
1214                 ;
1215             } else if (mem_path) {
1216 #if defined (__linux__) && !defined(TARGET_S390X)
1217                 if (block->fd) {
1218                     munmap(block->host, block->length);
1219                     close(block->fd);
1220                 } else {
1221                     qemu_anon_ram_free(block->host, block->length);
1222                 }
1223 #else
1224                 abort();
1225 #endif
1226             } else {
1227                 if (xen_enabled()) {
1228                     xen_invalidate_map_cache_entry(block->host);
1229                 } else {
1230                     qemu_anon_ram_free(block->host, block->length);
1231                 }
1232             }
1233             g_free(block);
1234             break;
1235         }
1236     }
1237     qemu_mutex_unlock_ramlist();
1238
1239 }
1240
1241 #ifndef _WIN32
1242 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1243 {
1244     RAMBlock *block;
1245     ram_addr_t offset;
1246     int flags;
1247     void *area, *vaddr;
1248
1249     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1250         offset = addr - block->offset;
1251         if (offset < block->length) {
1252             vaddr = block->host + offset;
1253             if (block->flags & RAM_PREALLOC_MASK) {
1254                 ;
1255             } else {
1256                 flags = MAP_FIXED;
1257                 munmap(vaddr, length);
1258                 if (mem_path) {
1259 #if defined(__linux__) && !defined(TARGET_S390X)
1260                     if (block->fd) {
1261 #ifdef MAP_POPULATE
1262                         flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1263                             MAP_PRIVATE;
1264 #else
1265                         flags |= MAP_PRIVATE;
1266 #endif
1267                         area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1268                                     flags, block->fd, offset);
1269                     } else {
1270                         flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1271                         area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1272                                     flags, -1, 0);
1273                     }
1274 #else
1275                     abort();
1276 #endif
1277                 } else {
1278 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1279                     flags |= MAP_SHARED | MAP_ANONYMOUS;
1280                     area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1281                                 flags, -1, 0);
1282 #else
1283                     flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1284                     area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1285                                 flags, -1, 0);
1286 #endif
1287                 }
1288                 if (area != vaddr) {
1289                     fprintf(stderr, "Could not remap addr: "
1290                             RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1291                             length, addr);
1292                     exit(1);
1293                 }
1294                 memory_try_enable_merging(vaddr, length);
1295                 qemu_ram_setup_dump(vaddr, length);
1296             }
1297             return;
1298         }
1299     }
1300 }
1301 #endif /* !_WIN32 */
1302
1303 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1304 {
1305     RAMBlock *block;
1306
1307     /* The list is protected by the iothread lock here.  */
1308     block = ram_list.mru_block;
1309     if (block && addr - block->offset < block->length) {
1310         goto found;
1311     }
1312     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1313         if (addr - block->offset < block->length) {
1314             goto found;
1315         }
1316     }
1317
1318     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1319     abort();
1320
1321 found:
1322     ram_list.mru_block = block;
1323     return block;
1324 }
1325
1326 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1327    With the exception of the softmmu code in this file, this should
1328    only be used for local memory (e.g. video ram) that the device owns,
1329    and knows it isn't going to access beyond the end of the block.
1330
1331    It should not be used for general purpose DMA.
1332    Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1333  */
1334 void *qemu_get_ram_ptr(ram_addr_t addr)
1335 {
1336     RAMBlock *block = qemu_get_ram_block(addr);
1337
1338     if (xen_enabled()) {
1339         /* We need to check if the requested address is in the RAM
1340          * because we don't want to map the entire memory in QEMU.
1341          * In that case just map until the end of the page.
1342          */
1343         if (block->offset == 0) {
1344             return xen_map_cache(addr, 0, 0);
1345         } else if (block->host == NULL) {
1346             block->host =
1347                 xen_map_cache(block->offset, block->length, 1);
1348         }
1349     }
1350     return block->host + (addr - block->offset);
1351 }
1352
1353 /* Return a host pointer to ram allocated with qemu_ram_alloc.  Same as
1354  * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1355  *
1356  * ??? Is this still necessary?
1357  */
1358 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1359 {
1360     RAMBlock *block;
1361
1362     /* The list is protected by the iothread lock here.  */
1363     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1364         if (addr - block->offset < block->length) {
1365             if (xen_enabled()) {
1366                 /* We need to check if the requested address is in the RAM
1367                  * because we don't want to map the entire memory in QEMU.
1368                  * In that case just map until the end of the page.
1369                  */
1370                 if (block->offset == 0) {
1371                     return xen_map_cache(addr, 0, 0);
1372                 } else if (block->host == NULL) {
1373                     block->host =
1374                         xen_map_cache(block->offset, block->length, 1);
1375                 }
1376             }
1377             return block->host + (addr - block->offset);
1378         }
1379     }
1380
1381     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1382     abort();
1383
1384     return NULL;
1385 }
1386
1387 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1388  * but takes a size argument */
1389 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1390 {
1391     if (*size == 0) {
1392         return NULL;
1393     }
1394     if (xen_enabled()) {
1395         return xen_map_cache(addr, *size, 1);
1396     } else {
1397         RAMBlock *block;
1398
1399         QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1400             if (addr - block->offset < block->length) {
1401                 if (addr - block->offset + *size > block->length)
1402                     *size = block->length - addr + block->offset;
1403                 return block->host + (addr - block->offset);
1404             }
1405         }
1406
1407         fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1408         abort();
1409     }
1410 }
1411
1412 /* Some of the softmmu routines need to translate from a host pointer
1413    (typically a TLB entry) back to a ram offset.  */
1414 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1415 {
1416     RAMBlock *block;
1417     uint8_t *host = ptr;
1418
1419     if (xen_enabled()) {
1420         *ram_addr = xen_ram_addr_from_mapcache(ptr);
1421         return qemu_get_ram_block(*ram_addr)->mr;
1422     }
1423
1424     block = ram_list.mru_block;
1425     if (block && block->host && host - block->host < block->length) {
1426         goto found;
1427     }
1428
1429     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1430         /* This case append when the block is not mapped. */
1431         if (block->host == NULL) {
1432             continue;
1433         }
1434         if (host - block->host < block->length) {
1435             goto found;
1436         }
1437     }
1438
1439     return NULL;
1440
1441 found:
1442     *ram_addr = block->offset + (host - block->host);
1443     return block->mr;
1444 }
1445
1446 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1447                                uint64_t val, unsigned size)
1448 {
1449     int dirty_flags;
1450     dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1451     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1452         tb_invalidate_phys_page_fast(ram_addr, size);
1453         dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1454     }
1455     switch (size) {
1456     case 1:
1457         stb_p(qemu_get_ram_ptr(ram_addr), val);
1458         break;
1459     case 2:
1460         stw_p(qemu_get_ram_ptr(ram_addr), val);
1461         break;
1462     case 4:
1463         stl_p(qemu_get_ram_ptr(ram_addr), val);
1464         break;
1465     default:
1466         abort();
1467     }
1468     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1469     cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1470     /* we remove the notdirty callback only if the code has been
1471        flushed */
1472     if (dirty_flags == 0xff)
1473         tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1474 }
1475
1476 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1477                                  unsigned size, bool is_write)
1478 {
1479     return is_write;
1480 }
1481
1482 static const MemoryRegionOps notdirty_mem_ops = {
1483     .write = notdirty_mem_write,
1484     .valid.accepts = notdirty_mem_accepts,
1485     .endianness = DEVICE_NATIVE_ENDIAN,
1486 };
1487
1488 /* Generate a debug exception if a watchpoint has been hit.  */
1489 static void check_watchpoint(int offset, int len_mask, int flags)
1490 {
1491     CPUArchState *env = cpu_single_env;
1492     target_ulong pc, cs_base;
1493     target_ulong vaddr;
1494     CPUWatchpoint *wp;
1495     int cpu_flags;
1496
1497     if (env->watchpoint_hit) {
1498         /* We re-entered the check after replacing the TB. Now raise
1499          * the debug interrupt so that is will trigger after the
1500          * current instruction. */
1501         cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1502         return;
1503     }
1504     vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1505     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1506         if ((vaddr == (wp->vaddr & len_mask) ||
1507              (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1508             wp->flags |= BP_WATCHPOINT_HIT;
1509             if (!env->watchpoint_hit) {
1510                 env->watchpoint_hit = wp;
1511                 tb_check_watchpoint(env);
1512                 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1513                     env->exception_index = EXCP_DEBUG;
1514                     cpu_loop_exit(env);
1515                 } else {
1516                     cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1517                     tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1518                     cpu_resume_from_signal(env, NULL);
1519                 }
1520             }
1521         } else {
1522             wp->flags &= ~BP_WATCHPOINT_HIT;
1523         }
1524     }
1525 }
1526
1527 /* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
1528    so these check for a hit then pass through to the normal out-of-line
1529    phys routines.  */
1530 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1531                                unsigned size)
1532 {
1533     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1534     switch (size) {
1535     case 1: return ldub_phys(addr);
1536     case 2: return lduw_phys(addr);
1537     case 4: return ldl_phys(addr);
1538     default: abort();
1539     }
1540 }
1541
1542 static void watch_mem_write(void *opaque, hwaddr addr,
1543                             uint64_t val, unsigned size)
1544 {
1545     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1546     switch (size) {
1547     case 1:
1548         stb_phys(addr, val);
1549         break;
1550     case 2:
1551         stw_phys(addr, val);
1552         break;
1553     case 4:
1554         stl_phys(addr, val);
1555         break;
1556     default: abort();
1557     }
1558 }
1559
1560 static const MemoryRegionOps watch_mem_ops = {
1561     .read = watch_mem_read,
1562     .write = watch_mem_write,
1563     .endianness = DEVICE_NATIVE_ENDIAN,
1564 };
1565
1566 static uint64_t subpage_read(void *opaque, hwaddr addr,
1567                              unsigned len)
1568 {
1569     subpage_t *subpage = opaque;
1570     uint8_t buf[4];
1571
1572 #if defined(DEBUG_SUBPAGE)
1573     printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1574            subpage, len, addr);
1575 #endif
1576     address_space_read(subpage->as, addr + subpage->base, buf, len);
1577     switch (len) {
1578     case 1:
1579         return ldub_p(buf);
1580     case 2:
1581         return lduw_p(buf);
1582     case 4:
1583         return ldl_p(buf);
1584     default:
1585         abort();
1586     }
1587 }
1588
1589 static void subpage_write(void *opaque, hwaddr addr,
1590                           uint64_t value, unsigned len)
1591 {
1592     subpage_t *subpage = opaque;
1593     uint8_t buf[4];
1594
1595 #if defined(DEBUG_SUBPAGE)
1596     printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1597            " value %"PRIx64"\n",
1598            __func__, subpage, len, addr, value);
1599 #endif
1600     switch (len) {
1601     case 1:
1602         stb_p(buf, value);
1603         break;
1604     case 2:
1605         stw_p(buf, value);
1606         break;
1607     case 4:
1608         stl_p(buf, value);
1609         break;
1610     default:
1611         abort();
1612     }
1613     address_space_write(subpage->as, addr + subpage->base, buf, len);
1614 }
1615
1616 static bool subpage_accepts(void *opaque, hwaddr addr,
1617                             unsigned size, bool is_write)
1618 {
1619     subpage_t *subpage = opaque;
1620 #if defined(DEBUG_SUBPAGE)
1621     printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1622            __func__, subpage, is_write ? 'w' : 'r', len, addr);
1623 #endif
1624
1625     return address_space_access_valid(subpage->as, addr + subpage->base,
1626                                       size, is_write);
1627 }
1628
1629 static const MemoryRegionOps subpage_ops = {
1630     .read = subpage_read,
1631     .write = subpage_write,
1632     .valid.accepts = subpage_accepts,
1633     .endianness = DEVICE_NATIVE_ENDIAN,
1634 };
1635
1636 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1637                              uint16_t section)
1638 {
1639     int idx, eidx;
1640
1641     if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1642         return -1;
1643     idx = SUBPAGE_IDX(start);
1644     eidx = SUBPAGE_IDX(end);
1645 #if defined(DEBUG_SUBPAGE)
1646     printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1647            mmio, start, end, idx, eidx, memory);
1648 #endif
1649     for (; idx <= eidx; idx++) {
1650         mmio->sub_section[idx] = section;
1651     }
1652
1653     return 0;
1654 }
1655
1656 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1657 {
1658     subpage_t *mmio;
1659
1660     mmio = g_malloc0(sizeof(subpage_t));
1661
1662     mmio->as = as;
1663     mmio->base = base;
1664     memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1665                           "subpage", TARGET_PAGE_SIZE);
1666     mmio->iomem.subpage = true;
1667 #if defined(DEBUG_SUBPAGE)
1668     printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1669            mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1670 #endif
1671     subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1672
1673     return mmio;
1674 }
1675
1676 static uint16_t dummy_section(MemoryRegion *mr)
1677 {
1678     MemoryRegionSection section = {
1679         .mr = mr,
1680         .offset_within_address_space = 0,
1681         .offset_within_region = 0,
1682         .size = int128_2_64(),
1683     };
1684
1685     return phys_section_add(&section);
1686 }
1687
1688 MemoryRegion *iotlb_to_region(hwaddr index)
1689 {
1690     return cur_map.sections[index & ~TARGET_PAGE_MASK].mr;
1691 }
1692
1693 static void io_mem_init(void)
1694 {
1695     memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1696     memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1697                           "unassigned", UINT64_MAX);
1698     memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1699                           "notdirty", UINT64_MAX);
1700     memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1701                           "watch", UINT64_MAX);
1702 }
1703
1704 static void mem_begin(MemoryListener *listener)
1705 {
1706     AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
1707
1708     d->phys_map.ptr = PHYS_MAP_NODE_NIL;
1709 }
1710
1711 static void core_begin(MemoryListener *listener)
1712 {
1713     uint16_t n;
1714
1715     memset(&next_map, 0, sizeof(next_map));
1716     n = dummy_section(&io_mem_unassigned);
1717     assert(n == PHYS_SECTION_UNASSIGNED);
1718     n = dummy_section(&io_mem_notdirty);
1719     assert(n == PHYS_SECTION_NOTDIRTY);
1720     n = dummy_section(&io_mem_rom);
1721     assert(n == PHYS_SECTION_ROM);
1722     n = dummy_section(&io_mem_watch);
1723     assert(n == PHYS_SECTION_WATCH);
1724 }
1725
1726 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1727  * All AddressSpaceDispatch instances have switched to the next map.
1728  */
1729 static void core_commit(MemoryListener *listener)
1730 {
1731     PhysPageMap info = cur_map;
1732     cur_map = next_map;
1733     phys_sections_clear(&info);
1734 }
1735
1736 static void tcg_commit(MemoryListener *listener)
1737 {
1738     CPUArchState *env;
1739
1740     /* since each CPU stores ram addresses in its TLB cache, we must
1741        reset the modified entries */
1742     /* XXX: slow ! */
1743     for(env = first_cpu; env != NULL; env = env->next_cpu) {
1744         tlb_flush(env, 1);
1745     }
1746 }
1747
1748 static void core_log_global_start(MemoryListener *listener)
1749 {
1750     cpu_physical_memory_set_dirty_tracking(1);
1751 }
1752
1753 static void core_log_global_stop(MemoryListener *listener)
1754 {
1755     cpu_physical_memory_set_dirty_tracking(0);
1756 }
1757
1758 static MemoryListener core_memory_listener = {
1759     .begin = core_begin,
1760     .commit = core_commit,
1761     .log_global_start = core_log_global_start,
1762     .log_global_stop = core_log_global_stop,
1763     .priority = 1,
1764 };
1765
1766 static MemoryListener tcg_memory_listener = {
1767     .commit = tcg_commit,
1768 };
1769
1770 void address_space_init_dispatch(AddressSpace *as)
1771 {
1772     AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1773
1774     d->phys_map  = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1775     d->listener = (MemoryListener) {
1776         .begin = mem_begin,
1777         .region_add = mem_add,
1778         .region_nop = mem_add,
1779         .priority = 0,
1780     };
1781     d->as = as;
1782     as->dispatch = d;
1783     memory_listener_register(&d->listener, as);
1784 }
1785
1786 void address_space_destroy_dispatch(AddressSpace *as)
1787 {
1788     AddressSpaceDispatch *d = as->dispatch;
1789
1790     memory_listener_unregister(&d->listener);
1791     g_free(d);
1792     as->dispatch = NULL;
1793 }
1794
1795 static void memory_map_init(void)
1796 {
1797     system_memory = g_malloc(sizeof(*system_memory));
1798     memory_region_init(system_memory, NULL, "system", INT64_MAX);
1799     address_space_init(&address_space_memory, system_memory, "memory");
1800
1801     system_io = g_malloc(sizeof(*system_io));
1802     memory_region_init(system_io, NULL, "io", 65536);
1803     address_space_init(&address_space_io, system_io, "I/O");
1804
1805     memory_listener_register(&core_memory_listener, &address_space_memory);
1806     memory_listener_register(&tcg_memory_listener, &address_space_memory);
1807 }
1808
1809 MemoryRegion *get_system_memory(void)
1810 {
1811     return system_memory;
1812 }
1813
1814 MemoryRegion *get_system_io(void)
1815 {
1816     return system_io;
1817 }
1818
1819 #endif /* !defined(CONFIG_USER_ONLY) */
1820
1821 /* physical memory access (slow version, mainly for debug) */
1822 #if defined(CONFIG_USER_ONLY)
1823 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1824                         uint8_t *buf, int len, int is_write)
1825 {
1826     int l, flags;
1827     target_ulong page;
1828     void * p;
1829
1830     while (len > 0) {
1831         page = addr & TARGET_PAGE_MASK;
1832         l = (page + TARGET_PAGE_SIZE) - addr;
1833         if (l > len)
1834             l = len;
1835         flags = page_get_flags(page);
1836         if (!(flags & PAGE_VALID))
1837             return -1;
1838         if (is_write) {
1839             if (!(flags & PAGE_WRITE))
1840                 return -1;
1841             /* XXX: this code should not depend on lock_user */
1842             if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1843                 return -1;
1844             memcpy(p, buf, l);
1845             unlock_user(p, addr, l);
1846         } else {
1847             if (!(flags & PAGE_READ))
1848                 return -1;
1849             /* XXX: this code should not depend on lock_user */
1850             if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1851                 return -1;
1852             memcpy(buf, p, l);
1853             unlock_user(p, addr, 0);
1854         }
1855         len -= l;
1856         buf += l;
1857         addr += l;
1858     }
1859     return 0;
1860 }
1861
1862 #else
1863
1864 static void invalidate_and_set_dirty(hwaddr addr,
1865                                      hwaddr length)
1866 {
1867     if (!cpu_physical_memory_is_dirty(addr)) {
1868         /* invalidate code */
1869         tb_invalidate_phys_page_range(addr, addr + length, 0);
1870         /* set dirty bit */
1871         cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1872     }
1873     xen_modified_memory(addr, length);
1874 }
1875
1876 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1877 {
1878     if (memory_region_is_ram(mr)) {
1879         return !(is_write && mr->readonly);
1880     }
1881     if (memory_region_is_romd(mr)) {
1882         return !is_write;
1883     }
1884
1885     return false;
1886 }
1887
1888 static inline int memory_access_size(MemoryRegion *mr, int l, hwaddr addr)
1889 {
1890     if (l >= 4 && (((addr & 3) == 0 || mr->ops->impl.unaligned))) {
1891         return 4;
1892     }
1893     if (l >= 2 && (((addr & 1) == 0) || mr->ops->impl.unaligned)) {
1894         return 2;
1895     }
1896     return 1;
1897 }
1898
1899 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1900                       int len, bool is_write)
1901 {
1902     hwaddr l;
1903     uint8_t *ptr;
1904     uint64_t val;
1905     hwaddr addr1;
1906     MemoryRegion *mr;
1907     bool error = false;
1908
1909     while (len > 0) {
1910         l = len;
1911         mr = address_space_translate(as, addr, &addr1, &l, is_write);
1912
1913         if (is_write) {
1914             if (!memory_access_is_direct(mr, is_write)) {
1915                 l = memory_access_size(mr, l, addr1);
1916                 /* XXX: could force cpu_single_env to NULL to avoid
1917                    potential bugs */
1918                 if (l == 4) {
1919                     /* 32 bit write access */
1920                     val = ldl_p(buf);
1921                     error |= io_mem_write(mr, addr1, val, 4);
1922                 } else if (l == 2) {
1923                     /* 16 bit write access */
1924                     val = lduw_p(buf);
1925                     error |= io_mem_write(mr, addr1, val, 2);
1926                 } else {
1927                     /* 8 bit write access */
1928                     val = ldub_p(buf);
1929                     error |= io_mem_write(mr, addr1, val, 1);
1930                 }
1931             } else {
1932                 addr1 += memory_region_get_ram_addr(mr);
1933                 /* RAM case */
1934                 ptr = qemu_get_ram_ptr(addr1);
1935                 memcpy(ptr, buf, l);
1936                 invalidate_and_set_dirty(addr1, l);
1937             }
1938         } else {
1939             if (!memory_access_is_direct(mr, is_write)) {
1940                 /* I/O case */
1941                 l = memory_access_size(mr, l, addr1);
1942                 if (l == 4) {
1943                     /* 32 bit read access */
1944                     error |= io_mem_read(mr, addr1, &val, 4);
1945                     stl_p(buf, val);
1946                 } else if (l == 2) {
1947                     /* 16 bit read access */
1948                     error |= io_mem_read(mr, addr1, &val, 2);
1949                     stw_p(buf, val);
1950                 } else {
1951                     /* 8 bit read access */
1952                     error |= io_mem_read(mr, addr1, &val, 1);
1953                     stb_p(buf, val);
1954                 }
1955             } else {
1956                 /* RAM case */
1957                 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1958                 memcpy(buf, ptr, l);
1959             }
1960         }
1961         len -= l;
1962         buf += l;
1963         addr += l;
1964     }
1965
1966     return error;
1967 }
1968
1969 bool address_space_write(AddressSpace *as, hwaddr addr,
1970                          const uint8_t *buf, int len)
1971 {
1972     return address_space_rw(as, addr, (uint8_t *)buf, len, true);
1973 }
1974
1975 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1976 {
1977     return address_space_rw(as, addr, buf, len, false);
1978 }
1979
1980
1981 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1982                             int len, int is_write)
1983 {
1984     address_space_rw(&address_space_memory, addr, buf, len, is_write);
1985 }
1986
1987 /* used for ROM loading : can write in RAM and ROM */
1988 void cpu_physical_memory_write_rom(hwaddr addr,
1989                                    const uint8_t *buf, int len)
1990 {
1991     hwaddr l;
1992     uint8_t *ptr;
1993     hwaddr addr1;
1994     MemoryRegion *mr;
1995
1996     while (len > 0) {
1997         l = len;
1998         mr = address_space_translate(&address_space_memory,
1999                                      addr, &addr1, &l, true);
2000
2001         if (!(memory_region_is_ram(mr) ||
2002               memory_region_is_romd(mr))) {
2003             /* do nothing */
2004         } else {
2005             addr1 += memory_region_get_ram_addr(mr);
2006             /* ROM/RAM case */
2007             ptr = qemu_get_ram_ptr(addr1);
2008             memcpy(ptr, buf, l);
2009             invalidate_and_set_dirty(addr1, l);
2010         }
2011         len -= l;
2012         buf += l;
2013         addr += l;
2014     }
2015 }
2016
2017 typedef struct {
2018     MemoryRegion *mr;
2019     void *buffer;
2020     hwaddr addr;
2021     hwaddr len;
2022 } BounceBuffer;
2023
2024 static BounceBuffer bounce;
2025
2026 typedef struct MapClient {
2027     void *opaque;
2028     void (*callback)(void *opaque);
2029     QLIST_ENTRY(MapClient) link;
2030 } MapClient;
2031
2032 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2033     = QLIST_HEAD_INITIALIZER(map_client_list);
2034
2035 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2036 {
2037     MapClient *client = g_malloc(sizeof(*client));
2038
2039     client->opaque = opaque;
2040     client->callback = callback;
2041     QLIST_INSERT_HEAD(&map_client_list, client, link);
2042     return client;
2043 }
2044
2045 static void cpu_unregister_map_client(void *_client)
2046 {
2047     MapClient *client = (MapClient *)_client;
2048
2049     QLIST_REMOVE(client, link);
2050     g_free(client);
2051 }
2052
2053 static void cpu_notify_map_clients(void)
2054 {
2055     MapClient *client;
2056
2057     while (!QLIST_EMPTY(&map_client_list)) {
2058         client = QLIST_FIRST(&map_client_list);
2059         client->callback(client->opaque);
2060         cpu_unregister_map_client(client);
2061     }
2062 }
2063
2064 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2065 {
2066     MemoryRegion *mr;
2067     hwaddr l, xlat;
2068
2069     while (len > 0) {
2070         l = len;
2071         mr = address_space_translate(as, addr, &xlat, &l, is_write);
2072         if (!memory_access_is_direct(mr, is_write)) {
2073             l = memory_access_size(mr, l, addr);
2074             if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2075                 return false;
2076             }
2077         }
2078
2079         len -= l;
2080         addr += l;
2081     }
2082     return true;
2083 }
2084
2085 /* Map a physical memory region into a host virtual address.
2086  * May map a subset of the requested range, given by and returned in *plen.
2087  * May return NULL if resources needed to perform the mapping are exhausted.
2088  * Use only for reads OR writes - not for read-modify-write operations.
2089  * Use cpu_register_map_client() to know when retrying the map operation is
2090  * likely to succeed.
2091  */
2092 void *address_space_map(AddressSpace *as,
2093                         hwaddr addr,
2094                         hwaddr *plen,
2095                         bool is_write)
2096 {
2097     hwaddr len = *plen;
2098     hwaddr done = 0;
2099     hwaddr l, xlat, base;
2100     MemoryRegion *mr, *this_mr;
2101     ram_addr_t raddr;
2102
2103     if (len == 0) {
2104         return NULL;
2105     }
2106
2107     l = len;
2108     mr = address_space_translate(as, addr, &xlat, &l, is_write);
2109     if (!memory_access_is_direct(mr, is_write)) {
2110         if (bounce.buffer) {
2111             return NULL;
2112         }
2113         bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2114         bounce.addr = addr;
2115         bounce.len = l;
2116
2117         memory_region_ref(mr);
2118         bounce.mr = mr;
2119         if (!is_write) {
2120             address_space_read(as, addr, bounce.buffer, l);
2121         }
2122
2123         *plen = l;
2124         return bounce.buffer;
2125     }
2126
2127     base = xlat;
2128     raddr = memory_region_get_ram_addr(mr);
2129
2130     for (;;) {
2131         len -= l;
2132         addr += l;
2133         done += l;
2134         if (len == 0) {
2135             break;
2136         }
2137
2138         l = len;
2139         this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2140         if (this_mr != mr || xlat != base + done) {
2141             break;
2142         }
2143     }
2144
2145     memory_region_ref(mr);
2146     *plen = done;
2147     return qemu_ram_ptr_length(raddr + base, plen);
2148 }
2149
2150 /* Unmaps a memory region previously mapped by address_space_map().
2151  * Will also mark the memory as dirty if is_write == 1.  access_len gives
2152  * the amount of memory that was actually read or written by the caller.
2153  */
2154 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2155                          int is_write, hwaddr access_len)
2156 {
2157     if (buffer != bounce.buffer) {
2158         MemoryRegion *mr;
2159         ram_addr_t addr1;
2160
2161         mr = qemu_ram_addr_from_host(buffer, &addr1);
2162         assert(mr != NULL);
2163         if (is_write) {
2164             while (access_len) {
2165                 unsigned l;
2166                 l = TARGET_PAGE_SIZE;
2167                 if (l > access_len)
2168                     l = access_len;
2169                 invalidate_and_set_dirty(addr1, l);
2170                 addr1 += l;
2171                 access_len -= l;
2172             }
2173         }
2174         if (xen_enabled()) {
2175             xen_invalidate_map_cache_entry(buffer);
2176         }
2177         memory_region_unref(mr);
2178         return;
2179     }
2180     if (is_write) {
2181         address_space_write(as, bounce.addr, bounce.buffer, access_len);
2182     }
2183     qemu_vfree(bounce.buffer);
2184     bounce.buffer = NULL;
2185     memory_region_unref(bounce.mr);
2186     cpu_notify_map_clients();
2187 }
2188
2189 void *cpu_physical_memory_map(hwaddr addr,
2190                               hwaddr *plen,
2191                               int is_write)
2192 {
2193     return address_space_map(&address_space_memory, addr, plen, is_write);
2194 }
2195
2196 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2197                                int is_write, hwaddr access_len)
2198 {
2199     return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2200 }
2201
2202 /* warning: addr must be aligned */
2203 static inline uint32_t ldl_phys_internal(hwaddr addr,
2204                                          enum device_endian endian)
2205 {
2206     uint8_t *ptr;
2207     uint64_t val;
2208     MemoryRegion *mr;
2209     hwaddr l = 4;
2210     hwaddr addr1;
2211
2212     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2213                                  false);
2214     if (l < 4 || !memory_access_is_direct(mr, false)) {
2215         /* I/O case */
2216         io_mem_read(mr, addr1, &val, 4);
2217 #if defined(TARGET_WORDS_BIGENDIAN)
2218         if (endian == DEVICE_LITTLE_ENDIAN) {
2219             val = bswap32(val);
2220         }
2221 #else
2222         if (endian == DEVICE_BIG_ENDIAN) {
2223             val = bswap32(val);
2224         }
2225 #endif
2226     } else {
2227         /* RAM case */
2228         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2229                                 & TARGET_PAGE_MASK)
2230                                + addr1);
2231         switch (endian) {
2232         case DEVICE_LITTLE_ENDIAN:
2233             val = ldl_le_p(ptr);
2234             break;
2235         case DEVICE_BIG_ENDIAN:
2236             val = ldl_be_p(ptr);
2237             break;
2238         default:
2239             val = ldl_p(ptr);
2240             break;
2241         }
2242     }
2243     return val;
2244 }
2245
2246 uint32_t ldl_phys(hwaddr addr)
2247 {
2248     return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2249 }
2250
2251 uint32_t ldl_le_phys(hwaddr addr)
2252 {
2253     return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2254 }
2255
2256 uint32_t ldl_be_phys(hwaddr addr)
2257 {
2258     return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2259 }
2260
2261 /* warning: addr must be aligned */
2262 static inline uint64_t ldq_phys_internal(hwaddr addr,
2263                                          enum device_endian endian)
2264 {
2265     uint8_t *ptr;
2266     uint64_t val;
2267     MemoryRegion *mr;
2268     hwaddr l = 8;
2269     hwaddr addr1;
2270
2271     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2272                                  false);
2273     if (l < 8 || !memory_access_is_direct(mr, false)) {
2274         /* I/O case */
2275         io_mem_read(mr, addr1, &val, 8);
2276 #if defined(TARGET_WORDS_BIGENDIAN)
2277         if (endian == DEVICE_LITTLE_ENDIAN) {
2278             val = bswap64(val);
2279         }
2280 #else
2281         if (endian == DEVICE_BIG_ENDIAN) {
2282             val = bswap64(val);
2283         }
2284 #endif
2285     } else {
2286         /* RAM case */
2287         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2288                                 & TARGET_PAGE_MASK)
2289                                + addr1);
2290         switch (endian) {
2291         case DEVICE_LITTLE_ENDIAN:
2292             val = ldq_le_p(ptr);
2293             break;
2294         case DEVICE_BIG_ENDIAN:
2295             val = ldq_be_p(ptr);
2296             break;
2297         default:
2298             val = ldq_p(ptr);
2299             break;
2300         }
2301     }
2302     return val;
2303 }
2304
2305 uint64_t ldq_phys(hwaddr addr)
2306 {
2307     return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2308 }
2309
2310 uint64_t ldq_le_phys(hwaddr addr)
2311 {
2312     return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2313 }
2314
2315 uint64_t ldq_be_phys(hwaddr addr)
2316 {
2317     return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2318 }
2319
2320 /* XXX: optimize */
2321 uint32_t ldub_phys(hwaddr addr)
2322 {
2323     uint8_t val;
2324     cpu_physical_memory_read(addr, &val, 1);
2325     return val;
2326 }
2327
2328 /* warning: addr must be aligned */
2329 static inline uint32_t lduw_phys_internal(hwaddr addr,
2330                                           enum device_endian endian)
2331 {
2332     uint8_t *ptr;
2333     uint64_t val;
2334     MemoryRegion *mr;
2335     hwaddr l = 2;
2336     hwaddr addr1;
2337
2338     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2339                                  false);
2340     if (l < 2 || !memory_access_is_direct(mr, false)) {
2341         /* I/O case */
2342         io_mem_read(mr, addr1, &val, 2);
2343 #if defined(TARGET_WORDS_BIGENDIAN)
2344         if (endian == DEVICE_LITTLE_ENDIAN) {
2345             val = bswap16(val);
2346         }
2347 #else
2348         if (endian == DEVICE_BIG_ENDIAN) {
2349             val = bswap16(val);
2350         }
2351 #endif
2352     } else {
2353         /* RAM case */
2354         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2355                                 & TARGET_PAGE_MASK)
2356                                + addr1);
2357         switch (endian) {
2358         case DEVICE_LITTLE_ENDIAN:
2359             val = lduw_le_p(ptr);
2360             break;
2361         case DEVICE_BIG_ENDIAN:
2362             val = lduw_be_p(ptr);
2363             break;
2364         default:
2365             val = lduw_p(ptr);
2366             break;
2367         }
2368     }
2369     return val;
2370 }
2371
2372 uint32_t lduw_phys(hwaddr addr)
2373 {
2374     return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2375 }
2376
2377 uint32_t lduw_le_phys(hwaddr addr)
2378 {
2379     return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2380 }
2381
2382 uint32_t lduw_be_phys(hwaddr addr)
2383 {
2384     return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2385 }
2386
2387 /* warning: addr must be aligned. The ram page is not masked as dirty
2388    and the code inside is not invalidated. It is useful if the dirty
2389    bits are used to track modified PTEs */
2390 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2391 {
2392     uint8_t *ptr;
2393     MemoryRegion *mr;
2394     hwaddr l = 4;
2395     hwaddr addr1;
2396
2397     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2398                                  true);
2399     if (l < 4 || !memory_access_is_direct(mr, true)) {
2400         io_mem_write(mr, addr1, val, 4);
2401     } else {
2402         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2403         ptr = qemu_get_ram_ptr(addr1);
2404         stl_p(ptr, val);
2405
2406         if (unlikely(in_migration)) {
2407             if (!cpu_physical_memory_is_dirty(addr1)) {
2408                 /* invalidate code */
2409                 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2410                 /* set dirty bit */
2411                 cpu_physical_memory_set_dirty_flags(
2412                     addr1, (0xff & ~CODE_DIRTY_FLAG));
2413             }
2414         }
2415     }
2416 }
2417
2418 /* warning: addr must be aligned */
2419 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2420                                      enum device_endian endian)
2421 {
2422     uint8_t *ptr;
2423     MemoryRegion *mr;
2424     hwaddr l = 4;
2425     hwaddr addr1;
2426
2427     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2428                                  true);
2429     if (l < 4 || !memory_access_is_direct(mr, true)) {
2430 #if defined(TARGET_WORDS_BIGENDIAN)
2431         if (endian == DEVICE_LITTLE_ENDIAN) {
2432             val = bswap32(val);
2433         }
2434 #else
2435         if (endian == DEVICE_BIG_ENDIAN) {
2436             val = bswap32(val);
2437         }
2438 #endif
2439         io_mem_write(mr, addr1, val, 4);
2440     } else {
2441         /* RAM case */
2442         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2443         ptr = qemu_get_ram_ptr(addr1);
2444         switch (endian) {
2445         case DEVICE_LITTLE_ENDIAN:
2446             stl_le_p(ptr, val);
2447             break;
2448         case DEVICE_BIG_ENDIAN:
2449             stl_be_p(ptr, val);
2450             break;
2451         default:
2452             stl_p(ptr, val);
2453             break;
2454         }
2455         invalidate_and_set_dirty(addr1, 4);
2456     }
2457 }
2458
2459 void stl_phys(hwaddr addr, uint32_t val)
2460 {
2461     stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2462 }
2463
2464 void stl_le_phys(hwaddr addr, uint32_t val)
2465 {
2466     stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2467 }
2468
2469 void stl_be_phys(hwaddr addr, uint32_t val)
2470 {
2471     stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2472 }
2473
2474 /* XXX: optimize */
2475 void stb_phys(hwaddr addr, uint32_t val)
2476 {
2477     uint8_t v = val;
2478     cpu_physical_memory_write(addr, &v, 1);
2479 }
2480
2481 /* warning: addr must be aligned */
2482 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2483                                      enum device_endian endian)
2484 {
2485     uint8_t *ptr;
2486     MemoryRegion *mr;
2487     hwaddr l = 2;
2488     hwaddr addr1;
2489
2490     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2491                                  true);
2492     if (l < 2 || !memory_access_is_direct(mr, true)) {
2493 #if defined(TARGET_WORDS_BIGENDIAN)
2494         if (endian == DEVICE_LITTLE_ENDIAN) {
2495             val = bswap16(val);
2496         }
2497 #else
2498         if (endian == DEVICE_BIG_ENDIAN) {
2499             val = bswap16(val);
2500         }
2501 #endif
2502         io_mem_write(mr, addr1, val, 2);
2503     } else {
2504         /* RAM case */
2505         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2506         ptr = qemu_get_ram_ptr(addr1);
2507         switch (endian) {
2508         case DEVICE_LITTLE_ENDIAN:
2509             stw_le_p(ptr, val);
2510             break;
2511         case DEVICE_BIG_ENDIAN:
2512             stw_be_p(ptr, val);
2513             break;
2514         default:
2515             stw_p(ptr, val);
2516             break;
2517         }
2518         invalidate_and_set_dirty(addr1, 2);
2519     }
2520 }
2521
2522 void stw_phys(hwaddr addr, uint32_t val)
2523 {
2524     stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2525 }
2526
2527 void stw_le_phys(hwaddr addr, uint32_t val)
2528 {
2529     stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2530 }
2531
2532 void stw_be_phys(hwaddr addr, uint32_t val)
2533 {
2534     stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2535 }
2536
2537 /* XXX: optimize */
2538 void stq_phys(hwaddr addr, uint64_t val)
2539 {
2540     val = tswap64(val);
2541     cpu_physical_memory_write(addr, &val, 8);
2542 }
2543
2544 void stq_le_phys(hwaddr addr, uint64_t val)
2545 {
2546     val = cpu_to_le64(val);
2547     cpu_physical_memory_write(addr, &val, 8);
2548 }
2549
2550 void stq_be_phys(hwaddr addr, uint64_t val)
2551 {
2552     val = cpu_to_be64(val);
2553     cpu_physical_memory_write(addr, &val, 8);
2554 }
2555
2556 /* virtual memory access for debug (includes writing to ROM) */
2557 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2558                         uint8_t *buf, int len, int is_write)
2559 {
2560     int l;
2561     hwaddr phys_addr;
2562     target_ulong page;
2563
2564     while (len > 0) {
2565         page = addr & TARGET_PAGE_MASK;
2566         phys_addr = cpu_get_phys_page_debug(env, page);
2567         /* if no physical page mapped, return an error */
2568         if (phys_addr == -1)
2569             return -1;
2570         l = (page + TARGET_PAGE_SIZE) - addr;
2571         if (l > len)
2572             l = len;
2573         phys_addr += (addr & ~TARGET_PAGE_MASK);
2574         if (is_write)
2575             cpu_physical_memory_write_rom(phys_addr, buf, l);
2576         else
2577             cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2578         len -= l;
2579         buf += l;
2580         addr += l;
2581     }
2582     return 0;
2583 }
2584 #endif
2585
2586 #if !defined(CONFIG_USER_ONLY)
2587
2588 /*
2589  * A helper function for the _utterly broken_ virtio device model to find out if
2590  * it's running on a big endian machine. Don't do this at home kids!
2591  */
2592 bool virtio_is_big_endian(void);
2593 bool virtio_is_big_endian(void)
2594 {
2595 #if defined(TARGET_WORDS_BIGENDIAN)
2596     return true;
2597 #else
2598     return false;
2599 #endif
2600 }
2601
2602 #endif
2603
2604 #ifndef CONFIG_USER_ONLY
2605 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2606 {
2607     MemoryRegion*mr;
2608     hwaddr l = 1;
2609
2610     mr = address_space_translate(&address_space_memory,
2611                                  phys_addr, &phys_addr, &l, false);
2612
2613     return !(memory_region_is_ram(mr) ||
2614              memory_region_is_romd(mr));
2615 }
2616
2617 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2618 {
2619     RAMBlock *block;
2620
2621     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2622         func(block->host, block->offset, block->length, opaque);
2623     }
2624 }
2625 #endif