exec.c

   1 /*
   2  *  Virtual page mapping
   3  *
   4  *  Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19 #include "config.h"
  20 #ifdef _WIN32
  21 #include <windows.h>
  22 #else
  23 #include <sys/types.h>
  24 #include <sys/mman.h>
  25 #endif
  26
  27 #include "qemu-common.h"
  28 #include "cpu.h"
  29 #include "tcg.h"
  30 #include "hw/hw.h"
  31 #include "hw/qdev.h"
  32 #include "qemu/osdep.h"
  33 #include "sysemu/kvm.h"
  34 #include "hw/xen/xen.h"
  35 #include "qemu/timer.h"
  36 #include "qemu/config-file.h"
  37 #include "exec/memory.h"
  38 #include "sysemu/dma.h"
  39 #include "exec/address-spaces.h"
  40 #if defined(CONFIG_USER_ONLY)
  41 #include <qemu.h>
  42 #else /* !CONFIG_USER_ONLY */
  43 #include "sysemu/xen-mapcache.h"
  44 #include "trace.h"
  45 #endif
  46 #include "exec/cpu-all.h"
  47
  48 #include "exec/cputlb.h"
  49 #include "translate-all.h"
  50
  51 #include "exec/memory-internal.h"
  52
  53 //#define DEBUG_SUBPAGE
  54
  55 #if !defined(CONFIG_USER_ONLY)
  56 int phys_ram_fd;
  57 static int in_migration;
  58
  59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
  60
  61 static MemoryRegion *system_memory;
  62 static MemoryRegion *system_io;
  63
  64 AddressSpace address_space_io;
  65 AddressSpace address_space_memory;
  66
  67 MemoryRegion io_mem_rom, io_mem_notdirty;
  68 static MemoryRegion io_mem_unassigned;
  69
  70 #endif
  71
  72 CPUArchState *first_cpu;
  73 /* current CPU in the current thread. It is only valid inside
  74    cpu_exec() */
  75 DEFINE_TLS(CPUArchState *,cpu_single_env);
  76 /* 0 = Do not count executed instructions.
  77    1 = Precise instruction counting.
  78    2 = Adaptive rate instruction counting.  */
  79 int use_icount;
  80
  81 #if !defined(CONFIG_USER_ONLY)
  82
  83 typedef struct PhysPageEntry PhysPageEntry;
  84
  85 struct PhysPageEntry {
  86     uint16_t is_leaf : 1;
  87      /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
  88     uint16_t ptr : 15;
  89 };
  90
  91 struct AddressSpaceDispatch {
  92     /* This is a multi-level map on the physical address space.
  93      * The bottom level has pointers to MemoryRegionSections.
  94      */
  95     PhysPageEntry phys_map;
  96     MemoryListener listener;
  97     AddressSpace *as;
  98 };
  99
 100 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
 101 typedef struct subpage_t {
 102     MemoryRegion iomem;
 103     AddressSpace *as;
 104     hwaddr base;
 105     uint16_t sub_section[TARGET_PAGE_SIZE];
 106 } subpage_t;
 107
 108 static MemoryRegionSection *phys_sections;
 109 static unsigned phys_sections_nb, phys_sections_nb_alloc;
 110 static uint16_t phys_section_unassigned;
 111 static uint16_t phys_section_notdirty;
 112 static uint16_t phys_section_rom;
 113 static uint16_t phys_section_watch;
 114
 115 /* Simple allocator for PhysPageEntry nodes */
 116 static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
 117 static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
 118
 119 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
 120
 121 static void io_mem_init(void);
 122 static void memory_map_init(void);
 123 static void *qemu_safe_ram_ptr(ram_addr_t addr);
 124
 125 static MemoryRegion io_mem_watch;
 126 #endif
 127
 128 #if !defined(CONFIG_USER_ONLY)
 129
 130 static void phys_map_node_reserve(unsigned nodes)
 131 {
 132     if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) {
 133         typedef PhysPageEntry Node[L2_SIZE];
 134         phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16);
 135         phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc,
 136                                       phys_map_nodes_nb + nodes);
 137         phys_map_nodes = g_renew(Node, phys_map_nodes,
 138                                  phys_map_nodes_nb_alloc);
 139     }
 140 }
 141
 142 static uint16_t phys_map_node_alloc(void)
 143 {
 144     unsigned i;
 145     uint16_t ret;
 146
 147     ret = phys_map_nodes_nb++;
 148     assert(ret != PHYS_MAP_NODE_NIL);
 149     assert(ret != phys_map_nodes_nb_alloc);
 150     for (i = 0; i < L2_SIZE; ++i) {
 151         phys_map_nodes[ret][i].is_leaf = 0;
 152         phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
 153     }
 154     return ret;
 155 }
 156
 157 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
 158                                 hwaddr *nb, uint16_t leaf,
 159                                 int level)
 160 {
 161     PhysPageEntry *p;
 162     int i;
 163     hwaddr step = (hwaddr)1 << (level * L2_BITS);
 164
 165     if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
 166         lp->ptr = phys_map_node_alloc();
 167         p = phys_map_nodes[lp->ptr];
 168         if (level == 0) {
 169             for (i = 0; i < L2_SIZE; i++) {
 170                 p[i].is_leaf = 1;
 171                 p[i].ptr = phys_section_unassigned;
 172             }
 173         }
 174     } else {
 175         p = phys_map_nodes[lp->ptr];
 176     }
 177     lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
 178
 179     while (*nb && lp < &p[L2_SIZE]) {
 180         if ((*index & (step - 1)) == 0 && *nb >= step) {
 181             lp->is_leaf = true;
 182             lp->ptr = leaf;
 183             *index += step;
 184             *nb -= step;
 185         } else {
 186             phys_page_set_level(lp, index, nb, leaf, level - 1);
 187         }
 188         ++lp;
 189     }
 190 }
 191
 192 static void phys_page_set(AddressSpaceDispatch *d,
 193                           hwaddr index, hwaddr nb,
 194                           uint16_t leaf)
 195 {
 196     /* Wildly overreserve - it doesn't matter much. */
 197     phys_map_node_reserve(3 * P_L2_LEVELS);
 198
 199     phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
 200 }
 201
 202 static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr index)
 203 {
 204     PhysPageEntry lp = d->phys_map;
 205     PhysPageEntry *p;
 206     int i;
 207
 208     for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
 209         if (lp.ptr == PHYS_MAP_NODE_NIL) {
 210             return &phys_sections[phys_section_unassigned];
 211         }
 212         p = phys_map_nodes[lp.ptr];
 213         lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
 214     }
 215     return &phys_sections[lp.ptr];
 216 }
 217
 218 bool memory_region_is_unassigned(MemoryRegion *mr)
 219 {
 220     return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
 221         && mr != &io_mem_watch;
 222 }
 223
 224 static MemoryRegionSection *address_space_lookup_region(AddressSpace *as,
 225                                                         hwaddr addr,
 226                                                         bool resolve_subpage)
 227 {
 228     MemoryRegionSection *section;
 229     subpage_t *subpage;
 230
 231     section = phys_page_find(as->dispatch, addr >> TARGET_PAGE_BITS);
 232     if (resolve_subpage && section->mr->subpage) {
 233         subpage = container_of(section->mr, subpage_t, iomem);
 234         section = &phys_sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
 235     }
 236     return section;
 237 }
 238
 239 static MemoryRegionSection *
 240 address_space_translate_internal(AddressSpace *as, hwaddr addr, hwaddr *xlat,
 241                                  hwaddr *plen, bool resolve_subpage)
 242 {
 243     MemoryRegionSection *section;
 244     Int128 diff;
 245
 246     section = address_space_lookup_region(as, addr, resolve_subpage);
 247     /* Compute offset within MemoryRegionSection */
 248     addr -= section->offset_within_address_space;
 249
 250     /* Compute offset within MemoryRegion */
 251     *xlat = addr + section->offset_within_region;
 252
 253     diff = int128_sub(section->mr->size, int128_make64(addr));
 254     *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
 255     return section;
 256 }
 257
 258 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
 259                                       hwaddr *xlat, hwaddr *plen,
 260                                       bool is_write)
 261 {
 262     IOMMUTLBEntry iotlb;
 263     MemoryRegionSection *section;
 264     MemoryRegion *mr;
 265     hwaddr len = *plen;
 266
 267     for (;;) {
 268         section = address_space_translate_internal(as, addr, &addr, plen, true);
 269         mr = section->mr;
 270
 271         if (!mr->iommu_ops) {
 272             break;
 273         }
 274
 275         iotlb = mr->iommu_ops->translate(mr, addr);
 276         addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
 277                 | (addr & iotlb.addr_mask));
 278         len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
 279         if (!(iotlb.perm & (1 << is_write))) {
 280             mr = &io_mem_unassigned;
 281             break;
 282         }
 283
 284         as = iotlb.target_as;
 285     }
 286
 287     *plen = len;
 288     *xlat = addr;
 289     return mr;
 290 }
 291
 292 MemoryRegionSection *
 293 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
 294                                   hwaddr *plen)
 295 {
 296     MemoryRegionSection *section;
 297     section = address_space_translate_internal(as, addr, xlat, plen, false);
 298
 299     assert(!section->mr->iommu_ops);
 300     return section;
 301 }
 302 #endif
 303
 304 void cpu_exec_init_all(void)
 305 {
 306 #if !defined(CONFIG_USER_ONLY)
 307     qemu_mutex_init(&ram_list.mutex);
 308     memory_map_init();
 309     io_mem_init();
 310 #endif
 311 }
 312
 313 #if !defined(CONFIG_USER_ONLY)
 314
 315 static int cpu_common_post_load(void *opaque, int version_id)
 316 {
 317     CPUState *cpu = opaque;
 318
 319     /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
 320        version_id is increased. */
 321     cpu->interrupt_request &= ~0x01;
 322     tlb_flush(cpu->env_ptr, 1);
 323
 324     return 0;
 325 }
 326
 327 const VMStateDescription vmstate_cpu_common = {
 328     .name = "cpu_common",
 329     .version_id = 1,
 330     .minimum_version_id = 1,
 331     .minimum_version_id_old = 1,
 332     .post_load = cpu_common_post_load,
 333     .fields      = (VMStateField []) {
 334         VMSTATE_UINT32(halted, CPUState),
 335         VMSTATE_UINT32(interrupt_request, CPUState),
 336         VMSTATE_END_OF_LIST()
 337     }
 338 };
 339
 340 #endif
 341
 342 CPUState *qemu_get_cpu(int index)
 343 {
 344     CPUArchState *env = first_cpu;
 345     CPUState *cpu = NULL;
 346
 347     while (env) {
 348         cpu = ENV_GET_CPU(env);
 349         if (cpu->cpu_index == index) {
 350             break;
 351         }
 352         env = env->next_cpu;
 353     }
 354
 355     return env ? cpu : NULL;
 356 }
 357
 358 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
 359 {
 360     CPUArchState *env = first_cpu;
 361
 362     while (env) {
 363         func(ENV_GET_CPU(env), data);
 364         env = env->next_cpu;
 365     }
 366 }
 367
 368 void cpu_exec_init(CPUArchState *env)
 369 {
 370     CPUState *cpu = ENV_GET_CPU(env);
 371     CPUClass *cc = CPU_GET_CLASS(cpu);
 372     CPUArchState **penv;
 373     int cpu_index;
 374
 375 #if defined(CONFIG_USER_ONLY)
 376     cpu_list_lock();
 377 #endif
 378     env->next_cpu = NULL;
 379     penv = &first_cpu;
 380     cpu_index = 0;
 381     while (*penv != NULL) {
 382         penv = &(*penv)->next_cpu;
 383         cpu_index++;
 384     }
 385     cpu->cpu_index = cpu_index;
 386     cpu->numa_node = 0;
 387     QTAILQ_INIT(&env->breakpoints);
 388     QTAILQ_INIT(&env->watchpoints);
 389 #ifndef CONFIG_USER_ONLY
 390     cpu->thread_id = qemu_get_thread_id();
 391 #endif
 392     *penv = env;
 393 #if defined(CONFIG_USER_ONLY)
 394     cpu_list_unlock();
 395 #endif
 396     vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
 397 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
 398     register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
 399                     cpu_save, cpu_load, env);
 400     assert(cc->vmsd == NULL);
 401 #endif
 402     if (cc->vmsd != NULL) {
 403         vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
 404     }
 405 }
 406
 407 #if defined(TARGET_HAS_ICE)
 408 #if defined(CONFIG_USER_ONLY)
 409 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
 410 {
 411     tb_invalidate_phys_page_range(pc, pc + 1, 0);
 412 }
 413 #else
 414 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
 415 {
 416     tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
 417             (pc & ~TARGET_PAGE_MASK));
 418 }
 419 #endif
 420 #endif /* TARGET_HAS_ICE */
 421
 422 #if defined(CONFIG_USER_ONLY)
 423 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 424
 425 {
 426 }
 427
 428 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 429                           int flags, CPUWatchpoint **watchpoint)
 430 {
 431     return -ENOSYS;
 432 }
 433 #else
 434 /* Add a watchpoint.  */
 435 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 436                           int flags, CPUWatchpoint **watchpoint)
 437 {
 438     target_ulong len_mask = ~(len - 1);
 439     CPUWatchpoint *wp;
 440
 441     /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
 442     if ((len & (len - 1)) || (addr & ~len_mask) ||
 443             len == 0 || len > TARGET_PAGE_SIZE) {
 444         fprintf(stderr, "qemu: tried to set invalid watchpoint at "
 445                 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
 446         return -EINVAL;
 447     }
 448     wp = g_malloc(sizeof(*wp));
 449
 450     wp->vaddr = addr;
 451     wp->len_mask = len_mask;
 452     wp->flags = flags;
 453
 454     /* keep all GDB-injected watchpoints in front */
 455     if (flags & BP_GDB)
 456         QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
 457     else
 458         QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
 459
 460     tlb_flush_page(env, addr);
 461
 462     if (watchpoint)
 463         *watchpoint = wp;
 464     return 0;
 465 }
 466
 467 /* Remove a specific watchpoint.  */
 468 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
 469                           int flags)
 470 {
 471     target_ulong len_mask = ~(len - 1);
 472     CPUWatchpoint *wp;
 473
 474     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 475         if (addr == wp->vaddr && len_mask == wp->len_mask
 476                 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
 477             cpu_watchpoint_remove_by_ref(env, wp);
 478             return 0;
 479         }
 480     }
 481     return -ENOENT;
 482 }
 483
 484 /* Remove a specific watchpoint by reference.  */
 485 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
 486 {
 487     QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
 488
 489     tlb_flush_page(env, watchpoint->vaddr);
 490
 491     g_free(watchpoint);
 492 }
 493
 494 /* Remove all matching watchpoints.  */
 495 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 496 {
 497     CPUWatchpoint *wp, *next;
 498
 499     QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
 500         if (wp->flags & mask)
 501             cpu_watchpoint_remove_by_ref(env, wp);
 502     }
 503 }
 504 #endif
 505
 506 /* Add a breakpoint.  */
 507 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
 508                           CPUBreakpoint **breakpoint)
 509 {
 510 #if defined(TARGET_HAS_ICE)
 511     CPUBreakpoint *bp;
 512
 513     bp = g_malloc(sizeof(*bp));
 514
 515     bp->pc = pc;
 516     bp->flags = flags;
 517
 518     /* keep all GDB-injected breakpoints in front */
 519     if (flags & BP_GDB)
 520         QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
 521     else
 522         QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
 523
 524     breakpoint_invalidate(env, pc);
 525
 526     if (breakpoint)
 527         *breakpoint = bp;
 528     return 0;
 529 #else
 530     return -ENOSYS;
 531 #endif
 532 }
 533
 534 /* Remove a specific breakpoint.  */
 535 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
 536 {
 537 #if defined(TARGET_HAS_ICE)
 538     CPUBreakpoint *bp;
 539
 540     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
 541         if (bp->pc == pc && bp->flags == flags) {
 542             cpu_breakpoint_remove_by_ref(env, bp);
 543             return 0;
 544         }
 545     }
 546     return -ENOENT;
 547 #else
 548     return -ENOSYS;
 549 #endif
 550 }
 551
 552 /* Remove a specific breakpoint by reference.  */
 553 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
 554 {
 555 #if defined(TARGET_HAS_ICE)
 556     QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
 557
 558     breakpoint_invalidate(env, breakpoint->pc);
 559
 560     g_free(breakpoint);
 561 #endif
 562 }
 563
 564 /* Remove all matching breakpoints. */
 565 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
 566 {
 567 #if defined(TARGET_HAS_ICE)
 568     CPUBreakpoint *bp, *next;
 569
 570     QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
 571         if (bp->flags & mask)
 572             cpu_breakpoint_remove_by_ref(env, bp);
 573     }
 574 #endif
 575 }
 576
 577 /* enable or disable single step mode. EXCP_DEBUG is returned by the
 578    CPU loop after each instruction */
 579 void cpu_single_step(CPUArchState *env, int enabled)
 580 {
 581 #if defined(TARGET_HAS_ICE)
 582     if (env->singlestep_enabled != enabled) {
 583         env->singlestep_enabled = enabled;
 584         if (kvm_enabled())
 585             kvm_update_guest_debug(env, 0);
 586         else {
 587             /* must flush all the translated code to avoid inconsistencies */
 588             /* XXX: only flush what is necessary */
 589             tb_flush(env);
 590         }
 591     }
 592 #endif
 593 }
 594
 595 void cpu_abort(CPUArchState *env, const char *fmt, ...)
 596 {
 597     CPUState *cpu = ENV_GET_CPU(env);
 598     va_list ap;
 599     va_list ap2;
 600
 601     va_start(ap, fmt);
 602     va_copy(ap2, ap);
 603     fprintf(stderr, "qemu: fatal: ");
 604     vfprintf(stderr, fmt, ap);
 605     fprintf(stderr, "\n");
 606     cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 607     if (qemu_log_enabled()) {
 608         qemu_log("qemu: fatal: ");
 609         qemu_log_vprintf(fmt, ap2);
 610         qemu_log("\n");
 611         log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 612         qemu_log_flush();
 613         qemu_log_close();
 614     }
 615     va_end(ap2);
 616     va_end(ap);
 617 #if defined(CONFIG_USER_ONLY)
 618     {
 619         struct sigaction act;
 620         sigfillset(&act.sa_mask);
 621         act.sa_handler = SIG_DFL;
 622         sigaction(SIGABRT, &act, NULL);
 623     }
 624 #endif
 625     abort();
 626 }
 627
 628 CPUArchState *cpu_copy(CPUArchState *env)
 629 {
 630     CPUArchState *new_env = cpu_init(env->cpu_model_str);
 631     CPUArchState *next_cpu = new_env->next_cpu;
 632 #if defined(TARGET_HAS_ICE)
 633     CPUBreakpoint *bp;
 634     CPUWatchpoint *wp;
 635 #endif
 636
 637     memcpy(new_env, env, sizeof(CPUArchState));
 638
 639     /* Preserve chaining. */
 640     new_env->next_cpu = next_cpu;
 641
 642     /* Clone all break/watchpoints.
 643        Note: Once we support ptrace with hw-debug register access, make sure
 644        BP_CPU break/watchpoints are handled correctly on clone. */
 645     QTAILQ_INIT(&env->breakpoints);
 646     QTAILQ_INIT(&env->watchpoints);
 647 #if defined(TARGET_HAS_ICE)
 648     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
 649         cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
 650     }
 651     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 652         cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
 653                               wp->flags, NULL);
 654     }
 655 #endif
 656
 657     return new_env;
 658 }
 659
 660 #if !defined(CONFIG_USER_ONLY)
 661 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
 662                                       uintptr_t length)
 663 {
 664     uintptr_t start1;
 665
 666     /* we modify the TLB cache so that the dirty bit will be set again
 667        when accessing the range */
 668     start1 = (uintptr_t)qemu_safe_ram_ptr(start);
 669     /* Check that we don't span multiple blocks - this breaks the
 670        address comparisons below.  */
 671     if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
 672             != (end - 1) - start) {
 673         abort();
 674     }
 675     cpu_tlb_reset_dirty_all(start1, length);
 676
 677 }
 678
 679 /* Note: start and end must be within the same ram block.  */
 680 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
 681                                      int dirty_flags)
 682 {
 683     uintptr_t length;
 684
 685     start &= TARGET_PAGE_MASK;
 686     end = TARGET_PAGE_ALIGN(end);
 687
 688     length = end - start;
 689     if (length == 0)
 690         return;
 691     cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
 692
 693     if (tcg_enabled()) {
 694         tlb_reset_dirty_range_all(start, end, length);
 695     }
 696 }
 697
 698 static int cpu_physical_memory_set_dirty_tracking(int enable)
 699 {
 700     int ret = 0;
 701     in_migration = enable;
 702     return ret;
 703 }
 704
 705 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
 706                                        MemoryRegionSection *section,
 707                                        target_ulong vaddr,
 708                                        hwaddr paddr, hwaddr xlat,
 709                                        int prot,
 710                                        target_ulong *address)
 711 {
 712     hwaddr iotlb;
 713     CPUWatchpoint *wp;
 714
 715     if (memory_region_is_ram(section->mr)) {
 716         /* Normal RAM.  */
 717         iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
 718             + xlat;
 719         if (!section->readonly) {
 720             iotlb |= phys_section_notdirty;
 721         } else {
 722             iotlb |= phys_section_rom;
 723         }
 724     } else {
 725         iotlb = section - phys_sections;
 726         iotlb += xlat;
 727     }
 728
 729     /* Make accesses to pages with watchpoints go via the
 730        watchpoint trap routines.  */
 731     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 732         if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
 733             /* Avoid trapping reads of pages with a write breakpoint. */
 734             if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
 735                 iotlb = phys_section_watch + paddr;
 736                 *address |= TLB_MMIO;
 737                 break;
 738             }
 739         }
 740     }
 741
 742     return iotlb;
 743 }
 744 #endif /* defined(CONFIG_USER_ONLY) */
 745
 746 #if !defined(CONFIG_USER_ONLY)
 747
 748 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
 749                              uint16_t section);
 750 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
 751
 752 static uint16_t phys_section_add(MemoryRegionSection *section)
 753 {
 754     /* The physical section number is ORed with a page-aligned
 755      * pointer to produce the iotlb entries.  Thus it should
 756      * never overflow into the page-aligned value.
 757      */
 758     assert(phys_sections_nb < TARGET_PAGE_SIZE);
 759
 760     if (phys_sections_nb == phys_sections_nb_alloc) {
 761         phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16);
 762         phys_sections = g_renew(MemoryRegionSection, phys_sections,
 763                                 phys_sections_nb_alloc);
 764     }
 765     phys_sections[phys_sections_nb] = *section;
 766     return phys_sections_nb++;
 767 }
 768
 769 static void phys_section_destroy(MemoryRegion *mr)
 770 {
 771     if (mr->subpage) {
 772         subpage_t *subpage = container_of(mr, subpage_t, iomem);
 773         memory_region_destroy(&subpage->iomem);
 774         g_free(subpage);
 775     }
 776 }
 777
 778 static void phys_sections_clear(void)
 779 {
 780     while (phys_sections_nb > 0) {
 781         MemoryRegionSection *section = &phys_sections[--phys_sections_nb];
 782         phys_section_destroy(section->mr);
 783     }
 784     phys_map_nodes_nb = 0;
 785 }
 786
 787 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
 788 {
 789     subpage_t *subpage;
 790     hwaddr base = section->offset_within_address_space
 791         & TARGET_PAGE_MASK;
 792     MemoryRegionSection *existing = phys_page_find(d, base >> TARGET_PAGE_BITS);
 793     MemoryRegionSection subsection = {
 794         .offset_within_address_space = base,
 795         .size = int128_make64(TARGET_PAGE_SIZE),
 796     };
 797     hwaddr start, end;
 798
 799     assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
 800
 801     if (!(existing->mr->subpage)) {
 802         subpage = subpage_init(d->as, base);
 803         subsection.mr = &subpage->iomem;
 804         phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
 805                       phys_section_add(&subsection));
 806     } else {
 807         subpage = container_of(existing->mr, subpage_t, iomem);
 808     }
 809     start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
 810     end = start + int128_get64(section->size) - 1;
 811     subpage_register(subpage, start, end, phys_section_add(section));
 812 }
 813
 814
 815 static void register_multipage(AddressSpaceDispatch *d,
 816                                MemoryRegionSection *section)
 817 {
 818     hwaddr start_addr = section->offset_within_address_space;
 819     uint16_t section_index = phys_section_add(section);
 820     uint64_t num_pages = int128_get64(int128_rshift(section->size,
 821                                                     TARGET_PAGE_BITS));
 822
 823     assert(num_pages);
 824     phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
 825 }
 826
 827 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
 828 {
 829     AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
 830     MemoryRegionSection now = *section, remain = *section;
 831     Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
 832
 833     if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
 834         uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
 835                        - now.offset_within_address_space;
 836
 837         now.size = int128_min(int128_make64(left), now.size);
 838         register_subpage(d, &now);
 839     } else {
 840         now.size = int128_zero();
 841     }
 842     while (int128_ne(remain.size, now.size)) {
 843         remain.size = int128_sub(remain.size, now.size);
 844         remain.offset_within_address_space += int128_get64(now.size);
 845         remain.offset_within_region += int128_get64(now.size);
 846         now = remain;
 847         if (int128_lt(remain.size, page_size)) {
 848             register_subpage(d, &now);
 849         } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
 850             now.size = page_size;
 851             register_subpage(d, &now);
 852         } else {
 853             now.size = int128_and(now.size, int128_neg(page_size));
 854             register_multipage(d, &now);
 855         }
 856     }
 857 }
 858
 859 void qemu_flush_coalesced_mmio_buffer(void)
 860 {
 861     if (kvm_enabled())
 862         kvm_flush_coalesced_mmio_buffer();
 863 }
 864
 865 void qemu_mutex_lock_ramlist(void)
 866 {
 867     qemu_mutex_lock(&ram_list.mutex);
 868 }
 869
 870 void qemu_mutex_unlock_ramlist(void)
 871 {
 872     qemu_mutex_unlock(&ram_list.mutex);
 873 }
 874
 875 #if defined(__linux__) && !defined(TARGET_S390X)
 876
 877 #include <sys/vfs.h>
 878
 879 #define HUGETLBFS_MAGIC       0x958458f6
 880
 881 static long gethugepagesize(const char *path)
 882 {
 883     struct statfs fs;
 884     int ret;
 885
 886     do {
 887         ret = statfs(path, &fs);
 888     } while (ret != 0 && errno == EINTR);
 889
 890     if (ret != 0) {
 891         perror(path);
 892         return 0;
 893     }
 894
 895     if (fs.f_type != HUGETLBFS_MAGIC)
 896         fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
 897
 898     return fs.f_bsize;
 899 }
 900
 901 static void *file_ram_alloc(RAMBlock *block,
 902                             ram_addr_t memory,
 903                             const char *path)
 904 {
 905     char *filename;
 906     char *sanitized_name;
 907     char *c;
 908     void *area;
 909     int fd;
 910 #ifdef MAP_POPULATE
 911     int flags;
 912 #endif
 913     unsigned long hpagesize;
 914
 915     hpagesize = gethugepagesize(path);
 916     if (!hpagesize) {
 917         return NULL;
 918     }
 919
 920     if (memory < hpagesize) {
 921         return NULL;
 922     }
 923
 924     if (kvm_enabled() && !kvm_has_sync_mmu()) {
 925         fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
 926         return NULL;
 927     }
 928
 929     /* Make name safe to use with mkstemp by replacing '/' with '_'. */
 930     sanitized_name = g_strdup(block->mr->name);
 931     for (c = sanitized_name; *c != '\0'; c++) {
 932         if (*c == '/')
 933             *c = '_';
 934     }
 935
 936     filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
 937                                sanitized_name);
 938     g_free(sanitized_name);
 939
 940     fd = mkstemp(filename);
 941     if (fd < 0) {
 942         perror("unable to create backing store for hugepages");
 943         g_free(filename);
 944         return NULL;
 945     }
 946     unlink(filename);
 947     g_free(filename);
 948
 949     memory = (memory+hpagesize-1) & ~(hpagesize-1);
 950
 951     /*
 952      * ftruncate is not supported by hugetlbfs in older
 953      * hosts, so don't bother bailing out on errors.
 954      * If anything goes wrong with it under other filesystems,
 955      * mmap will fail.
 956      */
 957     if (ftruncate(fd, memory))
 958         perror("ftruncate");
 959
 960 #ifdef MAP_POPULATE
 961     /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
 962      * MAP_PRIVATE is requested.  For mem_prealloc we mmap as MAP_SHARED
 963      * to sidestep this quirk.
 964      */
 965     flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
 966     area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
 967 #else
 968     area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
 969 #endif
 970     if (area == MAP_FAILED) {
 971         perror("file_ram_alloc: can't mmap RAM pages");
 972         close(fd);
 973         return (NULL);
 974     }
 975     block->fd = fd;
 976     return area;
 977 }
 978 #endif
 979
 980 static ram_addr_t find_ram_offset(ram_addr_t size)
 981 {
 982     RAMBlock *block, *next_block;
 983     ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
 984
 985     assert(size != 0); /* it would hand out same offset multiple times */
 986
 987     if (QTAILQ_EMPTY(&ram_list.blocks))
 988         return 0;
 989
 990     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
 991         ram_addr_t end, next = RAM_ADDR_MAX;
 992
 993         end = block->offset + block->length;
 994
 995         QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
 996             if (next_block->offset >= end) {
 997                 next = MIN(next, next_block->offset);
 998             }
 999         }
1000         if (next - end >= size && next - end < mingap) {
1001             offset = end;
1002             mingap = next - end;
1003         }
1004     }
1005
1006     if (offset == RAM_ADDR_MAX) {
1007         fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1008                 (uint64_t)size);
1009         abort();
1010     }
1011
1012     return offset;
1013 }
1014
1015 ram_addr_t last_ram_offset(void)
1016 {
1017     RAMBlock *block;
1018     ram_addr_t last = 0;
1019
1020     QTAILQ_FOREACH(block, &ram_list.blocks, next)
1021         last = MAX(last, block->offset + block->length);
1022
1023     return last;
1024 }
1025
1026 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1027 {
1028     int ret;
1029     QemuOpts *machine_opts;
1030
1031     /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1032     machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1033     if (machine_opts &&
1034         !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
1035         ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1036         if (ret) {
1037             perror("qemu_madvise");
1038             fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1039                             "but dump_guest_core=off specified\n");
1040         }
1041     }
1042 }
1043
1044 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1045 {
1046     RAMBlock *new_block, *block;
1047
1048     new_block = NULL;
1049     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1050         if (block->offset == addr) {
1051             new_block = block;
1052             break;
1053         }
1054     }
1055     assert(new_block);
1056     assert(!new_block->idstr[0]);
1057
1058     if (dev) {
1059         char *id = qdev_get_dev_path(dev);
1060         if (id) {
1061             snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1062             g_free(id);
1063         }
1064     }
1065     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1066
1067     /* This assumes the iothread lock is taken here too.  */
1068     qemu_mutex_lock_ramlist();
1069     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1070         if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1071             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1072                     new_block->idstr);
1073             abort();
1074         }
1075     }
1076     qemu_mutex_unlock_ramlist();
1077 }
1078
1079 static int memory_try_enable_merging(void *addr, size_t len)
1080 {
1081     QemuOpts *opts;
1082
1083     opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1084     if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
1085         /* disabled by the user */
1086         return 0;
1087     }
1088
1089     return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1090 }
1091
1092 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1093                                    MemoryRegion *mr)
1094 {
1095     RAMBlock *block, *new_block;
1096
1097     size = TARGET_PAGE_ALIGN(size);
1098     new_block = g_malloc0(sizeof(*new_block));
1099
1100     /* This assumes the iothread lock is taken here too.  */
1101     qemu_mutex_lock_ramlist();
1102     new_block->mr = mr;
1103     new_block->offset = find_ram_offset(size);
1104     if (host) {
1105         new_block->host = host;
1106         new_block->flags |= RAM_PREALLOC_MASK;
1107     } else {
1108         if (mem_path) {
1109 #if defined (__linux__) && !defined(TARGET_S390X)
1110             new_block->host = file_ram_alloc(new_block, size, mem_path);
1111             if (!new_block->host) {
1112                 new_block->host = qemu_anon_ram_alloc(size);
1113                 memory_try_enable_merging(new_block->host, size);
1114             }
1115 #else
1116             fprintf(stderr, "-mem-path option unsupported\n");
1117             exit(1);
1118 #endif
1119         } else {
1120             if (xen_enabled()) {
1121                 xen_ram_alloc(new_block->offset, size, mr);
1122             } else if (kvm_enabled()) {
1123                 /* some s390/kvm configurations have special constraints */
1124                 new_block->host = kvm_ram_alloc(size);
1125             } else {
1126                 new_block->host = qemu_anon_ram_alloc(size);
1127             }
1128             memory_try_enable_merging(new_block->host, size);
1129         }
1130     }
1131     new_block->length = size;
1132
1133     /* Keep the list sorted from biggest to smallest block.  */
1134     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1135         if (block->length < new_block->length) {
1136             break;
1137         }
1138     }
1139     if (block) {
1140         QTAILQ_INSERT_BEFORE(block, new_block, next);
1141     } else {
1142         QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1143     }
1144     ram_list.mru_block = NULL;
1145
1146     ram_list.version++;
1147     qemu_mutex_unlock_ramlist();
1148
1149     ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1150                                        last_ram_offset() >> TARGET_PAGE_BITS);
1151     memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1152            0, size >> TARGET_PAGE_BITS);
1153     cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1154
1155     qemu_ram_setup_dump(new_block->host, size);
1156     qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1157
1158     if (kvm_enabled())
1159         kvm_setup_guest_memory(new_block->host, size);
1160
1161     return new_block->offset;
1162 }
1163
1164 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1165 {
1166     return qemu_ram_alloc_from_ptr(size, NULL, mr);
1167 }
1168
1169 void qemu_ram_free_from_ptr(ram_addr_t addr)
1170 {
1171     RAMBlock *block;
1172
1173     /* This assumes the iothread lock is taken here too.  */
1174     qemu_mutex_lock_ramlist();
1175     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1176         if (addr == block->offset) {
1177             QTAILQ_REMOVE(&ram_list.blocks, block, next);
1178             ram_list.mru_block = NULL;
1179             ram_list.version++;
1180             g_free(block);
1181             break;
1182         }
1183     }
1184     qemu_mutex_unlock_ramlist();
1185 }
1186
1187 void qemu_ram_free(ram_addr_t addr)
1188 {
1189     RAMBlock *block;
1190
1191     /* This assumes the iothread lock is taken here too.  */
1192     qemu_mutex_lock_ramlist();
1193     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1194         if (addr == block->offset) {
1195             QTAILQ_REMOVE(&ram_list.blocks, block, next);
1196             ram_list.mru_block = NULL;
1197             ram_list.version++;
1198             if (block->flags & RAM_PREALLOC_MASK) {
1199                 ;
1200             } else if (mem_path) {
1201 #if defined (__linux__) && !defined(TARGET_S390X)
1202                 if (block->fd) {
1203                     munmap(block->host, block->length);
1204                     close(block->fd);
1205                 } else {
1206                     qemu_anon_ram_free(block->host, block->length);
1207                 }
1208 #else
1209                 abort();
1210 #endif
1211             } else {
1212                 if (xen_enabled()) {
1213                     xen_invalidate_map_cache_entry(block->host);
1214                 } else {
1215                     qemu_anon_ram_free(block->host, block->length);
1216                 }
1217             }
1218             g_free(block);
1219             break;
1220         }
1221     }
1222     qemu_mutex_unlock_ramlist();
1223
1224 }
1225
1226 #ifndef _WIN32
1227 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1228 {
1229     RAMBlock *block;
1230     ram_addr_t offset;
1231     int flags;
1232     void *area, *vaddr;
1233
1234     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1235         offset = addr - block->offset;
1236         if (offset < block->length) {
1237             vaddr = block->host + offset;
1238             if (block->flags & RAM_PREALLOC_MASK) {
1239                 ;
1240             } else {
1241                 flags = MAP_FIXED;
1242                 munmap(vaddr, length);
1243                 if (mem_path) {
1244 #if defined(__linux__) && !defined(TARGET_S390X)
1245                     if (block->fd) {
1246 #ifdef MAP_POPULATE
1247                         flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1248                             MAP_PRIVATE;
1249 #else
1250                         flags |= MAP_PRIVATE;
1251 #endif
1252                         area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1253                                     flags, block->fd, offset);
1254                     } else {
1255                         flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1256                         area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1257                                     flags, -1, 0);
1258                     }
1259 #else
1260                     abort();
1261 #endif
1262                 } else {
1263 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1264                     flags |= MAP_SHARED | MAP_ANONYMOUS;
1265                     area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1266                                 flags, -1, 0);
1267 #else
1268                     flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1269                     area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1270                                 flags, -1, 0);
1271 #endif
1272                 }
1273                 if (area != vaddr) {
1274                     fprintf(stderr, "Could not remap addr: "
1275                             RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1276                             length, addr);
1277                     exit(1);
1278                 }
1279                 memory_try_enable_merging(vaddr, length);
1280                 qemu_ram_setup_dump(vaddr, length);
1281             }
1282             return;
1283         }
1284     }
1285 }
1286 #endif /* !_WIN32 */
1287
1288 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1289    With the exception of the softmmu code in this file, this should
1290    only be used for local memory (e.g. video ram) that the device owns,
1291    and knows it isn't going to access beyond the end of the block.
1292
1293    It should not be used for general purpose DMA.
1294    Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1295  */
1296 void *qemu_get_ram_ptr(ram_addr_t addr)
1297 {
1298     RAMBlock *block;
1299
1300     /* The list is protected by the iothread lock here.  */
1301     block = ram_list.mru_block;
1302     if (block && addr - block->offset < block->length) {
1303         goto found;
1304     }
1305     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1306         if (addr - block->offset < block->length) {
1307             goto found;
1308         }
1309     }
1310
1311     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1312     abort();
1313
1314 found:
1315     ram_list.mru_block = block;
1316     if (xen_enabled()) {
1317         /* We need to check if the requested address is in the RAM
1318          * because we don't want to map the entire memory in QEMU.
1319          * In that case just map until the end of the page.
1320          */
1321         if (block->offset == 0) {
1322             return xen_map_cache(addr, 0, 0);
1323         } else if (block->host == NULL) {
1324             block->host =
1325                 xen_map_cache(block->offset, block->length, 1);
1326         }
1327     }
1328     return block->host + (addr - block->offset);
1329 }
1330
1331 /* Return a host pointer to ram allocated with qemu_ram_alloc.  Same as
1332  * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1333  *
1334  * ??? Is this still necessary?
1335  */
1336 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1337 {
1338     RAMBlock *block;
1339
1340     /* The list is protected by the iothread lock here.  */
1341     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1342         if (addr - block->offset < block->length) {
1343             if (xen_enabled()) {
1344                 /* We need to check if the requested address is in the RAM
1345                  * because we don't want to map the entire memory in QEMU.
1346                  * In that case just map until the end of the page.
1347                  */
1348                 if (block->offset == 0) {
1349                     return xen_map_cache(addr, 0, 0);
1350                 } else if (block->host == NULL) {
1351                     block->host =
1352                         xen_map_cache(block->offset, block->length, 1);
1353                 }
1354             }
1355             return block->host + (addr - block->offset);
1356         }
1357     }
1358
1359     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1360     abort();
1361
1362     return NULL;
1363 }
1364
1365 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1366  * but takes a size argument */
1367 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1368 {
1369     if (*size == 0) {
1370         return NULL;
1371     }
1372     if (xen_enabled()) {
1373         return xen_map_cache(addr, *size, 1);
1374     } else {
1375         RAMBlock *block;
1376
1377         QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1378             if (addr - block->offset < block->length) {
1379                 if (addr - block->offset + *size > block->length)
1380                     *size = block->length - addr + block->offset;
1381                 return block->host + (addr - block->offset);
1382             }
1383         }
1384
1385         fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1386         abort();
1387     }
1388 }
1389
1390 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1391 {
1392     RAMBlock *block;
1393     uint8_t *host = ptr;
1394
1395     if (xen_enabled()) {
1396         *ram_addr = xen_ram_addr_from_mapcache(ptr);
1397         return 0;
1398     }
1399
1400     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1401         /* This case append when the block is not mapped. */
1402         if (block->host == NULL) {
1403             continue;
1404         }
1405         if (host - block->host < block->length) {
1406             *ram_addr = block->offset + (host - block->host);
1407             return 0;
1408         }
1409     }
1410
1411     return -1;
1412 }
1413
1414 /* Some of the softmmu routines need to translate from a host pointer
1415    (typically a TLB entry) back to a ram offset.  */
1416 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
1417 {
1418     ram_addr_t ram_addr;
1419
1420     if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
1421         fprintf(stderr, "Bad ram pointer %p\n", ptr);
1422         abort();
1423     }
1424     return ram_addr;
1425 }
1426
1427 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1428                                uint64_t val, unsigned size)
1429 {
1430     int dirty_flags;
1431     dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1432     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1433         tb_invalidate_phys_page_fast(ram_addr, size);
1434         dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1435     }
1436     switch (size) {
1437     case 1:
1438         stb_p(qemu_get_ram_ptr(ram_addr), val);
1439         break;
1440     case 2:
1441         stw_p(qemu_get_ram_ptr(ram_addr), val);
1442         break;
1443     case 4:
1444         stl_p(qemu_get_ram_ptr(ram_addr), val);
1445         break;
1446     default:
1447         abort();
1448     }
1449     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1450     cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1451     /* we remove the notdirty callback only if the code has been
1452        flushed */
1453     if (dirty_flags == 0xff)
1454         tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1455 }
1456
1457 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1458                                  unsigned size, bool is_write)
1459 {
1460     return is_write;
1461 }
1462
1463 static const MemoryRegionOps notdirty_mem_ops = {
1464     .write = notdirty_mem_write,
1465     .valid.accepts = notdirty_mem_accepts,
1466     .endianness = DEVICE_NATIVE_ENDIAN,
1467 };
1468
1469 /* Generate a debug exception if a watchpoint has been hit.  */
1470 static void check_watchpoint(int offset, int len_mask, int flags)
1471 {
1472     CPUArchState *env = cpu_single_env;
1473     target_ulong pc, cs_base;
1474     target_ulong vaddr;
1475     CPUWatchpoint *wp;
1476     int cpu_flags;
1477
1478     if (env->watchpoint_hit) {
1479         /* We re-entered the check after replacing the TB. Now raise
1480          * the debug interrupt so that is will trigger after the
1481          * current instruction. */
1482         cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1483         return;
1484     }
1485     vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1486     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1487         if ((vaddr == (wp->vaddr & len_mask) ||
1488              (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1489             wp->flags |= BP_WATCHPOINT_HIT;
1490             if (!env->watchpoint_hit) {
1491                 env->watchpoint_hit = wp;
1492                 tb_check_watchpoint(env);
1493                 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1494                     env->exception_index = EXCP_DEBUG;
1495                     cpu_loop_exit(env);
1496                 } else {
1497                     cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1498                     tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1499                     cpu_resume_from_signal(env, NULL);
1500                 }
1501             }
1502         } else {
1503             wp->flags &= ~BP_WATCHPOINT_HIT;
1504         }
1505     }
1506 }
1507
1508 /* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
1509    so these check for a hit then pass through to the normal out-of-line
1510    phys routines.  */
1511 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1512                                unsigned size)
1513 {
1514     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1515     switch (size) {
1516     case 1: return ldub_phys(addr);
1517     case 2: return lduw_phys(addr);
1518     case 4: return ldl_phys(addr);
1519     default: abort();
1520     }
1521 }
1522
1523 static void watch_mem_write(void *opaque, hwaddr addr,
1524                             uint64_t val, unsigned size)
1525 {
1526     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1527     switch (size) {
1528     case 1:
1529         stb_phys(addr, val);
1530         break;
1531     case 2:
1532         stw_phys(addr, val);
1533         break;
1534     case 4:
1535         stl_phys(addr, val);
1536         break;
1537     default: abort();
1538     }
1539 }
1540
1541 static const MemoryRegionOps watch_mem_ops = {
1542     .read = watch_mem_read,
1543     .write = watch_mem_write,
1544     .endianness = DEVICE_NATIVE_ENDIAN,
1545 };
1546
1547 static uint64_t subpage_read(void *opaque, hwaddr addr,
1548                              unsigned len)
1549 {
1550     subpage_t *subpage = opaque;
1551     uint8_t buf[4];
1552
1553 #if defined(DEBUG_SUBPAGE)
1554     printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1555            subpage, len, addr);
1556 #endif
1557     address_space_read(subpage->as, addr + subpage->base, buf, len);
1558     switch (len) {
1559     case 1:
1560         return ldub_p(buf);
1561     case 2:
1562         return lduw_p(buf);
1563     case 4:
1564         return ldl_p(buf);
1565     default:
1566         abort();
1567     }
1568 }
1569
1570 static void subpage_write(void *opaque, hwaddr addr,
1571                           uint64_t value, unsigned len)
1572 {
1573     subpage_t *subpage = opaque;
1574     uint8_t buf[4];
1575
1576 #if defined(DEBUG_SUBPAGE)
1577     printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1578            " value %"PRIx64"\n",
1579            __func__, subpage, len, addr, value);
1580 #endif
1581     switch (len) {
1582     case 1:
1583         stb_p(buf, value);
1584         break;
1585     case 2:
1586         stw_p(buf, value);
1587         break;
1588     case 4:
1589         stl_p(buf, value);
1590         break;
1591     default:
1592         abort();
1593     }
1594     address_space_write(subpage->as, addr + subpage->base, buf, len);
1595 }
1596
1597 static bool subpage_accepts(void *opaque, hwaddr addr,
1598                             unsigned size, bool is_write)
1599 {
1600     subpage_t *subpage = opaque;
1601 #if defined(DEBUG_SUBPAGE)
1602     printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1603            __func__, subpage, is_write ? 'w' : 'r', len, addr);
1604 #endif
1605
1606     return address_space_access_valid(subpage->as, addr + subpage->base,
1607                                       size, is_write);
1608 }
1609
1610 static const MemoryRegionOps subpage_ops = {
1611     .read = subpage_read,
1612     .write = subpage_write,
1613     .valid.accepts = subpage_accepts,
1614     .endianness = DEVICE_NATIVE_ENDIAN,
1615 };
1616
1617 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1618                              uint16_t section)
1619 {
1620     int idx, eidx;
1621
1622     if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1623         return -1;
1624     idx = SUBPAGE_IDX(start);
1625     eidx = SUBPAGE_IDX(end);
1626 #if defined(DEBUG_SUBPAGE)
1627     printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1628            mmio, start, end, idx, eidx, memory);
1629 #endif
1630     for (; idx <= eidx; idx++) {
1631         mmio->sub_section[idx] = section;
1632     }
1633
1634     return 0;
1635 }
1636
1637 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1638 {
1639     subpage_t *mmio;
1640
1641     mmio = g_malloc0(sizeof(subpage_t));
1642
1643     mmio->as = as;
1644     mmio->base = base;
1645     memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1646                           "subpage", TARGET_PAGE_SIZE);
1647     mmio->iomem.subpage = true;
1648 #if defined(DEBUG_SUBPAGE)
1649     printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1650            mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1651 #endif
1652     subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, phys_section_unassigned);
1653
1654     return mmio;
1655 }
1656
1657 static uint16_t dummy_section(MemoryRegion *mr)
1658 {
1659     MemoryRegionSection section = {
1660         .mr = mr,
1661         .offset_within_address_space = 0,
1662         .offset_within_region = 0,
1663         .size = int128_2_64(),
1664     };
1665
1666     return phys_section_add(&section);
1667 }
1668
1669 MemoryRegion *iotlb_to_region(hwaddr index)
1670 {
1671     return phys_sections[index & ~TARGET_PAGE_MASK].mr;
1672 }
1673
1674 static void io_mem_init(void)
1675 {
1676     memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1677     memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1678                           "unassigned", UINT64_MAX);
1679     memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1680                           "notdirty", UINT64_MAX);
1681     memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1682                           "watch", UINT64_MAX);
1683 }
1684
1685 static void mem_begin(MemoryListener *listener)
1686 {
1687     AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
1688
1689     d->phys_map.ptr = PHYS_MAP_NODE_NIL;
1690 }
1691
1692 static void core_begin(MemoryListener *listener)
1693 {
1694     phys_sections_clear();
1695     phys_section_unassigned = dummy_section(&io_mem_unassigned);
1696     phys_section_notdirty = dummy_section(&io_mem_notdirty);
1697     phys_section_rom = dummy_section(&io_mem_rom);
1698     phys_section_watch = dummy_section(&io_mem_watch);
1699 }
1700
1701 static void tcg_commit(MemoryListener *listener)
1702 {
1703     CPUArchState *env;
1704
1705     /* since each CPU stores ram addresses in its TLB cache, we must
1706        reset the modified entries */
1707     /* XXX: slow ! */
1708     for(env = first_cpu; env != NULL; env = env->next_cpu) {
1709         tlb_flush(env, 1);
1710     }
1711 }
1712
1713 static void core_log_global_start(MemoryListener *listener)
1714 {
1715     cpu_physical_memory_set_dirty_tracking(1);
1716 }
1717
1718 static void core_log_global_stop(MemoryListener *listener)
1719 {
1720     cpu_physical_memory_set_dirty_tracking(0);
1721 }
1722
1723 static MemoryListener core_memory_listener = {
1724     .begin = core_begin,
1725     .log_global_start = core_log_global_start,
1726     .log_global_stop = core_log_global_stop,
1727     .priority = 1,
1728 };
1729
1730 static MemoryListener tcg_memory_listener = {
1731     .commit = tcg_commit,
1732 };
1733
1734 void address_space_init_dispatch(AddressSpace *as)
1735 {
1736     AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1737
1738     d->phys_map  = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1739     d->listener = (MemoryListener) {
1740         .begin = mem_begin,
1741         .region_add = mem_add,
1742         .region_nop = mem_add,
1743         .priority = 0,
1744     };
1745     d->as = as;
1746     as->dispatch = d;
1747     memory_listener_register(&d->listener, as);
1748 }
1749
1750 void address_space_destroy_dispatch(AddressSpace *as)
1751 {
1752     AddressSpaceDispatch *d = as->dispatch;
1753
1754     memory_listener_unregister(&d->listener);
1755     g_free(d);
1756     as->dispatch = NULL;
1757 }
1758
1759 static void memory_map_init(void)
1760 {
1761     system_memory = g_malloc(sizeof(*system_memory));
1762     memory_region_init(system_memory, NULL, "system", INT64_MAX);
1763     address_space_init(&address_space_memory, system_memory, "memory");
1764
1765     system_io = g_malloc(sizeof(*system_io));
1766     memory_region_init(system_io, NULL, "io", 65536);
1767     address_space_init(&address_space_io, system_io, "I/O");
1768
1769     memory_listener_register(&core_memory_listener, &address_space_memory);
1770     memory_listener_register(&tcg_memory_listener, &address_space_memory);
1771 }
1772
1773 MemoryRegion *get_system_memory(void)
1774 {
1775     return system_memory;
1776 }
1777
1778 MemoryRegion *get_system_io(void)
1779 {
1780     return system_io;
1781 }
1782
1783 #endif /* !defined(CONFIG_USER_ONLY) */
1784
1785 /* physical memory access (slow version, mainly for debug) */
1786 #if defined(CONFIG_USER_ONLY)
1787 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1788                         uint8_t *buf, int len, int is_write)
1789 {
1790     int l, flags;
1791     target_ulong page;
1792     void * p;
1793
1794     while (len > 0) {
1795         page = addr & TARGET_PAGE_MASK;
1796         l = (page + TARGET_PAGE_SIZE) - addr;
1797         if (l > len)
1798             l = len;
1799         flags = page_get_flags(page);
1800         if (!(flags & PAGE_VALID))
1801             return -1;
1802         if (is_write) {
1803             if (!(flags & PAGE_WRITE))
1804                 return -1;
1805             /* XXX: this code should not depend on lock_user */
1806             if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1807                 return -1;
1808             memcpy(p, buf, l);
1809             unlock_user(p, addr, l);
1810         } else {
1811             if (!(flags & PAGE_READ))
1812                 return -1;
1813             /* XXX: this code should not depend on lock_user */
1814             if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1815                 return -1;
1816             memcpy(buf, p, l);
1817             unlock_user(p, addr, 0);
1818         }
1819         len -= l;
1820         buf += l;
1821         addr += l;
1822     }
1823     return 0;
1824 }
1825
1826 #else
1827
1828 static void invalidate_and_set_dirty(hwaddr addr,
1829                                      hwaddr length)
1830 {
1831     if (!cpu_physical_memory_is_dirty(addr)) {
1832         /* invalidate code */
1833         tb_invalidate_phys_page_range(addr, addr + length, 0);
1834         /* set dirty bit */
1835         cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1836     }
1837     xen_modified_memory(addr, length);
1838 }
1839
1840 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1841 {
1842     if (memory_region_is_ram(mr)) {
1843         return !(is_write && mr->readonly);
1844     }
1845     if (memory_region_is_romd(mr)) {
1846         return !is_write;
1847     }
1848
1849     return false;
1850 }
1851
1852 static inline int memory_access_size(MemoryRegion *mr, int l, hwaddr addr)
1853 {
1854     if (l >= 4 && (((addr & 3) == 0 || mr->ops->impl.unaligned))) {
1855         return 4;
1856     }
1857     if (l >= 2 && (((addr & 1) == 0) || mr->ops->impl.unaligned)) {
1858         return 2;
1859     }
1860     return 1;
1861 }
1862
1863 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1864                       int len, bool is_write)
1865 {
1866     hwaddr l;
1867     uint8_t *ptr;
1868     uint64_t val;
1869     hwaddr addr1;
1870     MemoryRegion *mr;
1871     bool error = false;
1872
1873     while (len > 0) {
1874         l = len;
1875         mr = address_space_translate(as, addr, &addr1, &l, is_write);
1876
1877         if (is_write) {
1878             if (!memory_access_is_direct(mr, is_write)) {
1879                 l = memory_access_size(mr, l, addr1);
1880                 /* XXX: could force cpu_single_env to NULL to avoid
1881                    potential bugs */
1882                 if (l == 4) {
1883                     /* 32 bit write access */
1884                     val = ldl_p(buf);
1885                     error |= io_mem_write(mr, addr1, val, 4);
1886                 } else if (l == 2) {
1887                     /* 16 bit write access */
1888                     val = lduw_p(buf);
1889                     error |= io_mem_write(mr, addr1, val, 2);
1890                 } else {
1891                     /* 8 bit write access */
1892                     val = ldub_p(buf);
1893                     error |= io_mem_write(mr, addr1, val, 1);
1894                 }
1895             } else {
1896                 addr1 += memory_region_get_ram_addr(mr);
1897                 /* RAM case */
1898                 ptr = qemu_get_ram_ptr(addr1);
1899                 memcpy(ptr, buf, l);
1900                 invalidate_and_set_dirty(addr1, l);
1901             }
1902         } else {
1903             if (!memory_access_is_direct(mr, is_write)) {
1904                 /* I/O case */
1905                 l = memory_access_size(mr, l, addr1);
1906                 if (l == 4) {
1907                     /* 32 bit read access */
1908                     error |= io_mem_read(mr, addr1, &val, 4);
1909                     stl_p(buf, val);
1910                 } else if (l == 2) {
1911                     /* 16 bit read access */
1912                     error |= io_mem_read(mr, addr1, &val, 2);
1913                     stw_p(buf, val);
1914                 } else {
1915                     /* 8 bit read access */
1916                     error |= io_mem_read(mr, addr1, &val, 1);
1917                     stb_p(buf, val);
1918                 }
1919             } else {
1920                 /* RAM case */
1921                 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1922                 memcpy(buf, ptr, l);
1923             }
1924         }
1925         len -= l;
1926         buf += l;
1927         addr += l;
1928     }
1929
1930     return error;
1931 }
1932
1933 bool address_space_write(AddressSpace *as, hwaddr addr,
1934                          const uint8_t *buf, int len)
1935 {
1936     return address_space_rw(as, addr, (uint8_t *)buf, len, true);
1937 }
1938
1939 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1940 {
1941     return address_space_rw(as, addr, buf, len, false);
1942 }
1943
1944
1945 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1946                             int len, int is_write)
1947 {
1948     address_space_rw(&address_space_memory, addr, buf, len, is_write);
1949 }
1950
1951 /* used for ROM loading : can write in RAM and ROM */
1952 void cpu_physical_memory_write_rom(hwaddr addr,
1953                                    const uint8_t *buf, int len)
1954 {
1955     hwaddr l;
1956     uint8_t *ptr;
1957     hwaddr addr1;
1958     MemoryRegion *mr;
1959
1960     while (len > 0) {
1961         l = len;
1962         mr = address_space_translate(&address_space_memory,
1963                                      addr, &addr1, &l, true);
1964
1965         if (!(memory_region_is_ram(mr) ||
1966               memory_region_is_romd(mr))) {
1967             /* do nothing */
1968         } else {
1969             addr1 += memory_region_get_ram_addr(mr);
1970             /* ROM/RAM case */
1971             ptr = qemu_get_ram_ptr(addr1);
1972             memcpy(ptr, buf, l);
1973             invalidate_and_set_dirty(addr1, l);
1974         }
1975         len -= l;
1976         buf += l;
1977         addr += l;
1978     }
1979 }
1980
1981 typedef struct {
1982     void *buffer;
1983     hwaddr addr;
1984     hwaddr len;
1985 } BounceBuffer;
1986
1987 static BounceBuffer bounce;
1988
1989 typedef struct MapClient {
1990     void *opaque;
1991     void (*callback)(void *opaque);
1992     QLIST_ENTRY(MapClient) link;
1993 } MapClient;
1994
1995 static QLIST_HEAD(map_client_list, MapClient) map_client_list
1996     = QLIST_HEAD_INITIALIZER(map_client_list);
1997
1998 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
1999 {
2000     MapClient *client = g_malloc(sizeof(*client));
2001
2002     client->opaque = opaque;
2003     client->callback = callback;
2004     QLIST_INSERT_HEAD(&map_client_list, client, link);
2005     return client;
2006 }
2007
2008 static void cpu_unregister_map_client(void *_client)
2009 {
2010     MapClient *client = (MapClient *)_client;
2011
2012     QLIST_REMOVE(client, link);
2013     g_free(client);
2014 }
2015
2016 static void cpu_notify_map_clients(void)
2017 {
2018     MapClient *client;
2019
2020     while (!QLIST_EMPTY(&map_client_list)) {
2021         client = QLIST_FIRST(&map_client_list);
2022         client->callback(client->opaque);
2023         cpu_unregister_map_client(client);
2024     }
2025 }
2026
2027 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2028 {
2029     MemoryRegion *mr;
2030     hwaddr l, xlat;
2031
2032     while (len > 0) {
2033         l = len;
2034         mr = address_space_translate(as, addr, &xlat, &l, is_write);
2035         if (!memory_access_is_direct(mr, is_write)) {
2036             l = memory_access_size(mr, l, addr);
2037             if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2038                 return false;
2039             }
2040         }
2041
2042         len -= l;
2043         addr += l;
2044     }
2045     return true;
2046 }
2047
2048 /* Map a physical memory region into a host virtual address.
2049  * May map a subset of the requested range, given by and returned in *plen.
2050  * May return NULL if resources needed to perform the mapping are exhausted.
2051  * Use only for reads OR writes - not for read-modify-write operations.
2052  * Use cpu_register_map_client() to know when retrying the map operation is
2053  * likely to succeed.
2054  */
2055 void *address_space_map(AddressSpace *as,
2056                         hwaddr addr,
2057                         hwaddr *plen,
2058                         bool is_write)
2059 {
2060     hwaddr len = *plen;
2061     hwaddr todo = 0;
2062     hwaddr l, xlat;
2063     MemoryRegion *mr;
2064     ram_addr_t raddr = RAM_ADDR_MAX;
2065     ram_addr_t rlen;
2066     void *ret;
2067
2068     while (len > 0) {
2069         l = len;
2070         mr = address_space_translate(as, addr, &xlat, &l, is_write);
2071
2072         if (!memory_access_is_direct(mr, is_write)) {
2073             if (todo || bounce.buffer) {
2074                 break;
2075             }
2076             bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2077             bounce.addr = addr;
2078             bounce.len = l;
2079             if (!is_write) {
2080                 address_space_read(as, addr, bounce.buffer, l);
2081             }
2082
2083             *plen = l;
2084             return bounce.buffer;
2085         }
2086         if (!todo) {
2087             raddr = memory_region_get_ram_addr(mr) + xlat;
2088         } else {
2089             if (memory_region_get_ram_addr(mr) + xlat != raddr + todo) {
2090                 break;
2091             }
2092         }
2093
2094         len -= l;
2095         addr += l;
2096         todo += l;
2097     }
2098     rlen = todo;
2099     ret = qemu_ram_ptr_length(raddr, &rlen);
2100     *plen = rlen;
2101     return ret;
2102 }
2103
2104 /* Unmaps a memory region previously mapped by address_space_map().
2105  * Will also mark the memory as dirty if is_write == 1.  access_len gives
2106  * the amount of memory that was actually read or written by the caller.
2107  */
2108 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2109                          int is_write, hwaddr access_len)
2110 {
2111     if (buffer != bounce.buffer) {
2112         if (is_write) {
2113             ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
2114             while (access_len) {
2115                 unsigned l;
2116                 l = TARGET_PAGE_SIZE;
2117                 if (l > access_len)
2118                     l = access_len;
2119                 invalidate_and_set_dirty(addr1, l);
2120                 addr1 += l;
2121                 access_len -= l;
2122             }
2123         }
2124         if (xen_enabled()) {
2125             xen_invalidate_map_cache_entry(buffer);
2126         }
2127         return;
2128     }
2129     if (is_write) {
2130         address_space_write(as, bounce.addr, bounce.buffer, access_len);
2131     }
2132     qemu_vfree(bounce.buffer);
2133     bounce.buffer = NULL;
2134     cpu_notify_map_clients();
2135 }
2136
2137 void *cpu_physical_memory_map(hwaddr addr,
2138                               hwaddr *plen,
2139                               int is_write)
2140 {
2141     return address_space_map(&address_space_memory, addr, plen, is_write);
2142 }
2143
2144 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2145                                int is_write, hwaddr access_len)
2146 {
2147     return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2148 }
2149
2150 /* warning: addr must be aligned */
2151 static inline uint32_t ldl_phys_internal(hwaddr addr,
2152                                          enum device_endian endian)
2153 {
2154     uint8_t *ptr;
2155     uint64_t val;
2156     MemoryRegion *mr;
2157     hwaddr l = 4;
2158     hwaddr addr1;
2159
2160     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2161                                  false);
2162     if (l < 4 || !memory_access_is_direct(mr, false)) {
2163         /* I/O case */
2164         io_mem_read(mr, addr1, &val, 4);
2165 #if defined(TARGET_WORDS_BIGENDIAN)
2166         if (endian == DEVICE_LITTLE_ENDIAN) {
2167             val = bswap32(val);
2168         }
2169 #else
2170         if (endian == DEVICE_BIG_ENDIAN) {
2171             val = bswap32(val);
2172         }
2173 #endif
2174     } else {
2175         /* RAM case */
2176         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2177                                 & TARGET_PAGE_MASK)
2178                                + addr1);
2179         switch (endian) {
2180         case DEVICE_LITTLE_ENDIAN:
2181             val = ldl_le_p(ptr);
2182             break;
2183         case DEVICE_BIG_ENDIAN:
2184             val = ldl_be_p(ptr);
2185             break;
2186         default:
2187             val = ldl_p(ptr);
2188             break;
2189         }
2190     }
2191     return val;
2192 }
2193
2194 uint32_t ldl_phys(hwaddr addr)
2195 {
2196     return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2197 }
2198
2199 uint32_t ldl_le_phys(hwaddr addr)
2200 {
2201     return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2202 }
2203
2204 uint32_t ldl_be_phys(hwaddr addr)
2205 {
2206     return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2207 }
2208
2209 /* warning: addr must be aligned */
2210 static inline uint64_t ldq_phys_internal(hwaddr addr,
2211                                          enum device_endian endian)
2212 {
2213     uint8_t *ptr;
2214     uint64_t val;
2215     MemoryRegion *mr;
2216     hwaddr l = 8;
2217     hwaddr addr1;
2218
2219     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2220                                  false);
2221     if (l < 8 || !memory_access_is_direct(mr, false)) {
2222         /* I/O case */
2223         io_mem_read(mr, addr1, &val, 8);
2224 #if defined(TARGET_WORDS_BIGENDIAN)
2225         if (endian == DEVICE_LITTLE_ENDIAN) {
2226             val = bswap64(val);
2227         }
2228 #else
2229         if (endian == DEVICE_BIG_ENDIAN) {
2230             val = bswap64(val);
2231         }
2232 #endif
2233     } else {
2234         /* RAM case */
2235         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2236                                 & TARGET_PAGE_MASK)
2237                                + addr1);
2238         switch (endian) {
2239         case DEVICE_LITTLE_ENDIAN:
2240             val = ldq_le_p(ptr);
2241             break;
2242         case DEVICE_BIG_ENDIAN:
2243             val = ldq_be_p(ptr);
2244             break;
2245         default:
2246             val = ldq_p(ptr);
2247             break;
2248         }
2249     }
2250     return val;
2251 }
2252
2253 uint64_t ldq_phys(hwaddr addr)
2254 {
2255     return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2256 }
2257
2258 uint64_t ldq_le_phys(hwaddr addr)
2259 {
2260     return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2261 }
2262
2263 uint64_t ldq_be_phys(hwaddr addr)
2264 {
2265     return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2266 }
2267
2268 /* XXX: optimize */
2269 uint32_t ldub_phys(hwaddr addr)
2270 {
2271     uint8_t val;
2272     cpu_physical_memory_read(addr, &val, 1);
2273     return val;
2274 }
2275
2276 /* warning: addr must be aligned */
2277 static inline uint32_t lduw_phys_internal(hwaddr addr,
2278                                           enum device_endian endian)
2279 {
2280     uint8_t *ptr;
2281     uint64_t val;
2282     MemoryRegion *mr;
2283     hwaddr l = 2;
2284     hwaddr addr1;
2285
2286     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2287                                  false);
2288     if (l < 2 || !memory_access_is_direct(mr, false)) {
2289         /* I/O case */
2290         io_mem_read(mr, addr1, &val, 2);
2291 #if defined(TARGET_WORDS_BIGENDIAN)
2292         if (endian == DEVICE_LITTLE_ENDIAN) {
2293             val = bswap16(val);
2294         }
2295 #else
2296         if (endian == DEVICE_BIG_ENDIAN) {
2297             val = bswap16(val);
2298         }
2299 #endif
2300     } else {
2301         /* RAM case */
2302         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2303                                 & TARGET_PAGE_MASK)
2304                                + addr1);
2305         switch (endian) {
2306         case DEVICE_LITTLE_ENDIAN:
2307             val = lduw_le_p(ptr);
2308             break;
2309         case DEVICE_BIG_ENDIAN:
2310             val = lduw_be_p(ptr);
2311             break;
2312         default:
2313             val = lduw_p(ptr);
2314             break;
2315         }
2316     }
2317     return val;
2318 }
2319
2320 uint32_t lduw_phys(hwaddr addr)
2321 {
2322     return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2323 }
2324
2325 uint32_t lduw_le_phys(hwaddr addr)
2326 {
2327     return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2328 }
2329
2330 uint32_t lduw_be_phys(hwaddr addr)
2331 {
2332     return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2333 }
2334
2335 /* warning: addr must be aligned. The ram page is not masked as dirty
2336    and the code inside is not invalidated. It is useful if the dirty
2337    bits are used to track modified PTEs */
2338 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2339 {
2340     uint8_t *ptr;
2341     MemoryRegion *mr;
2342     hwaddr l = 4;
2343     hwaddr addr1;
2344
2345     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2346                                  true);
2347     if (l < 4 || !memory_access_is_direct(mr, true)) {
2348         io_mem_write(mr, addr1, val, 4);
2349     } else {
2350         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2351         ptr = qemu_get_ram_ptr(addr1);
2352         stl_p(ptr, val);
2353
2354         if (unlikely(in_migration)) {
2355             if (!cpu_physical_memory_is_dirty(addr1)) {
2356                 /* invalidate code */
2357                 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2358                 /* set dirty bit */
2359                 cpu_physical_memory_set_dirty_flags(
2360                     addr1, (0xff & ~CODE_DIRTY_FLAG));
2361             }
2362         }
2363     }
2364 }
2365
2366 /* warning: addr must be aligned */
2367 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2368                                      enum device_endian endian)
2369 {
2370     uint8_t *ptr;
2371     MemoryRegion *mr;
2372     hwaddr l = 4;
2373     hwaddr addr1;
2374
2375     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2376                                  true);
2377     if (l < 4 || !memory_access_is_direct(mr, true)) {
2378 #if defined(TARGET_WORDS_BIGENDIAN)
2379         if (endian == DEVICE_LITTLE_ENDIAN) {
2380             val = bswap32(val);
2381         }
2382 #else
2383         if (endian == DEVICE_BIG_ENDIAN) {
2384             val = bswap32(val);
2385         }
2386 #endif
2387         io_mem_write(mr, addr1, val, 4);
2388     } else {
2389         /* RAM case */
2390         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2391         ptr = qemu_get_ram_ptr(addr1);
2392         switch (endian) {
2393         case DEVICE_LITTLE_ENDIAN:
2394             stl_le_p(ptr, val);
2395             break;
2396         case DEVICE_BIG_ENDIAN:
2397             stl_be_p(ptr, val);
2398             break;
2399         default:
2400             stl_p(ptr, val);
2401             break;
2402         }
2403         invalidate_and_set_dirty(addr1, 4);
2404     }
2405 }
2406
2407 void stl_phys(hwaddr addr, uint32_t val)
2408 {
2409     stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2410 }
2411
2412 void stl_le_phys(hwaddr addr, uint32_t val)
2413 {
2414     stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2415 }
2416
2417 void stl_be_phys(hwaddr addr, uint32_t val)
2418 {
2419     stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2420 }
2421
2422 /* XXX: optimize */
2423 void stb_phys(hwaddr addr, uint32_t val)
2424 {
2425     uint8_t v = val;
2426     cpu_physical_memory_write(addr, &v, 1);
2427 }
2428
2429 /* warning: addr must be aligned */
2430 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2431                                      enum device_endian endian)
2432 {
2433     uint8_t *ptr;
2434     MemoryRegion *mr;
2435     hwaddr l = 2;
2436     hwaddr addr1;
2437
2438     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2439                                  true);
2440     if (l < 2 || !memory_access_is_direct(mr, true)) {
2441 #if defined(TARGET_WORDS_BIGENDIAN)
2442         if (endian == DEVICE_LITTLE_ENDIAN) {
2443             val = bswap16(val);
2444         }
2445 #else
2446         if (endian == DEVICE_BIG_ENDIAN) {
2447             val = bswap16(val);
2448         }
2449 #endif
2450         io_mem_write(mr, addr1, val, 2);
2451     } else {
2452         /* RAM case */
2453         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2454         ptr = qemu_get_ram_ptr(addr1);
2455         switch (endian) {
2456         case DEVICE_LITTLE_ENDIAN:
2457             stw_le_p(ptr, val);
2458             break;
2459         case DEVICE_BIG_ENDIAN:
2460             stw_be_p(ptr, val);
2461             break;
2462         default:
2463             stw_p(ptr, val);
2464             break;
2465         }
2466         invalidate_and_set_dirty(addr1, 2);
2467     }
2468 }
2469
2470 void stw_phys(hwaddr addr, uint32_t val)
2471 {
2472     stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2473 }
2474
2475 void stw_le_phys(hwaddr addr, uint32_t val)
2476 {
2477     stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2478 }
2479
2480 void stw_be_phys(hwaddr addr, uint32_t val)
2481 {
2482     stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2483 }
2484
2485 /* XXX: optimize */
2486 void stq_phys(hwaddr addr, uint64_t val)
2487 {
2488     val = tswap64(val);
2489     cpu_physical_memory_write(addr, &val, 8);
2490 }
2491
2492 void stq_le_phys(hwaddr addr, uint64_t val)
2493 {
2494     val = cpu_to_le64(val);
2495     cpu_physical_memory_write(addr, &val, 8);
2496 }
2497
2498 void stq_be_phys(hwaddr addr, uint64_t val)
2499 {
2500     val = cpu_to_be64(val);
2501     cpu_physical_memory_write(addr, &val, 8);
2502 }
2503
2504 /* virtual memory access for debug (includes writing to ROM) */
2505 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2506                         uint8_t *buf, int len, int is_write)
2507 {
2508     int l;
2509     hwaddr phys_addr;
2510     target_ulong page;
2511
2512     while (len > 0) {
2513         page = addr & TARGET_PAGE_MASK;
2514         phys_addr = cpu_get_phys_page_debug(env, page);
2515         /* if no physical page mapped, return an error */
2516         if (phys_addr == -1)
2517             return -1;
2518         l = (page + TARGET_PAGE_SIZE) - addr;
2519         if (l > len)
2520             l = len;
2521         phys_addr += (addr & ~TARGET_PAGE_MASK);
2522         if (is_write)
2523             cpu_physical_memory_write_rom(phys_addr, buf, l);
2524         else
2525             cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2526         len -= l;
2527         buf += l;
2528         addr += l;
2529     }
2530     return 0;
2531 }
2532 #endif
2533
2534 #if !defined(CONFIG_USER_ONLY)
2535
2536 /*
2537  * A helper function for the _utterly broken_ virtio device model to find out if
2538  * it's running on a big endian machine. Don't do this at home kids!
2539  */
2540 bool virtio_is_big_endian(void);
2541 bool virtio_is_big_endian(void)
2542 {
2543 #if defined(TARGET_WORDS_BIGENDIAN)
2544     return true;
2545 #else
2546     return false;
2547 #endif
2548 }
2549
2550 #endif
2551
2552 #ifndef CONFIG_USER_ONLY
2553 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2554 {
2555     MemoryRegion*mr;
2556     hwaddr l = 1;
2557
2558     mr = address_space_translate(&address_space_memory,
2559                                  phys_addr, &phys_addr, &l, false);
2560
2561     return !(memory_region_is_ram(mr) ||
2562              memory_region_is_romd(mr));
2563 }
2564
2565 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2566 {
2567     RAMBlock *block;
2568
2569     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2570         func(block->host, block->offset, block->length, opaque);
2571     }
2572 }
2573 #endif