exec.c

   1 /*
   2  *  Virtual page mapping
   3  *
   4  *  Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19 #include "config.h"
  20 #ifdef _WIN32
  21 #include <windows.h>
  22 #else
  23 #include <sys/types.h>
  24 #include <sys/mman.h>
  25 #endif
  26
  27 #include "qemu-common.h"
  28 #include "cpu.h"
  29 #include "tcg.h"
  30 #include "hw/hw.h"
  31 #include "hw/qdev.h"
  32 #include "qemu/osdep.h"
  33 #include "sysemu/kvm.h"
  34 #include "sysemu/sysemu.h"
  35 #include "hw/xen/xen.h"
  36 #include "qemu/timer.h"
  37 #include "qemu/config-file.h"
  38 #include "exec/memory.h"
  39 #include "sysemu/dma.h"
  40 #include "exec/address-spaces.h"
  41 #if defined(CONFIG_USER_ONLY)
  42 #include <qemu.h>
  43 #else /* !CONFIG_USER_ONLY */
  44 #include "sysemu/xen-mapcache.h"
  45 #include "trace.h"
  46 #endif
  47 #include "exec/cpu-all.h"
  48
  49 #include "exec/cputlb.h"
  50 #include "translate-all.h"
  51
  52 #include "exec/memory-internal.h"
  53
  54 //#define DEBUG_SUBPAGE
  55
  56 #if !defined(CONFIG_USER_ONLY)
  57 static int in_migration;
  58
  59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
  60
  61 static MemoryRegion *system_memory;
  62 static MemoryRegion *system_io;
  63
  64 AddressSpace address_space_io;
  65 AddressSpace address_space_memory;
  66
  67 MemoryRegion io_mem_rom, io_mem_notdirty;
  68 static MemoryRegion io_mem_unassigned;
  69
  70 #endif
  71
  72 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
  73 /* current CPU in the current thread. It is only valid inside
  74    cpu_exec() */
  75 DEFINE_TLS(CPUState *, current_cpu);
  76 /* 0 = Do not count executed instructions.
  77    1 = Precise instruction counting.
  78    2 = Adaptive rate instruction counting.  */
  79 int use_icount;
  80
  81 #if !defined(CONFIG_USER_ONLY)
  82
  83 typedef struct PhysPageEntry PhysPageEntry;
  84
  85 struct PhysPageEntry {
  86     uint16_t is_leaf : 1;
  87      /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
  88     uint16_t ptr : 15;
  89 };
  90
  91 typedef PhysPageEntry Node[L2_SIZE];
  92
  93 struct AddressSpaceDispatch {
  94     /* This is a multi-level map on the physical address space.
  95      * The bottom level has pointers to MemoryRegionSections.
  96      */
  97     PhysPageEntry phys_map;
  98     Node *nodes;
  99     MemoryRegionSection *sections;
 100     AddressSpace *as;
 101 };
 102
 103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
 104 typedef struct subpage_t {
 105     MemoryRegion iomem;
 106     AddressSpace *as;
 107     hwaddr base;
 108     uint16_t sub_section[TARGET_PAGE_SIZE];
 109 } subpage_t;
 110
 111 #define PHYS_SECTION_UNASSIGNED 0
 112 #define PHYS_SECTION_NOTDIRTY 1
 113 #define PHYS_SECTION_ROM 2
 114 #define PHYS_SECTION_WATCH 3
 115
 116 typedef struct PhysPageMap {
 117     unsigned sections_nb;
 118     unsigned sections_nb_alloc;
 119     unsigned nodes_nb;
 120     unsigned nodes_nb_alloc;
 121     Node *nodes;
 122     MemoryRegionSection *sections;
 123 } PhysPageMap;
 124
 125 static PhysPageMap *prev_map;
 126 static PhysPageMap next_map;
 127
 128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
 129
 130 static void io_mem_init(void);
 131 static void memory_map_init(void);
 132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
 133
 134 static MemoryRegion io_mem_watch;
 135 #endif
 136
 137 #if !defined(CONFIG_USER_ONLY)
 138
 139 static void phys_map_node_reserve(unsigned nodes)
 140 {
 141     if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
 142         next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
 143                                             16);
 144         next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
 145                                       next_map.nodes_nb + nodes);
 146         next_map.nodes = g_renew(Node, next_map.nodes,
 147                                  next_map.nodes_nb_alloc);
 148     }
 149 }
 150
 151 static uint16_t phys_map_node_alloc(void)
 152 {
 153     unsigned i;
 154     uint16_t ret;
 155
 156     ret = next_map.nodes_nb++;
 157     assert(ret != PHYS_MAP_NODE_NIL);
 158     assert(ret != next_map.nodes_nb_alloc);
 159     for (i = 0; i < L2_SIZE; ++i) {
 160         next_map.nodes[ret][i].is_leaf = 0;
 161         next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
 162     }
 163     return ret;
 164 }
 165
 166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
 167                                 hwaddr *nb, uint16_t leaf,
 168                                 int level)
 169 {
 170     PhysPageEntry *p;
 171     int i;
 172     hwaddr step = (hwaddr)1 << (level * L2_BITS);
 173
 174     if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
 175         lp->ptr = phys_map_node_alloc();
 176         p = next_map.nodes[lp->ptr];
 177         if (level == 0) {
 178             for (i = 0; i < L2_SIZE; i++) {
 179                 p[i].is_leaf = 1;
 180                 p[i].ptr = PHYS_SECTION_UNASSIGNED;
 181             }
 182         }
 183     } else {
 184         p = next_map.nodes[lp->ptr];
 185     }
 186     lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
 187
 188     while (*nb && lp < &p[L2_SIZE]) {
 189         if ((*index & (step - 1)) == 0 && *nb >= step) {
 190             lp->is_leaf = true;
 191             lp->ptr = leaf;
 192             *index += step;
 193             *nb -= step;
 194         } else {
 195             phys_page_set_level(lp, index, nb, leaf, level - 1);
 196         }
 197         ++lp;
 198     }
 199 }
 200
 201 static void phys_page_set(AddressSpaceDispatch *d,
 202                           hwaddr index, hwaddr nb,
 203                           uint16_t leaf)
 204 {
 205     /* Wildly overreserve - it doesn't matter much. */
 206     phys_map_node_reserve(3 * P_L2_LEVELS);
 207
 208     phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
 209 }
 210
 211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
 212                                            Node *nodes, MemoryRegionSection *sections)
 213 {
 214     PhysPageEntry *p;
 215     int i;
 216
 217     for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
 218         if (lp.ptr == PHYS_MAP_NODE_NIL) {
 219             return &sections[PHYS_SECTION_UNASSIGNED];
 220         }
 221         p = nodes[lp.ptr];
 222         lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
 223     }
 224     return &sections[lp.ptr];
 225 }
 226
 227 bool memory_region_is_unassigned(MemoryRegion *mr)
 228 {
 229     return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
 230         && mr != &io_mem_watch;
 231 }
 232
 233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
 234                                                         hwaddr addr,
 235                                                         bool resolve_subpage)
 236 {
 237     MemoryRegionSection *section;
 238     subpage_t *subpage;
 239
 240     section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
 241                              d->nodes, d->sections);
 242     if (resolve_subpage && section->mr->subpage) {
 243         subpage = container_of(section->mr, subpage_t, iomem);
 244         section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
 245     }
 246     return section;
 247 }
 248
 249 static MemoryRegionSection *
 250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
 251                                  hwaddr *plen, bool resolve_subpage)
 252 {
 253     MemoryRegionSection *section;
 254     Int128 diff;
 255
 256     section = address_space_lookup_region(d, addr, resolve_subpage);
 257     /* Compute offset within MemoryRegionSection */
 258     addr -= section->offset_within_address_space;
 259
 260     /* Compute offset within MemoryRegion */
 261     *xlat = addr + section->offset_within_region;
 262
 263     diff = int128_sub(section->mr->size, int128_make64(addr));
 264     *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
 265     return section;
 266 }
 267
 268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
 269                                       hwaddr *xlat, hwaddr *plen,
 270                                       bool is_write)
 271 {
 272     IOMMUTLBEntry iotlb;
 273     MemoryRegionSection *section;
 274     MemoryRegion *mr;
 275     hwaddr len = *plen;
 276
 277     for (;;) {
 278         section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
 279         mr = section->mr;
 280
 281         if (!mr->iommu_ops) {
 282             break;
 283         }
 284
 285         iotlb = mr->iommu_ops->translate(mr, addr);
 286         addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
 287                 | (addr & iotlb.addr_mask));
 288         len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
 289         if (!(iotlb.perm & (1 << is_write))) {
 290             mr = &io_mem_unassigned;
 291             break;
 292         }
 293
 294         as = iotlb.target_as;
 295     }
 296
 297     *plen = len;
 298     *xlat = addr;
 299     return mr;
 300 }
 301
 302 MemoryRegionSection *
 303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
 304                                   hwaddr *plen)
 305 {
 306     MemoryRegionSection *section;
 307     section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
 308
 309     assert(!section->mr->iommu_ops);
 310     return section;
 311 }
 312 #endif
 313
 314 void cpu_exec_init_all(void)
 315 {
 316 #if !defined(CONFIG_USER_ONLY)
 317     qemu_mutex_init(&ram_list.mutex);
 318     memory_map_init();
 319     io_mem_init();
 320 #endif
 321 }
 322
 323 #if !defined(CONFIG_USER_ONLY)
 324
 325 static int cpu_common_post_load(void *opaque, int version_id)
 326 {
 327     CPUState *cpu = opaque;
 328
 329     /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
 330        version_id is increased. */
 331     cpu->interrupt_request &= ~0x01;
 332     tlb_flush(cpu->env_ptr, 1);
 333
 334     return 0;
 335 }
 336
 337 const VMStateDescription vmstate_cpu_common = {
 338     .name = "cpu_common",
 339     .version_id = 1,
 340     .minimum_version_id = 1,
 341     .minimum_version_id_old = 1,
 342     .post_load = cpu_common_post_load,
 343     .fields      = (VMStateField []) {
 344         VMSTATE_UINT32(halted, CPUState),
 345         VMSTATE_UINT32(interrupt_request, CPUState),
 346         VMSTATE_END_OF_LIST()
 347     }
 348 };
 349
 350 #endif
 351
 352 CPUState *qemu_get_cpu(int index)
 353 {
 354     CPUState *cpu;
 355
 356     CPU_FOREACH(cpu) {
 357         if (cpu->cpu_index == index) {
 358             return cpu;
 359         }
 360     }
 361
 362     return NULL;
 363 }
 364
 365 void cpu_exec_init(CPUArchState *env)
 366 {
 367     CPUState *cpu = ENV_GET_CPU(env);
 368     CPUClass *cc = CPU_GET_CLASS(cpu);
 369     CPUState *some_cpu;
 370     int cpu_index;
 371
 372 #if defined(CONFIG_USER_ONLY)
 373     cpu_list_lock();
 374 #endif
 375     cpu_index = 0;
 376     CPU_FOREACH(some_cpu) {
 377         cpu_index++;
 378     }
 379     cpu->cpu_index = cpu_index;
 380     cpu->numa_node = 0;
 381     QTAILQ_INIT(&env->breakpoints);
 382     QTAILQ_INIT(&env->watchpoints);
 383 #ifndef CONFIG_USER_ONLY
 384     cpu->thread_id = qemu_get_thread_id();
 385 #endif
 386     QTAILQ_INSERT_TAIL(&cpus, cpu, node);
 387 #if defined(CONFIG_USER_ONLY)
 388     cpu_list_unlock();
 389 #endif
 390     if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
 391         vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
 392     }
 393 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
 394     register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
 395                     cpu_save, cpu_load, env);
 396     assert(cc->vmsd == NULL);
 397     assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
 398 #endif
 399     if (cc->vmsd != NULL) {
 400         vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
 401     }
 402 }
 403
 404 #if defined(TARGET_HAS_ICE)
 405 #if defined(CONFIG_USER_ONLY)
 406 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
 407 {
 408     tb_invalidate_phys_page_range(pc, pc + 1, 0);
 409 }
 410 #else
 411 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
 412 {
 413     tb_invalidate_phys_addr(cpu_get_phys_page_debug(cpu, pc) |
 414             (pc & ~TARGET_PAGE_MASK));
 415 }
 416 #endif
 417 #endif /* TARGET_HAS_ICE */
 418
 419 #if defined(CONFIG_USER_ONLY)
 420 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 421
 422 {
 423 }
 424
 425 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 426                           int flags, CPUWatchpoint **watchpoint)
 427 {
 428     return -ENOSYS;
 429 }
 430 #else
 431 /* Add a watchpoint.  */
 432 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 433                           int flags, CPUWatchpoint **watchpoint)
 434 {
 435     target_ulong len_mask = ~(len - 1);
 436     CPUWatchpoint *wp;
 437
 438     /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
 439     if ((len & (len - 1)) || (addr & ~len_mask) ||
 440             len == 0 || len > TARGET_PAGE_SIZE) {
 441         fprintf(stderr, "qemu: tried to set invalid watchpoint at "
 442                 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
 443         return -EINVAL;
 444     }
 445     wp = g_malloc(sizeof(*wp));
 446
 447     wp->vaddr = addr;
 448     wp->len_mask = len_mask;
 449     wp->flags = flags;
 450
 451     /* keep all GDB-injected watchpoints in front */
 452     if (flags & BP_GDB)
 453         QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
 454     else
 455         QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
 456
 457     tlb_flush_page(env, addr);
 458
 459     if (watchpoint)
 460         *watchpoint = wp;
 461     return 0;
 462 }
 463
 464 /* Remove a specific watchpoint.  */
 465 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
 466                           int flags)
 467 {
 468     target_ulong len_mask = ~(len - 1);
 469     CPUWatchpoint *wp;
 470
 471     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 472         if (addr == wp->vaddr && len_mask == wp->len_mask
 473                 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
 474             cpu_watchpoint_remove_by_ref(env, wp);
 475             return 0;
 476         }
 477     }
 478     return -ENOENT;
 479 }
 480
 481 /* Remove a specific watchpoint by reference.  */
 482 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
 483 {
 484     QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
 485
 486     tlb_flush_page(env, watchpoint->vaddr);
 487
 488     g_free(watchpoint);
 489 }
 490
 491 /* Remove all matching watchpoints.  */
 492 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 493 {
 494     CPUWatchpoint *wp, *next;
 495
 496     QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
 497         if (wp->flags & mask)
 498             cpu_watchpoint_remove_by_ref(env, wp);
 499     }
 500 }
 501 #endif
 502
 503 /* Add a breakpoint.  */
 504 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
 505                           CPUBreakpoint **breakpoint)
 506 {
 507 #if defined(TARGET_HAS_ICE)
 508     CPUBreakpoint *bp;
 509
 510     bp = g_malloc(sizeof(*bp));
 511
 512     bp->pc = pc;
 513     bp->flags = flags;
 514
 515     /* keep all GDB-injected breakpoints in front */
 516     if (flags & BP_GDB) {
 517         QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
 518     } else {
 519         QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
 520     }
 521
 522     breakpoint_invalidate(ENV_GET_CPU(env), pc);
 523
 524     if (breakpoint) {
 525         *breakpoint = bp;
 526     }
 527     return 0;
 528 #else
 529     return -ENOSYS;
 530 #endif
 531 }
 532
 533 /* Remove a specific breakpoint.  */
 534 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
 535 {
 536 #if defined(TARGET_HAS_ICE)
 537     CPUBreakpoint *bp;
 538
 539     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
 540         if (bp->pc == pc && bp->flags == flags) {
 541             cpu_breakpoint_remove_by_ref(env, bp);
 542             return 0;
 543         }
 544     }
 545     return -ENOENT;
 546 #else
 547     return -ENOSYS;
 548 #endif
 549 }
 550
 551 /* Remove a specific breakpoint by reference.  */
 552 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
 553 {
 554 #if defined(TARGET_HAS_ICE)
 555     QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
 556
 557     breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
 558
 559     g_free(breakpoint);
 560 #endif
 561 }
 562
 563 /* Remove all matching breakpoints. */
 564 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
 565 {
 566 #if defined(TARGET_HAS_ICE)
 567     CPUBreakpoint *bp, *next;
 568
 569     QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
 570         if (bp->flags & mask)
 571             cpu_breakpoint_remove_by_ref(env, bp);
 572     }
 573 #endif
 574 }
 575
 576 /* enable or disable single step mode. EXCP_DEBUG is returned by the
 577    CPU loop after each instruction */
 578 void cpu_single_step(CPUState *cpu, int enabled)
 579 {
 580 #if defined(TARGET_HAS_ICE)
 581     if (cpu->singlestep_enabled != enabled) {
 582         cpu->singlestep_enabled = enabled;
 583         if (kvm_enabled()) {
 584             kvm_update_guest_debug(cpu, 0);
 585         } else {
 586             /* must flush all the translated code to avoid inconsistencies */
 587             /* XXX: only flush what is necessary */
 588             CPUArchState *env = cpu->env_ptr;
 589             tb_flush(env);
 590         }
 591     }
 592 #endif
 593 }
 594
 595 void cpu_abort(CPUArchState *env, const char *fmt, ...)
 596 {
 597     CPUState *cpu = ENV_GET_CPU(env);
 598     va_list ap;
 599     va_list ap2;
 600
 601     va_start(ap, fmt);
 602     va_copy(ap2, ap);
 603     fprintf(stderr, "qemu: fatal: ");
 604     vfprintf(stderr, fmt, ap);
 605     fprintf(stderr, "\n");
 606     cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 607     if (qemu_log_enabled()) {
 608         qemu_log("qemu: fatal: ");
 609         qemu_log_vprintf(fmt, ap2);
 610         qemu_log("\n");
 611         log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 612         qemu_log_flush();
 613         qemu_log_close();
 614     }
 615     va_end(ap2);
 616     va_end(ap);
 617 #if defined(CONFIG_USER_ONLY)
 618     {
 619         struct sigaction act;
 620         sigfillset(&act.sa_mask);
 621         act.sa_handler = SIG_DFL;
 622         sigaction(SIGABRT, &act, NULL);
 623     }
 624 #endif
 625     abort();
 626 }
 627
 628 CPUArchState *cpu_copy(CPUArchState *env)
 629 {
 630     CPUArchState *new_env = cpu_init(env->cpu_model_str);
 631 #if defined(TARGET_HAS_ICE)
 632     CPUBreakpoint *bp;
 633     CPUWatchpoint *wp;
 634 #endif
 635
 636     /* Reset non arch specific state */
 637     cpu_reset(ENV_GET_CPU(new_env));
 638
 639     /* Copy arch specific state into the new CPU */
 640     memcpy(new_env, env, sizeof(CPUArchState));
 641
 642     /* Clone all break/watchpoints.
 643        Note: Once we support ptrace with hw-debug register access, make sure
 644        BP_CPU break/watchpoints are handled correctly on clone. */
 645     QTAILQ_INIT(&env->breakpoints);
 646     QTAILQ_INIT(&env->watchpoints);
 647 #if defined(TARGET_HAS_ICE)
 648     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
 649         cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
 650     }
 651     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 652         cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
 653                               wp->flags, NULL);
 654     }
 655 #endif
 656
 657     return new_env;
 658 }
 659
 660 #if !defined(CONFIG_USER_ONLY)
 661 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
 662                                       uintptr_t length)
 663 {
 664     uintptr_t start1;
 665
 666     /* we modify the TLB cache so that the dirty bit will be set again
 667        when accessing the range */
 668     start1 = (uintptr_t)qemu_safe_ram_ptr(start);
 669     /* Check that we don't span multiple blocks - this breaks the
 670        address comparisons below.  */
 671     if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
 672             != (end - 1) - start) {
 673         abort();
 674     }
 675     cpu_tlb_reset_dirty_all(start1, length);
 676
 677 }
 678
 679 /* Note: start and end must be within the same ram block.  */
 680 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
 681                                      int dirty_flags)
 682 {
 683     uintptr_t length;
 684
 685     start &= TARGET_PAGE_MASK;
 686     end = TARGET_PAGE_ALIGN(end);
 687
 688     length = end - start;
 689     if (length == 0)
 690         return;
 691     cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
 692
 693     if (tcg_enabled()) {
 694         tlb_reset_dirty_range_all(start, end, length);
 695     }
 696 }
 697
 698 static int cpu_physical_memory_set_dirty_tracking(int enable)
 699 {
 700     int ret = 0;
 701     in_migration = enable;
 702     return ret;
 703 }
 704
 705 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
 706                                        MemoryRegionSection *section,
 707                                        target_ulong vaddr,
 708                                        hwaddr paddr, hwaddr xlat,
 709                                        int prot,
 710                                        target_ulong *address)
 711 {
 712     hwaddr iotlb;
 713     CPUWatchpoint *wp;
 714
 715     if (memory_region_is_ram(section->mr)) {
 716         /* Normal RAM.  */
 717         iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
 718             + xlat;
 719         if (!section->readonly) {
 720             iotlb |= PHYS_SECTION_NOTDIRTY;
 721         } else {
 722             iotlb |= PHYS_SECTION_ROM;
 723         }
 724     } else {
 725         iotlb = section - address_space_memory.dispatch->sections;
 726         iotlb += xlat;
 727     }
 728
 729     /* Make accesses to pages with watchpoints go via the
 730        watchpoint trap routines.  */
 731     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 732         if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
 733             /* Avoid trapping reads of pages with a write breakpoint. */
 734             if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
 735                 iotlb = PHYS_SECTION_WATCH + paddr;
 736                 *address |= TLB_MMIO;
 737                 break;
 738             }
 739         }
 740     }
 741
 742     return iotlb;
 743 }
 744 #endif /* defined(CONFIG_USER_ONLY) */
 745
 746 #if !defined(CONFIG_USER_ONLY)
 747
 748 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
 749                              uint16_t section);
 750 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
 751
 752 static uint16_t phys_section_add(MemoryRegionSection *section)
 753 {
 754     /* The physical section number is ORed with a page-aligned
 755      * pointer to produce the iotlb entries.  Thus it should
 756      * never overflow into the page-aligned value.
 757      */
 758     assert(next_map.sections_nb < TARGET_PAGE_SIZE);
 759
 760     if (next_map.sections_nb == next_map.sections_nb_alloc) {
 761         next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
 762                                          16);
 763         next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
 764                                     next_map.sections_nb_alloc);
 765     }
 766     next_map.sections[next_map.sections_nb] = *section;
 767     memory_region_ref(section->mr);
 768     return next_map.sections_nb++;
 769 }
 770
 771 static void phys_section_destroy(MemoryRegion *mr)
 772 {
 773     memory_region_unref(mr);
 774
 775     if (mr->subpage) {
 776         subpage_t *subpage = container_of(mr, subpage_t, iomem);
 777         memory_region_destroy(&subpage->iomem);
 778         g_free(subpage);
 779     }
 780 }
 781
 782 static void phys_sections_free(PhysPageMap *map)
 783 {
 784     while (map->sections_nb > 0) {
 785         MemoryRegionSection *section = &map->sections[--map->sections_nb];
 786         phys_section_destroy(section->mr);
 787     }
 788     g_free(map->sections);
 789     g_free(map->nodes);
 790     g_free(map);
 791 }
 792
 793 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
 794 {
 795     subpage_t *subpage;
 796     hwaddr base = section->offset_within_address_space
 797         & TARGET_PAGE_MASK;
 798     MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
 799                                                    next_map.nodes, next_map.sections);
 800     MemoryRegionSection subsection = {
 801         .offset_within_address_space = base,
 802         .size = int128_make64(TARGET_PAGE_SIZE),
 803     };
 804     hwaddr start, end;
 805
 806     assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
 807
 808     if (!(existing->mr->subpage)) {
 809         subpage = subpage_init(d->as, base);
 810         subsection.mr = &subpage->iomem;
 811         phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
 812                       phys_section_add(&subsection));
 813     } else {
 814         subpage = container_of(existing->mr, subpage_t, iomem);
 815     }
 816     start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
 817     end = start + int128_get64(section->size) - 1;
 818     subpage_register(subpage, start, end, phys_section_add(section));
 819 }
 820
 821
 822 static void register_multipage(AddressSpaceDispatch *d,
 823                                MemoryRegionSection *section)
 824 {
 825     hwaddr start_addr = section->offset_within_address_space;
 826     uint16_t section_index = phys_section_add(section);
 827     uint64_t num_pages = int128_get64(int128_rshift(section->size,
 828                                                     TARGET_PAGE_BITS));
 829
 830     assert(num_pages);
 831     phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
 832 }
 833
 834 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
 835 {
 836     AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
 837     AddressSpaceDispatch *d = as->next_dispatch;
 838     MemoryRegionSection now = *section, remain = *section;
 839     Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
 840
 841     if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
 842         uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
 843                        - now.offset_within_address_space;
 844
 845         now.size = int128_min(int128_make64(left), now.size);
 846         register_subpage(d, &now);
 847     } else {
 848         now.size = int128_zero();
 849     }
 850     while (int128_ne(remain.size, now.size)) {
 851         remain.size = int128_sub(remain.size, now.size);
 852         remain.offset_within_address_space += int128_get64(now.size);
 853         remain.offset_within_region += int128_get64(now.size);
 854         now = remain;
 855         if (int128_lt(remain.size, page_size)) {
 856             register_subpage(d, &now);
 857         } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
 858             now.size = page_size;
 859             register_subpage(d, &now);
 860         } else {
 861             now.size = int128_and(now.size, int128_neg(page_size));
 862             register_multipage(d, &now);
 863         }
 864     }
 865 }
 866
 867 void qemu_flush_coalesced_mmio_buffer(void)
 868 {
 869     if (kvm_enabled())
 870         kvm_flush_coalesced_mmio_buffer();
 871 }
 872
 873 void qemu_mutex_lock_ramlist(void)
 874 {
 875     qemu_mutex_lock(&ram_list.mutex);
 876 }
 877
 878 void qemu_mutex_unlock_ramlist(void)
 879 {
 880     qemu_mutex_unlock(&ram_list.mutex);
 881 }
 882
 883 #if defined(__linux__) && !defined(TARGET_S390X)
 884
 885 #include <sys/vfs.h>
 886
 887 #define HUGETLBFS_MAGIC       0x958458f6
 888
 889 static long gethugepagesize(const char *path)
 890 {
 891     struct statfs fs;
 892     int ret;
 893
 894     do {
 895         ret = statfs(path, &fs);
 896     } while (ret != 0 && errno == EINTR);
 897
 898     if (ret != 0) {
 899         perror(path);
 900         return 0;
 901     }
 902
 903     if (fs.f_type != HUGETLBFS_MAGIC)
 904         fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
 905
 906     return fs.f_bsize;
 907 }
 908
 909 static void *file_ram_alloc(RAMBlock *block,
 910                             ram_addr_t memory,
 911                             const char *path)
 912 {
 913     char *filename;
 914     char *sanitized_name;
 915     char *c;
 916     void *area;
 917     int fd;
 918 #ifdef MAP_POPULATE
 919     int flags;
 920 #endif
 921     unsigned long hpagesize;
 922
 923     hpagesize = gethugepagesize(path);
 924     if (!hpagesize) {
 925         return NULL;
 926     }
 927
 928     if (memory < hpagesize) {
 929         return NULL;
 930     }
 931
 932     if (kvm_enabled() && !kvm_has_sync_mmu()) {
 933         fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
 934         return NULL;
 935     }
 936
 937     /* Make name safe to use with mkstemp by replacing '/' with '_'. */
 938     sanitized_name = g_strdup(block->mr->name);
 939     for (c = sanitized_name; *c != '\0'; c++) {
 940         if (*c == '/')
 941             *c = '_';
 942     }
 943
 944     filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
 945                                sanitized_name);
 946     g_free(sanitized_name);
 947
 948     fd = mkstemp(filename);
 949     if (fd < 0) {
 950         perror("unable to create backing store for hugepages");
 951         g_free(filename);
 952         return NULL;
 953     }
 954     unlink(filename);
 955     g_free(filename);
 956
 957     memory = (memory+hpagesize-1) & ~(hpagesize-1);
 958
 959     /*
 960      * ftruncate is not supported by hugetlbfs in older
 961      * hosts, so don't bother bailing out on errors.
 962      * If anything goes wrong with it under other filesystems,
 963      * mmap will fail.
 964      */
 965     if (ftruncate(fd, memory))
 966         perror("ftruncate");
 967
 968 #ifdef MAP_POPULATE
 969     /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
 970      * MAP_PRIVATE is requested.  For mem_prealloc we mmap as MAP_SHARED
 971      * to sidestep this quirk.
 972      */
 973     flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
 974     area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
 975 #else
 976     area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
 977 #endif
 978     if (area == MAP_FAILED) {
 979         perror("file_ram_alloc: can't mmap RAM pages");
 980         close(fd);
 981         return (NULL);
 982     }
 983     block->fd = fd;
 984     return area;
 985 }
 986 #endif
 987
 988 static ram_addr_t find_ram_offset(ram_addr_t size)
 989 {
 990     RAMBlock *block, *next_block;
 991     ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
 992
 993     assert(size != 0); /* it would hand out same offset multiple times */
 994
 995     if (QTAILQ_EMPTY(&ram_list.blocks))
 996         return 0;
 997
 998     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
 999         ram_addr_t end, next = RAM_ADDR_MAX;
1000
1001         end = block->offset + block->length;
1002
1003         QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1004             if (next_block->offset >= end) {
1005                 next = MIN(next, next_block->offset);
1006             }
1007         }
1008         if (next - end >= size && next - end < mingap) {
1009             offset = end;
1010             mingap = next - end;
1011         }
1012     }
1013
1014     if (offset == RAM_ADDR_MAX) {
1015         fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1016                 (uint64_t)size);
1017         abort();
1018     }
1019
1020     return offset;
1021 }
1022
1023 ram_addr_t last_ram_offset(void)
1024 {
1025     RAMBlock *block;
1026     ram_addr_t last = 0;
1027
1028     QTAILQ_FOREACH(block, &ram_list.blocks, next)
1029         last = MAX(last, block->offset + block->length);
1030
1031     return last;
1032 }
1033
1034 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1035 {
1036     int ret;
1037
1038     /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1039     if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1040                            "dump-guest-core", true)) {
1041         ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1042         if (ret) {
1043             perror("qemu_madvise");
1044             fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1045                             "but dump_guest_core=off specified\n");
1046         }
1047     }
1048 }
1049
1050 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1051 {
1052     RAMBlock *new_block, *block;
1053
1054     new_block = NULL;
1055     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1056         if (block->offset == addr) {
1057             new_block = block;
1058             break;
1059         }
1060     }
1061     assert(new_block);
1062     assert(!new_block->idstr[0]);
1063
1064     if (dev) {
1065         char *id = qdev_get_dev_path(dev);
1066         if (id) {
1067             snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1068             g_free(id);
1069         }
1070     }
1071     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1072
1073     /* This assumes the iothread lock is taken here too.  */
1074     qemu_mutex_lock_ramlist();
1075     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1076         if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1077             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1078                     new_block->idstr);
1079             abort();
1080         }
1081     }
1082     qemu_mutex_unlock_ramlist();
1083 }
1084
1085 static int memory_try_enable_merging(void *addr, size_t len)
1086 {
1087     if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1088         /* disabled by the user */
1089         return 0;
1090     }
1091
1092     return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1093 }
1094
1095 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1096                                    MemoryRegion *mr)
1097 {
1098     RAMBlock *block, *new_block;
1099
1100     size = TARGET_PAGE_ALIGN(size);
1101     new_block = g_malloc0(sizeof(*new_block));
1102     new_block->fd = -1;
1103
1104     /* This assumes the iothread lock is taken here too.  */
1105     qemu_mutex_lock_ramlist();
1106     new_block->mr = mr;
1107     new_block->offset = find_ram_offset(size);
1108     if (host) {
1109         new_block->host = host;
1110         new_block->flags |= RAM_PREALLOC_MASK;
1111     } else if (xen_enabled()) {
1112         if (mem_path) {
1113             fprintf(stderr, "-mem-path not supported with Xen\n");
1114             exit(1);
1115         }
1116         xen_ram_alloc(new_block->offset, size, mr);
1117     } else {
1118         if (mem_path) {
1119 #if defined (__linux__) && !defined(TARGET_S390X)
1120             new_block->host = file_ram_alloc(new_block, size, mem_path);
1121 #else
1122             fprintf(stderr, "-mem-path option unsupported\n");
1123             exit(1);
1124 #endif
1125         }
1126         if (!new_block->host) {
1127             if (kvm_enabled()) {
1128                 /* some s390/kvm configurations have special constraints */
1129                 new_block->host = kvm_ram_alloc(size);
1130             } else {
1131                 new_block->host = qemu_anon_ram_alloc(size);
1132             }
1133             memory_try_enable_merging(new_block->host, size);
1134         }
1135     }
1136     new_block->length = size;
1137
1138     /* Keep the list sorted from biggest to smallest block.  */
1139     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1140         if (block->length < new_block->length) {
1141             break;
1142         }
1143     }
1144     if (block) {
1145         QTAILQ_INSERT_BEFORE(block, new_block, next);
1146     } else {
1147         QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1148     }
1149     ram_list.mru_block = NULL;
1150
1151     ram_list.version++;
1152     qemu_mutex_unlock_ramlist();
1153
1154     ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1155                                        last_ram_offset() >> TARGET_PAGE_BITS);
1156     memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1157            0, size >> TARGET_PAGE_BITS);
1158     cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1159
1160     qemu_ram_setup_dump(new_block->host, size);
1161     qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1162
1163     if (kvm_enabled())
1164         kvm_setup_guest_memory(new_block->host, size);
1165
1166     return new_block->offset;
1167 }
1168
1169 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1170 {
1171     return qemu_ram_alloc_from_ptr(size, NULL, mr);
1172 }
1173
1174 void qemu_ram_free_from_ptr(ram_addr_t addr)
1175 {
1176     RAMBlock *block;
1177
1178     /* This assumes the iothread lock is taken here too.  */
1179     qemu_mutex_lock_ramlist();
1180     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1181         if (addr == block->offset) {
1182             QTAILQ_REMOVE(&ram_list.blocks, block, next);
1183             ram_list.mru_block = NULL;
1184             ram_list.version++;
1185             g_free(block);
1186             break;
1187         }
1188     }
1189     qemu_mutex_unlock_ramlist();
1190 }
1191
1192 void qemu_ram_free(ram_addr_t addr)
1193 {
1194     RAMBlock *block;
1195
1196     /* This assumes the iothread lock is taken here too.  */
1197     qemu_mutex_lock_ramlist();
1198     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1199         if (addr == block->offset) {
1200             QTAILQ_REMOVE(&ram_list.blocks, block, next);
1201             ram_list.mru_block = NULL;
1202             ram_list.version++;
1203             if (block->flags & RAM_PREALLOC_MASK) {
1204                 ;
1205             } else if (xen_enabled()) {
1206                 xen_invalidate_map_cache_entry(block->host);
1207             } else if (block->fd >= 0) {
1208                 munmap(block->host, block->length);
1209                 close(block->fd);
1210             } else {
1211                 qemu_anon_ram_free(block->host, block->length);
1212             }
1213             g_free(block);
1214             break;
1215         }
1216     }
1217     qemu_mutex_unlock_ramlist();
1218
1219 }
1220
1221 #ifndef _WIN32
1222 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1223 {
1224     RAMBlock *block;
1225     ram_addr_t offset;
1226     int flags;
1227     void *area, *vaddr;
1228
1229     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1230         offset = addr - block->offset;
1231         if (offset < block->length) {
1232             vaddr = block->host + offset;
1233             if (block->flags & RAM_PREALLOC_MASK) {
1234                 ;
1235             } else if (xen_enabled()) {
1236                 abort();
1237             } else {
1238                 flags = MAP_FIXED;
1239                 munmap(vaddr, length);
1240                 if (block->fd >= 0) {
1241 #ifdef MAP_POPULATE
1242                     flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1243                         MAP_PRIVATE;
1244 #else
1245                     flags |= MAP_PRIVATE;
1246 #endif
1247                     area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1248                                 flags, block->fd, offset);
1249                 } else {
1250 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1251                     flags |= MAP_SHARED | MAP_ANONYMOUS;
1252                     area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1253                                 flags, -1, 0);
1254 #else
1255                     flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1256                     area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1257                                 flags, -1, 0);
1258 #endif
1259                 }
1260                 if (area != vaddr) {
1261                     fprintf(stderr, "Could not remap addr: "
1262                             RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1263                             length, addr);
1264                     exit(1);
1265                 }
1266                 memory_try_enable_merging(vaddr, length);
1267                 qemu_ram_setup_dump(vaddr, length);
1268             }
1269             return;
1270         }
1271     }
1272 }
1273 #endif /* !_WIN32 */
1274
1275 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1276 {
1277     RAMBlock *block;
1278
1279     /* The list is protected by the iothread lock here.  */
1280     block = ram_list.mru_block;
1281     if (block && addr - block->offset < block->length) {
1282         goto found;
1283     }
1284     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1285         if (addr - block->offset < block->length) {
1286             goto found;
1287         }
1288     }
1289
1290     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1291     abort();
1292
1293 found:
1294     ram_list.mru_block = block;
1295     return block;
1296 }
1297
1298 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1299    With the exception of the softmmu code in this file, this should
1300    only be used for local memory (e.g. video ram) that the device owns,
1301    and knows it isn't going to access beyond the end of the block.
1302
1303    It should not be used for general purpose DMA.
1304    Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1305  */
1306 void *qemu_get_ram_ptr(ram_addr_t addr)
1307 {
1308     RAMBlock *block = qemu_get_ram_block(addr);
1309
1310     if (xen_enabled()) {
1311         /* We need to check if the requested address is in the RAM
1312          * because we don't want to map the entire memory in QEMU.
1313          * In that case just map until the end of the page.
1314          */
1315         if (block->offset == 0) {
1316             return xen_map_cache(addr, 0, 0);
1317         } else if (block->host == NULL) {
1318             block->host =
1319                 xen_map_cache(block->offset, block->length, 1);
1320         }
1321     }
1322     return block->host + (addr - block->offset);
1323 }
1324
1325 /* Return a host pointer to ram allocated with qemu_ram_alloc.  Same as
1326  * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1327  *
1328  * ??? Is this still necessary?
1329  */
1330 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1331 {
1332     RAMBlock *block;
1333
1334     /* The list is protected by the iothread lock here.  */
1335     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1336         if (addr - block->offset < block->length) {
1337             if (xen_enabled()) {
1338                 /* We need to check if the requested address is in the RAM
1339                  * because we don't want to map the entire memory in QEMU.
1340                  * In that case just map until the end of the page.
1341                  */
1342                 if (block->offset == 0) {
1343                     return xen_map_cache(addr, 0, 0);
1344                 } else if (block->host == NULL) {
1345                     block->host =
1346                         xen_map_cache(block->offset, block->length, 1);
1347                 }
1348             }
1349             return block->host + (addr - block->offset);
1350         }
1351     }
1352
1353     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1354     abort();
1355
1356     return NULL;
1357 }
1358
1359 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1360  * but takes a size argument */
1361 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1362 {
1363     if (*size == 0) {
1364         return NULL;
1365     }
1366     if (xen_enabled()) {
1367         return xen_map_cache(addr, *size, 1);
1368     } else {
1369         RAMBlock *block;
1370
1371         QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1372             if (addr - block->offset < block->length) {
1373                 if (addr - block->offset + *size > block->length)
1374                     *size = block->length - addr + block->offset;
1375                 return block->host + (addr - block->offset);
1376             }
1377         }
1378
1379         fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1380         abort();
1381     }
1382 }
1383
1384 /* Some of the softmmu routines need to translate from a host pointer
1385    (typically a TLB entry) back to a ram offset.  */
1386 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1387 {
1388     RAMBlock *block;
1389     uint8_t *host = ptr;
1390
1391     if (xen_enabled()) {
1392         *ram_addr = xen_ram_addr_from_mapcache(ptr);
1393         return qemu_get_ram_block(*ram_addr)->mr;
1394     }
1395
1396     block = ram_list.mru_block;
1397     if (block && block->host && host - block->host < block->length) {
1398         goto found;
1399     }
1400
1401     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1402         /* This case append when the block is not mapped. */
1403         if (block->host == NULL) {
1404             continue;
1405         }
1406         if (host - block->host < block->length) {
1407             goto found;
1408         }
1409     }
1410
1411     return NULL;
1412
1413 found:
1414     *ram_addr = block->offset + (host - block->host);
1415     return block->mr;
1416 }
1417
1418 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1419                                uint64_t val, unsigned size)
1420 {
1421     int dirty_flags;
1422     dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1423     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1424         tb_invalidate_phys_page_fast(ram_addr, size);
1425         dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1426     }
1427     switch (size) {
1428     case 1:
1429         stb_p(qemu_get_ram_ptr(ram_addr), val);
1430         break;
1431     case 2:
1432         stw_p(qemu_get_ram_ptr(ram_addr), val);
1433         break;
1434     case 4:
1435         stl_p(qemu_get_ram_ptr(ram_addr), val);
1436         break;
1437     default:
1438         abort();
1439     }
1440     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1441     cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1442     /* we remove the notdirty callback only if the code has been
1443        flushed */
1444     if (dirty_flags == 0xff) {
1445         CPUArchState *env = current_cpu->env_ptr;
1446         tlb_set_dirty(env, env->mem_io_vaddr);
1447     }
1448 }
1449
1450 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1451                                  unsigned size, bool is_write)
1452 {
1453     return is_write;
1454 }
1455
1456 static const MemoryRegionOps notdirty_mem_ops = {
1457     .write = notdirty_mem_write,
1458     .valid.accepts = notdirty_mem_accepts,
1459     .endianness = DEVICE_NATIVE_ENDIAN,
1460 };
1461
1462 /* Generate a debug exception if a watchpoint has been hit.  */
1463 static void check_watchpoint(int offset, int len_mask, int flags)
1464 {
1465     CPUArchState *env = current_cpu->env_ptr;
1466     target_ulong pc, cs_base;
1467     target_ulong vaddr;
1468     CPUWatchpoint *wp;
1469     int cpu_flags;
1470
1471     if (env->watchpoint_hit) {
1472         /* We re-entered the check after replacing the TB. Now raise
1473          * the debug interrupt so that is will trigger after the
1474          * current instruction. */
1475         cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1476         return;
1477     }
1478     vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1479     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1480         if ((vaddr == (wp->vaddr & len_mask) ||
1481              (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1482             wp->flags |= BP_WATCHPOINT_HIT;
1483             if (!env->watchpoint_hit) {
1484                 env->watchpoint_hit = wp;
1485                 tb_check_watchpoint(env);
1486                 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1487                     env->exception_index = EXCP_DEBUG;
1488                     cpu_loop_exit(env);
1489                 } else {
1490                     cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1491                     tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1492                     cpu_resume_from_signal(env, NULL);
1493                 }
1494             }
1495         } else {
1496             wp->flags &= ~BP_WATCHPOINT_HIT;
1497         }
1498     }
1499 }
1500
1501 /* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
1502    so these check for a hit then pass through to the normal out-of-line
1503    phys routines.  */
1504 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1505                                unsigned size)
1506 {
1507     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1508     switch (size) {
1509     case 1: return ldub_phys(addr);
1510     case 2: return lduw_phys(addr);
1511     case 4: return ldl_phys(addr);
1512     default: abort();
1513     }
1514 }
1515
1516 static void watch_mem_write(void *opaque, hwaddr addr,
1517                             uint64_t val, unsigned size)
1518 {
1519     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1520     switch (size) {
1521     case 1:
1522         stb_phys(addr, val);
1523         break;
1524     case 2:
1525         stw_phys(addr, val);
1526         break;
1527     case 4:
1528         stl_phys(addr, val);
1529         break;
1530     default: abort();
1531     }
1532 }
1533
1534 static const MemoryRegionOps watch_mem_ops = {
1535     .read = watch_mem_read,
1536     .write = watch_mem_write,
1537     .endianness = DEVICE_NATIVE_ENDIAN,
1538 };
1539
1540 static uint64_t subpage_read(void *opaque, hwaddr addr,
1541                              unsigned len)
1542 {
1543     subpage_t *subpage = opaque;
1544     uint8_t buf[4];
1545
1546 #if defined(DEBUG_SUBPAGE)
1547     printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1548            subpage, len, addr);
1549 #endif
1550     address_space_read(subpage->as, addr + subpage->base, buf, len);
1551     switch (len) {
1552     case 1:
1553         return ldub_p(buf);
1554     case 2:
1555         return lduw_p(buf);
1556     case 4:
1557         return ldl_p(buf);
1558     default:
1559         abort();
1560     }
1561 }
1562
1563 static void subpage_write(void *opaque, hwaddr addr,
1564                           uint64_t value, unsigned len)
1565 {
1566     subpage_t *subpage = opaque;
1567     uint8_t buf[4];
1568
1569 #if defined(DEBUG_SUBPAGE)
1570     printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1571            " value %"PRIx64"\n",
1572            __func__, subpage, len, addr, value);
1573 #endif
1574     switch (len) {
1575     case 1:
1576         stb_p(buf, value);
1577         break;
1578     case 2:
1579         stw_p(buf, value);
1580         break;
1581     case 4:
1582         stl_p(buf, value);
1583         break;
1584     default:
1585         abort();
1586     }
1587     address_space_write(subpage->as, addr + subpage->base, buf, len);
1588 }
1589
1590 static bool subpage_accepts(void *opaque, hwaddr addr,
1591                             unsigned size, bool is_write)
1592 {
1593     subpage_t *subpage = opaque;
1594 #if defined(DEBUG_SUBPAGE)
1595     printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1596            __func__, subpage, is_write ? 'w' : 'r', len, addr);
1597 #endif
1598
1599     return address_space_access_valid(subpage->as, addr + subpage->base,
1600                                       size, is_write);
1601 }
1602
1603 static const MemoryRegionOps subpage_ops = {
1604     .read = subpage_read,
1605     .write = subpage_write,
1606     .valid.accepts = subpage_accepts,
1607     .endianness = DEVICE_NATIVE_ENDIAN,
1608 };
1609
1610 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1611                              uint16_t section)
1612 {
1613     int idx, eidx;
1614
1615     if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1616         return -1;
1617     idx = SUBPAGE_IDX(start);
1618     eidx = SUBPAGE_IDX(end);
1619 #if defined(DEBUG_SUBPAGE)
1620     printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1621            mmio, start, end, idx, eidx, memory);
1622 #endif
1623     for (; idx <= eidx; idx++) {
1624         mmio->sub_section[idx] = section;
1625     }
1626
1627     return 0;
1628 }
1629
1630 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1631 {
1632     subpage_t *mmio;
1633
1634     mmio = g_malloc0(sizeof(subpage_t));
1635
1636     mmio->as = as;
1637     mmio->base = base;
1638     memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1639                           "subpage", TARGET_PAGE_SIZE);
1640     mmio->iomem.subpage = true;
1641 #if defined(DEBUG_SUBPAGE)
1642     printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1643            mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1644 #endif
1645     subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1646
1647     return mmio;
1648 }
1649
1650 static uint16_t dummy_section(MemoryRegion *mr)
1651 {
1652     MemoryRegionSection section = {
1653         .mr = mr,
1654         .offset_within_address_space = 0,
1655         .offset_within_region = 0,
1656         .size = int128_2_64(),
1657     };
1658
1659     return phys_section_add(&section);
1660 }
1661
1662 MemoryRegion *iotlb_to_region(hwaddr index)
1663 {
1664     return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1665 }
1666
1667 static void io_mem_init(void)
1668 {
1669     memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1670     memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1671                           "unassigned", UINT64_MAX);
1672     memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1673                           "notdirty", UINT64_MAX);
1674     memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1675                           "watch", UINT64_MAX);
1676 }
1677
1678 static void mem_begin(MemoryListener *listener)
1679 {
1680     AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1681     AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1682
1683     d->phys_map  = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1684     d->as = as;
1685     as->next_dispatch = d;
1686 }
1687
1688 static void mem_commit(MemoryListener *listener)
1689 {
1690     AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1691     AddressSpaceDispatch *cur = as->dispatch;
1692     AddressSpaceDispatch *next = as->next_dispatch;
1693
1694     next->nodes = next_map.nodes;
1695     next->sections = next_map.sections;
1696
1697     as->dispatch = next;
1698     g_free(cur);
1699 }
1700
1701 static void core_begin(MemoryListener *listener)
1702 {
1703     uint16_t n;
1704
1705     prev_map = g_new(PhysPageMap, 1);
1706     *prev_map = next_map;
1707
1708     memset(&next_map, 0, sizeof(next_map));
1709     n = dummy_section(&io_mem_unassigned);
1710     assert(n == PHYS_SECTION_UNASSIGNED);
1711     n = dummy_section(&io_mem_notdirty);
1712     assert(n == PHYS_SECTION_NOTDIRTY);
1713     n = dummy_section(&io_mem_rom);
1714     assert(n == PHYS_SECTION_ROM);
1715     n = dummy_section(&io_mem_watch);
1716     assert(n == PHYS_SECTION_WATCH);
1717 }
1718
1719 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1720  * All AddressSpaceDispatch instances have switched to the next map.
1721  */
1722 static void core_commit(MemoryListener *listener)
1723 {
1724     phys_sections_free(prev_map);
1725 }
1726
1727 static void tcg_commit(MemoryListener *listener)
1728 {
1729     CPUState *cpu;
1730
1731     /* since each CPU stores ram addresses in its TLB cache, we must
1732        reset the modified entries */
1733     /* XXX: slow ! */
1734     CPU_FOREACH(cpu) {
1735         CPUArchState *env = cpu->env_ptr;
1736
1737         tlb_flush(env, 1);
1738     }
1739 }
1740
1741 static void core_log_global_start(MemoryListener *listener)
1742 {
1743     cpu_physical_memory_set_dirty_tracking(1);
1744 }
1745
1746 static void core_log_global_stop(MemoryListener *listener)
1747 {
1748     cpu_physical_memory_set_dirty_tracking(0);
1749 }
1750
1751 static MemoryListener core_memory_listener = {
1752     .begin = core_begin,
1753     .commit = core_commit,
1754     .log_global_start = core_log_global_start,
1755     .log_global_stop = core_log_global_stop,
1756     .priority = 1,
1757 };
1758
1759 static MemoryListener tcg_memory_listener = {
1760     .commit = tcg_commit,
1761 };
1762
1763 void address_space_init_dispatch(AddressSpace *as)
1764 {
1765     as->dispatch = NULL;
1766     as->dispatch_listener = (MemoryListener) {
1767         .begin = mem_begin,
1768         .commit = mem_commit,
1769         .region_add = mem_add,
1770         .region_nop = mem_add,
1771         .priority = 0,
1772     };
1773     memory_listener_register(&as->dispatch_listener, as);
1774 }
1775
1776 void address_space_destroy_dispatch(AddressSpace *as)
1777 {
1778     AddressSpaceDispatch *d = as->dispatch;
1779
1780     memory_listener_unregister(&as->dispatch_listener);
1781     g_free(d);
1782     as->dispatch = NULL;
1783 }
1784
1785 static void memory_map_init(void)
1786 {
1787     system_memory = g_malloc(sizeof(*system_memory));
1788     memory_region_init(system_memory, NULL, "system", INT64_MAX);
1789     address_space_init(&address_space_memory, system_memory, "memory");
1790
1791     system_io = g_malloc(sizeof(*system_io));
1792     memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
1793                           65536);
1794     address_space_init(&address_space_io, system_io, "I/O");
1795
1796     memory_listener_register(&core_memory_listener, &address_space_memory);
1797     if (tcg_enabled()) {
1798         memory_listener_register(&tcg_memory_listener, &address_space_memory);
1799     }
1800 }
1801
1802 MemoryRegion *get_system_memory(void)
1803 {
1804     return system_memory;
1805 }
1806
1807 MemoryRegion *get_system_io(void)
1808 {
1809     return system_io;
1810 }
1811
1812 #endif /* !defined(CONFIG_USER_ONLY) */
1813
1814 /* physical memory access (slow version, mainly for debug) */
1815 #if defined(CONFIG_USER_ONLY)
1816 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1817                         uint8_t *buf, int len, int is_write)
1818 {
1819     int l, flags;
1820     target_ulong page;
1821     void * p;
1822
1823     while (len > 0) {
1824         page = addr & TARGET_PAGE_MASK;
1825         l = (page + TARGET_PAGE_SIZE) - addr;
1826         if (l > len)
1827             l = len;
1828         flags = page_get_flags(page);
1829         if (!(flags & PAGE_VALID))
1830             return -1;
1831         if (is_write) {
1832             if (!(flags & PAGE_WRITE))
1833                 return -1;
1834             /* XXX: this code should not depend on lock_user */
1835             if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1836                 return -1;
1837             memcpy(p, buf, l);
1838             unlock_user(p, addr, l);
1839         } else {
1840             if (!(flags & PAGE_READ))
1841                 return -1;
1842             /* XXX: this code should not depend on lock_user */
1843             if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1844                 return -1;
1845             memcpy(buf, p, l);
1846             unlock_user(p, addr, 0);
1847         }
1848         len -= l;
1849         buf += l;
1850         addr += l;
1851     }
1852     return 0;
1853 }
1854
1855 #else
1856
1857 static void invalidate_and_set_dirty(hwaddr addr,
1858                                      hwaddr length)
1859 {
1860     if (!cpu_physical_memory_is_dirty(addr)) {
1861         /* invalidate code */
1862         tb_invalidate_phys_page_range(addr, addr + length, 0);
1863         /* set dirty bit */
1864         cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1865     }
1866     xen_modified_memory(addr, length);
1867 }
1868
1869 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1870 {
1871     if (memory_region_is_ram(mr)) {
1872         return !(is_write && mr->readonly);
1873     }
1874     if (memory_region_is_romd(mr)) {
1875         return !is_write;
1876     }
1877
1878     return false;
1879 }
1880
1881 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1882 {
1883     unsigned access_size_max = mr->ops->valid.max_access_size;
1884
1885     /* Regions are assumed to support 1-4 byte accesses unless
1886        otherwise specified.  */
1887     if (access_size_max == 0) {
1888         access_size_max = 4;
1889     }
1890
1891     /* Bound the maximum access by the alignment of the address.  */
1892     if (!mr->ops->impl.unaligned) {
1893         unsigned align_size_max = addr & -addr;
1894         if (align_size_max != 0 && align_size_max < access_size_max) {
1895             access_size_max = align_size_max;
1896         }
1897     }
1898
1899     /* Don't attempt accesses larger than the maximum.  */
1900     if (l > access_size_max) {
1901         l = access_size_max;
1902     }
1903     if (l & (l - 1)) {
1904         l = 1 << (qemu_fls(l) - 1);
1905     }
1906
1907     return l;
1908 }
1909
1910 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1911                       int len, bool is_write)
1912 {
1913     hwaddr l;
1914     uint8_t *ptr;
1915     uint64_t val;
1916     hwaddr addr1;
1917     MemoryRegion *mr;
1918     bool error = false;
1919
1920     while (len > 0) {
1921         l = len;
1922         mr = address_space_translate(as, addr, &addr1, &l, is_write);
1923
1924         if (is_write) {
1925             if (!memory_access_is_direct(mr, is_write)) {
1926                 l = memory_access_size(mr, l, addr1);
1927                 /* XXX: could force current_cpu to NULL to avoid
1928                    potential bugs */
1929                 switch (l) {
1930                 case 8:
1931                     /* 64 bit write access */
1932                     val = ldq_p(buf);
1933                     error |= io_mem_write(mr, addr1, val, 8);
1934                     break;
1935                 case 4:
1936                     /* 32 bit write access */
1937                     val = ldl_p(buf);
1938                     error |= io_mem_write(mr, addr1, val, 4);
1939                     break;
1940                 case 2:
1941                     /* 16 bit write access */
1942                     val = lduw_p(buf);
1943                     error |= io_mem_write(mr, addr1, val, 2);
1944                     break;
1945                 case 1:
1946                     /* 8 bit write access */
1947                     val = ldub_p(buf);
1948                     error |= io_mem_write(mr, addr1, val, 1);
1949                     break;
1950                 default:
1951                     abort();
1952                 }
1953             } else {
1954                 addr1 += memory_region_get_ram_addr(mr);
1955                 /* RAM case */
1956                 ptr = qemu_get_ram_ptr(addr1);
1957                 memcpy(ptr, buf, l);
1958                 invalidate_and_set_dirty(addr1, l);
1959             }
1960         } else {
1961             if (!memory_access_is_direct(mr, is_write)) {
1962                 /* I/O case */
1963                 l = memory_access_size(mr, l, addr1);
1964                 switch (l) {
1965                 case 8:
1966                     /* 64 bit read access */
1967                     error |= io_mem_read(mr, addr1, &val, 8);
1968                     stq_p(buf, val);
1969                     break;
1970                 case 4:
1971                     /* 32 bit read access */
1972                     error |= io_mem_read(mr, addr1, &val, 4);
1973                     stl_p(buf, val);
1974                     break;
1975                 case 2:
1976                     /* 16 bit read access */
1977                     error |= io_mem_read(mr, addr1, &val, 2);
1978                     stw_p(buf, val);
1979                     break;
1980                 case 1:
1981                     /* 8 bit read access */
1982                     error |= io_mem_read(mr, addr1, &val, 1);
1983                     stb_p(buf, val);
1984                     break;
1985                 default:
1986                     abort();
1987                 }
1988             } else {
1989                 /* RAM case */
1990                 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1991                 memcpy(buf, ptr, l);
1992             }
1993         }
1994         len -= l;
1995         buf += l;
1996         addr += l;
1997     }
1998
1999     return error;
2000 }
2001
2002 bool address_space_write(AddressSpace *as, hwaddr addr,
2003                          const uint8_t *buf, int len)
2004 {
2005     return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2006 }
2007
2008 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2009 {
2010     return address_space_rw(as, addr, buf, len, false);
2011 }
2012
2013
2014 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2015                             int len, int is_write)
2016 {
2017     address_space_rw(&address_space_memory, addr, buf, len, is_write);
2018 }
2019
2020 /* used for ROM loading : can write in RAM and ROM */
2021 void cpu_physical_memory_write_rom(hwaddr addr,
2022                                    const uint8_t *buf, int len)
2023 {
2024     hwaddr l;
2025     uint8_t *ptr;
2026     hwaddr addr1;
2027     MemoryRegion *mr;
2028
2029     while (len > 0) {
2030         l = len;
2031         mr = address_space_translate(&address_space_memory,
2032                                      addr, &addr1, &l, true);
2033
2034         if (!(memory_region_is_ram(mr) ||
2035               memory_region_is_romd(mr))) {
2036             /* do nothing */
2037         } else {
2038             addr1 += memory_region_get_ram_addr(mr);
2039             /* ROM/RAM case */
2040             ptr = qemu_get_ram_ptr(addr1);
2041             memcpy(ptr, buf, l);
2042             invalidate_and_set_dirty(addr1, l);
2043         }
2044         len -= l;
2045         buf += l;
2046         addr += l;
2047     }
2048 }
2049
2050 typedef struct {
2051     MemoryRegion *mr;
2052     void *buffer;
2053     hwaddr addr;
2054     hwaddr len;
2055 } BounceBuffer;
2056
2057 static BounceBuffer bounce;
2058
2059 typedef struct MapClient {
2060     void *opaque;
2061     void (*callback)(void *opaque);
2062     QLIST_ENTRY(MapClient) link;
2063 } MapClient;
2064
2065 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2066     = QLIST_HEAD_INITIALIZER(map_client_list);
2067
2068 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2069 {
2070     MapClient *client = g_malloc(sizeof(*client));
2071
2072     client->opaque = opaque;
2073     client->callback = callback;
2074     QLIST_INSERT_HEAD(&map_client_list, client, link);
2075     return client;
2076 }
2077
2078 static void cpu_unregister_map_client(void *_client)
2079 {
2080     MapClient *client = (MapClient *)_client;
2081
2082     QLIST_REMOVE(client, link);
2083     g_free(client);
2084 }
2085
2086 static void cpu_notify_map_clients(void)
2087 {
2088     MapClient *client;
2089
2090     while (!QLIST_EMPTY(&map_client_list)) {
2091         client = QLIST_FIRST(&map_client_list);
2092         client->callback(client->opaque);
2093         cpu_unregister_map_client(client);
2094     }
2095 }
2096
2097 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2098 {
2099     MemoryRegion *mr;
2100     hwaddr l, xlat;
2101
2102     while (len > 0) {
2103         l = len;
2104         mr = address_space_translate(as, addr, &xlat, &l, is_write);
2105         if (!memory_access_is_direct(mr, is_write)) {
2106             l = memory_access_size(mr, l, addr);
2107             if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2108                 return false;
2109             }
2110         }
2111
2112         len -= l;
2113         addr += l;
2114     }
2115     return true;
2116 }
2117
2118 /* Map a physical memory region into a host virtual address.
2119  * May map a subset of the requested range, given by and returned in *plen.
2120  * May return NULL if resources needed to perform the mapping are exhausted.
2121  * Use only for reads OR writes - not for read-modify-write operations.
2122  * Use cpu_register_map_client() to know when retrying the map operation is
2123  * likely to succeed.
2124  */
2125 void *address_space_map(AddressSpace *as,
2126                         hwaddr addr,
2127                         hwaddr *plen,
2128                         bool is_write)
2129 {
2130     hwaddr len = *plen;
2131     hwaddr done = 0;
2132     hwaddr l, xlat, base;
2133     MemoryRegion *mr, *this_mr;
2134     ram_addr_t raddr;
2135
2136     if (len == 0) {
2137         return NULL;
2138     }
2139
2140     l = len;
2141     mr = address_space_translate(as, addr, &xlat, &l, is_write);
2142     if (!memory_access_is_direct(mr, is_write)) {
2143         if (bounce.buffer) {
2144             return NULL;
2145         }
2146         bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2147         bounce.addr = addr;
2148         bounce.len = l;
2149
2150         memory_region_ref(mr);
2151         bounce.mr = mr;
2152         if (!is_write) {
2153             address_space_read(as, addr, bounce.buffer, l);
2154         }
2155
2156         *plen = l;
2157         return bounce.buffer;
2158     }
2159
2160     base = xlat;
2161     raddr = memory_region_get_ram_addr(mr);
2162
2163     for (;;) {
2164         len -= l;
2165         addr += l;
2166         done += l;
2167         if (len == 0) {
2168             break;
2169         }
2170
2171         l = len;
2172         this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2173         if (this_mr != mr || xlat != base + done) {
2174             break;
2175         }
2176     }
2177
2178     memory_region_ref(mr);
2179     *plen = done;
2180     return qemu_ram_ptr_length(raddr + base, plen);
2181 }
2182
2183 /* Unmaps a memory region previously mapped by address_space_map().
2184  * Will also mark the memory as dirty if is_write == 1.  access_len gives
2185  * the amount of memory that was actually read or written by the caller.
2186  */
2187 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2188                          int is_write, hwaddr access_len)
2189 {
2190     if (buffer != bounce.buffer) {
2191         MemoryRegion *mr;
2192         ram_addr_t addr1;
2193
2194         mr = qemu_ram_addr_from_host(buffer, &addr1);
2195         assert(mr != NULL);
2196         if (is_write) {
2197             while (access_len) {
2198                 unsigned l;
2199                 l = TARGET_PAGE_SIZE;
2200                 if (l > access_len)
2201                     l = access_len;
2202                 invalidate_and_set_dirty(addr1, l);
2203                 addr1 += l;
2204                 access_len -= l;
2205             }
2206         }
2207         if (xen_enabled()) {
2208             xen_invalidate_map_cache_entry(buffer);
2209         }
2210         memory_region_unref(mr);
2211         return;
2212     }
2213     if (is_write) {
2214         address_space_write(as, bounce.addr, bounce.buffer, access_len);
2215     }
2216     qemu_vfree(bounce.buffer);
2217     bounce.buffer = NULL;
2218     memory_region_unref(bounce.mr);
2219     cpu_notify_map_clients();
2220 }
2221
2222 void *cpu_physical_memory_map(hwaddr addr,
2223                               hwaddr *plen,
2224                               int is_write)
2225 {
2226     return address_space_map(&address_space_memory, addr, plen, is_write);
2227 }
2228
2229 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2230                                int is_write, hwaddr access_len)
2231 {
2232     return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2233 }
2234
2235 /* warning: addr must be aligned */
2236 static inline uint32_t ldl_phys_internal(hwaddr addr,
2237                                          enum device_endian endian)
2238 {
2239     uint8_t *ptr;
2240     uint64_t val;
2241     MemoryRegion *mr;
2242     hwaddr l = 4;
2243     hwaddr addr1;
2244
2245     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2246                                  false);
2247     if (l < 4 || !memory_access_is_direct(mr, false)) {
2248         /* I/O case */
2249         io_mem_read(mr, addr1, &val, 4);
2250 #if defined(TARGET_WORDS_BIGENDIAN)
2251         if (endian == DEVICE_LITTLE_ENDIAN) {
2252             val = bswap32(val);
2253         }
2254 #else
2255         if (endian == DEVICE_BIG_ENDIAN) {
2256             val = bswap32(val);
2257         }
2258 #endif
2259     } else {
2260         /* RAM case */
2261         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2262                                 & TARGET_PAGE_MASK)
2263                                + addr1);
2264         switch (endian) {
2265         case DEVICE_LITTLE_ENDIAN:
2266             val = ldl_le_p(ptr);
2267             break;
2268         case DEVICE_BIG_ENDIAN:
2269             val = ldl_be_p(ptr);
2270             break;
2271         default:
2272             val = ldl_p(ptr);
2273             break;
2274         }
2275     }
2276     return val;
2277 }
2278
2279 uint32_t ldl_phys(hwaddr addr)
2280 {
2281     return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2282 }
2283
2284 uint32_t ldl_le_phys(hwaddr addr)
2285 {
2286     return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2287 }
2288
2289 uint32_t ldl_be_phys(hwaddr addr)
2290 {
2291     return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2292 }
2293
2294 /* warning: addr must be aligned */
2295 static inline uint64_t ldq_phys_internal(hwaddr addr,
2296                                          enum device_endian endian)
2297 {
2298     uint8_t *ptr;
2299     uint64_t val;
2300     MemoryRegion *mr;
2301     hwaddr l = 8;
2302     hwaddr addr1;
2303
2304     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2305                                  false);
2306     if (l < 8 || !memory_access_is_direct(mr, false)) {
2307         /* I/O case */
2308         io_mem_read(mr, addr1, &val, 8);
2309 #if defined(TARGET_WORDS_BIGENDIAN)
2310         if (endian == DEVICE_LITTLE_ENDIAN) {
2311             val = bswap64(val);
2312         }
2313 #else
2314         if (endian == DEVICE_BIG_ENDIAN) {
2315             val = bswap64(val);
2316         }
2317 #endif
2318     } else {
2319         /* RAM case */
2320         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2321                                 & TARGET_PAGE_MASK)
2322                                + addr1);
2323         switch (endian) {
2324         case DEVICE_LITTLE_ENDIAN:
2325             val = ldq_le_p(ptr);
2326             break;
2327         case DEVICE_BIG_ENDIAN:
2328             val = ldq_be_p(ptr);
2329             break;
2330         default:
2331             val = ldq_p(ptr);
2332             break;
2333         }
2334     }
2335     return val;
2336 }
2337
2338 uint64_t ldq_phys(hwaddr addr)
2339 {
2340     return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2341 }
2342
2343 uint64_t ldq_le_phys(hwaddr addr)
2344 {
2345     return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2346 }
2347
2348 uint64_t ldq_be_phys(hwaddr addr)
2349 {
2350     return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2351 }
2352
2353 /* XXX: optimize */
2354 uint32_t ldub_phys(hwaddr addr)
2355 {
2356     uint8_t val;
2357     cpu_physical_memory_read(addr, &val, 1);
2358     return val;
2359 }
2360
2361 /* warning: addr must be aligned */
2362 static inline uint32_t lduw_phys_internal(hwaddr addr,
2363                                           enum device_endian endian)
2364 {
2365     uint8_t *ptr;
2366     uint64_t val;
2367     MemoryRegion *mr;
2368     hwaddr l = 2;
2369     hwaddr addr1;
2370
2371     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2372                                  false);
2373     if (l < 2 || !memory_access_is_direct(mr, false)) {
2374         /* I/O case */
2375         io_mem_read(mr, addr1, &val, 2);
2376 #if defined(TARGET_WORDS_BIGENDIAN)
2377         if (endian == DEVICE_LITTLE_ENDIAN) {
2378             val = bswap16(val);
2379         }
2380 #else
2381         if (endian == DEVICE_BIG_ENDIAN) {
2382             val = bswap16(val);
2383         }
2384 #endif
2385     } else {
2386         /* RAM case */
2387         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2388                                 & TARGET_PAGE_MASK)
2389                                + addr1);
2390         switch (endian) {
2391         case DEVICE_LITTLE_ENDIAN:
2392             val = lduw_le_p(ptr);
2393             break;
2394         case DEVICE_BIG_ENDIAN:
2395             val = lduw_be_p(ptr);
2396             break;
2397         default:
2398             val = lduw_p(ptr);
2399             break;
2400         }
2401     }
2402     return val;
2403 }
2404
2405 uint32_t lduw_phys(hwaddr addr)
2406 {
2407     return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2408 }
2409
2410 uint32_t lduw_le_phys(hwaddr addr)
2411 {
2412     return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2413 }
2414
2415 uint32_t lduw_be_phys(hwaddr addr)
2416 {
2417     return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2418 }
2419
2420 /* warning: addr must be aligned. The ram page is not masked as dirty
2421    and the code inside is not invalidated. It is useful if the dirty
2422    bits are used to track modified PTEs */
2423 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2424 {
2425     uint8_t *ptr;
2426     MemoryRegion *mr;
2427     hwaddr l = 4;
2428     hwaddr addr1;
2429
2430     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2431                                  true);
2432     if (l < 4 || !memory_access_is_direct(mr, true)) {
2433         io_mem_write(mr, addr1, val, 4);
2434     } else {
2435         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2436         ptr = qemu_get_ram_ptr(addr1);
2437         stl_p(ptr, val);
2438
2439         if (unlikely(in_migration)) {
2440             if (!cpu_physical_memory_is_dirty(addr1)) {
2441                 /* invalidate code */
2442                 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2443                 /* set dirty bit */
2444                 cpu_physical_memory_set_dirty_flags(
2445                     addr1, (0xff & ~CODE_DIRTY_FLAG));
2446             }
2447         }
2448     }
2449 }
2450
2451 /* warning: addr must be aligned */
2452 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2453                                      enum device_endian endian)
2454 {
2455     uint8_t *ptr;
2456     MemoryRegion *mr;
2457     hwaddr l = 4;
2458     hwaddr addr1;
2459
2460     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2461                                  true);
2462     if (l < 4 || !memory_access_is_direct(mr, true)) {
2463 #if defined(TARGET_WORDS_BIGENDIAN)
2464         if (endian == DEVICE_LITTLE_ENDIAN) {
2465             val = bswap32(val);
2466         }
2467 #else
2468         if (endian == DEVICE_BIG_ENDIAN) {
2469             val = bswap32(val);
2470         }
2471 #endif
2472         io_mem_write(mr, addr1, val, 4);
2473     } else {
2474         /* RAM case */
2475         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2476         ptr = qemu_get_ram_ptr(addr1);
2477         switch (endian) {
2478         case DEVICE_LITTLE_ENDIAN:
2479             stl_le_p(ptr, val);
2480             break;
2481         case DEVICE_BIG_ENDIAN:
2482             stl_be_p(ptr, val);
2483             break;
2484         default:
2485             stl_p(ptr, val);
2486             break;
2487         }
2488         invalidate_and_set_dirty(addr1, 4);
2489     }
2490 }
2491
2492 void stl_phys(hwaddr addr, uint32_t val)
2493 {
2494     stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2495 }
2496
2497 void stl_le_phys(hwaddr addr, uint32_t val)
2498 {
2499     stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2500 }
2501
2502 void stl_be_phys(hwaddr addr, uint32_t val)
2503 {
2504     stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2505 }
2506
2507 /* XXX: optimize */
2508 void stb_phys(hwaddr addr, uint32_t val)
2509 {
2510     uint8_t v = val;
2511     cpu_physical_memory_write(addr, &v, 1);
2512 }
2513
2514 /* warning: addr must be aligned */
2515 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2516                                      enum device_endian endian)
2517 {
2518     uint8_t *ptr;
2519     MemoryRegion *mr;
2520     hwaddr l = 2;
2521     hwaddr addr1;
2522
2523     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2524                                  true);
2525     if (l < 2 || !memory_access_is_direct(mr, true)) {
2526 #if defined(TARGET_WORDS_BIGENDIAN)
2527         if (endian == DEVICE_LITTLE_ENDIAN) {
2528             val = bswap16(val);
2529         }
2530 #else
2531         if (endian == DEVICE_BIG_ENDIAN) {
2532             val = bswap16(val);
2533         }
2534 #endif
2535         io_mem_write(mr, addr1, val, 2);
2536     } else {
2537         /* RAM case */
2538         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2539         ptr = qemu_get_ram_ptr(addr1);
2540         switch (endian) {
2541         case DEVICE_LITTLE_ENDIAN:
2542             stw_le_p(ptr, val);
2543             break;
2544         case DEVICE_BIG_ENDIAN:
2545             stw_be_p(ptr, val);
2546             break;
2547         default:
2548             stw_p(ptr, val);
2549             break;
2550         }
2551         invalidate_and_set_dirty(addr1, 2);
2552     }
2553 }
2554
2555 void stw_phys(hwaddr addr, uint32_t val)
2556 {
2557     stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2558 }
2559
2560 void stw_le_phys(hwaddr addr, uint32_t val)
2561 {
2562     stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2563 }
2564
2565 void stw_be_phys(hwaddr addr, uint32_t val)
2566 {
2567     stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2568 }
2569
2570 /* XXX: optimize */
2571 void stq_phys(hwaddr addr, uint64_t val)
2572 {
2573     val = tswap64(val);
2574     cpu_physical_memory_write(addr, &val, 8);
2575 }
2576
2577 void stq_le_phys(hwaddr addr, uint64_t val)
2578 {
2579     val = cpu_to_le64(val);
2580     cpu_physical_memory_write(addr, &val, 8);
2581 }
2582
2583 void stq_be_phys(hwaddr addr, uint64_t val)
2584 {
2585     val = cpu_to_be64(val);
2586     cpu_physical_memory_write(addr, &val, 8);
2587 }
2588
2589 /* virtual memory access for debug (includes writing to ROM) */
2590 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2591                         uint8_t *buf, int len, int is_write)
2592 {
2593     int l;
2594     hwaddr phys_addr;
2595     target_ulong page;
2596
2597     while (len > 0) {
2598         page = addr & TARGET_PAGE_MASK;
2599         phys_addr = cpu_get_phys_page_debug(cpu, page);
2600         /* if no physical page mapped, return an error */
2601         if (phys_addr == -1)
2602             return -1;
2603         l = (page + TARGET_PAGE_SIZE) - addr;
2604         if (l > len)
2605             l = len;
2606         phys_addr += (addr & ~TARGET_PAGE_MASK);
2607         if (is_write)
2608             cpu_physical_memory_write_rom(phys_addr, buf, l);
2609         else
2610             cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2611         len -= l;
2612         buf += l;
2613         addr += l;
2614     }
2615     return 0;
2616 }
2617 #endif
2618
2619 #if !defined(CONFIG_USER_ONLY)
2620
2621 /*
2622  * A helper function for the _utterly broken_ virtio device model to find out if
2623  * it's running on a big endian machine. Don't do this at home kids!
2624  */
2625 bool virtio_is_big_endian(void);
2626 bool virtio_is_big_endian(void)
2627 {
2628 #if defined(TARGET_WORDS_BIGENDIAN)
2629     return true;
2630 #else
2631     return false;
2632 #endif
2633 }
2634
2635 #endif
2636
2637 #ifndef CONFIG_USER_ONLY
2638 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2639 {
2640     MemoryRegion*mr;
2641     hwaddr l = 1;
2642
2643     mr = address_space_translate(&address_space_memory,
2644                                  phys_addr, &phys_addr, &l, false);
2645
2646     return !(memory_region_is_ram(mr) ||
2647              memory_region_is_romd(mr));
2648 }
2649
2650 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2651 {
2652     RAMBlock *block;
2653
2654     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2655         func(block->host, block->offset, block->length, opaque);
2656     }
2657 }
2658 #endif