exec.c

   1 /*
   2  *  Virtual page mapping
   3  *
   4  *  Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19 #include "config.h"
  20 #ifdef _WIN32
  21 #include <windows.h>
  22 #else
  23 #include <sys/types.h>
  24 #include <sys/mman.h>
  25 #endif
  26
  27 #include "qemu-common.h"
  28 #include "cpu.h"
  29 #include "tcg.h"
  30 #include "hw/hw.h"
  31 #include "hw/qdev.h"
  32 #include "qemu/osdep.h"
  33 #include "sysemu/kvm.h"
  34 #include "sysemu/sysemu.h"
  35 #include "hw/xen/xen.h"
  36 #include "qemu/timer.h"
  37 #include "qemu/config-file.h"
  38 #include "exec/memory.h"
  39 #include "sysemu/dma.h"
  40 #include "exec/address-spaces.h"
  41 #if defined(CONFIG_USER_ONLY)
  42 #include <qemu.h>
  43 #else /* !CONFIG_USER_ONLY */
  44 #include "sysemu/xen-mapcache.h"
  45 #include "trace.h"
  46 #endif
  47 #include "exec/cpu-all.h"
  48
  49 #include "exec/cputlb.h"
  50 #include "translate-all.h"
  51
  52 #include "exec/memory-internal.h"
  53
  54 //#define DEBUG_SUBPAGE
  55
  56 #if !defined(CONFIG_USER_ONLY)
  57 static int in_migration;
  58
  59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
  60
  61 static MemoryRegion *system_memory;
  62 static MemoryRegion *system_io;
  63
  64 AddressSpace address_space_io;
  65 AddressSpace address_space_memory;
  66
  67 MemoryRegion io_mem_rom, io_mem_notdirty;
  68 static MemoryRegion io_mem_unassigned;
  69
  70 #endif
  71
  72 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
  73 /* current CPU in the current thread. It is only valid inside
  74    cpu_exec() */
  75 DEFINE_TLS(CPUState *, current_cpu);
  76 /* 0 = Do not count executed instructions.
  77    1 = Precise instruction counting.
  78    2 = Adaptive rate instruction counting.  */
  79 int use_icount;
  80
  81 #if !defined(CONFIG_USER_ONLY)
  82
  83 typedef struct PhysPageEntry PhysPageEntry;
  84
  85 struct PhysPageEntry {
  86     uint16_t is_leaf : 1;
  87      /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
  88     uint16_t ptr : 15;
  89 };
  90
  91 typedef PhysPageEntry Node[L2_SIZE];
  92
  93 struct AddressSpaceDispatch {
  94     /* This is a multi-level map on the physical address space.
  95      * The bottom level has pointers to MemoryRegionSections.
  96      */
  97     PhysPageEntry phys_map;
  98     Node *nodes;
  99     MemoryRegionSection *sections;
 100     AddressSpace *as;
 101 };
 102
 103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
 104 typedef struct subpage_t {
 105     MemoryRegion iomem;
 106     AddressSpace *as;
 107     hwaddr base;
 108     uint16_t sub_section[TARGET_PAGE_SIZE];
 109 } subpage_t;
 110
 111 #define PHYS_SECTION_UNASSIGNED 0
 112 #define PHYS_SECTION_NOTDIRTY 1
 113 #define PHYS_SECTION_ROM 2
 114 #define PHYS_SECTION_WATCH 3
 115
 116 typedef struct PhysPageMap {
 117     unsigned sections_nb;
 118     unsigned sections_nb_alloc;
 119     unsigned nodes_nb;
 120     unsigned nodes_nb_alloc;
 121     Node *nodes;
 122     MemoryRegionSection *sections;
 123 } PhysPageMap;
 124
 125 static PhysPageMap *prev_map;
 126 static PhysPageMap next_map;
 127
 128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
 129
 130 static void io_mem_init(void);
 131 static void memory_map_init(void);
 132
 133 static MemoryRegion io_mem_watch;
 134 #endif
 135
 136 #if !defined(CONFIG_USER_ONLY)
 137
 138 static void phys_map_node_reserve(unsigned nodes)
 139 {
 140     if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
 141         next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
 142                                             16);
 143         next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
 144                                       next_map.nodes_nb + nodes);
 145         next_map.nodes = g_renew(Node, next_map.nodes,
 146                                  next_map.nodes_nb_alloc);
 147     }
 148 }
 149
 150 static uint16_t phys_map_node_alloc(void)
 151 {
 152     unsigned i;
 153     uint16_t ret;
 154
 155     ret = next_map.nodes_nb++;
 156     assert(ret != PHYS_MAP_NODE_NIL);
 157     assert(ret != next_map.nodes_nb_alloc);
 158     for (i = 0; i < L2_SIZE; ++i) {
 159         next_map.nodes[ret][i].is_leaf = 0;
 160         next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
 161     }
 162     return ret;
 163 }
 164
 165 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
 166                                 hwaddr *nb, uint16_t leaf,
 167                                 int level)
 168 {
 169     PhysPageEntry *p;
 170     int i;
 171     hwaddr step = (hwaddr)1 << (level * L2_BITS);
 172
 173     if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
 174         lp->ptr = phys_map_node_alloc();
 175         p = next_map.nodes[lp->ptr];
 176         if (level == 0) {
 177             for (i = 0; i < L2_SIZE; i++) {
 178                 p[i].is_leaf = 1;
 179                 p[i].ptr = PHYS_SECTION_UNASSIGNED;
 180             }
 181         }
 182     } else {
 183         p = next_map.nodes[lp->ptr];
 184     }
 185     lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
 186
 187     while (*nb && lp < &p[L2_SIZE]) {
 188         if ((*index & (step - 1)) == 0 && *nb >= step) {
 189             lp->is_leaf = true;
 190             lp->ptr = leaf;
 191             *index += step;
 192             *nb -= step;
 193         } else {
 194             phys_page_set_level(lp, index, nb, leaf, level - 1);
 195         }
 196         ++lp;
 197     }
 198 }
 199
 200 static void phys_page_set(AddressSpaceDispatch *d,
 201                           hwaddr index, hwaddr nb,
 202                           uint16_t leaf)
 203 {
 204     /* Wildly overreserve - it doesn't matter much. */
 205     phys_map_node_reserve(3 * P_L2_LEVELS);
 206
 207     phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
 208 }
 209
 210 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
 211                                            Node *nodes, MemoryRegionSection *sections)
 212 {
 213     PhysPageEntry *p;
 214     int i;
 215
 216     for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
 217         if (lp.ptr == PHYS_MAP_NODE_NIL) {
 218             return &sections[PHYS_SECTION_UNASSIGNED];
 219         }
 220         p = nodes[lp.ptr];
 221         lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
 222     }
 223     return &sections[lp.ptr];
 224 }
 225
 226 bool memory_region_is_unassigned(MemoryRegion *mr)
 227 {
 228     return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
 229         && mr != &io_mem_watch;
 230 }
 231
 232 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
 233                                                         hwaddr addr,
 234                                                         bool resolve_subpage)
 235 {
 236     MemoryRegionSection *section;
 237     subpage_t *subpage;
 238
 239     section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
 240                              d->nodes, d->sections);
 241     if (resolve_subpage && section->mr->subpage) {
 242         subpage = container_of(section->mr, subpage_t, iomem);
 243         section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
 244     }
 245     return section;
 246 }
 247
 248 static MemoryRegionSection *
 249 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
 250                                  hwaddr *plen, bool resolve_subpage)
 251 {
 252     MemoryRegionSection *section;
 253     Int128 diff;
 254
 255     section = address_space_lookup_region(d, addr, resolve_subpage);
 256     /* Compute offset within MemoryRegionSection */
 257     addr -= section->offset_within_address_space;
 258
 259     /* Compute offset within MemoryRegion */
 260     *xlat = addr + section->offset_within_region;
 261
 262     diff = int128_sub(section->mr->size, int128_make64(addr));
 263     *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
 264     return section;
 265 }
 266
 267 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
 268                                       hwaddr *xlat, hwaddr *plen,
 269                                       bool is_write)
 270 {
 271     IOMMUTLBEntry iotlb;
 272     MemoryRegionSection *section;
 273     MemoryRegion *mr;
 274     hwaddr len = *plen;
 275
 276     for (;;) {
 277         section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
 278         mr = section->mr;
 279
 280         if (!mr->iommu_ops) {
 281             break;
 282         }
 283
 284         iotlb = mr->iommu_ops->translate(mr, addr);
 285         addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
 286                 | (addr & iotlb.addr_mask));
 287         len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
 288         if (!(iotlb.perm & (1 << is_write))) {
 289             mr = &io_mem_unassigned;
 290             break;
 291         }
 292
 293         as = iotlb.target_as;
 294     }
 295
 296     *plen = len;
 297     *xlat = addr;
 298     return mr;
 299 }
 300
 301 MemoryRegionSection *
 302 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
 303                                   hwaddr *plen)
 304 {
 305     MemoryRegionSection *section;
 306     section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
 307
 308     assert(!section->mr->iommu_ops);
 309     return section;
 310 }
 311 #endif
 312
 313 void cpu_exec_init_all(void)
 314 {
 315 #if !defined(CONFIG_USER_ONLY)
 316     qemu_mutex_init(&ram_list.mutex);
 317     memory_map_init();
 318     io_mem_init();
 319 #endif
 320 }
 321
 322 #if !defined(CONFIG_USER_ONLY)
 323
 324 static int cpu_common_post_load(void *opaque, int version_id)
 325 {
 326     CPUState *cpu = opaque;
 327
 328     /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
 329        version_id is increased. */
 330     cpu->interrupt_request &= ~0x01;
 331     tlb_flush(cpu->env_ptr, 1);
 332
 333     return 0;
 334 }
 335
 336 const VMStateDescription vmstate_cpu_common = {
 337     .name = "cpu_common",
 338     .version_id = 1,
 339     .minimum_version_id = 1,
 340     .minimum_version_id_old = 1,
 341     .post_load = cpu_common_post_load,
 342     .fields      = (VMStateField []) {
 343         VMSTATE_UINT32(halted, CPUState),
 344         VMSTATE_UINT32(interrupt_request, CPUState),
 345         VMSTATE_END_OF_LIST()
 346     }
 347 };
 348
 349 #endif
 350
 351 CPUState *qemu_get_cpu(int index)
 352 {
 353     CPUState *cpu;
 354
 355     CPU_FOREACH(cpu) {
 356         if (cpu->cpu_index == index) {
 357             return cpu;
 358         }
 359     }
 360
 361     return NULL;
 362 }
 363
 364 void cpu_exec_init(CPUArchState *env)
 365 {
 366     CPUState *cpu = ENV_GET_CPU(env);
 367     CPUClass *cc = CPU_GET_CLASS(cpu);
 368     CPUState *some_cpu;
 369     int cpu_index;
 370
 371 #if defined(CONFIG_USER_ONLY)
 372     cpu_list_lock();
 373 #endif
 374     cpu_index = 0;
 375     CPU_FOREACH(some_cpu) {
 376         cpu_index++;
 377     }
 378     cpu->cpu_index = cpu_index;
 379     cpu->numa_node = 0;
 380     QTAILQ_INIT(&env->breakpoints);
 381     QTAILQ_INIT(&env->watchpoints);
 382 #ifndef CONFIG_USER_ONLY
 383     cpu->thread_id = qemu_get_thread_id();
 384 #endif
 385     QTAILQ_INSERT_TAIL(&cpus, cpu, node);
 386 #if defined(CONFIG_USER_ONLY)
 387     cpu_list_unlock();
 388 #endif
 389     if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
 390         vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
 391     }
 392 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
 393     register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
 394                     cpu_save, cpu_load, env);
 395     assert(cc->vmsd == NULL);
 396     assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
 397 #endif
 398     if (cc->vmsd != NULL) {
 399         vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
 400     }
 401 }
 402
 403 #if defined(TARGET_HAS_ICE)
 404 #if defined(CONFIG_USER_ONLY)
 405 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
 406 {
 407     tb_invalidate_phys_page_range(pc, pc + 1, 0);
 408 }
 409 #else
 410 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
 411 {
 412     tb_invalidate_phys_addr(cpu_get_phys_page_debug(cpu, pc) |
 413             (pc & ~TARGET_PAGE_MASK));
 414 }
 415 #endif
 416 #endif /* TARGET_HAS_ICE */
 417
 418 #if defined(CONFIG_USER_ONLY)
 419 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 420
 421 {
 422 }
 423
 424 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 425                           int flags, CPUWatchpoint **watchpoint)
 426 {
 427     return -ENOSYS;
 428 }
 429 #else
 430 /* Add a watchpoint.  */
 431 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 432                           int flags, CPUWatchpoint **watchpoint)
 433 {
 434     target_ulong len_mask = ~(len - 1);
 435     CPUWatchpoint *wp;
 436
 437     /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
 438     if ((len & (len - 1)) || (addr & ~len_mask) ||
 439             len == 0 || len > TARGET_PAGE_SIZE) {
 440         fprintf(stderr, "qemu: tried to set invalid watchpoint at "
 441                 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
 442         return -EINVAL;
 443     }
 444     wp = g_malloc(sizeof(*wp));
 445
 446     wp->vaddr = addr;
 447     wp->len_mask = len_mask;
 448     wp->flags = flags;
 449
 450     /* keep all GDB-injected watchpoints in front */
 451     if (flags & BP_GDB)
 452         QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
 453     else
 454         QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
 455
 456     tlb_flush_page(env, addr);
 457
 458     if (watchpoint)
 459         *watchpoint = wp;
 460     return 0;
 461 }
 462
 463 /* Remove a specific watchpoint.  */
 464 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
 465                           int flags)
 466 {
 467     target_ulong len_mask = ~(len - 1);
 468     CPUWatchpoint *wp;
 469
 470     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 471         if (addr == wp->vaddr && len_mask == wp->len_mask
 472                 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
 473             cpu_watchpoint_remove_by_ref(env, wp);
 474             return 0;
 475         }
 476     }
 477     return -ENOENT;
 478 }
 479
 480 /* Remove a specific watchpoint by reference.  */
 481 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
 482 {
 483     QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
 484
 485     tlb_flush_page(env, watchpoint->vaddr);
 486
 487     g_free(watchpoint);
 488 }
 489
 490 /* Remove all matching watchpoints.  */
 491 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 492 {
 493     CPUWatchpoint *wp, *next;
 494
 495     QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
 496         if (wp->flags & mask)
 497             cpu_watchpoint_remove_by_ref(env, wp);
 498     }
 499 }
 500 #endif
 501
 502 /* Add a breakpoint.  */
 503 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
 504                           CPUBreakpoint **breakpoint)
 505 {
 506 #if defined(TARGET_HAS_ICE)
 507     CPUBreakpoint *bp;
 508
 509     bp = g_malloc(sizeof(*bp));
 510
 511     bp->pc = pc;
 512     bp->flags = flags;
 513
 514     /* keep all GDB-injected breakpoints in front */
 515     if (flags & BP_GDB) {
 516         QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
 517     } else {
 518         QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
 519     }
 520
 521     breakpoint_invalidate(ENV_GET_CPU(env), pc);
 522
 523     if (breakpoint) {
 524         *breakpoint = bp;
 525     }
 526     return 0;
 527 #else
 528     return -ENOSYS;
 529 #endif
 530 }
 531
 532 /* Remove a specific breakpoint.  */
 533 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
 534 {
 535 #if defined(TARGET_HAS_ICE)
 536     CPUBreakpoint *bp;
 537
 538     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
 539         if (bp->pc == pc && bp->flags == flags) {
 540             cpu_breakpoint_remove_by_ref(env, bp);
 541             return 0;
 542         }
 543     }
 544     return -ENOENT;
 545 #else
 546     return -ENOSYS;
 547 #endif
 548 }
 549
 550 /* Remove a specific breakpoint by reference.  */
 551 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
 552 {
 553 #if defined(TARGET_HAS_ICE)
 554     QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
 555
 556     breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
 557
 558     g_free(breakpoint);
 559 #endif
 560 }
 561
 562 /* Remove all matching breakpoints. */
 563 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
 564 {
 565 #if defined(TARGET_HAS_ICE)
 566     CPUBreakpoint *bp, *next;
 567
 568     QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
 569         if (bp->flags & mask)
 570             cpu_breakpoint_remove_by_ref(env, bp);
 571     }
 572 #endif
 573 }
 574
 575 /* enable or disable single step mode. EXCP_DEBUG is returned by the
 576    CPU loop after each instruction */
 577 void cpu_single_step(CPUState *cpu, int enabled)
 578 {
 579 #if defined(TARGET_HAS_ICE)
 580     if (cpu->singlestep_enabled != enabled) {
 581         cpu->singlestep_enabled = enabled;
 582         if (kvm_enabled()) {
 583             kvm_update_guest_debug(cpu, 0);
 584         } else {
 585             /* must flush all the translated code to avoid inconsistencies */
 586             /* XXX: only flush what is necessary */
 587             CPUArchState *env = cpu->env_ptr;
 588             tb_flush(env);
 589         }
 590     }
 591 #endif
 592 }
 593
 594 void cpu_abort(CPUArchState *env, const char *fmt, ...)
 595 {
 596     CPUState *cpu = ENV_GET_CPU(env);
 597     va_list ap;
 598     va_list ap2;
 599
 600     va_start(ap, fmt);
 601     va_copy(ap2, ap);
 602     fprintf(stderr, "qemu: fatal: ");
 603     vfprintf(stderr, fmt, ap);
 604     fprintf(stderr, "\n");
 605     cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 606     if (qemu_log_enabled()) {
 607         qemu_log("qemu: fatal: ");
 608         qemu_log_vprintf(fmt, ap2);
 609         qemu_log("\n");
 610         log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 611         qemu_log_flush();
 612         qemu_log_close();
 613     }
 614     va_end(ap2);
 615     va_end(ap);
 616 #if defined(CONFIG_USER_ONLY)
 617     {
 618         struct sigaction act;
 619         sigfillset(&act.sa_mask);
 620         act.sa_handler = SIG_DFL;
 621         sigaction(SIGABRT, &act, NULL);
 622     }
 623 #endif
 624     abort();
 625 }
 626
 627 #if !defined(CONFIG_USER_ONLY)
 628 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
 629 {
 630     RAMBlock *block;
 631
 632     /* The list is protected by the iothread lock here.  */
 633     block = ram_list.mru_block;
 634     if (block && addr - block->offset < block->length) {
 635         goto found;
 636     }
 637     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
 638         if (addr - block->offset < block->length) {
 639             goto found;
 640         }
 641     }
 642
 643     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
 644     abort();
 645
 646 found:
 647     ram_list.mru_block = block;
 648     return block;
 649 }
 650
 651 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
 652                                       uintptr_t length)
 653 {
 654     RAMBlock *block;
 655     ram_addr_t start1;
 656
 657     block = qemu_get_ram_block(start);
 658     assert(block == qemu_get_ram_block(end - 1));
 659     start1 = (uintptr_t)block->host + (start - block->offset);
 660     cpu_tlb_reset_dirty_all(start1, length);
 661 }
 662
 663 /* Note: start and end must be within the same ram block.  */
 664 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
 665                                      int dirty_flags)
 666 {
 667     uintptr_t length;
 668
 669     start &= TARGET_PAGE_MASK;
 670     end = TARGET_PAGE_ALIGN(end);
 671
 672     length = end - start;
 673     if (length == 0)
 674         return;
 675     cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
 676
 677     if (tcg_enabled()) {
 678         tlb_reset_dirty_range_all(start, end, length);
 679     }
 680 }
 681
 682 static int cpu_physical_memory_set_dirty_tracking(int enable)
 683 {
 684     int ret = 0;
 685     in_migration = enable;
 686     return ret;
 687 }
 688
 689 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
 690                                        MemoryRegionSection *section,
 691                                        target_ulong vaddr,
 692                                        hwaddr paddr, hwaddr xlat,
 693                                        int prot,
 694                                        target_ulong *address)
 695 {
 696     hwaddr iotlb;
 697     CPUWatchpoint *wp;
 698
 699     if (memory_region_is_ram(section->mr)) {
 700         /* Normal RAM.  */
 701         iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
 702             + xlat;
 703         if (!section->readonly) {
 704             iotlb |= PHYS_SECTION_NOTDIRTY;
 705         } else {
 706             iotlb |= PHYS_SECTION_ROM;
 707         }
 708     } else {
 709         iotlb = section - address_space_memory.dispatch->sections;
 710         iotlb += xlat;
 711     }
 712
 713     /* Make accesses to pages with watchpoints go via the
 714        watchpoint trap routines.  */
 715     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 716         if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
 717             /* Avoid trapping reads of pages with a write breakpoint. */
 718             if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
 719                 iotlb = PHYS_SECTION_WATCH + paddr;
 720                 *address |= TLB_MMIO;
 721                 break;
 722             }
 723         }
 724     }
 725
 726     return iotlb;
 727 }
 728 #endif /* defined(CONFIG_USER_ONLY) */
 729
 730 #if !defined(CONFIG_USER_ONLY)
 731
 732 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
 733                              uint16_t section);
 734 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
 735
 736 static void *(*phys_mem_alloc)(size_t size) = qemu_anon_ram_alloc;
 737
 738 /*
 739  * Set a custom physical guest memory alloator.
 740  * Accelerators with unusual needs may need this.  Hopefully, we can
 741  * get rid of it eventually.
 742  */
 743 void phys_mem_set_alloc(void *(*alloc)(size_t))
 744 {
 745     phys_mem_alloc = alloc;
 746 }
 747
 748 static uint16_t phys_section_add(MemoryRegionSection *section)
 749 {
 750     /* The physical section number is ORed with a page-aligned
 751      * pointer to produce the iotlb entries.  Thus it should
 752      * never overflow into the page-aligned value.
 753      */
 754     assert(next_map.sections_nb < TARGET_PAGE_SIZE);
 755
 756     if (next_map.sections_nb == next_map.sections_nb_alloc) {
 757         next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
 758                                          16);
 759         next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
 760                                     next_map.sections_nb_alloc);
 761     }
 762     next_map.sections[next_map.sections_nb] = *section;
 763     memory_region_ref(section->mr);
 764     return next_map.sections_nb++;
 765 }
 766
 767 static void phys_section_destroy(MemoryRegion *mr)
 768 {
 769     memory_region_unref(mr);
 770
 771     if (mr->subpage) {
 772         subpage_t *subpage = container_of(mr, subpage_t, iomem);
 773         memory_region_destroy(&subpage->iomem);
 774         g_free(subpage);
 775     }
 776 }
 777
 778 static void phys_sections_free(PhysPageMap *map)
 779 {
 780     while (map->sections_nb > 0) {
 781         MemoryRegionSection *section = &map->sections[--map->sections_nb];
 782         phys_section_destroy(section->mr);
 783     }
 784     g_free(map->sections);
 785     g_free(map->nodes);
 786     g_free(map);
 787 }
 788
 789 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
 790 {
 791     subpage_t *subpage;
 792     hwaddr base = section->offset_within_address_space
 793         & TARGET_PAGE_MASK;
 794     MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
 795                                                    next_map.nodes, next_map.sections);
 796     MemoryRegionSection subsection = {
 797         .offset_within_address_space = base,
 798         .size = int128_make64(TARGET_PAGE_SIZE),
 799     };
 800     hwaddr start, end;
 801
 802     assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
 803
 804     if (!(existing->mr->subpage)) {
 805         subpage = subpage_init(d->as, base);
 806         subsection.mr = &subpage->iomem;
 807         phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
 808                       phys_section_add(&subsection));
 809     } else {
 810         subpage = container_of(existing->mr, subpage_t, iomem);
 811     }
 812     start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
 813     end = start + int128_get64(section->size) - 1;
 814     subpage_register(subpage, start, end, phys_section_add(section));
 815 }
 816
 817
 818 static void register_multipage(AddressSpaceDispatch *d,
 819                                MemoryRegionSection *section)
 820 {
 821     hwaddr start_addr = section->offset_within_address_space;
 822     uint16_t section_index = phys_section_add(section);
 823     uint64_t num_pages = int128_get64(int128_rshift(section->size,
 824                                                     TARGET_PAGE_BITS));
 825
 826     assert(num_pages);
 827     phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
 828 }
 829
 830 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
 831 {
 832     AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
 833     AddressSpaceDispatch *d = as->next_dispatch;
 834     MemoryRegionSection now = *section, remain = *section;
 835     Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
 836
 837     if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
 838         uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
 839                        - now.offset_within_address_space;
 840
 841         now.size = int128_min(int128_make64(left), now.size);
 842         register_subpage(d, &now);
 843     } else {
 844         now.size = int128_zero();
 845     }
 846     while (int128_ne(remain.size, now.size)) {
 847         remain.size = int128_sub(remain.size, now.size);
 848         remain.offset_within_address_space += int128_get64(now.size);
 849         remain.offset_within_region += int128_get64(now.size);
 850         now = remain;
 851         if (int128_lt(remain.size, page_size)) {
 852             register_subpage(d, &now);
 853         } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
 854             now.size = page_size;
 855             register_subpage(d, &now);
 856         } else {
 857             now.size = int128_and(now.size, int128_neg(page_size));
 858             register_multipage(d, &now);
 859         }
 860     }
 861 }
 862
 863 void qemu_flush_coalesced_mmio_buffer(void)
 864 {
 865     if (kvm_enabled())
 866         kvm_flush_coalesced_mmio_buffer();
 867 }
 868
 869 void qemu_mutex_lock_ramlist(void)
 870 {
 871     qemu_mutex_lock(&ram_list.mutex);
 872 }
 873
 874 void qemu_mutex_unlock_ramlist(void)
 875 {
 876     qemu_mutex_unlock(&ram_list.mutex);
 877 }
 878
 879 #ifdef __linux__
 880
 881 #include <sys/vfs.h>
 882
 883 #define HUGETLBFS_MAGIC       0x958458f6
 884
 885 static long gethugepagesize(const char *path)
 886 {
 887     struct statfs fs;
 888     int ret;
 889
 890     do {
 891         ret = statfs(path, &fs);
 892     } while (ret != 0 && errno == EINTR);
 893
 894     if (ret != 0) {
 895         perror(path);
 896         return 0;
 897     }
 898
 899     if (fs.f_type != HUGETLBFS_MAGIC)
 900         fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
 901
 902     return fs.f_bsize;
 903 }
 904
 905 static void *file_ram_alloc(RAMBlock *block,
 906                             ram_addr_t memory,
 907                             const char *path)
 908 {
 909     char *filename;
 910     char *sanitized_name;
 911     char *c;
 912     void *area;
 913     int fd;
 914 #ifdef MAP_POPULATE
 915     int flags;
 916 #endif
 917     unsigned long hpagesize;
 918
 919     hpagesize = gethugepagesize(path);
 920     if (!hpagesize) {
 921         return NULL;
 922     }
 923
 924     if (memory < hpagesize) {
 925         return NULL;
 926     }
 927
 928     if (kvm_enabled() && !kvm_has_sync_mmu()) {
 929         fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
 930         return NULL;
 931     }
 932
 933     /* Make name safe to use with mkstemp by replacing '/' with '_'. */
 934     sanitized_name = g_strdup(block->mr->name);
 935     for (c = sanitized_name; *c != '\0'; c++) {
 936         if (*c == '/')
 937             *c = '_';
 938     }
 939
 940     filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
 941                                sanitized_name);
 942     g_free(sanitized_name);
 943
 944     fd = mkstemp(filename);
 945     if (fd < 0) {
 946         perror("unable to create backing store for hugepages");
 947         g_free(filename);
 948         return NULL;
 949     }
 950     unlink(filename);
 951     g_free(filename);
 952
 953     memory = (memory+hpagesize-1) & ~(hpagesize-1);
 954
 955     /*
 956      * ftruncate is not supported by hugetlbfs in older
 957      * hosts, so don't bother bailing out on errors.
 958      * If anything goes wrong with it under other filesystems,
 959      * mmap will fail.
 960      */
 961     if (ftruncate(fd, memory))
 962         perror("ftruncate");
 963
 964 #ifdef MAP_POPULATE
 965     /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
 966      * MAP_PRIVATE is requested.  For mem_prealloc we mmap as MAP_SHARED
 967      * to sidestep this quirk.
 968      */
 969     flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
 970     area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
 971 #else
 972     area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
 973 #endif
 974     if (area == MAP_FAILED) {
 975         perror("file_ram_alloc: can't mmap RAM pages");
 976         close(fd);
 977         return (NULL);
 978     }
 979     block->fd = fd;
 980     return area;
 981 }
 982 #else
 983 static void *file_ram_alloc(RAMBlock *block,
 984                             ram_addr_t memory,
 985                             const char *path)
 986 {
 987     fprintf(stderr, "-mem-path not supported on this host\n");
 988     exit(1);
 989 }
 990 #endif
 991
 992 static ram_addr_t find_ram_offset(ram_addr_t size)
 993 {
 994     RAMBlock *block, *next_block;
 995     ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
 996
 997     assert(size != 0); /* it would hand out same offset multiple times */
 998
 999     if (QTAILQ_EMPTY(&ram_list.blocks))
1000         return 0;
1001
1002     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1003         ram_addr_t end, next = RAM_ADDR_MAX;
1004
1005         end = block->offset + block->length;
1006
1007         QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1008             if (next_block->offset >= end) {
1009                 next = MIN(next, next_block->offset);
1010             }
1011         }
1012         if (next - end >= size && next - end < mingap) {
1013             offset = end;
1014             mingap = next - end;
1015         }
1016     }
1017
1018     if (offset == RAM_ADDR_MAX) {
1019         fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1020                 (uint64_t)size);
1021         abort();
1022     }
1023
1024     return offset;
1025 }
1026
1027 ram_addr_t last_ram_offset(void)
1028 {
1029     RAMBlock *block;
1030     ram_addr_t last = 0;
1031
1032     QTAILQ_FOREACH(block, &ram_list.blocks, next)
1033         last = MAX(last, block->offset + block->length);
1034
1035     return last;
1036 }
1037
1038 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1039 {
1040     int ret;
1041
1042     /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1043     if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1044                            "dump-guest-core", true)) {
1045         ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1046         if (ret) {
1047             perror("qemu_madvise");
1048             fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1049                             "but dump_guest_core=off specified\n");
1050         }
1051     }
1052 }
1053
1054 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1055 {
1056     RAMBlock *new_block, *block;
1057
1058     new_block = NULL;
1059     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1060         if (block->offset == addr) {
1061             new_block = block;
1062             break;
1063         }
1064     }
1065     assert(new_block);
1066     assert(!new_block->idstr[0]);
1067
1068     if (dev) {
1069         char *id = qdev_get_dev_path(dev);
1070         if (id) {
1071             snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1072             g_free(id);
1073         }
1074     }
1075     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1076
1077     /* This assumes the iothread lock is taken here too.  */
1078     qemu_mutex_lock_ramlist();
1079     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1080         if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1081             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1082                     new_block->idstr);
1083             abort();
1084         }
1085     }
1086     qemu_mutex_unlock_ramlist();
1087 }
1088
1089 static int memory_try_enable_merging(void *addr, size_t len)
1090 {
1091     if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1092         /* disabled by the user */
1093         return 0;
1094     }
1095
1096     return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1097 }
1098
1099 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1100                                    MemoryRegion *mr)
1101 {
1102     RAMBlock *block, *new_block;
1103
1104     size = TARGET_PAGE_ALIGN(size);
1105     new_block = g_malloc0(sizeof(*new_block));
1106     new_block->fd = -1;
1107
1108     /* This assumes the iothread lock is taken here too.  */
1109     qemu_mutex_lock_ramlist();
1110     new_block->mr = mr;
1111     new_block->offset = find_ram_offset(size);
1112     if (host) {
1113         new_block->host = host;
1114         new_block->flags |= RAM_PREALLOC_MASK;
1115     } else if (xen_enabled()) {
1116         if (mem_path) {
1117             fprintf(stderr, "-mem-path not supported with Xen\n");
1118             exit(1);
1119         }
1120         xen_ram_alloc(new_block->offset, size, mr);
1121     } else {
1122         if (mem_path) {
1123             if (phys_mem_alloc != qemu_anon_ram_alloc) {
1124                 /*
1125                  * file_ram_alloc() needs to allocate just like
1126                  * phys_mem_alloc, but we haven't bothered to provide
1127                  * a hook there.
1128                  */
1129                 fprintf(stderr,
1130                         "-mem-path not supported with this accelerator\n");
1131                 exit(1);
1132             }
1133             new_block->host = file_ram_alloc(new_block, size, mem_path);
1134         }
1135         if (!new_block->host) {
1136             new_block->host = phys_mem_alloc(size);
1137             if (!new_block->host) {
1138                 fprintf(stderr, "Cannot set up guest memory '%s': %s\n",
1139                         new_block->mr->name, strerror(errno));
1140                 exit(1);
1141             }
1142             memory_try_enable_merging(new_block->host, size);
1143         }
1144     }
1145     new_block->length = size;
1146
1147     /* Keep the list sorted from biggest to smallest block.  */
1148     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1149         if (block->length < new_block->length) {
1150             break;
1151         }
1152     }
1153     if (block) {
1154         QTAILQ_INSERT_BEFORE(block, new_block, next);
1155     } else {
1156         QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1157     }
1158     ram_list.mru_block = NULL;
1159
1160     ram_list.version++;
1161     qemu_mutex_unlock_ramlist();
1162
1163     ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1164                                        last_ram_offset() >> TARGET_PAGE_BITS);
1165     memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1166            0, size >> TARGET_PAGE_BITS);
1167     cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1168
1169     qemu_ram_setup_dump(new_block->host, size);
1170     qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1171     qemu_madvise(new_block->host, size, QEMU_MADV_DONTFORK);
1172
1173     if (kvm_enabled())
1174         kvm_setup_guest_memory(new_block->host, size);
1175
1176     return new_block->offset;
1177 }
1178
1179 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1180 {
1181     return qemu_ram_alloc_from_ptr(size, NULL, mr);
1182 }
1183
1184 void qemu_ram_free_from_ptr(ram_addr_t addr)
1185 {
1186     RAMBlock *block;
1187
1188     /* This assumes the iothread lock is taken here too.  */
1189     qemu_mutex_lock_ramlist();
1190     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1191         if (addr == block->offset) {
1192             QTAILQ_REMOVE(&ram_list.blocks, block, next);
1193             ram_list.mru_block = NULL;
1194             ram_list.version++;
1195             g_free(block);
1196             break;
1197         }
1198     }
1199     qemu_mutex_unlock_ramlist();
1200 }
1201
1202 void qemu_ram_free(ram_addr_t addr)
1203 {
1204     RAMBlock *block;
1205
1206     /* This assumes the iothread lock is taken here too.  */
1207     qemu_mutex_lock_ramlist();
1208     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1209         if (addr == block->offset) {
1210             QTAILQ_REMOVE(&ram_list.blocks, block, next);
1211             ram_list.mru_block = NULL;
1212             ram_list.version++;
1213             if (block->flags & RAM_PREALLOC_MASK) {
1214                 ;
1215             } else if (xen_enabled()) {
1216                 xen_invalidate_map_cache_entry(block->host);
1217 #ifndef _WIN32
1218             } else if (block->fd >= 0) {
1219                 munmap(block->host, block->length);
1220                 close(block->fd);
1221 #endif
1222             } else {
1223                 qemu_anon_ram_free(block->host, block->length);
1224             }
1225             g_free(block);
1226             break;
1227         }
1228     }
1229     qemu_mutex_unlock_ramlist();
1230
1231 }
1232
1233 #ifndef _WIN32
1234 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1235 {
1236     RAMBlock *block;
1237     ram_addr_t offset;
1238     int flags;
1239     void *area, *vaddr;
1240
1241     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1242         offset = addr - block->offset;
1243         if (offset < block->length) {
1244             vaddr = block->host + offset;
1245             if (block->flags & RAM_PREALLOC_MASK) {
1246                 ;
1247             } else if (xen_enabled()) {
1248                 abort();
1249             } else {
1250                 flags = MAP_FIXED;
1251                 munmap(vaddr, length);
1252                 if (block->fd >= 0) {
1253 #ifdef MAP_POPULATE
1254                     flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1255                         MAP_PRIVATE;
1256 #else
1257                     flags |= MAP_PRIVATE;
1258 #endif
1259                     area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1260                                 flags, block->fd, offset);
1261                 } else {
1262                     /*
1263                      * Remap needs to match alloc.  Accelerators that
1264                      * set phys_mem_alloc never remap.  If they did,
1265                      * we'd need a remap hook here.
1266                      */
1267                     assert(phys_mem_alloc == qemu_anon_ram_alloc);
1268
1269                     flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1270                     area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1271                                 flags, -1, 0);
1272                 }
1273                 if (area != vaddr) {
1274                     fprintf(stderr, "Could not remap addr: "
1275                             RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1276                             length, addr);
1277                     exit(1);
1278                 }
1279                 memory_try_enable_merging(vaddr, length);
1280                 qemu_ram_setup_dump(vaddr, length);
1281             }
1282             return;
1283         }
1284     }
1285 }
1286 #endif /* !_WIN32 */
1287
1288 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1289    With the exception of the softmmu code in this file, this should
1290    only be used for local memory (e.g. video ram) that the device owns,
1291    and knows it isn't going to access beyond the end of the block.
1292
1293    It should not be used for general purpose DMA.
1294    Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1295  */
1296 void *qemu_get_ram_ptr(ram_addr_t addr)
1297 {
1298     RAMBlock *block = qemu_get_ram_block(addr);
1299
1300     if (xen_enabled()) {
1301         /* We need to check if the requested address is in the RAM
1302          * because we don't want to map the entire memory in QEMU.
1303          * In that case just map until the end of the page.
1304          */
1305         if (block->offset == 0) {
1306             return xen_map_cache(addr, 0, 0);
1307         } else if (block->host == NULL) {
1308             block->host =
1309                 xen_map_cache(block->offset, block->length, 1);
1310         }
1311     }
1312     return block->host + (addr - block->offset);
1313 }
1314
1315 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1316  * but takes a size argument */
1317 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1318 {
1319     if (*size == 0) {
1320         return NULL;
1321     }
1322     if (xen_enabled()) {
1323         return xen_map_cache(addr, *size, 1);
1324     } else {
1325         RAMBlock *block;
1326
1327         QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1328             if (addr - block->offset < block->length) {
1329                 if (addr - block->offset + *size > block->length)
1330                     *size = block->length - addr + block->offset;
1331                 return block->host + (addr - block->offset);
1332             }
1333         }
1334
1335         fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1336         abort();
1337     }
1338 }
1339
1340 /* Some of the softmmu routines need to translate from a host pointer
1341    (typically a TLB entry) back to a ram offset.  */
1342 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1343 {
1344     RAMBlock *block;
1345     uint8_t *host = ptr;
1346
1347     if (xen_enabled()) {
1348         *ram_addr = xen_ram_addr_from_mapcache(ptr);
1349         return qemu_get_ram_block(*ram_addr)->mr;
1350     }
1351
1352     block = ram_list.mru_block;
1353     if (block && block->host && host - block->host < block->length) {
1354         goto found;
1355     }
1356
1357     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1358         /* This case append when the block is not mapped. */
1359         if (block->host == NULL) {
1360             continue;
1361         }
1362         if (host - block->host < block->length) {
1363             goto found;
1364         }
1365     }
1366
1367     return NULL;
1368
1369 found:
1370     *ram_addr = block->offset + (host - block->host);
1371     return block->mr;
1372 }
1373
1374 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1375                                uint64_t val, unsigned size)
1376 {
1377     int dirty_flags;
1378     dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1379     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1380         tb_invalidate_phys_page_fast(ram_addr, size);
1381         dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1382     }
1383     switch (size) {
1384     case 1:
1385         stb_p(qemu_get_ram_ptr(ram_addr), val);
1386         break;
1387     case 2:
1388         stw_p(qemu_get_ram_ptr(ram_addr), val);
1389         break;
1390     case 4:
1391         stl_p(qemu_get_ram_ptr(ram_addr), val);
1392         break;
1393     default:
1394         abort();
1395     }
1396     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1397     cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1398     /* we remove the notdirty callback only if the code has been
1399        flushed */
1400     if (dirty_flags == 0xff) {
1401         CPUArchState *env = current_cpu->env_ptr;
1402         tlb_set_dirty(env, env->mem_io_vaddr);
1403     }
1404 }
1405
1406 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1407                                  unsigned size, bool is_write)
1408 {
1409     return is_write;
1410 }
1411
1412 static const MemoryRegionOps notdirty_mem_ops = {
1413     .write = notdirty_mem_write,
1414     .valid.accepts = notdirty_mem_accepts,
1415     .endianness = DEVICE_NATIVE_ENDIAN,
1416 };
1417
1418 /* Generate a debug exception if a watchpoint has been hit.  */
1419 static void check_watchpoint(int offset, int len_mask, int flags)
1420 {
1421     CPUArchState *env = current_cpu->env_ptr;
1422     target_ulong pc, cs_base;
1423     target_ulong vaddr;
1424     CPUWatchpoint *wp;
1425     int cpu_flags;
1426
1427     if (env->watchpoint_hit) {
1428         /* We re-entered the check after replacing the TB. Now raise
1429          * the debug interrupt so that is will trigger after the
1430          * current instruction. */
1431         cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1432         return;
1433     }
1434     vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1435     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1436         if ((vaddr == (wp->vaddr & len_mask) ||
1437              (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1438             wp->flags |= BP_WATCHPOINT_HIT;
1439             if (!env->watchpoint_hit) {
1440                 env->watchpoint_hit = wp;
1441                 tb_check_watchpoint(env);
1442                 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1443                     env->exception_index = EXCP_DEBUG;
1444                     cpu_loop_exit(env);
1445                 } else {
1446                     cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1447                     tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1448                     cpu_resume_from_signal(env, NULL);
1449                 }
1450             }
1451         } else {
1452             wp->flags &= ~BP_WATCHPOINT_HIT;
1453         }
1454     }
1455 }
1456
1457 /* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
1458    so these check for a hit then pass through to the normal out-of-line
1459    phys routines.  */
1460 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1461                                unsigned size)
1462 {
1463     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1464     switch (size) {
1465     case 1: return ldub_phys(addr);
1466     case 2: return lduw_phys(addr);
1467     case 4: return ldl_phys(addr);
1468     default: abort();
1469     }
1470 }
1471
1472 static void watch_mem_write(void *opaque, hwaddr addr,
1473                             uint64_t val, unsigned size)
1474 {
1475     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1476     switch (size) {
1477     case 1:
1478         stb_phys(addr, val);
1479         break;
1480     case 2:
1481         stw_phys(addr, val);
1482         break;
1483     case 4:
1484         stl_phys(addr, val);
1485         break;
1486     default: abort();
1487     }
1488 }
1489
1490 static const MemoryRegionOps watch_mem_ops = {
1491     .read = watch_mem_read,
1492     .write = watch_mem_write,
1493     .endianness = DEVICE_NATIVE_ENDIAN,
1494 };
1495
1496 static uint64_t subpage_read(void *opaque, hwaddr addr,
1497                              unsigned len)
1498 {
1499     subpage_t *subpage = opaque;
1500     uint8_t buf[4];
1501
1502 #if defined(DEBUG_SUBPAGE)
1503     printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
1504            subpage, len, addr);
1505 #endif
1506     address_space_read(subpage->as, addr + subpage->base, buf, len);
1507     switch (len) {
1508     case 1:
1509         return ldub_p(buf);
1510     case 2:
1511         return lduw_p(buf);
1512     case 4:
1513         return ldl_p(buf);
1514     default:
1515         abort();
1516     }
1517 }
1518
1519 static void subpage_write(void *opaque, hwaddr addr,
1520                           uint64_t value, unsigned len)
1521 {
1522     subpage_t *subpage = opaque;
1523     uint8_t buf[4];
1524
1525 #if defined(DEBUG_SUBPAGE)
1526     printf("%s: subpage %p len %u addr " TARGET_FMT_plx
1527            " value %"PRIx64"\n",
1528            __func__, subpage, len, addr, value);
1529 #endif
1530     switch (len) {
1531     case 1:
1532         stb_p(buf, value);
1533         break;
1534     case 2:
1535         stw_p(buf, value);
1536         break;
1537     case 4:
1538         stl_p(buf, value);
1539         break;
1540     default:
1541         abort();
1542     }
1543     address_space_write(subpage->as, addr + subpage->base, buf, len);
1544 }
1545
1546 static bool subpage_accepts(void *opaque, hwaddr addr,
1547                             unsigned len, bool is_write)
1548 {
1549     subpage_t *subpage = opaque;
1550 #if defined(DEBUG_SUBPAGE)
1551     printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
1552            __func__, subpage, is_write ? 'w' : 'r', len, addr);
1553 #endif
1554
1555     return address_space_access_valid(subpage->as, addr + subpage->base,
1556                                       len, is_write);
1557 }
1558
1559 static const MemoryRegionOps subpage_ops = {
1560     .read = subpage_read,
1561     .write = subpage_write,
1562     .valid.accepts = subpage_accepts,
1563     .endianness = DEVICE_NATIVE_ENDIAN,
1564 };
1565
1566 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1567                              uint16_t section)
1568 {
1569     int idx, eidx;
1570
1571     if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1572         return -1;
1573     idx = SUBPAGE_IDX(start);
1574     eidx = SUBPAGE_IDX(end);
1575 #if defined(DEBUG_SUBPAGE)
1576     printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
1577            __func__, mmio, start, end, idx, eidx, section);
1578 #endif
1579     for (; idx <= eidx; idx++) {
1580         mmio->sub_section[idx] = section;
1581     }
1582
1583     return 0;
1584 }
1585
1586 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1587 {
1588     subpage_t *mmio;
1589
1590     mmio = g_malloc0(sizeof(subpage_t));
1591
1592     mmio->as = as;
1593     mmio->base = base;
1594     memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1595                           "subpage", TARGET_PAGE_SIZE);
1596     mmio->iomem.subpage = true;
1597 #if defined(DEBUG_SUBPAGE)
1598     printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
1599            mmio, base, TARGET_PAGE_SIZE);
1600 #endif
1601     subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1602
1603     return mmio;
1604 }
1605
1606 static uint16_t dummy_section(MemoryRegion *mr)
1607 {
1608     MemoryRegionSection section = {
1609         .mr = mr,
1610         .offset_within_address_space = 0,
1611         .offset_within_region = 0,
1612         .size = int128_2_64(),
1613     };
1614
1615     return phys_section_add(&section);
1616 }
1617
1618 MemoryRegion *iotlb_to_region(hwaddr index)
1619 {
1620     return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1621 }
1622
1623 static void io_mem_init(void)
1624 {
1625     memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1626     memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1627                           "unassigned", UINT64_MAX);
1628     memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1629                           "notdirty", UINT64_MAX);
1630     memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1631                           "watch", UINT64_MAX);
1632 }
1633
1634 static void mem_begin(MemoryListener *listener)
1635 {
1636     AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1637     AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1638
1639     d->phys_map  = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1640     d->as = as;
1641     as->next_dispatch = d;
1642 }
1643
1644 static void mem_commit(MemoryListener *listener)
1645 {
1646     AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1647     AddressSpaceDispatch *cur = as->dispatch;
1648     AddressSpaceDispatch *next = as->next_dispatch;
1649
1650     next->nodes = next_map.nodes;
1651     next->sections = next_map.sections;
1652
1653     as->dispatch = next;
1654     g_free(cur);
1655 }
1656
1657 static void core_begin(MemoryListener *listener)
1658 {
1659     uint16_t n;
1660
1661     prev_map = g_new(PhysPageMap, 1);
1662     *prev_map = next_map;
1663
1664     memset(&next_map, 0, sizeof(next_map));
1665     n = dummy_section(&io_mem_unassigned);
1666     assert(n == PHYS_SECTION_UNASSIGNED);
1667     n = dummy_section(&io_mem_notdirty);
1668     assert(n == PHYS_SECTION_NOTDIRTY);
1669     n = dummy_section(&io_mem_rom);
1670     assert(n == PHYS_SECTION_ROM);
1671     n = dummy_section(&io_mem_watch);
1672     assert(n == PHYS_SECTION_WATCH);
1673 }
1674
1675 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1676  * All AddressSpaceDispatch instances have switched to the next map.
1677  */
1678 static void core_commit(MemoryListener *listener)
1679 {
1680     phys_sections_free(prev_map);
1681 }
1682
1683 static void tcg_commit(MemoryListener *listener)
1684 {
1685     CPUState *cpu;
1686
1687     /* since each CPU stores ram addresses in its TLB cache, we must
1688        reset the modified entries */
1689     /* XXX: slow ! */
1690     CPU_FOREACH(cpu) {
1691         CPUArchState *env = cpu->env_ptr;
1692
1693         tlb_flush(env, 1);
1694     }
1695 }
1696
1697 static void core_log_global_start(MemoryListener *listener)
1698 {
1699     cpu_physical_memory_set_dirty_tracking(1);
1700 }
1701
1702 static void core_log_global_stop(MemoryListener *listener)
1703 {
1704     cpu_physical_memory_set_dirty_tracking(0);
1705 }
1706
1707 static MemoryListener core_memory_listener = {
1708     .begin = core_begin,
1709     .commit = core_commit,
1710     .log_global_start = core_log_global_start,
1711     .log_global_stop = core_log_global_stop,
1712     .priority = 1,
1713 };
1714
1715 static MemoryListener tcg_memory_listener = {
1716     .commit = tcg_commit,
1717 };
1718
1719 void address_space_init_dispatch(AddressSpace *as)
1720 {
1721     as->dispatch = NULL;
1722     as->dispatch_listener = (MemoryListener) {
1723         .begin = mem_begin,
1724         .commit = mem_commit,
1725         .region_add = mem_add,
1726         .region_nop = mem_add,
1727         .priority = 0,
1728     };
1729     memory_listener_register(&as->dispatch_listener, as);
1730 }
1731
1732 void address_space_destroy_dispatch(AddressSpace *as)
1733 {
1734     AddressSpaceDispatch *d = as->dispatch;
1735
1736     memory_listener_unregister(&as->dispatch_listener);
1737     g_free(d);
1738     as->dispatch = NULL;
1739 }
1740
1741 static void memory_map_init(void)
1742 {
1743     system_memory = g_malloc(sizeof(*system_memory));
1744     memory_region_init(system_memory, NULL, "system", INT64_MAX);
1745     address_space_init(&address_space_memory, system_memory, "memory");
1746
1747     system_io = g_malloc(sizeof(*system_io));
1748     memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
1749                           65536);
1750     address_space_init(&address_space_io, system_io, "I/O");
1751
1752     memory_listener_register(&core_memory_listener, &address_space_memory);
1753     if (tcg_enabled()) {
1754         memory_listener_register(&tcg_memory_listener, &address_space_memory);
1755     }
1756 }
1757
1758 MemoryRegion *get_system_memory(void)
1759 {
1760     return system_memory;
1761 }
1762
1763 MemoryRegion *get_system_io(void)
1764 {
1765     return system_io;
1766 }
1767
1768 #endif /* !defined(CONFIG_USER_ONLY) */
1769
1770 /* physical memory access (slow version, mainly for debug) */
1771 #if defined(CONFIG_USER_ONLY)
1772 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1773                         uint8_t *buf, int len, int is_write)
1774 {
1775     int l, flags;
1776     target_ulong page;
1777     void * p;
1778
1779     while (len > 0) {
1780         page = addr & TARGET_PAGE_MASK;
1781         l = (page + TARGET_PAGE_SIZE) - addr;
1782         if (l > len)
1783             l = len;
1784         flags = page_get_flags(page);
1785         if (!(flags & PAGE_VALID))
1786             return -1;
1787         if (is_write) {
1788             if (!(flags & PAGE_WRITE))
1789                 return -1;
1790             /* XXX: this code should not depend on lock_user */
1791             if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1792                 return -1;
1793             memcpy(p, buf, l);
1794             unlock_user(p, addr, l);
1795         } else {
1796             if (!(flags & PAGE_READ))
1797                 return -1;
1798             /* XXX: this code should not depend on lock_user */
1799             if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1800                 return -1;
1801             memcpy(buf, p, l);
1802             unlock_user(p, addr, 0);
1803         }
1804         len -= l;
1805         buf += l;
1806         addr += l;
1807     }
1808     return 0;
1809 }
1810
1811 #else
1812
1813 static void invalidate_and_set_dirty(hwaddr addr,
1814                                      hwaddr length)
1815 {
1816     if (!cpu_physical_memory_is_dirty(addr)) {
1817         /* invalidate code */
1818         tb_invalidate_phys_page_range(addr, addr + length, 0);
1819         /* set dirty bit */
1820         cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1821     }
1822     xen_modified_memory(addr, length);
1823 }
1824
1825 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1826 {
1827     if (memory_region_is_ram(mr)) {
1828         return !(is_write && mr->readonly);
1829     }
1830     if (memory_region_is_romd(mr)) {
1831         return !is_write;
1832     }
1833
1834     return false;
1835 }
1836
1837 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1838 {
1839     unsigned access_size_max = mr->ops->valid.max_access_size;
1840
1841     /* Regions are assumed to support 1-4 byte accesses unless
1842        otherwise specified.  */
1843     if (access_size_max == 0) {
1844         access_size_max = 4;
1845     }
1846
1847     /* Bound the maximum access by the alignment of the address.  */
1848     if (!mr->ops->impl.unaligned) {
1849         unsigned align_size_max = addr & -addr;
1850         if (align_size_max != 0 && align_size_max < access_size_max) {
1851             access_size_max = align_size_max;
1852         }
1853     }
1854
1855     /* Don't attempt accesses larger than the maximum.  */
1856     if (l > access_size_max) {
1857         l = access_size_max;
1858     }
1859     if (l & (l - 1)) {
1860         l = 1 << (qemu_fls(l) - 1);
1861     }
1862
1863     return l;
1864 }
1865
1866 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1867                       int len, bool is_write)
1868 {
1869     hwaddr l;
1870     uint8_t *ptr;
1871     uint64_t val;
1872     hwaddr addr1;
1873     MemoryRegion *mr;
1874     bool error = false;
1875
1876     while (len > 0) {
1877         l = len;
1878         mr = address_space_translate(as, addr, &addr1, &l, is_write);
1879
1880         if (is_write) {
1881             if (!memory_access_is_direct(mr, is_write)) {
1882                 l = memory_access_size(mr, l, addr1);
1883                 /* XXX: could force current_cpu to NULL to avoid
1884                    potential bugs */
1885                 switch (l) {
1886                 case 8:
1887                     /* 64 bit write access */
1888                     val = ldq_p(buf);
1889                     error |= io_mem_write(mr, addr1, val, 8);
1890                     break;
1891                 case 4:
1892                     /* 32 bit write access */
1893                     val = ldl_p(buf);
1894                     error |= io_mem_write(mr, addr1, val, 4);
1895                     break;
1896                 case 2:
1897                     /* 16 bit write access */
1898                     val = lduw_p(buf);
1899                     error |= io_mem_write(mr, addr1, val, 2);
1900                     break;
1901                 case 1:
1902                     /* 8 bit write access */
1903                     val = ldub_p(buf);
1904                     error |= io_mem_write(mr, addr1, val, 1);
1905                     break;
1906                 default:
1907                     abort();
1908                 }
1909             } else {
1910                 addr1 += memory_region_get_ram_addr(mr);
1911                 /* RAM case */
1912                 ptr = qemu_get_ram_ptr(addr1);
1913                 memcpy(ptr, buf, l);
1914                 invalidate_and_set_dirty(addr1, l);
1915             }
1916         } else {
1917             if (!memory_access_is_direct(mr, is_write)) {
1918                 /* I/O case */
1919                 l = memory_access_size(mr, l, addr1);
1920                 switch (l) {
1921                 case 8:
1922                     /* 64 bit read access */
1923                     error |= io_mem_read(mr, addr1, &val, 8);
1924                     stq_p(buf, val);
1925                     break;
1926                 case 4:
1927                     /* 32 bit read access */
1928                     error |= io_mem_read(mr, addr1, &val, 4);
1929                     stl_p(buf, val);
1930                     break;
1931                 case 2:
1932                     /* 16 bit read access */
1933                     error |= io_mem_read(mr, addr1, &val, 2);
1934                     stw_p(buf, val);
1935                     break;
1936                 case 1:
1937                     /* 8 bit read access */
1938                     error |= io_mem_read(mr, addr1, &val, 1);
1939                     stb_p(buf, val);
1940                     break;
1941                 default:
1942                     abort();
1943                 }
1944             } else {
1945                 /* RAM case */
1946                 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1947                 memcpy(buf, ptr, l);
1948             }
1949         }
1950         len -= l;
1951         buf += l;
1952         addr += l;
1953     }
1954
1955     return error;
1956 }
1957
1958 bool address_space_write(AddressSpace *as, hwaddr addr,
1959                          const uint8_t *buf, int len)
1960 {
1961     return address_space_rw(as, addr, (uint8_t *)buf, len, true);
1962 }
1963
1964 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1965 {
1966     return address_space_rw(as, addr, buf, len, false);
1967 }
1968
1969
1970 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1971                             int len, int is_write)
1972 {
1973     address_space_rw(&address_space_memory, addr, buf, len, is_write);
1974 }
1975
1976 /* used for ROM loading : can write in RAM and ROM */
1977 void cpu_physical_memory_write_rom(hwaddr addr,
1978                                    const uint8_t *buf, int len)
1979 {
1980     hwaddr l;
1981     uint8_t *ptr;
1982     hwaddr addr1;
1983     MemoryRegion *mr;
1984
1985     while (len > 0) {
1986         l = len;
1987         mr = address_space_translate(&address_space_memory,
1988                                      addr, &addr1, &l, true);
1989
1990         if (!(memory_region_is_ram(mr) ||
1991               memory_region_is_romd(mr))) {
1992             /* do nothing */
1993         } else {
1994             addr1 += memory_region_get_ram_addr(mr);
1995             /* ROM/RAM case */
1996             ptr = qemu_get_ram_ptr(addr1);
1997             memcpy(ptr, buf, l);
1998             invalidate_and_set_dirty(addr1, l);
1999         }
2000         len -= l;
2001         buf += l;
2002         addr += l;
2003     }
2004 }
2005
2006 typedef struct {
2007     MemoryRegion *mr;
2008     void *buffer;
2009     hwaddr addr;
2010     hwaddr len;
2011 } BounceBuffer;
2012
2013 static BounceBuffer bounce;
2014
2015 typedef struct MapClient {
2016     void *opaque;
2017     void (*callback)(void *opaque);
2018     QLIST_ENTRY(MapClient) link;
2019 } MapClient;
2020
2021 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2022     = QLIST_HEAD_INITIALIZER(map_client_list);
2023
2024 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2025 {
2026     MapClient *client = g_malloc(sizeof(*client));
2027
2028     client->opaque = opaque;
2029     client->callback = callback;
2030     QLIST_INSERT_HEAD(&map_client_list, client, link);
2031     return client;
2032 }
2033
2034 static void cpu_unregister_map_client(void *_client)
2035 {
2036     MapClient *client = (MapClient *)_client;
2037
2038     QLIST_REMOVE(client, link);
2039     g_free(client);
2040 }
2041
2042 static void cpu_notify_map_clients(void)
2043 {
2044     MapClient *client;
2045
2046     while (!QLIST_EMPTY(&map_client_list)) {
2047         client = QLIST_FIRST(&map_client_list);
2048         client->callback(client->opaque);
2049         cpu_unregister_map_client(client);
2050     }
2051 }
2052
2053 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2054 {
2055     MemoryRegion *mr;
2056     hwaddr l, xlat;
2057
2058     while (len > 0) {
2059         l = len;
2060         mr = address_space_translate(as, addr, &xlat, &l, is_write);
2061         if (!memory_access_is_direct(mr, is_write)) {
2062             l = memory_access_size(mr, l, addr);
2063             if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2064                 return false;
2065             }
2066         }
2067
2068         len -= l;
2069         addr += l;
2070     }
2071     return true;
2072 }
2073
2074 /* Map a physical memory region into a host virtual address.
2075  * May map a subset of the requested range, given by and returned in *plen.
2076  * May return NULL if resources needed to perform the mapping are exhausted.
2077  * Use only for reads OR writes - not for read-modify-write operations.
2078  * Use cpu_register_map_client() to know when retrying the map operation is
2079  * likely to succeed.
2080  */
2081 void *address_space_map(AddressSpace *as,
2082                         hwaddr addr,
2083                         hwaddr *plen,
2084                         bool is_write)
2085 {
2086     hwaddr len = *plen;
2087     hwaddr done = 0;
2088     hwaddr l, xlat, base;
2089     MemoryRegion *mr, *this_mr;
2090     ram_addr_t raddr;
2091
2092     if (len == 0) {
2093         return NULL;
2094     }
2095
2096     l = len;
2097     mr = address_space_translate(as, addr, &xlat, &l, is_write);
2098     if (!memory_access_is_direct(mr, is_write)) {
2099         if (bounce.buffer) {
2100             return NULL;
2101         }
2102         bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2103         bounce.addr = addr;
2104         bounce.len = l;
2105
2106         memory_region_ref(mr);
2107         bounce.mr = mr;
2108         if (!is_write) {
2109             address_space_read(as, addr, bounce.buffer, l);
2110         }
2111
2112         *plen = l;
2113         return bounce.buffer;
2114     }
2115
2116     base = xlat;
2117     raddr = memory_region_get_ram_addr(mr);
2118
2119     for (;;) {
2120         len -= l;
2121         addr += l;
2122         done += l;
2123         if (len == 0) {
2124             break;
2125         }
2126
2127         l = len;
2128         this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2129         if (this_mr != mr || xlat != base + done) {
2130             break;
2131         }
2132     }
2133
2134     memory_region_ref(mr);
2135     *plen = done;
2136     return qemu_ram_ptr_length(raddr + base, plen);
2137 }
2138
2139 /* Unmaps a memory region previously mapped by address_space_map().
2140  * Will also mark the memory as dirty if is_write == 1.  access_len gives
2141  * the amount of memory that was actually read or written by the caller.
2142  */
2143 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2144                          int is_write, hwaddr access_len)
2145 {
2146     if (buffer != bounce.buffer) {
2147         MemoryRegion *mr;
2148         ram_addr_t addr1;
2149
2150         mr = qemu_ram_addr_from_host(buffer, &addr1);
2151         assert(mr != NULL);
2152         if (is_write) {
2153             while (access_len) {
2154                 unsigned l;
2155                 l = TARGET_PAGE_SIZE;
2156                 if (l > access_len)
2157                     l = access_len;
2158                 invalidate_and_set_dirty(addr1, l);
2159                 addr1 += l;
2160                 access_len -= l;
2161             }
2162         }
2163         if (xen_enabled()) {
2164             xen_invalidate_map_cache_entry(buffer);
2165         }
2166         memory_region_unref(mr);
2167         return;
2168     }
2169     if (is_write) {
2170         address_space_write(as, bounce.addr, bounce.buffer, access_len);
2171     }
2172     qemu_vfree(bounce.buffer);
2173     bounce.buffer = NULL;
2174     memory_region_unref(bounce.mr);
2175     cpu_notify_map_clients();
2176 }
2177
2178 void *cpu_physical_memory_map(hwaddr addr,
2179                               hwaddr *plen,
2180                               int is_write)
2181 {
2182     return address_space_map(&address_space_memory, addr, plen, is_write);
2183 }
2184
2185 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2186                                int is_write, hwaddr access_len)
2187 {
2188     return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2189 }
2190
2191 /* warning: addr must be aligned */
2192 static inline uint32_t ldl_phys_internal(hwaddr addr,
2193                                          enum device_endian endian)
2194 {
2195     uint8_t *ptr;
2196     uint64_t val;
2197     MemoryRegion *mr;
2198     hwaddr l = 4;
2199     hwaddr addr1;
2200
2201     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2202                                  false);
2203     if (l < 4 || !memory_access_is_direct(mr, false)) {
2204         /* I/O case */
2205         io_mem_read(mr, addr1, &val, 4);
2206 #if defined(TARGET_WORDS_BIGENDIAN)
2207         if (endian == DEVICE_LITTLE_ENDIAN) {
2208             val = bswap32(val);
2209         }
2210 #else
2211         if (endian == DEVICE_BIG_ENDIAN) {
2212             val = bswap32(val);
2213         }
2214 #endif
2215     } else {
2216         /* RAM case */
2217         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2218                                 & TARGET_PAGE_MASK)
2219                                + addr1);
2220         switch (endian) {
2221         case DEVICE_LITTLE_ENDIAN:
2222             val = ldl_le_p(ptr);
2223             break;
2224         case DEVICE_BIG_ENDIAN:
2225             val = ldl_be_p(ptr);
2226             break;
2227         default:
2228             val = ldl_p(ptr);
2229             break;
2230         }
2231     }
2232     return val;
2233 }
2234
2235 uint32_t ldl_phys(hwaddr addr)
2236 {
2237     return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2238 }
2239
2240 uint32_t ldl_le_phys(hwaddr addr)
2241 {
2242     return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2243 }
2244
2245 uint32_t ldl_be_phys(hwaddr addr)
2246 {
2247     return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2248 }
2249
2250 /* warning: addr must be aligned */
2251 static inline uint64_t ldq_phys_internal(hwaddr addr,
2252                                          enum device_endian endian)
2253 {
2254     uint8_t *ptr;
2255     uint64_t val;
2256     MemoryRegion *mr;
2257     hwaddr l = 8;
2258     hwaddr addr1;
2259
2260     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2261                                  false);
2262     if (l < 8 || !memory_access_is_direct(mr, false)) {
2263         /* I/O case */
2264         io_mem_read(mr, addr1, &val, 8);
2265 #if defined(TARGET_WORDS_BIGENDIAN)
2266         if (endian == DEVICE_LITTLE_ENDIAN) {
2267             val = bswap64(val);
2268         }
2269 #else
2270         if (endian == DEVICE_BIG_ENDIAN) {
2271             val = bswap64(val);
2272         }
2273 #endif
2274     } else {
2275         /* RAM case */
2276         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2277                                 & TARGET_PAGE_MASK)
2278                                + addr1);
2279         switch (endian) {
2280         case DEVICE_LITTLE_ENDIAN:
2281             val = ldq_le_p(ptr);
2282             break;
2283         case DEVICE_BIG_ENDIAN:
2284             val = ldq_be_p(ptr);
2285             break;
2286         default:
2287             val = ldq_p(ptr);
2288             break;
2289         }
2290     }
2291     return val;
2292 }
2293
2294 uint64_t ldq_phys(hwaddr addr)
2295 {
2296     return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2297 }
2298
2299 uint64_t ldq_le_phys(hwaddr addr)
2300 {
2301     return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2302 }
2303
2304 uint64_t ldq_be_phys(hwaddr addr)
2305 {
2306     return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2307 }
2308
2309 /* XXX: optimize */
2310 uint32_t ldub_phys(hwaddr addr)
2311 {
2312     uint8_t val;
2313     cpu_physical_memory_read(addr, &val, 1);
2314     return val;
2315 }
2316
2317 /* warning: addr must be aligned */
2318 static inline uint32_t lduw_phys_internal(hwaddr addr,
2319                                           enum device_endian endian)
2320 {
2321     uint8_t *ptr;
2322     uint64_t val;
2323     MemoryRegion *mr;
2324     hwaddr l = 2;
2325     hwaddr addr1;
2326
2327     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2328                                  false);
2329     if (l < 2 || !memory_access_is_direct(mr, false)) {
2330         /* I/O case */
2331         io_mem_read(mr, addr1, &val, 2);
2332 #if defined(TARGET_WORDS_BIGENDIAN)
2333         if (endian == DEVICE_LITTLE_ENDIAN) {
2334             val = bswap16(val);
2335         }
2336 #else
2337         if (endian == DEVICE_BIG_ENDIAN) {
2338             val = bswap16(val);
2339         }
2340 #endif
2341     } else {
2342         /* RAM case */
2343         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2344                                 & TARGET_PAGE_MASK)
2345                                + addr1);
2346         switch (endian) {
2347         case DEVICE_LITTLE_ENDIAN:
2348             val = lduw_le_p(ptr);
2349             break;
2350         case DEVICE_BIG_ENDIAN:
2351             val = lduw_be_p(ptr);
2352             break;
2353         default:
2354             val = lduw_p(ptr);
2355             break;
2356         }
2357     }
2358     return val;
2359 }
2360
2361 uint32_t lduw_phys(hwaddr addr)
2362 {
2363     return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2364 }
2365
2366 uint32_t lduw_le_phys(hwaddr addr)
2367 {
2368     return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2369 }
2370
2371 uint32_t lduw_be_phys(hwaddr addr)
2372 {
2373     return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2374 }
2375
2376 /* warning: addr must be aligned. The ram page is not masked as dirty
2377    and the code inside is not invalidated. It is useful if the dirty
2378    bits are used to track modified PTEs */
2379 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2380 {
2381     uint8_t *ptr;
2382     MemoryRegion *mr;
2383     hwaddr l = 4;
2384     hwaddr addr1;
2385
2386     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2387                                  true);
2388     if (l < 4 || !memory_access_is_direct(mr, true)) {
2389         io_mem_write(mr, addr1, val, 4);
2390     } else {
2391         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2392         ptr = qemu_get_ram_ptr(addr1);
2393         stl_p(ptr, val);
2394
2395         if (unlikely(in_migration)) {
2396             if (!cpu_physical_memory_is_dirty(addr1)) {
2397                 /* invalidate code */
2398                 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2399                 /* set dirty bit */
2400                 cpu_physical_memory_set_dirty_flags(
2401                     addr1, (0xff & ~CODE_DIRTY_FLAG));
2402             }
2403         }
2404     }
2405 }
2406
2407 /* warning: addr must be aligned */
2408 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2409                                      enum device_endian endian)
2410 {
2411     uint8_t *ptr;
2412     MemoryRegion *mr;
2413     hwaddr l = 4;
2414     hwaddr addr1;
2415
2416     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2417                                  true);
2418     if (l < 4 || !memory_access_is_direct(mr, true)) {
2419 #if defined(TARGET_WORDS_BIGENDIAN)
2420         if (endian == DEVICE_LITTLE_ENDIAN) {
2421             val = bswap32(val);
2422         }
2423 #else
2424         if (endian == DEVICE_BIG_ENDIAN) {
2425             val = bswap32(val);
2426         }
2427 #endif
2428         io_mem_write(mr, addr1, val, 4);
2429     } else {
2430         /* RAM case */
2431         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2432         ptr = qemu_get_ram_ptr(addr1);
2433         switch (endian) {
2434         case DEVICE_LITTLE_ENDIAN:
2435             stl_le_p(ptr, val);
2436             break;
2437         case DEVICE_BIG_ENDIAN:
2438             stl_be_p(ptr, val);
2439             break;
2440         default:
2441             stl_p(ptr, val);
2442             break;
2443         }
2444         invalidate_and_set_dirty(addr1, 4);
2445     }
2446 }
2447
2448 void stl_phys(hwaddr addr, uint32_t val)
2449 {
2450     stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2451 }
2452
2453 void stl_le_phys(hwaddr addr, uint32_t val)
2454 {
2455     stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2456 }
2457
2458 void stl_be_phys(hwaddr addr, uint32_t val)
2459 {
2460     stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2461 }
2462
2463 /* XXX: optimize */
2464 void stb_phys(hwaddr addr, uint32_t val)
2465 {
2466     uint8_t v = val;
2467     cpu_physical_memory_write(addr, &v, 1);
2468 }
2469
2470 /* warning: addr must be aligned */
2471 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2472                                      enum device_endian endian)
2473 {
2474     uint8_t *ptr;
2475     MemoryRegion *mr;
2476     hwaddr l = 2;
2477     hwaddr addr1;
2478
2479     mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2480                                  true);
2481     if (l < 2 || !memory_access_is_direct(mr, true)) {
2482 #if defined(TARGET_WORDS_BIGENDIAN)
2483         if (endian == DEVICE_LITTLE_ENDIAN) {
2484             val = bswap16(val);
2485         }
2486 #else
2487         if (endian == DEVICE_BIG_ENDIAN) {
2488             val = bswap16(val);
2489         }
2490 #endif
2491         io_mem_write(mr, addr1, val, 2);
2492     } else {
2493         /* RAM case */
2494         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2495         ptr = qemu_get_ram_ptr(addr1);
2496         switch (endian) {
2497         case DEVICE_LITTLE_ENDIAN:
2498             stw_le_p(ptr, val);
2499             break;
2500         case DEVICE_BIG_ENDIAN:
2501             stw_be_p(ptr, val);
2502             break;
2503         default:
2504             stw_p(ptr, val);
2505             break;
2506         }
2507         invalidate_and_set_dirty(addr1, 2);
2508     }
2509 }
2510
2511 void stw_phys(hwaddr addr, uint32_t val)
2512 {
2513     stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2514 }
2515
2516 void stw_le_phys(hwaddr addr, uint32_t val)
2517 {
2518     stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2519 }
2520
2521 void stw_be_phys(hwaddr addr, uint32_t val)
2522 {
2523     stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2524 }
2525
2526 /* XXX: optimize */
2527 void stq_phys(hwaddr addr, uint64_t val)
2528 {
2529     val = tswap64(val);
2530     cpu_physical_memory_write(addr, &val, 8);
2531 }
2532
2533 void stq_le_phys(hwaddr addr, uint64_t val)
2534 {
2535     val = cpu_to_le64(val);
2536     cpu_physical_memory_write(addr, &val, 8);
2537 }
2538
2539 void stq_be_phys(hwaddr addr, uint64_t val)
2540 {
2541     val = cpu_to_be64(val);
2542     cpu_physical_memory_write(addr, &val, 8);
2543 }
2544
2545 /* virtual memory access for debug (includes writing to ROM) */
2546 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2547                         uint8_t *buf, int len, int is_write)
2548 {
2549     int l;
2550     hwaddr phys_addr;
2551     target_ulong page;
2552
2553     while (len > 0) {
2554         page = addr & TARGET_PAGE_MASK;
2555         phys_addr = cpu_get_phys_page_debug(cpu, page);
2556         /* if no physical page mapped, return an error */
2557         if (phys_addr == -1)
2558             return -1;
2559         l = (page + TARGET_PAGE_SIZE) - addr;
2560         if (l > len)
2561             l = len;
2562         phys_addr += (addr & ~TARGET_PAGE_MASK);
2563         if (is_write)
2564             cpu_physical_memory_write_rom(phys_addr, buf, l);
2565         else
2566             cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2567         len -= l;
2568         buf += l;
2569         addr += l;
2570     }
2571     return 0;
2572 }
2573 #endif
2574
2575 #if !defined(CONFIG_USER_ONLY)
2576
2577 /*
2578  * A helper function for the _utterly broken_ virtio device model to find out if
2579  * it's running on a big endian machine. Don't do this at home kids!
2580  */
2581 bool virtio_is_big_endian(void);
2582 bool virtio_is_big_endian(void)
2583 {
2584 #if defined(TARGET_WORDS_BIGENDIAN)
2585     return true;
2586 #else
2587     return false;
2588 #endif
2589 }
2590
2591 #endif
2592
2593 #ifndef CONFIG_USER_ONLY
2594 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2595 {
2596     MemoryRegion*mr;
2597     hwaddr l = 1;
2598
2599     mr = address_space_translate(&address_space_memory,
2600                                  phys_addr, &phys_addr, &l, false);
2601
2602     return !(memory_region_is_ram(mr) ||
2603              memory_region_is_romd(mr));
2604 }
2605
2606 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2607 {
2608     RAMBlock *block;
2609
2610     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2611         func(block->host, block->offset, block->length, opaque);
2612     }
2613 }
2614 #endif