]> git.proxmox.com Git - qemu.git/blob - exec.c
projects / qemu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
exec: do tcg_commit only when tcg_enabled
[qemu.git] / exec.c
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
51
52 #include "exec/memory-internal.h"
53
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 typedef PhysPageEntry Node[L2_SIZE];
92
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101 };
102
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
110
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
115
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
124
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
127
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
133
134 static MemoryRegion io_mem_watch;
135 #endif
136
137 #if !defined(CONFIG_USER_ONLY)
138
139 static void phys_map_node_reserve(unsigned nodes)
140 {
141 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
142 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
143 16);
144 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
145 next_map.nodes_nb + nodes);
146 next_map.nodes = g_renew(Node, next_map.nodes,
147 next_map.nodes_nb_alloc);
148 }
149 }
150
151 static uint16_t phys_map_node_alloc(void)
152 {
153 unsigned i;
154 uint16_t ret;
155
156 ret = next_map.nodes_nb++;
157 assert(ret != PHYS_MAP_NODE_NIL);
158 assert(ret != next_map.nodes_nb_alloc);
159 for (i = 0; i < L2_SIZE; ++i) {
160 next_map.nodes[ret][i].is_leaf = 0;
161 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
162 }
163 return ret;
164 }
165
166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
167 hwaddr *nb, uint16_t leaf,
168 int level)
169 {
170 PhysPageEntry *p;
171 int i;
172 hwaddr step = (hwaddr)1 << (level * L2_BITS);
173
174 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
175 lp->ptr = phys_map_node_alloc();
176 p = next_map.nodes[lp->ptr];
177 if (level == 0) {
178 for (i = 0; i < L2_SIZE; i++) {
179 p[i].is_leaf = 1;
180 p[i].ptr = PHYS_SECTION_UNASSIGNED;
181 }
182 }
183 } else {
184 p = next_map.nodes[lp->ptr];
185 }
186 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
187
188 while (*nb && lp < &p[L2_SIZE]) {
189 if ((*index & (step - 1)) == 0 && *nb >= step) {
190 lp->is_leaf = true;
191 lp->ptr = leaf;
192 *index += step;
193 *nb -= step;
194 } else {
195 phys_page_set_level(lp, index, nb, leaf, level - 1);
196 }
197 ++lp;
198 }
199 }
200
201 static void phys_page_set(AddressSpaceDispatch *d,
202 hwaddr index, hwaddr nb,
203 uint16_t leaf)
204 {
205 /* Wildly overreserve - it doesn't matter much. */
206 phys_map_node_reserve(3 * P_L2_LEVELS);
207
208 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
209 }
210
211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
212 Node *nodes, MemoryRegionSection *sections)
213 {
214 PhysPageEntry *p;
215 int i;
216
217 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
218 if (lp.ptr == PHYS_MAP_NODE_NIL) {
219 return &sections[PHYS_SECTION_UNASSIGNED];
220 }
221 p = nodes[lp.ptr];
222 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
223 }
224 return &sections[lp.ptr];
225 }
226
227 bool memory_region_is_unassigned(MemoryRegion *mr)
228 {
229 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
230 && mr != &io_mem_watch;
231 }
232
233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
234 hwaddr addr,
235 bool resolve_subpage)
236 {
237 MemoryRegionSection *section;
238 subpage_t *subpage;
239
240 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
241 d->nodes, d->sections);
242 if (resolve_subpage && section->mr->subpage) {
243 subpage = container_of(section->mr, subpage_t, iomem);
244 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
245 }
246 return section;
247 }
248
249 static MemoryRegionSection *
250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
251 hwaddr *plen, bool resolve_subpage)
252 {
253 MemoryRegionSection *section;
254 Int128 diff;
255
256 section = address_space_lookup_region(d, addr, resolve_subpage);
257 /* Compute offset within MemoryRegionSection */
258 addr -= section->offset_within_address_space;
259
260 /* Compute offset within MemoryRegion */
261 *xlat = addr + section->offset_within_region;
262
263 diff = int128_sub(section->mr->size, int128_make64(addr));
264 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
265 return section;
266 }
267
268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
269 hwaddr *xlat, hwaddr *plen,
270 bool is_write)
271 {
272 IOMMUTLBEntry iotlb;
273 MemoryRegionSection *section;
274 MemoryRegion *mr;
275 hwaddr len = *plen;
276
277 for (;;) {
278 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
279 mr = section->mr;
280
281 if (!mr->iommu_ops) {
282 break;
283 }
284
285 iotlb = mr->iommu_ops->translate(mr, addr);
286 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
287 | (addr & iotlb.addr_mask));
288 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
289 if (!(iotlb.perm & (1 << is_write))) {
290 mr = &io_mem_unassigned;
291 break;
292 }
293
294 as = iotlb.target_as;
295 }
296
297 *plen = len;
298 *xlat = addr;
299 return mr;
300 }
301
302 MemoryRegionSection *
303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
304 hwaddr *plen)
305 {
306 MemoryRegionSection *section;
307 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
308
309 assert(!section->mr->iommu_ops);
310 return section;
311 }
312 #endif
313
314 void cpu_exec_init_all(void)
315 {
316 #if !defined(CONFIG_USER_ONLY)
317 qemu_mutex_init(&ram_list.mutex);
318 memory_map_init();
319 io_mem_init();
320 #endif
321 }
322
323 #if !defined(CONFIG_USER_ONLY)
324
325 static int cpu_common_post_load(void *opaque, int version_id)
326 {
327 CPUState *cpu = opaque;
328
329 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
330 version_id is increased. */
331 cpu->interrupt_request &= ~0x01;
332 tlb_flush(cpu->env_ptr, 1);
333
334 return 0;
335 }
336
337 const VMStateDescription vmstate_cpu_common = {
338 .name = "cpu_common",
339 .version_id = 1,
340 .minimum_version_id = 1,
341 .minimum_version_id_old = 1,
342 .post_load = cpu_common_post_load,
343 .fields = (VMStateField []) {
344 VMSTATE_UINT32(halted, CPUState),
345 VMSTATE_UINT32(interrupt_request, CPUState),
346 VMSTATE_END_OF_LIST()
347 }
348 };
349
350 #endif
351
352 CPUState *qemu_get_cpu(int index)
353 {
354 CPUState *cpu;
355
356 CPU_FOREACH(cpu) {
357 if (cpu->cpu_index == index) {
358 return cpu;
359 }
360 }
361
362 return NULL;
363 }
364
365 void cpu_exec_init(CPUArchState *env)
366 {
367 CPUState *cpu = ENV_GET_CPU(env);
368 CPUClass *cc = CPU_GET_CLASS(cpu);
369 CPUState *some_cpu;
370 int cpu_index;
371
372 #if defined(CONFIG_USER_ONLY)
373 cpu_list_lock();
374 #endif
375 cpu_index = 0;
376 CPU_FOREACH(some_cpu) {
377 cpu_index++;
378 }
379 cpu->cpu_index = cpu_index;
380 cpu->numa_node = 0;
381 QTAILQ_INIT(&env->breakpoints);
382 QTAILQ_INIT(&env->watchpoints);
383 #ifndef CONFIG_USER_ONLY
384 cpu->thread_id = qemu_get_thread_id();
385 #endif
386 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
387 #if defined(CONFIG_USER_ONLY)
388 cpu_list_unlock();
389 #endif
390 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
391 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
392 }
393 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
394 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
395 cpu_save, cpu_load, env);
396 assert(cc->vmsd == NULL);
397 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
398 #endif
399 if (cc->vmsd != NULL) {
400 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
401 }
402 }
403
404 #if defined(TARGET_HAS_ICE)
405 #if defined(CONFIG_USER_ONLY)
406 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
407 {
408 tb_invalidate_phys_page_range(pc, pc + 1, 0);
409 }
410 #else
411 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
412 {
413 tb_invalidate_phys_addr(cpu_get_phys_page_debug(cpu, pc) |
414 (pc & ~TARGET_PAGE_MASK));
415 }
416 #endif
417 #endif /* TARGET_HAS_ICE */
418
419 #if defined(CONFIG_USER_ONLY)
420 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
421
422 {
423 }
424
425 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
426 int flags, CPUWatchpoint **watchpoint)
427 {
428 return -ENOSYS;
429 }
430 #else
431 /* Add a watchpoint. */
432 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
433 int flags, CPUWatchpoint **watchpoint)
434 {
435 target_ulong len_mask = ~(len - 1);
436 CPUWatchpoint *wp;
437
438 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
439 if ((len & (len - 1)) || (addr & ~len_mask) ||
440 len == 0 || len > TARGET_PAGE_SIZE) {
441 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
442 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
443 return -EINVAL;
444 }
445 wp = g_malloc(sizeof(*wp));
446
447 wp->vaddr = addr;
448 wp->len_mask = len_mask;
449 wp->flags = flags;
450
451 /* keep all GDB-injected watchpoints in front */
452 if (flags & BP_GDB)
453 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
454 else
455 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
456
457 tlb_flush_page(env, addr);
458
459 if (watchpoint)
460 *watchpoint = wp;
461 return 0;
462 }
463
464 /* Remove a specific watchpoint. */
465 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
466 int flags)
467 {
468 target_ulong len_mask = ~(len - 1);
469 CPUWatchpoint *wp;
470
471 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
472 if (addr == wp->vaddr && len_mask == wp->len_mask
473 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
474 cpu_watchpoint_remove_by_ref(env, wp);
475 return 0;
476 }
477 }
478 return -ENOENT;
479 }
480
481 /* Remove a specific watchpoint by reference. */
482 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
483 {
484 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
485
486 tlb_flush_page(env, watchpoint->vaddr);
487
488 g_free(watchpoint);
489 }
490
491 /* Remove all matching watchpoints. */
492 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
493 {
494 CPUWatchpoint *wp, *next;
495
496 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
497 if (wp->flags & mask)
498 cpu_watchpoint_remove_by_ref(env, wp);
499 }
500 }
501 #endif
502
503 /* Add a breakpoint. */
504 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
505 CPUBreakpoint **breakpoint)
506 {
507 #if defined(TARGET_HAS_ICE)
508 CPUBreakpoint *bp;
509
510 bp = g_malloc(sizeof(*bp));
511
512 bp->pc = pc;
513 bp->flags = flags;
514
515 /* keep all GDB-injected breakpoints in front */
516 if (flags & BP_GDB) {
517 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
518 } else {
519 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
520 }
521
522 breakpoint_invalidate(ENV_GET_CPU(env), pc);
523
524 if (breakpoint) {
525 *breakpoint = bp;
526 }
527 return 0;
528 #else
529 return -ENOSYS;
530 #endif
531 }
532
533 /* Remove a specific breakpoint. */
534 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
535 {
536 #if defined(TARGET_HAS_ICE)
537 CPUBreakpoint *bp;
538
539 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
540 if (bp->pc == pc && bp->flags == flags) {
541 cpu_breakpoint_remove_by_ref(env, bp);
542 return 0;
543 }
544 }
545 return -ENOENT;
546 #else
547 return -ENOSYS;
548 #endif
549 }
550
551 /* Remove a specific breakpoint by reference. */
552 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
553 {
554 #if defined(TARGET_HAS_ICE)
555 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
556
557 breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
558
559 g_free(breakpoint);
560 #endif
561 }
562
563 /* Remove all matching breakpoints. */
564 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
565 {
566 #if defined(TARGET_HAS_ICE)
567 CPUBreakpoint *bp, *next;
568
569 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
570 if (bp->flags & mask)
571 cpu_breakpoint_remove_by_ref(env, bp);
572 }
573 #endif
574 }
575
576 /* enable or disable single step mode. EXCP_DEBUG is returned by the
577 CPU loop after each instruction */
578 void cpu_single_step(CPUState *cpu, int enabled)
579 {
580 #if defined(TARGET_HAS_ICE)
581 if (cpu->singlestep_enabled != enabled) {
582 cpu->singlestep_enabled = enabled;
583 if (kvm_enabled()) {
584 kvm_update_guest_debug(cpu, 0);
585 } else {
586 /* must flush all the translated code to avoid inconsistencies */
587 /* XXX: only flush what is necessary */
588 CPUArchState *env = cpu->env_ptr;
589 tb_flush(env);
590 }
591 }
592 #endif
593 }
594
595 void cpu_abort(CPUArchState *env, const char *fmt, ...)
596 {
597 CPUState *cpu = ENV_GET_CPU(env);
598 va_list ap;
599 va_list ap2;
600
601 va_start(ap, fmt);
602 va_copy(ap2, ap);
603 fprintf(stderr, "qemu: fatal: ");
604 vfprintf(stderr, fmt, ap);
605 fprintf(stderr, "\n");
606 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
607 if (qemu_log_enabled()) {
608 qemu_log("qemu: fatal: ");
609 qemu_log_vprintf(fmt, ap2);
610 qemu_log("\n");
611 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
612 qemu_log_flush();
613 qemu_log_close();
614 }
615 va_end(ap2);
616 va_end(ap);
617 #if defined(CONFIG_USER_ONLY)
618 {
619 struct sigaction act;
620 sigfillset(&act.sa_mask);
621 act.sa_handler = SIG_DFL;
622 sigaction(SIGABRT, &act, NULL);
623 }
624 #endif
625 abort();
626 }
627
628 CPUArchState *cpu_copy(CPUArchState *env)
629 {
630 CPUArchState *new_env = cpu_init(env->cpu_model_str);
631 #if defined(TARGET_HAS_ICE)
632 CPUBreakpoint *bp;
633 CPUWatchpoint *wp;
634 #endif
635
636 /* Reset non arch specific state */
637 cpu_reset(ENV_GET_CPU(new_env));
638
639 /* Copy arch specific state into the new CPU */
640 memcpy(new_env, env, sizeof(CPUArchState));
641
642 /* Clone all break/watchpoints.
643 Note: Once we support ptrace with hw-debug register access, make sure
644 BP_CPU break/watchpoints are handled correctly on clone. */
645 QTAILQ_INIT(&env->breakpoints);
646 QTAILQ_INIT(&env->watchpoints);
647 #if defined(TARGET_HAS_ICE)
648 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
649 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
650 }
651 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
652 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
653 wp->flags, NULL);
654 }
655 #endif
656
657 return new_env;
658 }
659
660 #if !defined(CONFIG_USER_ONLY)
661 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
662 uintptr_t length)
663 {
664 uintptr_t start1;
665
666 /* we modify the TLB cache so that the dirty bit will be set again
667 when accessing the range */
668 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
669 /* Check that we don't span multiple blocks - this breaks the
670 address comparisons below. */
671 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
672 != (end - 1) - start) {
673 abort();
674 }
675 cpu_tlb_reset_dirty_all(start1, length);
676
677 }
678
679 /* Note: start and end must be within the same ram block. */
680 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
681 int dirty_flags)
682 {
683 uintptr_t length;
684
685 start &= TARGET_PAGE_MASK;
686 end = TARGET_PAGE_ALIGN(end);
687
688 length = end - start;
689 if (length == 0)
690 return;
691 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
692
693 if (tcg_enabled()) {
694 tlb_reset_dirty_range_all(start, end, length);
695 }
696 }
697
698 static int cpu_physical_memory_set_dirty_tracking(int enable)
699 {
700 int ret = 0;
701 in_migration = enable;
702 return ret;
703 }
704
705 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
706 MemoryRegionSection *section,
707 target_ulong vaddr,
708 hwaddr paddr, hwaddr xlat,
709 int prot,
710 target_ulong *address)
711 {
712 hwaddr iotlb;
713 CPUWatchpoint *wp;
714
715 if (memory_region_is_ram(section->mr)) {
716 /* Normal RAM. */
717 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
718 + xlat;
719 if (!section->readonly) {
720 iotlb |= PHYS_SECTION_NOTDIRTY;
721 } else {
722 iotlb |= PHYS_SECTION_ROM;
723 }
724 } else {
725 iotlb = section - address_space_memory.dispatch->sections;
726 iotlb += xlat;
727 }
728
729 /* Make accesses to pages with watchpoints go via the
730 watchpoint trap routines. */
731 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
732 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
733 /* Avoid trapping reads of pages with a write breakpoint. */
734 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
735 iotlb = PHYS_SECTION_WATCH + paddr;
736 *address |= TLB_MMIO;
737 break;
738 }
739 }
740 }
741
742 return iotlb;
743 }
744 #endif /* defined(CONFIG_USER_ONLY) */
745
746 #if !defined(CONFIG_USER_ONLY)
747
748 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
749 uint16_t section);
750 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
751
752 static uint16_t phys_section_add(MemoryRegionSection *section)
753 {
754 /* The physical section number is ORed with a page-aligned
755 * pointer to produce the iotlb entries. Thus it should
756 * never overflow into the page-aligned value.
757 */
758 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
759
760 if (next_map.sections_nb == next_map.sections_nb_alloc) {
761 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
762 16);
763 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
764 next_map.sections_nb_alloc);
765 }
766 next_map.sections[next_map.sections_nb] = *section;
767 memory_region_ref(section->mr);
768 return next_map.sections_nb++;
769 }
770
771 static void phys_section_destroy(MemoryRegion *mr)
772 {
773 memory_region_unref(mr);
774
775 if (mr->subpage) {
776 subpage_t *subpage = container_of(mr, subpage_t, iomem);
777 memory_region_destroy(&subpage->iomem);
778 g_free(subpage);
779 }
780 }
781
782 static void phys_sections_free(PhysPageMap *map)
783 {
784 while (map->sections_nb > 0) {
785 MemoryRegionSection *section = &map->sections[--map->sections_nb];
786 phys_section_destroy(section->mr);
787 }
788 g_free(map->sections);
789 g_free(map->nodes);
790 g_free(map);
791 }
792
793 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
794 {
795 subpage_t *subpage;
796 hwaddr base = section->offset_within_address_space
797 & TARGET_PAGE_MASK;
798 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
799 next_map.nodes, next_map.sections);
800 MemoryRegionSection subsection = {
801 .offset_within_address_space = base,
802 .size = int128_make64(TARGET_PAGE_SIZE),
803 };
804 hwaddr start, end;
805
806 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
807
808 if (!(existing->mr->subpage)) {
809 subpage = subpage_init(d->as, base);
810 subsection.mr = &subpage->iomem;
811 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
812 phys_section_add(&subsection));
813 } else {
814 subpage = container_of(existing->mr, subpage_t, iomem);
815 }
816 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
817 end = start + int128_get64(section->size) - 1;
818 subpage_register(subpage, start, end, phys_section_add(section));
819 }
820
821
822 static void register_multipage(AddressSpaceDispatch *d,
823 MemoryRegionSection *section)
824 {
825 hwaddr start_addr = section->offset_within_address_space;
826 uint16_t section_index = phys_section_add(section);
827 uint64_t num_pages = int128_get64(int128_rshift(section->size,
828 TARGET_PAGE_BITS));
829
830 assert(num_pages);
831 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
832 }
833
834 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
835 {
836 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
837 AddressSpaceDispatch *d = as->next_dispatch;
838 MemoryRegionSection now = *section, remain = *section;
839 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
840
841 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
842 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
843 - now.offset_within_address_space;
844
845 now.size = int128_min(int128_make64(left), now.size);
846 register_subpage(d, &now);
847 } else {
848 now.size = int128_zero();
849 }
850 while (int128_ne(remain.size, now.size)) {
851 remain.size = int128_sub(remain.size, now.size);
852 remain.offset_within_address_space += int128_get64(now.size);
853 remain.offset_within_region += int128_get64(now.size);
854 now = remain;
855 if (int128_lt(remain.size, page_size)) {
856 register_subpage(d, &now);
857 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
858 now.size = page_size;
859 register_subpage(d, &now);
860 } else {
861 now.size = int128_and(now.size, int128_neg(page_size));
862 register_multipage(d, &now);
863 }
864 }
865 }
866
867 void qemu_flush_coalesced_mmio_buffer(void)
868 {
869 if (kvm_enabled())
870 kvm_flush_coalesced_mmio_buffer();
871 }
872
873 void qemu_mutex_lock_ramlist(void)
874 {
875 qemu_mutex_lock(&ram_list.mutex);
876 }
877
878 void qemu_mutex_unlock_ramlist(void)
879 {
880 qemu_mutex_unlock(&ram_list.mutex);
881 }
882
883 #if defined(__linux__) && !defined(TARGET_S390X)
884
885 #include <sys/vfs.h>
886
887 #define HUGETLBFS_MAGIC 0x958458f6
888
889 static long gethugepagesize(const char *path)
890 {
891 struct statfs fs;
892 int ret;
893
894 do {
895 ret = statfs(path, &fs);
896 } while (ret != 0 && errno == EINTR);
897
898 if (ret != 0) {
899 perror(path);
900 return 0;
901 }
902
903 if (fs.f_type != HUGETLBFS_MAGIC)
904 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
905
906 return fs.f_bsize;
907 }
908
909 static void *file_ram_alloc(RAMBlock *block,
910 ram_addr_t memory,
911 const char *path)
912 {
913 char *filename;
914 char *sanitized_name;
915 char *c;
916 void *area;
917 int fd;
918 #ifdef MAP_POPULATE
919 int flags;
920 #endif
921 unsigned long hpagesize;
922
923 hpagesize = gethugepagesize(path);
924 if (!hpagesize) {
925 return NULL;
926 }
927
928 if (memory < hpagesize) {
929 return NULL;
930 }
931
932 if (kvm_enabled() && !kvm_has_sync_mmu()) {
933 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
934 return NULL;
935 }
936
937 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
938 sanitized_name = g_strdup(block->mr->name);
939 for (c = sanitized_name; *c != '\0'; c++) {
940 if (*c == '/')
941 *c = '_';
942 }
943
944 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
945 sanitized_name);
946 g_free(sanitized_name);
947
948 fd = mkstemp(filename);
949 if (fd < 0) {
950 perror("unable to create backing store for hugepages");
951 g_free(filename);
952 return NULL;
953 }
954 unlink(filename);
955 g_free(filename);
956
957 memory = (memory+hpagesize-1) & ~(hpagesize-1);
958
959 /*
960 * ftruncate is not supported by hugetlbfs in older
961 * hosts, so don't bother bailing out on errors.
962 * If anything goes wrong with it under other filesystems,
963 * mmap will fail.
964 */
965 if (ftruncate(fd, memory))
966 perror("ftruncate");
967
968 #ifdef MAP_POPULATE
969 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
970 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
971 * to sidestep this quirk.
972 */
973 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
974 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
975 #else
976 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
977 #endif
978 if (area == MAP_FAILED) {
979 perror("file_ram_alloc: can't mmap RAM pages");
980 close(fd);
981 return (NULL);
982 }
983 block->fd = fd;
984 return area;
985 }
986 #endif
987
988 static ram_addr_t find_ram_offset(ram_addr_t size)
989 {
990 RAMBlock *block, *next_block;
991 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
992
993 assert(size != 0); /* it would hand out same offset multiple times */
994
995 if (QTAILQ_EMPTY(&ram_list.blocks))
996 return 0;
997
998 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
999 ram_addr_t end, next = RAM_ADDR_MAX;
1000
1001 end = block->offset + block->length;
1002
1003 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1004 if (next_block->offset >= end) {
1005 next = MIN(next, next_block->offset);
1006 }
1007 }
1008 if (next - end >= size && next - end < mingap) {
1009 offset = end;
1010 mingap = next - end;
1011 }
1012 }
1013
1014 if (offset == RAM_ADDR_MAX) {
1015 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1016 (uint64_t)size);
1017 abort();
1018 }
1019
1020 return offset;
1021 }
1022
1023 ram_addr_t last_ram_offset(void)
1024 {
1025 RAMBlock *block;
1026 ram_addr_t last = 0;
1027
1028 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1029 last = MAX(last, block->offset + block->length);
1030
1031 return last;
1032 }
1033
1034 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1035 {
1036 int ret;
1037
1038 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1039 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1040 "dump-guest-core", true)) {
1041 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1042 if (ret) {
1043 perror("qemu_madvise");
1044 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1045 "but dump_guest_core=off specified\n");
1046 }
1047 }
1048 }
1049
1050 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1051 {
1052 RAMBlock *new_block, *block;
1053
1054 new_block = NULL;
1055 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1056 if (block->offset == addr) {
1057 new_block = block;
1058 break;
1059 }
1060 }
1061 assert(new_block);
1062 assert(!new_block->idstr[0]);
1063
1064 if (dev) {
1065 char *id = qdev_get_dev_path(dev);
1066 if (id) {
1067 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1068 g_free(id);
1069 }
1070 }
1071 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1072
1073 /* This assumes the iothread lock is taken here too. */
1074 qemu_mutex_lock_ramlist();
1075 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1076 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1077 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1078 new_block->idstr);
1079 abort();
1080 }
1081 }
1082 qemu_mutex_unlock_ramlist();
1083 }
1084
1085 static int memory_try_enable_merging(void *addr, size_t len)
1086 {
1087 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1088 /* disabled by the user */
1089 return 0;
1090 }
1091
1092 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1093 }
1094
1095 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1096 MemoryRegion *mr)
1097 {
1098 RAMBlock *block, *new_block;
1099
1100 size = TARGET_PAGE_ALIGN(size);
1101 new_block = g_malloc0(sizeof(*new_block));
1102
1103 /* This assumes the iothread lock is taken here too. */
1104 qemu_mutex_lock_ramlist();
1105 new_block->mr = mr;
1106 new_block->offset = find_ram_offset(size);
1107 if (host) {
1108 new_block->host = host;
1109 new_block->flags |= RAM_PREALLOC_MASK;
1110 } else {
1111 if (mem_path) {
1112 #if defined (__linux__) && !defined(TARGET_S390X)
1113 new_block->host = file_ram_alloc(new_block, size, mem_path);
1114 if (!new_block->host) {
1115 new_block->host = qemu_anon_ram_alloc(size);
1116 memory_try_enable_merging(new_block->host, size);
1117 }
1118 #else
1119 fprintf(stderr, "-mem-path option unsupported\n");
1120 exit(1);
1121 #endif
1122 } else {
1123 if (xen_enabled()) {
1124 xen_ram_alloc(new_block->offset, size, mr);
1125 } else if (kvm_enabled()) {
1126 /* some s390/kvm configurations have special constraints */
1127 new_block->host = kvm_ram_alloc(size);
1128 } else {
1129 new_block->host = qemu_anon_ram_alloc(size);
1130 }
1131 memory_try_enable_merging(new_block->host, size);
1132 }
1133 }
1134 new_block->length = size;
1135
1136 /* Keep the list sorted from biggest to smallest block. */
1137 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1138 if (block->length < new_block->length) {
1139 break;
1140 }
1141 }
1142 if (block) {
1143 QTAILQ_INSERT_BEFORE(block, new_block, next);
1144 } else {
1145 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1146 }
1147 ram_list.mru_block = NULL;
1148
1149 ram_list.version++;
1150 qemu_mutex_unlock_ramlist();
1151
1152 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1153 last_ram_offset() >> TARGET_PAGE_BITS);
1154 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1155 0, size >> TARGET_PAGE_BITS);
1156 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1157
1158 qemu_ram_setup_dump(new_block->host, size);
1159 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1160
1161 if (kvm_enabled())
1162 kvm_setup_guest_memory(new_block->host, size);
1163
1164 return new_block->offset;
1165 }
1166
1167 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1168 {
1169 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1170 }
1171
1172 void qemu_ram_free_from_ptr(ram_addr_t addr)
1173 {
1174 RAMBlock *block;
1175
1176 /* This assumes the iothread lock is taken here too. */
1177 qemu_mutex_lock_ramlist();
1178 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1179 if (addr == block->offset) {
1180 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1181 ram_list.mru_block = NULL;
1182 ram_list.version++;
1183 g_free(block);
1184 break;
1185 }
1186 }
1187 qemu_mutex_unlock_ramlist();
1188 }
1189
1190 void qemu_ram_free(ram_addr_t addr)
1191 {
1192 RAMBlock *block;
1193
1194 /* This assumes the iothread lock is taken here too. */
1195 qemu_mutex_lock_ramlist();
1196 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1197 if (addr == block->offset) {
1198 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1199 ram_list.mru_block = NULL;
1200 ram_list.version++;
1201 if (block->flags & RAM_PREALLOC_MASK) {
1202 ;
1203 } else if (mem_path) {
1204 #if defined (__linux__) && !defined(TARGET_S390X)
1205 if (block->fd) {
1206 munmap(block->host, block->length);
1207 close(block->fd);
1208 } else {
1209 qemu_anon_ram_free(block->host, block->length);
1210 }
1211 #else
1212 abort();
1213 #endif
1214 } else {
1215 if (xen_enabled()) {
1216 xen_invalidate_map_cache_entry(block->host);
1217 } else {
1218 qemu_anon_ram_free(block->host, block->length);
1219 }
1220 }
1221 g_free(block);
1222 break;
1223 }
1224 }
1225 qemu_mutex_unlock_ramlist();
1226
1227 }
1228
1229 #ifndef _WIN32
1230 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1231 {
1232 RAMBlock *block;
1233 ram_addr_t offset;
1234 int flags;
1235 void *area, *vaddr;
1236
1237 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1238 offset = addr - block->offset;
1239 if (offset < block->length) {
1240 vaddr = block->host + offset;
1241 if (block->flags & RAM_PREALLOC_MASK) {
1242 ;
1243 } else {
1244 flags = MAP_FIXED;
1245 munmap(vaddr, length);
1246 if (mem_path) {
1247 #if defined(__linux__) && !defined(TARGET_S390X)
1248 if (block->fd) {
1249 #ifdef MAP_POPULATE
1250 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1251 MAP_PRIVATE;
1252 #else
1253 flags |= MAP_PRIVATE;
1254 #endif
1255 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1256 flags, block->fd, offset);
1257 } else {
1258 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1259 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1260 flags, -1, 0);
1261 }
1262 #else
1263 abort();
1264 #endif
1265 } else {
1266 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1267 flags |= MAP_SHARED | MAP_ANONYMOUS;
1268 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1269 flags, -1, 0);
1270 #else
1271 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1272 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1273 flags, -1, 0);
1274 #endif
1275 }
1276 if (area != vaddr) {
1277 fprintf(stderr, "Could not remap addr: "
1278 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1279 length, addr);
1280 exit(1);
1281 }
1282 memory_try_enable_merging(vaddr, length);
1283 qemu_ram_setup_dump(vaddr, length);
1284 }
1285 return;
1286 }
1287 }
1288 }
1289 #endif /* !_WIN32 */
1290
1291 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1292 {
1293 RAMBlock *block;
1294
1295 /* The list is protected by the iothread lock here. */
1296 block = ram_list.mru_block;
1297 if (block && addr - block->offset < block->length) {
1298 goto found;
1299 }
1300 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1301 if (addr - block->offset < block->length) {
1302 goto found;
1303 }
1304 }
1305
1306 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1307 abort();
1308
1309 found:
1310 ram_list.mru_block = block;
1311 return block;
1312 }
1313
1314 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1315 With the exception of the softmmu code in this file, this should
1316 only be used for local memory (e.g. video ram) that the device owns,
1317 and knows it isn't going to access beyond the end of the block.
1318
1319 It should not be used for general purpose DMA.
1320 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1321 */
1322 void *qemu_get_ram_ptr(ram_addr_t addr)
1323 {
1324 RAMBlock *block = qemu_get_ram_block(addr);
1325
1326 if (xen_enabled()) {
1327 /* We need to check if the requested address is in the RAM
1328 * because we don't want to map the entire memory in QEMU.
1329 * In that case just map until the end of the page.
1330 */
1331 if (block->offset == 0) {
1332 return xen_map_cache(addr, 0, 0);
1333 } else if (block->host == NULL) {
1334 block->host =
1335 xen_map_cache(block->offset, block->length, 1);
1336 }
1337 }
1338 return block->host + (addr - block->offset);
1339 }
1340
1341 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1342 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1343 *
1344 * ??? Is this still necessary?
1345 */
1346 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1347 {
1348 RAMBlock *block;
1349
1350 /* The list is protected by the iothread lock here. */
1351 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1352 if (addr - block->offset < block->length) {
1353 if (xen_enabled()) {
1354 /* We need to check if the requested address is in the RAM
1355 * because we don't want to map the entire memory in QEMU.
1356 * In that case just map until the end of the page.
1357 */
1358 if (block->offset == 0) {
1359 return xen_map_cache(addr, 0, 0);
1360 } else if (block->host == NULL) {
1361 block->host =
1362 xen_map_cache(block->offset, block->length, 1);
1363 }
1364 }
1365 return block->host + (addr - block->offset);
1366 }
1367 }
1368
1369 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1370 abort();
1371
1372 return NULL;
1373 }
1374
1375 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1376 * but takes a size argument */
1377 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1378 {
1379 if (*size == 0) {
1380 return NULL;
1381 }
1382 if (xen_enabled()) {
1383 return xen_map_cache(addr, *size, 1);
1384 } else {
1385 RAMBlock *block;
1386
1387 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1388 if (addr - block->offset < block->length) {
1389 if (addr - block->offset + *size > block->length)
1390 *size = block->length - addr + block->offset;
1391 return block->host + (addr - block->offset);
1392 }
1393 }
1394
1395 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1396 abort();
1397 }
1398 }
1399
1400 /* Some of the softmmu routines need to translate from a host pointer
1401 (typically a TLB entry) back to a ram offset. */
1402 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1403 {
1404 RAMBlock *block;
1405 uint8_t *host = ptr;
1406
1407 if (xen_enabled()) {
1408 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1409 return qemu_get_ram_block(*ram_addr)->mr;
1410 }
1411
1412 block = ram_list.mru_block;
1413 if (block && block->host && host - block->host < block->length) {
1414 goto found;
1415 }
1416
1417 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1418 /* This case append when the block is not mapped. */
1419 if (block->host == NULL) {
1420 continue;
1421 }
1422 if (host - block->host < block->length) {
1423 goto found;
1424 }
1425 }
1426
1427 return NULL;
1428
1429 found:
1430 *ram_addr = block->offset + (host - block->host);
1431 return block->mr;
1432 }
1433
1434 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1435 uint64_t val, unsigned size)
1436 {
1437 int dirty_flags;
1438 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1439 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1440 tb_invalidate_phys_page_fast(ram_addr, size);
1441 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1442 }
1443 switch (size) {
1444 case 1:
1445 stb_p(qemu_get_ram_ptr(ram_addr), val);
1446 break;
1447 case 2:
1448 stw_p(qemu_get_ram_ptr(ram_addr), val);
1449 break;
1450 case 4:
1451 stl_p(qemu_get_ram_ptr(ram_addr), val);
1452 break;
1453 default:
1454 abort();
1455 }
1456 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1457 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1458 /* we remove the notdirty callback only if the code has been
1459 flushed */
1460 if (dirty_flags == 0xff) {
1461 CPUArchState *env = current_cpu->env_ptr;
1462 tlb_set_dirty(env, env->mem_io_vaddr);
1463 }
1464 }
1465
1466 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1467 unsigned size, bool is_write)
1468 {
1469 return is_write;
1470 }
1471
1472 static const MemoryRegionOps notdirty_mem_ops = {
1473 .write = notdirty_mem_write,
1474 .valid.accepts = notdirty_mem_accepts,
1475 .endianness = DEVICE_NATIVE_ENDIAN,
1476 };
1477
1478 /* Generate a debug exception if a watchpoint has been hit. */
1479 static void check_watchpoint(int offset, int len_mask, int flags)
1480 {
1481 CPUArchState *env = current_cpu->env_ptr;
1482 target_ulong pc, cs_base;
1483 target_ulong vaddr;
1484 CPUWatchpoint *wp;
1485 int cpu_flags;
1486
1487 if (env->watchpoint_hit) {
1488 /* We re-entered the check after replacing the TB. Now raise
1489 * the debug interrupt so that is will trigger after the
1490 * current instruction. */
1491 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1492 return;
1493 }
1494 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1495 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1496 if ((vaddr == (wp->vaddr & len_mask) ||
1497 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1498 wp->flags |= BP_WATCHPOINT_HIT;
1499 if (!env->watchpoint_hit) {
1500 env->watchpoint_hit = wp;
1501 tb_check_watchpoint(env);
1502 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1503 env->exception_index = EXCP_DEBUG;
1504 cpu_loop_exit(env);
1505 } else {
1506 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1507 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1508 cpu_resume_from_signal(env, NULL);
1509 }
1510 }
1511 } else {
1512 wp->flags &= ~BP_WATCHPOINT_HIT;
1513 }
1514 }
1515 }
1516
1517 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1518 so these check for a hit then pass through to the normal out-of-line
1519 phys routines. */
1520 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1521 unsigned size)
1522 {
1523 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1524 switch (size) {
1525 case 1: return ldub_phys(addr);
1526 case 2: return lduw_phys(addr);
1527 case 4: return ldl_phys(addr);
1528 default: abort();
1529 }
1530 }
1531
1532 static void watch_mem_write(void *opaque, hwaddr addr,
1533 uint64_t val, unsigned size)
1534 {
1535 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1536 switch (size) {
1537 case 1:
1538 stb_phys(addr, val);
1539 break;
1540 case 2:
1541 stw_phys(addr, val);
1542 break;
1543 case 4:
1544 stl_phys(addr, val);
1545 break;
1546 default: abort();
1547 }
1548 }
1549
1550 static const MemoryRegionOps watch_mem_ops = {
1551 .read = watch_mem_read,
1552 .write = watch_mem_write,
1553 .endianness = DEVICE_NATIVE_ENDIAN,
1554 };
1555
1556 static uint64_t subpage_read(void *opaque, hwaddr addr,
1557 unsigned len)
1558 {
1559 subpage_t *subpage = opaque;
1560 uint8_t buf[4];
1561
1562 #if defined(DEBUG_SUBPAGE)
1563 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1564 subpage, len, addr);
1565 #endif
1566 address_space_read(subpage->as, addr + subpage->base, buf, len);
1567 switch (len) {
1568 case 1:
1569 return ldub_p(buf);
1570 case 2:
1571 return lduw_p(buf);
1572 case 4:
1573 return ldl_p(buf);
1574 default:
1575 abort();
1576 }
1577 }
1578
1579 static void subpage_write(void *opaque, hwaddr addr,
1580 uint64_t value, unsigned len)
1581 {
1582 subpage_t *subpage = opaque;
1583 uint8_t buf[4];
1584
1585 #if defined(DEBUG_SUBPAGE)
1586 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1587 " value %"PRIx64"\n",
1588 __func__, subpage, len, addr, value);
1589 #endif
1590 switch (len) {
1591 case 1:
1592 stb_p(buf, value);
1593 break;
1594 case 2:
1595 stw_p(buf, value);
1596 break;
1597 case 4:
1598 stl_p(buf, value);
1599 break;
1600 default:
1601 abort();
1602 }
1603 address_space_write(subpage->as, addr + subpage->base, buf, len);
1604 }
1605
1606 static bool subpage_accepts(void *opaque, hwaddr addr,
1607 unsigned size, bool is_write)
1608 {
1609 subpage_t *subpage = opaque;
1610 #if defined(DEBUG_SUBPAGE)
1611 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1612 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1613 #endif
1614
1615 return address_space_access_valid(subpage->as, addr + subpage->base,
1616 size, is_write);
1617 }
1618
1619 static const MemoryRegionOps subpage_ops = {
1620 .read = subpage_read,
1621 .write = subpage_write,
1622 .valid.accepts = subpage_accepts,
1623 .endianness = DEVICE_NATIVE_ENDIAN,
1624 };
1625
1626 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1627 uint16_t section)
1628 {
1629 int idx, eidx;
1630
1631 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1632 return -1;
1633 idx = SUBPAGE_IDX(start);
1634 eidx = SUBPAGE_IDX(end);
1635 #if defined(DEBUG_SUBPAGE)
1636 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1637 mmio, start, end, idx, eidx, memory);
1638 #endif
1639 for (; idx <= eidx; idx++) {
1640 mmio->sub_section[idx] = section;
1641 }
1642
1643 return 0;
1644 }
1645
1646 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1647 {
1648 subpage_t *mmio;
1649
1650 mmio = g_malloc0(sizeof(subpage_t));
1651
1652 mmio->as = as;
1653 mmio->base = base;
1654 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1655 "subpage", TARGET_PAGE_SIZE);
1656 mmio->iomem.subpage = true;
1657 #if defined(DEBUG_SUBPAGE)
1658 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1659 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1660 #endif
1661 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1662
1663 return mmio;
1664 }
1665
1666 static uint16_t dummy_section(MemoryRegion *mr)
1667 {
1668 MemoryRegionSection section = {
1669 .mr = mr,
1670 .offset_within_address_space = 0,
1671 .offset_within_region = 0,
1672 .size = int128_2_64(),
1673 };
1674
1675 return phys_section_add(&section);
1676 }
1677
1678 MemoryRegion *iotlb_to_region(hwaddr index)
1679 {
1680 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1681 }
1682
1683 static void io_mem_init(void)
1684 {
1685 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1686 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1687 "unassigned", UINT64_MAX);
1688 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1689 "notdirty", UINT64_MAX);
1690 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1691 "watch", UINT64_MAX);
1692 }
1693
1694 static void mem_begin(MemoryListener *listener)
1695 {
1696 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1697 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1698
1699 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1700 d->as = as;
1701 as->next_dispatch = d;
1702 }
1703
1704 static void mem_commit(MemoryListener *listener)
1705 {
1706 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1707 AddressSpaceDispatch *cur = as->dispatch;
1708 AddressSpaceDispatch *next = as->next_dispatch;
1709
1710 next->nodes = next_map.nodes;
1711 next->sections = next_map.sections;
1712
1713 as->dispatch = next;
1714 g_free(cur);
1715 }
1716
1717 static void core_begin(MemoryListener *listener)
1718 {
1719 uint16_t n;
1720
1721 prev_map = g_new(PhysPageMap, 1);
1722 *prev_map = next_map;
1723
1724 memset(&next_map, 0, sizeof(next_map));
1725 n = dummy_section(&io_mem_unassigned);
1726 assert(n == PHYS_SECTION_UNASSIGNED);
1727 n = dummy_section(&io_mem_notdirty);
1728 assert(n == PHYS_SECTION_NOTDIRTY);
1729 n = dummy_section(&io_mem_rom);
1730 assert(n == PHYS_SECTION_ROM);
1731 n = dummy_section(&io_mem_watch);
1732 assert(n == PHYS_SECTION_WATCH);
1733 }
1734
1735 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1736 * All AddressSpaceDispatch instances have switched to the next map.
1737 */
1738 static void core_commit(MemoryListener *listener)
1739 {
1740 phys_sections_free(prev_map);
1741 }
1742
1743 static void tcg_commit(MemoryListener *listener)
1744 {
1745 CPUState *cpu;
1746
1747 /* since each CPU stores ram addresses in its TLB cache, we must
1748 reset the modified entries */
1749 /* XXX: slow ! */
1750 CPU_FOREACH(cpu) {
1751 CPUArchState *env = cpu->env_ptr;
1752
1753 tlb_flush(env, 1);
1754 }
1755 }
1756
1757 static void core_log_global_start(MemoryListener *listener)
1758 {
1759 cpu_physical_memory_set_dirty_tracking(1);
1760 }
1761
1762 static void core_log_global_stop(MemoryListener *listener)
1763 {
1764 cpu_physical_memory_set_dirty_tracking(0);
1765 }
1766
1767 static MemoryListener core_memory_listener = {
1768 .begin = core_begin,
1769 .commit = core_commit,
1770 .log_global_start = core_log_global_start,
1771 .log_global_stop = core_log_global_stop,
1772 .priority = 1,
1773 };
1774
1775 static MemoryListener tcg_memory_listener = {
1776 .commit = tcg_commit,
1777 };
1778
1779 void address_space_init_dispatch(AddressSpace *as)
1780 {
1781 as->dispatch = NULL;
1782 as->dispatch_listener = (MemoryListener) {
1783 .begin = mem_begin,
1784 .commit = mem_commit,
1785 .region_add = mem_add,
1786 .region_nop = mem_add,
1787 .priority = 0,
1788 };
1789 memory_listener_register(&as->dispatch_listener, as);
1790 }
1791
1792 void address_space_destroy_dispatch(AddressSpace *as)
1793 {
1794 AddressSpaceDispatch *d = as->dispatch;
1795
1796 memory_listener_unregister(&as->dispatch_listener);
1797 g_free(d);
1798 as->dispatch = NULL;
1799 }
1800
1801 static void memory_map_init(void)
1802 {
1803 system_memory = g_malloc(sizeof(*system_memory));
1804 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1805 address_space_init(&address_space_memory, system_memory, "memory");
1806
1807 system_io = g_malloc(sizeof(*system_io));
1808 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
1809 65536);
1810 address_space_init(&address_space_io, system_io, "I/O");
1811
1812 memory_listener_register(&core_memory_listener, &address_space_memory);
1813 if (tcg_enabled()) {
1814 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1815 }
1816 }
1817
1818 MemoryRegion *get_system_memory(void)
1819 {
1820 return system_memory;
1821 }
1822
1823 MemoryRegion *get_system_io(void)
1824 {
1825 return system_io;
1826 }
1827
1828 #endif /* !defined(CONFIG_USER_ONLY) */
1829
1830 /* physical memory access (slow version, mainly for debug) */
1831 #if defined(CONFIG_USER_ONLY)
1832 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1833 uint8_t *buf, int len, int is_write)
1834 {
1835 int l, flags;
1836 target_ulong page;
1837 void * p;
1838
1839 while (len > 0) {
1840 page = addr & TARGET_PAGE_MASK;
1841 l = (page + TARGET_PAGE_SIZE) - addr;
1842 if (l > len)
1843 l = len;
1844 flags = page_get_flags(page);
1845 if (!(flags & PAGE_VALID))
1846 return -1;
1847 if (is_write) {
1848 if (!(flags & PAGE_WRITE))
1849 return -1;
1850 /* XXX: this code should not depend on lock_user */
1851 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1852 return -1;
1853 memcpy(p, buf, l);
1854 unlock_user(p, addr, l);
1855 } else {
1856 if (!(flags & PAGE_READ))
1857 return -1;
1858 /* XXX: this code should not depend on lock_user */
1859 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1860 return -1;
1861 memcpy(buf, p, l);
1862 unlock_user(p, addr, 0);
1863 }
1864 len -= l;
1865 buf += l;
1866 addr += l;
1867 }
1868 return 0;
1869 }
1870
1871 #else
1872
1873 static void invalidate_and_set_dirty(hwaddr addr,
1874 hwaddr length)
1875 {
1876 if (!cpu_physical_memory_is_dirty(addr)) {
1877 /* invalidate code */
1878 tb_invalidate_phys_page_range(addr, addr + length, 0);
1879 /* set dirty bit */
1880 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1881 }
1882 xen_modified_memory(addr, length);
1883 }
1884
1885 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1886 {
1887 if (memory_region_is_ram(mr)) {
1888 return !(is_write && mr->readonly);
1889 }
1890 if (memory_region_is_romd(mr)) {
1891 return !is_write;
1892 }
1893
1894 return false;
1895 }
1896
1897 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1898 {
1899 unsigned access_size_max = mr->ops->valid.max_access_size;
1900
1901 /* Regions are assumed to support 1-4 byte accesses unless
1902 otherwise specified. */
1903 if (access_size_max == 0) {
1904 access_size_max = 4;
1905 }
1906
1907 /* Bound the maximum access by the alignment of the address. */
1908 if (!mr->ops->impl.unaligned) {
1909 unsigned align_size_max = addr & -addr;
1910 if (align_size_max != 0 && align_size_max < access_size_max) {
1911 access_size_max = align_size_max;
1912 }
1913 }
1914
1915 /* Don't attempt accesses larger than the maximum. */
1916 if (l > access_size_max) {
1917 l = access_size_max;
1918 }
1919 if (l & (l - 1)) {
1920 l = 1 << (qemu_fls(l) - 1);
1921 }
1922
1923 return l;
1924 }
1925
1926 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1927 int len, bool is_write)
1928 {
1929 hwaddr l;
1930 uint8_t *ptr;
1931 uint64_t val;
1932 hwaddr addr1;
1933 MemoryRegion *mr;
1934 bool error = false;
1935
1936 while (len > 0) {
1937 l = len;
1938 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1939
1940 if (is_write) {
1941 if (!memory_access_is_direct(mr, is_write)) {
1942 l = memory_access_size(mr, l, addr1);
1943 /* XXX: could force current_cpu to NULL to avoid
1944 potential bugs */
1945 switch (l) {
1946 case 8:
1947 /* 64 bit write access */
1948 val = ldq_p(buf);
1949 error |= io_mem_write(mr, addr1, val, 8);
1950 break;
1951 case 4:
1952 /* 32 bit write access */
1953 val = ldl_p(buf);
1954 error |= io_mem_write(mr, addr1, val, 4);
1955 break;
1956 case 2:
1957 /* 16 bit write access */
1958 val = lduw_p(buf);
1959 error |= io_mem_write(mr, addr1, val, 2);
1960 break;
1961 case 1:
1962 /* 8 bit write access */
1963 val = ldub_p(buf);
1964 error |= io_mem_write(mr, addr1, val, 1);
1965 break;
1966 default:
1967 abort();
1968 }
1969 } else {
1970 addr1 += memory_region_get_ram_addr(mr);
1971 /* RAM case */
1972 ptr = qemu_get_ram_ptr(addr1);
1973 memcpy(ptr, buf, l);
1974 invalidate_and_set_dirty(addr1, l);
1975 }
1976 } else {
1977 if (!memory_access_is_direct(mr, is_write)) {
1978 /* I/O case */
1979 l = memory_access_size(mr, l, addr1);
1980 switch (l) {
1981 case 8:
1982 /* 64 bit read access */
1983 error |= io_mem_read(mr, addr1, &val, 8);
1984 stq_p(buf, val);
1985 break;
1986 case 4:
1987 /* 32 bit read access */
1988 error |= io_mem_read(mr, addr1, &val, 4);
1989 stl_p(buf, val);
1990 break;
1991 case 2:
1992 /* 16 bit read access */
1993 error |= io_mem_read(mr, addr1, &val, 2);
1994 stw_p(buf, val);
1995 break;
1996 case 1:
1997 /* 8 bit read access */
1998 error |= io_mem_read(mr, addr1, &val, 1);
1999 stb_p(buf, val);
2000 break;
2001 default:
2002 abort();
2003 }
2004 } else {
2005 /* RAM case */
2006 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2007 memcpy(buf, ptr, l);
2008 }
2009 }
2010 len -= l;
2011 buf += l;
2012 addr += l;
2013 }
2014
2015 return error;
2016 }
2017
2018 bool address_space_write(AddressSpace *as, hwaddr addr,
2019 const uint8_t *buf, int len)
2020 {
2021 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2022 }
2023
2024 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2025 {
2026 return address_space_rw(as, addr, buf, len, false);
2027 }
2028
2029
2030 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2031 int len, int is_write)
2032 {
2033 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2034 }
2035
2036 /* used for ROM loading : can write in RAM and ROM */
2037 void cpu_physical_memory_write_rom(hwaddr addr,
2038 const uint8_t *buf, int len)
2039 {
2040 hwaddr l;
2041 uint8_t *ptr;
2042 hwaddr addr1;
2043 MemoryRegion *mr;
2044
2045 while (len > 0) {
2046 l = len;
2047 mr = address_space_translate(&address_space_memory,
2048 addr, &addr1, &l, true);
2049
2050 if (!(memory_region_is_ram(mr) ||
2051 memory_region_is_romd(mr))) {
2052 /* do nothing */
2053 } else {
2054 addr1 += memory_region_get_ram_addr(mr);
2055 /* ROM/RAM case */
2056 ptr = qemu_get_ram_ptr(addr1);
2057 memcpy(ptr, buf, l);
2058 invalidate_and_set_dirty(addr1, l);
2059 }
2060 len -= l;
2061 buf += l;
2062 addr += l;
2063 }
2064 }
2065
2066 typedef struct {
2067 MemoryRegion *mr;
2068 void *buffer;
2069 hwaddr addr;
2070 hwaddr len;
2071 } BounceBuffer;
2072
2073 static BounceBuffer bounce;
2074
2075 typedef struct MapClient {
2076 void *opaque;
2077 void (*callback)(void *opaque);
2078 QLIST_ENTRY(MapClient) link;
2079 } MapClient;
2080
2081 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2082 = QLIST_HEAD_INITIALIZER(map_client_list);
2083
2084 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2085 {
2086 MapClient *client = g_malloc(sizeof(*client));
2087
2088 client->opaque = opaque;
2089 client->callback = callback;
2090 QLIST_INSERT_HEAD(&map_client_list, client, link);
2091 return client;
2092 }
2093
2094 static void cpu_unregister_map_client(void *_client)
2095 {
2096 MapClient *client = (MapClient *)_client;
2097
2098 QLIST_REMOVE(client, link);
2099 g_free(client);
2100 }
2101
2102 static void cpu_notify_map_clients(void)
2103 {
2104 MapClient *client;
2105
2106 while (!QLIST_EMPTY(&map_client_list)) {
2107 client = QLIST_FIRST(&map_client_list);
2108 client->callback(client->opaque);
2109 cpu_unregister_map_client(client);
2110 }
2111 }
2112
2113 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2114 {
2115 MemoryRegion *mr;
2116 hwaddr l, xlat;
2117
2118 while (len > 0) {
2119 l = len;
2120 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2121 if (!memory_access_is_direct(mr, is_write)) {
2122 l = memory_access_size(mr, l, addr);
2123 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2124 return false;
2125 }
2126 }
2127
2128 len -= l;
2129 addr += l;
2130 }
2131 return true;
2132 }
2133
2134 /* Map a physical memory region into a host virtual address.
2135 * May map a subset of the requested range, given by and returned in *plen.
2136 * May return NULL if resources needed to perform the mapping are exhausted.
2137 * Use only for reads OR writes - not for read-modify-write operations.
2138 * Use cpu_register_map_client() to know when retrying the map operation is
2139 * likely to succeed.
2140 */
2141 void *address_space_map(AddressSpace *as,
2142 hwaddr addr,
2143 hwaddr *plen,
2144 bool is_write)
2145 {
2146 hwaddr len = *plen;
2147 hwaddr done = 0;
2148 hwaddr l, xlat, base;
2149 MemoryRegion *mr, *this_mr;
2150 ram_addr_t raddr;
2151
2152 if (len == 0) {
2153 return NULL;
2154 }
2155
2156 l = len;
2157 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2158 if (!memory_access_is_direct(mr, is_write)) {
2159 if (bounce.buffer) {
2160 return NULL;
2161 }
2162 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2163 bounce.addr = addr;
2164 bounce.len = l;
2165
2166 memory_region_ref(mr);
2167 bounce.mr = mr;
2168 if (!is_write) {
2169 address_space_read(as, addr, bounce.buffer, l);
2170 }
2171
2172 *plen = l;
2173 return bounce.buffer;
2174 }
2175
2176 base = xlat;
2177 raddr = memory_region_get_ram_addr(mr);
2178
2179 for (;;) {
2180 len -= l;
2181 addr += l;
2182 done += l;
2183 if (len == 0) {
2184 break;
2185 }
2186
2187 l = len;
2188 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2189 if (this_mr != mr || xlat != base + done) {
2190 break;
2191 }
2192 }
2193
2194 memory_region_ref(mr);
2195 *plen = done;
2196 return qemu_ram_ptr_length(raddr + base, plen);
2197 }
2198
2199 /* Unmaps a memory region previously mapped by address_space_map().
2200 * Will also mark the memory as dirty if is_write == 1. access_len gives
2201 * the amount of memory that was actually read or written by the caller.
2202 */
2203 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2204 int is_write, hwaddr access_len)
2205 {
2206 if (buffer != bounce.buffer) {
2207 MemoryRegion *mr;
2208 ram_addr_t addr1;
2209
2210 mr = qemu_ram_addr_from_host(buffer, &addr1);
2211 assert(mr != NULL);
2212 if (is_write) {
2213 while (access_len) {
2214 unsigned l;
2215 l = TARGET_PAGE_SIZE;
2216 if (l > access_len)
2217 l = access_len;
2218 invalidate_and_set_dirty(addr1, l);
2219 addr1 += l;
2220 access_len -= l;
2221 }
2222 }
2223 if (xen_enabled()) {
2224 xen_invalidate_map_cache_entry(buffer);
2225 }
2226 memory_region_unref(mr);
2227 return;
2228 }
2229 if (is_write) {
2230 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2231 }
2232 qemu_vfree(bounce.buffer);
2233 bounce.buffer = NULL;
2234 memory_region_unref(bounce.mr);
2235 cpu_notify_map_clients();
2236 }
2237
2238 void *cpu_physical_memory_map(hwaddr addr,
2239 hwaddr *plen,
2240 int is_write)
2241 {
2242 return address_space_map(&address_space_memory, addr, plen, is_write);
2243 }
2244
2245 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2246 int is_write, hwaddr access_len)
2247 {
2248 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2249 }
2250
2251 /* warning: addr must be aligned */
2252 static inline uint32_t ldl_phys_internal(hwaddr addr,
2253 enum device_endian endian)
2254 {
2255 uint8_t *ptr;
2256 uint64_t val;
2257 MemoryRegion *mr;
2258 hwaddr l = 4;
2259 hwaddr addr1;
2260
2261 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2262 false);
2263 if (l < 4 || !memory_access_is_direct(mr, false)) {
2264 /* I/O case */
2265 io_mem_read(mr, addr1, &val, 4);
2266 #if defined(TARGET_WORDS_BIGENDIAN)
2267 if (endian == DEVICE_LITTLE_ENDIAN) {
2268 val = bswap32(val);
2269 }
2270 #else
2271 if (endian == DEVICE_BIG_ENDIAN) {
2272 val = bswap32(val);
2273 }
2274 #endif
2275 } else {
2276 /* RAM case */
2277 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2278 & TARGET_PAGE_MASK)
2279 + addr1);
2280 switch (endian) {
2281 case DEVICE_LITTLE_ENDIAN:
2282 val = ldl_le_p(ptr);
2283 break;
2284 case DEVICE_BIG_ENDIAN:
2285 val = ldl_be_p(ptr);
2286 break;
2287 default:
2288 val = ldl_p(ptr);
2289 break;
2290 }
2291 }
2292 return val;
2293 }
2294
2295 uint32_t ldl_phys(hwaddr addr)
2296 {
2297 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2298 }
2299
2300 uint32_t ldl_le_phys(hwaddr addr)
2301 {
2302 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2303 }
2304
2305 uint32_t ldl_be_phys(hwaddr addr)
2306 {
2307 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2308 }
2309
2310 /* warning: addr must be aligned */
2311 static inline uint64_t ldq_phys_internal(hwaddr addr,
2312 enum device_endian endian)
2313 {
2314 uint8_t *ptr;
2315 uint64_t val;
2316 MemoryRegion *mr;
2317 hwaddr l = 8;
2318 hwaddr addr1;
2319
2320 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2321 false);
2322 if (l < 8 || !memory_access_is_direct(mr, false)) {
2323 /* I/O case */
2324 io_mem_read(mr, addr1, &val, 8);
2325 #if defined(TARGET_WORDS_BIGENDIAN)
2326 if (endian == DEVICE_LITTLE_ENDIAN) {
2327 val = bswap64(val);
2328 }
2329 #else
2330 if (endian == DEVICE_BIG_ENDIAN) {
2331 val = bswap64(val);
2332 }
2333 #endif
2334 } else {
2335 /* RAM case */
2336 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2337 & TARGET_PAGE_MASK)
2338 + addr1);
2339 switch (endian) {
2340 case DEVICE_LITTLE_ENDIAN:
2341 val = ldq_le_p(ptr);
2342 break;
2343 case DEVICE_BIG_ENDIAN:
2344 val = ldq_be_p(ptr);
2345 break;
2346 default:
2347 val = ldq_p(ptr);
2348 break;
2349 }
2350 }
2351 return val;
2352 }
2353
2354 uint64_t ldq_phys(hwaddr addr)
2355 {
2356 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2357 }
2358
2359 uint64_t ldq_le_phys(hwaddr addr)
2360 {
2361 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2362 }
2363
2364 uint64_t ldq_be_phys(hwaddr addr)
2365 {
2366 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2367 }
2368
2369 /* XXX: optimize */
2370 uint32_t ldub_phys(hwaddr addr)
2371 {
2372 uint8_t val;
2373 cpu_physical_memory_read(addr, &val, 1);
2374 return val;
2375 }
2376
2377 /* warning: addr must be aligned */
2378 static inline uint32_t lduw_phys_internal(hwaddr addr,
2379 enum device_endian endian)
2380 {
2381 uint8_t *ptr;
2382 uint64_t val;
2383 MemoryRegion *mr;
2384 hwaddr l = 2;
2385 hwaddr addr1;
2386
2387 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2388 false);
2389 if (l < 2 || !memory_access_is_direct(mr, false)) {
2390 /* I/O case */
2391 io_mem_read(mr, addr1, &val, 2);
2392 #if defined(TARGET_WORDS_BIGENDIAN)
2393 if (endian == DEVICE_LITTLE_ENDIAN) {
2394 val = bswap16(val);
2395 }
2396 #else
2397 if (endian == DEVICE_BIG_ENDIAN) {
2398 val = bswap16(val);
2399 }
2400 #endif
2401 } else {
2402 /* RAM case */
2403 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2404 & TARGET_PAGE_MASK)
2405 + addr1);
2406 switch (endian) {
2407 case DEVICE_LITTLE_ENDIAN:
2408 val = lduw_le_p(ptr);
2409 break;
2410 case DEVICE_BIG_ENDIAN:
2411 val = lduw_be_p(ptr);
2412 break;
2413 default:
2414 val = lduw_p(ptr);
2415 break;
2416 }
2417 }
2418 return val;
2419 }
2420
2421 uint32_t lduw_phys(hwaddr addr)
2422 {
2423 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2424 }
2425
2426 uint32_t lduw_le_phys(hwaddr addr)
2427 {
2428 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2429 }
2430
2431 uint32_t lduw_be_phys(hwaddr addr)
2432 {
2433 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2434 }
2435
2436 /* warning: addr must be aligned. The ram page is not masked as dirty
2437 and the code inside is not invalidated. It is useful if the dirty
2438 bits are used to track modified PTEs */
2439 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2440 {
2441 uint8_t *ptr;
2442 MemoryRegion *mr;
2443 hwaddr l = 4;
2444 hwaddr addr1;
2445
2446 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2447 true);
2448 if (l < 4 || !memory_access_is_direct(mr, true)) {
2449 io_mem_write(mr, addr1, val, 4);
2450 } else {
2451 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2452 ptr = qemu_get_ram_ptr(addr1);
2453 stl_p(ptr, val);
2454
2455 if (unlikely(in_migration)) {
2456 if (!cpu_physical_memory_is_dirty(addr1)) {
2457 /* invalidate code */
2458 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2459 /* set dirty bit */
2460 cpu_physical_memory_set_dirty_flags(
2461 addr1, (0xff & ~CODE_DIRTY_FLAG));
2462 }
2463 }
2464 }
2465 }
2466
2467 /* warning: addr must be aligned */
2468 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2469 enum device_endian endian)
2470 {
2471 uint8_t *ptr;
2472 MemoryRegion *mr;
2473 hwaddr l = 4;
2474 hwaddr addr1;
2475
2476 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2477 true);
2478 if (l < 4 || !memory_access_is_direct(mr, true)) {
2479 #if defined(TARGET_WORDS_BIGENDIAN)
2480 if (endian == DEVICE_LITTLE_ENDIAN) {
2481 val = bswap32(val);
2482 }
2483 #else
2484 if (endian == DEVICE_BIG_ENDIAN) {
2485 val = bswap32(val);
2486 }
2487 #endif
2488 io_mem_write(mr, addr1, val, 4);
2489 } else {
2490 /* RAM case */
2491 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2492 ptr = qemu_get_ram_ptr(addr1);
2493 switch (endian) {
2494 case DEVICE_LITTLE_ENDIAN:
2495 stl_le_p(ptr, val);
2496 break;
2497 case DEVICE_BIG_ENDIAN:
2498 stl_be_p(ptr, val);
2499 break;
2500 default:
2501 stl_p(ptr, val);
2502 break;
2503 }
2504 invalidate_and_set_dirty(addr1, 4);
2505 }
2506 }
2507
2508 void stl_phys(hwaddr addr, uint32_t val)
2509 {
2510 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2511 }
2512
2513 void stl_le_phys(hwaddr addr, uint32_t val)
2514 {
2515 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2516 }
2517
2518 void stl_be_phys(hwaddr addr, uint32_t val)
2519 {
2520 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2521 }
2522
2523 /* XXX: optimize */
2524 void stb_phys(hwaddr addr, uint32_t val)
2525 {
2526 uint8_t v = val;
2527 cpu_physical_memory_write(addr, &v, 1);
2528 }
2529
2530 /* warning: addr must be aligned */
2531 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2532 enum device_endian endian)
2533 {
2534 uint8_t *ptr;
2535 MemoryRegion *mr;
2536 hwaddr l = 2;
2537 hwaddr addr1;
2538
2539 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2540 true);
2541 if (l < 2 || !memory_access_is_direct(mr, true)) {
2542 #if defined(TARGET_WORDS_BIGENDIAN)
2543 if (endian == DEVICE_LITTLE_ENDIAN) {
2544 val = bswap16(val);
2545 }
2546 #else
2547 if (endian == DEVICE_BIG_ENDIAN) {
2548 val = bswap16(val);
2549 }
2550 #endif
2551 io_mem_write(mr, addr1, val, 2);
2552 } else {
2553 /* RAM case */
2554 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2555 ptr = qemu_get_ram_ptr(addr1);
2556 switch (endian) {
2557 case DEVICE_LITTLE_ENDIAN:
2558 stw_le_p(ptr, val);
2559 break;
2560 case DEVICE_BIG_ENDIAN:
2561 stw_be_p(ptr, val);
2562 break;
2563 default:
2564 stw_p(ptr, val);
2565 break;
2566 }
2567 invalidate_and_set_dirty(addr1, 2);
2568 }
2569 }
2570
2571 void stw_phys(hwaddr addr, uint32_t val)
2572 {
2573 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2574 }
2575
2576 void stw_le_phys(hwaddr addr, uint32_t val)
2577 {
2578 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2579 }
2580
2581 void stw_be_phys(hwaddr addr, uint32_t val)
2582 {
2583 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2584 }
2585
2586 /* XXX: optimize */
2587 void stq_phys(hwaddr addr, uint64_t val)
2588 {
2589 val = tswap64(val);
2590 cpu_physical_memory_write(addr, &val, 8);
2591 }
2592
2593 void stq_le_phys(hwaddr addr, uint64_t val)
2594 {
2595 val = cpu_to_le64(val);
2596 cpu_physical_memory_write(addr, &val, 8);
2597 }
2598
2599 void stq_be_phys(hwaddr addr, uint64_t val)
2600 {
2601 val = cpu_to_be64(val);
2602 cpu_physical_memory_write(addr, &val, 8);
2603 }
2604
2605 /* virtual memory access for debug (includes writing to ROM) */
2606 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2607 uint8_t *buf, int len, int is_write)
2608 {
2609 int l;
2610 hwaddr phys_addr;
2611 target_ulong page;
2612
2613 while (len > 0) {
2614 page = addr & TARGET_PAGE_MASK;
2615 phys_addr = cpu_get_phys_page_debug(cpu, page);
2616 /* if no physical page mapped, return an error */
2617 if (phys_addr == -1)
2618 return -1;
2619 l = (page + TARGET_PAGE_SIZE) - addr;
2620 if (l > len)
2621 l = len;
2622 phys_addr += (addr & ~TARGET_PAGE_MASK);
2623 if (is_write)
2624 cpu_physical_memory_write_rom(phys_addr, buf, l);
2625 else
2626 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2627 len -= l;
2628 buf += l;
2629 addr += l;
2630 }
2631 return 0;
2632 }
2633 #endif
2634
2635 #if !defined(CONFIG_USER_ONLY)
2636
2637 /*
2638 * A helper function for the _utterly broken_ virtio device model to find out if
2639 * it's running on a big endian machine. Don't do this at home kids!
2640 */
2641 bool virtio_is_big_endian(void);
2642 bool virtio_is_big_endian(void)
2643 {
2644 #if defined(TARGET_WORDS_BIGENDIAN)
2645 return true;
2646 #else
2647 return false;
2648 #endif
2649 }
2650
2651 #endif
2652
2653 #ifndef CONFIG_USER_ONLY
2654 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2655 {
2656 MemoryRegion*mr;
2657 hwaddr l = 1;
2658
2659 mr = address_space_translate(&address_space_memory,
2660 phys_addr, &phys_addr, &l, false);
2661
2662 return !(memory_region_is_ram(mr) ||
2663 memory_region_is_romd(mr));
2664 }
2665
2666 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2667 {
2668 RAMBlock *block;
2669
2670 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2671 func(block->host, block->offset, block->length, opaque);
2672 }
2673 }
2674 #endif
Qemu copy for testing