]> git.proxmox.com Git - qemu.git/blob - exec.c
projects / qemu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
cpu: Change cpu_single_step() argument to CPUState
[qemu.git] / exec.c
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
51
52 #include "exec/memory-internal.h"
53
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 CPUState *first_cpu;
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 typedef PhysPageEntry Node[L2_SIZE];
92
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101 };
102
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
110
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
115
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
124
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
127
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
133
134 static MemoryRegion io_mem_watch;
135 #endif
136
137 #if !defined(CONFIG_USER_ONLY)
138
139 static void phys_map_node_reserve(unsigned nodes)
140 {
141 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
142 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
143 16);
144 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
145 next_map.nodes_nb + nodes);
146 next_map.nodes = g_renew(Node, next_map.nodes,
147 next_map.nodes_nb_alloc);
148 }
149 }
150
151 static uint16_t phys_map_node_alloc(void)
152 {
153 unsigned i;
154 uint16_t ret;
155
156 ret = next_map.nodes_nb++;
157 assert(ret != PHYS_MAP_NODE_NIL);
158 assert(ret != next_map.nodes_nb_alloc);
159 for (i = 0; i < L2_SIZE; ++i) {
160 next_map.nodes[ret][i].is_leaf = 0;
161 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
162 }
163 return ret;
164 }
165
166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
167 hwaddr *nb, uint16_t leaf,
168 int level)
169 {
170 PhysPageEntry *p;
171 int i;
172 hwaddr step = (hwaddr)1 << (level * L2_BITS);
173
174 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
175 lp->ptr = phys_map_node_alloc();
176 p = next_map.nodes[lp->ptr];
177 if (level == 0) {
178 for (i = 0; i < L2_SIZE; i++) {
179 p[i].is_leaf = 1;
180 p[i].ptr = PHYS_SECTION_UNASSIGNED;
181 }
182 }
183 } else {
184 p = next_map.nodes[lp->ptr];
185 }
186 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
187
188 while (*nb && lp < &p[L2_SIZE]) {
189 if ((*index & (step - 1)) == 0 && *nb >= step) {
190 lp->is_leaf = true;
191 lp->ptr = leaf;
192 *index += step;
193 *nb -= step;
194 } else {
195 phys_page_set_level(lp, index, nb, leaf, level - 1);
196 }
197 ++lp;
198 }
199 }
200
201 static void phys_page_set(AddressSpaceDispatch *d,
202 hwaddr index, hwaddr nb,
203 uint16_t leaf)
204 {
205 /* Wildly overreserve - it doesn't matter much. */
206 phys_map_node_reserve(3 * P_L2_LEVELS);
207
208 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
209 }
210
211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
212 Node *nodes, MemoryRegionSection *sections)
213 {
214 PhysPageEntry *p;
215 int i;
216
217 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
218 if (lp.ptr == PHYS_MAP_NODE_NIL) {
219 return &sections[PHYS_SECTION_UNASSIGNED];
220 }
221 p = nodes[lp.ptr];
222 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
223 }
224 return &sections[lp.ptr];
225 }
226
227 bool memory_region_is_unassigned(MemoryRegion *mr)
228 {
229 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
230 && mr != &io_mem_watch;
231 }
232
233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
234 hwaddr addr,
235 bool resolve_subpage)
236 {
237 MemoryRegionSection *section;
238 subpage_t *subpage;
239
240 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
241 d->nodes, d->sections);
242 if (resolve_subpage && section->mr->subpage) {
243 subpage = container_of(section->mr, subpage_t, iomem);
244 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
245 }
246 return section;
247 }
248
249 static MemoryRegionSection *
250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
251 hwaddr *plen, bool resolve_subpage)
252 {
253 MemoryRegionSection *section;
254 Int128 diff;
255
256 section = address_space_lookup_region(d, addr, resolve_subpage);
257 /* Compute offset within MemoryRegionSection */
258 addr -= section->offset_within_address_space;
259
260 /* Compute offset within MemoryRegion */
261 *xlat = addr + section->offset_within_region;
262
263 diff = int128_sub(section->mr->size, int128_make64(addr));
264 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
265 return section;
266 }
267
268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
269 hwaddr *xlat, hwaddr *plen,
270 bool is_write)
271 {
272 IOMMUTLBEntry iotlb;
273 MemoryRegionSection *section;
274 MemoryRegion *mr;
275 hwaddr len = *plen;
276
277 for (;;) {
278 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
279 mr = section->mr;
280
281 if (!mr->iommu_ops) {
282 break;
283 }
284
285 iotlb = mr->iommu_ops->translate(mr, addr);
286 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
287 | (addr & iotlb.addr_mask));
288 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
289 if (!(iotlb.perm & (1 << is_write))) {
290 mr = &io_mem_unassigned;
291 break;
292 }
293
294 as = iotlb.target_as;
295 }
296
297 *plen = len;
298 *xlat = addr;
299 return mr;
300 }
301
302 MemoryRegionSection *
303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
304 hwaddr *plen)
305 {
306 MemoryRegionSection *section;
307 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
308
309 assert(!section->mr->iommu_ops);
310 return section;
311 }
312 #endif
313
314 void cpu_exec_init_all(void)
315 {
316 #if !defined(CONFIG_USER_ONLY)
317 qemu_mutex_init(&ram_list.mutex);
318 memory_map_init();
319 io_mem_init();
320 #endif
321 }
322
323 #if !defined(CONFIG_USER_ONLY)
324
325 static int cpu_common_post_load(void *opaque, int version_id)
326 {
327 CPUState *cpu = opaque;
328
329 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
330 version_id is increased. */
331 cpu->interrupt_request &= ~0x01;
332 tlb_flush(cpu->env_ptr, 1);
333
334 return 0;
335 }
336
337 const VMStateDescription vmstate_cpu_common = {
338 .name = "cpu_common",
339 .version_id = 1,
340 .minimum_version_id = 1,
341 .minimum_version_id_old = 1,
342 .post_load = cpu_common_post_load,
343 .fields = (VMStateField []) {
344 VMSTATE_UINT32(halted, CPUState),
345 VMSTATE_UINT32(interrupt_request, CPUState),
346 VMSTATE_END_OF_LIST()
347 }
348 };
349
350 #endif
351
352 CPUState *qemu_get_cpu(int index)
353 {
354 CPUState *cpu = first_cpu;
355
356 while (cpu) {
357 if (cpu->cpu_index == index) {
358 break;
359 }
360 cpu = cpu->next_cpu;
361 }
362
363 return cpu;
364 }
365
366 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
367 {
368 CPUState *cpu;
369
370 cpu = first_cpu;
371 while (cpu) {
372 func(cpu, data);
373 cpu = cpu->next_cpu;
374 }
375 }
376
377 void cpu_exec_init(CPUArchState *env)
378 {
379 CPUState *cpu = ENV_GET_CPU(env);
380 CPUClass *cc = CPU_GET_CLASS(cpu);
381 CPUState **pcpu;
382 int cpu_index;
383
384 #if defined(CONFIG_USER_ONLY)
385 cpu_list_lock();
386 #endif
387 cpu->next_cpu = NULL;
388 pcpu = &first_cpu;
389 cpu_index = 0;
390 while (*pcpu != NULL) {
391 pcpu = &(*pcpu)->next_cpu;
392 cpu_index++;
393 }
394 cpu->cpu_index = cpu_index;
395 cpu->numa_node = 0;
396 QTAILQ_INIT(&env->breakpoints);
397 QTAILQ_INIT(&env->watchpoints);
398 #ifndef CONFIG_USER_ONLY
399 cpu->thread_id = qemu_get_thread_id();
400 #endif
401 *pcpu = cpu;
402 #if defined(CONFIG_USER_ONLY)
403 cpu_list_unlock();
404 #endif
405 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
406 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
407 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
408 cpu_save, cpu_load, env);
409 assert(cc->vmsd == NULL);
410 #endif
411 if (cc->vmsd != NULL) {
412 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
413 }
414 }
415
416 #if defined(TARGET_HAS_ICE)
417 #if defined(CONFIG_USER_ONLY)
418 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
419 {
420 tb_invalidate_phys_page_range(pc, pc + 1, 0);
421 }
422 #else
423 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
424 {
425 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
426 (pc & ~TARGET_PAGE_MASK));
427 }
428 #endif
429 #endif /* TARGET_HAS_ICE */
430
431 #if defined(CONFIG_USER_ONLY)
432 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
433
434 {
435 }
436
437 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
438 int flags, CPUWatchpoint **watchpoint)
439 {
440 return -ENOSYS;
441 }
442 #else
443 /* Add a watchpoint. */
444 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
445 int flags, CPUWatchpoint **watchpoint)
446 {
447 target_ulong len_mask = ~(len - 1);
448 CPUWatchpoint *wp;
449
450 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
451 if ((len & (len - 1)) || (addr & ~len_mask) ||
452 len == 0 || len > TARGET_PAGE_SIZE) {
453 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
454 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
455 return -EINVAL;
456 }
457 wp = g_malloc(sizeof(*wp));
458
459 wp->vaddr = addr;
460 wp->len_mask = len_mask;
461 wp->flags = flags;
462
463 /* keep all GDB-injected watchpoints in front */
464 if (flags & BP_GDB)
465 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
466 else
467 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
468
469 tlb_flush_page(env, addr);
470
471 if (watchpoint)
472 *watchpoint = wp;
473 return 0;
474 }
475
476 /* Remove a specific watchpoint. */
477 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
478 int flags)
479 {
480 target_ulong len_mask = ~(len - 1);
481 CPUWatchpoint *wp;
482
483 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
484 if (addr == wp->vaddr && len_mask == wp->len_mask
485 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
486 cpu_watchpoint_remove_by_ref(env, wp);
487 return 0;
488 }
489 }
490 return -ENOENT;
491 }
492
493 /* Remove a specific watchpoint by reference. */
494 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
495 {
496 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
497
498 tlb_flush_page(env, watchpoint->vaddr);
499
500 g_free(watchpoint);
501 }
502
503 /* Remove all matching watchpoints. */
504 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
505 {
506 CPUWatchpoint *wp, *next;
507
508 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
509 if (wp->flags & mask)
510 cpu_watchpoint_remove_by_ref(env, wp);
511 }
512 }
513 #endif
514
515 /* Add a breakpoint. */
516 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
517 CPUBreakpoint **breakpoint)
518 {
519 #if defined(TARGET_HAS_ICE)
520 CPUBreakpoint *bp;
521
522 bp = g_malloc(sizeof(*bp));
523
524 bp->pc = pc;
525 bp->flags = flags;
526
527 /* keep all GDB-injected breakpoints in front */
528 if (flags & BP_GDB)
529 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
530 else
531 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
532
533 breakpoint_invalidate(env, pc);
534
535 if (breakpoint)
536 *breakpoint = bp;
537 return 0;
538 #else
539 return -ENOSYS;
540 #endif
541 }
542
543 /* Remove a specific breakpoint. */
544 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
545 {
546 #if defined(TARGET_HAS_ICE)
547 CPUBreakpoint *bp;
548
549 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
550 if (bp->pc == pc && bp->flags == flags) {
551 cpu_breakpoint_remove_by_ref(env, bp);
552 return 0;
553 }
554 }
555 return -ENOENT;
556 #else
557 return -ENOSYS;
558 #endif
559 }
560
561 /* Remove a specific breakpoint by reference. */
562 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
563 {
564 #if defined(TARGET_HAS_ICE)
565 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
566
567 breakpoint_invalidate(env, breakpoint->pc);
568
569 g_free(breakpoint);
570 #endif
571 }
572
573 /* Remove all matching breakpoints. */
574 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
575 {
576 #if defined(TARGET_HAS_ICE)
577 CPUBreakpoint *bp, *next;
578
579 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
580 if (bp->flags & mask)
581 cpu_breakpoint_remove_by_ref(env, bp);
582 }
583 #endif
584 }
585
586 /* enable or disable single step mode. EXCP_DEBUG is returned by the
587 CPU loop after each instruction */
588 void cpu_single_step(CPUState *cpu, int enabled)
589 {
590 #if defined(TARGET_HAS_ICE)
591 CPUArchState *env = cpu->env_ptr;
592
593 if (cpu->singlestep_enabled != enabled) {
594 cpu->singlestep_enabled = enabled;
595 if (kvm_enabled()) {
596 kvm_update_guest_debug(env, 0);
597 } else {
598 /* must flush all the translated code to avoid inconsistencies */
599 /* XXX: only flush what is necessary */
600 tb_flush(env);
601 }
602 }
603 #endif
604 }
605
606 void cpu_abort(CPUArchState *env, const char *fmt, ...)
607 {
608 CPUState *cpu = ENV_GET_CPU(env);
609 va_list ap;
610 va_list ap2;
611
612 va_start(ap, fmt);
613 va_copy(ap2, ap);
614 fprintf(stderr, "qemu: fatal: ");
615 vfprintf(stderr, fmt, ap);
616 fprintf(stderr, "\n");
617 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
618 if (qemu_log_enabled()) {
619 qemu_log("qemu: fatal: ");
620 qemu_log_vprintf(fmt, ap2);
621 qemu_log("\n");
622 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
623 qemu_log_flush();
624 qemu_log_close();
625 }
626 va_end(ap2);
627 va_end(ap);
628 #if defined(CONFIG_USER_ONLY)
629 {
630 struct sigaction act;
631 sigfillset(&act.sa_mask);
632 act.sa_handler = SIG_DFL;
633 sigaction(SIGABRT, &act, NULL);
634 }
635 #endif
636 abort();
637 }
638
639 CPUArchState *cpu_copy(CPUArchState *env)
640 {
641 CPUArchState *new_env = cpu_init(env->cpu_model_str);
642 #if defined(TARGET_HAS_ICE)
643 CPUBreakpoint *bp;
644 CPUWatchpoint *wp;
645 #endif
646
647 memcpy(new_env, env, sizeof(CPUArchState));
648
649 /* Clone all break/watchpoints.
650 Note: Once we support ptrace with hw-debug register access, make sure
651 BP_CPU break/watchpoints are handled correctly on clone. */
652 QTAILQ_INIT(&env->breakpoints);
653 QTAILQ_INIT(&env->watchpoints);
654 #if defined(TARGET_HAS_ICE)
655 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
656 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
657 }
658 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
659 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
660 wp->flags, NULL);
661 }
662 #endif
663
664 return new_env;
665 }
666
667 #if !defined(CONFIG_USER_ONLY)
668 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
669 uintptr_t length)
670 {
671 uintptr_t start1;
672
673 /* we modify the TLB cache so that the dirty bit will be set again
674 when accessing the range */
675 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
676 /* Check that we don't span multiple blocks - this breaks the
677 address comparisons below. */
678 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
679 != (end - 1) - start) {
680 abort();
681 }
682 cpu_tlb_reset_dirty_all(start1, length);
683
684 }
685
686 /* Note: start and end must be within the same ram block. */
687 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
688 int dirty_flags)
689 {
690 uintptr_t length;
691
692 start &= TARGET_PAGE_MASK;
693 end = TARGET_PAGE_ALIGN(end);
694
695 length = end - start;
696 if (length == 0)
697 return;
698 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
699
700 if (tcg_enabled()) {
701 tlb_reset_dirty_range_all(start, end, length);
702 }
703 }
704
705 static int cpu_physical_memory_set_dirty_tracking(int enable)
706 {
707 int ret = 0;
708 in_migration = enable;
709 return ret;
710 }
711
712 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
713 MemoryRegionSection *section,
714 target_ulong vaddr,
715 hwaddr paddr, hwaddr xlat,
716 int prot,
717 target_ulong *address)
718 {
719 hwaddr iotlb;
720 CPUWatchpoint *wp;
721
722 if (memory_region_is_ram(section->mr)) {
723 /* Normal RAM. */
724 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
725 + xlat;
726 if (!section->readonly) {
727 iotlb |= PHYS_SECTION_NOTDIRTY;
728 } else {
729 iotlb |= PHYS_SECTION_ROM;
730 }
731 } else {
732 iotlb = section - address_space_memory.dispatch->sections;
733 iotlb += xlat;
734 }
735
736 /* Make accesses to pages with watchpoints go via the
737 watchpoint trap routines. */
738 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
739 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
740 /* Avoid trapping reads of pages with a write breakpoint. */
741 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
742 iotlb = PHYS_SECTION_WATCH + paddr;
743 *address |= TLB_MMIO;
744 break;
745 }
746 }
747 }
748
749 return iotlb;
750 }
751 #endif /* defined(CONFIG_USER_ONLY) */
752
753 #if !defined(CONFIG_USER_ONLY)
754
755 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
756 uint16_t section);
757 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
758
759 static uint16_t phys_section_add(MemoryRegionSection *section)
760 {
761 /* The physical section number is ORed with a page-aligned
762 * pointer to produce the iotlb entries. Thus it should
763 * never overflow into the page-aligned value.
764 */
765 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
766
767 if (next_map.sections_nb == next_map.sections_nb_alloc) {
768 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
769 16);
770 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
771 next_map.sections_nb_alloc);
772 }
773 next_map.sections[next_map.sections_nb] = *section;
774 memory_region_ref(section->mr);
775 return next_map.sections_nb++;
776 }
777
778 static void phys_section_destroy(MemoryRegion *mr)
779 {
780 memory_region_unref(mr);
781
782 if (mr->subpage) {
783 subpage_t *subpage = container_of(mr, subpage_t, iomem);
784 memory_region_destroy(&subpage->iomem);
785 g_free(subpage);
786 }
787 }
788
789 static void phys_sections_free(PhysPageMap *map)
790 {
791 while (map->sections_nb > 0) {
792 MemoryRegionSection *section = &map->sections[--map->sections_nb];
793 phys_section_destroy(section->mr);
794 }
795 g_free(map->sections);
796 g_free(map->nodes);
797 g_free(map);
798 }
799
800 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
801 {
802 subpage_t *subpage;
803 hwaddr base = section->offset_within_address_space
804 & TARGET_PAGE_MASK;
805 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
806 next_map.nodes, next_map.sections);
807 MemoryRegionSection subsection = {
808 .offset_within_address_space = base,
809 .size = int128_make64(TARGET_PAGE_SIZE),
810 };
811 hwaddr start, end;
812
813 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
814
815 if (!(existing->mr->subpage)) {
816 subpage = subpage_init(d->as, base);
817 subsection.mr = &subpage->iomem;
818 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
819 phys_section_add(&subsection));
820 } else {
821 subpage = container_of(existing->mr, subpage_t, iomem);
822 }
823 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
824 end = start + int128_get64(section->size) - 1;
825 subpage_register(subpage, start, end, phys_section_add(section));
826 }
827
828
829 static void register_multipage(AddressSpaceDispatch *d,
830 MemoryRegionSection *section)
831 {
832 hwaddr start_addr = section->offset_within_address_space;
833 uint16_t section_index = phys_section_add(section);
834 uint64_t num_pages = int128_get64(int128_rshift(section->size,
835 TARGET_PAGE_BITS));
836
837 assert(num_pages);
838 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
839 }
840
841 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
842 {
843 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
844 AddressSpaceDispatch *d = as->next_dispatch;
845 MemoryRegionSection now = *section, remain = *section;
846 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
847
848 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
849 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
850 - now.offset_within_address_space;
851
852 now.size = int128_min(int128_make64(left), now.size);
853 register_subpage(d, &now);
854 } else {
855 now.size = int128_zero();
856 }
857 while (int128_ne(remain.size, now.size)) {
858 remain.size = int128_sub(remain.size, now.size);
859 remain.offset_within_address_space += int128_get64(now.size);
860 remain.offset_within_region += int128_get64(now.size);
861 now = remain;
862 if (int128_lt(remain.size, page_size)) {
863 register_subpage(d, &now);
864 } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
865 now.size = page_size;
866 register_subpage(d, &now);
867 } else {
868 now.size = int128_and(now.size, int128_neg(page_size));
869 register_multipage(d, &now);
870 }
871 }
872 }
873
874 void qemu_flush_coalesced_mmio_buffer(void)
875 {
876 if (kvm_enabled())
877 kvm_flush_coalesced_mmio_buffer();
878 }
879
880 void qemu_mutex_lock_ramlist(void)
881 {
882 qemu_mutex_lock(&ram_list.mutex);
883 }
884
885 void qemu_mutex_unlock_ramlist(void)
886 {
887 qemu_mutex_unlock(&ram_list.mutex);
888 }
889
890 #if defined(__linux__) && !defined(TARGET_S390X)
891
892 #include <sys/vfs.h>
893
894 #define HUGETLBFS_MAGIC 0x958458f6
895
896 static long gethugepagesize(const char *path)
897 {
898 struct statfs fs;
899 int ret;
900
901 do {
902 ret = statfs(path, &fs);
903 } while (ret != 0 && errno == EINTR);
904
905 if (ret != 0) {
906 perror(path);
907 return 0;
908 }
909
910 if (fs.f_type != HUGETLBFS_MAGIC)
911 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
912
913 return fs.f_bsize;
914 }
915
916 static void *file_ram_alloc(RAMBlock *block,
917 ram_addr_t memory,
918 const char *path)
919 {
920 char *filename;
921 char *sanitized_name;
922 char *c;
923 void *area;
924 int fd;
925 #ifdef MAP_POPULATE
926 int flags;
927 #endif
928 unsigned long hpagesize;
929
930 hpagesize = gethugepagesize(path);
931 if (!hpagesize) {
932 return NULL;
933 }
934
935 if (memory < hpagesize) {
936 return NULL;
937 }
938
939 if (kvm_enabled() && !kvm_has_sync_mmu()) {
940 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
941 return NULL;
942 }
943
944 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
945 sanitized_name = g_strdup(block->mr->name);
946 for (c = sanitized_name; *c != '\0'; c++) {
947 if (*c == '/')
948 *c = '_';
949 }
950
951 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
952 sanitized_name);
953 g_free(sanitized_name);
954
955 fd = mkstemp(filename);
956 if (fd < 0) {
957 perror("unable to create backing store for hugepages");
958 g_free(filename);
959 return NULL;
960 }
961 unlink(filename);
962 g_free(filename);
963
964 memory = (memory+hpagesize-1) & ~(hpagesize-1);
965
966 /*
967 * ftruncate is not supported by hugetlbfs in older
968 * hosts, so don't bother bailing out on errors.
969 * If anything goes wrong with it under other filesystems,
970 * mmap will fail.
971 */
972 if (ftruncate(fd, memory))
973 perror("ftruncate");
974
975 #ifdef MAP_POPULATE
976 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
977 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
978 * to sidestep this quirk.
979 */
980 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
981 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
982 #else
983 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
984 #endif
985 if (area == MAP_FAILED) {
986 perror("file_ram_alloc: can't mmap RAM pages");
987 close(fd);
988 return (NULL);
989 }
990 block->fd = fd;
991 return area;
992 }
993 #endif
994
995 static ram_addr_t find_ram_offset(ram_addr_t size)
996 {
997 RAMBlock *block, *next_block;
998 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
999
1000 assert(size != 0); /* it would hand out same offset multiple times */
1001
1002 if (QTAILQ_EMPTY(&ram_list.blocks))
1003 return 0;
1004
1005 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1006 ram_addr_t end, next = RAM_ADDR_MAX;
1007
1008 end = block->offset + block->length;
1009
1010 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1011 if (next_block->offset >= end) {
1012 next = MIN(next, next_block->offset);
1013 }
1014 }
1015 if (next - end >= size && next - end < mingap) {
1016 offset = end;
1017 mingap = next - end;
1018 }
1019 }
1020
1021 if (offset == RAM_ADDR_MAX) {
1022 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1023 (uint64_t)size);
1024 abort();
1025 }
1026
1027 return offset;
1028 }
1029
1030 ram_addr_t last_ram_offset(void)
1031 {
1032 RAMBlock *block;
1033 ram_addr_t last = 0;
1034
1035 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1036 last = MAX(last, block->offset + block->length);
1037
1038 return last;
1039 }
1040
1041 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1042 {
1043 int ret;
1044
1045 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1046 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1047 "dump-guest-core", true)) {
1048 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1049 if (ret) {
1050 perror("qemu_madvise");
1051 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1052 "but dump_guest_core=off specified\n");
1053 }
1054 }
1055 }
1056
1057 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1058 {
1059 RAMBlock *new_block, *block;
1060
1061 new_block = NULL;
1062 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1063 if (block->offset == addr) {
1064 new_block = block;
1065 break;
1066 }
1067 }
1068 assert(new_block);
1069 assert(!new_block->idstr[0]);
1070
1071 if (dev) {
1072 char *id = qdev_get_dev_path(dev);
1073 if (id) {
1074 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1075 g_free(id);
1076 }
1077 }
1078 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1079
1080 /* This assumes the iothread lock is taken here too. */
1081 qemu_mutex_lock_ramlist();
1082 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1083 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1084 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1085 new_block->idstr);
1086 abort();
1087 }
1088 }
1089 qemu_mutex_unlock_ramlist();
1090 }
1091
1092 static int memory_try_enable_merging(void *addr, size_t len)
1093 {
1094 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1095 /* disabled by the user */
1096 return 0;
1097 }
1098
1099 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1100 }
1101
1102 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1103 MemoryRegion *mr)
1104 {
1105 RAMBlock *block, *new_block;
1106
1107 size = TARGET_PAGE_ALIGN(size);
1108 new_block = g_malloc0(sizeof(*new_block));
1109
1110 /* This assumes the iothread lock is taken here too. */
1111 qemu_mutex_lock_ramlist();
1112 new_block->mr = mr;
1113 new_block->offset = find_ram_offset(size);
1114 if (host) {
1115 new_block->host = host;
1116 new_block->flags |= RAM_PREALLOC_MASK;
1117 } else {
1118 if (mem_path) {
1119 #if defined (__linux__) && !defined(TARGET_S390X)
1120 new_block->host = file_ram_alloc(new_block, size, mem_path);
1121 if (!new_block->host) {
1122 new_block->host = qemu_anon_ram_alloc(size);
1123 memory_try_enable_merging(new_block->host, size);
1124 }
1125 #else
1126 fprintf(stderr, "-mem-path option unsupported\n");
1127 exit(1);
1128 #endif
1129 } else {
1130 if (xen_enabled()) {
1131 xen_ram_alloc(new_block->offset, size, mr);
1132 } else if (kvm_enabled()) {
1133 /* some s390/kvm configurations have special constraints */
1134 new_block->host = kvm_ram_alloc(size);
1135 } else {
1136 new_block->host = qemu_anon_ram_alloc(size);
1137 }
1138 memory_try_enable_merging(new_block->host, size);
1139 }
1140 }
1141 new_block->length = size;
1142
1143 /* Keep the list sorted from biggest to smallest block. */
1144 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1145 if (block->length < new_block->length) {
1146 break;
1147 }
1148 }
1149 if (block) {
1150 QTAILQ_INSERT_BEFORE(block, new_block, next);
1151 } else {
1152 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1153 }
1154 ram_list.mru_block = NULL;
1155
1156 ram_list.version++;
1157 qemu_mutex_unlock_ramlist();
1158
1159 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1160 last_ram_offset() >> TARGET_PAGE_BITS);
1161 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1162 0, size >> TARGET_PAGE_BITS);
1163 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1164
1165 qemu_ram_setup_dump(new_block->host, size);
1166 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1167
1168 if (kvm_enabled())
1169 kvm_setup_guest_memory(new_block->host, size);
1170
1171 return new_block->offset;
1172 }
1173
1174 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1175 {
1176 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1177 }
1178
1179 void qemu_ram_free_from_ptr(ram_addr_t addr)
1180 {
1181 RAMBlock *block;
1182
1183 /* This assumes the iothread lock is taken here too. */
1184 qemu_mutex_lock_ramlist();
1185 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1186 if (addr == block->offset) {
1187 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1188 ram_list.mru_block = NULL;
1189 ram_list.version++;
1190 g_free(block);
1191 break;
1192 }
1193 }
1194 qemu_mutex_unlock_ramlist();
1195 }
1196
1197 void qemu_ram_free(ram_addr_t addr)
1198 {
1199 RAMBlock *block;
1200
1201 /* This assumes the iothread lock is taken here too. */
1202 qemu_mutex_lock_ramlist();
1203 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1204 if (addr == block->offset) {
1205 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1206 ram_list.mru_block = NULL;
1207 ram_list.version++;
1208 if (block->flags & RAM_PREALLOC_MASK) {
1209 ;
1210 } else if (mem_path) {
1211 #if defined (__linux__) && !defined(TARGET_S390X)
1212 if (block->fd) {
1213 munmap(block->host, block->length);
1214 close(block->fd);
1215 } else {
1216 qemu_anon_ram_free(block->host, block->length);
1217 }
1218 #else
1219 abort();
1220 #endif
1221 } else {
1222 if (xen_enabled()) {
1223 xen_invalidate_map_cache_entry(block->host);
1224 } else {
1225 qemu_anon_ram_free(block->host, block->length);
1226 }
1227 }
1228 g_free(block);
1229 break;
1230 }
1231 }
1232 qemu_mutex_unlock_ramlist();
1233
1234 }
1235
1236 #ifndef _WIN32
1237 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1238 {
1239 RAMBlock *block;
1240 ram_addr_t offset;
1241 int flags;
1242 void *area, *vaddr;
1243
1244 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1245 offset = addr - block->offset;
1246 if (offset < block->length) {
1247 vaddr = block->host + offset;
1248 if (block->flags & RAM_PREALLOC_MASK) {
1249 ;
1250 } else {
1251 flags = MAP_FIXED;
1252 munmap(vaddr, length);
1253 if (mem_path) {
1254 #if defined(__linux__) && !defined(TARGET_S390X)
1255 if (block->fd) {
1256 #ifdef MAP_POPULATE
1257 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1258 MAP_PRIVATE;
1259 #else
1260 flags |= MAP_PRIVATE;
1261 #endif
1262 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1263 flags, block->fd, offset);
1264 } else {
1265 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1266 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1267 flags, -1, 0);
1268 }
1269 #else
1270 abort();
1271 #endif
1272 } else {
1273 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1274 flags |= MAP_SHARED | MAP_ANONYMOUS;
1275 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1276 flags, -1, 0);
1277 #else
1278 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1279 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1280 flags, -1, 0);
1281 #endif
1282 }
1283 if (area != vaddr) {
1284 fprintf(stderr, "Could not remap addr: "
1285 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1286 length, addr);
1287 exit(1);
1288 }
1289 memory_try_enable_merging(vaddr, length);
1290 qemu_ram_setup_dump(vaddr, length);
1291 }
1292 return;
1293 }
1294 }
1295 }
1296 #endif /* !_WIN32 */
1297
1298 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1299 {
1300 RAMBlock *block;
1301
1302 /* The list is protected by the iothread lock here. */
1303 block = ram_list.mru_block;
1304 if (block && addr - block->offset < block->length) {
1305 goto found;
1306 }
1307 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1308 if (addr - block->offset < block->length) {
1309 goto found;
1310 }
1311 }
1312
1313 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1314 abort();
1315
1316 found:
1317 ram_list.mru_block = block;
1318 return block;
1319 }
1320
1321 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1322 With the exception of the softmmu code in this file, this should
1323 only be used for local memory (e.g. video ram) that the device owns,
1324 and knows it isn't going to access beyond the end of the block.
1325
1326 It should not be used for general purpose DMA.
1327 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1328 */
1329 void *qemu_get_ram_ptr(ram_addr_t addr)
1330 {
1331 RAMBlock *block = qemu_get_ram_block(addr);
1332
1333 if (xen_enabled()) {
1334 /* We need to check if the requested address is in the RAM
1335 * because we don't want to map the entire memory in QEMU.
1336 * In that case just map until the end of the page.
1337 */
1338 if (block->offset == 0) {
1339 return xen_map_cache(addr, 0, 0);
1340 } else if (block->host == NULL) {
1341 block->host =
1342 xen_map_cache(block->offset, block->length, 1);
1343 }
1344 }
1345 return block->host + (addr - block->offset);
1346 }
1347
1348 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1349 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1350 *
1351 * ??? Is this still necessary?
1352 */
1353 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1354 {
1355 RAMBlock *block;
1356
1357 /* The list is protected by the iothread lock here. */
1358 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1359 if (addr - block->offset < block->length) {
1360 if (xen_enabled()) {
1361 /* We need to check if the requested address is in the RAM
1362 * because we don't want to map the entire memory in QEMU.
1363 * In that case just map until the end of the page.
1364 */
1365 if (block->offset == 0) {
1366 return xen_map_cache(addr, 0, 0);
1367 } else if (block->host == NULL) {
1368 block->host =
1369 xen_map_cache(block->offset, block->length, 1);
1370 }
1371 }
1372 return block->host + (addr - block->offset);
1373 }
1374 }
1375
1376 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1377 abort();
1378
1379 return NULL;
1380 }
1381
1382 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1383 * but takes a size argument */
1384 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1385 {
1386 if (*size == 0) {
1387 return NULL;
1388 }
1389 if (xen_enabled()) {
1390 return xen_map_cache(addr, *size, 1);
1391 } else {
1392 RAMBlock *block;
1393
1394 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1395 if (addr - block->offset < block->length) {
1396 if (addr - block->offset + *size > block->length)
1397 *size = block->length - addr + block->offset;
1398 return block->host + (addr - block->offset);
1399 }
1400 }
1401
1402 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1403 abort();
1404 }
1405 }
1406
1407 /* Some of the softmmu routines need to translate from a host pointer
1408 (typically a TLB entry) back to a ram offset. */
1409 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1410 {
1411 RAMBlock *block;
1412 uint8_t *host = ptr;
1413
1414 if (xen_enabled()) {
1415 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1416 return qemu_get_ram_block(*ram_addr)->mr;
1417 }
1418
1419 block = ram_list.mru_block;
1420 if (block && block->host && host - block->host < block->length) {
1421 goto found;
1422 }
1423
1424 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1425 /* This case append when the block is not mapped. */
1426 if (block->host == NULL) {
1427 continue;
1428 }
1429 if (host - block->host < block->length) {
1430 goto found;
1431 }
1432 }
1433
1434 return NULL;
1435
1436 found:
1437 *ram_addr = block->offset + (host - block->host);
1438 return block->mr;
1439 }
1440
1441 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1442 uint64_t val, unsigned size)
1443 {
1444 int dirty_flags;
1445 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1446 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1447 tb_invalidate_phys_page_fast(ram_addr, size);
1448 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1449 }
1450 switch (size) {
1451 case 1:
1452 stb_p(qemu_get_ram_ptr(ram_addr), val);
1453 break;
1454 case 2:
1455 stw_p(qemu_get_ram_ptr(ram_addr), val);
1456 break;
1457 case 4:
1458 stl_p(qemu_get_ram_ptr(ram_addr), val);
1459 break;
1460 default:
1461 abort();
1462 }
1463 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1464 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1465 /* we remove the notdirty callback only if the code has been
1466 flushed */
1467 if (dirty_flags == 0xff) {
1468 CPUArchState *env = current_cpu->env_ptr;
1469 tlb_set_dirty(env, env->mem_io_vaddr);
1470 }
1471 }
1472
1473 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1474 unsigned size, bool is_write)
1475 {
1476 return is_write;
1477 }
1478
1479 static const MemoryRegionOps notdirty_mem_ops = {
1480 .write = notdirty_mem_write,
1481 .valid.accepts = notdirty_mem_accepts,
1482 .endianness = DEVICE_NATIVE_ENDIAN,
1483 };
1484
1485 /* Generate a debug exception if a watchpoint has been hit. */
1486 static void check_watchpoint(int offset, int len_mask, int flags)
1487 {
1488 CPUArchState *env = current_cpu->env_ptr;
1489 target_ulong pc, cs_base;
1490 target_ulong vaddr;
1491 CPUWatchpoint *wp;
1492 int cpu_flags;
1493
1494 if (env->watchpoint_hit) {
1495 /* We re-entered the check after replacing the TB. Now raise
1496 * the debug interrupt so that is will trigger after the
1497 * current instruction. */
1498 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1499 return;
1500 }
1501 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1502 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1503 if ((vaddr == (wp->vaddr & len_mask) ||
1504 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1505 wp->flags |= BP_WATCHPOINT_HIT;
1506 if (!env->watchpoint_hit) {
1507 env->watchpoint_hit = wp;
1508 tb_check_watchpoint(env);
1509 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1510 env->exception_index = EXCP_DEBUG;
1511 cpu_loop_exit(env);
1512 } else {
1513 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1514 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1515 cpu_resume_from_signal(env, NULL);
1516 }
1517 }
1518 } else {
1519 wp->flags &= ~BP_WATCHPOINT_HIT;
1520 }
1521 }
1522 }
1523
1524 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1525 so these check for a hit then pass through to the normal out-of-line
1526 phys routines. */
1527 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1528 unsigned size)
1529 {
1530 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1531 switch (size) {
1532 case 1: return ldub_phys(addr);
1533 case 2: return lduw_phys(addr);
1534 case 4: return ldl_phys(addr);
1535 default: abort();
1536 }
1537 }
1538
1539 static void watch_mem_write(void *opaque, hwaddr addr,
1540 uint64_t val, unsigned size)
1541 {
1542 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1543 switch (size) {
1544 case 1:
1545 stb_phys(addr, val);
1546 break;
1547 case 2:
1548 stw_phys(addr, val);
1549 break;
1550 case 4:
1551 stl_phys(addr, val);
1552 break;
1553 default: abort();
1554 }
1555 }
1556
1557 static const MemoryRegionOps watch_mem_ops = {
1558 .read = watch_mem_read,
1559 .write = watch_mem_write,
1560 .endianness = DEVICE_NATIVE_ENDIAN,
1561 };
1562
1563 static uint64_t subpage_read(void *opaque, hwaddr addr,
1564 unsigned len)
1565 {
1566 subpage_t *subpage = opaque;
1567 uint8_t buf[4];
1568
1569 #if defined(DEBUG_SUBPAGE)
1570 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1571 subpage, len, addr);
1572 #endif
1573 address_space_read(subpage->as, addr + subpage->base, buf, len);
1574 switch (len) {
1575 case 1:
1576 return ldub_p(buf);
1577 case 2:
1578 return lduw_p(buf);
1579 case 4:
1580 return ldl_p(buf);
1581 default:
1582 abort();
1583 }
1584 }
1585
1586 static void subpage_write(void *opaque, hwaddr addr,
1587 uint64_t value, unsigned len)
1588 {
1589 subpage_t *subpage = opaque;
1590 uint8_t buf[4];
1591
1592 #if defined(DEBUG_SUBPAGE)
1593 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1594 " value %"PRIx64"\n",
1595 __func__, subpage, len, addr, value);
1596 #endif
1597 switch (len) {
1598 case 1:
1599 stb_p(buf, value);
1600 break;
1601 case 2:
1602 stw_p(buf, value);
1603 break;
1604 case 4:
1605 stl_p(buf, value);
1606 break;
1607 default:
1608 abort();
1609 }
1610 address_space_write(subpage->as, addr + subpage->base, buf, len);
1611 }
1612
1613 static bool subpage_accepts(void *opaque, hwaddr addr,
1614 unsigned size, bool is_write)
1615 {
1616 subpage_t *subpage = opaque;
1617 #if defined(DEBUG_SUBPAGE)
1618 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1619 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1620 #endif
1621
1622 return address_space_access_valid(subpage->as, addr + subpage->base,
1623 size, is_write);
1624 }
1625
1626 static const MemoryRegionOps subpage_ops = {
1627 .read = subpage_read,
1628 .write = subpage_write,
1629 .valid.accepts = subpage_accepts,
1630 .endianness = DEVICE_NATIVE_ENDIAN,
1631 };
1632
1633 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1634 uint16_t section)
1635 {
1636 int idx, eidx;
1637
1638 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1639 return -1;
1640 idx = SUBPAGE_IDX(start);
1641 eidx = SUBPAGE_IDX(end);
1642 #if defined(DEBUG_SUBPAGE)
1643 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1644 mmio, start, end, idx, eidx, memory);
1645 #endif
1646 for (; idx <= eidx; idx++) {
1647 mmio->sub_section[idx] = section;
1648 }
1649
1650 return 0;
1651 }
1652
1653 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1654 {
1655 subpage_t *mmio;
1656
1657 mmio = g_malloc0(sizeof(subpage_t));
1658
1659 mmio->as = as;
1660 mmio->base = base;
1661 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1662 "subpage", TARGET_PAGE_SIZE);
1663 mmio->iomem.subpage = true;
1664 #if defined(DEBUG_SUBPAGE)
1665 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1666 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1667 #endif
1668 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1669
1670 return mmio;
1671 }
1672
1673 static uint16_t dummy_section(MemoryRegion *mr)
1674 {
1675 MemoryRegionSection section = {
1676 .mr = mr,
1677 .offset_within_address_space = 0,
1678 .offset_within_region = 0,
1679 .size = int128_2_64(),
1680 };
1681
1682 return phys_section_add(&section);
1683 }
1684
1685 MemoryRegion *iotlb_to_region(hwaddr index)
1686 {
1687 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1688 }
1689
1690 static void io_mem_init(void)
1691 {
1692 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1693 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1694 "unassigned", UINT64_MAX);
1695 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1696 "notdirty", UINT64_MAX);
1697 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1698 "watch", UINT64_MAX);
1699 }
1700
1701 static void mem_begin(MemoryListener *listener)
1702 {
1703 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1704 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1705
1706 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1707 d->as = as;
1708 as->next_dispatch = d;
1709 }
1710
1711 static void mem_commit(MemoryListener *listener)
1712 {
1713 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1714 AddressSpaceDispatch *cur = as->dispatch;
1715 AddressSpaceDispatch *next = as->next_dispatch;
1716
1717 next->nodes = next_map.nodes;
1718 next->sections = next_map.sections;
1719
1720 as->dispatch = next;
1721 g_free(cur);
1722 }
1723
1724 static void core_begin(MemoryListener *listener)
1725 {
1726 uint16_t n;
1727
1728 prev_map = g_new(PhysPageMap, 1);
1729 *prev_map = next_map;
1730
1731 memset(&next_map, 0, sizeof(next_map));
1732 n = dummy_section(&io_mem_unassigned);
1733 assert(n == PHYS_SECTION_UNASSIGNED);
1734 n = dummy_section(&io_mem_notdirty);
1735 assert(n == PHYS_SECTION_NOTDIRTY);
1736 n = dummy_section(&io_mem_rom);
1737 assert(n == PHYS_SECTION_ROM);
1738 n = dummy_section(&io_mem_watch);
1739 assert(n == PHYS_SECTION_WATCH);
1740 }
1741
1742 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1743 * All AddressSpaceDispatch instances have switched to the next map.
1744 */
1745 static void core_commit(MemoryListener *listener)
1746 {
1747 phys_sections_free(prev_map);
1748 }
1749
1750 static void tcg_commit(MemoryListener *listener)
1751 {
1752 CPUState *cpu;
1753
1754 /* since each CPU stores ram addresses in its TLB cache, we must
1755 reset the modified entries */
1756 /* XXX: slow ! */
1757 for (cpu = first_cpu; cpu != NULL; cpu = cpu->next_cpu) {
1758 CPUArchState *env = cpu->env_ptr;
1759
1760 tlb_flush(env, 1);
1761 }
1762 }
1763
1764 static void core_log_global_start(MemoryListener *listener)
1765 {
1766 cpu_physical_memory_set_dirty_tracking(1);
1767 }
1768
1769 static void core_log_global_stop(MemoryListener *listener)
1770 {
1771 cpu_physical_memory_set_dirty_tracking(0);
1772 }
1773
1774 static MemoryListener core_memory_listener = {
1775 .begin = core_begin,
1776 .commit = core_commit,
1777 .log_global_start = core_log_global_start,
1778 .log_global_stop = core_log_global_stop,
1779 .priority = 1,
1780 };
1781
1782 static MemoryListener tcg_memory_listener = {
1783 .commit = tcg_commit,
1784 };
1785
1786 void address_space_init_dispatch(AddressSpace *as)
1787 {
1788 as->dispatch = NULL;
1789 as->dispatch_listener = (MemoryListener) {
1790 .begin = mem_begin,
1791 .commit = mem_commit,
1792 .region_add = mem_add,
1793 .region_nop = mem_add,
1794 .priority = 0,
1795 };
1796 memory_listener_register(&as->dispatch_listener, as);
1797 }
1798
1799 void address_space_destroy_dispatch(AddressSpace *as)
1800 {
1801 AddressSpaceDispatch *d = as->dispatch;
1802
1803 memory_listener_unregister(&as->dispatch_listener);
1804 g_free(d);
1805 as->dispatch = NULL;
1806 }
1807
1808 static void memory_map_init(void)
1809 {
1810 system_memory = g_malloc(sizeof(*system_memory));
1811 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1812 address_space_init(&address_space_memory, system_memory, "memory");
1813
1814 system_io = g_malloc(sizeof(*system_io));
1815 memory_region_init(system_io, NULL, "io", 65536);
1816 address_space_init(&address_space_io, system_io, "I/O");
1817
1818 memory_listener_register(&core_memory_listener, &address_space_memory);
1819 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1820 }
1821
1822 MemoryRegion *get_system_memory(void)
1823 {
1824 return system_memory;
1825 }
1826
1827 MemoryRegion *get_system_io(void)
1828 {
1829 return system_io;
1830 }
1831
1832 #endif /* !defined(CONFIG_USER_ONLY) */
1833
1834 /* physical memory access (slow version, mainly for debug) */
1835 #if defined(CONFIG_USER_ONLY)
1836 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1837 uint8_t *buf, int len, int is_write)
1838 {
1839 int l, flags;
1840 target_ulong page;
1841 void * p;
1842
1843 while (len > 0) {
1844 page = addr & TARGET_PAGE_MASK;
1845 l = (page + TARGET_PAGE_SIZE) - addr;
1846 if (l > len)
1847 l = len;
1848 flags = page_get_flags(page);
1849 if (!(flags & PAGE_VALID))
1850 return -1;
1851 if (is_write) {
1852 if (!(flags & PAGE_WRITE))
1853 return -1;
1854 /* XXX: this code should not depend on lock_user */
1855 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1856 return -1;
1857 memcpy(p, buf, l);
1858 unlock_user(p, addr, l);
1859 } else {
1860 if (!(flags & PAGE_READ))
1861 return -1;
1862 /* XXX: this code should not depend on lock_user */
1863 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1864 return -1;
1865 memcpy(buf, p, l);
1866 unlock_user(p, addr, 0);
1867 }
1868 len -= l;
1869 buf += l;
1870 addr += l;
1871 }
1872 return 0;
1873 }
1874
1875 #else
1876
1877 static void invalidate_and_set_dirty(hwaddr addr,
1878 hwaddr length)
1879 {
1880 if (!cpu_physical_memory_is_dirty(addr)) {
1881 /* invalidate code */
1882 tb_invalidate_phys_page_range(addr, addr + length, 0);
1883 /* set dirty bit */
1884 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1885 }
1886 xen_modified_memory(addr, length);
1887 }
1888
1889 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1890 {
1891 if (memory_region_is_ram(mr)) {
1892 return !(is_write && mr->readonly);
1893 }
1894 if (memory_region_is_romd(mr)) {
1895 return !is_write;
1896 }
1897
1898 return false;
1899 }
1900
1901 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1902 {
1903 unsigned access_size_max = mr->ops->valid.max_access_size;
1904
1905 /* Regions are assumed to support 1-4 byte accesses unless
1906 otherwise specified. */
1907 if (access_size_max == 0) {
1908 access_size_max = 4;
1909 }
1910
1911 /* Bound the maximum access by the alignment of the address. */
1912 if (!mr->ops->impl.unaligned) {
1913 unsigned align_size_max = addr & -addr;
1914 if (align_size_max != 0 && align_size_max < access_size_max) {
1915 access_size_max = align_size_max;
1916 }
1917 }
1918
1919 /* Don't attempt accesses larger than the maximum. */
1920 if (l > access_size_max) {
1921 l = access_size_max;
1922 }
1923
1924 return l;
1925 }
1926
1927 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1928 int len, bool is_write)
1929 {
1930 hwaddr l;
1931 uint8_t *ptr;
1932 uint64_t val;
1933 hwaddr addr1;
1934 MemoryRegion *mr;
1935 bool error = false;
1936
1937 while (len > 0) {
1938 l = len;
1939 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1940
1941 if (is_write) {
1942 if (!memory_access_is_direct(mr, is_write)) {
1943 l = memory_access_size(mr, l, addr1);
1944 /* XXX: could force current_cpu to NULL to avoid
1945 potential bugs */
1946 switch (l) {
1947 case 8:
1948 /* 64 bit write access */
1949 val = ldq_p(buf);
1950 error |= io_mem_write(mr, addr1, val, 8);
1951 break;
1952 case 4:
1953 /* 32 bit write access */
1954 val = ldl_p(buf);
1955 error |= io_mem_write(mr, addr1, val, 4);
1956 break;
1957 case 2:
1958 /* 16 bit write access */
1959 val = lduw_p(buf);
1960 error |= io_mem_write(mr, addr1, val, 2);
1961 break;
1962 case 1:
1963 /* 8 bit write access */
1964 val = ldub_p(buf);
1965 error |= io_mem_write(mr, addr1, val, 1);
1966 break;
1967 default:
1968 abort();
1969 }
1970 } else {
1971 addr1 += memory_region_get_ram_addr(mr);
1972 /* RAM case */
1973 ptr = qemu_get_ram_ptr(addr1);
1974 memcpy(ptr, buf, l);
1975 invalidate_and_set_dirty(addr1, l);
1976 }
1977 } else {
1978 if (!memory_access_is_direct(mr, is_write)) {
1979 /* I/O case */
1980 l = memory_access_size(mr, l, addr1);
1981 switch (l) {
1982 case 8:
1983 /* 64 bit read access */
1984 error |= io_mem_read(mr, addr1, &val, 8);
1985 stq_p(buf, val);
1986 break;
1987 case 4:
1988 /* 32 bit read access */
1989 error |= io_mem_read(mr, addr1, &val, 4);
1990 stl_p(buf, val);
1991 break;
1992 case 2:
1993 /* 16 bit read access */
1994 error |= io_mem_read(mr, addr1, &val, 2);
1995 stw_p(buf, val);
1996 break;
1997 case 1:
1998 /* 8 bit read access */
1999 error |= io_mem_read(mr, addr1, &val, 1);
2000 stb_p(buf, val);
2001 break;
2002 default:
2003 abort();
2004 }
2005 } else {
2006 /* RAM case */
2007 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2008 memcpy(buf, ptr, l);
2009 }
2010 }
2011 len -= l;
2012 buf += l;
2013 addr += l;
2014 }
2015
2016 return error;
2017 }
2018
2019 bool address_space_write(AddressSpace *as, hwaddr addr,
2020 const uint8_t *buf, int len)
2021 {
2022 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2023 }
2024
2025 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2026 {
2027 return address_space_rw(as, addr, buf, len, false);
2028 }
2029
2030
2031 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2032 int len, int is_write)
2033 {
2034 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2035 }
2036
2037 /* used for ROM loading : can write in RAM and ROM */
2038 void cpu_physical_memory_write_rom(hwaddr addr,
2039 const uint8_t *buf, int len)
2040 {
2041 hwaddr l;
2042 uint8_t *ptr;
2043 hwaddr addr1;
2044 MemoryRegion *mr;
2045
2046 while (len > 0) {
2047 l = len;
2048 mr = address_space_translate(&address_space_memory,
2049 addr, &addr1, &l, true);
2050
2051 if (!(memory_region_is_ram(mr) ||
2052 memory_region_is_romd(mr))) {
2053 /* do nothing */
2054 } else {
2055 addr1 += memory_region_get_ram_addr(mr);
2056 /* ROM/RAM case */
2057 ptr = qemu_get_ram_ptr(addr1);
2058 memcpy(ptr, buf, l);
2059 invalidate_and_set_dirty(addr1, l);
2060 }
2061 len -= l;
2062 buf += l;
2063 addr += l;
2064 }
2065 }
2066
2067 typedef struct {
2068 MemoryRegion *mr;
2069 void *buffer;
2070 hwaddr addr;
2071 hwaddr len;
2072 } BounceBuffer;
2073
2074 static BounceBuffer bounce;
2075
2076 typedef struct MapClient {
2077 void *opaque;
2078 void (*callback)(void *opaque);
2079 QLIST_ENTRY(MapClient) link;
2080 } MapClient;
2081
2082 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2083 = QLIST_HEAD_INITIALIZER(map_client_list);
2084
2085 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2086 {
2087 MapClient *client = g_malloc(sizeof(*client));
2088
2089 client->opaque = opaque;
2090 client->callback = callback;
2091 QLIST_INSERT_HEAD(&map_client_list, client, link);
2092 return client;
2093 }
2094
2095 static void cpu_unregister_map_client(void *_client)
2096 {
2097 MapClient *client = (MapClient *)_client;
2098
2099 QLIST_REMOVE(client, link);
2100 g_free(client);
2101 }
2102
2103 static void cpu_notify_map_clients(void)
2104 {
2105 MapClient *client;
2106
2107 while (!QLIST_EMPTY(&map_client_list)) {
2108 client = QLIST_FIRST(&map_client_list);
2109 client->callback(client->opaque);
2110 cpu_unregister_map_client(client);
2111 }
2112 }
2113
2114 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2115 {
2116 MemoryRegion *mr;
2117 hwaddr l, xlat;
2118
2119 while (len > 0) {
2120 l = len;
2121 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2122 if (!memory_access_is_direct(mr, is_write)) {
2123 l = memory_access_size(mr, l, addr);
2124 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2125 return false;
2126 }
2127 }
2128
2129 len -= l;
2130 addr += l;
2131 }
2132 return true;
2133 }
2134
2135 /* Map a physical memory region into a host virtual address.
2136 * May map a subset of the requested range, given by and returned in *plen.
2137 * May return NULL if resources needed to perform the mapping are exhausted.
2138 * Use only for reads OR writes - not for read-modify-write operations.
2139 * Use cpu_register_map_client() to know when retrying the map operation is
2140 * likely to succeed.
2141 */
2142 void *address_space_map(AddressSpace *as,
2143 hwaddr addr,
2144 hwaddr *plen,
2145 bool is_write)
2146 {
2147 hwaddr len = *plen;
2148 hwaddr done = 0;
2149 hwaddr l, xlat, base;
2150 MemoryRegion *mr, *this_mr;
2151 ram_addr_t raddr;
2152
2153 if (len == 0) {
2154 return NULL;
2155 }
2156
2157 l = len;
2158 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2159 if (!memory_access_is_direct(mr, is_write)) {
2160 if (bounce.buffer) {
2161 return NULL;
2162 }
2163 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2164 bounce.addr = addr;
2165 bounce.len = l;
2166
2167 memory_region_ref(mr);
2168 bounce.mr = mr;
2169 if (!is_write) {
2170 address_space_read(as, addr, bounce.buffer, l);
2171 }
2172
2173 *plen = l;
2174 return bounce.buffer;
2175 }
2176
2177 base = xlat;
2178 raddr = memory_region_get_ram_addr(mr);
2179
2180 for (;;) {
2181 len -= l;
2182 addr += l;
2183 done += l;
2184 if (len == 0) {
2185 break;
2186 }
2187
2188 l = len;
2189 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2190 if (this_mr != mr || xlat != base + done) {
2191 break;
2192 }
2193 }
2194
2195 memory_region_ref(mr);
2196 *plen = done;
2197 return qemu_ram_ptr_length(raddr + base, plen);
2198 }
2199
2200 /* Unmaps a memory region previously mapped by address_space_map().
2201 * Will also mark the memory as dirty if is_write == 1. access_len gives
2202 * the amount of memory that was actually read or written by the caller.
2203 */
2204 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2205 int is_write, hwaddr access_len)
2206 {
2207 if (buffer != bounce.buffer) {
2208 MemoryRegion *mr;
2209 ram_addr_t addr1;
2210
2211 mr = qemu_ram_addr_from_host(buffer, &addr1);
2212 assert(mr != NULL);
2213 if (is_write) {
2214 while (access_len) {
2215 unsigned l;
2216 l = TARGET_PAGE_SIZE;
2217 if (l > access_len)
2218 l = access_len;
2219 invalidate_and_set_dirty(addr1, l);
2220 addr1 += l;
2221 access_len -= l;
2222 }
2223 }
2224 if (xen_enabled()) {
2225 xen_invalidate_map_cache_entry(buffer);
2226 }
2227 memory_region_unref(mr);
2228 return;
2229 }
2230 if (is_write) {
2231 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2232 }
2233 qemu_vfree(bounce.buffer);
2234 bounce.buffer = NULL;
2235 memory_region_unref(bounce.mr);
2236 cpu_notify_map_clients();
2237 }
2238
2239 void *cpu_physical_memory_map(hwaddr addr,
2240 hwaddr *plen,
2241 int is_write)
2242 {
2243 return address_space_map(&address_space_memory, addr, plen, is_write);
2244 }
2245
2246 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2247 int is_write, hwaddr access_len)
2248 {
2249 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2250 }
2251
2252 /* warning: addr must be aligned */
2253 static inline uint32_t ldl_phys_internal(hwaddr addr,
2254 enum device_endian endian)
2255 {
2256 uint8_t *ptr;
2257 uint64_t val;
2258 MemoryRegion *mr;
2259 hwaddr l = 4;
2260 hwaddr addr1;
2261
2262 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2263 false);
2264 if (l < 4 || !memory_access_is_direct(mr, false)) {
2265 /* I/O case */
2266 io_mem_read(mr, addr1, &val, 4);
2267 #if defined(TARGET_WORDS_BIGENDIAN)
2268 if (endian == DEVICE_LITTLE_ENDIAN) {
2269 val = bswap32(val);
2270 }
2271 #else
2272 if (endian == DEVICE_BIG_ENDIAN) {
2273 val = bswap32(val);
2274 }
2275 #endif
2276 } else {
2277 /* RAM case */
2278 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2279 & TARGET_PAGE_MASK)
2280 + addr1);
2281 switch (endian) {
2282 case DEVICE_LITTLE_ENDIAN:
2283 val = ldl_le_p(ptr);
2284 break;
2285 case DEVICE_BIG_ENDIAN:
2286 val = ldl_be_p(ptr);
2287 break;
2288 default:
2289 val = ldl_p(ptr);
2290 break;
2291 }
2292 }
2293 return val;
2294 }
2295
2296 uint32_t ldl_phys(hwaddr addr)
2297 {
2298 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2299 }
2300
2301 uint32_t ldl_le_phys(hwaddr addr)
2302 {
2303 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2304 }
2305
2306 uint32_t ldl_be_phys(hwaddr addr)
2307 {
2308 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2309 }
2310
2311 /* warning: addr must be aligned */
2312 static inline uint64_t ldq_phys_internal(hwaddr addr,
2313 enum device_endian endian)
2314 {
2315 uint8_t *ptr;
2316 uint64_t val;
2317 MemoryRegion *mr;
2318 hwaddr l = 8;
2319 hwaddr addr1;
2320
2321 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2322 false);
2323 if (l < 8 || !memory_access_is_direct(mr, false)) {
2324 /* I/O case */
2325 io_mem_read(mr, addr1, &val, 8);
2326 #if defined(TARGET_WORDS_BIGENDIAN)
2327 if (endian == DEVICE_LITTLE_ENDIAN) {
2328 val = bswap64(val);
2329 }
2330 #else
2331 if (endian == DEVICE_BIG_ENDIAN) {
2332 val = bswap64(val);
2333 }
2334 #endif
2335 } else {
2336 /* RAM case */
2337 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2338 & TARGET_PAGE_MASK)
2339 + addr1);
2340 switch (endian) {
2341 case DEVICE_LITTLE_ENDIAN:
2342 val = ldq_le_p(ptr);
2343 break;
2344 case DEVICE_BIG_ENDIAN:
2345 val = ldq_be_p(ptr);
2346 break;
2347 default:
2348 val = ldq_p(ptr);
2349 break;
2350 }
2351 }
2352 return val;
2353 }
2354
2355 uint64_t ldq_phys(hwaddr addr)
2356 {
2357 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2358 }
2359
2360 uint64_t ldq_le_phys(hwaddr addr)
2361 {
2362 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2363 }
2364
2365 uint64_t ldq_be_phys(hwaddr addr)
2366 {
2367 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2368 }
2369
2370 /* XXX: optimize */
2371 uint32_t ldub_phys(hwaddr addr)
2372 {
2373 uint8_t val;
2374 cpu_physical_memory_read(addr, &val, 1);
2375 return val;
2376 }
2377
2378 /* warning: addr must be aligned */
2379 static inline uint32_t lduw_phys_internal(hwaddr addr,
2380 enum device_endian endian)
2381 {
2382 uint8_t *ptr;
2383 uint64_t val;
2384 MemoryRegion *mr;
2385 hwaddr l = 2;
2386 hwaddr addr1;
2387
2388 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2389 false);
2390 if (l < 2 || !memory_access_is_direct(mr, false)) {
2391 /* I/O case */
2392 io_mem_read(mr, addr1, &val, 2);
2393 #if defined(TARGET_WORDS_BIGENDIAN)
2394 if (endian == DEVICE_LITTLE_ENDIAN) {
2395 val = bswap16(val);
2396 }
2397 #else
2398 if (endian == DEVICE_BIG_ENDIAN) {
2399 val = bswap16(val);
2400 }
2401 #endif
2402 } else {
2403 /* RAM case */
2404 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2405 & TARGET_PAGE_MASK)
2406 + addr1);
2407 switch (endian) {
2408 case DEVICE_LITTLE_ENDIAN:
2409 val = lduw_le_p(ptr);
2410 break;
2411 case DEVICE_BIG_ENDIAN:
2412 val = lduw_be_p(ptr);
2413 break;
2414 default:
2415 val = lduw_p(ptr);
2416 break;
2417 }
2418 }
2419 return val;
2420 }
2421
2422 uint32_t lduw_phys(hwaddr addr)
2423 {
2424 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2425 }
2426
2427 uint32_t lduw_le_phys(hwaddr addr)
2428 {
2429 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2430 }
2431
2432 uint32_t lduw_be_phys(hwaddr addr)
2433 {
2434 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2435 }
2436
2437 /* warning: addr must be aligned. The ram page is not masked as dirty
2438 and the code inside is not invalidated. It is useful if the dirty
2439 bits are used to track modified PTEs */
2440 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2441 {
2442 uint8_t *ptr;
2443 MemoryRegion *mr;
2444 hwaddr l = 4;
2445 hwaddr addr1;
2446
2447 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2448 true);
2449 if (l < 4 || !memory_access_is_direct(mr, true)) {
2450 io_mem_write(mr, addr1, val, 4);
2451 } else {
2452 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2453 ptr = qemu_get_ram_ptr(addr1);
2454 stl_p(ptr, val);
2455
2456 if (unlikely(in_migration)) {
2457 if (!cpu_physical_memory_is_dirty(addr1)) {
2458 /* invalidate code */
2459 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2460 /* set dirty bit */
2461 cpu_physical_memory_set_dirty_flags(
2462 addr1, (0xff & ~CODE_DIRTY_FLAG));
2463 }
2464 }
2465 }
2466 }
2467
2468 /* warning: addr must be aligned */
2469 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2470 enum device_endian endian)
2471 {
2472 uint8_t *ptr;
2473 MemoryRegion *mr;
2474 hwaddr l = 4;
2475 hwaddr addr1;
2476
2477 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2478 true);
2479 if (l < 4 || !memory_access_is_direct(mr, true)) {
2480 #if defined(TARGET_WORDS_BIGENDIAN)
2481 if (endian == DEVICE_LITTLE_ENDIAN) {
2482 val = bswap32(val);
2483 }
2484 #else
2485 if (endian == DEVICE_BIG_ENDIAN) {
2486 val = bswap32(val);
2487 }
2488 #endif
2489 io_mem_write(mr, addr1, val, 4);
2490 } else {
2491 /* RAM case */
2492 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2493 ptr = qemu_get_ram_ptr(addr1);
2494 switch (endian) {
2495 case DEVICE_LITTLE_ENDIAN:
2496 stl_le_p(ptr, val);
2497 break;
2498 case DEVICE_BIG_ENDIAN:
2499 stl_be_p(ptr, val);
2500 break;
2501 default:
2502 stl_p(ptr, val);
2503 break;
2504 }
2505 invalidate_and_set_dirty(addr1, 4);
2506 }
2507 }
2508
2509 void stl_phys(hwaddr addr, uint32_t val)
2510 {
2511 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2512 }
2513
2514 void stl_le_phys(hwaddr addr, uint32_t val)
2515 {
2516 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2517 }
2518
2519 void stl_be_phys(hwaddr addr, uint32_t val)
2520 {
2521 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2522 }
2523
2524 /* XXX: optimize */
2525 void stb_phys(hwaddr addr, uint32_t val)
2526 {
2527 uint8_t v = val;
2528 cpu_physical_memory_write(addr, &v, 1);
2529 }
2530
2531 /* warning: addr must be aligned */
2532 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2533 enum device_endian endian)
2534 {
2535 uint8_t *ptr;
2536 MemoryRegion *mr;
2537 hwaddr l = 2;
2538 hwaddr addr1;
2539
2540 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2541 true);
2542 if (l < 2 || !memory_access_is_direct(mr, true)) {
2543 #if defined(TARGET_WORDS_BIGENDIAN)
2544 if (endian == DEVICE_LITTLE_ENDIAN) {
2545 val = bswap16(val);
2546 }
2547 #else
2548 if (endian == DEVICE_BIG_ENDIAN) {
2549 val = bswap16(val);
2550 }
2551 #endif
2552 io_mem_write(mr, addr1, val, 2);
2553 } else {
2554 /* RAM case */
2555 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2556 ptr = qemu_get_ram_ptr(addr1);
2557 switch (endian) {
2558 case DEVICE_LITTLE_ENDIAN:
2559 stw_le_p(ptr, val);
2560 break;
2561 case DEVICE_BIG_ENDIAN:
2562 stw_be_p(ptr, val);
2563 break;
2564 default:
2565 stw_p(ptr, val);
2566 break;
2567 }
2568 invalidate_and_set_dirty(addr1, 2);
2569 }
2570 }
2571
2572 void stw_phys(hwaddr addr, uint32_t val)
2573 {
2574 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2575 }
2576
2577 void stw_le_phys(hwaddr addr, uint32_t val)
2578 {
2579 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2580 }
2581
2582 void stw_be_phys(hwaddr addr, uint32_t val)
2583 {
2584 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2585 }
2586
2587 /* XXX: optimize */
2588 void stq_phys(hwaddr addr, uint64_t val)
2589 {
2590 val = tswap64(val);
2591 cpu_physical_memory_write(addr, &val, 8);
2592 }
2593
2594 void stq_le_phys(hwaddr addr, uint64_t val)
2595 {
2596 val = cpu_to_le64(val);
2597 cpu_physical_memory_write(addr, &val, 8);
2598 }
2599
2600 void stq_be_phys(hwaddr addr, uint64_t val)
2601 {
2602 val = cpu_to_be64(val);
2603 cpu_physical_memory_write(addr, &val, 8);
2604 }
2605
2606 /* virtual memory access for debug (includes writing to ROM) */
2607 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2608 uint8_t *buf, int len, int is_write)
2609 {
2610 int l;
2611 hwaddr phys_addr;
2612 target_ulong page;
2613
2614 while (len > 0) {
2615 page = addr & TARGET_PAGE_MASK;
2616 phys_addr = cpu_get_phys_page_debug(env, page);
2617 /* if no physical page mapped, return an error */
2618 if (phys_addr == -1)
2619 return -1;
2620 l = (page + TARGET_PAGE_SIZE) - addr;
2621 if (l > len)
2622 l = len;
2623 phys_addr += (addr & ~TARGET_PAGE_MASK);
2624 if (is_write)
2625 cpu_physical_memory_write_rom(phys_addr, buf, l);
2626 else
2627 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2628 len -= l;
2629 buf += l;
2630 addr += l;
2631 }
2632 return 0;
2633 }
2634 #endif
2635
2636 #if !defined(CONFIG_USER_ONLY)
2637
2638 /*
2639 * A helper function for the _utterly broken_ virtio device model to find out if
2640 * it's running on a big endian machine. Don't do this at home kids!
2641 */
2642 bool virtio_is_big_endian(void);
2643 bool virtio_is_big_endian(void)
2644 {
2645 #if defined(TARGET_WORDS_BIGENDIAN)
2646 return true;
2647 #else
2648 return false;
2649 #endif
2650 }
2651
2652 #endif
2653
2654 #ifndef CONFIG_USER_ONLY
2655 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2656 {
2657 MemoryRegion*mr;
2658 hwaddr l = 1;
2659
2660 mr = address_space_translate(&address_space_memory,
2661 phys_addr, &phys_addr, &l, false);
2662
2663 return !(memory_region_is_ram(mr) ||
2664 memory_region_is_romd(mr));
2665 }
2666
2667 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2668 {
2669 RAMBlock *block;
2670
2671 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2672 func(block->host, block->offset, block->length, opaque);
2673 }
2674 }
2675 #endif
Qemu copy for testing