]> git.proxmox.com Git - qemu.git/blob - exec.c
projects / qemu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
log: Change log_cpu_state[_mask]() argument to CPUState
[qemu.git] / exec.c
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "hw/xen/xen.h"
35 #include "qemu/timer.h"
36 #include "qemu/config-file.h"
37 #include "exec/memory.h"
38 #include "sysemu/dma.h"
39 #include "exec/address-spaces.h"
40 #if defined(CONFIG_USER_ONLY)
41 #include <qemu.h>
42 #else /* !CONFIG_USER_ONLY */
43 #include "sysemu/xen-mapcache.h"
44 #include "trace.h"
45 #endif
46 #include "exec/cpu-all.h"
47
48 #include "exec/cputlb.h"
49 #include "translate-all.h"
50
51 #include "exec/memory-internal.h"
52
53 //#define DEBUG_SUBPAGE
54
55 #if !defined(CONFIG_USER_ONLY)
56 int phys_ram_fd;
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 CPUState *first_cpu;
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 typedef PhysPageEntry Node[L2_SIZE];
92
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101 };
102
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
110
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
115
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
124
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
127
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
133
134 static MemoryRegion io_mem_watch;
135 #endif
136
137 #if !defined(CONFIG_USER_ONLY)
138
139 static void phys_map_node_reserve(unsigned nodes)
140 {
141 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
142 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
143 16);
144 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
145 next_map.nodes_nb + nodes);
146 next_map.nodes = g_renew(Node, next_map.nodes,
147 next_map.nodes_nb_alloc);
148 }
149 }
150
151 static uint16_t phys_map_node_alloc(void)
152 {
153 unsigned i;
154 uint16_t ret;
155
156 ret = next_map.nodes_nb++;
157 assert(ret != PHYS_MAP_NODE_NIL);
158 assert(ret != next_map.nodes_nb_alloc);
159 for (i = 0; i < L2_SIZE; ++i) {
160 next_map.nodes[ret][i].is_leaf = 0;
161 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
162 }
163 return ret;
164 }
165
166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
167 hwaddr *nb, uint16_t leaf,
168 int level)
169 {
170 PhysPageEntry *p;
171 int i;
172 hwaddr step = (hwaddr)1 << (level * L2_BITS);
173
174 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
175 lp->ptr = phys_map_node_alloc();
176 p = next_map.nodes[lp->ptr];
177 if (level == 0) {
178 for (i = 0; i < L2_SIZE; i++) {
179 p[i].is_leaf = 1;
180 p[i].ptr = PHYS_SECTION_UNASSIGNED;
181 }
182 }
183 } else {
184 p = next_map.nodes[lp->ptr];
185 }
186 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
187
188 while (*nb && lp < &p[L2_SIZE]) {
189 if ((*index & (step - 1)) == 0 && *nb >= step) {
190 lp->is_leaf = true;
191 lp->ptr = leaf;
192 *index += step;
193 *nb -= step;
194 } else {
195 phys_page_set_level(lp, index, nb, leaf, level - 1);
196 }
197 ++lp;
198 }
199 }
200
201 static void phys_page_set(AddressSpaceDispatch *d,
202 hwaddr index, hwaddr nb,
203 uint16_t leaf)
204 {
205 /* Wildly overreserve - it doesn't matter much. */
206 phys_map_node_reserve(3 * P_L2_LEVELS);
207
208 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
209 }
210
211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
212 Node *nodes, MemoryRegionSection *sections)
213 {
214 PhysPageEntry *p;
215 int i;
216
217 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
218 if (lp.ptr == PHYS_MAP_NODE_NIL) {
219 return &sections[PHYS_SECTION_UNASSIGNED];
220 }
221 p = nodes[lp.ptr];
222 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
223 }
224 return &sections[lp.ptr];
225 }
226
227 bool memory_region_is_unassigned(MemoryRegion *mr)
228 {
229 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
230 && mr != &io_mem_watch;
231 }
232
233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
234 hwaddr addr,
235 bool resolve_subpage)
236 {
237 MemoryRegionSection *section;
238 subpage_t *subpage;
239
240 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
241 d->nodes, d->sections);
242 if (resolve_subpage && section->mr->subpage) {
243 subpage = container_of(section->mr, subpage_t, iomem);
244 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
245 }
246 return section;
247 }
248
249 static MemoryRegionSection *
250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
251 hwaddr *plen, bool resolve_subpage)
252 {
253 MemoryRegionSection *section;
254 Int128 diff;
255
256 section = address_space_lookup_region(d, addr, resolve_subpage);
257 /* Compute offset within MemoryRegionSection */
258 addr -= section->offset_within_address_space;
259
260 /* Compute offset within MemoryRegion */
261 *xlat = addr + section->offset_within_region;
262
263 diff = int128_sub(section->mr->size, int128_make64(addr));
264 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
265 return section;
266 }
267
268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
269 hwaddr *xlat, hwaddr *plen,
270 bool is_write)
271 {
272 IOMMUTLBEntry iotlb;
273 MemoryRegionSection *section;
274 MemoryRegion *mr;
275 hwaddr len = *plen;
276
277 for (;;) {
278 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
279 mr = section->mr;
280
281 if (!mr->iommu_ops) {
282 break;
283 }
284
285 iotlb = mr->iommu_ops->translate(mr, addr);
286 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
287 | (addr & iotlb.addr_mask));
288 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
289 if (!(iotlb.perm & (1 << is_write))) {
290 mr = &io_mem_unassigned;
291 break;
292 }
293
294 as = iotlb.target_as;
295 }
296
297 *plen = len;
298 *xlat = addr;
299 return mr;
300 }
301
302 MemoryRegionSection *
303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
304 hwaddr *plen)
305 {
306 MemoryRegionSection *section;
307 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
308
309 assert(!section->mr->iommu_ops);
310 return section;
311 }
312 #endif
313
314 void cpu_exec_init_all(void)
315 {
316 #if !defined(CONFIG_USER_ONLY)
317 qemu_mutex_init(&ram_list.mutex);
318 memory_map_init();
319 io_mem_init();
320 #endif
321 }
322
323 #if !defined(CONFIG_USER_ONLY)
324
325 static int cpu_common_post_load(void *opaque, int version_id)
326 {
327 CPUState *cpu = opaque;
328
329 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
330 version_id is increased. */
331 cpu->interrupt_request &= ~0x01;
332 tlb_flush(cpu->env_ptr, 1);
333
334 return 0;
335 }
336
337 const VMStateDescription vmstate_cpu_common = {
338 .name = "cpu_common",
339 .version_id = 1,
340 .minimum_version_id = 1,
341 .minimum_version_id_old = 1,
342 .post_load = cpu_common_post_load,
343 .fields = (VMStateField []) {
344 VMSTATE_UINT32(halted, CPUState),
345 VMSTATE_UINT32(interrupt_request, CPUState),
346 VMSTATE_END_OF_LIST()
347 }
348 };
349
350 #endif
351
352 CPUState *qemu_get_cpu(int index)
353 {
354 CPUState *cpu = first_cpu;
355
356 while (cpu) {
357 if (cpu->cpu_index == index) {
358 break;
359 }
360 cpu = cpu->next_cpu;
361 }
362
363 return cpu;
364 }
365
366 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
367 {
368 CPUState *cpu;
369
370 cpu = first_cpu;
371 while (cpu) {
372 func(cpu, data);
373 cpu = cpu->next_cpu;
374 }
375 }
376
377 void cpu_exec_init(CPUArchState *env)
378 {
379 CPUState *cpu = ENV_GET_CPU(env);
380 CPUClass *cc = CPU_GET_CLASS(cpu);
381 CPUState **pcpu;
382 int cpu_index;
383
384 #if defined(CONFIG_USER_ONLY)
385 cpu_list_lock();
386 #endif
387 cpu->next_cpu = NULL;
388 pcpu = &first_cpu;
389 cpu_index = 0;
390 while (*pcpu != NULL) {
391 pcpu = &(*pcpu)->next_cpu;
392 cpu_index++;
393 }
394 cpu->cpu_index = cpu_index;
395 cpu->numa_node = 0;
396 QTAILQ_INIT(&env->breakpoints);
397 QTAILQ_INIT(&env->watchpoints);
398 #ifndef CONFIG_USER_ONLY
399 cpu->thread_id = qemu_get_thread_id();
400 #endif
401 *pcpu = cpu;
402 #if defined(CONFIG_USER_ONLY)
403 cpu_list_unlock();
404 #endif
405 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
406 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
407 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
408 cpu_save, cpu_load, env);
409 assert(cc->vmsd == NULL);
410 #endif
411 if (cc->vmsd != NULL) {
412 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
413 }
414 }
415
416 #if defined(TARGET_HAS_ICE)
417 #if defined(CONFIG_USER_ONLY)
418 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
419 {
420 tb_invalidate_phys_page_range(pc, pc + 1, 0);
421 }
422 #else
423 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
424 {
425 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
426 (pc & ~TARGET_PAGE_MASK));
427 }
428 #endif
429 #endif /* TARGET_HAS_ICE */
430
431 #if defined(CONFIG_USER_ONLY)
432 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
433
434 {
435 }
436
437 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
438 int flags, CPUWatchpoint **watchpoint)
439 {
440 return -ENOSYS;
441 }
442 #else
443 /* Add a watchpoint. */
444 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
445 int flags, CPUWatchpoint **watchpoint)
446 {
447 target_ulong len_mask = ~(len - 1);
448 CPUWatchpoint *wp;
449
450 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
451 if ((len & (len - 1)) || (addr & ~len_mask) ||
452 len == 0 || len > TARGET_PAGE_SIZE) {
453 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
454 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
455 return -EINVAL;
456 }
457 wp = g_malloc(sizeof(*wp));
458
459 wp->vaddr = addr;
460 wp->len_mask = len_mask;
461 wp->flags = flags;
462
463 /* keep all GDB-injected watchpoints in front */
464 if (flags & BP_GDB)
465 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
466 else
467 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
468
469 tlb_flush_page(env, addr);
470
471 if (watchpoint)
472 *watchpoint = wp;
473 return 0;
474 }
475
476 /* Remove a specific watchpoint. */
477 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
478 int flags)
479 {
480 target_ulong len_mask = ~(len - 1);
481 CPUWatchpoint *wp;
482
483 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
484 if (addr == wp->vaddr && len_mask == wp->len_mask
485 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
486 cpu_watchpoint_remove_by_ref(env, wp);
487 return 0;
488 }
489 }
490 return -ENOENT;
491 }
492
493 /* Remove a specific watchpoint by reference. */
494 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
495 {
496 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
497
498 tlb_flush_page(env, watchpoint->vaddr);
499
500 g_free(watchpoint);
501 }
502
503 /* Remove all matching watchpoints. */
504 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
505 {
506 CPUWatchpoint *wp, *next;
507
508 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
509 if (wp->flags & mask)
510 cpu_watchpoint_remove_by_ref(env, wp);
511 }
512 }
513 #endif
514
515 /* Add a breakpoint. */
516 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
517 CPUBreakpoint **breakpoint)
518 {
519 #if defined(TARGET_HAS_ICE)
520 CPUBreakpoint *bp;
521
522 bp = g_malloc(sizeof(*bp));
523
524 bp->pc = pc;
525 bp->flags = flags;
526
527 /* keep all GDB-injected breakpoints in front */
528 if (flags & BP_GDB)
529 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
530 else
531 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
532
533 breakpoint_invalidate(env, pc);
534
535 if (breakpoint)
536 *breakpoint = bp;
537 return 0;
538 #else
539 return -ENOSYS;
540 #endif
541 }
542
543 /* Remove a specific breakpoint. */
544 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
545 {
546 #if defined(TARGET_HAS_ICE)
547 CPUBreakpoint *bp;
548
549 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
550 if (bp->pc == pc && bp->flags == flags) {
551 cpu_breakpoint_remove_by_ref(env, bp);
552 return 0;
553 }
554 }
555 return -ENOENT;
556 #else
557 return -ENOSYS;
558 #endif
559 }
560
561 /* Remove a specific breakpoint by reference. */
562 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
563 {
564 #if defined(TARGET_HAS_ICE)
565 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
566
567 breakpoint_invalidate(env, breakpoint->pc);
568
569 g_free(breakpoint);
570 #endif
571 }
572
573 /* Remove all matching breakpoints. */
574 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
575 {
576 #if defined(TARGET_HAS_ICE)
577 CPUBreakpoint *bp, *next;
578
579 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
580 if (bp->flags & mask)
581 cpu_breakpoint_remove_by_ref(env, bp);
582 }
583 #endif
584 }
585
586 /* enable or disable single step mode. EXCP_DEBUG is returned by the
587 CPU loop after each instruction */
588 void cpu_single_step(CPUArchState *env, int enabled)
589 {
590 #if defined(TARGET_HAS_ICE)
591 if (env->singlestep_enabled != enabled) {
592 env->singlestep_enabled = enabled;
593 if (kvm_enabled())
594 kvm_update_guest_debug(env, 0);
595 else {
596 /* must flush all the translated code to avoid inconsistencies */
597 /* XXX: only flush what is necessary */
598 tb_flush(env);
599 }
600 }
601 #endif
602 }
603
604 void cpu_abort(CPUArchState *env, const char *fmt, ...)
605 {
606 CPUState *cpu = ENV_GET_CPU(env);
607 va_list ap;
608 va_list ap2;
609
610 va_start(ap, fmt);
611 va_copy(ap2, ap);
612 fprintf(stderr, "qemu: fatal: ");
613 vfprintf(stderr, fmt, ap);
614 fprintf(stderr, "\n");
615 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
616 if (qemu_log_enabled()) {
617 qemu_log("qemu: fatal: ");
618 qemu_log_vprintf(fmt, ap2);
619 qemu_log("\n");
620 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
621 qemu_log_flush();
622 qemu_log_close();
623 }
624 va_end(ap2);
625 va_end(ap);
626 #if defined(CONFIG_USER_ONLY)
627 {
628 struct sigaction act;
629 sigfillset(&act.sa_mask);
630 act.sa_handler = SIG_DFL;
631 sigaction(SIGABRT, &act, NULL);
632 }
633 #endif
634 abort();
635 }
636
637 CPUArchState *cpu_copy(CPUArchState *env)
638 {
639 CPUArchState *new_env = cpu_init(env->cpu_model_str);
640 #if defined(TARGET_HAS_ICE)
641 CPUBreakpoint *bp;
642 CPUWatchpoint *wp;
643 #endif
644
645 memcpy(new_env, env, sizeof(CPUArchState));
646
647 /* Clone all break/watchpoints.
648 Note: Once we support ptrace with hw-debug register access, make sure
649 BP_CPU break/watchpoints are handled correctly on clone. */
650 QTAILQ_INIT(&env->breakpoints);
651 QTAILQ_INIT(&env->watchpoints);
652 #if defined(TARGET_HAS_ICE)
653 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
654 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
655 }
656 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
657 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
658 wp->flags, NULL);
659 }
660 #endif
661
662 return new_env;
663 }
664
665 #if !defined(CONFIG_USER_ONLY)
666 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
667 uintptr_t length)
668 {
669 uintptr_t start1;
670
671 /* we modify the TLB cache so that the dirty bit will be set again
672 when accessing the range */
673 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
674 /* Check that we don't span multiple blocks - this breaks the
675 address comparisons below. */
676 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
677 != (end - 1) - start) {
678 abort();
679 }
680 cpu_tlb_reset_dirty_all(start1, length);
681
682 }
683
684 /* Note: start and end must be within the same ram block. */
685 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
686 int dirty_flags)
687 {
688 uintptr_t length;
689
690 start &= TARGET_PAGE_MASK;
691 end = TARGET_PAGE_ALIGN(end);
692
693 length = end - start;
694 if (length == 0)
695 return;
696 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
697
698 if (tcg_enabled()) {
699 tlb_reset_dirty_range_all(start, end, length);
700 }
701 }
702
703 static int cpu_physical_memory_set_dirty_tracking(int enable)
704 {
705 int ret = 0;
706 in_migration = enable;
707 return ret;
708 }
709
710 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
711 MemoryRegionSection *section,
712 target_ulong vaddr,
713 hwaddr paddr, hwaddr xlat,
714 int prot,
715 target_ulong *address)
716 {
717 hwaddr iotlb;
718 CPUWatchpoint *wp;
719
720 if (memory_region_is_ram(section->mr)) {
721 /* Normal RAM. */
722 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
723 + xlat;
724 if (!section->readonly) {
725 iotlb |= PHYS_SECTION_NOTDIRTY;
726 } else {
727 iotlb |= PHYS_SECTION_ROM;
728 }
729 } else {
730 iotlb = section - address_space_memory.dispatch->sections;
731 iotlb += xlat;
732 }
733
734 /* Make accesses to pages with watchpoints go via the
735 watchpoint trap routines. */
736 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
737 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
738 /* Avoid trapping reads of pages with a write breakpoint. */
739 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
740 iotlb = PHYS_SECTION_WATCH + paddr;
741 *address |= TLB_MMIO;
742 break;
743 }
744 }
745 }
746
747 return iotlb;
748 }
749 #endif /* defined(CONFIG_USER_ONLY) */
750
751 #if !defined(CONFIG_USER_ONLY)
752
753 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
754 uint16_t section);
755 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
756
757 static uint16_t phys_section_add(MemoryRegionSection *section)
758 {
759 /* The physical section number is ORed with a page-aligned
760 * pointer to produce the iotlb entries. Thus it should
761 * never overflow into the page-aligned value.
762 */
763 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
764
765 if (next_map.sections_nb == next_map.sections_nb_alloc) {
766 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
767 16);
768 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
769 next_map.sections_nb_alloc);
770 }
771 next_map.sections[next_map.sections_nb] = *section;
772 memory_region_ref(section->mr);
773 return next_map.sections_nb++;
774 }
775
776 static void phys_section_destroy(MemoryRegion *mr)
777 {
778 memory_region_unref(mr);
779
780 if (mr->subpage) {
781 subpage_t *subpage = container_of(mr, subpage_t, iomem);
782 memory_region_destroy(&subpage->iomem);
783 g_free(subpage);
784 }
785 }
786
787 static void phys_sections_free(PhysPageMap *map)
788 {
789 while (map->sections_nb > 0) {
790 MemoryRegionSection *section = &map->sections[--map->sections_nb];
791 phys_section_destroy(section->mr);
792 }
793 g_free(map->sections);
794 g_free(map->nodes);
795 g_free(map);
796 }
797
798 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
799 {
800 subpage_t *subpage;
801 hwaddr base = section->offset_within_address_space
802 & TARGET_PAGE_MASK;
803 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
804 next_map.nodes, next_map.sections);
805 MemoryRegionSection subsection = {
806 .offset_within_address_space = base,
807 .size = int128_make64(TARGET_PAGE_SIZE),
808 };
809 hwaddr start, end;
810
811 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
812
813 if (!(existing->mr->subpage)) {
814 subpage = subpage_init(d->as, base);
815 subsection.mr = &subpage->iomem;
816 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
817 phys_section_add(&subsection));
818 } else {
819 subpage = container_of(existing->mr, subpage_t, iomem);
820 }
821 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
822 end = start + int128_get64(section->size) - 1;
823 subpage_register(subpage, start, end, phys_section_add(section));
824 }
825
826
827 static void register_multipage(AddressSpaceDispatch *d,
828 MemoryRegionSection *section)
829 {
830 hwaddr start_addr = section->offset_within_address_space;
831 uint16_t section_index = phys_section_add(section);
832 uint64_t num_pages = int128_get64(int128_rshift(section->size,
833 TARGET_PAGE_BITS));
834
835 assert(num_pages);
836 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
837 }
838
839 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
840 {
841 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
842 AddressSpaceDispatch *d = as->next_dispatch;
843 MemoryRegionSection now = *section, remain = *section;
844 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
845
846 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
847 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
848 - now.offset_within_address_space;
849
850 now.size = int128_min(int128_make64(left), now.size);
851 register_subpage(d, &now);
852 } else {
853 now.size = int128_zero();
854 }
855 while (int128_ne(remain.size, now.size)) {
856 remain.size = int128_sub(remain.size, now.size);
857 remain.offset_within_address_space += int128_get64(now.size);
858 remain.offset_within_region += int128_get64(now.size);
859 now = remain;
860 if (int128_lt(remain.size, page_size)) {
861 register_subpage(d, &now);
862 } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
863 now.size = page_size;
864 register_subpage(d, &now);
865 } else {
866 now.size = int128_and(now.size, int128_neg(page_size));
867 register_multipage(d, &now);
868 }
869 }
870 }
871
872 void qemu_flush_coalesced_mmio_buffer(void)
873 {
874 if (kvm_enabled())
875 kvm_flush_coalesced_mmio_buffer();
876 }
877
878 void qemu_mutex_lock_ramlist(void)
879 {
880 qemu_mutex_lock(&ram_list.mutex);
881 }
882
883 void qemu_mutex_unlock_ramlist(void)
884 {
885 qemu_mutex_unlock(&ram_list.mutex);
886 }
887
888 #if defined(__linux__) && !defined(TARGET_S390X)
889
890 #include <sys/vfs.h>
891
892 #define HUGETLBFS_MAGIC 0x958458f6
893
894 static long gethugepagesize(const char *path)
895 {
896 struct statfs fs;
897 int ret;
898
899 do {
900 ret = statfs(path, &fs);
901 } while (ret != 0 && errno == EINTR);
902
903 if (ret != 0) {
904 perror(path);
905 return 0;
906 }
907
908 if (fs.f_type != HUGETLBFS_MAGIC)
909 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
910
911 return fs.f_bsize;
912 }
913
914 static void *file_ram_alloc(RAMBlock *block,
915 ram_addr_t memory,
916 const char *path)
917 {
918 char *filename;
919 char *sanitized_name;
920 char *c;
921 void *area;
922 int fd;
923 #ifdef MAP_POPULATE
924 int flags;
925 #endif
926 unsigned long hpagesize;
927
928 hpagesize = gethugepagesize(path);
929 if (!hpagesize) {
930 return NULL;
931 }
932
933 if (memory < hpagesize) {
934 return NULL;
935 }
936
937 if (kvm_enabled() && !kvm_has_sync_mmu()) {
938 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
939 return NULL;
940 }
941
942 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
943 sanitized_name = g_strdup(block->mr->name);
944 for (c = sanitized_name; *c != '\0'; c++) {
945 if (*c == '/')
946 *c = '_';
947 }
948
949 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
950 sanitized_name);
951 g_free(sanitized_name);
952
953 fd = mkstemp(filename);
954 if (fd < 0) {
955 perror("unable to create backing store for hugepages");
956 g_free(filename);
957 return NULL;
958 }
959 unlink(filename);
960 g_free(filename);
961
962 memory = (memory+hpagesize-1) & ~(hpagesize-1);
963
964 /*
965 * ftruncate is not supported by hugetlbfs in older
966 * hosts, so don't bother bailing out on errors.
967 * If anything goes wrong with it under other filesystems,
968 * mmap will fail.
969 */
970 if (ftruncate(fd, memory))
971 perror("ftruncate");
972
973 #ifdef MAP_POPULATE
974 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
975 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
976 * to sidestep this quirk.
977 */
978 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
979 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
980 #else
981 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
982 #endif
983 if (area == MAP_FAILED) {
984 perror("file_ram_alloc: can't mmap RAM pages");
985 close(fd);
986 return (NULL);
987 }
988 block->fd = fd;
989 return area;
990 }
991 #endif
992
993 static ram_addr_t find_ram_offset(ram_addr_t size)
994 {
995 RAMBlock *block, *next_block;
996 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
997
998 assert(size != 0); /* it would hand out same offset multiple times */
999
1000 if (QTAILQ_EMPTY(&ram_list.blocks))
1001 return 0;
1002
1003 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1004 ram_addr_t end, next = RAM_ADDR_MAX;
1005
1006 end = block->offset + block->length;
1007
1008 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1009 if (next_block->offset >= end) {
1010 next = MIN(next, next_block->offset);
1011 }
1012 }
1013 if (next - end >= size && next - end < mingap) {
1014 offset = end;
1015 mingap = next - end;
1016 }
1017 }
1018
1019 if (offset == RAM_ADDR_MAX) {
1020 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1021 (uint64_t)size);
1022 abort();
1023 }
1024
1025 return offset;
1026 }
1027
1028 ram_addr_t last_ram_offset(void)
1029 {
1030 RAMBlock *block;
1031 ram_addr_t last = 0;
1032
1033 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1034 last = MAX(last, block->offset + block->length);
1035
1036 return last;
1037 }
1038
1039 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1040 {
1041 int ret;
1042 QemuOpts *machine_opts;
1043
1044 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1045 machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1046 if (machine_opts &&
1047 !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
1048 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1049 if (ret) {
1050 perror("qemu_madvise");
1051 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1052 "but dump_guest_core=off specified\n");
1053 }
1054 }
1055 }
1056
1057 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1058 {
1059 RAMBlock *new_block, *block;
1060
1061 new_block = NULL;
1062 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1063 if (block->offset == addr) {
1064 new_block = block;
1065 break;
1066 }
1067 }
1068 assert(new_block);
1069 assert(!new_block->idstr[0]);
1070
1071 if (dev) {
1072 char *id = qdev_get_dev_path(dev);
1073 if (id) {
1074 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1075 g_free(id);
1076 }
1077 }
1078 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1079
1080 /* This assumes the iothread lock is taken here too. */
1081 qemu_mutex_lock_ramlist();
1082 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1083 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1084 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1085 new_block->idstr);
1086 abort();
1087 }
1088 }
1089 qemu_mutex_unlock_ramlist();
1090 }
1091
1092 static int memory_try_enable_merging(void *addr, size_t len)
1093 {
1094 QemuOpts *opts;
1095
1096 opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1097 if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
1098 /* disabled by the user */
1099 return 0;
1100 }
1101
1102 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1103 }
1104
1105 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1106 MemoryRegion *mr)
1107 {
1108 RAMBlock *block, *new_block;
1109
1110 size = TARGET_PAGE_ALIGN(size);
1111 new_block = g_malloc0(sizeof(*new_block));
1112
1113 /* This assumes the iothread lock is taken here too. */
1114 qemu_mutex_lock_ramlist();
1115 new_block->mr = mr;
1116 new_block->offset = find_ram_offset(size);
1117 if (host) {
1118 new_block->host = host;
1119 new_block->flags |= RAM_PREALLOC_MASK;
1120 } else {
1121 if (mem_path) {
1122 #if defined (__linux__) && !defined(TARGET_S390X)
1123 new_block->host = file_ram_alloc(new_block, size, mem_path);
1124 if (!new_block->host) {
1125 new_block->host = qemu_anon_ram_alloc(size);
1126 memory_try_enable_merging(new_block->host, size);
1127 }
1128 #else
1129 fprintf(stderr, "-mem-path option unsupported\n");
1130 exit(1);
1131 #endif
1132 } else {
1133 if (xen_enabled()) {
1134 xen_ram_alloc(new_block->offset, size, mr);
1135 } else if (kvm_enabled()) {
1136 /* some s390/kvm configurations have special constraints */
1137 new_block->host = kvm_ram_alloc(size);
1138 } else {
1139 new_block->host = qemu_anon_ram_alloc(size);
1140 }
1141 memory_try_enable_merging(new_block->host, size);
1142 }
1143 }
1144 new_block->length = size;
1145
1146 /* Keep the list sorted from biggest to smallest block. */
1147 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1148 if (block->length < new_block->length) {
1149 break;
1150 }
1151 }
1152 if (block) {
1153 QTAILQ_INSERT_BEFORE(block, new_block, next);
1154 } else {
1155 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1156 }
1157 ram_list.mru_block = NULL;
1158
1159 ram_list.version++;
1160 qemu_mutex_unlock_ramlist();
1161
1162 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1163 last_ram_offset() >> TARGET_PAGE_BITS);
1164 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1165 0, size >> TARGET_PAGE_BITS);
1166 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1167
1168 qemu_ram_setup_dump(new_block->host, size);
1169 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1170
1171 if (kvm_enabled())
1172 kvm_setup_guest_memory(new_block->host, size);
1173
1174 return new_block->offset;
1175 }
1176
1177 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1178 {
1179 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1180 }
1181
1182 void qemu_ram_free_from_ptr(ram_addr_t addr)
1183 {
1184 RAMBlock *block;
1185
1186 /* This assumes the iothread lock is taken here too. */
1187 qemu_mutex_lock_ramlist();
1188 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1189 if (addr == block->offset) {
1190 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1191 ram_list.mru_block = NULL;
1192 ram_list.version++;
1193 g_free(block);
1194 break;
1195 }
1196 }
1197 qemu_mutex_unlock_ramlist();
1198 }
1199
1200 void qemu_ram_free(ram_addr_t addr)
1201 {
1202 RAMBlock *block;
1203
1204 /* This assumes the iothread lock is taken here too. */
1205 qemu_mutex_lock_ramlist();
1206 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1207 if (addr == block->offset) {
1208 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1209 ram_list.mru_block = NULL;
1210 ram_list.version++;
1211 if (block->flags & RAM_PREALLOC_MASK) {
1212 ;
1213 } else if (mem_path) {
1214 #if defined (__linux__) && !defined(TARGET_S390X)
1215 if (block->fd) {
1216 munmap(block->host, block->length);
1217 close(block->fd);
1218 } else {
1219 qemu_anon_ram_free(block->host, block->length);
1220 }
1221 #else
1222 abort();
1223 #endif
1224 } else {
1225 if (xen_enabled()) {
1226 xen_invalidate_map_cache_entry(block->host);
1227 } else {
1228 qemu_anon_ram_free(block->host, block->length);
1229 }
1230 }
1231 g_free(block);
1232 break;
1233 }
1234 }
1235 qemu_mutex_unlock_ramlist();
1236
1237 }
1238
1239 #ifndef _WIN32
1240 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1241 {
1242 RAMBlock *block;
1243 ram_addr_t offset;
1244 int flags;
1245 void *area, *vaddr;
1246
1247 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1248 offset = addr - block->offset;
1249 if (offset < block->length) {
1250 vaddr = block->host + offset;
1251 if (block->flags & RAM_PREALLOC_MASK) {
1252 ;
1253 } else {
1254 flags = MAP_FIXED;
1255 munmap(vaddr, length);
1256 if (mem_path) {
1257 #if defined(__linux__) && !defined(TARGET_S390X)
1258 if (block->fd) {
1259 #ifdef MAP_POPULATE
1260 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1261 MAP_PRIVATE;
1262 #else
1263 flags |= MAP_PRIVATE;
1264 #endif
1265 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1266 flags, block->fd, offset);
1267 } else {
1268 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1269 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1270 flags, -1, 0);
1271 }
1272 #else
1273 abort();
1274 #endif
1275 } else {
1276 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1277 flags |= MAP_SHARED | MAP_ANONYMOUS;
1278 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1279 flags, -1, 0);
1280 #else
1281 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1282 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1283 flags, -1, 0);
1284 #endif
1285 }
1286 if (area != vaddr) {
1287 fprintf(stderr, "Could not remap addr: "
1288 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1289 length, addr);
1290 exit(1);
1291 }
1292 memory_try_enable_merging(vaddr, length);
1293 qemu_ram_setup_dump(vaddr, length);
1294 }
1295 return;
1296 }
1297 }
1298 }
1299 #endif /* !_WIN32 */
1300
1301 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1302 {
1303 RAMBlock *block;
1304
1305 /* The list is protected by the iothread lock here. */
1306 block = ram_list.mru_block;
1307 if (block && addr - block->offset < block->length) {
1308 goto found;
1309 }
1310 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1311 if (addr - block->offset < block->length) {
1312 goto found;
1313 }
1314 }
1315
1316 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1317 abort();
1318
1319 found:
1320 ram_list.mru_block = block;
1321 return block;
1322 }
1323
1324 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1325 With the exception of the softmmu code in this file, this should
1326 only be used for local memory (e.g. video ram) that the device owns,
1327 and knows it isn't going to access beyond the end of the block.
1328
1329 It should not be used for general purpose DMA.
1330 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1331 */
1332 void *qemu_get_ram_ptr(ram_addr_t addr)
1333 {
1334 RAMBlock *block = qemu_get_ram_block(addr);
1335
1336 if (xen_enabled()) {
1337 /* We need to check if the requested address is in the RAM
1338 * because we don't want to map the entire memory in QEMU.
1339 * In that case just map until the end of the page.
1340 */
1341 if (block->offset == 0) {
1342 return xen_map_cache(addr, 0, 0);
1343 } else if (block->host == NULL) {
1344 block->host =
1345 xen_map_cache(block->offset, block->length, 1);
1346 }
1347 }
1348 return block->host + (addr - block->offset);
1349 }
1350
1351 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1352 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1353 *
1354 * ??? Is this still necessary?
1355 */
1356 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1357 {
1358 RAMBlock *block;
1359
1360 /* The list is protected by the iothread lock here. */
1361 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1362 if (addr - block->offset < block->length) {
1363 if (xen_enabled()) {
1364 /* We need to check if the requested address is in the RAM
1365 * because we don't want to map the entire memory in QEMU.
1366 * In that case just map until the end of the page.
1367 */
1368 if (block->offset == 0) {
1369 return xen_map_cache(addr, 0, 0);
1370 } else if (block->host == NULL) {
1371 block->host =
1372 xen_map_cache(block->offset, block->length, 1);
1373 }
1374 }
1375 return block->host + (addr - block->offset);
1376 }
1377 }
1378
1379 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1380 abort();
1381
1382 return NULL;
1383 }
1384
1385 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1386 * but takes a size argument */
1387 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1388 {
1389 if (*size == 0) {
1390 return NULL;
1391 }
1392 if (xen_enabled()) {
1393 return xen_map_cache(addr, *size, 1);
1394 } else {
1395 RAMBlock *block;
1396
1397 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1398 if (addr - block->offset < block->length) {
1399 if (addr - block->offset + *size > block->length)
1400 *size = block->length - addr + block->offset;
1401 return block->host + (addr - block->offset);
1402 }
1403 }
1404
1405 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1406 abort();
1407 }
1408 }
1409
1410 /* Some of the softmmu routines need to translate from a host pointer
1411 (typically a TLB entry) back to a ram offset. */
1412 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1413 {
1414 RAMBlock *block;
1415 uint8_t *host = ptr;
1416
1417 if (xen_enabled()) {
1418 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1419 return qemu_get_ram_block(*ram_addr)->mr;
1420 }
1421
1422 block = ram_list.mru_block;
1423 if (block && block->host && host - block->host < block->length) {
1424 goto found;
1425 }
1426
1427 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1428 /* This case append when the block is not mapped. */
1429 if (block->host == NULL) {
1430 continue;
1431 }
1432 if (host - block->host < block->length) {
1433 goto found;
1434 }
1435 }
1436
1437 return NULL;
1438
1439 found:
1440 *ram_addr = block->offset + (host - block->host);
1441 return block->mr;
1442 }
1443
1444 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1445 uint64_t val, unsigned size)
1446 {
1447 int dirty_flags;
1448 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1449 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1450 tb_invalidate_phys_page_fast(ram_addr, size);
1451 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1452 }
1453 switch (size) {
1454 case 1:
1455 stb_p(qemu_get_ram_ptr(ram_addr), val);
1456 break;
1457 case 2:
1458 stw_p(qemu_get_ram_ptr(ram_addr), val);
1459 break;
1460 case 4:
1461 stl_p(qemu_get_ram_ptr(ram_addr), val);
1462 break;
1463 default:
1464 abort();
1465 }
1466 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1467 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1468 /* we remove the notdirty callback only if the code has been
1469 flushed */
1470 if (dirty_flags == 0xff) {
1471 CPUArchState *env = current_cpu->env_ptr;
1472 tlb_set_dirty(env, env->mem_io_vaddr);
1473 }
1474 }
1475
1476 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1477 unsigned size, bool is_write)
1478 {
1479 return is_write;
1480 }
1481
1482 static const MemoryRegionOps notdirty_mem_ops = {
1483 .write = notdirty_mem_write,
1484 .valid.accepts = notdirty_mem_accepts,
1485 .endianness = DEVICE_NATIVE_ENDIAN,
1486 };
1487
1488 /* Generate a debug exception if a watchpoint has been hit. */
1489 static void check_watchpoint(int offset, int len_mask, int flags)
1490 {
1491 CPUArchState *env = current_cpu->env_ptr;
1492 target_ulong pc, cs_base;
1493 target_ulong vaddr;
1494 CPUWatchpoint *wp;
1495 int cpu_flags;
1496
1497 if (env->watchpoint_hit) {
1498 /* We re-entered the check after replacing the TB. Now raise
1499 * the debug interrupt so that is will trigger after the
1500 * current instruction. */
1501 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1502 return;
1503 }
1504 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1505 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1506 if ((vaddr == (wp->vaddr & len_mask) ||
1507 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1508 wp->flags |= BP_WATCHPOINT_HIT;
1509 if (!env->watchpoint_hit) {
1510 env->watchpoint_hit = wp;
1511 tb_check_watchpoint(env);
1512 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1513 env->exception_index = EXCP_DEBUG;
1514 cpu_loop_exit(env);
1515 } else {
1516 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1517 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1518 cpu_resume_from_signal(env, NULL);
1519 }
1520 }
1521 } else {
1522 wp->flags &= ~BP_WATCHPOINT_HIT;
1523 }
1524 }
1525 }
1526
1527 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1528 so these check for a hit then pass through to the normal out-of-line
1529 phys routines. */
1530 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1531 unsigned size)
1532 {
1533 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1534 switch (size) {
1535 case 1: return ldub_phys(addr);
1536 case 2: return lduw_phys(addr);
1537 case 4: return ldl_phys(addr);
1538 default: abort();
1539 }
1540 }
1541
1542 static void watch_mem_write(void *opaque, hwaddr addr,
1543 uint64_t val, unsigned size)
1544 {
1545 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1546 switch (size) {
1547 case 1:
1548 stb_phys(addr, val);
1549 break;
1550 case 2:
1551 stw_phys(addr, val);
1552 break;
1553 case 4:
1554 stl_phys(addr, val);
1555 break;
1556 default: abort();
1557 }
1558 }
1559
1560 static const MemoryRegionOps watch_mem_ops = {
1561 .read = watch_mem_read,
1562 .write = watch_mem_write,
1563 .endianness = DEVICE_NATIVE_ENDIAN,
1564 };
1565
1566 static uint64_t subpage_read(void *opaque, hwaddr addr,
1567 unsigned len)
1568 {
1569 subpage_t *subpage = opaque;
1570 uint8_t buf[4];
1571
1572 #if defined(DEBUG_SUBPAGE)
1573 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1574 subpage, len, addr);
1575 #endif
1576 address_space_read(subpage->as, addr + subpage->base, buf, len);
1577 switch (len) {
1578 case 1:
1579 return ldub_p(buf);
1580 case 2:
1581 return lduw_p(buf);
1582 case 4:
1583 return ldl_p(buf);
1584 default:
1585 abort();
1586 }
1587 }
1588
1589 static void subpage_write(void *opaque, hwaddr addr,
1590 uint64_t value, unsigned len)
1591 {
1592 subpage_t *subpage = opaque;
1593 uint8_t buf[4];
1594
1595 #if defined(DEBUG_SUBPAGE)
1596 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1597 " value %"PRIx64"\n",
1598 __func__, subpage, len, addr, value);
1599 #endif
1600 switch (len) {
1601 case 1:
1602 stb_p(buf, value);
1603 break;
1604 case 2:
1605 stw_p(buf, value);
1606 break;
1607 case 4:
1608 stl_p(buf, value);
1609 break;
1610 default:
1611 abort();
1612 }
1613 address_space_write(subpage->as, addr + subpage->base, buf, len);
1614 }
1615
1616 static bool subpage_accepts(void *opaque, hwaddr addr,
1617 unsigned size, bool is_write)
1618 {
1619 subpage_t *subpage = opaque;
1620 #if defined(DEBUG_SUBPAGE)
1621 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1622 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1623 #endif
1624
1625 return address_space_access_valid(subpage->as, addr + subpage->base,
1626 size, is_write);
1627 }
1628
1629 static const MemoryRegionOps subpage_ops = {
1630 .read = subpage_read,
1631 .write = subpage_write,
1632 .valid.accepts = subpage_accepts,
1633 .endianness = DEVICE_NATIVE_ENDIAN,
1634 };
1635
1636 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1637 uint16_t section)
1638 {
1639 int idx, eidx;
1640
1641 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1642 return -1;
1643 idx = SUBPAGE_IDX(start);
1644 eidx = SUBPAGE_IDX(end);
1645 #if defined(DEBUG_SUBPAGE)
1646 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1647 mmio, start, end, idx, eidx, memory);
1648 #endif
1649 for (; idx <= eidx; idx++) {
1650 mmio->sub_section[idx] = section;
1651 }
1652
1653 return 0;
1654 }
1655
1656 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1657 {
1658 subpage_t *mmio;
1659
1660 mmio = g_malloc0(sizeof(subpage_t));
1661
1662 mmio->as = as;
1663 mmio->base = base;
1664 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1665 "subpage", TARGET_PAGE_SIZE);
1666 mmio->iomem.subpage = true;
1667 #if defined(DEBUG_SUBPAGE)
1668 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1669 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1670 #endif
1671 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1672
1673 return mmio;
1674 }
1675
1676 static uint16_t dummy_section(MemoryRegion *mr)
1677 {
1678 MemoryRegionSection section = {
1679 .mr = mr,
1680 .offset_within_address_space = 0,
1681 .offset_within_region = 0,
1682 .size = int128_2_64(),
1683 };
1684
1685 return phys_section_add(&section);
1686 }
1687
1688 MemoryRegion *iotlb_to_region(hwaddr index)
1689 {
1690 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1691 }
1692
1693 static void io_mem_init(void)
1694 {
1695 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1696 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1697 "unassigned", UINT64_MAX);
1698 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1699 "notdirty", UINT64_MAX);
1700 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1701 "watch", UINT64_MAX);
1702 }
1703
1704 static void mem_begin(MemoryListener *listener)
1705 {
1706 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1707 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1708
1709 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1710 d->as = as;
1711 as->next_dispatch = d;
1712 }
1713
1714 static void mem_commit(MemoryListener *listener)
1715 {
1716 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1717 AddressSpaceDispatch *cur = as->dispatch;
1718 AddressSpaceDispatch *next = as->next_dispatch;
1719
1720 next->nodes = next_map.nodes;
1721 next->sections = next_map.sections;
1722
1723 as->dispatch = next;
1724 g_free(cur);
1725 }
1726
1727 static void core_begin(MemoryListener *listener)
1728 {
1729 uint16_t n;
1730
1731 prev_map = g_new(PhysPageMap, 1);
1732 *prev_map = next_map;
1733
1734 memset(&next_map, 0, sizeof(next_map));
1735 n = dummy_section(&io_mem_unassigned);
1736 assert(n == PHYS_SECTION_UNASSIGNED);
1737 n = dummy_section(&io_mem_notdirty);
1738 assert(n == PHYS_SECTION_NOTDIRTY);
1739 n = dummy_section(&io_mem_rom);
1740 assert(n == PHYS_SECTION_ROM);
1741 n = dummy_section(&io_mem_watch);
1742 assert(n == PHYS_SECTION_WATCH);
1743 }
1744
1745 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1746 * All AddressSpaceDispatch instances have switched to the next map.
1747 */
1748 static void core_commit(MemoryListener *listener)
1749 {
1750 phys_sections_free(prev_map);
1751 }
1752
1753 static void tcg_commit(MemoryListener *listener)
1754 {
1755 CPUState *cpu;
1756
1757 /* since each CPU stores ram addresses in its TLB cache, we must
1758 reset the modified entries */
1759 /* XXX: slow ! */
1760 for (cpu = first_cpu; cpu != NULL; cpu = cpu->next_cpu) {
1761 CPUArchState *env = cpu->env_ptr;
1762
1763 tlb_flush(env, 1);
1764 }
1765 }
1766
1767 static void core_log_global_start(MemoryListener *listener)
1768 {
1769 cpu_physical_memory_set_dirty_tracking(1);
1770 }
1771
1772 static void core_log_global_stop(MemoryListener *listener)
1773 {
1774 cpu_physical_memory_set_dirty_tracking(0);
1775 }
1776
1777 static MemoryListener core_memory_listener = {
1778 .begin = core_begin,
1779 .commit = core_commit,
1780 .log_global_start = core_log_global_start,
1781 .log_global_stop = core_log_global_stop,
1782 .priority = 1,
1783 };
1784
1785 static MemoryListener tcg_memory_listener = {
1786 .commit = tcg_commit,
1787 };
1788
1789 void address_space_init_dispatch(AddressSpace *as)
1790 {
1791 as->dispatch = NULL;
1792 as->dispatch_listener = (MemoryListener) {
1793 .begin = mem_begin,
1794 .commit = mem_commit,
1795 .region_add = mem_add,
1796 .region_nop = mem_add,
1797 .priority = 0,
1798 };
1799 memory_listener_register(&as->dispatch_listener, as);
1800 }
1801
1802 void address_space_destroy_dispatch(AddressSpace *as)
1803 {
1804 AddressSpaceDispatch *d = as->dispatch;
1805
1806 memory_listener_unregister(&as->dispatch_listener);
1807 g_free(d);
1808 as->dispatch = NULL;
1809 }
1810
1811 static void memory_map_init(void)
1812 {
1813 system_memory = g_malloc(sizeof(*system_memory));
1814 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1815 address_space_init(&address_space_memory, system_memory, "memory");
1816
1817 system_io = g_malloc(sizeof(*system_io));
1818 memory_region_init(system_io, NULL, "io", 65536);
1819 address_space_init(&address_space_io, system_io, "I/O");
1820
1821 memory_listener_register(&core_memory_listener, &address_space_memory);
1822 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1823 }
1824
1825 MemoryRegion *get_system_memory(void)
1826 {
1827 return system_memory;
1828 }
1829
1830 MemoryRegion *get_system_io(void)
1831 {
1832 return system_io;
1833 }
1834
1835 #endif /* !defined(CONFIG_USER_ONLY) */
1836
1837 /* physical memory access (slow version, mainly for debug) */
1838 #if defined(CONFIG_USER_ONLY)
1839 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1840 uint8_t *buf, int len, int is_write)
1841 {
1842 int l, flags;
1843 target_ulong page;
1844 void * p;
1845
1846 while (len > 0) {
1847 page = addr & TARGET_PAGE_MASK;
1848 l = (page + TARGET_PAGE_SIZE) - addr;
1849 if (l > len)
1850 l = len;
1851 flags = page_get_flags(page);
1852 if (!(flags & PAGE_VALID))
1853 return -1;
1854 if (is_write) {
1855 if (!(flags & PAGE_WRITE))
1856 return -1;
1857 /* XXX: this code should not depend on lock_user */
1858 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1859 return -1;
1860 memcpy(p, buf, l);
1861 unlock_user(p, addr, l);
1862 } else {
1863 if (!(flags & PAGE_READ))
1864 return -1;
1865 /* XXX: this code should not depend on lock_user */
1866 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1867 return -1;
1868 memcpy(buf, p, l);
1869 unlock_user(p, addr, 0);
1870 }
1871 len -= l;
1872 buf += l;
1873 addr += l;
1874 }
1875 return 0;
1876 }
1877
1878 #else
1879
1880 static void invalidate_and_set_dirty(hwaddr addr,
1881 hwaddr length)
1882 {
1883 if (!cpu_physical_memory_is_dirty(addr)) {
1884 /* invalidate code */
1885 tb_invalidate_phys_page_range(addr, addr + length, 0);
1886 /* set dirty bit */
1887 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1888 }
1889 xen_modified_memory(addr, length);
1890 }
1891
1892 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1893 {
1894 if (memory_region_is_ram(mr)) {
1895 return !(is_write && mr->readonly);
1896 }
1897 if (memory_region_is_romd(mr)) {
1898 return !is_write;
1899 }
1900
1901 return false;
1902 }
1903
1904 static inline int memory_access_size(MemoryRegion *mr, int l, hwaddr addr)
1905 {
1906 if (l >= 4 && (((addr & 3) == 0 || mr->ops->impl.unaligned))) {
1907 return 4;
1908 }
1909 if (l >= 2 && (((addr & 1) == 0) || mr->ops->impl.unaligned)) {
1910 return 2;
1911 }
1912 return 1;
1913 }
1914
1915 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1916 int len, bool is_write)
1917 {
1918 hwaddr l;
1919 uint8_t *ptr;
1920 uint64_t val;
1921 hwaddr addr1;
1922 MemoryRegion *mr;
1923 bool error = false;
1924
1925 while (len > 0) {
1926 l = len;
1927 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1928
1929 if (is_write) {
1930 if (!memory_access_is_direct(mr, is_write)) {
1931 l = memory_access_size(mr, l, addr1);
1932 /* XXX: could force current_cpu to NULL to avoid
1933 potential bugs */
1934 if (l == 4) {
1935 /* 32 bit write access */
1936 val = ldl_p(buf);
1937 error |= io_mem_write(mr, addr1, val, 4);
1938 } else if (l == 2) {
1939 /* 16 bit write access */
1940 val = lduw_p(buf);
1941 error |= io_mem_write(mr, addr1, val, 2);
1942 } else {
1943 /* 8 bit write access */
1944 val = ldub_p(buf);
1945 error |= io_mem_write(mr, addr1, val, 1);
1946 }
1947 } else {
1948 addr1 += memory_region_get_ram_addr(mr);
1949 /* RAM case */
1950 ptr = qemu_get_ram_ptr(addr1);
1951 memcpy(ptr, buf, l);
1952 invalidate_and_set_dirty(addr1, l);
1953 }
1954 } else {
1955 if (!memory_access_is_direct(mr, is_write)) {
1956 /* I/O case */
1957 l = memory_access_size(mr, l, addr1);
1958 if (l == 4) {
1959 /* 32 bit read access */
1960 error |= io_mem_read(mr, addr1, &val, 4);
1961 stl_p(buf, val);
1962 } else if (l == 2) {
1963 /* 16 bit read access */
1964 error |= io_mem_read(mr, addr1, &val, 2);
1965 stw_p(buf, val);
1966 } else {
1967 /* 8 bit read access */
1968 error |= io_mem_read(mr, addr1, &val, 1);
1969 stb_p(buf, val);
1970 }
1971 } else {
1972 /* RAM case */
1973 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1974 memcpy(buf, ptr, l);
1975 }
1976 }
1977 len -= l;
1978 buf += l;
1979 addr += l;
1980 }
1981
1982 return error;
1983 }
1984
1985 bool address_space_write(AddressSpace *as, hwaddr addr,
1986 const uint8_t *buf, int len)
1987 {
1988 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
1989 }
1990
1991 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1992 {
1993 return address_space_rw(as, addr, buf, len, false);
1994 }
1995
1996
1997 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1998 int len, int is_write)
1999 {
2000 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2001 }
2002
2003 /* used for ROM loading : can write in RAM and ROM */
2004 void cpu_physical_memory_write_rom(hwaddr addr,
2005 const uint8_t *buf, int len)
2006 {
2007 hwaddr l;
2008 uint8_t *ptr;
2009 hwaddr addr1;
2010 MemoryRegion *mr;
2011
2012 while (len > 0) {
2013 l = len;
2014 mr = address_space_translate(&address_space_memory,
2015 addr, &addr1, &l, true);
2016
2017 if (!(memory_region_is_ram(mr) ||
2018 memory_region_is_romd(mr))) {
2019 /* do nothing */
2020 } else {
2021 addr1 += memory_region_get_ram_addr(mr);
2022 /* ROM/RAM case */
2023 ptr = qemu_get_ram_ptr(addr1);
2024 memcpy(ptr, buf, l);
2025 invalidate_and_set_dirty(addr1, l);
2026 }
2027 len -= l;
2028 buf += l;
2029 addr += l;
2030 }
2031 }
2032
2033 typedef struct {
2034 MemoryRegion *mr;
2035 void *buffer;
2036 hwaddr addr;
2037 hwaddr len;
2038 } BounceBuffer;
2039
2040 static BounceBuffer bounce;
2041
2042 typedef struct MapClient {
2043 void *opaque;
2044 void (*callback)(void *opaque);
2045 QLIST_ENTRY(MapClient) link;
2046 } MapClient;
2047
2048 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2049 = QLIST_HEAD_INITIALIZER(map_client_list);
2050
2051 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2052 {
2053 MapClient *client = g_malloc(sizeof(*client));
2054
2055 client->opaque = opaque;
2056 client->callback = callback;
2057 QLIST_INSERT_HEAD(&map_client_list, client, link);
2058 return client;
2059 }
2060
2061 static void cpu_unregister_map_client(void *_client)
2062 {
2063 MapClient *client = (MapClient *)_client;
2064
2065 QLIST_REMOVE(client, link);
2066 g_free(client);
2067 }
2068
2069 static void cpu_notify_map_clients(void)
2070 {
2071 MapClient *client;
2072
2073 while (!QLIST_EMPTY(&map_client_list)) {
2074 client = QLIST_FIRST(&map_client_list);
2075 client->callback(client->opaque);
2076 cpu_unregister_map_client(client);
2077 }
2078 }
2079
2080 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2081 {
2082 MemoryRegion *mr;
2083 hwaddr l, xlat;
2084
2085 while (len > 0) {
2086 l = len;
2087 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2088 if (!memory_access_is_direct(mr, is_write)) {
2089 l = memory_access_size(mr, l, addr);
2090 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2091 return false;
2092 }
2093 }
2094
2095 len -= l;
2096 addr += l;
2097 }
2098 return true;
2099 }
2100
2101 /* Map a physical memory region into a host virtual address.
2102 * May map a subset of the requested range, given by and returned in *plen.
2103 * May return NULL if resources needed to perform the mapping are exhausted.
2104 * Use only for reads OR writes - not for read-modify-write operations.
2105 * Use cpu_register_map_client() to know when retrying the map operation is
2106 * likely to succeed.
2107 */
2108 void *address_space_map(AddressSpace *as,
2109 hwaddr addr,
2110 hwaddr *plen,
2111 bool is_write)
2112 {
2113 hwaddr len = *plen;
2114 hwaddr done = 0;
2115 hwaddr l, xlat, base;
2116 MemoryRegion *mr, *this_mr;
2117 ram_addr_t raddr;
2118
2119 if (len == 0) {
2120 return NULL;
2121 }
2122
2123 l = len;
2124 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2125 if (!memory_access_is_direct(mr, is_write)) {
2126 if (bounce.buffer) {
2127 return NULL;
2128 }
2129 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2130 bounce.addr = addr;
2131 bounce.len = l;
2132
2133 memory_region_ref(mr);
2134 bounce.mr = mr;
2135 if (!is_write) {
2136 address_space_read(as, addr, bounce.buffer, l);
2137 }
2138
2139 *plen = l;
2140 return bounce.buffer;
2141 }
2142
2143 base = xlat;
2144 raddr = memory_region_get_ram_addr(mr);
2145
2146 for (;;) {
2147 len -= l;
2148 addr += l;
2149 done += l;
2150 if (len == 0) {
2151 break;
2152 }
2153
2154 l = len;
2155 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2156 if (this_mr != mr || xlat != base + done) {
2157 break;
2158 }
2159 }
2160
2161 memory_region_ref(mr);
2162 *plen = done;
2163 return qemu_ram_ptr_length(raddr + base, plen);
2164 }
2165
2166 /* Unmaps a memory region previously mapped by address_space_map().
2167 * Will also mark the memory as dirty if is_write == 1. access_len gives
2168 * the amount of memory that was actually read or written by the caller.
2169 */
2170 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2171 int is_write, hwaddr access_len)
2172 {
2173 if (buffer != bounce.buffer) {
2174 MemoryRegion *mr;
2175 ram_addr_t addr1;
2176
2177 mr = qemu_ram_addr_from_host(buffer, &addr1);
2178 assert(mr != NULL);
2179 if (is_write) {
2180 while (access_len) {
2181 unsigned l;
2182 l = TARGET_PAGE_SIZE;
2183 if (l > access_len)
2184 l = access_len;
2185 invalidate_and_set_dirty(addr1, l);
2186 addr1 += l;
2187 access_len -= l;
2188 }
2189 }
2190 if (xen_enabled()) {
2191 xen_invalidate_map_cache_entry(buffer);
2192 }
2193 memory_region_unref(mr);
2194 return;
2195 }
2196 if (is_write) {
2197 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2198 }
2199 qemu_vfree(bounce.buffer);
2200 bounce.buffer = NULL;
2201 memory_region_unref(bounce.mr);
2202 cpu_notify_map_clients();
2203 }
2204
2205 void *cpu_physical_memory_map(hwaddr addr,
2206 hwaddr *plen,
2207 int is_write)
2208 {
2209 return address_space_map(&address_space_memory, addr, plen, is_write);
2210 }
2211
2212 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2213 int is_write, hwaddr access_len)
2214 {
2215 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2216 }
2217
2218 /* warning: addr must be aligned */
2219 static inline uint32_t ldl_phys_internal(hwaddr addr,
2220 enum device_endian endian)
2221 {
2222 uint8_t *ptr;
2223 uint64_t val;
2224 MemoryRegion *mr;
2225 hwaddr l = 4;
2226 hwaddr addr1;
2227
2228 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2229 false);
2230 if (l < 4 || !memory_access_is_direct(mr, false)) {
2231 /* I/O case */
2232 io_mem_read(mr, addr1, &val, 4);
2233 #if defined(TARGET_WORDS_BIGENDIAN)
2234 if (endian == DEVICE_LITTLE_ENDIAN) {
2235 val = bswap32(val);
2236 }
2237 #else
2238 if (endian == DEVICE_BIG_ENDIAN) {
2239 val = bswap32(val);
2240 }
2241 #endif
2242 } else {
2243 /* RAM case */
2244 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2245 & TARGET_PAGE_MASK)
2246 + addr1);
2247 switch (endian) {
2248 case DEVICE_LITTLE_ENDIAN:
2249 val = ldl_le_p(ptr);
2250 break;
2251 case DEVICE_BIG_ENDIAN:
2252 val = ldl_be_p(ptr);
2253 break;
2254 default:
2255 val = ldl_p(ptr);
2256 break;
2257 }
2258 }
2259 return val;
2260 }
2261
2262 uint32_t ldl_phys(hwaddr addr)
2263 {
2264 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2265 }
2266
2267 uint32_t ldl_le_phys(hwaddr addr)
2268 {
2269 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2270 }
2271
2272 uint32_t ldl_be_phys(hwaddr addr)
2273 {
2274 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2275 }
2276
2277 /* warning: addr must be aligned */
2278 static inline uint64_t ldq_phys_internal(hwaddr addr,
2279 enum device_endian endian)
2280 {
2281 uint8_t *ptr;
2282 uint64_t val;
2283 MemoryRegion *mr;
2284 hwaddr l = 8;
2285 hwaddr addr1;
2286
2287 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2288 false);
2289 if (l < 8 || !memory_access_is_direct(mr, false)) {
2290 /* I/O case */
2291 io_mem_read(mr, addr1, &val, 8);
2292 #if defined(TARGET_WORDS_BIGENDIAN)
2293 if (endian == DEVICE_LITTLE_ENDIAN) {
2294 val = bswap64(val);
2295 }
2296 #else
2297 if (endian == DEVICE_BIG_ENDIAN) {
2298 val = bswap64(val);
2299 }
2300 #endif
2301 } else {
2302 /* RAM case */
2303 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2304 & TARGET_PAGE_MASK)
2305 + addr1);
2306 switch (endian) {
2307 case DEVICE_LITTLE_ENDIAN:
2308 val = ldq_le_p(ptr);
2309 break;
2310 case DEVICE_BIG_ENDIAN:
2311 val = ldq_be_p(ptr);
2312 break;
2313 default:
2314 val = ldq_p(ptr);
2315 break;
2316 }
2317 }
2318 return val;
2319 }
2320
2321 uint64_t ldq_phys(hwaddr addr)
2322 {
2323 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2324 }
2325
2326 uint64_t ldq_le_phys(hwaddr addr)
2327 {
2328 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2329 }
2330
2331 uint64_t ldq_be_phys(hwaddr addr)
2332 {
2333 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2334 }
2335
2336 /* XXX: optimize */
2337 uint32_t ldub_phys(hwaddr addr)
2338 {
2339 uint8_t val;
2340 cpu_physical_memory_read(addr, &val, 1);
2341 return val;
2342 }
2343
2344 /* warning: addr must be aligned */
2345 static inline uint32_t lduw_phys_internal(hwaddr addr,
2346 enum device_endian endian)
2347 {
2348 uint8_t *ptr;
2349 uint64_t val;
2350 MemoryRegion *mr;
2351 hwaddr l = 2;
2352 hwaddr addr1;
2353
2354 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2355 false);
2356 if (l < 2 || !memory_access_is_direct(mr, false)) {
2357 /* I/O case */
2358 io_mem_read(mr, addr1, &val, 2);
2359 #if defined(TARGET_WORDS_BIGENDIAN)
2360 if (endian == DEVICE_LITTLE_ENDIAN) {
2361 val = bswap16(val);
2362 }
2363 #else
2364 if (endian == DEVICE_BIG_ENDIAN) {
2365 val = bswap16(val);
2366 }
2367 #endif
2368 } else {
2369 /* RAM case */
2370 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2371 & TARGET_PAGE_MASK)
2372 + addr1);
2373 switch (endian) {
2374 case DEVICE_LITTLE_ENDIAN:
2375 val = lduw_le_p(ptr);
2376 break;
2377 case DEVICE_BIG_ENDIAN:
2378 val = lduw_be_p(ptr);
2379 break;
2380 default:
2381 val = lduw_p(ptr);
2382 break;
2383 }
2384 }
2385 return val;
2386 }
2387
2388 uint32_t lduw_phys(hwaddr addr)
2389 {
2390 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2391 }
2392
2393 uint32_t lduw_le_phys(hwaddr addr)
2394 {
2395 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2396 }
2397
2398 uint32_t lduw_be_phys(hwaddr addr)
2399 {
2400 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2401 }
2402
2403 /* warning: addr must be aligned. The ram page is not masked as dirty
2404 and the code inside is not invalidated. It is useful if the dirty
2405 bits are used to track modified PTEs */
2406 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2407 {
2408 uint8_t *ptr;
2409 MemoryRegion *mr;
2410 hwaddr l = 4;
2411 hwaddr addr1;
2412
2413 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2414 true);
2415 if (l < 4 || !memory_access_is_direct(mr, true)) {
2416 io_mem_write(mr, addr1, val, 4);
2417 } else {
2418 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2419 ptr = qemu_get_ram_ptr(addr1);
2420 stl_p(ptr, val);
2421
2422 if (unlikely(in_migration)) {
2423 if (!cpu_physical_memory_is_dirty(addr1)) {
2424 /* invalidate code */
2425 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2426 /* set dirty bit */
2427 cpu_physical_memory_set_dirty_flags(
2428 addr1, (0xff & ~CODE_DIRTY_FLAG));
2429 }
2430 }
2431 }
2432 }
2433
2434 /* warning: addr must be aligned */
2435 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2436 enum device_endian endian)
2437 {
2438 uint8_t *ptr;
2439 MemoryRegion *mr;
2440 hwaddr l = 4;
2441 hwaddr addr1;
2442
2443 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2444 true);
2445 if (l < 4 || !memory_access_is_direct(mr, true)) {
2446 #if defined(TARGET_WORDS_BIGENDIAN)
2447 if (endian == DEVICE_LITTLE_ENDIAN) {
2448 val = bswap32(val);
2449 }
2450 #else
2451 if (endian == DEVICE_BIG_ENDIAN) {
2452 val = bswap32(val);
2453 }
2454 #endif
2455 io_mem_write(mr, addr1, val, 4);
2456 } else {
2457 /* RAM case */
2458 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2459 ptr = qemu_get_ram_ptr(addr1);
2460 switch (endian) {
2461 case DEVICE_LITTLE_ENDIAN:
2462 stl_le_p(ptr, val);
2463 break;
2464 case DEVICE_BIG_ENDIAN:
2465 stl_be_p(ptr, val);
2466 break;
2467 default:
2468 stl_p(ptr, val);
2469 break;
2470 }
2471 invalidate_and_set_dirty(addr1, 4);
2472 }
2473 }
2474
2475 void stl_phys(hwaddr addr, uint32_t val)
2476 {
2477 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2478 }
2479
2480 void stl_le_phys(hwaddr addr, uint32_t val)
2481 {
2482 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2483 }
2484
2485 void stl_be_phys(hwaddr addr, uint32_t val)
2486 {
2487 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2488 }
2489
2490 /* XXX: optimize */
2491 void stb_phys(hwaddr addr, uint32_t val)
2492 {
2493 uint8_t v = val;
2494 cpu_physical_memory_write(addr, &v, 1);
2495 }
2496
2497 /* warning: addr must be aligned */
2498 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2499 enum device_endian endian)
2500 {
2501 uint8_t *ptr;
2502 MemoryRegion *mr;
2503 hwaddr l = 2;
2504 hwaddr addr1;
2505
2506 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2507 true);
2508 if (l < 2 || !memory_access_is_direct(mr, true)) {
2509 #if defined(TARGET_WORDS_BIGENDIAN)
2510 if (endian == DEVICE_LITTLE_ENDIAN) {
2511 val = bswap16(val);
2512 }
2513 #else
2514 if (endian == DEVICE_BIG_ENDIAN) {
2515 val = bswap16(val);
2516 }
2517 #endif
2518 io_mem_write(mr, addr1, val, 2);
2519 } else {
2520 /* RAM case */
2521 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2522 ptr = qemu_get_ram_ptr(addr1);
2523 switch (endian) {
2524 case DEVICE_LITTLE_ENDIAN:
2525 stw_le_p(ptr, val);
2526 break;
2527 case DEVICE_BIG_ENDIAN:
2528 stw_be_p(ptr, val);
2529 break;
2530 default:
2531 stw_p(ptr, val);
2532 break;
2533 }
2534 invalidate_and_set_dirty(addr1, 2);
2535 }
2536 }
2537
2538 void stw_phys(hwaddr addr, uint32_t val)
2539 {
2540 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2541 }
2542
2543 void stw_le_phys(hwaddr addr, uint32_t val)
2544 {
2545 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2546 }
2547
2548 void stw_be_phys(hwaddr addr, uint32_t val)
2549 {
2550 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2551 }
2552
2553 /* XXX: optimize */
2554 void stq_phys(hwaddr addr, uint64_t val)
2555 {
2556 val = tswap64(val);
2557 cpu_physical_memory_write(addr, &val, 8);
2558 }
2559
2560 void stq_le_phys(hwaddr addr, uint64_t val)
2561 {
2562 val = cpu_to_le64(val);
2563 cpu_physical_memory_write(addr, &val, 8);
2564 }
2565
2566 void stq_be_phys(hwaddr addr, uint64_t val)
2567 {
2568 val = cpu_to_be64(val);
2569 cpu_physical_memory_write(addr, &val, 8);
2570 }
2571
2572 /* virtual memory access for debug (includes writing to ROM) */
2573 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2574 uint8_t *buf, int len, int is_write)
2575 {
2576 int l;
2577 hwaddr phys_addr;
2578 target_ulong page;
2579
2580 while (len > 0) {
2581 page = addr & TARGET_PAGE_MASK;
2582 phys_addr = cpu_get_phys_page_debug(env, page);
2583 /* if no physical page mapped, return an error */
2584 if (phys_addr == -1)
2585 return -1;
2586 l = (page + TARGET_PAGE_SIZE) - addr;
2587 if (l > len)
2588 l = len;
2589 phys_addr += (addr & ~TARGET_PAGE_MASK);
2590 if (is_write)
2591 cpu_physical_memory_write_rom(phys_addr, buf, l);
2592 else
2593 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2594 len -= l;
2595 buf += l;
2596 addr += l;
2597 }
2598 return 0;
2599 }
2600 #endif
2601
2602 #if !defined(CONFIG_USER_ONLY)
2603
2604 /*
2605 * A helper function for the _utterly broken_ virtio device model to find out if
2606 * it's running on a big endian machine. Don't do this at home kids!
2607 */
2608 bool virtio_is_big_endian(void);
2609 bool virtio_is_big_endian(void)
2610 {
2611 #if defined(TARGET_WORDS_BIGENDIAN)
2612 return true;
2613 #else
2614 return false;
2615 #endif
2616 }
2617
2618 #endif
2619
2620 #ifndef CONFIG_USER_ONLY
2621 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2622 {
2623 MemoryRegion*mr;
2624 hwaddr l = 1;
2625
2626 mr = address_space_translate(&address_space_memory,
2627 phys_addr, &phys_addr, &l, false);
2628
2629 return !(memory_region_is_ram(mr) ||
2630 memory_region_is_romd(mr));
2631 }
2632
2633 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2634 {
2635 RAMBlock *block;
2636
2637 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2638 func(block->host, block->offset, block->length, opaque);
2639 }
2640 }
2641 #endif
Qemu copy for testing