]> git.proxmox.com Git - qemu.git/blob - exec.c
projects / qemu.git / blob
? search:


482dd801df238c7b63823c560fd964e11a73e55d
[qemu.git] / exec.c
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
51
52 #include "exec/memory-internal.h"
53
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 typedef PhysPageEntry Node[L2_SIZE];
92
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101 };
102
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
110
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
115
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
124
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
127
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
133
134 static MemoryRegion io_mem_watch;
135 #endif
136
137 #if !defined(CONFIG_USER_ONLY)
138
139 static void phys_map_node_reserve(unsigned nodes)
140 {
141 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
142 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
143 16);
144 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
145 next_map.nodes_nb + nodes);
146 next_map.nodes = g_renew(Node, next_map.nodes,
147 next_map.nodes_nb_alloc);
148 }
149 }
150
151 static uint16_t phys_map_node_alloc(void)
152 {
153 unsigned i;
154 uint16_t ret;
155
156 ret = next_map.nodes_nb++;
157 assert(ret != PHYS_MAP_NODE_NIL);
158 assert(ret != next_map.nodes_nb_alloc);
159 for (i = 0; i < L2_SIZE; ++i) {
160 next_map.nodes[ret][i].is_leaf = 0;
161 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
162 }
163 return ret;
164 }
165
166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
167 hwaddr *nb, uint16_t leaf,
168 int level)
169 {
170 PhysPageEntry *p;
171 int i;
172 hwaddr step = (hwaddr)1 << (level * L2_BITS);
173
174 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
175 lp->ptr = phys_map_node_alloc();
176 p = next_map.nodes[lp->ptr];
177 if (level == 0) {
178 for (i = 0; i < L2_SIZE; i++) {
179 p[i].is_leaf = 1;
180 p[i].ptr = PHYS_SECTION_UNASSIGNED;
181 }
182 }
183 } else {
184 p = next_map.nodes[lp->ptr];
185 }
186 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
187
188 while (*nb && lp < &p[L2_SIZE]) {
189 if ((*index & (step - 1)) == 0 && *nb >= step) {
190 lp->is_leaf = true;
191 lp->ptr = leaf;
192 *index += step;
193 *nb -= step;
194 } else {
195 phys_page_set_level(lp, index, nb, leaf, level - 1);
196 }
197 ++lp;
198 }
199 }
200
201 static void phys_page_set(AddressSpaceDispatch *d,
202 hwaddr index, hwaddr nb,
203 uint16_t leaf)
204 {
205 /* Wildly overreserve - it doesn't matter much. */
206 phys_map_node_reserve(3 * P_L2_LEVELS);
207
208 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
209 }
210
211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
212 Node *nodes, MemoryRegionSection *sections)
213 {
214 PhysPageEntry *p;
215 int i;
216
217 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
218 if (lp.ptr == PHYS_MAP_NODE_NIL) {
219 return &sections[PHYS_SECTION_UNASSIGNED];
220 }
221 p = nodes[lp.ptr];
222 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
223 }
224 return &sections[lp.ptr];
225 }
226
227 bool memory_region_is_unassigned(MemoryRegion *mr)
228 {
229 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
230 && mr != &io_mem_watch;
231 }
232
233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
234 hwaddr addr,
235 bool resolve_subpage)
236 {
237 MemoryRegionSection *section;
238 subpage_t *subpage;
239
240 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
241 d->nodes, d->sections);
242 if (resolve_subpage && section->mr->subpage) {
243 subpage = container_of(section->mr, subpage_t, iomem);
244 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
245 }
246 return section;
247 }
248
249 static MemoryRegionSection *
250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
251 hwaddr *plen, bool resolve_subpage)
252 {
253 MemoryRegionSection *section;
254 Int128 diff;
255
256 section = address_space_lookup_region(d, addr, resolve_subpage);
257 /* Compute offset within MemoryRegionSection */
258 addr -= section->offset_within_address_space;
259
260 /* Compute offset within MemoryRegion */
261 *xlat = addr + section->offset_within_region;
262
263 diff = int128_sub(section->mr->size, int128_make64(addr));
264 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
265 return section;
266 }
267
268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
269 hwaddr *xlat, hwaddr *plen,
270 bool is_write)
271 {
272 IOMMUTLBEntry iotlb;
273 MemoryRegionSection *section;
274 MemoryRegion *mr;
275 hwaddr len = *plen;
276
277 for (;;) {
278 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
279 mr = section->mr;
280
281 if (!mr->iommu_ops) {
282 break;
283 }
284
285 iotlb = mr->iommu_ops->translate(mr, addr);
286 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
287 | (addr & iotlb.addr_mask));
288 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
289 if (!(iotlb.perm & (1 << is_write))) {
290 mr = &io_mem_unassigned;
291 break;
292 }
293
294 as = iotlb.target_as;
295 }
296
297 *plen = len;
298 *xlat = addr;
299 return mr;
300 }
301
302 MemoryRegionSection *
303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
304 hwaddr *plen)
305 {
306 MemoryRegionSection *section;
307 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
308
309 assert(!section->mr->iommu_ops);
310 return section;
311 }
312 #endif
313
314 void cpu_exec_init_all(void)
315 {
316 #if !defined(CONFIG_USER_ONLY)
317 qemu_mutex_init(&ram_list.mutex);
318 memory_map_init();
319 io_mem_init();
320 #endif
321 }
322
323 #if !defined(CONFIG_USER_ONLY)
324
325 static int cpu_common_post_load(void *opaque, int version_id)
326 {
327 CPUState *cpu = opaque;
328
329 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
330 version_id is increased. */
331 cpu->interrupt_request &= ~0x01;
332 tlb_flush(cpu->env_ptr, 1);
333
334 return 0;
335 }
336
337 const VMStateDescription vmstate_cpu_common = {
338 .name = "cpu_common",
339 .version_id = 1,
340 .minimum_version_id = 1,
341 .minimum_version_id_old = 1,
342 .post_load = cpu_common_post_load,
343 .fields = (VMStateField []) {
344 VMSTATE_UINT32(halted, CPUState),
345 VMSTATE_UINT32(interrupt_request, CPUState),
346 VMSTATE_END_OF_LIST()
347 }
348 };
349
350 #endif
351
352 CPUState *qemu_get_cpu(int index)
353 {
354 CPUState *cpu;
355
356 CPU_FOREACH(cpu) {
357 if (cpu->cpu_index == index) {
358 return cpu;
359 }
360 }
361
362 return NULL;
363 }
364
365 void cpu_exec_init(CPUArchState *env)
366 {
367 CPUState *cpu = ENV_GET_CPU(env);
368 CPUClass *cc = CPU_GET_CLASS(cpu);
369 CPUState *some_cpu;
370 int cpu_index;
371
372 #if defined(CONFIG_USER_ONLY)
373 cpu_list_lock();
374 #endif
375 cpu_index = 0;
376 CPU_FOREACH(some_cpu) {
377 cpu_index++;
378 }
379 cpu->cpu_index = cpu_index;
380 cpu->numa_node = 0;
381 QTAILQ_INIT(&env->breakpoints);
382 QTAILQ_INIT(&env->watchpoints);
383 #ifndef CONFIG_USER_ONLY
384 cpu->thread_id = qemu_get_thread_id();
385 #endif
386 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
387 #if defined(CONFIG_USER_ONLY)
388 cpu_list_unlock();
389 #endif
390 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
391 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
392 }
393 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
394 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
395 cpu_save, cpu_load, env);
396 assert(cc->vmsd == NULL);
397 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
398 #endif
399 if (cc->vmsd != NULL) {
400 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
401 }
402 }
403
404 #if defined(TARGET_HAS_ICE)
405 #if defined(CONFIG_USER_ONLY)
406 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
407 {
408 tb_invalidate_phys_page_range(pc, pc + 1, 0);
409 }
410 #else
411 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
412 {
413 tb_invalidate_phys_addr(cpu_get_phys_page_debug(cpu, pc) |
414 (pc & ~TARGET_PAGE_MASK));
415 }
416 #endif
417 #endif /* TARGET_HAS_ICE */
418
419 #if defined(CONFIG_USER_ONLY)
420 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
421
422 {
423 }
424
425 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
426 int flags, CPUWatchpoint **watchpoint)
427 {
428 return -ENOSYS;
429 }
430 #else
431 /* Add a watchpoint. */
432 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
433 int flags, CPUWatchpoint **watchpoint)
434 {
435 target_ulong len_mask = ~(len - 1);
436 CPUWatchpoint *wp;
437
438 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
439 if ((len & (len - 1)) || (addr & ~len_mask) ||
440 len == 0 || len > TARGET_PAGE_SIZE) {
441 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
442 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
443 return -EINVAL;
444 }
445 wp = g_malloc(sizeof(*wp));
446
447 wp->vaddr = addr;
448 wp->len_mask = len_mask;
449 wp->flags = flags;
450
451 /* keep all GDB-injected watchpoints in front */
452 if (flags & BP_GDB)
453 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
454 else
455 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
456
457 tlb_flush_page(env, addr);
458
459 if (watchpoint)
460 *watchpoint = wp;
461 return 0;
462 }
463
464 /* Remove a specific watchpoint. */
465 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
466 int flags)
467 {
468 target_ulong len_mask = ~(len - 1);
469 CPUWatchpoint *wp;
470
471 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
472 if (addr == wp->vaddr && len_mask == wp->len_mask
473 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
474 cpu_watchpoint_remove_by_ref(env, wp);
475 return 0;
476 }
477 }
478 return -ENOENT;
479 }
480
481 /* Remove a specific watchpoint by reference. */
482 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
483 {
484 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
485
486 tlb_flush_page(env, watchpoint->vaddr);
487
488 g_free(watchpoint);
489 }
490
491 /* Remove all matching watchpoints. */
492 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
493 {
494 CPUWatchpoint *wp, *next;
495
496 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
497 if (wp->flags & mask)
498 cpu_watchpoint_remove_by_ref(env, wp);
499 }
500 }
501 #endif
502
503 /* Add a breakpoint. */
504 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
505 CPUBreakpoint **breakpoint)
506 {
507 #if defined(TARGET_HAS_ICE)
508 CPUBreakpoint *bp;
509
510 bp = g_malloc(sizeof(*bp));
511
512 bp->pc = pc;
513 bp->flags = flags;
514
515 /* keep all GDB-injected breakpoints in front */
516 if (flags & BP_GDB) {
517 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
518 } else {
519 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
520 }
521
522 breakpoint_invalidate(ENV_GET_CPU(env), pc);
523
524 if (breakpoint) {
525 *breakpoint = bp;
526 }
527 return 0;
528 #else
529 return -ENOSYS;
530 #endif
531 }
532
533 /* Remove a specific breakpoint. */
534 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
535 {
536 #if defined(TARGET_HAS_ICE)
537 CPUBreakpoint *bp;
538
539 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
540 if (bp->pc == pc && bp->flags == flags) {
541 cpu_breakpoint_remove_by_ref(env, bp);
542 return 0;
543 }
544 }
545 return -ENOENT;
546 #else
547 return -ENOSYS;
548 #endif
549 }
550
551 /* Remove a specific breakpoint by reference. */
552 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
553 {
554 #if defined(TARGET_HAS_ICE)
555 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
556
557 breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
558
559 g_free(breakpoint);
560 #endif
561 }
562
563 /* Remove all matching breakpoints. */
564 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
565 {
566 #if defined(TARGET_HAS_ICE)
567 CPUBreakpoint *bp, *next;
568
569 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
570 if (bp->flags & mask)
571 cpu_breakpoint_remove_by_ref(env, bp);
572 }
573 #endif
574 }
575
576 /* enable or disable single step mode. EXCP_DEBUG is returned by the
577 CPU loop after each instruction */
578 void cpu_single_step(CPUState *cpu, int enabled)
579 {
580 #if defined(TARGET_HAS_ICE)
581 if (cpu->singlestep_enabled != enabled) {
582 cpu->singlestep_enabled = enabled;
583 if (kvm_enabled()) {
584 kvm_update_guest_debug(cpu, 0);
585 } else {
586 /* must flush all the translated code to avoid inconsistencies */
587 /* XXX: only flush what is necessary */
588 CPUArchState *env = cpu->env_ptr;
589 tb_flush(env);
590 }
591 }
592 #endif
593 }
594
595 void cpu_abort(CPUArchState *env, const char *fmt, ...)
596 {
597 CPUState *cpu = ENV_GET_CPU(env);
598 va_list ap;
599 va_list ap2;
600
601 va_start(ap, fmt);
602 va_copy(ap2, ap);
603 fprintf(stderr, "qemu: fatal: ");
604 vfprintf(stderr, fmt, ap);
605 fprintf(stderr, "\n");
606 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
607 if (qemu_log_enabled()) {
608 qemu_log("qemu: fatal: ");
609 qemu_log_vprintf(fmt, ap2);
610 qemu_log("\n");
611 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
612 qemu_log_flush();
613 qemu_log_close();
614 }
615 va_end(ap2);
616 va_end(ap);
617 #if defined(CONFIG_USER_ONLY)
618 {
619 struct sigaction act;
620 sigfillset(&act.sa_mask);
621 act.sa_handler = SIG_DFL;
622 sigaction(SIGABRT, &act, NULL);
623 }
624 #endif
625 abort();
626 }
627
628 CPUArchState *cpu_copy(CPUArchState *env)
629 {
630 CPUArchState *new_env = cpu_init(env->cpu_model_str);
631 #if defined(TARGET_HAS_ICE)
632 CPUBreakpoint *bp;
633 CPUWatchpoint *wp;
634 #endif
635
636 /* Reset non arch specific state */
637 cpu_reset(ENV_GET_CPU(new_env));
638
639 /* Copy arch specific state into the new CPU */
640 memcpy(new_env, env, sizeof(CPUArchState));
641
642 /* Clone all break/watchpoints.
643 Note: Once we support ptrace with hw-debug register access, make sure
644 BP_CPU break/watchpoints are handled correctly on clone. */
645 QTAILQ_INIT(&env->breakpoints);
646 QTAILQ_INIT(&env->watchpoints);
647 #if defined(TARGET_HAS_ICE)
648 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
649 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
650 }
651 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
652 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
653 wp->flags, NULL);
654 }
655 #endif
656
657 return new_env;
658 }
659
660 #if !defined(CONFIG_USER_ONLY)
661 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
662 uintptr_t length)
663 {
664 uintptr_t start1;
665
666 /* we modify the TLB cache so that the dirty bit will be set again
667 when accessing the range */
668 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
669 /* Check that we don't span multiple blocks - this breaks the
670 address comparisons below. */
671 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
672 != (end - 1) - start) {
673 abort();
674 }
675 cpu_tlb_reset_dirty_all(start1, length);
676
677 }
678
679 /* Note: start and end must be within the same ram block. */
680 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
681 int dirty_flags)
682 {
683 uintptr_t length;
684
685 start &= TARGET_PAGE_MASK;
686 end = TARGET_PAGE_ALIGN(end);
687
688 length = end - start;
689 if (length == 0)
690 return;
691 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
692
693 if (tcg_enabled()) {
694 tlb_reset_dirty_range_all(start, end, length);
695 }
696 }
697
698 static int cpu_physical_memory_set_dirty_tracking(int enable)
699 {
700 int ret = 0;
701 in_migration = enable;
702 return ret;
703 }
704
705 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
706 MemoryRegionSection *section,
707 target_ulong vaddr,
708 hwaddr paddr, hwaddr xlat,
709 int prot,
710 target_ulong *address)
711 {
712 hwaddr iotlb;
713 CPUWatchpoint *wp;
714
715 if (memory_region_is_ram(section->mr)) {
716 /* Normal RAM. */
717 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
718 + xlat;
719 if (!section->readonly) {
720 iotlb |= PHYS_SECTION_NOTDIRTY;
721 } else {
722 iotlb |= PHYS_SECTION_ROM;
723 }
724 } else {
725 iotlb = section - address_space_memory.dispatch->sections;
726 iotlb += xlat;
727 }
728
729 /* Make accesses to pages with watchpoints go via the
730 watchpoint trap routines. */
731 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
732 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
733 /* Avoid trapping reads of pages with a write breakpoint. */
734 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
735 iotlb = PHYS_SECTION_WATCH + paddr;
736 *address |= TLB_MMIO;
737 break;
738 }
739 }
740 }
741
742 return iotlb;
743 }
744 #endif /* defined(CONFIG_USER_ONLY) */
745
746 #if !defined(CONFIG_USER_ONLY)
747
748 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
749 uint16_t section);
750 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
751
752 static uint16_t phys_section_add(MemoryRegionSection *section)
753 {
754 /* The physical section number is ORed with a page-aligned
755 * pointer to produce the iotlb entries. Thus it should
756 * never overflow into the page-aligned value.
757 */
758 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
759
760 if (next_map.sections_nb == next_map.sections_nb_alloc) {
761 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
762 16);
763 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
764 next_map.sections_nb_alloc);
765 }
766 next_map.sections[next_map.sections_nb] = *section;
767 memory_region_ref(section->mr);
768 return next_map.sections_nb++;
769 }
770
771 static void phys_section_destroy(MemoryRegion *mr)
772 {
773 memory_region_unref(mr);
774
775 if (mr->subpage) {
776 subpage_t *subpage = container_of(mr, subpage_t, iomem);
777 memory_region_destroy(&subpage->iomem);
778 g_free(subpage);
779 }
780 }
781
782 static void phys_sections_free(PhysPageMap *map)
783 {
784 while (map->sections_nb > 0) {
785 MemoryRegionSection *section = &map->sections[--map->sections_nb];
786 phys_section_destroy(section->mr);
787 }
788 g_free(map->sections);
789 g_free(map->nodes);
790 g_free(map);
791 }
792
793 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
794 {
795 subpage_t *subpage;
796 hwaddr base = section->offset_within_address_space
797 & TARGET_PAGE_MASK;
798 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
799 next_map.nodes, next_map.sections);
800 MemoryRegionSection subsection = {
801 .offset_within_address_space = base,
802 .size = int128_make64(TARGET_PAGE_SIZE),
803 };
804 hwaddr start, end;
805
806 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
807
808 if (!(existing->mr->subpage)) {
809 subpage = subpage_init(d->as, base);
810 subsection.mr = &subpage->iomem;
811 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
812 phys_section_add(&subsection));
813 } else {
814 subpage = container_of(existing->mr, subpage_t, iomem);
815 }
816 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
817 end = start + int128_get64(section->size) - 1;
818 subpage_register(subpage, start, end, phys_section_add(section));
819 }
820
821
822 static void register_multipage(AddressSpaceDispatch *d,
823 MemoryRegionSection *section)
824 {
825 hwaddr start_addr = section->offset_within_address_space;
826 uint16_t section_index = phys_section_add(section);
827 uint64_t num_pages = int128_get64(int128_rshift(section->size,
828 TARGET_PAGE_BITS));
829
830 assert(num_pages);
831 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
832 }
833
834 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
835 {
836 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
837 AddressSpaceDispatch *d = as->next_dispatch;
838 MemoryRegionSection now = *section, remain = *section;
839 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
840
841 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
842 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
843 - now.offset_within_address_space;
844
845 now.size = int128_min(int128_make64(left), now.size);
846 register_subpage(d, &now);
847 } else {
848 now.size = int128_zero();
849 }
850 while (int128_ne(remain.size, now.size)) {
851 remain.size = int128_sub(remain.size, now.size);
852 remain.offset_within_address_space += int128_get64(now.size);
853 remain.offset_within_region += int128_get64(now.size);
854 now = remain;
855 if (int128_lt(remain.size, page_size)) {
856 register_subpage(d, &now);
857 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
858 now.size = page_size;
859 register_subpage(d, &now);
860 } else {
861 now.size = int128_and(now.size, int128_neg(page_size));
862 register_multipage(d, &now);
863 }
864 }
865 }
866
867 void qemu_flush_coalesced_mmio_buffer(void)
868 {
869 if (kvm_enabled())
870 kvm_flush_coalesced_mmio_buffer();
871 }
872
873 void qemu_mutex_lock_ramlist(void)
874 {
875 qemu_mutex_lock(&ram_list.mutex);
876 }
877
878 void qemu_mutex_unlock_ramlist(void)
879 {
880 qemu_mutex_unlock(&ram_list.mutex);
881 }
882
883 #if defined(__linux__) && !defined(TARGET_S390X)
884
885 #include <sys/vfs.h>
886
887 #define HUGETLBFS_MAGIC 0x958458f6
888
889 static long gethugepagesize(const char *path)
890 {
891 struct statfs fs;
892 int ret;
893
894 do {
895 ret = statfs(path, &fs);
896 } while (ret != 0 && errno == EINTR);
897
898 if (ret != 0) {
899 perror(path);
900 return 0;
901 }
902
903 if (fs.f_type != HUGETLBFS_MAGIC)
904 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
905
906 return fs.f_bsize;
907 }
908
909 static void *file_ram_alloc(RAMBlock *block,
910 ram_addr_t memory,
911 const char *path)
912 {
913 char *filename;
914 char *sanitized_name;
915 char *c;
916 void *area;
917 int fd;
918 #ifdef MAP_POPULATE
919 int flags;
920 #endif
921 unsigned long hpagesize;
922
923 hpagesize = gethugepagesize(path);
924 if (!hpagesize) {
925 return NULL;
926 }
927
928 if (memory < hpagesize) {
929 return NULL;
930 }
931
932 if (kvm_enabled() && !kvm_has_sync_mmu()) {
933 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
934 return NULL;
935 }
936
937 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
938 sanitized_name = g_strdup(block->mr->name);
939 for (c = sanitized_name; *c != '\0'; c++) {
940 if (*c == '/')
941 *c = '_';
942 }
943
944 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
945 sanitized_name);
946 g_free(sanitized_name);
947
948 fd = mkstemp(filename);
949 if (fd < 0) {
950 perror("unable to create backing store for hugepages");
951 g_free(filename);
952 return NULL;
953 }
954 unlink(filename);
955 g_free(filename);
956
957 memory = (memory+hpagesize-1) & ~(hpagesize-1);
958
959 /*
960 * ftruncate is not supported by hugetlbfs in older
961 * hosts, so don't bother bailing out on errors.
962 * If anything goes wrong with it under other filesystems,
963 * mmap will fail.
964 */
965 if (ftruncate(fd, memory))
966 perror("ftruncate");
967
968 #ifdef MAP_POPULATE
969 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
970 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
971 * to sidestep this quirk.
972 */
973 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
974 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
975 #else
976 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
977 #endif
978 if (area == MAP_FAILED) {
979 perror("file_ram_alloc: can't mmap RAM pages");
980 close(fd);
981 return (NULL);
982 }
983 block->fd = fd;
984 return area;
985 }
986 #endif
987
988 static ram_addr_t find_ram_offset(ram_addr_t size)
989 {
990 RAMBlock *block, *next_block;
991 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
992
993 assert(size != 0); /* it would hand out same offset multiple times */
994
995 if (QTAILQ_EMPTY(&ram_list.blocks))
996 return 0;
997
998 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
999 ram_addr_t end, next = RAM_ADDR_MAX;
1000
1001 end = block->offset + block->length;
1002
1003 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1004 if (next_block->offset >= end) {
1005 next = MIN(next, next_block->offset);
1006 }
1007 }
1008 if (next - end >= size && next - end < mingap) {
1009 offset = end;
1010 mingap = next - end;
1011 }
1012 }
1013
1014 if (offset == RAM_ADDR_MAX) {
1015 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1016 (uint64_t)size);
1017 abort();
1018 }
1019
1020 return offset;
1021 }
1022
1023 ram_addr_t last_ram_offset(void)
1024 {
1025 RAMBlock *block;
1026 ram_addr_t last = 0;
1027
1028 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1029 last = MAX(last, block->offset + block->length);
1030
1031 return last;
1032 }
1033
1034 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1035 {
1036 int ret;
1037
1038 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1039 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1040 "dump-guest-core", true)) {
1041 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1042 if (ret) {
1043 perror("qemu_madvise");
1044 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1045 "but dump_guest_core=off specified\n");
1046 }
1047 }
1048 }
1049
1050 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1051 {
1052 RAMBlock *new_block, *block;
1053
1054 new_block = NULL;
1055 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1056 if (block->offset == addr) {
1057 new_block = block;
1058 break;
1059 }
1060 }
1061 assert(new_block);
1062 assert(!new_block->idstr[0]);
1063
1064 if (dev) {
1065 char *id = qdev_get_dev_path(dev);
1066 if (id) {
1067 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1068 g_free(id);
1069 }
1070 }
1071 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1072
1073 /* This assumes the iothread lock is taken here too. */
1074 qemu_mutex_lock_ramlist();
1075 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1076 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1077 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1078 new_block->idstr);
1079 abort();
1080 }
1081 }
1082 qemu_mutex_unlock_ramlist();
1083 }
1084
1085 static int memory_try_enable_merging(void *addr, size_t len)
1086 {
1087 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1088 /* disabled by the user */
1089 return 0;
1090 }
1091
1092 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1093 }
1094
1095 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1096 MemoryRegion *mr)
1097 {
1098 RAMBlock *block, *new_block;
1099
1100 size = TARGET_PAGE_ALIGN(size);
1101 new_block = g_malloc0(sizeof(*new_block));
1102
1103 /* This assumes the iothread lock is taken here too. */
1104 qemu_mutex_lock_ramlist();
1105 new_block->mr = mr;
1106 new_block->offset = find_ram_offset(size);
1107 if (host) {
1108 new_block->host = host;
1109 new_block->flags |= RAM_PREALLOC_MASK;
1110 } else if (xen_enabled()) {
1111 if (mem_path) {
1112 fprintf(stderr, "-mem-path not supported with Xen\n");
1113 exit(1);
1114 }
1115 xen_ram_alloc(new_block->offset, size, mr);
1116 } else {
1117 if (mem_path) {
1118 #if defined (__linux__) && !defined(TARGET_S390X)
1119 new_block->host = file_ram_alloc(new_block, size, mem_path);
1120 #else
1121 fprintf(stderr, "-mem-path option unsupported\n");
1122 exit(1);
1123 #endif
1124 }
1125 if (!new_block->host) {
1126 if (kvm_enabled()) {
1127 /* some s390/kvm configurations have special constraints */
1128 new_block->host = kvm_ram_alloc(size);
1129 } else {
1130 new_block->host = qemu_anon_ram_alloc(size);
1131 }
1132 memory_try_enable_merging(new_block->host, size);
1133 }
1134 }
1135 new_block->length = size;
1136
1137 /* Keep the list sorted from biggest to smallest block. */
1138 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1139 if (block->length < new_block->length) {
1140 break;
1141 }
1142 }
1143 if (block) {
1144 QTAILQ_INSERT_BEFORE(block, new_block, next);
1145 } else {
1146 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1147 }
1148 ram_list.mru_block = NULL;
1149
1150 ram_list.version++;
1151 qemu_mutex_unlock_ramlist();
1152
1153 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1154 last_ram_offset() >> TARGET_PAGE_BITS);
1155 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1156 0, size >> TARGET_PAGE_BITS);
1157 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1158
1159 qemu_ram_setup_dump(new_block->host, size);
1160 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1161
1162 if (kvm_enabled())
1163 kvm_setup_guest_memory(new_block->host, size);
1164
1165 return new_block->offset;
1166 }
1167
1168 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1169 {
1170 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1171 }
1172
1173 void qemu_ram_free_from_ptr(ram_addr_t addr)
1174 {
1175 RAMBlock *block;
1176
1177 /* This assumes the iothread lock is taken here too. */
1178 qemu_mutex_lock_ramlist();
1179 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1180 if (addr == block->offset) {
1181 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1182 ram_list.mru_block = NULL;
1183 ram_list.version++;
1184 g_free(block);
1185 break;
1186 }
1187 }
1188 qemu_mutex_unlock_ramlist();
1189 }
1190
1191 void qemu_ram_free(ram_addr_t addr)
1192 {
1193 RAMBlock *block;
1194
1195 /* This assumes the iothread lock is taken here too. */
1196 qemu_mutex_lock_ramlist();
1197 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1198 if (addr == block->offset) {
1199 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1200 ram_list.mru_block = NULL;
1201 ram_list.version++;
1202 if (block->flags & RAM_PREALLOC_MASK) {
1203 ;
1204 } else if (xen_enabled()) {
1205 xen_invalidate_map_cache_entry(block->host);
1206 } else if (mem_path) {
1207 #if defined (__linux__) && !defined(TARGET_S390X)
1208 if (block->fd) {
1209 munmap(block->host, block->length);
1210 close(block->fd);
1211 } else {
1212 qemu_anon_ram_free(block->host, block->length);
1213 }
1214 #else
1215 abort();
1216 #endif
1217 } else {
1218 qemu_anon_ram_free(block->host, block->length);
1219 }
1220 g_free(block);
1221 break;
1222 }
1223 }
1224 qemu_mutex_unlock_ramlist();
1225
1226 }
1227
1228 #ifndef _WIN32
1229 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1230 {
1231 RAMBlock *block;
1232 ram_addr_t offset;
1233 int flags;
1234 void *area, *vaddr;
1235
1236 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1237 offset = addr - block->offset;
1238 if (offset < block->length) {
1239 vaddr = block->host + offset;
1240 if (block->flags & RAM_PREALLOC_MASK) {
1241 ;
1242 } else if (xen_enabled()) {
1243 abort();
1244 } else {
1245 flags = MAP_FIXED;
1246 munmap(vaddr, length);
1247 if (mem_path) {
1248 #if defined(__linux__) && !defined(TARGET_S390X)
1249 if (block->fd) {
1250 #ifdef MAP_POPULATE
1251 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1252 MAP_PRIVATE;
1253 #else
1254 flags |= MAP_PRIVATE;
1255 #endif
1256 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1257 flags, block->fd, offset);
1258 } else {
1259 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1260 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1261 flags, -1, 0);
1262 }
1263 #else
1264 abort();
1265 #endif
1266 } else {
1267 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1268 flags |= MAP_SHARED | MAP_ANONYMOUS;
1269 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1270 flags, -1, 0);
1271 #else
1272 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1273 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1274 flags, -1, 0);
1275 #endif
1276 }
1277 if (area != vaddr) {
1278 fprintf(stderr, "Could not remap addr: "
1279 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1280 length, addr);
1281 exit(1);
1282 }
1283 memory_try_enable_merging(vaddr, length);
1284 qemu_ram_setup_dump(vaddr, length);
1285 }
1286 return;
1287 }
1288 }
1289 }
1290 #endif /* !_WIN32 */
1291
1292 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1293 {
1294 RAMBlock *block;
1295
1296 /* The list is protected by the iothread lock here. */
1297 block = ram_list.mru_block;
1298 if (block && addr - block->offset < block->length) {
1299 goto found;
1300 }
1301 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1302 if (addr - block->offset < block->length) {
1303 goto found;
1304 }
1305 }
1306
1307 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1308 abort();
1309
1310 found:
1311 ram_list.mru_block = block;
1312 return block;
1313 }
1314
1315 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1316 With the exception of the softmmu code in this file, this should
1317 only be used for local memory (e.g. video ram) that the device owns,
1318 and knows it isn't going to access beyond the end of the block.
1319
1320 It should not be used for general purpose DMA.
1321 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1322 */
1323 void *qemu_get_ram_ptr(ram_addr_t addr)
1324 {
1325 RAMBlock *block = qemu_get_ram_block(addr);
1326
1327 if (xen_enabled()) {
1328 /* We need to check if the requested address is in the RAM
1329 * because we don't want to map the entire memory in QEMU.
1330 * In that case just map until the end of the page.
1331 */
1332 if (block->offset == 0) {
1333 return xen_map_cache(addr, 0, 0);
1334 } else if (block->host == NULL) {
1335 block->host =
1336 xen_map_cache(block->offset, block->length, 1);
1337 }
1338 }
1339 return block->host + (addr - block->offset);
1340 }
1341
1342 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1343 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1344 *
1345 * ??? Is this still necessary?
1346 */
1347 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1348 {
1349 RAMBlock *block;
1350
1351 /* The list is protected by the iothread lock here. */
1352 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1353 if (addr - block->offset < block->length) {
1354 if (xen_enabled()) {
1355 /* We need to check if the requested address is in the RAM
1356 * because we don't want to map the entire memory in QEMU.
1357 * In that case just map until the end of the page.
1358 */
1359 if (block->offset == 0) {
1360 return xen_map_cache(addr, 0, 0);
1361 } else if (block->host == NULL) {
1362 block->host =
1363 xen_map_cache(block->offset, block->length, 1);
1364 }
1365 }
1366 return block->host + (addr - block->offset);
1367 }
1368 }
1369
1370 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1371 abort();
1372
1373 return NULL;
1374 }
1375
1376 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1377 * but takes a size argument */
1378 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1379 {
1380 if (*size == 0) {
1381 return NULL;
1382 }
1383 if (xen_enabled()) {
1384 return xen_map_cache(addr, *size, 1);
1385 } else {
1386 RAMBlock *block;
1387
1388 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1389 if (addr - block->offset < block->length) {
1390 if (addr - block->offset + *size > block->length)
1391 *size = block->length - addr + block->offset;
1392 return block->host + (addr - block->offset);
1393 }
1394 }
1395
1396 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1397 abort();
1398 }
1399 }
1400
1401 /* Some of the softmmu routines need to translate from a host pointer
1402 (typically a TLB entry) back to a ram offset. */
1403 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1404 {
1405 RAMBlock *block;
1406 uint8_t *host = ptr;
1407
1408 if (xen_enabled()) {
1409 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1410 return qemu_get_ram_block(*ram_addr)->mr;
1411 }
1412
1413 block = ram_list.mru_block;
1414 if (block && block->host && host - block->host < block->length) {
1415 goto found;
1416 }
1417
1418 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1419 /* This case append when the block is not mapped. */
1420 if (block->host == NULL) {
1421 continue;
1422 }
1423 if (host - block->host < block->length) {
1424 goto found;
1425 }
1426 }
1427
1428 return NULL;
1429
1430 found:
1431 *ram_addr = block->offset + (host - block->host);
1432 return block->mr;
1433 }
1434
1435 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1436 uint64_t val, unsigned size)
1437 {
1438 int dirty_flags;
1439 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1440 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1441 tb_invalidate_phys_page_fast(ram_addr, size);
1442 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1443 }
1444 switch (size) {
1445 case 1:
1446 stb_p(qemu_get_ram_ptr(ram_addr), val);
1447 break;
1448 case 2:
1449 stw_p(qemu_get_ram_ptr(ram_addr), val);
1450 break;
1451 case 4:
1452 stl_p(qemu_get_ram_ptr(ram_addr), val);
1453 break;
1454 default:
1455 abort();
1456 }
1457 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1458 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1459 /* we remove the notdirty callback only if the code has been
1460 flushed */
1461 if (dirty_flags == 0xff) {
1462 CPUArchState *env = current_cpu->env_ptr;
1463 tlb_set_dirty(env, env->mem_io_vaddr);
1464 }
1465 }
1466
1467 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1468 unsigned size, bool is_write)
1469 {
1470 return is_write;
1471 }
1472
1473 static const MemoryRegionOps notdirty_mem_ops = {
1474 .write = notdirty_mem_write,
1475 .valid.accepts = notdirty_mem_accepts,
1476 .endianness = DEVICE_NATIVE_ENDIAN,
1477 };
1478
1479 /* Generate a debug exception if a watchpoint has been hit. */
1480 static void check_watchpoint(int offset, int len_mask, int flags)
1481 {
1482 CPUArchState *env = current_cpu->env_ptr;
1483 target_ulong pc, cs_base;
1484 target_ulong vaddr;
1485 CPUWatchpoint *wp;
1486 int cpu_flags;
1487
1488 if (env->watchpoint_hit) {
1489 /* We re-entered the check after replacing the TB. Now raise
1490 * the debug interrupt so that is will trigger after the
1491 * current instruction. */
1492 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1493 return;
1494 }
1495 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1496 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1497 if ((vaddr == (wp->vaddr & len_mask) ||
1498 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1499 wp->flags |= BP_WATCHPOINT_HIT;
1500 if (!env->watchpoint_hit) {
1501 env->watchpoint_hit = wp;
1502 tb_check_watchpoint(env);
1503 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1504 env->exception_index = EXCP_DEBUG;
1505 cpu_loop_exit(env);
1506 } else {
1507 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1508 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1509 cpu_resume_from_signal(env, NULL);
1510 }
1511 }
1512 } else {
1513 wp->flags &= ~BP_WATCHPOINT_HIT;
1514 }
1515 }
1516 }
1517
1518 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1519 so these check for a hit then pass through to the normal out-of-line
1520 phys routines. */
1521 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1522 unsigned size)
1523 {
1524 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1525 switch (size) {
1526 case 1: return ldub_phys(addr);
1527 case 2: return lduw_phys(addr);
1528 case 4: return ldl_phys(addr);
1529 default: abort();
1530 }
1531 }
1532
1533 static void watch_mem_write(void *opaque, hwaddr addr,
1534 uint64_t val, unsigned size)
1535 {
1536 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1537 switch (size) {
1538 case 1:
1539 stb_phys(addr, val);
1540 break;
1541 case 2:
1542 stw_phys(addr, val);
1543 break;
1544 case 4:
1545 stl_phys(addr, val);
1546 break;
1547 default: abort();
1548 }
1549 }
1550
1551 static const MemoryRegionOps watch_mem_ops = {
1552 .read = watch_mem_read,
1553 .write = watch_mem_write,
1554 .endianness = DEVICE_NATIVE_ENDIAN,
1555 };
1556
1557 static uint64_t subpage_read(void *opaque, hwaddr addr,
1558 unsigned len)
1559 {
1560 subpage_t *subpage = opaque;
1561 uint8_t buf[4];
1562
1563 #if defined(DEBUG_SUBPAGE)
1564 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1565 subpage, len, addr);
1566 #endif
1567 address_space_read(subpage->as, addr + subpage->base, buf, len);
1568 switch (len) {
1569 case 1:
1570 return ldub_p(buf);
1571 case 2:
1572 return lduw_p(buf);
1573 case 4:
1574 return ldl_p(buf);
1575 default:
1576 abort();
1577 }
1578 }
1579
1580 static void subpage_write(void *opaque, hwaddr addr,
1581 uint64_t value, unsigned len)
1582 {
1583 subpage_t *subpage = opaque;
1584 uint8_t buf[4];
1585
1586 #if defined(DEBUG_SUBPAGE)
1587 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1588 " value %"PRIx64"\n",
1589 __func__, subpage, len, addr, value);
1590 #endif
1591 switch (len) {
1592 case 1:
1593 stb_p(buf, value);
1594 break;
1595 case 2:
1596 stw_p(buf, value);
1597 break;
1598 case 4:
1599 stl_p(buf, value);
1600 break;
1601 default:
1602 abort();
1603 }
1604 address_space_write(subpage->as, addr + subpage->base, buf, len);
1605 }
1606
1607 static bool subpage_accepts(void *opaque, hwaddr addr,
1608 unsigned size, bool is_write)
1609 {
1610 subpage_t *subpage = opaque;
1611 #if defined(DEBUG_SUBPAGE)
1612 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1613 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1614 #endif
1615
1616 return address_space_access_valid(subpage->as, addr + subpage->base,
1617 size, is_write);
1618 }
1619
1620 static const MemoryRegionOps subpage_ops = {
1621 .read = subpage_read,
1622 .write = subpage_write,
1623 .valid.accepts = subpage_accepts,
1624 .endianness = DEVICE_NATIVE_ENDIAN,
1625 };
1626
1627 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1628 uint16_t section)
1629 {
1630 int idx, eidx;
1631
1632 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1633 return -1;
1634 idx = SUBPAGE_IDX(start);
1635 eidx = SUBPAGE_IDX(end);
1636 #if defined(DEBUG_SUBPAGE)
1637 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1638 mmio, start, end, idx, eidx, memory);
1639 #endif
1640 for (; idx <= eidx; idx++) {
1641 mmio->sub_section[idx] = section;
1642 }
1643
1644 return 0;
1645 }
1646
1647 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1648 {
1649 subpage_t *mmio;
1650
1651 mmio = g_malloc0(sizeof(subpage_t));
1652
1653 mmio->as = as;
1654 mmio->base = base;
1655 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1656 "subpage", TARGET_PAGE_SIZE);
1657 mmio->iomem.subpage = true;
1658 #if defined(DEBUG_SUBPAGE)
1659 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1660 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1661 #endif
1662 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1663
1664 return mmio;
1665 }
1666
1667 static uint16_t dummy_section(MemoryRegion *mr)
1668 {
1669 MemoryRegionSection section = {
1670 .mr = mr,
1671 .offset_within_address_space = 0,
1672 .offset_within_region = 0,
1673 .size = int128_2_64(),
1674 };
1675
1676 return phys_section_add(&section);
1677 }
1678
1679 MemoryRegion *iotlb_to_region(hwaddr index)
1680 {
1681 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1682 }
1683
1684 static void io_mem_init(void)
1685 {
1686 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1687 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1688 "unassigned", UINT64_MAX);
1689 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1690 "notdirty", UINT64_MAX);
1691 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1692 "watch", UINT64_MAX);
1693 }
1694
1695 static void mem_begin(MemoryListener *listener)
1696 {
1697 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1698 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1699
1700 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1701 d->as = as;
1702 as->next_dispatch = d;
1703 }
1704
1705 static void mem_commit(MemoryListener *listener)
1706 {
1707 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1708 AddressSpaceDispatch *cur = as->dispatch;
1709 AddressSpaceDispatch *next = as->next_dispatch;
1710
1711 next->nodes = next_map.nodes;
1712 next->sections = next_map.sections;
1713
1714 as->dispatch = next;
1715 g_free(cur);
1716 }
1717
1718 static void core_begin(MemoryListener *listener)
1719 {
1720 uint16_t n;
1721
1722 prev_map = g_new(PhysPageMap, 1);
1723 *prev_map = next_map;
1724
1725 memset(&next_map, 0, sizeof(next_map));
1726 n = dummy_section(&io_mem_unassigned);
1727 assert(n == PHYS_SECTION_UNASSIGNED);
1728 n = dummy_section(&io_mem_notdirty);
1729 assert(n == PHYS_SECTION_NOTDIRTY);
1730 n = dummy_section(&io_mem_rom);
1731 assert(n == PHYS_SECTION_ROM);
1732 n = dummy_section(&io_mem_watch);
1733 assert(n == PHYS_SECTION_WATCH);
1734 }
1735
1736 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1737 * All AddressSpaceDispatch instances have switched to the next map.
1738 */
1739 static void core_commit(MemoryListener *listener)
1740 {
1741 phys_sections_free(prev_map);
1742 }
1743
1744 static void tcg_commit(MemoryListener *listener)
1745 {
1746 CPUState *cpu;
1747
1748 /* since each CPU stores ram addresses in its TLB cache, we must
1749 reset the modified entries */
1750 /* XXX: slow ! */
1751 CPU_FOREACH(cpu) {
1752 CPUArchState *env = cpu->env_ptr;
1753
1754 tlb_flush(env, 1);
1755 }
1756 }
1757
1758 static void core_log_global_start(MemoryListener *listener)
1759 {
1760 cpu_physical_memory_set_dirty_tracking(1);
1761 }
1762
1763 static void core_log_global_stop(MemoryListener *listener)
1764 {
1765 cpu_physical_memory_set_dirty_tracking(0);
1766 }
1767
1768 static MemoryListener core_memory_listener = {
1769 .begin = core_begin,
1770 .commit = core_commit,
1771 .log_global_start = core_log_global_start,
1772 .log_global_stop = core_log_global_stop,
1773 .priority = 1,
1774 };
1775
1776 static MemoryListener tcg_memory_listener = {
1777 .commit = tcg_commit,
1778 };
1779
1780 void address_space_init_dispatch(AddressSpace *as)
1781 {
1782 as->dispatch = NULL;
1783 as->dispatch_listener = (MemoryListener) {
1784 .begin = mem_begin,
1785 .commit = mem_commit,
1786 .region_add = mem_add,
1787 .region_nop = mem_add,
1788 .priority = 0,
1789 };
1790 memory_listener_register(&as->dispatch_listener, as);
1791 }
1792
1793 void address_space_destroy_dispatch(AddressSpace *as)
1794 {
1795 AddressSpaceDispatch *d = as->dispatch;
1796
1797 memory_listener_unregister(&as->dispatch_listener);
1798 g_free(d);
1799 as->dispatch = NULL;
1800 }
1801
1802 static void memory_map_init(void)
1803 {
1804 system_memory = g_malloc(sizeof(*system_memory));
1805 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1806 address_space_init(&address_space_memory, system_memory, "memory");
1807
1808 system_io = g_malloc(sizeof(*system_io));
1809 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
1810 65536);
1811 address_space_init(&address_space_io, system_io, "I/O");
1812
1813 memory_listener_register(&core_memory_listener, &address_space_memory);
1814 if (tcg_enabled()) {
1815 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1816 }
1817 }
1818
1819 MemoryRegion *get_system_memory(void)
1820 {
1821 return system_memory;
1822 }
1823
1824 MemoryRegion *get_system_io(void)
1825 {
1826 return system_io;
1827 }
1828
1829 #endif /* !defined(CONFIG_USER_ONLY) */
1830
1831 /* physical memory access (slow version, mainly for debug) */
1832 #if defined(CONFIG_USER_ONLY)
1833 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1834 uint8_t *buf, int len, int is_write)
1835 {
1836 int l, flags;
1837 target_ulong page;
1838 void * p;
1839
1840 while (len > 0) {
1841 page = addr & TARGET_PAGE_MASK;
1842 l = (page + TARGET_PAGE_SIZE) - addr;
1843 if (l > len)
1844 l = len;
1845 flags = page_get_flags(page);
1846 if (!(flags & PAGE_VALID))
1847 return -1;
1848 if (is_write) {
1849 if (!(flags & PAGE_WRITE))
1850 return -1;
1851 /* XXX: this code should not depend on lock_user */
1852 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1853 return -1;
1854 memcpy(p, buf, l);
1855 unlock_user(p, addr, l);
1856 } else {
1857 if (!(flags & PAGE_READ))
1858 return -1;
1859 /* XXX: this code should not depend on lock_user */
1860 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1861 return -1;
1862 memcpy(buf, p, l);
1863 unlock_user(p, addr, 0);
1864 }
1865 len -= l;
1866 buf += l;
1867 addr += l;
1868 }
1869 return 0;
1870 }
1871
1872 #else
1873
1874 static void invalidate_and_set_dirty(hwaddr addr,
1875 hwaddr length)
1876 {
1877 if (!cpu_physical_memory_is_dirty(addr)) {
1878 /* invalidate code */
1879 tb_invalidate_phys_page_range(addr, addr + length, 0);
1880 /* set dirty bit */
1881 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1882 }
1883 xen_modified_memory(addr, length);
1884 }
1885
1886 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1887 {
1888 if (memory_region_is_ram(mr)) {
1889 return !(is_write && mr->readonly);
1890 }
1891 if (memory_region_is_romd(mr)) {
1892 return !is_write;
1893 }
1894
1895 return false;
1896 }
1897
1898 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1899 {
1900 unsigned access_size_max = mr->ops->valid.max_access_size;
1901
1902 /* Regions are assumed to support 1-4 byte accesses unless
1903 otherwise specified. */
1904 if (access_size_max == 0) {
1905 access_size_max = 4;
1906 }
1907
1908 /* Bound the maximum access by the alignment of the address. */
1909 if (!mr->ops->impl.unaligned) {
1910 unsigned align_size_max = addr & -addr;
1911 if (align_size_max != 0 && align_size_max < access_size_max) {
1912 access_size_max = align_size_max;
1913 }
1914 }
1915
1916 /* Don't attempt accesses larger than the maximum. */
1917 if (l > access_size_max) {
1918 l = access_size_max;
1919 }
1920 if (l & (l - 1)) {
1921 l = 1 << (qemu_fls(l) - 1);
1922 }
1923
1924 return l;
1925 }
1926
1927 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1928 int len, bool is_write)
1929 {
1930 hwaddr l;
1931 uint8_t *ptr;
1932 uint64_t val;
1933 hwaddr addr1;
1934 MemoryRegion *mr;
1935 bool error = false;
1936
1937 while (len > 0) {
1938 l = len;
1939 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1940
1941 if (is_write) {
1942 if (!memory_access_is_direct(mr, is_write)) {
1943 l = memory_access_size(mr, l, addr1);
1944 /* XXX: could force current_cpu to NULL to avoid
1945 potential bugs */
1946 switch (l) {
1947 case 8:
1948 /* 64 bit write access */
1949 val = ldq_p(buf);
1950 error |= io_mem_write(mr, addr1, val, 8);
1951 break;
1952 case 4:
1953 /* 32 bit write access */
1954 val = ldl_p(buf);
1955 error |= io_mem_write(mr, addr1, val, 4);
1956 break;
1957 case 2:
1958 /* 16 bit write access */
1959 val = lduw_p(buf);
1960 error |= io_mem_write(mr, addr1, val, 2);
1961 break;
1962 case 1:
1963 /* 8 bit write access */
1964 val = ldub_p(buf);
1965 error |= io_mem_write(mr, addr1, val, 1);
1966 break;
1967 default:
1968 abort();
1969 }
1970 } else {
1971 addr1 += memory_region_get_ram_addr(mr);
1972 /* RAM case */
1973 ptr = qemu_get_ram_ptr(addr1);
1974 memcpy(ptr, buf, l);
1975 invalidate_and_set_dirty(addr1, l);
1976 }
1977 } else {
1978 if (!memory_access_is_direct(mr, is_write)) {
1979 /* I/O case */
1980 l = memory_access_size(mr, l, addr1);
1981 switch (l) {
1982 case 8:
1983 /* 64 bit read access */
1984 error |= io_mem_read(mr, addr1, &val, 8);
1985 stq_p(buf, val);
1986 break;
1987 case 4:
1988 /* 32 bit read access */
1989 error |= io_mem_read(mr, addr1, &val, 4);
1990 stl_p(buf, val);
1991 break;
1992 case 2:
1993 /* 16 bit read access */
1994 error |= io_mem_read(mr, addr1, &val, 2);
1995 stw_p(buf, val);
1996 break;
1997 case 1:
1998 /* 8 bit read access */
1999 error |= io_mem_read(mr, addr1, &val, 1);
2000 stb_p(buf, val);
2001 break;
2002 default:
2003 abort();
2004 }
2005 } else {
2006 /* RAM case */
2007 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2008 memcpy(buf, ptr, l);
2009 }
2010 }
2011 len -= l;
2012 buf += l;
2013 addr += l;
2014 }
2015
2016 return error;
2017 }
2018
2019 bool address_space_write(AddressSpace *as, hwaddr addr,
2020 const uint8_t *buf, int len)
2021 {
2022 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2023 }
2024
2025 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2026 {
2027 return address_space_rw(as, addr, buf, len, false);
2028 }
2029
2030
2031 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2032 int len, int is_write)
2033 {
2034 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2035 }
2036
2037 /* used for ROM loading : can write in RAM and ROM */
2038 void cpu_physical_memory_write_rom(hwaddr addr,
2039 const uint8_t *buf, int len)
2040 {
2041 hwaddr l;
2042 uint8_t *ptr;
2043 hwaddr addr1;
2044 MemoryRegion *mr;
2045
2046 while (len > 0) {
2047 l = len;
2048 mr = address_space_translate(&address_space_memory,
2049 addr, &addr1, &l, true);
2050
2051 if (!(memory_region_is_ram(mr) ||
2052 memory_region_is_romd(mr))) {
2053 /* do nothing */
2054 } else {
2055 addr1 += memory_region_get_ram_addr(mr);
2056 /* ROM/RAM case */
2057 ptr = qemu_get_ram_ptr(addr1);
2058 memcpy(ptr, buf, l);
2059 invalidate_and_set_dirty(addr1, l);
2060 }
2061 len -= l;
2062 buf += l;
2063 addr += l;
2064 }
2065 }
2066
2067 typedef struct {
2068 MemoryRegion *mr;
2069 void *buffer;
2070 hwaddr addr;
2071 hwaddr len;
2072 } BounceBuffer;
2073
2074 static BounceBuffer bounce;
2075
2076 typedef struct MapClient {
2077 void *opaque;
2078 void (*callback)(void *opaque);
2079 QLIST_ENTRY(MapClient) link;
2080 } MapClient;
2081
2082 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2083 = QLIST_HEAD_INITIALIZER(map_client_list);
2084
2085 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2086 {
2087 MapClient *client = g_malloc(sizeof(*client));
2088
2089 client->opaque = opaque;
2090 client->callback = callback;
2091 QLIST_INSERT_HEAD(&map_client_list, client, link);
2092 return client;
2093 }
2094
2095 static void cpu_unregister_map_client(void *_client)
2096 {
2097 MapClient *client = (MapClient *)_client;
2098
2099 QLIST_REMOVE(client, link);
2100 g_free(client);
2101 }
2102
2103 static void cpu_notify_map_clients(void)
2104 {
2105 MapClient *client;
2106
2107 while (!QLIST_EMPTY(&map_client_list)) {
2108 client = QLIST_FIRST(&map_client_list);
2109 client->callback(client->opaque);
2110 cpu_unregister_map_client(client);
2111 }
2112 }
2113
2114 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2115 {
2116 MemoryRegion *mr;
2117 hwaddr l, xlat;
2118
2119 while (len > 0) {
2120 l = len;
2121 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2122 if (!memory_access_is_direct(mr, is_write)) {
2123 l = memory_access_size(mr, l, addr);
2124 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2125 return false;
2126 }
2127 }
2128
2129 len -= l;
2130 addr += l;
2131 }
2132 return true;
2133 }
2134
2135 /* Map a physical memory region into a host virtual address.
2136 * May map a subset of the requested range, given by and returned in *plen.
2137 * May return NULL if resources needed to perform the mapping are exhausted.
2138 * Use only for reads OR writes - not for read-modify-write operations.
2139 * Use cpu_register_map_client() to know when retrying the map operation is
2140 * likely to succeed.
2141 */
2142 void *address_space_map(AddressSpace *as,
2143 hwaddr addr,
2144 hwaddr *plen,
2145 bool is_write)
2146 {
2147 hwaddr len = *plen;
2148 hwaddr done = 0;
2149 hwaddr l, xlat, base;
2150 MemoryRegion *mr, *this_mr;
2151 ram_addr_t raddr;
2152
2153 if (len == 0) {
2154 return NULL;
2155 }
2156
2157 l = len;
2158 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2159 if (!memory_access_is_direct(mr, is_write)) {
2160 if (bounce.buffer) {
2161 return NULL;
2162 }
2163 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2164 bounce.addr = addr;
2165 bounce.len = l;
2166
2167 memory_region_ref(mr);
2168 bounce.mr = mr;
2169 if (!is_write) {
2170 address_space_read(as, addr, bounce.buffer, l);
2171 }
2172
2173 *plen = l;
2174 return bounce.buffer;
2175 }
2176
2177 base = xlat;
2178 raddr = memory_region_get_ram_addr(mr);
2179
2180 for (;;) {
2181 len -= l;
2182 addr += l;
2183 done += l;
2184 if (len == 0) {
2185 break;
2186 }
2187
2188 l = len;
2189 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2190 if (this_mr != mr || xlat != base + done) {
2191 break;
2192 }
2193 }
2194
2195 memory_region_ref(mr);
2196 *plen = done;
2197 return qemu_ram_ptr_length(raddr + base, plen);
2198 }
2199
2200 /* Unmaps a memory region previously mapped by address_space_map().
2201 * Will also mark the memory as dirty if is_write == 1. access_len gives
2202 * the amount of memory that was actually read or written by the caller.
2203 */
2204 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2205 int is_write, hwaddr access_len)
2206 {
2207 if (buffer != bounce.buffer) {
2208 MemoryRegion *mr;
2209 ram_addr_t addr1;
2210
2211 mr = qemu_ram_addr_from_host(buffer, &addr1);
2212 assert(mr != NULL);
2213 if (is_write) {
2214 while (access_len) {
2215 unsigned l;
2216 l = TARGET_PAGE_SIZE;
2217 if (l > access_len)
2218 l = access_len;
2219 invalidate_and_set_dirty(addr1, l);
2220 addr1 += l;
2221 access_len -= l;
2222 }
2223 }
2224 if (xen_enabled()) {
2225 xen_invalidate_map_cache_entry(buffer);
2226 }
2227 memory_region_unref(mr);
2228 return;
2229 }
2230 if (is_write) {
2231 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2232 }
2233 qemu_vfree(bounce.buffer);
2234 bounce.buffer = NULL;
2235 memory_region_unref(bounce.mr);
2236 cpu_notify_map_clients();
2237 }
2238
2239 void *cpu_physical_memory_map(hwaddr addr,
2240 hwaddr *plen,
2241 int is_write)
2242 {
2243 return address_space_map(&address_space_memory, addr, plen, is_write);
2244 }
2245
2246 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2247 int is_write, hwaddr access_len)
2248 {
2249 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2250 }
2251
2252 /* warning: addr must be aligned */
2253 static inline uint32_t ldl_phys_internal(hwaddr addr,
2254 enum device_endian endian)
2255 {
2256 uint8_t *ptr;
2257 uint64_t val;
2258 MemoryRegion *mr;
2259 hwaddr l = 4;
2260 hwaddr addr1;
2261
2262 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2263 false);
2264 if (l < 4 || !memory_access_is_direct(mr, false)) {
2265 /* I/O case */
2266 io_mem_read(mr, addr1, &val, 4);
2267 #if defined(TARGET_WORDS_BIGENDIAN)
2268 if (endian == DEVICE_LITTLE_ENDIAN) {
2269 val = bswap32(val);
2270 }
2271 #else
2272 if (endian == DEVICE_BIG_ENDIAN) {
2273 val = bswap32(val);
2274 }
2275 #endif
2276 } else {
2277 /* RAM case */
2278 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2279 & TARGET_PAGE_MASK)
2280 + addr1);
2281 switch (endian) {
2282 case DEVICE_LITTLE_ENDIAN:
2283 val = ldl_le_p(ptr);
2284 break;
2285 case DEVICE_BIG_ENDIAN:
2286 val = ldl_be_p(ptr);
2287 break;
2288 default:
2289 val = ldl_p(ptr);
2290 break;
2291 }
2292 }
2293 return val;
2294 }
2295
2296 uint32_t ldl_phys(hwaddr addr)
2297 {
2298 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2299 }
2300
2301 uint32_t ldl_le_phys(hwaddr addr)
2302 {
2303 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2304 }
2305
2306 uint32_t ldl_be_phys(hwaddr addr)
2307 {
2308 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2309 }
2310
2311 /* warning: addr must be aligned */
2312 static inline uint64_t ldq_phys_internal(hwaddr addr,
2313 enum device_endian endian)
2314 {
2315 uint8_t *ptr;
2316 uint64_t val;
2317 MemoryRegion *mr;
2318 hwaddr l = 8;
2319 hwaddr addr1;
2320
2321 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2322 false);
2323 if (l < 8 || !memory_access_is_direct(mr, false)) {
2324 /* I/O case */
2325 io_mem_read(mr, addr1, &val, 8);
2326 #if defined(TARGET_WORDS_BIGENDIAN)
2327 if (endian == DEVICE_LITTLE_ENDIAN) {
2328 val = bswap64(val);
2329 }
2330 #else
2331 if (endian == DEVICE_BIG_ENDIAN) {
2332 val = bswap64(val);
2333 }
2334 #endif
2335 } else {
2336 /* RAM case */
2337 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2338 & TARGET_PAGE_MASK)
2339 + addr1);
2340 switch (endian) {
2341 case DEVICE_LITTLE_ENDIAN:
2342 val = ldq_le_p(ptr);
2343 break;
2344 case DEVICE_BIG_ENDIAN:
2345 val = ldq_be_p(ptr);
2346 break;
2347 default:
2348 val = ldq_p(ptr);
2349 break;
2350 }
2351 }
2352 return val;
2353 }
2354
2355 uint64_t ldq_phys(hwaddr addr)
2356 {
2357 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2358 }
2359
2360 uint64_t ldq_le_phys(hwaddr addr)
2361 {
2362 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2363 }
2364
2365 uint64_t ldq_be_phys(hwaddr addr)
2366 {
2367 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2368 }
2369
2370 /* XXX: optimize */
2371 uint32_t ldub_phys(hwaddr addr)
2372 {
2373 uint8_t val;
2374 cpu_physical_memory_read(addr, &val, 1);
2375 return val;
2376 }
2377
2378 /* warning: addr must be aligned */
2379 static inline uint32_t lduw_phys_internal(hwaddr addr,
2380 enum device_endian endian)
2381 {
2382 uint8_t *ptr;
2383 uint64_t val;
2384 MemoryRegion *mr;
2385 hwaddr l = 2;
2386 hwaddr addr1;
2387
2388 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2389 false);
2390 if (l < 2 || !memory_access_is_direct(mr, false)) {
2391 /* I/O case */
2392 io_mem_read(mr, addr1, &val, 2);
2393 #if defined(TARGET_WORDS_BIGENDIAN)
2394 if (endian == DEVICE_LITTLE_ENDIAN) {
2395 val = bswap16(val);
2396 }
2397 #else
2398 if (endian == DEVICE_BIG_ENDIAN) {
2399 val = bswap16(val);
2400 }
2401 #endif
2402 } else {
2403 /* RAM case */
2404 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2405 & TARGET_PAGE_MASK)
2406 + addr1);
2407 switch (endian) {
2408 case DEVICE_LITTLE_ENDIAN:
2409 val = lduw_le_p(ptr);
2410 break;
2411 case DEVICE_BIG_ENDIAN:
2412 val = lduw_be_p(ptr);
2413 break;
2414 default:
2415 val = lduw_p(ptr);
2416 break;
2417 }
2418 }
2419 return val;
2420 }
2421
2422 uint32_t lduw_phys(hwaddr addr)
2423 {
2424 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2425 }
2426
2427 uint32_t lduw_le_phys(hwaddr addr)
2428 {
2429 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2430 }
2431
2432 uint32_t lduw_be_phys(hwaddr addr)
2433 {
2434 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2435 }
2436
2437 /* warning: addr must be aligned. The ram page is not masked as dirty
2438 and the code inside is not invalidated. It is useful if the dirty
2439 bits are used to track modified PTEs */
2440 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2441 {
2442 uint8_t *ptr;
2443 MemoryRegion *mr;
2444 hwaddr l = 4;
2445 hwaddr addr1;
2446
2447 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2448 true);
2449 if (l < 4 || !memory_access_is_direct(mr, true)) {
2450 io_mem_write(mr, addr1, val, 4);
2451 } else {
2452 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2453 ptr = qemu_get_ram_ptr(addr1);
2454 stl_p(ptr, val);
2455
2456 if (unlikely(in_migration)) {
2457 if (!cpu_physical_memory_is_dirty(addr1)) {
2458 /* invalidate code */
2459 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2460 /* set dirty bit */
2461 cpu_physical_memory_set_dirty_flags(
2462 addr1, (0xff & ~CODE_DIRTY_FLAG));
2463 }
2464 }
2465 }
2466 }
2467
2468 /* warning: addr must be aligned */
2469 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2470 enum device_endian endian)
2471 {
2472 uint8_t *ptr;
2473 MemoryRegion *mr;
2474 hwaddr l = 4;
2475 hwaddr addr1;
2476
2477 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2478 true);
2479 if (l < 4 || !memory_access_is_direct(mr, true)) {
2480 #if defined(TARGET_WORDS_BIGENDIAN)
2481 if (endian == DEVICE_LITTLE_ENDIAN) {
2482 val = bswap32(val);
2483 }
2484 #else
2485 if (endian == DEVICE_BIG_ENDIAN) {
2486 val = bswap32(val);
2487 }
2488 #endif
2489 io_mem_write(mr, addr1, val, 4);
2490 } else {
2491 /* RAM case */
2492 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2493 ptr = qemu_get_ram_ptr(addr1);
2494 switch (endian) {
2495 case DEVICE_LITTLE_ENDIAN:
2496 stl_le_p(ptr, val);
2497 break;
2498 case DEVICE_BIG_ENDIAN:
2499 stl_be_p(ptr, val);
2500 break;
2501 default:
2502 stl_p(ptr, val);
2503 break;
2504 }
2505 invalidate_and_set_dirty(addr1, 4);
2506 }
2507 }
2508
2509 void stl_phys(hwaddr addr, uint32_t val)
2510 {
2511 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2512 }
2513
2514 void stl_le_phys(hwaddr addr, uint32_t val)
2515 {
2516 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2517 }
2518
2519 void stl_be_phys(hwaddr addr, uint32_t val)
2520 {
2521 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2522 }
2523
2524 /* XXX: optimize */
2525 void stb_phys(hwaddr addr, uint32_t val)
2526 {
2527 uint8_t v = val;
2528 cpu_physical_memory_write(addr, &v, 1);
2529 }
2530
2531 /* warning: addr must be aligned */
2532 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2533 enum device_endian endian)
2534 {
2535 uint8_t *ptr;
2536 MemoryRegion *mr;
2537 hwaddr l = 2;
2538 hwaddr addr1;
2539
2540 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2541 true);
2542 if (l < 2 || !memory_access_is_direct(mr, true)) {
2543 #if defined(TARGET_WORDS_BIGENDIAN)
2544 if (endian == DEVICE_LITTLE_ENDIAN) {
2545 val = bswap16(val);
2546 }
2547 #else
2548 if (endian == DEVICE_BIG_ENDIAN) {
2549 val = bswap16(val);
2550 }
2551 #endif
2552 io_mem_write(mr, addr1, val, 2);
2553 } else {
2554 /* RAM case */
2555 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2556 ptr = qemu_get_ram_ptr(addr1);
2557 switch (endian) {
2558 case DEVICE_LITTLE_ENDIAN:
2559 stw_le_p(ptr, val);
2560 break;
2561 case DEVICE_BIG_ENDIAN:
2562 stw_be_p(ptr, val);
2563 break;
2564 default:
2565 stw_p(ptr, val);
2566 break;
2567 }
2568 invalidate_and_set_dirty(addr1, 2);
2569 }
2570 }
2571
2572 void stw_phys(hwaddr addr, uint32_t val)
2573 {
2574 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2575 }
2576
2577 void stw_le_phys(hwaddr addr, uint32_t val)
2578 {
2579 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2580 }
2581
2582 void stw_be_phys(hwaddr addr, uint32_t val)
2583 {
2584 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2585 }
2586
2587 /* XXX: optimize */
2588 void stq_phys(hwaddr addr, uint64_t val)
2589 {
2590 val = tswap64(val);
2591 cpu_physical_memory_write(addr, &val, 8);
2592 }
2593
2594 void stq_le_phys(hwaddr addr, uint64_t val)
2595 {
2596 val = cpu_to_le64(val);
2597 cpu_physical_memory_write(addr, &val, 8);
2598 }
2599
2600 void stq_be_phys(hwaddr addr, uint64_t val)
2601 {
2602 val = cpu_to_be64(val);
2603 cpu_physical_memory_write(addr, &val, 8);
2604 }
2605
2606 /* virtual memory access for debug (includes writing to ROM) */
2607 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2608 uint8_t *buf, int len, int is_write)
2609 {
2610 int l;
2611 hwaddr phys_addr;
2612 target_ulong page;
2613
2614 while (len > 0) {
2615 page = addr & TARGET_PAGE_MASK;
2616 phys_addr = cpu_get_phys_page_debug(cpu, page);
2617 /* if no physical page mapped, return an error */
2618 if (phys_addr == -1)
2619 return -1;
2620 l = (page + TARGET_PAGE_SIZE) - addr;
2621 if (l > len)
2622 l = len;
2623 phys_addr += (addr & ~TARGET_PAGE_MASK);
2624 if (is_write)
2625 cpu_physical_memory_write_rom(phys_addr, buf, l);
2626 else
2627 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2628 len -= l;
2629 buf += l;
2630 addr += l;
2631 }
2632 return 0;
2633 }
2634 #endif
2635
2636 #if !defined(CONFIG_USER_ONLY)
2637
2638 /*
2639 * A helper function for the _utterly broken_ virtio device model to find out if
2640 * it's running on a big endian machine. Don't do this at home kids!
2641 */
2642 bool virtio_is_big_endian(void);
2643 bool virtio_is_big_endian(void)
2644 {
2645 #if defined(TARGET_WORDS_BIGENDIAN)
2646 return true;
2647 #else
2648 return false;
2649 #endif
2650 }
2651
2652 #endif
2653
2654 #ifndef CONFIG_USER_ONLY
2655 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2656 {
2657 MemoryRegion*mr;
2658 hwaddr l = 1;
2659
2660 mr = address_space_translate(&address_space_memory,
2661 phys_addr, &phys_addr, &l, false);
2662
2663 return !(memory_region_is_ram(mr) ||
2664 memory_region_is_romd(mr));
2665 }
2666
2667 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2668 {
2669 RAMBlock *block;
2670
2671 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2672 func(block->host, block->offset, block->length, opaque);
2673 }
2674 }
2675 #endif
Qemu copy for testing