]> git.proxmox.com Git - qemu.git/blob - exec.c
projects / qemu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge remote-tracking branch 'kraxel/seabios-1.7.3' into staging
[qemu.git] / exec.c
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
51
52 #include "exec/memory-internal.h"
53
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 CPUState *first_cpu;
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUState *, current_cpu);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 typedef PhysPageEntry Node[L2_SIZE];
92
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101 };
102
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
110
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
115
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
124
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
127
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
133
134 static MemoryRegion io_mem_watch;
135 #endif
136
137 #if !defined(CONFIG_USER_ONLY)
138
139 static void phys_map_node_reserve(unsigned nodes)
140 {
141 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
142 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
143 16);
144 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
145 next_map.nodes_nb + nodes);
146 next_map.nodes = g_renew(Node, next_map.nodes,
147 next_map.nodes_nb_alloc);
148 }
149 }
150
151 static uint16_t phys_map_node_alloc(void)
152 {
153 unsigned i;
154 uint16_t ret;
155
156 ret = next_map.nodes_nb++;
157 assert(ret != PHYS_MAP_NODE_NIL);
158 assert(ret != next_map.nodes_nb_alloc);
159 for (i = 0; i < L2_SIZE; ++i) {
160 next_map.nodes[ret][i].is_leaf = 0;
161 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
162 }
163 return ret;
164 }
165
166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
167 hwaddr *nb, uint16_t leaf,
168 int level)
169 {
170 PhysPageEntry *p;
171 int i;
172 hwaddr step = (hwaddr)1 << (level * L2_BITS);
173
174 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
175 lp->ptr = phys_map_node_alloc();
176 p = next_map.nodes[lp->ptr];
177 if (level == 0) {
178 for (i = 0; i < L2_SIZE; i++) {
179 p[i].is_leaf = 1;
180 p[i].ptr = PHYS_SECTION_UNASSIGNED;
181 }
182 }
183 } else {
184 p = next_map.nodes[lp->ptr];
185 }
186 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
187
188 while (*nb && lp < &p[L2_SIZE]) {
189 if ((*index & (step - 1)) == 0 && *nb >= step) {
190 lp->is_leaf = true;
191 lp->ptr = leaf;
192 *index += step;
193 *nb -= step;
194 } else {
195 phys_page_set_level(lp, index, nb, leaf, level - 1);
196 }
197 ++lp;
198 }
199 }
200
201 static void phys_page_set(AddressSpaceDispatch *d,
202 hwaddr index, hwaddr nb,
203 uint16_t leaf)
204 {
205 /* Wildly overreserve - it doesn't matter much. */
206 phys_map_node_reserve(3 * P_L2_LEVELS);
207
208 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
209 }
210
211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
212 Node *nodes, MemoryRegionSection *sections)
213 {
214 PhysPageEntry *p;
215 int i;
216
217 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
218 if (lp.ptr == PHYS_MAP_NODE_NIL) {
219 return &sections[PHYS_SECTION_UNASSIGNED];
220 }
221 p = nodes[lp.ptr];
222 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
223 }
224 return &sections[lp.ptr];
225 }
226
227 bool memory_region_is_unassigned(MemoryRegion *mr)
228 {
229 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
230 && mr != &io_mem_watch;
231 }
232
233 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
234 hwaddr addr,
235 bool resolve_subpage)
236 {
237 MemoryRegionSection *section;
238 subpage_t *subpage;
239
240 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
241 d->nodes, d->sections);
242 if (resolve_subpage && section->mr->subpage) {
243 subpage = container_of(section->mr, subpage_t, iomem);
244 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
245 }
246 return section;
247 }
248
249 static MemoryRegionSection *
250 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
251 hwaddr *plen, bool resolve_subpage)
252 {
253 MemoryRegionSection *section;
254 Int128 diff;
255
256 section = address_space_lookup_region(d, addr, resolve_subpage);
257 /* Compute offset within MemoryRegionSection */
258 addr -= section->offset_within_address_space;
259
260 /* Compute offset within MemoryRegion */
261 *xlat = addr + section->offset_within_region;
262
263 diff = int128_sub(section->mr->size, int128_make64(addr));
264 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
265 return section;
266 }
267
268 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
269 hwaddr *xlat, hwaddr *plen,
270 bool is_write)
271 {
272 IOMMUTLBEntry iotlb;
273 MemoryRegionSection *section;
274 MemoryRegion *mr;
275 hwaddr len = *plen;
276
277 for (;;) {
278 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
279 mr = section->mr;
280
281 if (!mr->iommu_ops) {
282 break;
283 }
284
285 iotlb = mr->iommu_ops->translate(mr, addr);
286 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
287 | (addr & iotlb.addr_mask));
288 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
289 if (!(iotlb.perm & (1 << is_write))) {
290 mr = &io_mem_unassigned;
291 break;
292 }
293
294 as = iotlb.target_as;
295 }
296
297 *plen = len;
298 *xlat = addr;
299 return mr;
300 }
301
302 MemoryRegionSection *
303 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
304 hwaddr *plen)
305 {
306 MemoryRegionSection *section;
307 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
308
309 assert(!section->mr->iommu_ops);
310 return section;
311 }
312 #endif
313
314 void cpu_exec_init_all(void)
315 {
316 #if !defined(CONFIG_USER_ONLY)
317 qemu_mutex_init(&ram_list.mutex);
318 memory_map_init();
319 io_mem_init();
320 #endif
321 }
322
323 #if !defined(CONFIG_USER_ONLY)
324
325 static int cpu_common_post_load(void *opaque, int version_id)
326 {
327 CPUState *cpu = opaque;
328
329 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
330 version_id is increased. */
331 cpu->interrupt_request &= ~0x01;
332 tlb_flush(cpu->env_ptr, 1);
333
334 return 0;
335 }
336
337 const VMStateDescription vmstate_cpu_common = {
338 .name = "cpu_common",
339 .version_id = 1,
340 .minimum_version_id = 1,
341 .minimum_version_id_old = 1,
342 .post_load = cpu_common_post_load,
343 .fields = (VMStateField []) {
344 VMSTATE_UINT32(halted, CPUState),
345 VMSTATE_UINT32(interrupt_request, CPUState),
346 VMSTATE_END_OF_LIST()
347 }
348 };
349
350 #endif
351
352 CPUState *qemu_get_cpu(int index)
353 {
354 CPUState *cpu = first_cpu;
355
356 while (cpu) {
357 if (cpu->cpu_index == index) {
358 break;
359 }
360 cpu = cpu->next_cpu;
361 }
362
363 return cpu;
364 }
365
366 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
367 {
368 CPUState *cpu;
369
370 cpu = first_cpu;
371 while (cpu) {
372 func(cpu, data);
373 cpu = cpu->next_cpu;
374 }
375 }
376
377 void cpu_exec_init(CPUArchState *env)
378 {
379 CPUState *cpu = ENV_GET_CPU(env);
380 CPUClass *cc = CPU_GET_CLASS(cpu);
381 CPUState **pcpu;
382 int cpu_index;
383
384 #if defined(CONFIG_USER_ONLY)
385 cpu_list_lock();
386 #endif
387 cpu->next_cpu = NULL;
388 pcpu = &first_cpu;
389 cpu_index = 0;
390 while (*pcpu != NULL) {
391 pcpu = &(*pcpu)->next_cpu;
392 cpu_index++;
393 }
394 cpu->cpu_index = cpu_index;
395 cpu->numa_node = 0;
396 QTAILQ_INIT(&env->breakpoints);
397 QTAILQ_INIT(&env->watchpoints);
398 #ifndef CONFIG_USER_ONLY
399 cpu->thread_id = qemu_get_thread_id();
400 #endif
401 *pcpu = cpu;
402 #if defined(CONFIG_USER_ONLY)
403 cpu_list_unlock();
404 #endif
405 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
406 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
407 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
408 cpu_save, cpu_load, env);
409 assert(cc->vmsd == NULL);
410 #endif
411 if (cc->vmsd != NULL) {
412 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
413 }
414 }
415
416 #if defined(TARGET_HAS_ICE)
417 #if defined(CONFIG_USER_ONLY)
418 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
419 {
420 tb_invalidate_phys_page_range(pc, pc + 1, 0);
421 }
422 #else
423 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
424 {
425 tb_invalidate_phys_addr(cpu_get_phys_page_debug(cpu, pc) |
426 (pc & ~TARGET_PAGE_MASK));
427 }
428 #endif
429 #endif /* TARGET_HAS_ICE */
430
431 #if defined(CONFIG_USER_ONLY)
432 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
433
434 {
435 }
436
437 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
438 int flags, CPUWatchpoint **watchpoint)
439 {
440 return -ENOSYS;
441 }
442 #else
443 /* Add a watchpoint. */
444 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
445 int flags, CPUWatchpoint **watchpoint)
446 {
447 target_ulong len_mask = ~(len - 1);
448 CPUWatchpoint *wp;
449
450 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
451 if ((len & (len - 1)) || (addr & ~len_mask) ||
452 len == 0 || len > TARGET_PAGE_SIZE) {
453 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
454 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
455 return -EINVAL;
456 }
457 wp = g_malloc(sizeof(*wp));
458
459 wp->vaddr = addr;
460 wp->len_mask = len_mask;
461 wp->flags = flags;
462
463 /* keep all GDB-injected watchpoints in front */
464 if (flags & BP_GDB)
465 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
466 else
467 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
468
469 tlb_flush_page(env, addr);
470
471 if (watchpoint)
472 *watchpoint = wp;
473 return 0;
474 }
475
476 /* Remove a specific watchpoint. */
477 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
478 int flags)
479 {
480 target_ulong len_mask = ~(len - 1);
481 CPUWatchpoint *wp;
482
483 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
484 if (addr == wp->vaddr && len_mask == wp->len_mask
485 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
486 cpu_watchpoint_remove_by_ref(env, wp);
487 return 0;
488 }
489 }
490 return -ENOENT;
491 }
492
493 /* Remove a specific watchpoint by reference. */
494 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
495 {
496 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
497
498 tlb_flush_page(env, watchpoint->vaddr);
499
500 g_free(watchpoint);
501 }
502
503 /* Remove all matching watchpoints. */
504 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
505 {
506 CPUWatchpoint *wp, *next;
507
508 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
509 if (wp->flags & mask)
510 cpu_watchpoint_remove_by_ref(env, wp);
511 }
512 }
513 #endif
514
515 /* Add a breakpoint. */
516 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
517 CPUBreakpoint **breakpoint)
518 {
519 #if defined(TARGET_HAS_ICE)
520 CPUBreakpoint *bp;
521
522 bp = g_malloc(sizeof(*bp));
523
524 bp->pc = pc;
525 bp->flags = flags;
526
527 /* keep all GDB-injected breakpoints in front */
528 if (flags & BP_GDB) {
529 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
530 } else {
531 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
532 }
533
534 breakpoint_invalidate(ENV_GET_CPU(env), pc);
535
536 if (breakpoint) {
537 *breakpoint = bp;
538 }
539 return 0;
540 #else
541 return -ENOSYS;
542 #endif
543 }
544
545 /* Remove a specific breakpoint. */
546 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
547 {
548 #if defined(TARGET_HAS_ICE)
549 CPUBreakpoint *bp;
550
551 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
552 if (bp->pc == pc && bp->flags == flags) {
553 cpu_breakpoint_remove_by_ref(env, bp);
554 return 0;
555 }
556 }
557 return -ENOENT;
558 #else
559 return -ENOSYS;
560 #endif
561 }
562
563 /* Remove a specific breakpoint by reference. */
564 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
565 {
566 #if defined(TARGET_HAS_ICE)
567 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
568
569 breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
570
571 g_free(breakpoint);
572 #endif
573 }
574
575 /* Remove all matching breakpoints. */
576 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
577 {
578 #if defined(TARGET_HAS_ICE)
579 CPUBreakpoint *bp, *next;
580
581 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
582 if (bp->flags & mask)
583 cpu_breakpoint_remove_by_ref(env, bp);
584 }
585 #endif
586 }
587
588 /* enable or disable single step mode. EXCP_DEBUG is returned by the
589 CPU loop after each instruction */
590 void cpu_single_step(CPUState *cpu, int enabled)
591 {
592 #if defined(TARGET_HAS_ICE)
593 CPUArchState *env = cpu->env_ptr;
594
595 if (cpu->singlestep_enabled != enabled) {
596 cpu->singlestep_enabled = enabled;
597 if (kvm_enabled()) {
598 kvm_update_guest_debug(env, 0);
599 } else {
600 /* must flush all the translated code to avoid inconsistencies */
601 /* XXX: only flush what is necessary */
602 tb_flush(env);
603 }
604 }
605 #endif
606 }
607
608 void cpu_abort(CPUArchState *env, const char *fmt, ...)
609 {
610 CPUState *cpu = ENV_GET_CPU(env);
611 va_list ap;
612 va_list ap2;
613
614 va_start(ap, fmt);
615 va_copy(ap2, ap);
616 fprintf(stderr, "qemu: fatal: ");
617 vfprintf(stderr, fmt, ap);
618 fprintf(stderr, "\n");
619 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
620 if (qemu_log_enabled()) {
621 qemu_log("qemu: fatal: ");
622 qemu_log_vprintf(fmt, ap2);
623 qemu_log("\n");
624 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
625 qemu_log_flush();
626 qemu_log_close();
627 }
628 va_end(ap2);
629 va_end(ap);
630 #if defined(CONFIG_USER_ONLY)
631 {
632 struct sigaction act;
633 sigfillset(&act.sa_mask);
634 act.sa_handler = SIG_DFL;
635 sigaction(SIGABRT, &act, NULL);
636 }
637 #endif
638 abort();
639 }
640
641 CPUArchState *cpu_copy(CPUArchState *env)
642 {
643 CPUArchState *new_env = cpu_init(env->cpu_model_str);
644 #if defined(TARGET_HAS_ICE)
645 CPUBreakpoint *bp;
646 CPUWatchpoint *wp;
647 #endif
648
649 /* Reset non arch specific state */
650 cpu_reset(ENV_GET_CPU(new_env));
651
652 /* Copy arch specific state into the new CPU */
653 memcpy(new_env, env, sizeof(CPUArchState));
654
655 /* Clone all break/watchpoints.
656 Note: Once we support ptrace with hw-debug register access, make sure
657 BP_CPU break/watchpoints are handled correctly on clone. */
658 QTAILQ_INIT(&env->breakpoints);
659 QTAILQ_INIT(&env->watchpoints);
660 #if defined(TARGET_HAS_ICE)
661 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
662 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
663 }
664 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
665 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
666 wp->flags, NULL);
667 }
668 #endif
669
670 return new_env;
671 }
672
673 #if !defined(CONFIG_USER_ONLY)
674 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
675 uintptr_t length)
676 {
677 uintptr_t start1;
678
679 /* we modify the TLB cache so that the dirty bit will be set again
680 when accessing the range */
681 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
682 /* Check that we don't span multiple blocks - this breaks the
683 address comparisons below. */
684 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
685 != (end - 1) - start) {
686 abort();
687 }
688 cpu_tlb_reset_dirty_all(start1, length);
689
690 }
691
692 /* Note: start and end must be within the same ram block. */
693 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
694 int dirty_flags)
695 {
696 uintptr_t length;
697
698 start &= TARGET_PAGE_MASK;
699 end = TARGET_PAGE_ALIGN(end);
700
701 length = end - start;
702 if (length == 0)
703 return;
704 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
705
706 if (tcg_enabled()) {
707 tlb_reset_dirty_range_all(start, end, length);
708 }
709 }
710
711 static int cpu_physical_memory_set_dirty_tracking(int enable)
712 {
713 int ret = 0;
714 in_migration = enable;
715 return ret;
716 }
717
718 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
719 MemoryRegionSection *section,
720 target_ulong vaddr,
721 hwaddr paddr, hwaddr xlat,
722 int prot,
723 target_ulong *address)
724 {
725 hwaddr iotlb;
726 CPUWatchpoint *wp;
727
728 if (memory_region_is_ram(section->mr)) {
729 /* Normal RAM. */
730 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
731 + xlat;
732 if (!section->readonly) {
733 iotlb |= PHYS_SECTION_NOTDIRTY;
734 } else {
735 iotlb |= PHYS_SECTION_ROM;
736 }
737 } else {
738 iotlb = section - address_space_memory.dispatch->sections;
739 iotlb += xlat;
740 }
741
742 /* Make accesses to pages with watchpoints go via the
743 watchpoint trap routines. */
744 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
745 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
746 /* Avoid trapping reads of pages with a write breakpoint. */
747 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
748 iotlb = PHYS_SECTION_WATCH + paddr;
749 *address |= TLB_MMIO;
750 break;
751 }
752 }
753 }
754
755 return iotlb;
756 }
757 #endif /* defined(CONFIG_USER_ONLY) */
758
759 #if !defined(CONFIG_USER_ONLY)
760
761 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
762 uint16_t section);
763 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
764
765 static uint16_t phys_section_add(MemoryRegionSection *section)
766 {
767 /* The physical section number is ORed with a page-aligned
768 * pointer to produce the iotlb entries. Thus it should
769 * never overflow into the page-aligned value.
770 */
771 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
772
773 if (next_map.sections_nb == next_map.sections_nb_alloc) {
774 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
775 16);
776 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
777 next_map.sections_nb_alloc);
778 }
779 next_map.sections[next_map.sections_nb] = *section;
780 memory_region_ref(section->mr);
781 return next_map.sections_nb++;
782 }
783
784 static void phys_section_destroy(MemoryRegion *mr)
785 {
786 memory_region_unref(mr);
787
788 if (mr->subpage) {
789 subpage_t *subpage = container_of(mr, subpage_t, iomem);
790 memory_region_destroy(&subpage->iomem);
791 g_free(subpage);
792 }
793 }
794
795 static void phys_sections_free(PhysPageMap *map)
796 {
797 while (map->sections_nb > 0) {
798 MemoryRegionSection *section = &map->sections[--map->sections_nb];
799 phys_section_destroy(section->mr);
800 }
801 g_free(map->sections);
802 g_free(map->nodes);
803 g_free(map);
804 }
805
806 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
807 {
808 subpage_t *subpage;
809 hwaddr base = section->offset_within_address_space
810 & TARGET_PAGE_MASK;
811 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
812 next_map.nodes, next_map.sections);
813 MemoryRegionSection subsection = {
814 .offset_within_address_space = base,
815 .size = int128_make64(TARGET_PAGE_SIZE),
816 };
817 hwaddr start, end;
818
819 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
820
821 if (!(existing->mr->subpage)) {
822 subpage = subpage_init(d->as, base);
823 subsection.mr = &subpage->iomem;
824 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
825 phys_section_add(&subsection));
826 } else {
827 subpage = container_of(existing->mr, subpage_t, iomem);
828 }
829 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
830 end = start + int128_get64(section->size) - 1;
831 subpage_register(subpage, start, end, phys_section_add(section));
832 }
833
834
835 static void register_multipage(AddressSpaceDispatch *d,
836 MemoryRegionSection *section)
837 {
838 hwaddr start_addr = section->offset_within_address_space;
839 uint16_t section_index = phys_section_add(section);
840 uint64_t num_pages = int128_get64(int128_rshift(section->size,
841 TARGET_PAGE_BITS));
842
843 assert(num_pages);
844 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
845 }
846
847 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
848 {
849 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
850 AddressSpaceDispatch *d = as->next_dispatch;
851 MemoryRegionSection now = *section, remain = *section;
852 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
853
854 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
855 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
856 - now.offset_within_address_space;
857
858 now.size = int128_min(int128_make64(left), now.size);
859 register_subpage(d, &now);
860 } else {
861 now.size = int128_zero();
862 }
863 while (int128_ne(remain.size, now.size)) {
864 remain.size = int128_sub(remain.size, now.size);
865 remain.offset_within_address_space += int128_get64(now.size);
866 remain.offset_within_region += int128_get64(now.size);
867 now = remain;
868 if (int128_lt(remain.size, page_size)) {
869 register_subpage(d, &now);
870 } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
871 now.size = page_size;
872 register_subpage(d, &now);
873 } else {
874 now.size = int128_and(now.size, int128_neg(page_size));
875 register_multipage(d, &now);
876 }
877 }
878 }
879
880 void qemu_flush_coalesced_mmio_buffer(void)
881 {
882 if (kvm_enabled())
883 kvm_flush_coalesced_mmio_buffer();
884 }
885
886 void qemu_mutex_lock_ramlist(void)
887 {
888 qemu_mutex_lock(&ram_list.mutex);
889 }
890
891 void qemu_mutex_unlock_ramlist(void)
892 {
893 qemu_mutex_unlock(&ram_list.mutex);
894 }
895
896 #if defined(__linux__) && !defined(TARGET_S390X)
897
898 #include <sys/vfs.h>
899
900 #define HUGETLBFS_MAGIC 0x958458f6
901
902 static long gethugepagesize(const char *path)
903 {
904 struct statfs fs;
905 int ret;
906
907 do {
908 ret = statfs(path, &fs);
909 } while (ret != 0 && errno == EINTR);
910
911 if (ret != 0) {
912 perror(path);
913 return 0;
914 }
915
916 if (fs.f_type != HUGETLBFS_MAGIC)
917 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
918
919 return fs.f_bsize;
920 }
921
922 static void *file_ram_alloc(RAMBlock *block,
923 ram_addr_t memory,
924 const char *path)
925 {
926 char *filename;
927 char *sanitized_name;
928 char *c;
929 void *area;
930 int fd;
931 #ifdef MAP_POPULATE
932 int flags;
933 #endif
934 unsigned long hpagesize;
935
936 hpagesize = gethugepagesize(path);
937 if (!hpagesize) {
938 return NULL;
939 }
940
941 if (memory < hpagesize) {
942 return NULL;
943 }
944
945 if (kvm_enabled() && !kvm_has_sync_mmu()) {
946 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
947 return NULL;
948 }
949
950 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
951 sanitized_name = g_strdup(block->mr->name);
952 for (c = sanitized_name; *c != '\0'; c++) {
953 if (*c == '/')
954 *c = '_';
955 }
956
957 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
958 sanitized_name);
959 g_free(sanitized_name);
960
961 fd = mkstemp(filename);
962 if (fd < 0) {
963 perror("unable to create backing store for hugepages");
964 g_free(filename);
965 return NULL;
966 }
967 unlink(filename);
968 g_free(filename);
969
970 memory = (memory+hpagesize-1) & ~(hpagesize-1);
971
972 /*
973 * ftruncate is not supported by hugetlbfs in older
974 * hosts, so don't bother bailing out on errors.
975 * If anything goes wrong with it under other filesystems,
976 * mmap will fail.
977 */
978 if (ftruncate(fd, memory))
979 perror("ftruncate");
980
981 #ifdef MAP_POPULATE
982 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
983 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
984 * to sidestep this quirk.
985 */
986 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
987 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
988 #else
989 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
990 #endif
991 if (area == MAP_FAILED) {
992 perror("file_ram_alloc: can't mmap RAM pages");
993 close(fd);
994 return (NULL);
995 }
996 block->fd = fd;
997 return area;
998 }
999 #endif
1000
1001 static ram_addr_t find_ram_offset(ram_addr_t size)
1002 {
1003 RAMBlock *block, *next_block;
1004 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1005
1006 assert(size != 0); /* it would hand out same offset multiple times */
1007
1008 if (QTAILQ_EMPTY(&ram_list.blocks))
1009 return 0;
1010
1011 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1012 ram_addr_t end, next = RAM_ADDR_MAX;
1013
1014 end = block->offset + block->length;
1015
1016 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1017 if (next_block->offset >= end) {
1018 next = MIN(next, next_block->offset);
1019 }
1020 }
1021 if (next - end >= size && next - end < mingap) {
1022 offset = end;
1023 mingap = next - end;
1024 }
1025 }
1026
1027 if (offset == RAM_ADDR_MAX) {
1028 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1029 (uint64_t)size);
1030 abort();
1031 }
1032
1033 return offset;
1034 }
1035
1036 ram_addr_t last_ram_offset(void)
1037 {
1038 RAMBlock *block;
1039 ram_addr_t last = 0;
1040
1041 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1042 last = MAX(last, block->offset + block->length);
1043
1044 return last;
1045 }
1046
1047 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1048 {
1049 int ret;
1050
1051 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1052 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1053 "dump-guest-core", true)) {
1054 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1055 if (ret) {
1056 perror("qemu_madvise");
1057 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1058 "but dump_guest_core=off specified\n");
1059 }
1060 }
1061 }
1062
1063 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1064 {
1065 RAMBlock *new_block, *block;
1066
1067 new_block = NULL;
1068 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1069 if (block->offset == addr) {
1070 new_block = block;
1071 break;
1072 }
1073 }
1074 assert(new_block);
1075 assert(!new_block->idstr[0]);
1076
1077 if (dev) {
1078 char *id = qdev_get_dev_path(dev);
1079 if (id) {
1080 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1081 g_free(id);
1082 }
1083 }
1084 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1085
1086 /* This assumes the iothread lock is taken here too. */
1087 qemu_mutex_lock_ramlist();
1088 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1089 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1090 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1091 new_block->idstr);
1092 abort();
1093 }
1094 }
1095 qemu_mutex_unlock_ramlist();
1096 }
1097
1098 static int memory_try_enable_merging(void *addr, size_t len)
1099 {
1100 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1101 /* disabled by the user */
1102 return 0;
1103 }
1104
1105 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1106 }
1107
1108 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1109 MemoryRegion *mr)
1110 {
1111 RAMBlock *block, *new_block;
1112
1113 size = TARGET_PAGE_ALIGN(size);
1114 new_block = g_malloc0(sizeof(*new_block));
1115
1116 /* This assumes the iothread lock is taken here too. */
1117 qemu_mutex_lock_ramlist();
1118 new_block->mr = mr;
1119 new_block->offset = find_ram_offset(size);
1120 if (host) {
1121 new_block->host = host;
1122 new_block->flags |= RAM_PREALLOC_MASK;
1123 } else {
1124 if (mem_path) {
1125 #if defined (__linux__) && !defined(TARGET_S390X)
1126 new_block->host = file_ram_alloc(new_block, size, mem_path);
1127 if (!new_block->host) {
1128 new_block->host = qemu_anon_ram_alloc(size);
1129 memory_try_enable_merging(new_block->host, size);
1130 }
1131 #else
1132 fprintf(stderr, "-mem-path option unsupported\n");
1133 exit(1);
1134 #endif
1135 } else {
1136 if (xen_enabled()) {
1137 xen_ram_alloc(new_block->offset, size, mr);
1138 } else if (kvm_enabled()) {
1139 /* some s390/kvm configurations have special constraints */
1140 new_block->host = kvm_ram_alloc(size);
1141 } else {
1142 new_block->host = qemu_anon_ram_alloc(size);
1143 }
1144 memory_try_enable_merging(new_block->host, size);
1145 }
1146 }
1147 new_block->length = size;
1148
1149 /* Keep the list sorted from biggest to smallest block. */
1150 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1151 if (block->length < new_block->length) {
1152 break;
1153 }
1154 }
1155 if (block) {
1156 QTAILQ_INSERT_BEFORE(block, new_block, next);
1157 } else {
1158 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1159 }
1160 ram_list.mru_block = NULL;
1161
1162 ram_list.version++;
1163 qemu_mutex_unlock_ramlist();
1164
1165 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1166 last_ram_offset() >> TARGET_PAGE_BITS);
1167 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1168 0, size >> TARGET_PAGE_BITS);
1169 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1170
1171 qemu_ram_setup_dump(new_block->host, size);
1172 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1173
1174 if (kvm_enabled())
1175 kvm_setup_guest_memory(new_block->host, size);
1176
1177 return new_block->offset;
1178 }
1179
1180 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1181 {
1182 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1183 }
1184
1185 void qemu_ram_free_from_ptr(ram_addr_t addr)
1186 {
1187 RAMBlock *block;
1188
1189 /* This assumes the iothread lock is taken here too. */
1190 qemu_mutex_lock_ramlist();
1191 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1192 if (addr == block->offset) {
1193 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1194 ram_list.mru_block = NULL;
1195 ram_list.version++;
1196 g_free(block);
1197 break;
1198 }
1199 }
1200 qemu_mutex_unlock_ramlist();
1201 }
1202
1203 void qemu_ram_free(ram_addr_t addr)
1204 {
1205 RAMBlock *block;
1206
1207 /* This assumes the iothread lock is taken here too. */
1208 qemu_mutex_lock_ramlist();
1209 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1210 if (addr == block->offset) {
1211 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1212 ram_list.mru_block = NULL;
1213 ram_list.version++;
1214 if (block->flags & RAM_PREALLOC_MASK) {
1215 ;
1216 } else if (mem_path) {
1217 #if defined (__linux__) && !defined(TARGET_S390X)
1218 if (block->fd) {
1219 munmap(block->host, block->length);
1220 close(block->fd);
1221 } else {
1222 qemu_anon_ram_free(block->host, block->length);
1223 }
1224 #else
1225 abort();
1226 #endif
1227 } else {
1228 if (xen_enabled()) {
1229 xen_invalidate_map_cache_entry(block->host);
1230 } else {
1231 qemu_anon_ram_free(block->host, block->length);
1232 }
1233 }
1234 g_free(block);
1235 break;
1236 }
1237 }
1238 qemu_mutex_unlock_ramlist();
1239
1240 }
1241
1242 #ifndef _WIN32
1243 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1244 {
1245 RAMBlock *block;
1246 ram_addr_t offset;
1247 int flags;
1248 void *area, *vaddr;
1249
1250 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1251 offset = addr - block->offset;
1252 if (offset < block->length) {
1253 vaddr = block->host + offset;
1254 if (block->flags & RAM_PREALLOC_MASK) {
1255 ;
1256 } else {
1257 flags = MAP_FIXED;
1258 munmap(vaddr, length);
1259 if (mem_path) {
1260 #if defined(__linux__) && !defined(TARGET_S390X)
1261 if (block->fd) {
1262 #ifdef MAP_POPULATE
1263 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1264 MAP_PRIVATE;
1265 #else
1266 flags |= MAP_PRIVATE;
1267 #endif
1268 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1269 flags, block->fd, offset);
1270 } else {
1271 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1272 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1273 flags, -1, 0);
1274 }
1275 #else
1276 abort();
1277 #endif
1278 } else {
1279 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1280 flags |= MAP_SHARED | MAP_ANONYMOUS;
1281 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1282 flags, -1, 0);
1283 #else
1284 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1285 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1286 flags, -1, 0);
1287 #endif
1288 }
1289 if (area != vaddr) {
1290 fprintf(stderr, "Could not remap addr: "
1291 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1292 length, addr);
1293 exit(1);
1294 }
1295 memory_try_enable_merging(vaddr, length);
1296 qemu_ram_setup_dump(vaddr, length);
1297 }
1298 return;
1299 }
1300 }
1301 }
1302 #endif /* !_WIN32 */
1303
1304 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1305 {
1306 RAMBlock *block;
1307
1308 /* The list is protected by the iothread lock here. */
1309 block = ram_list.mru_block;
1310 if (block && addr - block->offset < block->length) {
1311 goto found;
1312 }
1313 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1314 if (addr - block->offset < block->length) {
1315 goto found;
1316 }
1317 }
1318
1319 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1320 abort();
1321
1322 found:
1323 ram_list.mru_block = block;
1324 return block;
1325 }
1326
1327 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1328 With the exception of the softmmu code in this file, this should
1329 only be used for local memory (e.g. video ram) that the device owns,
1330 and knows it isn't going to access beyond the end of the block.
1331
1332 It should not be used for general purpose DMA.
1333 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1334 */
1335 void *qemu_get_ram_ptr(ram_addr_t addr)
1336 {
1337 RAMBlock *block = qemu_get_ram_block(addr);
1338
1339 if (xen_enabled()) {
1340 /* We need to check if the requested address is in the RAM
1341 * because we don't want to map the entire memory in QEMU.
1342 * In that case just map until the end of the page.
1343 */
1344 if (block->offset == 0) {
1345 return xen_map_cache(addr, 0, 0);
1346 } else if (block->host == NULL) {
1347 block->host =
1348 xen_map_cache(block->offset, block->length, 1);
1349 }
1350 }
1351 return block->host + (addr - block->offset);
1352 }
1353
1354 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1355 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1356 *
1357 * ??? Is this still necessary?
1358 */
1359 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1360 {
1361 RAMBlock *block;
1362
1363 /* The list is protected by the iothread lock here. */
1364 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1365 if (addr - block->offset < block->length) {
1366 if (xen_enabled()) {
1367 /* We need to check if the requested address is in the RAM
1368 * because we don't want to map the entire memory in QEMU.
1369 * In that case just map until the end of the page.
1370 */
1371 if (block->offset == 0) {
1372 return xen_map_cache(addr, 0, 0);
1373 } else if (block->host == NULL) {
1374 block->host =
1375 xen_map_cache(block->offset, block->length, 1);
1376 }
1377 }
1378 return block->host + (addr - block->offset);
1379 }
1380 }
1381
1382 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1383 abort();
1384
1385 return NULL;
1386 }
1387
1388 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1389 * but takes a size argument */
1390 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1391 {
1392 if (*size == 0) {
1393 return NULL;
1394 }
1395 if (xen_enabled()) {
1396 return xen_map_cache(addr, *size, 1);
1397 } else {
1398 RAMBlock *block;
1399
1400 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1401 if (addr - block->offset < block->length) {
1402 if (addr - block->offset + *size > block->length)
1403 *size = block->length - addr + block->offset;
1404 return block->host + (addr - block->offset);
1405 }
1406 }
1407
1408 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1409 abort();
1410 }
1411 }
1412
1413 /* Some of the softmmu routines need to translate from a host pointer
1414 (typically a TLB entry) back to a ram offset. */
1415 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1416 {
1417 RAMBlock *block;
1418 uint8_t *host = ptr;
1419
1420 if (xen_enabled()) {
1421 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1422 return qemu_get_ram_block(*ram_addr)->mr;
1423 }
1424
1425 block = ram_list.mru_block;
1426 if (block && block->host && host - block->host < block->length) {
1427 goto found;
1428 }
1429
1430 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1431 /* This case append when the block is not mapped. */
1432 if (block->host == NULL) {
1433 continue;
1434 }
1435 if (host - block->host < block->length) {
1436 goto found;
1437 }
1438 }
1439
1440 return NULL;
1441
1442 found:
1443 *ram_addr = block->offset + (host - block->host);
1444 return block->mr;
1445 }
1446
1447 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1448 uint64_t val, unsigned size)
1449 {
1450 int dirty_flags;
1451 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1452 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1453 tb_invalidate_phys_page_fast(ram_addr, size);
1454 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1455 }
1456 switch (size) {
1457 case 1:
1458 stb_p(qemu_get_ram_ptr(ram_addr), val);
1459 break;
1460 case 2:
1461 stw_p(qemu_get_ram_ptr(ram_addr), val);
1462 break;
1463 case 4:
1464 stl_p(qemu_get_ram_ptr(ram_addr), val);
1465 break;
1466 default:
1467 abort();
1468 }
1469 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1470 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1471 /* we remove the notdirty callback only if the code has been
1472 flushed */
1473 if (dirty_flags == 0xff) {
1474 CPUArchState *env = current_cpu->env_ptr;
1475 tlb_set_dirty(env, env->mem_io_vaddr);
1476 }
1477 }
1478
1479 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1480 unsigned size, bool is_write)
1481 {
1482 return is_write;
1483 }
1484
1485 static const MemoryRegionOps notdirty_mem_ops = {
1486 .write = notdirty_mem_write,
1487 .valid.accepts = notdirty_mem_accepts,
1488 .endianness = DEVICE_NATIVE_ENDIAN,
1489 };
1490
1491 /* Generate a debug exception if a watchpoint has been hit. */
1492 static void check_watchpoint(int offset, int len_mask, int flags)
1493 {
1494 CPUArchState *env = current_cpu->env_ptr;
1495 target_ulong pc, cs_base;
1496 target_ulong vaddr;
1497 CPUWatchpoint *wp;
1498 int cpu_flags;
1499
1500 if (env->watchpoint_hit) {
1501 /* We re-entered the check after replacing the TB. Now raise
1502 * the debug interrupt so that is will trigger after the
1503 * current instruction. */
1504 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1505 return;
1506 }
1507 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1508 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1509 if ((vaddr == (wp->vaddr & len_mask) ||
1510 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1511 wp->flags |= BP_WATCHPOINT_HIT;
1512 if (!env->watchpoint_hit) {
1513 env->watchpoint_hit = wp;
1514 tb_check_watchpoint(env);
1515 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1516 env->exception_index = EXCP_DEBUG;
1517 cpu_loop_exit(env);
1518 } else {
1519 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1520 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1521 cpu_resume_from_signal(env, NULL);
1522 }
1523 }
1524 } else {
1525 wp->flags &= ~BP_WATCHPOINT_HIT;
1526 }
1527 }
1528 }
1529
1530 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1531 so these check for a hit then pass through to the normal out-of-line
1532 phys routines. */
1533 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1534 unsigned size)
1535 {
1536 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1537 switch (size) {
1538 case 1: return ldub_phys(addr);
1539 case 2: return lduw_phys(addr);
1540 case 4: return ldl_phys(addr);
1541 default: abort();
1542 }
1543 }
1544
1545 static void watch_mem_write(void *opaque, hwaddr addr,
1546 uint64_t val, unsigned size)
1547 {
1548 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1549 switch (size) {
1550 case 1:
1551 stb_phys(addr, val);
1552 break;
1553 case 2:
1554 stw_phys(addr, val);
1555 break;
1556 case 4:
1557 stl_phys(addr, val);
1558 break;
1559 default: abort();
1560 }
1561 }
1562
1563 static const MemoryRegionOps watch_mem_ops = {
1564 .read = watch_mem_read,
1565 .write = watch_mem_write,
1566 .endianness = DEVICE_NATIVE_ENDIAN,
1567 };
1568
1569 static uint64_t subpage_read(void *opaque, hwaddr addr,
1570 unsigned len)
1571 {
1572 subpage_t *subpage = opaque;
1573 uint8_t buf[4];
1574
1575 #if defined(DEBUG_SUBPAGE)
1576 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1577 subpage, len, addr);
1578 #endif
1579 address_space_read(subpage->as, addr + subpage->base, buf, len);
1580 switch (len) {
1581 case 1:
1582 return ldub_p(buf);
1583 case 2:
1584 return lduw_p(buf);
1585 case 4:
1586 return ldl_p(buf);
1587 default:
1588 abort();
1589 }
1590 }
1591
1592 static void subpage_write(void *opaque, hwaddr addr,
1593 uint64_t value, unsigned len)
1594 {
1595 subpage_t *subpage = opaque;
1596 uint8_t buf[4];
1597
1598 #if defined(DEBUG_SUBPAGE)
1599 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1600 " value %"PRIx64"\n",
1601 __func__, subpage, len, addr, value);
1602 #endif
1603 switch (len) {
1604 case 1:
1605 stb_p(buf, value);
1606 break;
1607 case 2:
1608 stw_p(buf, value);
1609 break;
1610 case 4:
1611 stl_p(buf, value);
1612 break;
1613 default:
1614 abort();
1615 }
1616 address_space_write(subpage->as, addr + subpage->base, buf, len);
1617 }
1618
1619 static bool subpage_accepts(void *opaque, hwaddr addr,
1620 unsigned size, bool is_write)
1621 {
1622 subpage_t *subpage = opaque;
1623 #if defined(DEBUG_SUBPAGE)
1624 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1625 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1626 #endif
1627
1628 return address_space_access_valid(subpage->as, addr + subpage->base,
1629 size, is_write);
1630 }
1631
1632 static const MemoryRegionOps subpage_ops = {
1633 .read = subpage_read,
1634 .write = subpage_write,
1635 .valid.accepts = subpage_accepts,
1636 .endianness = DEVICE_NATIVE_ENDIAN,
1637 };
1638
1639 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1640 uint16_t section)
1641 {
1642 int idx, eidx;
1643
1644 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1645 return -1;
1646 idx = SUBPAGE_IDX(start);
1647 eidx = SUBPAGE_IDX(end);
1648 #if defined(DEBUG_SUBPAGE)
1649 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1650 mmio, start, end, idx, eidx, memory);
1651 #endif
1652 for (; idx <= eidx; idx++) {
1653 mmio->sub_section[idx] = section;
1654 }
1655
1656 return 0;
1657 }
1658
1659 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1660 {
1661 subpage_t *mmio;
1662
1663 mmio = g_malloc0(sizeof(subpage_t));
1664
1665 mmio->as = as;
1666 mmio->base = base;
1667 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1668 "subpage", TARGET_PAGE_SIZE);
1669 mmio->iomem.subpage = true;
1670 #if defined(DEBUG_SUBPAGE)
1671 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1672 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1673 #endif
1674 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1675
1676 return mmio;
1677 }
1678
1679 static uint16_t dummy_section(MemoryRegion *mr)
1680 {
1681 MemoryRegionSection section = {
1682 .mr = mr,
1683 .offset_within_address_space = 0,
1684 .offset_within_region = 0,
1685 .size = int128_2_64(),
1686 };
1687
1688 return phys_section_add(&section);
1689 }
1690
1691 MemoryRegion *iotlb_to_region(hwaddr index)
1692 {
1693 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1694 }
1695
1696 static void io_mem_init(void)
1697 {
1698 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1699 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1700 "unassigned", UINT64_MAX);
1701 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1702 "notdirty", UINT64_MAX);
1703 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1704 "watch", UINT64_MAX);
1705 }
1706
1707 static void mem_begin(MemoryListener *listener)
1708 {
1709 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1710 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1711
1712 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1713 d->as = as;
1714 as->next_dispatch = d;
1715 }
1716
1717 static void mem_commit(MemoryListener *listener)
1718 {
1719 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1720 AddressSpaceDispatch *cur = as->dispatch;
1721 AddressSpaceDispatch *next = as->next_dispatch;
1722
1723 next->nodes = next_map.nodes;
1724 next->sections = next_map.sections;
1725
1726 as->dispatch = next;
1727 g_free(cur);
1728 }
1729
1730 static void core_begin(MemoryListener *listener)
1731 {
1732 uint16_t n;
1733
1734 prev_map = g_new(PhysPageMap, 1);
1735 *prev_map = next_map;
1736
1737 memset(&next_map, 0, sizeof(next_map));
1738 n = dummy_section(&io_mem_unassigned);
1739 assert(n == PHYS_SECTION_UNASSIGNED);
1740 n = dummy_section(&io_mem_notdirty);
1741 assert(n == PHYS_SECTION_NOTDIRTY);
1742 n = dummy_section(&io_mem_rom);
1743 assert(n == PHYS_SECTION_ROM);
1744 n = dummy_section(&io_mem_watch);
1745 assert(n == PHYS_SECTION_WATCH);
1746 }
1747
1748 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1749 * All AddressSpaceDispatch instances have switched to the next map.
1750 */
1751 static void core_commit(MemoryListener *listener)
1752 {
1753 phys_sections_free(prev_map);
1754 }
1755
1756 static void tcg_commit(MemoryListener *listener)
1757 {
1758 CPUState *cpu;
1759
1760 /* since each CPU stores ram addresses in its TLB cache, we must
1761 reset the modified entries */
1762 /* XXX: slow ! */
1763 for (cpu = first_cpu; cpu != NULL; cpu = cpu->next_cpu) {
1764 CPUArchState *env = cpu->env_ptr;
1765
1766 tlb_flush(env, 1);
1767 }
1768 }
1769
1770 static void core_log_global_start(MemoryListener *listener)
1771 {
1772 cpu_physical_memory_set_dirty_tracking(1);
1773 }
1774
1775 static void core_log_global_stop(MemoryListener *listener)
1776 {
1777 cpu_physical_memory_set_dirty_tracking(0);
1778 }
1779
1780 static MemoryListener core_memory_listener = {
1781 .begin = core_begin,
1782 .commit = core_commit,
1783 .log_global_start = core_log_global_start,
1784 .log_global_stop = core_log_global_stop,
1785 .priority = 1,
1786 };
1787
1788 static MemoryListener tcg_memory_listener = {
1789 .commit = tcg_commit,
1790 };
1791
1792 void address_space_init_dispatch(AddressSpace *as)
1793 {
1794 as->dispatch = NULL;
1795 as->dispatch_listener = (MemoryListener) {
1796 .begin = mem_begin,
1797 .commit = mem_commit,
1798 .region_add = mem_add,
1799 .region_nop = mem_add,
1800 .priority = 0,
1801 };
1802 memory_listener_register(&as->dispatch_listener, as);
1803 }
1804
1805 void address_space_destroy_dispatch(AddressSpace *as)
1806 {
1807 AddressSpaceDispatch *d = as->dispatch;
1808
1809 memory_listener_unregister(&as->dispatch_listener);
1810 g_free(d);
1811 as->dispatch = NULL;
1812 }
1813
1814 static void memory_map_init(void)
1815 {
1816 system_memory = g_malloc(sizeof(*system_memory));
1817 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1818 address_space_init(&address_space_memory, system_memory, "memory");
1819
1820 system_io = g_malloc(sizeof(*system_io));
1821 memory_region_init(system_io, NULL, "io", 65536);
1822 address_space_init(&address_space_io, system_io, "I/O");
1823
1824 memory_listener_register(&core_memory_listener, &address_space_memory);
1825 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1826 }
1827
1828 MemoryRegion *get_system_memory(void)
1829 {
1830 return system_memory;
1831 }
1832
1833 MemoryRegion *get_system_io(void)
1834 {
1835 return system_io;
1836 }
1837
1838 #endif /* !defined(CONFIG_USER_ONLY) */
1839
1840 /* physical memory access (slow version, mainly for debug) */
1841 #if defined(CONFIG_USER_ONLY)
1842 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1843 uint8_t *buf, int len, int is_write)
1844 {
1845 int l, flags;
1846 target_ulong page;
1847 void * p;
1848
1849 while (len > 0) {
1850 page = addr & TARGET_PAGE_MASK;
1851 l = (page + TARGET_PAGE_SIZE) - addr;
1852 if (l > len)
1853 l = len;
1854 flags = page_get_flags(page);
1855 if (!(flags & PAGE_VALID))
1856 return -1;
1857 if (is_write) {
1858 if (!(flags & PAGE_WRITE))
1859 return -1;
1860 /* XXX: this code should not depend on lock_user */
1861 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1862 return -1;
1863 memcpy(p, buf, l);
1864 unlock_user(p, addr, l);
1865 } else {
1866 if (!(flags & PAGE_READ))
1867 return -1;
1868 /* XXX: this code should not depend on lock_user */
1869 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1870 return -1;
1871 memcpy(buf, p, l);
1872 unlock_user(p, addr, 0);
1873 }
1874 len -= l;
1875 buf += l;
1876 addr += l;
1877 }
1878 return 0;
1879 }
1880
1881 #else
1882
1883 static void invalidate_and_set_dirty(hwaddr addr,
1884 hwaddr length)
1885 {
1886 if (!cpu_physical_memory_is_dirty(addr)) {
1887 /* invalidate code */
1888 tb_invalidate_phys_page_range(addr, addr + length, 0);
1889 /* set dirty bit */
1890 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1891 }
1892 xen_modified_memory(addr, length);
1893 }
1894
1895 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1896 {
1897 if (memory_region_is_ram(mr)) {
1898 return !(is_write && mr->readonly);
1899 }
1900 if (memory_region_is_romd(mr)) {
1901 return !is_write;
1902 }
1903
1904 return false;
1905 }
1906
1907 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1908 {
1909 unsigned access_size_max = mr->ops->valid.max_access_size;
1910
1911 /* Regions are assumed to support 1-4 byte accesses unless
1912 otherwise specified. */
1913 if (access_size_max == 0) {
1914 access_size_max = 4;
1915 }
1916
1917 /* Bound the maximum access by the alignment of the address. */
1918 if (!mr->ops->impl.unaligned) {
1919 unsigned align_size_max = addr & -addr;
1920 if (align_size_max != 0 && align_size_max < access_size_max) {
1921 access_size_max = align_size_max;
1922 }
1923 }
1924
1925 /* Don't attempt accesses larger than the maximum. */
1926 if (l > access_size_max) {
1927 l = access_size_max;
1928 }
1929
1930 return l;
1931 }
1932
1933 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1934 int len, bool is_write)
1935 {
1936 hwaddr l;
1937 uint8_t *ptr;
1938 uint64_t val;
1939 hwaddr addr1;
1940 MemoryRegion *mr;
1941 bool error = false;
1942
1943 while (len > 0) {
1944 l = len;
1945 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1946
1947 if (is_write) {
1948 if (!memory_access_is_direct(mr, is_write)) {
1949 l = memory_access_size(mr, l, addr1);
1950 /* XXX: could force current_cpu to NULL to avoid
1951 potential bugs */
1952 switch (l) {
1953 case 8:
1954 /* 64 bit write access */
1955 val = ldq_p(buf);
1956 error |= io_mem_write(mr, addr1, val, 8);
1957 break;
1958 case 4:
1959 /* 32 bit write access */
1960 val = ldl_p(buf);
1961 error |= io_mem_write(mr, addr1, val, 4);
1962 break;
1963 case 2:
1964 /* 16 bit write access */
1965 val = lduw_p(buf);
1966 error |= io_mem_write(mr, addr1, val, 2);
1967 break;
1968 case 1:
1969 /* 8 bit write access */
1970 val = ldub_p(buf);
1971 error |= io_mem_write(mr, addr1, val, 1);
1972 break;
1973 default:
1974 abort();
1975 }
1976 } else {
1977 addr1 += memory_region_get_ram_addr(mr);
1978 /* RAM case */
1979 ptr = qemu_get_ram_ptr(addr1);
1980 memcpy(ptr, buf, l);
1981 invalidate_and_set_dirty(addr1, l);
1982 }
1983 } else {
1984 if (!memory_access_is_direct(mr, is_write)) {
1985 /* I/O case */
1986 l = memory_access_size(mr, l, addr1);
1987 switch (l) {
1988 case 8:
1989 /* 64 bit read access */
1990 error |= io_mem_read(mr, addr1, &val, 8);
1991 stq_p(buf, val);
1992 break;
1993 case 4:
1994 /* 32 bit read access */
1995 error |= io_mem_read(mr, addr1, &val, 4);
1996 stl_p(buf, val);
1997 break;
1998 case 2:
1999 /* 16 bit read access */
2000 error |= io_mem_read(mr, addr1, &val, 2);
2001 stw_p(buf, val);
2002 break;
2003 case 1:
2004 /* 8 bit read access */
2005 error |= io_mem_read(mr, addr1, &val, 1);
2006 stb_p(buf, val);
2007 break;
2008 default:
2009 abort();
2010 }
2011 } else {
2012 /* RAM case */
2013 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2014 memcpy(buf, ptr, l);
2015 }
2016 }
2017 len -= l;
2018 buf += l;
2019 addr += l;
2020 }
2021
2022 return error;
2023 }
2024
2025 bool address_space_write(AddressSpace *as, hwaddr addr,
2026 const uint8_t *buf, int len)
2027 {
2028 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2029 }
2030
2031 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2032 {
2033 return address_space_rw(as, addr, buf, len, false);
2034 }
2035
2036
2037 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2038 int len, int is_write)
2039 {
2040 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2041 }
2042
2043 /* used for ROM loading : can write in RAM and ROM */
2044 void cpu_physical_memory_write_rom(hwaddr addr,
2045 const uint8_t *buf, int len)
2046 {
2047 hwaddr l;
2048 uint8_t *ptr;
2049 hwaddr addr1;
2050 MemoryRegion *mr;
2051
2052 while (len > 0) {
2053 l = len;
2054 mr = address_space_translate(&address_space_memory,
2055 addr, &addr1, &l, true);
2056
2057 if (!(memory_region_is_ram(mr) ||
2058 memory_region_is_romd(mr))) {
2059 /* do nothing */
2060 } else {
2061 addr1 += memory_region_get_ram_addr(mr);
2062 /* ROM/RAM case */
2063 ptr = qemu_get_ram_ptr(addr1);
2064 memcpy(ptr, buf, l);
2065 invalidate_and_set_dirty(addr1, l);
2066 }
2067 len -= l;
2068 buf += l;
2069 addr += l;
2070 }
2071 }
2072
2073 typedef struct {
2074 MemoryRegion *mr;
2075 void *buffer;
2076 hwaddr addr;
2077 hwaddr len;
2078 } BounceBuffer;
2079
2080 static BounceBuffer bounce;
2081
2082 typedef struct MapClient {
2083 void *opaque;
2084 void (*callback)(void *opaque);
2085 QLIST_ENTRY(MapClient) link;
2086 } MapClient;
2087
2088 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2089 = QLIST_HEAD_INITIALIZER(map_client_list);
2090
2091 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2092 {
2093 MapClient *client = g_malloc(sizeof(*client));
2094
2095 client->opaque = opaque;
2096 client->callback = callback;
2097 QLIST_INSERT_HEAD(&map_client_list, client, link);
2098 return client;
2099 }
2100
2101 static void cpu_unregister_map_client(void *_client)
2102 {
2103 MapClient *client = (MapClient *)_client;
2104
2105 QLIST_REMOVE(client, link);
2106 g_free(client);
2107 }
2108
2109 static void cpu_notify_map_clients(void)
2110 {
2111 MapClient *client;
2112
2113 while (!QLIST_EMPTY(&map_client_list)) {
2114 client = QLIST_FIRST(&map_client_list);
2115 client->callback(client->opaque);
2116 cpu_unregister_map_client(client);
2117 }
2118 }
2119
2120 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2121 {
2122 MemoryRegion *mr;
2123 hwaddr l, xlat;
2124
2125 while (len > 0) {
2126 l = len;
2127 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2128 if (!memory_access_is_direct(mr, is_write)) {
2129 l = memory_access_size(mr, l, addr);
2130 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2131 return false;
2132 }
2133 }
2134
2135 len -= l;
2136 addr += l;
2137 }
2138 return true;
2139 }
2140
2141 /* Map a physical memory region into a host virtual address.
2142 * May map a subset of the requested range, given by and returned in *plen.
2143 * May return NULL if resources needed to perform the mapping are exhausted.
2144 * Use only for reads OR writes - not for read-modify-write operations.
2145 * Use cpu_register_map_client() to know when retrying the map operation is
2146 * likely to succeed.
2147 */
2148 void *address_space_map(AddressSpace *as,
2149 hwaddr addr,
2150 hwaddr *plen,
2151 bool is_write)
2152 {
2153 hwaddr len = *plen;
2154 hwaddr done = 0;
2155 hwaddr l, xlat, base;
2156 MemoryRegion *mr, *this_mr;
2157 ram_addr_t raddr;
2158
2159 if (len == 0) {
2160 return NULL;
2161 }
2162
2163 l = len;
2164 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2165 if (!memory_access_is_direct(mr, is_write)) {
2166 if (bounce.buffer) {
2167 return NULL;
2168 }
2169 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2170 bounce.addr = addr;
2171 bounce.len = l;
2172
2173 memory_region_ref(mr);
2174 bounce.mr = mr;
2175 if (!is_write) {
2176 address_space_read(as, addr, bounce.buffer, l);
2177 }
2178
2179 *plen = l;
2180 return bounce.buffer;
2181 }
2182
2183 base = xlat;
2184 raddr = memory_region_get_ram_addr(mr);
2185
2186 for (;;) {
2187 len -= l;
2188 addr += l;
2189 done += l;
2190 if (len == 0) {
2191 break;
2192 }
2193
2194 l = len;
2195 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2196 if (this_mr != mr || xlat != base + done) {
2197 break;
2198 }
2199 }
2200
2201 memory_region_ref(mr);
2202 *plen = done;
2203 return qemu_ram_ptr_length(raddr + base, plen);
2204 }
2205
2206 /* Unmaps a memory region previously mapped by address_space_map().
2207 * Will also mark the memory as dirty if is_write == 1. access_len gives
2208 * the amount of memory that was actually read or written by the caller.
2209 */
2210 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2211 int is_write, hwaddr access_len)
2212 {
2213 if (buffer != bounce.buffer) {
2214 MemoryRegion *mr;
2215 ram_addr_t addr1;
2216
2217 mr = qemu_ram_addr_from_host(buffer, &addr1);
2218 assert(mr != NULL);
2219 if (is_write) {
2220 while (access_len) {
2221 unsigned l;
2222 l = TARGET_PAGE_SIZE;
2223 if (l > access_len)
2224 l = access_len;
2225 invalidate_and_set_dirty(addr1, l);
2226 addr1 += l;
2227 access_len -= l;
2228 }
2229 }
2230 if (xen_enabled()) {
2231 xen_invalidate_map_cache_entry(buffer);
2232 }
2233 memory_region_unref(mr);
2234 return;
2235 }
2236 if (is_write) {
2237 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2238 }
2239 qemu_vfree(bounce.buffer);
2240 bounce.buffer = NULL;
2241 memory_region_unref(bounce.mr);
2242 cpu_notify_map_clients();
2243 }
2244
2245 void *cpu_physical_memory_map(hwaddr addr,
2246 hwaddr *plen,
2247 int is_write)
2248 {
2249 return address_space_map(&address_space_memory, addr, plen, is_write);
2250 }
2251
2252 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2253 int is_write, hwaddr access_len)
2254 {
2255 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2256 }
2257
2258 /* warning: addr must be aligned */
2259 static inline uint32_t ldl_phys_internal(hwaddr addr,
2260 enum device_endian endian)
2261 {
2262 uint8_t *ptr;
2263 uint64_t val;
2264 MemoryRegion *mr;
2265 hwaddr l = 4;
2266 hwaddr addr1;
2267
2268 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2269 false);
2270 if (l < 4 || !memory_access_is_direct(mr, false)) {
2271 /* I/O case */
2272 io_mem_read(mr, addr1, &val, 4);
2273 #if defined(TARGET_WORDS_BIGENDIAN)
2274 if (endian == DEVICE_LITTLE_ENDIAN) {
2275 val = bswap32(val);
2276 }
2277 #else
2278 if (endian == DEVICE_BIG_ENDIAN) {
2279 val = bswap32(val);
2280 }
2281 #endif
2282 } else {
2283 /* RAM case */
2284 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2285 & TARGET_PAGE_MASK)
2286 + addr1);
2287 switch (endian) {
2288 case DEVICE_LITTLE_ENDIAN:
2289 val = ldl_le_p(ptr);
2290 break;
2291 case DEVICE_BIG_ENDIAN:
2292 val = ldl_be_p(ptr);
2293 break;
2294 default:
2295 val = ldl_p(ptr);
2296 break;
2297 }
2298 }
2299 return val;
2300 }
2301
2302 uint32_t ldl_phys(hwaddr addr)
2303 {
2304 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2305 }
2306
2307 uint32_t ldl_le_phys(hwaddr addr)
2308 {
2309 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2310 }
2311
2312 uint32_t ldl_be_phys(hwaddr addr)
2313 {
2314 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2315 }
2316
2317 /* warning: addr must be aligned */
2318 static inline uint64_t ldq_phys_internal(hwaddr addr,
2319 enum device_endian endian)
2320 {
2321 uint8_t *ptr;
2322 uint64_t val;
2323 MemoryRegion *mr;
2324 hwaddr l = 8;
2325 hwaddr addr1;
2326
2327 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2328 false);
2329 if (l < 8 || !memory_access_is_direct(mr, false)) {
2330 /* I/O case */
2331 io_mem_read(mr, addr1, &val, 8);
2332 #if defined(TARGET_WORDS_BIGENDIAN)
2333 if (endian == DEVICE_LITTLE_ENDIAN) {
2334 val = bswap64(val);
2335 }
2336 #else
2337 if (endian == DEVICE_BIG_ENDIAN) {
2338 val = bswap64(val);
2339 }
2340 #endif
2341 } else {
2342 /* RAM case */
2343 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2344 & TARGET_PAGE_MASK)
2345 + addr1);
2346 switch (endian) {
2347 case DEVICE_LITTLE_ENDIAN:
2348 val = ldq_le_p(ptr);
2349 break;
2350 case DEVICE_BIG_ENDIAN:
2351 val = ldq_be_p(ptr);
2352 break;
2353 default:
2354 val = ldq_p(ptr);
2355 break;
2356 }
2357 }
2358 return val;
2359 }
2360
2361 uint64_t ldq_phys(hwaddr addr)
2362 {
2363 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2364 }
2365
2366 uint64_t ldq_le_phys(hwaddr addr)
2367 {
2368 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2369 }
2370
2371 uint64_t ldq_be_phys(hwaddr addr)
2372 {
2373 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2374 }
2375
2376 /* XXX: optimize */
2377 uint32_t ldub_phys(hwaddr addr)
2378 {
2379 uint8_t val;
2380 cpu_physical_memory_read(addr, &val, 1);
2381 return val;
2382 }
2383
2384 /* warning: addr must be aligned */
2385 static inline uint32_t lduw_phys_internal(hwaddr addr,
2386 enum device_endian endian)
2387 {
2388 uint8_t *ptr;
2389 uint64_t val;
2390 MemoryRegion *mr;
2391 hwaddr l = 2;
2392 hwaddr addr1;
2393
2394 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2395 false);
2396 if (l < 2 || !memory_access_is_direct(mr, false)) {
2397 /* I/O case */
2398 io_mem_read(mr, addr1, &val, 2);
2399 #if defined(TARGET_WORDS_BIGENDIAN)
2400 if (endian == DEVICE_LITTLE_ENDIAN) {
2401 val = bswap16(val);
2402 }
2403 #else
2404 if (endian == DEVICE_BIG_ENDIAN) {
2405 val = bswap16(val);
2406 }
2407 #endif
2408 } else {
2409 /* RAM case */
2410 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2411 & TARGET_PAGE_MASK)
2412 + addr1);
2413 switch (endian) {
2414 case DEVICE_LITTLE_ENDIAN:
2415 val = lduw_le_p(ptr);
2416 break;
2417 case DEVICE_BIG_ENDIAN:
2418 val = lduw_be_p(ptr);
2419 break;
2420 default:
2421 val = lduw_p(ptr);
2422 break;
2423 }
2424 }
2425 return val;
2426 }
2427
2428 uint32_t lduw_phys(hwaddr addr)
2429 {
2430 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2431 }
2432
2433 uint32_t lduw_le_phys(hwaddr addr)
2434 {
2435 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2436 }
2437
2438 uint32_t lduw_be_phys(hwaddr addr)
2439 {
2440 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2441 }
2442
2443 /* warning: addr must be aligned. The ram page is not masked as dirty
2444 and the code inside is not invalidated. It is useful if the dirty
2445 bits are used to track modified PTEs */
2446 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2447 {
2448 uint8_t *ptr;
2449 MemoryRegion *mr;
2450 hwaddr l = 4;
2451 hwaddr addr1;
2452
2453 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2454 true);
2455 if (l < 4 || !memory_access_is_direct(mr, true)) {
2456 io_mem_write(mr, addr1, val, 4);
2457 } else {
2458 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2459 ptr = qemu_get_ram_ptr(addr1);
2460 stl_p(ptr, val);
2461
2462 if (unlikely(in_migration)) {
2463 if (!cpu_physical_memory_is_dirty(addr1)) {
2464 /* invalidate code */
2465 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2466 /* set dirty bit */
2467 cpu_physical_memory_set_dirty_flags(
2468 addr1, (0xff & ~CODE_DIRTY_FLAG));
2469 }
2470 }
2471 }
2472 }
2473
2474 /* warning: addr must be aligned */
2475 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2476 enum device_endian endian)
2477 {
2478 uint8_t *ptr;
2479 MemoryRegion *mr;
2480 hwaddr l = 4;
2481 hwaddr addr1;
2482
2483 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2484 true);
2485 if (l < 4 || !memory_access_is_direct(mr, true)) {
2486 #if defined(TARGET_WORDS_BIGENDIAN)
2487 if (endian == DEVICE_LITTLE_ENDIAN) {
2488 val = bswap32(val);
2489 }
2490 #else
2491 if (endian == DEVICE_BIG_ENDIAN) {
2492 val = bswap32(val);
2493 }
2494 #endif
2495 io_mem_write(mr, addr1, val, 4);
2496 } else {
2497 /* RAM case */
2498 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2499 ptr = qemu_get_ram_ptr(addr1);
2500 switch (endian) {
2501 case DEVICE_LITTLE_ENDIAN:
2502 stl_le_p(ptr, val);
2503 break;
2504 case DEVICE_BIG_ENDIAN:
2505 stl_be_p(ptr, val);
2506 break;
2507 default:
2508 stl_p(ptr, val);
2509 break;
2510 }
2511 invalidate_and_set_dirty(addr1, 4);
2512 }
2513 }
2514
2515 void stl_phys(hwaddr addr, uint32_t val)
2516 {
2517 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2518 }
2519
2520 void stl_le_phys(hwaddr addr, uint32_t val)
2521 {
2522 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2523 }
2524
2525 void stl_be_phys(hwaddr addr, uint32_t val)
2526 {
2527 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2528 }
2529
2530 /* XXX: optimize */
2531 void stb_phys(hwaddr addr, uint32_t val)
2532 {
2533 uint8_t v = val;
2534 cpu_physical_memory_write(addr, &v, 1);
2535 }
2536
2537 /* warning: addr must be aligned */
2538 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2539 enum device_endian endian)
2540 {
2541 uint8_t *ptr;
2542 MemoryRegion *mr;
2543 hwaddr l = 2;
2544 hwaddr addr1;
2545
2546 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2547 true);
2548 if (l < 2 || !memory_access_is_direct(mr, true)) {
2549 #if defined(TARGET_WORDS_BIGENDIAN)
2550 if (endian == DEVICE_LITTLE_ENDIAN) {
2551 val = bswap16(val);
2552 }
2553 #else
2554 if (endian == DEVICE_BIG_ENDIAN) {
2555 val = bswap16(val);
2556 }
2557 #endif
2558 io_mem_write(mr, addr1, val, 2);
2559 } else {
2560 /* RAM case */
2561 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2562 ptr = qemu_get_ram_ptr(addr1);
2563 switch (endian) {
2564 case DEVICE_LITTLE_ENDIAN:
2565 stw_le_p(ptr, val);
2566 break;
2567 case DEVICE_BIG_ENDIAN:
2568 stw_be_p(ptr, val);
2569 break;
2570 default:
2571 stw_p(ptr, val);
2572 break;
2573 }
2574 invalidate_and_set_dirty(addr1, 2);
2575 }
2576 }
2577
2578 void stw_phys(hwaddr addr, uint32_t val)
2579 {
2580 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2581 }
2582
2583 void stw_le_phys(hwaddr addr, uint32_t val)
2584 {
2585 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2586 }
2587
2588 void stw_be_phys(hwaddr addr, uint32_t val)
2589 {
2590 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2591 }
2592
2593 /* XXX: optimize */
2594 void stq_phys(hwaddr addr, uint64_t val)
2595 {
2596 val = tswap64(val);
2597 cpu_physical_memory_write(addr, &val, 8);
2598 }
2599
2600 void stq_le_phys(hwaddr addr, uint64_t val)
2601 {
2602 val = cpu_to_le64(val);
2603 cpu_physical_memory_write(addr, &val, 8);
2604 }
2605
2606 void stq_be_phys(hwaddr addr, uint64_t val)
2607 {
2608 val = cpu_to_be64(val);
2609 cpu_physical_memory_write(addr, &val, 8);
2610 }
2611
2612 /* virtual memory access for debug (includes writing to ROM) */
2613 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2614 uint8_t *buf, int len, int is_write)
2615 {
2616 int l;
2617 hwaddr phys_addr;
2618 target_ulong page;
2619
2620 while (len > 0) {
2621 page = addr & TARGET_PAGE_MASK;
2622 phys_addr = cpu_get_phys_page_debug(cpu, page);
2623 /* if no physical page mapped, return an error */
2624 if (phys_addr == -1)
2625 return -1;
2626 l = (page + TARGET_PAGE_SIZE) - addr;
2627 if (l > len)
2628 l = len;
2629 phys_addr += (addr & ~TARGET_PAGE_MASK);
2630 if (is_write)
2631 cpu_physical_memory_write_rom(phys_addr, buf, l);
2632 else
2633 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2634 len -= l;
2635 buf += l;
2636 addr += l;
2637 }
2638 return 0;
2639 }
2640 #endif
2641
2642 #if !defined(CONFIG_USER_ONLY)
2643
2644 /*
2645 * A helper function for the _utterly broken_ virtio device model to find out if
2646 * it's running on a big endian machine. Don't do this at home kids!
2647 */
2648 bool virtio_is_big_endian(void);
2649 bool virtio_is_big_endian(void)
2650 {
2651 #if defined(TARGET_WORDS_BIGENDIAN)
2652 return true;
2653 #else
2654 return false;
2655 #endif
2656 }
2657
2658 #endif
2659
2660 #ifndef CONFIG_USER_ONLY
2661 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2662 {
2663 MemoryRegion*mr;
2664 hwaddr l = 1;
2665
2666 mr = address_space_translate(&address_space_memory,
2667 phys_addr, &phys_addr, &l, false);
2668
2669 return !(memory_region_is_ram(mr) ||
2670 memory_region_is_romd(mr));
2671 }
2672
2673 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2674 {
2675 RAMBlock *block;
2676
2677 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2678 func(block->host, block->offset, block->length, opaque);
2679 }
2680 }
2681 #endif
Qemu copy for testing