]> git.proxmox.com Git - qemu.git/blob - exec.c
projects / qemu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
exec: remove cur_map
[qemu.git] / exec.c
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "hw/xen/xen.h"
35 #include "qemu/timer.h"
36 #include "qemu/config-file.h"
37 #include "exec/memory.h"
38 #include "sysemu/dma.h"
39 #include "exec/address-spaces.h"
40 #if defined(CONFIG_USER_ONLY)
41 #include <qemu.h>
42 #else /* !CONFIG_USER_ONLY */
43 #include "sysemu/xen-mapcache.h"
44 #include "trace.h"
45 #endif
46 #include "exec/cpu-all.h"
47
48 #include "exec/cputlb.h"
49 #include "translate-all.h"
50
51 #include "exec/memory-internal.h"
52
53 //#define DEBUG_SUBPAGE
54
55 #if !defined(CONFIG_USER_ONLY)
56 int phys_ram_fd;
57 static int in_migration;
58
59 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61 static MemoryRegion *system_memory;
62 static MemoryRegion *system_io;
63
64 AddressSpace address_space_io;
65 AddressSpace address_space_memory;
66
67 MemoryRegion io_mem_rom, io_mem_notdirty;
68 static MemoryRegion io_mem_unassigned;
69
70 #endif
71
72 CPUArchState *first_cpu;
73 /* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75 DEFINE_TLS(CPUArchState *,cpu_single_env);
76 /* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79 int use_icount;
80
81 #if !defined(CONFIG_USER_ONLY)
82
83 typedef struct PhysPageEntry PhysPageEntry;
84
85 struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89 };
90
91 typedef PhysPageEntry Node[L2_SIZE];
92
93 struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101 };
102
103 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104 typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109 } subpage_t;
110
111 #define PHYS_SECTION_UNASSIGNED 0
112 #define PHYS_SECTION_NOTDIRTY 1
113 #define PHYS_SECTION_ROM 2
114 #define PHYS_SECTION_WATCH 3
115
116 typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123 } PhysPageMap;
124
125 static PhysPageMap *prev_map;
126 static PhysPageMap next_map;
127
128 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130 static void io_mem_init(void);
131 static void memory_map_init(void);
132 static void *qemu_safe_ram_ptr(ram_addr_t addr);
133
134 static MemoryRegion io_mem_watch;
135 #endif
136
137 #if !defined(CONFIG_USER_ONLY)
138
139 static void phys_map_node_reserve(unsigned nodes)
140 {
141 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
142 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
143 16);
144 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
145 next_map.nodes_nb + nodes);
146 next_map.nodes = g_renew(Node, next_map.nodes,
147 next_map.nodes_nb_alloc);
148 }
149 }
150
151 static uint16_t phys_map_node_alloc(void)
152 {
153 unsigned i;
154 uint16_t ret;
155
156 ret = next_map.nodes_nb++;
157 assert(ret != PHYS_MAP_NODE_NIL);
158 assert(ret != next_map.nodes_nb_alloc);
159 for (i = 0; i < L2_SIZE; ++i) {
160 next_map.nodes[ret][i].is_leaf = 0;
161 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
162 }
163 return ret;
164 }
165
166 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
167 hwaddr *nb, uint16_t leaf,
168 int level)
169 {
170 PhysPageEntry *p;
171 int i;
172 hwaddr step = (hwaddr)1 << (level * L2_BITS);
173
174 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
175 lp->ptr = phys_map_node_alloc();
176 p = next_map.nodes[lp->ptr];
177 if (level == 0) {
178 for (i = 0; i < L2_SIZE; i++) {
179 p[i].is_leaf = 1;
180 p[i].ptr = PHYS_SECTION_UNASSIGNED;
181 }
182 }
183 } else {
184 p = next_map.nodes[lp->ptr];
185 }
186 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
187
188 while (*nb && lp < &p[L2_SIZE]) {
189 if ((*index & (step - 1)) == 0 && *nb >= step) {
190 lp->is_leaf = true;
191 lp->ptr = leaf;
192 *index += step;
193 *nb -= step;
194 } else {
195 phys_page_set_level(lp, index, nb, leaf, level - 1);
196 }
197 ++lp;
198 }
199 }
200
201 static void phys_page_set(AddressSpaceDispatch *d,
202 hwaddr index, hwaddr nb,
203 uint16_t leaf)
204 {
205 /* Wildly overreserve - it doesn't matter much. */
206 phys_map_node_reserve(3 * P_L2_LEVELS);
207
208 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
209 }
210
211 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
212 Node *nodes, MemoryRegionSection *sections)
213 {
214 PhysPageEntry *p;
215 int i;
216
217 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
218 if (lp.ptr == PHYS_MAP_NODE_NIL) {
219 return &sections[PHYS_SECTION_UNASSIGNED];
220 }
221 p = nodes[lp.ptr];
222 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
223 }
224 return &sections[lp.ptr];
225 }
226
227 bool memory_region_is_unassigned(MemoryRegion *mr)
228 {
229 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
230 && mr != &io_mem_watch;
231 }
232
233 static MemoryRegionSection *address_space_lookup_region(AddressSpace *as,
234 hwaddr addr,
235 bool resolve_subpage)
236 {
237 AddressSpaceDispatch *d = as->dispatch;
238 MemoryRegionSection *section;
239 subpage_t *subpage;
240
241 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
242 d->nodes, d->sections);
243 if (resolve_subpage && section->mr->subpage) {
244 subpage = container_of(section->mr, subpage_t, iomem);
245 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
246 }
247 return section;
248 }
249
250 static MemoryRegionSection *
251 address_space_translate_internal(AddressSpace *as, hwaddr addr, hwaddr *xlat,
252 hwaddr *plen, bool resolve_subpage)
253 {
254 MemoryRegionSection *section;
255 Int128 diff;
256
257 section = address_space_lookup_region(as, addr, resolve_subpage);
258 /* Compute offset within MemoryRegionSection */
259 addr -= section->offset_within_address_space;
260
261 /* Compute offset within MemoryRegion */
262 *xlat = addr + section->offset_within_region;
263
264 diff = int128_sub(section->mr->size, int128_make64(addr));
265 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
266 return section;
267 }
268
269 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
270 hwaddr *xlat, hwaddr *plen,
271 bool is_write)
272 {
273 IOMMUTLBEntry iotlb;
274 MemoryRegionSection *section;
275 MemoryRegion *mr;
276 hwaddr len = *plen;
277
278 for (;;) {
279 section = address_space_translate_internal(as, addr, &addr, plen, true);
280 mr = section->mr;
281
282 if (!mr->iommu_ops) {
283 break;
284 }
285
286 iotlb = mr->iommu_ops->translate(mr, addr);
287 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
288 | (addr & iotlb.addr_mask));
289 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
290 if (!(iotlb.perm & (1 << is_write))) {
291 mr = &io_mem_unassigned;
292 break;
293 }
294
295 as = iotlb.target_as;
296 }
297
298 *plen = len;
299 *xlat = addr;
300 return mr;
301 }
302
303 MemoryRegionSection *
304 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
305 hwaddr *plen)
306 {
307 MemoryRegionSection *section;
308 section = address_space_translate_internal(as, addr, xlat, plen, false);
309
310 assert(!section->mr->iommu_ops);
311 return section;
312 }
313 #endif
314
315 void cpu_exec_init_all(void)
316 {
317 #if !defined(CONFIG_USER_ONLY)
318 qemu_mutex_init(&ram_list.mutex);
319 memory_map_init();
320 io_mem_init();
321 #endif
322 }
323
324 #if !defined(CONFIG_USER_ONLY)
325
326 static int cpu_common_post_load(void *opaque, int version_id)
327 {
328 CPUState *cpu = opaque;
329
330 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
331 version_id is increased. */
332 cpu->interrupt_request &= ~0x01;
333 tlb_flush(cpu->env_ptr, 1);
334
335 return 0;
336 }
337
338 const VMStateDescription vmstate_cpu_common = {
339 .name = "cpu_common",
340 .version_id = 1,
341 .minimum_version_id = 1,
342 .minimum_version_id_old = 1,
343 .post_load = cpu_common_post_load,
344 .fields = (VMStateField []) {
345 VMSTATE_UINT32(halted, CPUState),
346 VMSTATE_UINT32(interrupt_request, CPUState),
347 VMSTATE_END_OF_LIST()
348 }
349 };
350
351 #endif
352
353 CPUState *qemu_get_cpu(int index)
354 {
355 CPUArchState *env = first_cpu;
356 CPUState *cpu = NULL;
357
358 while (env) {
359 cpu = ENV_GET_CPU(env);
360 if (cpu->cpu_index == index) {
361 break;
362 }
363 env = env->next_cpu;
364 }
365
366 return env ? cpu : NULL;
367 }
368
369 void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
370 {
371 CPUArchState *env = first_cpu;
372
373 while (env) {
374 func(ENV_GET_CPU(env), data);
375 env = env->next_cpu;
376 }
377 }
378
379 void cpu_exec_init(CPUArchState *env)
380 {
381 CPUState *cpu = ENV_GET_CPU(env);
382 CPUClass *cc = CPU_GET_CLASS(cpu);
383 CPUArchState **penv;
384 int cpu_index;
385
386 #if defined(CONFIG_USER_ONLY)
387 cpu_list_lock();
388 #endif
389 env->next_cpu = NULL;
390 penv = &first_cpu;
391 cpu_index = 0;
392 while (*penv != NULL) {
393 penv = &(*penv)->next_cpu;
394 cpu_index++;
395 }
396 cpu->cpu_index = cpu_index;
397 cpu->numa_node = 0;
398 QTAILQ_INIT(&env->breakpoints);
399 QTAILQ_INIT(&env->watchpoints);
400 #ifndef CONFIG_USER_ONLY
401 cpu->thread_id = qemu_get_thread_id();
402 #endif
403 *penv = env;
404 #if defined(CONFIG_USER_ONLY)
405 cpu_list_unlock();
406 #endif
407 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
408 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
409 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
410 cpu_save, cpu_load, env);
411 assert(cc->vmsd == NULL);
412 #endif
413 if (cc->vmsd != NULL) {
414 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
415 }
416 }
417
418 #if defined(TARGET_HAS_ICE)
419 #if defined(CONFIG_USER_ONLY)
420 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
421 {
422 tb_invalidate_phys_page_range(pc, pc + 1, 0);
423 }
424 #else
425 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
426 {
427 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
428 (pc & ~TARGET_PAGE_MASK));
429 }
430 #endif
431 #endif /* TARGET_HAS_ICE */
432
433 #if defined(CONFIG_USER_ONLY)
434 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
435
436 {
437 }
438
439 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
440 int flags, CPUWatchpoint **watchpoint)
441 {
442 return -ENOSYS;
443 }
444 #else
445 /* Add a watchpoint. */
446 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
447 int flags, CPUWatchpoint **watchpoint)
448 {
449 target_ulong len_mask = ~(len - 1);
450 CPUWatchpoint *wp;
451
452 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
453 if ((len & (len - 1)) || (addr & ~len_mask) ||
454 len == 0 || len > TARGET_PAGE_SIZE) {
455 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
456 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
457 return -EINVAL;
458 }
459 wp = g_malloc(sizeof(*wp));
460
461 wp->vaddr = addr;
462 wp->len_mask = len_mask;
463 wp->flags = flags;
464
465 /* keep all GDB-injected watchpoints in front */
466 if (flags & BP_GDB)
467 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
468 else
469 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
470
471 tlb_flush_page(env, addr);
472
473 if (watchpoint)
474 *watchpoint = wp;
475 return 0;
476 }
477
478 /* Remove a specific watchpoint. */
479 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
480 int flags)
481 {
482 target_ulong len_mask = ~(len - 1);
483 CPUWatchpoint *wp;
484
485 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
486 if (addr == wp->vaddr && len_mask == wp->len_mask
487 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
488 cpu_watchpoint_remove_by_ref(env, wp);
489 return 0;
490 }
491 }
492 return -ENOENT;
493 }
494
495 /* Remove a specific watchpoint by reference. */
496 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
497 {
498 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
499
500 tlb_flush_page(env, watchpoint->vaddr);
501
502 g_free(watchpoint);
503 }
504
505 /* Remove all matching watchpoints. */
506 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
507 {
508 CPUWatchpoint *wp, *next;
509
510 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
511 if (wp->flags & mask)
512 cpu_watchpoint_remove_by_ref(env, wp);
513 }
514 }
515 #endif
516
517 /* Add a breakpoint. */
518 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
519 CPUBreakpoint **breakpoint)
520 {
521 #if defined(TARGET_HAS_ICE)
522 CPUBreakpoint *bp;
523
524 bp = g_malloc(sizeof(*bp));
525
526 bp->pc = pc;
527 bp->flags = flags;
528
529 /* keep all GDB-injected breakpoints in front */
530 if (flags & BP_GDB)
531 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
532 else
533 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
534
535 breakpoint_invalidate(env, pc);
536
537 if (breakpoint)
538 *breakpoint = bp;
539 return 0;
540 #else
541 return -ENOSYS;
542 #endif
543 }
544
545 /* Remove a specific breakpoint. */
546 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
547 {
548 #if defined(TARGET_HAS_ICE)
549 CPUBreakpoint *bp;
550
551 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
552 if (bp->pc == pc && bp->flags == flags) {
553 cpu_breakpoint_remove_by_ref(env, bp);
554 return 0;
555 }
556 }
557 return -ENOENT;
558 #else
559 return -ENOSYS;
560 #endif
561 }
562
563 /* Remove a specific breakpoint by reference. */
564 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
565 {
566 #if defined(TARGET_HAS_ICE)
567 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
568
569 breakpoint_invalidate(env, breakpoint->pc);
570
571 g_free(breakpoint);
572 #endif
573 }
574
575 /* Remove all matching breakpoints. */
576 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
577 {
578 #if defined(TARGET_HAS_ICE)
579 CPUBreakpoint *bp, *next;
580
581 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
582 if (bp->flags & mask)
583 cpu_breakpoint_remove_by_ref(env, bp);
584 }
585 #endif
586 }
587
588 /* enable or disable single step mode. EXCP_DEBUG is returned by the
589 CPU loop after each instruction */
590 void cpu_single_step(CPUArchState *env, int enabled)
591 {
592 #if defined(TARGET_HAS_ICE)
593 if (env->singlestep_enabled != enabled) {
594 env->singlestep_enabled = enabled;
595 if (kvm_enabled())
596 kvm_update_guest_debug(env, 0);
597 else {
598 /* must flush all the translated code to avoid inconsistencies */
599 /* XXX: only flush what is necessary */
600 tb_flush(env);
601 }
602 }
603 #endif
604 }
605
606 void cpu_abort(CPUArchState *env, const char *fmt, ...)
607 {
608 CPUState *cpu = ENV_GET_CPU(env);
609 va_list ap;
610 va_list ap2;
611
612 va_start(ap, fmt);
613 va_copy(ap2, ap);
614 fprintf(stderr, "qemu: fatal: ");
615 vfprintf(stderr, fmt, ap);
616 fprintf(stderr, "\n");
617 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
618 if (qemu_log_enabled()) {
619 qemu_log("qemu: fatal: ");
620 qemu_log_vprintf(fmt, ap2);
621 qemu_log("\n");
622 log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
623 qemu_log_flush();
624 qemu_log_close();
625 }
626 va_end(ap2);
627 va_end(ap);
628 #if defined(CONFIG_USER_ONLY)
629 {
630 struct sigaction act;
631 sigfillset(&act.sa_mask);
632 act.sa_handler = SIG_DFL;
633 sigaction(SIGABRT, &act, NULL);
634 }
635 #endif
636 abort();
637 }
638
639 CPUArchState *cpu_copy(CPUArchState *env)
640 {
641 CPUArchState *new_env = cpu_init(env->cpu_model_str);
642 CPUArchState *next_cpu = new_env->next_cpu;
643 #if defined(TARGET_HAS_ICE)
644 CPUBreakpoint *bp;
645 CPUWatchpoint *wp;
646 #endif
647
648 memcpy(new_env, env, sizeof(CPUArchState));
649
650 /* Preserve chaining. */
651 new_env->next_cpu = next_cpu;
652
653 /* Clone all break/watchpoints.
654 Note: Once we support ptrace with hw-debug register access, make sure
655 BP_CPU break/watchpoints are handled correctly on clone. */
656 QTAILQ_INIT(&env->breakpoints);
657 QTAILQ_INIT(&env->watchpoints);
658 #if defined(TARGET_HAS_ICE)
659 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
660 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
661 }
662 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
663 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
664 wp->flags, NULL);
665 }
666 #endif
667
668 return new_env;
669 }
670
671 #if !defined(CONFIG_USER_ONLY)
672 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
673 uintptr_t length)
674 {
675 uintptr_t start1;
676
677 /* we modify the TLB cache so that the dirty bit will be set again
678 when accessing the range */
679 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
680 /* Check that we don't span multiple blocks - this breaks the
681 address comparisons below. */
682 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
683 != (end - 1) - start) {
684 abort();
685 }
686 cpu_tlb_reset_dirty_all(start1, length);
687
688 }
689
690 /* Note: start and end must be within the same ram block. */
691 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
692 int dirty_flags)
693 {
694 uintptr_t length;
695
696 start &= TARGET_PAGE_MASK;
697 end = TARGET_PAGE_ALIGN(end);
698
699 length = end - start;
700 if (length == 0)
701 return;
702 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
703
704 if (tcg_enabled()) {
705 tlb_reset_dirty_range_all(start, end, length);
706 }
707 }
708
709 static int cpu_physical_memory_set_dirty_tracking(int enable)
710 {
711 int ret = 0;
712 in_migration = enable;
713 return ret;
714 }
715
716 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
717 MemoryRegionSection *section,
718 target_ulong vaddr,
719 hwaddr paddr, hwaddr xlat,
720 int prot,
721 target_ulong *address)
722 {
723 hwaddr iotlb;
724 CPUWatchpoint *wp;
725
726 if (memory_region_is_ram(section->mr)) {
727 /* Normal RAM. */
728 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
729 + xlat;
730 if (!section->readonly) {
731 iotlb |= PHYS_SECTION_NOTDIRTY;
732 } else {
733 iotlb |= PHYS_SECTION_ROM;
734 }
735 } else {
736 iotlb = section - address_space_memory.dispatch->sections;
737 iotlb += xlat;
738 }
739
740 /* Make accesses to pages with watchpoints go via the
741 watchpoint trap routines. */
742 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
743 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
744 /* Avoid trapping reads of pages with a write breakpoint. */
745 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
746 iotlb = PHYS_SECTION_WATCH + paddr;
747 *address |= TLB_MMIO;
748 break;
749 }
750 }
751 }
752
753 return iotlb;
754 }
755 #endif /* defined(CONFIG_USER_ONLY) */
756
757 #if !defined(CONFIG_USER_ONLY)
758
759 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
760 uint16_t section);
761 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
762
763 static uint16_t phys_section_add(MemoryRegionSection *section)
764 {
765 /* The physical section number is ORed with a page-aligned
766 * pointer to produce the iotlb entries. Thus it should
767 * never overflow into the page-aligned value.
768 */
769 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
770
771 if (next_map.sections_nb == next_map.sections_nb_alloc) {
772 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
773 16);
774 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
775 next_map.sections_nb_alloc);
776 }
777 next_map.sections[next_map.sections_nb] = *section;
778 memory_region_ref(section->mr);
779 return next_map.sections_nb++;
780 }
781
782 static void phys_section_destroy(MemoryRegion *mr)
783 {
784 memory_region_unref(mr);
785
786 if (mr->subpage) {
787 subpage_t *subpage = container_of(mr, subpage_t, iomem);
788 memory_region_destroy(&subpage->iomem);
789 g_free(subpage);
790 }
791 }
792
793 static void phys_sections_free(PhysPageMap *map)
794 {
795 while (map->sections_nb > 0) {
796 MemoryRegionSection *section = &map->sections[--map->sections_nb];
797 phys_section_destroy(section->mr);
798 }
799 g_free(map->sections);
800 g_free(map->nodes);
801 g_free(map);
802 }
803
804 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
805 {
806 subpage_t *subpage;
807 hwaddr base = section->offset_within_address_space
808 & TARGET_PAGE_MASK;
809 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
810 next_map.nodes, next_map.sections);
811 MemoryRegionSection subsection = {
812 .offset_within_address_space = base,
813 .size = int128_make64(TARGET_PAGE_SIZE),
814 };
815 hwaddr start, end;
816
817 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
818
819 if (!(existing->mr->subpage)) {
820 subpage = subpage_init(d->as, base);
821 subsection.mr = &subpage->iomem;
822 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
823 phys_section_add(&subsection));
824 } else {
825 subpage = container_of(existing->mr, subpage_t, iomem);
826 }
827 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
828 end = start + int128_get64(section->size) - 1;
829 subpage_register(subpage, start, end, phys_section_add(section));
830 }
831
832
833 static void register_multipage(AddressSpaceDispatch *d,
834 MemoryRegionSection *section)
835 {
836 hwaddr start_addr = section->offset_within_address_space;
837 uint16_t section_index = phys_section_add(section);
838 uint64_t num_pages = int128_get64(int128_rshift(section->size,
839 TARGET_PAGE_BITS));
840
841 assert(num_pages);
842 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
843 }
844
845 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
846 {
847 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
848 AddressSpaceDispatch *d = as->next_dispatch;
849 MemoryRegionSection now = *section, remain = *section;
850 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
851
852 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
853 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
854 - now.offset_within_address_space;
855
856 now.size = int128_min(int128_make64(left), now.size);
857 register_subpage(d, &now);
858 } else {
859 now.size = int128_zero();
860 }
861 while (int128_ne(remain.size, now.size)) {
862 remain.size = int128_sub(remain.size, now.size);
863 remain.offset_within_address_space += int128_get64(now.size);
864 remain.offset_within_region += int128_get64(now.size);
865 now = remain;
866 if (int128_lt(remain.size, page_size)) {
867 register_subpage(d, &now);
868 } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
869 now.size = page_size;
870 register_subpage(d, &now);
871 } else {
872 now.size = int128_and(now.size, int128_neg(page_size));
873 register_multipage(d, &now);
874 }
875 }
876 }
877
878 void qemu_flush_coalesced_mmio_buffer(void)
879 {
880 if (kvm_enabled())
881 kvm_flush_coalesced_mmio_buffer();
882 }
883
884 void qemu_mutex_lock_ramlist(void)
885 {
886 qemu_mutex_lock(&ram_list.mutex);
887 }
888
889 void qemu_mutex_unlock_ramlist(void)
890 {
891 qemu_mutex_unlock(&ram_list.mutex);
892 }
893
894 #if defined(__linux__) && !defined(TARGET_S390X)
895
896 #include <sys/vfs.h>
897
898 #define HUGETLBFS_MAGIC 0x958458f6
899
900 static long gethugepagesize(const char *path)
901 {
902 struct statfs fs;
903 int ret;
904
905 do {
906 ret = statfs(path, &fs);
907 } while (ret != 0 && errno == EINTR);
908
909 if (ret != 0) {
910 perror(path);
911 return 0;
912 }
913
914 if (fs.f_type != HUGETLBFS_MAGIC)
915 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
916
917 return fs.f_bsize;
918 }
919
920 static void *file_ram_alloc(RAMBlock *block,
921 ram_addr_t memory,
922 const char *path)
923 {
924 char *filename;
925 char *sanitized_name;
926 char *c;
927 void *area;
928 int fd;
929 #ifdef MAP_POPULATE
930 int flags;
931 #endif
932 unsigned long hpagesize;
933
934 hpagesize = gethugepagesize(path);
935 if (!hpagesize) {
936 return NULL;
937 }
938
939 if (memory < hpagesize) {
940 return NULL;
941 }
942
943 if (kvm_enabled() && !kvm_has_sync_mmu()) {
944 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
945 return NULL;
946 }
947
948 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
949 sanitized_name = g_strdup(block->mr->name);
950 for (c = sanitized_name; *c != '\0'; c++) {
951 if (*c == '/')
952 *c = '_';
953 }
954
955 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
956 sanitized_name);
957 g_free(sanitized_name);
958
959 fd = mkstemp(filename);
960 if (fd < 0) {
961 perror("unable to create backing store for hugepages");
962 g_free(filename);
963 return NULL;
964 }
965 unlink(filename);
966 g_free(filename);
967
968 memory = (memory+hpagesize-1) & ~(hpagesize-1);
969
970 /*
971 * ftruncate is not supported by hugetlbfs in older
972 * hosts, so don't bother bailing out on errors.
973 * If anything goes wrong with it under other filesystems,
974 * mmap will fail.
975 */
976 if (ftruncate(fd, memory))
977 perror("ftruncate");
978
979 #ifdef MAP_POPULATE
980 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
981 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
982 * to sidestep this quirk.
983 */
984 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
985 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
986 #else
987 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
988 #endif
989 if (area == MAP_FAILED) {
990 perror("file_ram_alloc: can't mmap RAM pages");
991 close(fd);
992 return (NULL);
993 }
994 block->fd = fd;
995 return area;
996 }
997 #endif
998
999 static ram_addr_t find_ram_offset(ram_addr_t size)
1000 {
1001 RAMBlock *block, *next_block;
1002 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1003
1004 assert(size != 0); /* it would hand out same offset multiple times */
1005
1006 if (QTAILQ_EMPTY(&ram_list.blocks))
1007 return 0;
1008
1009 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1010 ram_addr_t end, next = RAM_ADDR_MAX;
1011
1012 end = block->offset + block->length;
1013
1014 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1015 if (next_block->offset >= end) {
1016 next = MIN(next, next_block->offset);
1017 }
1018 }
1019 if (next - end >= size && next - end < mingap) {
1020 offset = end;
1021 mingap = next - end;
1022 }
1023 }
1024
1025 if (offset == RAM_ADDR_MAX) {
1026 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1027 (uint64_t)size);
1028 abort();
1029 }
1030
1031 return offset;
1032 }
1033
1034 ram_addr_t last_ram_offset(void)
1035 {
1036 RAMBlock *block;
1037 ram_addr_t last = 0;
1038
1039 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1040 last = MAX(last, block->offset + block->length);
1041
1042 return last;
1043 }
1044
1045 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1046 {
1047 int ret;
1048 QemuOpts *machine_opts;
1049
1050 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1051 machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1052 if (machine_opts &&
1053 !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
1054 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1055 if (ret) {
1056 perror("qemu_madvise");
1057 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1058 "but dump_guest_core=off specified\n");
1059 }
1060 }
1061 }
1062
1063 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1064 {
1065 RAMBlock *new_block, *block;
1066
1067 new_block = NULL;
1068 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1069 if (block->offset == addr) {
1070 new_block = block;
1071 break;
1072 }
1073 }
1074 assert(new_block);
1075 assert(!new_block->idstr[0]);
1076
1077 if (dev) {
1078 char *id = qdev_get_dev_path(dev);
1079 if (id) {
1080 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1081 g_free(id);
1082 }
1083 }
1084 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1085
1086 /* This assumes the iothread lock is taken here too. */
1087 qemu_mutex_lock_ramlist();
1088 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1089 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1090 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1091 new_block->idstr);
1092 abort();
1093 }
1094 }
1095 qemu_mutex_unlock_ramlist();
1096 }
1097
1098 static int memory_try_enable_merging(void *addr, size_t len)
1099 {
1100 QemuOpts *opts;
1101
1102 opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1103 if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
1104 /* disabled by the user */
1105 return 0;
1106 }
1107
1108 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1109 }
1110
1111 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1112 MemoryRegion *mr)
1113 {
1114 RAMBlock *block, *new_block;
1115
1116 size = TARGET_PAGE_ALIGN(size);
1117 new_block = g_malloc0(sizeof(*new_block));
1118
1119 /* This assumes the iothread lock is taken here too. */
1120 qemu_mutex_lock_ramlist();
1121 new_block->mr = mr;
1122 new_block->offset = find_ram_offset(size);
1123 if (host) {
1124 new_block->host = host;
1125 new_block->flags |= RAM_PREALLOC_MASK;
1126 } else {
1127 if (mem_path) {
1128 #if defined (__linux__) && !defined(TARGET_S390X)
1129 new_block->host = file_ram_alloc(new_block, size, mem_path);
1130 if (!new_block->host) {
1131 new_block->host = qemu_anon_ram_alloc(size);
1132 memory_try_enable_merging(new_block->host, size);
1133 }
1134 #else
1135 fprintf(stderr, "-mem-path option unsupported\n");
1136 exit(1);
1137 #endif
1138 } else {
1139 if (xen_enabled()) {
1140 xen_ram_alloc(new_block->offset, size, mr);
1141 } else if (kvm_enabled()) {
1142 /* some s390/kvm configurations have special constraints */
1143 new_block->host = kvm_ram_alloc(size);
1144 } else {
1145 new_block->host = qemu_anon_ram_alloc(size);
1146 }
1147 memory_try_enable_merging(new_block->host, size);
1148 }
1149 }
1150 new_block->length = size;
1151
1152 /* Keep the list sorted from biggest to smallest block. */
1153 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1154 if (block->length < new_block->length) {
1155 break;
1156 }
1157 }
1158 if (block) {
1159 QTAILQ_INSERT_BEFORE(block, new_block, next);
1160 } else {
1161 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1162 }
1163 ram_list.mru_block = NULL;
1164
1165 ram_list.version++;
1166 qemu_mutex_unlock_ramlist();
1167
1168 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1169 last_ram_offset() >> TARGET_PAGE_BITS);
1170 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1171 0, size >> TARGET_PAGE_BITS);
1172 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1173
1174 qemu_ram_setup_dump(new_block->host, size);
1175 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1176
1177 if (kvm_enabled())
1178 kvm_setup_guest_memory(new_block->host, size);
1179
1180 return new_block->offset;
1181 }
1182
1183 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1184 {
1185 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1186 }
1187
1188 void qemu_ram_free_from_ptr(ram_addr_t addr)
1189 {
1190 RAMBlock *block;
1191
1192 /* This assumes the iothread lock is taken here too. */
1193 qemu_mutex_lock_ramlist();
1194 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1195 if (addr == block->offset) {
1196 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1197 ram_list.mru_block = NULL;
1198 ram_list.version++;
1199 g_free(block);
1200 break;
1201 }
1202 }
1203 qemu_mutex_unlock_ramlist();
1204 }
1205
1206 void qemu_ram_free(ram_addr_t addr)
1207 {
1208 RAMBlock *block;
1209
1210 /* This assumes the iothread lock is taken here too. */
1211 qemu_mutex_lock_ramlist();
1212 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1213 if (addr == block->offset) {
1214 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1215 ram_list.mru_block = NULL;
1216 ram_list.version++;
1217 if (block->flags & RAM_PREALLOC_MASK) {
1218 ;
1219 } else if (mem_path) {
1220 #if defined (__linux__) && !defined(TARGET_S390X)
1221 if (block->fd) {
1222 munmap(block->host, block->length);
1223 close(block->fd);
1224 } else {
1225 qemu_anon_ram_free(block->host, block->length);
1226 }
1227 #else
1228 abort();
1229 #endif
1230 } else {
1231 if (xen_enabled()) {
1232 xen_invalidate_map_cache_entry(block->host);
1233 } else {
1234 qemu_anon_ram_free(block->host, block->length);
1235 }
1236 }
1237 g_free(block);
1238 break;
1239 }
1240 }
1241 qemu_mutex_unlock_ramlist();
1242
1243 }
1244
1245 #ifndef _WIN32
1246 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1247 {
1248 RAMBlock *block;
1249 ram_addr_t offset;
1250 int flags;
1251 void *area, *vaddr;
1252
1253 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1254 offset = addr - block->offset;
1255 if (offset < block->length) {
1256 vaddr = block->host + offset;
1257 if (block->flags & RAM_PREALLOC_MASK) {
1258 ;
1259 } else {
1260 flags = MAP_FIXED;
1261 munmap(vaddr, length);
1262 if (mem_path) {
1263 #if defined(__linux__) && !defined(TARGET_S390X)
1264 if (block->fd) {
1265 #ifdef MAP_POPULATE
1266 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1267 MAP_PRIVATE;
1268 #else
1269 flags |= MAP_PRIVATE;
1270 #endif
1271 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1272 flags, block->fd, offset);
1273 } else {
1274 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1275 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1276 flags, -1, 0);
1277 }
1278 #else
1279 abort();
1280 #endif
1281 } else {
1282 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1283 flags |= MAP_SHARED | MAP_ANONYMOUS;
1284 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1285 flags, -1, 0);
1286 #else
1287 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1288 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1289 flags, -1, 0);
1290 #endif
1291 }
1292 if (area != vaddr) {
1293 fprintf(stderr, "Could not remap addr: "
1294 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1295 length, addr);
1296 exit(1);
1297 }
1298 memory_try_enable_merging(vaddr, length);
1299 qemu_ram_setup_dump(vaddr, length);
1300 }
1301 return;
1302 }
1303 }
1304 }
1305 #endif /* !_WIN32 */
1306
1307 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1308 {
1309 RAMBlock *block;
1310
1311 /* The list is protected by the iothread lock here. */
1312 block = ram_list.mru_block;
1313 if (block && addr - block->offset < block->length) {
1314 goto found;
1315 }
1316 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1317 if (addr - block->offset < block->length) {
1318 goto found;
1319 }
1320 }
1321
1322 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1323 abort();
1324
1325 found:
1326 ram_list.mru_block = block;
1327 return block;
1328 }
1329
1330 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1331 With the exception of the softmmu code in this file, this should
1332 only be used for local memory (e.g. video ram) that the device owns,
1333 and knows it isn't going to access beyond the end of the block.
1334
1335 It should not be used for general purpose DMA.
1336 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1337 */
1338 void *qemu_get_ram_ptr(ram_addr_t addr)
1339 {
1340 RAMBlock *block = qemu_get_ram_block(addr);
1341
1342 if (xen_enabled()) {
1343 /* We need to check if the requested address is in the RAM
1344 * because we don't want to map the entire memory in QEMU.
1345 * In that case just map until the end of the page.
1346 */
1347 if (block->offset == 0) {
1348 return xen_map_cache(addr, 0, 0);
1349 } else if (block->host == NULL) {
1350 block->host =
1351 xen_map_cache(block->offset, block->length, 1);
1352 }
1353 }
1354 return block->host + (addr - block->offset);
1355 }
1356
1357 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1358 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1359 *
1360 * ??? Is this still necessary?
1361 */
1362 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1363 {
1364 RAMBlock *block;
1365
1366 /* The list is protected by the iothread lock here. */
1367 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1368 if (addr - block->offset < block->length) {
1369 if (xen_enabled()) {
1370 /* We need to check if the requested address is in the RAM
1371 * because we don't want to map the entire memory in QEMU.
1372 * In that case just map until the end of the page.
1373 */
1374 if (block->offset == 0) {
1375 return xen_map_cache(addr, 0, 0);
1376 } else if (block->host == NULL) {
1377 block->host =
1378 xen_map_cache(block->offset, block->length, 1);
1379 }
1380 }
1381 return block->host + (addr - block->offset);
1382 }
1383 }
1384
1385 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1386 abort();
1387
1388 return NULL;
1389 }
1390
1391 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1392 * but takes a size argument */
1393 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1394 {
1395 if (*size == 0) {
1396 return NULL;
1397 }
1398 if (xen_enabled()) {
1399 return xen_map_cache(addr, *size, 1);
1400 } else {
1401 RAMBlock *block;
1402
1403 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1404 if (addr - block->offset < block->length) {
1405 if (addr - block->offset + *size > block->length)
1406 *size = block->length - addr + block->offset;
1407 return block->host + (addr - block->offset);
1408 }
1409 }
1410
1411 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1412 abort();
1413 }
1414 }
1415
1416 /* Some of the softmmu routines need to translate from a host pointer
1417 (typically a TLB entry) back to a ram offset. */
1418 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1419 {
1420 RAMBlock *block;
1421 uint8_t *host = ptr;
1422
1423 if (xen_enabled()) {
1424 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1425 return qemu_get_ram_block(*ram_addr)->mr;
1426 }
1427
1428 block = ram_list.mru_block;
1429 if (block && block->host && host - block->host < block->length) {
1430 goto found;
1431 }
1432
1433 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1434 /* This case append when the block is not mapped. */
1435 if (block->host == NULL) {
1436 continue;
1437 }
1438 if (host - block->host < block->length) {
1439 goto found;
1440 }
1441 }
1442
1443 return NULL;
1444
1445 found:
1446 *ram_addr = block->offset + (host - block->host);
1447 return block->mr;
1448 }
1449
1450 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1451 uint64_t val, unsigned size)
1452 {
1453 int dirty_flags;
1454 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1455 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1456 tb_invalidate_phys_page_fast(ram_addr, size);
1457 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1458 }
1459 switch (size) {
1460 case 1:
1461 stb_p(qemu_get_ram_ptr(ram_addr), val);
1462 break;
1463 case 2:
1464 stw_p(qemu_get_ram_ptr(ram_addr), val);
1465 break;
1466 case 4:
1467 stl_p(qemu_get_ram_ptr(ram_addr), val);
1468 break;
1469 default:
1470 abort();
1471 }
1472 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1473 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1474 /* we remove the notdirty callback only if the code has been
1475 flushed */
1476 if (dirty_flags == 0xff)
1477 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1478 }
1479
1480 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1481 unsigned size, bool is_write)
1482 {
1483 return is_write;
1484 }
1485
1486 static const MemoryRegionOps notdirty_mem_ops = {
1487 .write = notdirty_mem_write,
1488 .valid.accepts = notdirty_mem_accepts,
1489 .endianness = DEVICE_NATIVE_ENDIAN,
1490 };
1491
1492 /* Generate a debug exception if a watchpoint has been hit. */
1493 static void check_watchpoint(int offset, int len_mask, int flags)
1494 {
1495 CPUArchState *env = cpu_single_env;
1496 target_ulong pc, cs_base;
1497 target_ulong vaddr;
1498 CPUWatchpoint *wp;
1499 int cpu_flags;
1500
1501 if (env->watchpoint_hit) {
1502 /* We re-entered the check after replacing the TB. Now raise
1503 * the debug interrupt so that is will trigger after the
1504 * current instruction. */
1505 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1506 return;
1507 }
1508 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1509 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1510 if ((vaddr == (wp->vaddr & len_mask) ||
1511 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1512 wp->flags |= BP_WATCHPOINT_HIT;
1513 if (!env->watchpoint_hit) {
1514 env->watchpoint_hit = wp;
1515 tb_check_watchpoint(env);
1516 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1517 env->exception_index = EXCP_DEBUG;
1518 cpu_loop_exit(env);
1519 } else {
1520 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1521 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1522 cpu_resume_from_signal(env, NULL);
1523 }
1524 }
1525 } else {
1526 wp->flags &= ~BP_WATCHPOINT_HIT;
1527 }
1528 }
1529 }
1530
1531 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1532 so these check for a hit then pass through to the normal out-of-line
1533 phys routines. */
1534 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1535 unsigned size)
1536 {
1537 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1538 switch (size) {
1539 case 1: return ldub_phys(addr);
1540 case 2: return lduw_phys(addr);
1541 case 4: return ldl_phys(addr);
1542 default: abort();
1543 }
1544 }
1545
1546 static void watch_mem_write(void *opaque, hwaddr addr,
1547 uint64_t val, unsigned size)
1548 {
1549 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1550 switch (size) {
1551 case 1:
1552 stb_phys(addr, val);
1553 break;
1554 case 2:
1555 stw_phys(addr, val);
1556 break;
1557 case 4:
1558 stl_phys(addr, val);
1559 break;
1560 default: abort();
1561 }
1562 }
1563
1564 static const MemoryRegionOps watch_mem_ops = {
1565 .read = watch_mem_read,
1566 .write = watch_mem_write,
1567 .endianness = DEVICE_NATIVE_ENDIAN,
1568 };
1569
1570 static uint64_t subpage_read(void *opaque, hwaddr addr,
1571 unsigned len)
1572 {
1573 subpage_t *subpage = opaque;
1574 uint8_t buf[4];
1575
1576 #if defined(DEBUG_SUBPAGE)
1577 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1578 subpage, len, addr);
1579 #endif
1580 address_space_read(subpage->as, addr + subpage->base, buf, len);
1581 switch (len) {
1582 case 1:
1583 return ldub_p(buf);
1584 case 2:
1585 return lduw_p(buf);
1586 case 4:
1587 return ldl_p(buf);
1588 default:
1589 abort();
1590 }
1591 }
1592
1593 static void subpage_write(void *opaque, hwaddr addr,
1594 uint64_t value, unsigned len)
1595 {
1596 subpage_t *subpage = opaque;
1597 uint8_t buf[4];
1598
1599 #if defined(DEBUG_SUBPAGE)
1600 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1601 " value %"PRIx64"\n",
1602 __func__, subpage, len, addr, value);
1603 #endif
1604 switch (len) {
1605 case 1:
1606 stb_p(buf, value);
1607 break;
1608 case 2:
1609 stw_p(buf, value);
1610 break;
1611 case 4:
1612 stl_p(buf, value);
1613 break;
1614 default:
1615 abort();
1616 }
1617 address_space_write(subpage->as, addr + subpage->base, buf, len);
1618 }
1619
1620 static bool subpage_accepts(void *opaque, hwaddr addr,
1621 unsigned size, bool is_write)
1622 {
1623 subpage_t *subpage = opaque;
1624 #if defined(DEBUG_SUBPAGE)
1625 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1626 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1627 #endif
1628
1629 return address_space_access_valid(subpage->as, addr + subpage->base,
1630 size, is_write);
1631 }
1632
1633 static const MemoryRegionOps subpage_ops = {
1634 .read = subpage_read,
1635 .write = subpage_write,
1636 .valid.accepts = subpage_accepts,
1637 .endianness = DEVICE_NATIVE_ENDIAN,
1638 };
1639
1640 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1641 uint16_t section)
1642 {
1643 int idx, eidx;
1644
1645 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1646 return -1;
1647 idx = SUBPAGE_IDX(start);
1648 eidx = SUBPAGE_IDX(end);
1649 #if defined(DEBUG_SUBPAGE)
1650 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1651 mmio, start, end, idx, eidx, memory);
1652 #endif
1653 for (; idx <= eidx; idx++) {
1654 mmio->sub_section[idx] = section;
1655 }
1656
1657 return 0;
1658 }
1659
1660 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1661 {
1662 subpage_t *mmio;
1663
1664 mmio = g_malloc0(sizeof(subpage_t));
1665
1666 mmio->as = as;
1667 mmio->base = base;
1668 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1669 "subpage", TARGET_PAGE_SIZE);
1670 mmio->iomem.subpage = true;
1671 #if defined(DEBUG_SUBPAGE)
1672 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1673 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1674 #endif
1675 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1676
1677 return mmio;
1678 }
1679
1680 static uint16_t dummy_section(MemoryRegion *mr)
1681 {
1682 MemoryRegionSection section = {
1683 .mr = mr,
1684 .offset_within_address_space = 0,
1685 .offset_within_region = 0,
1686 .size = int128_2_64(),
1687 };
1688
1689 return phys_section_add(&section);
1690 }
1691
1692 MemoryRegion *iotlb_to_region(hwaddr index)
1693 {
1694 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1695 }
1696
1697 static void io_mem_init(void)
1698 {
1699 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1700 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1701 "unassigned", UINT64_MAX);
1702 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1703 "notdirty", UINT64_MAX);
1704 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1705 "watch", UINT64_MAX);
1706 }
1707
1708 static void mem_begin(MemoryListener *listener)
1709 {
1710 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1711 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1712
1713 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1714 d->as = as;
1715 as->next_dispatch = d;
1716 }
1717
1718 static void mem_commit(MemoryListener *listener)
1719 {
1720 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1721 AddressSpaceDispatch *cur = as->dispatch;
1722 AddressSpaceDispatch *next = as->next_dispatch;
1723
1724 next->nodes = next_map.nodes;
1725 next->sections = next_map.sections;
1726
1727 as->dispatch = next;
1728 g_free(cur);
1729 }
1730
1731 static void core_begin(MemoryListener *listener)
1732 {
1733 uint16_t n;
1734
1735 prev_map = g_new(PhysPageMap, 1);
1736 *prev_map = next_map;
1737
1738 memset(&next_map, 0, sizeof(next_map));
1739 n = dummy_section(&io_mem_unassigned);
1740 assert(n == PHYS_SECTION_UNASSIGNED);
1741 n = dummy_section(&io_mem_notdirty);
1742 assert(n == PHYS_SECTION_NOTDIRTY);
1743 n = dummy_section(&io_mem_rom);
1744 assert(n == PHYS_SECTION_ROM);
1745 n = dummy_section(&io_mem_watch);
1746 assert(n == PHYS_SECTION_WATCH);
1747 }
1748
1749 /* This listener's commit run after the other AddressSpaceDispatch listeners'.
1750 * All AddressSpaceDispatch instances have switched to the next map.
1751 */
1752 static void core_commit(MemoryListener *listener)
1753 {
1754 phys_sections_free(prev_map);
1755 }
1756
1757 static void tcg_commit(MemoryListener *listener)
1758 {
1759 CPUArchState *env;
1760
1761 /* since each CPU stores ram addresses in its TLB cache, we must
1762 reset the modified entries */
1763 /* XXX: slow ! */
1764 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1765 tlb_flush(env, 1);
1766 }
1767 }
1768
1769 static void core_log_global_start(MemoryListener *listener)
1770 {
1771 cpu_physical_memory_set_dirty_tracking(1);
1772 }
1773
1774 static void core_log_global_stop(MemoryListener *listener)
1775 {
1776 cpu_physical_memory_set_dirty_tracking(0);
1777 }
1778
1779 static MemoryListener core_memory_listener = {
1780 .begin = core_begin,
1781 .commit = core_commit,
1782 .log_global_start = core_log_global_start,
1783 .log_global_stop = core_log_global_stop,
1784 .priority = 1,
1785 };
1786
1787 static MemoryListener tcg_memory_listener = {
1788 .commit = tcg_commit,
1789 };
1790
1791 void address_space_init_dispatch(AddressSpace *as)
1792 {
1793 as->dispatch = NULL;
1794 as->dispatch_listener = (MemoryListener) {
1795 .begin = mem_begin,
1796 .commit = mem_commit,
1797 .region_add = mem_add,
1798 .region_nop = mem_add,
1799 .priority = 0,
1800 };
1801 memory_listener_register(&as->dispatch_listener, as);
1802 }
1803
1804 void address_space_destroy_dispatch(AddressSpace *as)
1805 {
1806 AddressSpaceDispatch *d = as->dispatch;
1807
1808 memory_listener_unregister(&as->dispatch_listener);
1809 g_free(d);
1810 as->dispatch = NULL;
1811 }
1812
1813 static void memory_map_init(void)
1814 {
1815 system_memory = g_malloc(sizeof(*system_memory));
1816 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1817 address_space_init(&address_space_memory, system_memory, "memory");
1818
1819 system_io = g_malloc(sizeof(*system_io));
1820 memory_region_init(system_io, NULL, "io", 65536);
1821 address_space_init(&address_space_io, system_io, "I/O");
1822
1823 memory_listener_register(&core_memory_listener, &address_space_memory);
1824 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1825 }
1826
1827 MemoryRegion *get_system_memory(void)
1828 {
1829 return system_memory;
1830 }
1831
1832 MemoryRegion *get_system_io(void)
1833 {
1834 return system_io;
1835 }
1836
1837 #endif /* !defined(CONFIG_USER_ONLY) */
1838
1839 /* physical memory access (slow version, mainly for debug) */
1840 #if defined(CONFIG_USER_ONLY)
1841 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1842 uint8_t *buf, int len, int is_write)
1843 {
1844 int l, flags;
1845 target_ulong page;
1846 void * p;
1847
1848 while (len > 0) {
1849 page = addr & TARGET_PAGE_MASK;
1850 l = (page + TARGET_PAGE_SIZE) - addr;
1851 if (l > len)
1852 l = len;
1853 flags = page_get_flags(page);
1854 if (!(flags & PAGE_VALID))
1855 return -1;
1856 if (is_write) {
1857 if (!(flags & PAGE_WRITE))
1858 return -1;
1859 /* XXX: this code should not depend on lock_user */
1860 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1861 return -1;
1862 memcpy(p, buf, l);
1863 unlock_user(p, addr, l);
1864 } else {
1865 if (!(flags & PAGE_READ))
1866 return -1;
1867 /* XXX: this code should not depend on lock_user */
1868 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1869 return -1;
1870 memcpy(buf, p, l);
1871 unlock_user(p, addr, 0);
1872 }
1873 len -= l;
1874 buf += l;
1875 addr += l;
1876 }
1877 return 0;
1878 }
1879
1880 #else
1881
1882 static void invalidate_and_set_dirty(hwaddr addr,
1883 hwaddr length)
1884 {
1885 if (!cpu_physical_memory_is_dirty(addr)) {
1886 /* invalidate code */
1887 tb_invalidate_phys_page_range(addr, addr + length, 0);
1888 /* set dirty bit */
1889 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1890 }
1891 xen_modified_memory(addr, length);
1892 }
1893
1894 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1895 {
1896 if (memory_region_is_ram(mr)) {
1897 return !(is_write && mr->readonly);
1898 }
1899 if (memory_region_is_romd(mr)) {
1900 return !is_write;
1901 }
1902
1903 return false;
1904 }
1905
1906 static inline int memory_access_size(MemoryRegion *mr, int l, hwaddr addr)
1907 {
1908 if (l >= 4 && (((addr & 3) == 0 || mr->ops->impl.unaligned))) {
1909 return 4;
1910 }
1911 if (l >= 2 && (((addr & 1) == 0) || mr->ops->impl.unaligned)) {
1912 return 2;
1913 }
1914 return 1;
1915 }
1916
1917 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1918 int len, bool is_write)
1919 {
1920 hwaddr l;
1921 uint8_t *ptr;
1922 uint64_t val;
1923 hwaddr addr1;
1924 MemoryRegion *mr;
1925 bool error = false;
1926
1927 while (len > 0) {
1928 l = len;
1929 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1930
1931 if (is_write) {
1932 if (!memory_access_is_direct(mr, is_write)) {
1933 l = memory_access_size(mr, l, addr1);
1934 /* XXX: could force cpu_single_env to NULL to avoid
1935 potential bugs */
1936 if (l == 4) {
1937 /* 32 bit write access */
1938 val = ldl_p(buf);
1939 error |= io_mem_write(mr, addr1, val, 4);
1940 } else if (l == 2) {
1941 /* 16 bit write access */
1942 val = lduw_p(buf);
1943 error |= io_mem_write(mr, addr1, val, 2);
1944 } else {
1945 /* 8 bit write access */
1946 val = ldub_p(buf);
1947 error |= io_mem_write(mr, addr1, val, 1);
1948 }
1949 } else {
1950 addr1 += memory_region_get_ram_addr(mr);
1951 /* RAM case */
1952 ptr = qemu_get_ram_ptr(addr1);
1953 memcpy(ptr, buf, l);
1954 invalidate_and_set_dirty(addr1, l);
1955 }
1956 } else {
1957 if (!memory_access_is_direct(mr, is_write)) {
1958 /* I/O case */
1959 l = memory_access_size(mr, l, addr1);
1960 if (l == 4) {
1961 /* 32 bit read access */
1962 error |= io_mem_read(mr, addr1, &val, 4);
1963 stl_p(buf, val);
1964 } else if (l == 2) {
1965 /* 16 bit read access */
1966 error |= io_mem_read(mr, addr1, &val, 2);
1967 stw_p(buf, val);
1968 } else {
1969 /* 8 bit read access */
1970 error |= io_mem_read(mr, addr1, &val, 1);
1971 stb_p(buf, val);
1972 }
1973 } else {
1974 /* RAM case */
1975 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
1976 memcpy(buf, ptr, l);
1977 }
1978 }
1979 len -= l;
1980 buf += l;
1981 addr += l;
1982 }
1983
1984 return error;
1985 }
1986
1987 bool address_space_write(AddressSpace *as, hwaddr addr,
1988 const uint8_t *buf, int len)
1989 {
1990 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
1991 }
1992
1993 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1994 {
1995 return address_space_rw(as, addr, buf, len, false);
1996 }
1997
1998
1999 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2000 int len, int is_write)
2001 {
2002 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2003 }
2004
2005 /* used for ROM loading : can write in RAM and ROM */
2006 void cpu_physical_memory_write_rom(hwaddr addr,
2007 const uint8_t *buf, int len)
2008 {
2009 hwaddr l;
2010 uint8_t *ptr;
2011 hwaddr addr1;
2012 MemoryRegion *mr;
2013
2014 while (len > 0) {
2015 l = len;
2016 mr = address_space_translate(&address_space_memory,
2017 addr, &addr1, &l, true);
2018
2019 if (!(memory_region_is_ram(mr) ||
2020 memory_region_is_romd(mr))) {
2021 /* do nothing */
2022 } else {
2023 addr1 += memory_region_get_ram_addr(mr);
2024 /* ROM/RAM case */
2025 ptr = qemu_get_ram_ptr(addr1);
2026 memcpy(ptr, buf, l);
2027 invalidate_and_set_dirty(addr1, l);
2028 }
2029 len -= l;
2030 buf += l;
2031 addr += l;
2032 }
2033 }
2034
2035 typedef struct {
2036 MemoryRegion *mr;
2037 void *buffer;
2038 hwaddr addr;
2039 hwaddr len;
2040 } BounceBuffer;
2041
2042 static BounceBuffer bounce;
2043
2044 typedef struct MapClient {
2045 void *opaque;
2046 void (*callback)(void *opaque);
2047 QLIST_ENTRY(MapClient) link;
2048 } MapClient;
2049
2050 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2051 = QLIST_HEAD_INITIALIZER(map_client_list);
2052
2053 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2054 {
2055 MapClient *client = g_malloc(sizeof(*client));
2056
2057 client->opaque = opaque;
2058 client->callback = callback;
2059 QLIST_INSERT_HEAD(&map_client_list, client, link);
2060 return client;
2061 }
2062
2063 static void cpu_unregister_map_client(void *_client)
2064 {
2065 MapClient *client = (MapClient *)_client;
2066
2067 QLIST_REMOVE(client, link);
2068 g_free(client);
2069 }
2070
2071 static void cpu_notify_map_clients(void)
2072 {
2073 MapClient *client;
2074
2075 while (!QLIST_EMPTY(&map_client_list)) {
2076 client = QLIST_FIRST(&map_client_list);
2077 client->callback(client->opaque);
2078 cpu_unregister_map_client(client);
2079 }
2080 }
2081
2082 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2083 {
2084 MemoryRegion *mr;
2085 hwaddr l, xlat;
2086
2087 while (len > 0) {
2088 l = len;
2089 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2090 if (!memory_access_is_direct(mr, is_write)) {
2091 l = memory_access_size(mr, l, addr);
2092 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2093 return false;
2094 }
2095 }
2096
2097 len -= l;
2098 addr += l;
2099 }
2100 return true;
2101 }
2102
2103 /* Map a physical memory region into a host virtual address.
2104 * May map a subset of the requested range, given by and returned in *plen.
2105 * May return NULL if resources needed to perform the mapping are exhausted.
2106 * Use only for reads OR writes - not for read-modify-write operations.
2107 * Use cpu_register_map_client() to know when retrying the map operation is
2108 * likely to succeed.
2109 */
2110 void *address_space_map(AddressSpace *as,
2111 hwaddr addr,
2112 hwaddr *plen,
2113 bool is_write)
2114 {
2115 hwaddr len = *plen;
2116 hwaddr done = 0;
2117 hwaddr l, xlat, base;
2118 MemoryRegion *mr, *this_mr;
2119 ram_addr_t raddr;
2120
2121 if (len == 0) {
2122 return NULL;
2123 }
2124
2125 l = len;
2126 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2127 if (!memory_access_is_direct(mr, is_write)) {
2128 if (bounce.buffer) {
2129 return NULL;
2130 }
2131 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2132 bounce.addr = addr;
2133 bounce.len = l;
2134
2135 memory_region_ref(mr);
2136 bounce.mr = mr;
2137 if (!is_write) {
2138 address_space_read(as, addr, bounce.buffer, l);
2139 }
2140
2141 *plen = l;
2142 return bounce.buffer;
2143 }
2144
2145 base = xlat;
2146 raddr = memory_region_get_ram_addr(mr);
2147
2148 for (;;) {
2149 len -= l;
2150 addr += l;
2151 done += l;
2152 if (len == 0) {
2153 break;
2154 }
2155
2156 l = len;
2157 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2158 if (this_mr != mr || xlat != base + done) {
2159 break;
2160 }
2161 }
2162
2163 memory_region_ref(mr);
2164 *plen = done;
2165 return qemu_ram_ptr_length(raddr + base, plen);
2166 }
2167
2168 /* Unmaps a memory region previously mapped by address_space_map().
2169 * Will also mark the memory as dirty if is_write == 1. access_len gives
2170 * the amount of memory that was actually read or written by the caller.
2171 */
2172 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2173 int is_write, hwaddr access_len)
2174 {
2175 if (buffer != bounce.buffer) {
2176 MemoryRegion *mr;
2177 ram_addr_t addr1;
2178
2179 mr = qemu_ram_addr_from_host(buffer, &addr1);
2180 assert(mr != NULL);
2181 if (is_write) {
2182 while (access_len) {
2183 unsigned l;
2184 l = TARGET_PAGE_SIZE;
2185 if (l > access_len)
2186 l = access_len;
2187 invalidate_and_set_dirty(addr1, l);
2188 addr1 += l;
2189 access_len -= l;
2190 }
2191 }
2192 if (xen_enabled()) {
2193 xen_invalidate_map_cache_entry(buffer);
2194 }
2195 memory_region_unref(mr);
2196 return;
2197 }
2198 if (is_write) {
2199 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2200 }
2201 qemu_vfree(bounce.buffer);
2202 bounce.buffer = NULL;
2203 memory_region_unref(bounce.mr);
2204 cpu_notify_map_clients();
2205 }
2206
2207 void *cpu_physical_memory_map(hwaddr addr,
2208 hwaddr *plen,
2209 int is_write)
2210 {
2211 return address_space_map(&address_space_memory, addr, plen, is_write);
2212 }
2213
2214 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2215 int is_write, hwaddr access_len)
2216 {
2217 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2218 }
2219
2220 /* warning: addr must be aligned */
2221 static inline uint32_t ldl_phys_internal(hwaddr addr,
2222 enum device_endian endian)
2223 {
2224 uint8_t *ptr;
2225 uint64_t val;
2226 MemoryRegion *mr;
2227 hwaddr l = 4;
2228 hwaddr addr1;
2229
2230 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2231 false);
2232 if (l < 4 || !memory_access_is_direct(mr, false)) {
2233 /* I/O case */
2234 io_mem_read(mr, addr1, &val, 4);
2235 #if defined(TARGET_WORDS_BIGENDIAN)
2236 if (endian == DEVICE_LITTLE_ENDIAN) {
2237 val = bswap32(val);
2238 }
2239 #else
2240 if (endian == DEVICE_BIG_ENDIAN) {
2241 val = bswap32(val);
2242 }
2243 #endif
2244 } else {
2245 /* RAM case */
2246 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2247 & TARGET_PAGE_MASK)
2248 + addr1);
2249 switch (endian) {
2250 case DEVICE_LITTLE_ENDIAN:
2251 val = ldl_le_p(ptr);
2252 break;
2253 case DEVICE_BIG_ENDIAN:
2254 val = ldl_be_p(ptr);
2255 break;
2256 default:
2257 val = ldl_p(ptr);
2258 break;
2259 }
2260 }
2261 return val;
2262 }
2263
2264 uint32_t ldl_phys(hwaddr addr)
2265 {
2266 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2267 }
2268
2269 uint32_t ldl_le_phys(hwaddr addr)
2270 {
2271 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2272 }
2273
2274 uint32_t ldl_be_phys(hwaddr addr)
2275 {
2276 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2277 }
2278
2279 /* warning: addr must be aligned */
2280 static inline uint64_t ldq_phys_internal(hwaddr addr,
2281 enum device_endian endian)
2282 {
2283 uint8_t *ptr;
2284 uint64_t val;
2285 MemoryRegion *mr;
2286 hwaddr l = 8;
2287 hwaddr addr1;
2288
2289 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2290 false);
2291 if (l < 8 || !memory_access_is_direct(mr, false)) {
2292 /* I/O case */
2293 io_mem_read(mr, addr1, &val, 8);
2294 #if defined(TARGET_WORDS_BIGENDIAN)
2295 if (endian == DEVICE_LITTLE_ENDIAN) {
2296 val = bswap64(val);
2297 }
2298 #else
2299 if (endian == DEVICE_BIG_ENDIAN) {
2300 val = bswap64(val);
2301 }
2302 #endif
2303 } else {
2304 /* RAM case */
2305 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2306 & TARGET_PAGE_MASK)
2307 + addr1);
2308 switch (endian) {
2309 case DEVICE_LITTLE_ENDIAN:
2310 val = ldq_le_p(ptr);
2311 break;
2312 case DEVICE_BIG_ENDIAN:
2313 val = ldq_be_p(ptr);
2314 break;
2315 default:
2316 val = ldq_p(ptr);
2317 break;
2318 }
2319 }
2320 return val;
2321 }
2322
2323 uint64_t ldq_phys(hwaddr addr)
2324 {
2325 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2326 }
2327
2328 uint64_t ldq_le_phys(hwaddr addr)
2329 {
2330 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2331 }
2332
2333 uint64_t ldq_be_phys(hwaddr addr)
2334 {
2335 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2336 }
2337
2338 /* XXX: optimize */
2339 uint32_t ldub_phys(hwaddr addr)
2340 {
2341 uint8_t val;
2342 cpu_physical_memory_read(addr, &val, 1);
2343 return val;
2344 }
2345
2346 /* warning: addr must be aligned */
2347 static inline uint32_t lduw_phys_internal(hwaddr addr,
2348 enum device_endian endian)
2349 {
2350 uint8_t *ptr;
2351 uint64_t val;
2352 MemoryRegion *mr;
2353 hwaddr l = 2;
2354 hwaddr addr1;
2355
2356 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2357 false);
2358 if (l < 2 || !memory_access_is_direct(mr, false)) {
2359 /* I/O case */
2360 io_mem_read(mr, addr1, &val, 2);
2361 #if defined(TARGET_WORDS_BIGENDIAN)
2362 if (endian == DEVICE_LITTLE_ENDIAN) {
2363 val = bswap16(val);
2364 }
2365 #else
2366 if (endian == DEVICE_BIG_ENDIAN) {
2367 val = bswap16(val);
2368 }
2369 #endif
2370 } else {
2371 /* RAM case */
2372 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2373 & TARGET_PAGE_MASK)
2374 + addr1);
2375 switch (endian) {
2376 case DEVICE_LITTLE_ENDIAN:
2377 val = lduw_le_p(ptr);
2378 break;
2379 case DEVICE_BIG_ENDIAN:
2380 val = lduw_be_p(ptr);
2381 break;
2382 default:
2383 val = lduw_p(ptr);
2384 break;
2385 }
2386 }
2387 return val;
2388 }
2389
2390 uint32_t lduw_phys(hwaddr addr)
2391 {
2392 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2393 }
2394
2395 uint32_t lduw_le_phys(hwaddr addr)
2396 {
2397 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2398 }
2399
2400 uint32_t lduw_be_phys(hwaddr addr)
2401 {
2402 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2403 }
2404
2405 /* warning: addr must be aligned. The ram page is not masked as dirty
2406 and the code inside is not invalidated. It is useful if the dirty
2407 bits are used to track modified PTEs */
2408 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2409 {
2410 uint8_t *ptr;
2411 MemoryRegion *mr;
2412 hwaddr l = 4;
2413 hwaddr addr1;
2414
2415 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2416 true);
2417 if (l < 4 || !memory_access_is_direct(mr, true)) {
2418 io_mem_write(mr, addr1, val, 4);
2419 } else {
2420 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2421 ptr = qemu_get_ram_ptr(addr1);
2422 stl_p(ptr, val);
2423
2424 if (unlikely(in_migration)) {
2425 if (!cpu_physical_memory_is_dirty(addr1)) {
2426 /* invalidate code */
2427 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2428 /* set dirty bit */
2429 cpu_physical_memory_set_dirty_flags(
2430 addr1, (0xff & ~CODE_DIRTY_FLAG));
2431 }
2432 }
2433 }
2434 }
2435
2436 /* warning: addr must be aligned */
2437 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2438 enum device_endian endian)
2439 {
2440 uint8_t *ptr;
2441 MemoryRegion *mr;
2442 hwaddr l = 4;
2443 hwaddr addr1;
2444
2445 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2446 true);
2447 if (l < 4 || !memory_access_is_direct(mr, true)) {
2448 #if defined(TARGET_WORDS_BIGENDIAN)
2449 if (endian == DEVICE_LITTLE_ENDIAN) {
2450 val = bswap32(val);
2451 }
2452 #else
2453 if (endian == DEVICE_BIG_ENDIAN) {
2454 val = bswap32(val);
2455 }
2456 #endif
2457 io_mem_write(mr, addr1, val, 4);
2458 } else {
2459 /* RAM case */
2460 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2461 ptr = qemu_get_ram_ptr(addr1);
2462 switch (endian) {
2463 case DEVICE_LITTLE_ENDIAN:
2464 stl_le_p(ptr, val);
2465 break;
2466 case DEVICE_BIG_ENDIAN:
2467 stl_be_p(ptr, val);
2468 break;
2469 default:
2470 stl_p(ptr, val);
2471 break;
2472 }
2473 invalidate_and_set_dirty(addr1, 4);
2474 }
2475 }
2476
2477 void stl_phys(hwaddr addr, uint32_t val)
2478 {
2479 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2480 }
2481
2482 void stl_le_phys(hwaddr addr, uint32_t val)
2483 {
2484 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2485 }
2486
2487 void stl_be_phys(hwaddr addr, uint32_t val)
2488 {
2489 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2490 }
2491
2492 /* XXX: optimize */
2493 void stb_phys(hwaddr addr, uint32_t val)
2494 {
2495 uint8_t v = val;
2496 cpu_physical_memory_write(addr, &v, 1);
2497 }
2498
2499 /* warning: addr must be aligned */
2500 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2501 enum device_endian endian)
2502 {
2503 uint8_t *ptr;
2504 MemoryRegion *mr;
2505 hwaddr l = 2;
2506 hwaddr addr1;
2507
2508 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2509 true);
2510 if (l < 2 || !memory_access_is_direct(mr, true)) {
2511 #if defined(TARGET_WORDS_BIGENDIAN)
2512 if (endian == DEVICE_LITTLE_ENDIAN) {
2513 val = bswap16(val);
2514 }
2515 #else
2516 if (endian == DEVICE_BIG_ENDIAN) {
2517 val = bswap16(val);
2518 }
2519 #endif
2520 io_mem_write(mr, addr1, val, 2);
2521 } else {
2522 /* RAM case */
2523 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2524 ptr = qemu_get_ram_ptr(addr1);
2525 switch (endian) {
2526 case DEVICE_LITTLE_ENDIAN:
2527 stw_le_p(ptr, val);
2528 break;
2529 case DEVICE_BIG_ENDIAN:
2530 stw_be_p(ptr, val);
2531 break;
2532 default:
2533 stw_p(ptr, val);
2534 break;
2535 }
2536 invalidate_and_set_dirty(addr1, 2);
2537 }
2538 }
2539
2540 void stw_phys(hwaddr addr, uint32_t val)
2541 {
2542 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2543 }
2544
2545 void stw_le_phys(hwaddr addr, uint32_t val)
2546 {
2547 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2548 }
2549
2550 void stw_be_phys(hwaddr addr, uint32_t val)
2551 {
2552 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2553 }
2554
2555 /* XXX: optimize */
2556 void stq_phys(hwaddr addr, uint64_t val)
2557 {
2558 val = tswap64(val);
2559 cpu_physical_memory_write(addr, &val, 8);
2560 }
2561
2562 void stq_le_phys(hwaddr addr, uint64_t val)
2563 {
2564 val = cpu_to_le64(val);
2565 cpu_physical_memory_write(addr, &val, 8);
2566 }
2567
2568 void stq_be_phys(hwaddr addr, uint64_t val)
2569 {
2570 val = cpu_to_be64(val);
2571 cpu_physical_memory_write(addr, &val, 8);
2572 }
2573
2574 /* virtual memory access for debug (includes writing to ROM) */
2575 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2576 uint8_t *buf, int len, int is_write)
2577 {
2578 int l;
2579 hwaddr phys_addr;
2580 target_ulong page;
2581
2582 while (len > 0) {
2583 page = addr & TARGET_PAGE_MASK;
2584 phys_addr = cpu_get_phys_page_debug(env, page);
2585 /* if no physical page mapped, return an error */
2586 if (phys_addr == -1)
2587 return -1;
2588 l = (page + TARGET_PAGE_SIZE) - addr;
2589 if (l > len)
2590 l = len;
2591 phys_addr += (addr & ~TARGET_PAGE_MASK);
2592 if (is_write)
2593 cpu_physical_memory_write_rom(phys_addr, buf, l);
2594 else
2595 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2596 len -= l;
2597 buf += l;
2598 addr += l;
2599 }
2600 return 0;
2601 }
2602 #endif
2603
2604 #if !defined(CONFIG_USER_ONLY)
2605
2606 /*
2607 * A helper function for the _utterly broken_ virtio device model to find out if
2608 * it's running on a big endian machine. Don't do this at home kids!
2609 */
2610 bool virtio_is_big_endian(void);
2611 bool virtio_is_big_endian(void)
2612 {
2613 #if defined(TARGET_WORDS_BIGENDIAN)
2614 return true;
2615 #else
2616 return false;
2617 #endif
2618 }
2619
2620 #endif
2621
2622 #ifndef CONFIG_USER_ONLY
2623 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2624 {
2625 MemoryRegion*mr;
2626 hwaddr l = 1;
2627
2628 mr = address_space_translate(&address_space_memory,
2629 phys_addr, &phys_addr, &l, false);
2630
2631 return !(memory_region_is_ram(mr) ||
2632 memory_region_is_romd(mr));
2633 }
2634
2635 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2636 {
2637 RAMBlock *block;
2638
2639 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2640 func(block->host, block->offset, block->length, opaque);
2641 }
2642 }
2643 #endif
Qemu copy for testing