]> git.proxmox.com Git - mirror_qemu.git/blame_incremental - exec.c
projects / mirror_qemu.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
rdma: don't use negative index to array
[mirror_qemu.git] / exec.c
... / ...
CommitLineData
1/*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19#include "config.h"
20#ifdef _WIN32
21#include <windows.h>
22#else
23#include <sys/types.h>
24#include <sys/mman.h>
25#endif
26
27#include "qemu-common.h"
28#include "cpu.h"
29#include "tcg.h"
30#include "hw/hw.h"
31#include "hw/qdev.h"
32#include "qemu/osdep.h"
33#include "sysemu/kvm.h"
34#include "sysemu/sysemu.h"
35#include "hw/xen/xen.h"
36#include "qemu/timer.h"
37#include "qemu/config-file.h"
38#include "exec/memory.h"
39#include "sysemu/dma.h"
40#include "exec/address-spaces.h"
41#if defined(CONFIG_USER_ONLY)
42#include <qemu.h>
43#else /* !CONFIG_USER_ONLY */
44#include "sysemu/xen-mapcache.h"
45#include "trace.h"
46#endif
47#include "exec/cpu-all.h"
48
49#include "exec/cputlb.h"
50#include "translate-all.h"
51
52#include "exec/memory-internal.h"
53
54//#define DEBUG_SUBPAGE
55
56#if !defined(CONFIG_USER_ONLY)
57static int in_migration;
58
59RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
60
61static MemoryRegion *system_memory;
62static MemoryRegion *system_io;
63
64AddressSpace address_space_io;
65AddressSpace address_space_memory;
66
67MemoryRegion io_mem_rom, io_mem_notdirty;
68static MemoryRegion io_mem_unassigned;
69
70#endif
71
72CPUState *first_cpu;
73/* current CPU in the current thread. It is only valid inside
74 cpu_exec() */
75DEFINE_TLS(CPUState *, current_cpu);
76/* 0 = Do not count executed instructions.
77 1 = Precise instruction counting.
78 2 = Adaptive rate instruction counting. */
79int use_icount;
80
81#if !defined(CONFIG_USER_ONLY)
82
83typedef struct PhysPageEntry PhysPageEntry;
84
85struct PhysPageEntry {
86 uint16_t is_leaf : 1;
87 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
88 uint16_t ptr : 15;
89};
90
91typedef PhysPageEntry Node[L2_SIZE];
92
93struct AddressSpaceDispatch {
94 /* This is a multi-level map on the physical address space.
95 * The bottom level has pointers to MemoryRegionSections.
96 */
97 PhysPageEntry phys_map;
98 Node *nodes;
99 MemoryRegionSection *sections;
100 AddressSpace *as;
101};
102
103#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
104typedef struct subpage_t {
105 MemoryRegion iomem;
106 AddressSpace *as;
107 hwaddr base;
108 uint16_t sub_section[TARGET_PAGE_SIZE];
109} subpage_t;
110
111#define PHYS_SECTION_UNASSIGNED 0
112#define PHYS_SECTION_NOTDIRTY 1
113#define PHYS_SECTION_ROM 2
114#define PHYS_SECTION_WATCH 3
115
116typedef struct PhysPageMap {
117 unsigned sections_nb;
118 unsigned sections_nb_alloc;
119 unsigned nodes_nb;
120 unsigned nodes_nb_alloc;
121 Node *nodes;
122 MemoryRegionSection *sections;
123} PhysPageMap;
124
125static PhysPageMap *prev_map;
126static PhysPageMap next_map;
127
128#define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
129
130static void io_mem_init(void);
131static void memory_map_init(void);
132static void *qemu_safe_ram_ptr(ram_addr_t addr);
133
134static MemoryRegion io_mem_watch;
135#endif
136
137#if !defined(CONFIG_USER_ONLY)
138
139static void phys_map_node_reserve(unsigned nodes)
140{
141 if (next_map.nodes_nb + nodes > next_map.nodes_nb_alloc) {
142 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc * 2,
143 16);
144 next_map.nodes_nb_alloc = MAX(next_map.nodes_nb_alloc,
145 next_map.nodes_nb + nodes);
146 next_map.nodes = g_renew(Node, next_map.nodes,
147 next_map.nodes_nb_alloc);
148 }
149}
150
151static uint16_t phys_map_node_alloc(void)
152{
153 unsigned i;
154 uint16_t ret;
155
156 ret = next_map.nodes_nb++;
157 assert(ret != PHYS_MAP_NODE_NIL);
158 assert(ret != next_map.nodes_nb_alloc);
159 for (i = 0; i < L2_SIZE; ++i) {
160 next_map.nodes[ret][i].is_leaf = 0;
161 next_map.nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
162 }
163 return ret;
164}
165
166static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
167 hwaddr *nb, uint16_t leaf,
168 int level)
169{
170 PhysPageEntry *p;
171 int i;
172 hwaddr step = (hwaddr)1 << (level * L2_BITS);
173
174 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
175 lp->ptr = phys_map_node_alloc();
176 p = next_map.nodes[lp->ptr];
177 if (level == 0) {
178 for (i = 0; i < L2_SIZE; i++) {
179 p[i].is_leaf = 1;
180 p[i].ptr = PHYS_SECTION_UNASSIGNED;
181 }
182 }
183 } else {
184 p = next_map.nodes[lp->ptr];
185 }
186 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
187
188 while (*nb && lp < &p[L2_SIZE]) {
189 if ((*index & (step - 1)) == 0 && *nb >= step) {
190 lp->is_leaf = true;
191 lp->ptr = leaf;
192 *index += step;
193 *nb -= step;
194 } else {
195 phys_page_set_level(lp, index, nb, leaf, level - 1);
196 }
197 ++lp;
198 }
199}
200
201static void phys_page_set(AddressSpaceDispatch *d,
202 hwaddr index, hwaddr nb,
203 uint16_t leaf)
204{
205 /* Wildly overreserve - it doesn't matter much. */
206 phys_map_node_reserve(3 * P_L2_LEVELS);
207
208 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
209}
210
211static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr index,
212 Node *nodes, MemoryRegionSection *sections)
213{
214 PhysPageEntry *p;
215 int i;
216
217 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
218 if (lp.ptr == PHYS_MAP_NODE_NIL) {
219 return &sections[PHYS_SECTION_UNASSIGNED];
220 }
221 p = nodes[lp.ptr];
222 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
223 }
224 return &sections[lp.ptr];
225}
226
227bool memory_region_is_unassigned(MemoryRegion *mr)
228{
229 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
230 && mr != &io_mem_watch;
231}
232
233static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
234 hwaddr addr,
235 bool resolve_subpage)
236{
237 MemoryRegionSection *section;
238 subpage_t *subpage;
239
240 section = phys_page_find(d->phys_map, addr >> TARGET_PAGE_BITS,
241 d->nodes, d->sections);
242 if (resolve_subpage && section->mr->subpage) {
243 subpage = container_of(section->mr, subpage_t, iomem);
244 section = &d->sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
245 }
246 return section;
247}
248
249static MemoryRegionSection *
250address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
251 hwaddr *plen, bool resolve_subpage)
252{
253 MemoryRegionSection *section;
254 Int128 diff;
255
256 section = address_space_lookup_region(d, addr, resolve_subpage);
257 /* Compute offset within MemoryRegionSection */
258 addr -= section->offset_within_address_space;
259
260 /* Compute offset within MemoryRegion */
261 *xlat = addr + section->offset_within_region;
262
263 diff = int128_sub(section->mr->size, int128_make64(addr));
264 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
265 return section;
266}
267
268MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
269 hwaddr *xlat, hwaddr *plen,
270 bool is_write)
271{
272 IOMMUTLBEntry iotlb;
273 MemoryRegionSection *section;
274 MemoryRegion *mr;
275 hwaddr len = *plen;
276
277 for (;;) {
278 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
279 mr = section->mr;
280
281 if (!mr->iommu_ops) {
282 break;
283 }
284
285 iotlb = mr->iommu_ops->translate(mr, addr);
286 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
287 | (addr & iotlb.addr_mask));
288 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
289 if (!(iotlb.perm & (1 << is_write))) {
290 mr = &io_mem_unassigned;
291 break;
292 }
293
294 as = iotlb.target_as;
295 }
296
297 *plen = len;
298 *xlat = addr;
299 return mr;
300}
301
302MemoryRegionSection *
303address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
304 hwaddr *plen)
305{
306 MemoryRegionSection *section;
307 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
308
309 assert(!section->mr->iommu_ops);
310 return section;
311}
312#endif
313
314void cpu_exec_init_all(void)
315{
316#if !defined(CONFIG_USER_ONLY)
317 qemu_mutex_init(&ram_list.mutex);
318 memory_map_init();
319 io_mem_init();
320#endif
321}
322
323#if !defined(CONFIG_USER_ONLY)
324
325static int cpu_common_post_load(void *opaque, int version_id)
326{
327 CPUState *cpu = opaque;
328
329 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
330 version_id is increased. */
331 cpu->interrupt_request &= ~0x01;
332 tlb_flush(cpu->env_ptr, 1);
333
334 return 0;
335}
336
337const VMStateDescription vmstate_cpu_common = {
338 .name = "cpu_common",
339 .version_id = 1,
340 .minimum_version_id = 1,
341 .minimum_version_id_old = 1,
342 .post_load = cpu_common_post_load,
343 .fields = (VMStateField []) {
344 VMSTATE_UINT32(halted, CPUState),
345 VMSTATE_UINT32(interrupt_request, CPUState),
346 VMSTATE_END_OF_LIST()
347 }
348};
349
350#endif
351
352CPUState *qemu_get_cpu(int index)
353{
354 CPUState *cpu = first_cpu;
355
356 while (cpu) {
357 if (cpu->cpu_index == index) {
358 break;
359 }
360 cpu = cpu->next_cpu;
361 }
362
363 return cpu;
364}
365
366void qemu_for_each_cpu(void (*func)(CPUState *cpu, void *data), void *data)
367{
368 CPUState *cpu;
369
370 cpu = first_cpu;
371 while (cpu) {
372 func(cpu, data);
373 cpu = cpu->next_cpu;
374 }
375}
376
377void cpu_exec_init(CPUArchState *env)
378{
379 CPUState *cpu = ENV_GET_CPU(env);
380 CPUClass *cc = CPU_GET_CLASS(cpu);
381 CPUState **pcpu;
382 int cpu_index;
383
384#if defined(CONFIG_USER_ONLY)
385 cpu_list_lock();
386#endif
387 cpu->next_cpu = NULL;
388 pcpu = &first_cpu;
389 cpu_index = 0;
390 while (*pcpu != NULL) {
391 pcpu = &(*pcpu)->next_cpu;
392 cpu_index++;
393 }
394 cpu->cpu_index = cpu_index;
395 cpu->numa_node = 0;
396 QTAILQ_INIT(&env->breakpoints);
397 QTAILQ_INIT(&env->watchpoints);
398#ifndef CONFIG_USER_ONLY
399 cpu->thread_id = qemu_get_thread_id();
400#endif
401 *pcpu = cpu;
402#if defined(CONFIG_USER_ONLY)
403 cpu_list_unlock();
404#endif
405 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
406 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
407 }
408#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
409 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
410 cpu_save, cpu_load, env);
411 assert(cc->vmsd == NULL);
412 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
413#endif
414 if (cc->vmsd != NULL) {
415 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
416 }
417}
418
419#if defined(TARGET_HAS_ICE)
420#if defined(CONFIG_USER_ONLY)
421static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
422{
423 tb_invalidate_phys_page_range(pc, pc + 1, 0);
424}
425#else
426static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
427{
428 tb_invalidate_phys_addr(cpu_get_phys_page_debug(cpu, pc) |
429 (pc & ~TARGET_PAGE_MASK));
430}
431#endif
432#endif /* TARGET_HAS_ICE */
433
434#if defined(CONFIG_USER_ONLY)
435void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
436
437{
438}
439
440int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
441 int flags, CPUWatchpoint **watchpoint)
442{
443 return -ENOSYS;
444}
445#else
446/* Add a watchpoint. */
447int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
448 int flags, CPUWatchpoint **watchpoint)
449{
450 target_ulong len_mask = ~(len - 1);
451 CPUWatchpoint *wp;
452
453 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
454 if ((len & (len - 1)) || (addr & ~len_mask) ||
455 len == 0 || len > TARGET_PAGE_SIZE) {
456 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
457 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
458 return -EINVAL;
459 }
460 wp = g_malloc(sizeof(*wp));
461
462 wp->vaddr = addr;
463 wp->len_mask = len_mask;
464 wp->flags = flags;
465
466 /* keep all GDB-injected watchpoints in front */
467 if (flags & BP_GDB)
468 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
469 else
470 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
471
472 tlb_flush_page(env, addr);
473
474 if (watchpoint)
475 *watchpoint = wp;
476 return 0;
477}
478
479/* Remove a specific watchpoint. */
480int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
481 int flags)
482{
483 target_ulong len_mask = ~(len - 1);
484 CPUWatchpoint *wp;
485
486 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
487 if (addr == wp->vaddr && len_mask == wp->len_mask
488 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
489 cpu_watchpoint_remove_by_ref(env, wp);
490 return 0;
491 }
492 }
493 return -ENOENT;
494}
495
496/* Remove a specific watchpoint by reference. */
497void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
498{
499 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
500
501 tlb_flush_page(env, watchpoint->vaddr);
502
503 g_free(watchpoint);
504}
505
506/* Remove all matching watchpoints. */
507void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
508{
509 CPUWatchpoint *wp, *next;
510
511 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
512 if (wp->flags & mask)
513 cpu_watchpoint_remove_by_ref(env, wp);
514 }
515}
516#endif
517
518/* Add a breakpoint. */
519int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
520 CPUBreakpoint **breakpoint)
521{
522#if defined(TARGET_HAS_ICE)
523 CPUBreakpoint *bp;
524
525 bp = g_malloc(sizeof(*bp));
526
527 bp->pc = pc;
528 bp->flags = flags;
529
530 /* keep all GDB-injected breakpoints in front */
531 if (flags & BP_GDB) {
532 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
533 } else {
534 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
535 }
536
537 breakpoint_invalidate(ENV_GET_CPU(env), pc);
538
539 if (breakpoint) {
540 *breakpoint = bp;
541 }
542 return 0;
543#else
544 return -ENOSYS;
545#endif
546}
547
548/* Remove a specific breakpoint. */
549int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
550{
551#if defined(TARGET_HAS_ICE)
552 CPUBreakpoint *bp;
553
554 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
555 if (bp->pc == pc && bp->flags == flags) {
556 cpu_breakpoint_remove_by_ref(env, bp);
557 return 0;
558 }
559 }
560 return -ENOENT;
561#else
562 return -ENOSYS;
563#endif
564}
565
566/* Remove a specific breakpoint by reference. */
567void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
568{
569#if defined(TARGET_HAS_ICE)
570 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
571
572 breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
573
574 g_free(breakpoint);
575#endif
576}
577
578/* Remove all matching breakpoints. */
579void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
580{
581#if defined(TARGET_HAS_ICE)
582 CPUBreakpoint *bp, *next;
583
584 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
585 if (bp->flags & mask)
586 cpu_breakpoint_remove_by_ref(env, bp);
587 }
588#endif
589}
590
591/* enable or disable single step mode. EXCP_DEBUG is returned by the
592 CPU loop after each instruction */
593void cpu_single_step(CPUState *cpu, int enabled)
594{
595#if defined(TARGET_HAS_ICE)
596 if (cpu->singlestep_enabled != enabled) {
597 cpu->singlestep_enabled = enabled;
598 if (kvm_enabled()) {
599 kvm_update_guest_debug(cpu, 0);
600 } else {
601 /* must flush all the translated code to avoid inconsistencies */
602 /* XXX: only flush what is necessary */
603 CPUArchState *env = cpu->env_ptr;
604 tb_flush(env);
605 }
606 }
607#endif
608}
609
610void cpu_abort(CPUArchState *env, const char *fmt, ...)
611{
612 CPUState *cpu = ENV_GET_CPU(env);
613 va_list ap;
614 va_list ap2;
615
616 va_start(ap, fmt);
617 va_copy(ap2, ap);
618 fprintf(stderr, "qemu: fatal: ");
619 vfprintf(stderr, fmt, ap);
620 fprintf(stderr, "\n");
621 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
622 if (qemu_log_enabled()) {
623 qemu_log("qemu: fatal: ");
624 qemu_log_vprintf(fmt, ap2);
625 qemu_log("\n");
626 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
627 qemu_log_flush();
628 qemu_log_close();
629 }
630 va_end(ap2);
631 va_end(ap);
632#if defined(CONFIG_USER_ONLY)
633 {
634 struct sigaction act;
635 sigfillset(&act.sa_mask);
636 act.sa_handler = SIG_DFL;
637 sigaction(SIGABRT, &act, NULL);
638 }
639#endif
640 abort();
641}
642
643CPUArchState *cpu_copy(CPUArchState *env)
644{
645 CPUArchState *new_env = cpu_init(env->cpu_model_str);
646#if defined(TARGET_HAS_ICE)
647 CPUBreakpoint *bp;
648 CPUWatchpoint *wp;
649#endif
650
651 /* Reset non arch specific state */
652 cpu_reset(ENV_GET_CPU(new_env));
653
654 /* Copy arch specific state into the new CPU */
655 memcpy(new_env, env, sizeof(CPUArchState));
656
657 /* Clone all break/watchpoints.
658 Note: Once we support ptrace with hw-debug register access, make sure
659 BP_CPU break/watchpoints are handled correctly on clone. */
660 QTAILQ_INIT(&env->breakpoints);
661 QTAILQ_INIT(&env->watchpoints);
662#if defined(TARGET_HAS_ICE)
663 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
664 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
665 }
666 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
667 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
668 wp->flags, NULL);
669 }
670#endif
671
672 return new_env;
673}
674
675#if !defined(CONFIG_USER_ONLY)
676static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
677 uintptr_t length)
678{
679 uintptr_t start1;
680
681 /* we modify the TLB cache so that the dirty bit will be set again
682 when accessing the range */
683 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
684 /* Check that we don't span multiple blocks - this breaks the
685 address comparisons below. */
686 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
687 != (end - 1) - start) {
688 abort();
689 }
690 cpu_tlb_reset_dirty_all(start1, length);
691
692}
693
694/* Note: start and end must be within the same ram block. */
695void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
696 int dirty_flags)
697{
698 uintptr_t length;
699
700 start &= TARGET_PAGE_MASK;
701 end = TARGET_PAGE_ALIGN(end);
702
703 length = end - start;
704 if (length == 0)
705 return;
706 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
707
708 if (tcg_enabled()) {
709 tlb_reset_dirty_range_all(start, end, length);
710 }
711}
712
713static int cpu_physical_memory_set_dirty_tracking(int enable)
714{
715 int ret = 0;
716 in_migration = enable;
717 return ret;
718}
719
720hwaddr memory_region_section_get_iotlb(CPUArchState *env,
721 MemoryRegionSection *section,
722 target_ulong vaddr,
723 hwaddr paddr, hwaddr xlat,
724 int prot,
725 target_ulong *address)
726{
727 hwaddr iotlb;
728 CPUWatchpoint *wp;
729
730 if (memory_region_is_ram(section->mr)) {
731 /* Normal RAM. */
732 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
733 + xlat;
734 if (!section->readonly) {
735 iotlb |= PHYS_SECTION_NOTDIRTY;
736 } else {
737 iotlb |= PHYS_SECTION_ROM;
738 }
739 } else {
740 iotlb = section - address_space_memory.dispatch->sections;
741 iotlb += xlat;
742 }
743
744 /* Make accesses to pages with watchpoints go via the
745 watchpoint trap routines. */
746 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
747 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
748 /* Avoid trapping reads of pages with a write breakpoint. */
749 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
750 iotlb = PHYS_SECTION_WATCH + paddr;
751 *address |= TLB_MMIO;
752 break;
753 }
754 }
755 }
756
757 return iotlb;
758}
759#endif /* defined(CONFIG_USER_ONLY) */
760
761#if !defined(CONFIG_USER_ONLY)
762
763static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
764 uint16_t section);
765static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
766
767static uint16_t phys_section_add(MemoryRegionSection *section)
768{
769 /* The physical section number is ORed with a page-aligned
770 * pointer to produce the iotlb entries. Thus it should
771 * never overflow into the page-aligned value.
772 */
773 assert(next_map.sections_nb < TARGET_PAGE_SIZE);
774
775 if (next_map.sections_nb == next_map.sections_nb_alloc) {
776 next_map.sections_nb_alloc = MAX(next_map.sections_nb_alloc * 2,
777 16);
778 next_map.sections = g_renew(MemoryRegionSection, next_map.sections,
779 next_map.sections_nb_alloc);
780 }
781 next_map.sections[next_map.sections_nb] = *section;
782 memory_region_ref(section->mr);
783 return next_map.sections_nb++;
784}
785
786static void phys_section_destroy(MemoryRegion *mr)
787{
788 memory_region_unref(mr);
789
790 if (mr->subpage) {
791 subpage_t *subpage = container_of(mr, subpage_t, iomem);
792 memory_region_destroy(&subpage->iomem);
793 g_free(subpage);
794 }
795}
796
797static void phys_sections_free(PhysPageMap *map)
798{
799 while (map->sections_nb > 0) {
800 MemoryRegionSection *section = &map->sections[--map->sections_nb];
801 phys_section_destroy(section->mr);
802 }
803 g_free(map->sections);
804 g_free(map->nodes);
805 g_free(map);
806}
807
808static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
809{
810 subpage_t *subpage;
811 hwaddr base = section->offset_within_address_space
812 & TARGET_PAGE_MASK;
813 MemoryRegionSection *existing = phys_page_find(d->phys_map, base >> TARGET_PAGE_BITS,
814 next_map.nodes, next_map.sections);
815 MemoryRegionSection subsection = {
816 .offset_within_address_space = base,
817 .size = int128_make64(TARGET_PAGE_SIZE),
818 };
819 hwaddr start, end;
820
821 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
822
823 if (!(existing->mr->subpage)) {
824 subpage = subpage_init(d->as, base);
825 subsection.mr = &subpage->iomem;
826 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
827 phys_section_add(&subsection));
828 } else {
829 subpage = container_of(existing->mr, subpage_t, iomem);
830 }
831 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
832 end = start + int128_get64(section->size) - 1;
833 subpage_register(subpage, start, end, phys_section_add(section));
834}
835
836
837static void register_multipage(AddressSpaceDispatch *d,
838 MemoryRegionSection *section)
839{
840 hwaddr start_addr = section->offset_within_address_space;
841 uint16_t section_index = phys_section_add(section);
842 uint64_t num_pages = int128_get64(int128_rshift(section->size,
843 TARGET_PAGE_BITS));
844
845 assert(num_pages);
846 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
847}
848
849static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
850{
851 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
852 AddressSpaceDispatch *d = as->next_dispatch;
853 MemoryRegionSection now = *section, remain = *section;
854 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
855
856 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
857 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
858 - now.offset_within_address_space;
859
860 now.size = int128_min(int128_make64(left), now.size);
861 register_subpage(d, &now);
862 } else {
863 now.size = int128_zero();
864 }
865 while (int128_ne(remain.size, now.size)) {
866 remain.size = int128_sub(remain.size, now.size);
867 remain.offset_within_address_space += int128_get64(now.size);
868 remain.offset_within_region += int128_get64(now.size);
869 now = remain;
870 if (int128_lt(remain.size, page_size)) {
871 register_subpage(d, &now);
872 } else if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
873 now.size = page_size;
874 register_subpage(d, &now);
875 } else {
876 now.size = int128_and(now.size, int128_neg(page_size));
877 register_multipage(d, &now);
878 }
879 }
880}
881
882void qemu_flush_coalesced_mmio_buffer(void)
883{
884 if (kvm_enabled())
885 kvm_flush_coalesced_mmio_buffer();
886}
887
888void qemu_mutex_lock_ramlist(void)
889{
890 qemu_mutex_lock(&ram_list.mutex);
891}
892
893void qemu_mutex_unlock_ramlist(void)
894{
895 qemu_mutex_unlock(&ram_list.mutex);
896}
897
898#if defined(__linux__) && !defined(TARGET_S390X)
899
900#include <sys/vfs.h>
901
902#define HUGETLBFS_MAGIC 0x958458f6
903
904static long gethugepagesize(const char *path)
905{
906 struct statfs fs;
907 int ret;
908
909 do {
910 ret = statfs(path, &fs);
911 } while (ret != 0 && errno == EINTR);
912
913 if (ret != 0) {
914 perror(path);
915 return 0;
916 }
917
918 if (fs.f_type != HUGETLBFS_MAGIC)
919 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
920
921 return fs.f_bsize;
922}
923
924static void *file_ram_alloc(RAMBlock *block,
925 ram_addr_t memory,
926 const char *path)
927{
928 char *filename;
929 char *sanitized_name;
930 char *c;
931 void *area;
932 int fd;
933#ifdef MAP_POPULATE
934 int flags;
935#endif
936 unsigned long hpagesize;
937
938 hpagesize = gethugepagesize(path);
939 if (!hpagesize) {
940 return NULL;
941 }
942
943 if (memory < hpagesize) {
944 return NULL;
945 }
946
947 if (kvm_enabled() && !kvm_has_sync_mmu()) {
948 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
949 return NULL;
950 }
951
952 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
953 sanitized_name = g_strdup(block->mr->name);
954 for (c = sanitized_name; *c != '\0'; c++) {
955 if (*c == '/')
956 *c = '_';
957 }
958
959 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
960 sanitized_name);
961 g_free(sanitized_name);
962
963 fd = mkstemp(filename);
964 if (fd < 0) {
965 perror("unable to create backing store for hugepages");
966 g_free(filename);
967 return NULL;
968 }
969 unlink(filename);
970 g_free(filename);
971
972 memory = (memory+hpagesize-1) & ~(hpagesize-1);
973
974 /*
975 * ftruncate is not supported by hugetlbfs in older
976 * hosts, so don't bother bailing out on errors.
977 * If anything goes wrong with it under other filesystems,
978 * mmap will fail.
979 */
980 if (ftruncate(fd, memory))
981 perror("ftruncate");
982
983#ifdef MAP_POPULATE
984 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
985 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
986 * to sidestep this quirk.
987 */
988 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
989 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
990#else
991 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
992#endif
993 if (area == MAP_FAILED) {
994 perror("file_ram_alloc: can't mmap RAM pages");
995 close(fd);
996 return (NULL);
997 }
998 block->fd = fd;
999 return area;
1000}
1001#endif
1002
1003static ram_addr_t find_ram_offset(ram_addr_t size)
1004{
1005 RAMBlock *block, *next_block;
1006 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1007
1008 assert(size != 0); /* it would hand out same offset multiple times */
1009
1010 if (QTAILQ_EMPTY(&ram_list.blocks))
1011 return 0;
1012
1013 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1014 ram_addr_t end, next = RAM_ADDR_MAX;
1015
1016 end = block->offset + block->length;
1017
1018 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1019 if (next_block->offset >= end) {
1020 next = MIN(next, next_block->offset);
1021 }
1022 }
1023 if (next - end >= size && next - end < mingap) {
1024 offset = end;
1025 mingap = next - end;
1026 }
1027 }
1028
1029 if (offset == RAM_ADDR_MAX) {
1030 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1031 (uint64_t)size);
1032 abort();
1033 }
1034
1035 return offset;
1036}
1037
1038ram_addr_t last_ram_offset(void)
1039{
1040 RAMBlock *block;
1041 ram_addr_t last = 0;
1042
1043 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1044 last = MAX(last, block->offset + block->length);
1045
1046 return last;
1047}
1048
1049static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1050{
1051 int ret;
1052
1053 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1054 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1055 "dump-guest-core", true)) {
1056 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1057 if (ret) {
1058 perror("qemu_madvise");
1059 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1060 "but dump_guest_core=off specified\n");
1061 }
1062 }
1063}
1064
1065void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1066{
1067 RAMBlock *new_block, *block;
1068
1069 new_block = NULL;
1070 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1071 if (block->offset == addr) {
1072 new_block = block;
1073 break;
1074 }
1075 }
1076 assert(new_block);
1077 assert(!new_block->idstr[0]);
1078
1079 if (dev) {
1080 char *id = qdev_get_dev_path(dev);
1081 if (id) {
1082 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1083 g_free(id);
1084 }
1085 }
1086 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1087
1088 /* This assumes the iothread lock is taken here too. */
1089 qemu_mutex_lock_ramlist();
1090 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1091 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1092 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1093 new_block->idstr);
1094 abort();
1095 }
1096 }
1097 qemu_mutex_unlock_ramlist();
1098}
1099
1100static int memory_try_enable_merging(void *addr, size_t len)
1101{
1102 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1103 /* disabled by the user */
1104 return 0;
1105 }
1106
1107 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1108}
1109
1110ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1111 MemoryRegion *mr)
1112{
1113 RAMBlock *block, *new_block;
1114
1115 size = TARGET_PAGE_ALIGN(size);
1116 new_block = g_malloc0(sizeof(*new_block));
1117
1118 /* This assumes the iothread lock is taken here too. */
1119 qemu_mutex_lock_ramlist();
1120 new_block->mr = mr;
1121 new_block->offset = find_ram_offset(size);
1122 if (host) {
1123 new_block->host = host;
1124 new_block->flags |= RAM_PREALLOC_MASK;
1125 } else {
1126 if (mem_path) {
1127#if defined (__linux__) && !defined(TARGET_S390X)
1128 new_block->host = file_ram_alloc(new_block, size, mem_path);
1129 if (!new_block->host) {
1130 new_block->host = qemu_anon_ram_alloc(size);
1131 memory_try_enable_merging(new_block->host, size);
1132 }
1133#else
1134 fprintf(stderr, "-mem-path option unsupported\n");
1135 exit(1);
1136#endif
1137 } else {
1138 if (xen_enabled()) {
1139 xen_ram_alloc(new_block->offset, size, mr);
1140 } else if (kvm_enabled()) {
1141 /* some s390/kvm configurations have special constraints */
1142 new_block->host = kvm_ram_alloc(size);
1143 } else {
1144 new_block->host = qemu_anon_ram_alloc(size);
1145 }
1146 memory_try_enable_merging(new_block->host, size);
1147 }
1148 }
1149 new_block->length = size;
1150
1151 /* Keep the list sorted from biggest to smallest block. */
1152 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1153 if (block->length < new_block->length) {
1154 break;
1155 }
1156 }
1157 if (block) {
1158 QTAILQ_INSERT_BEFORE(block, new_block, next);
1159 } else {
1160 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1161 }
1162 ram_list.mru_block = NULL;
1163
1164 ram_list.version++;
1165 qemu_mutex_unlock_ramlist();
1166
1167 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1168 last_ram_offset() >> TARGET_PAGE_BITS);
1169 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1170 0, size >> TARGET_PAGE_BITS);
1171 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1172
1173 qemu_ram_setup_dump(new_block->host, size);
1174 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1175
1176 if (kvm_enabled())
1177 kvm_setup_guest_memory(new_block->host, size);
1178
1179 return new_block->offset;
1180}
1181
1182ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1183{
1184 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1185}
1186
1187void qemu_ram_free_from_ptr(ram_addr_t addr)
1188{
1189 RAMBlock *block;
1190
1191 /* This assumes the iothread lock is taken here too. */
1192 qemu_mutex_lock_ramlist();
1193 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1194 if (addr == block->offset) {
1195 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1196 ram_list.mru_block = NULL;
1197 ram_list.version++;
1198 g_free(block);
1199 break;
1200 }
1201 }
1202 qemu_mutex_unlock_ramlist();
1203}
1204
1205void qemu_ram_free(ram_addr_t addr)
1206{
1207 RAMBlock *block;
1208
1209 /* This assumes the iothread lock is taken here too. */
1210 qemu_mutex_lock_ramlist();
1211 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1212 if (addr == block->offset) {
1213 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1214 ram_list.mru_block = NULL;
1215 ram_list.version++;
1216 if (block->flags & RAM_PREALLOC_MASK) {
1217 ;
1218 } else if (mem_path) {
1219#if defined (__linux__) && !defined(TARGET_S390X)
1220 if (block->fd) {
1221 munmap(block->host, block->length);
1222 close(block->fd);
1223 } else {
1224 qemu_anon_ram_free(block->host, block->length);
1225 }
1226#else
1227 abort();
1228#endif
1229 } else {
1230 if (xen_enabled()) {
1231 xen_invalidate_map_cache_entry(block->host);
1232 } else {
1233 qemu_anon_ram_free(block->host, block->length);
1234 }
1235 }
1236 g_free(block);
1237 break;
1238 }
1239 }
1240 qemu_mutex_unlock_ramlist();
1241
1242}
1243
1244#ifndef _WIN32
1245void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1246{
1247 RAMBlock *block;
1248 ram_addr_t offset;
1249 int flags;
1250 void *area, *vaddr;
1251
1252 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1253 offset = addr - block->offset;
1254 if (offset < block->length) {
1255 vaddr = block->host + offset;
1256 if (block->flags & RAM_PREALLOC_MASK) {
1257 ;
1258 } else {
1259 flags = MAP_FIXED;
1260 munmap(vaddr, length);
1261 if (mem_path) {
1262#if defined(__linux__) && !defined(TARGET_S390X)
1263 if (block->fd) {
1264#ifdef MAP_POPULATE
1265 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1266 MAP_PRIVATE;
1267#else
1268 flags |= MAP_PRIVATE;
1269#endif
1270 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1271 flags, block->fd, offset);
1272 } else {
1273 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1274 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1275 flags, -1, 0);
1276 }
1277#else
1278 abort();
1279#endif
1280 } else {
1281#if defined(TARGET_S390X) && defined(CONFIG_KVM)
1282 flags |= MAP_SHARED | MAP_ANONYMOUS;
1283 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1284 flags, -1, 0);
1285#else
1286 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1287 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1288 flags, -1, 0);
1289#endif
1290 }
1291 if (area != vaddr) {
1292 fprintf(stderr, "Could not remap addr: "
1293 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1294 length, addr);
1295 exit(1);
1296 }
1297 memory_try_enable_merging(vaddr, length);
1298 qemu_ram_setup_dump(vaddr, length);
1299 }
1300 return;
1301 }
1302 }
1303}
1304#endif /* !_WIN32 */
1305
1306static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
1307{
1308 RAMBlock *block;
1309
1310 /* The list is protected by the iothread lock here. */
1311 block = ram_list.mru_block;
1312 if (block && addr - block->offset < block->length) {
1313 goto found;
1314 }
1315 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1316 if (addr - block->offset < block->length) {
1317 goto found;
1318 }
1319 }
1320
1321 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1322 abort();
1323
1324found:
1325 ram_list.mru_block = block;
1326 return block;
1327}
1328
1329/* Return a host pointer to ram allocated with qemu_ram_alloc.
1330 With the exception of the softmmu code in this file, this should
1331 only be used for local memory (e.g. video ram) that the device owns,
1332 and knows it isn't going to access beyond the end of the block.
1333
1334 It should not be used for general purpose DMA.
1335 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1336 */
1337void *qemu_get_ram_ptr(ram_addr_t addr)
1338{
1339 RAMBlock *block = qemu_get_ram_block(addr);
1340
1341 if (xen_enabled()) {
1342 /* We need to check if the requested address is in the RAM
1343 * because we don't want to map the entire memory in QEMU.
1344 * In that case just map until the end of the page.
1345 */
1346 if (block->offset == 0) {
1347 return xen_map_cache(addr, 0, 0);
1348 } else if (block->host == NULL) {
1349 block->host =
1350 xen_map_cache(block->offset, block->length, 1);
1351 }
1352 }
1353 return block->host + (addr - block->offset);
1354}
1355
1356/* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1357 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1358 *
1359 * ??? Is this still necessary?
1360 */
1361static void *qemu_safe_ram_ptr(ram_addr_t addr)
1362{
1363 RAMBlock *block;
1364
1365 /* The list is protected by the iothread lock here. */
1366 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1367 if (addr - block->offset < block->length) {
1368 if (xen_enabled()) {
1369 /* We need to check if the requested address is in the RAM
1370 * because we don't want to map the entire memory in QEMU.
1371 * In that case just map until the end of the page.
1372 */
1373 if (block->offset == 0) {
1374 return xen_map_cache(addr, 0, 0);
1375 } else if (block->host == NULL) {
1376 block->host =
1377 xen_map_cache(block->offset, block->length, 1);
1378 }
1379 }
1380 return block->host + (addr - block->offset);
1381 }
1382 }
1383
1384 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1385 abort();
1386
1387 return NULL;
1388}
1389
1390/* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1391 * but takes a size argument */
1392static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1393{
1394 if (*size == 0) {
1395 return NULL;
1396 }
1397 if (xen_enabled()) {
1398 return xen_map_cache(addr, *size, 1);
1399 } else {
1400 RAMBlock *block;
1401
1402 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1403 if (addr - block->offset < block->length) {
1404 if (addr - block->offset + *size > block->length)
1405 *size = block->length - addr + block->offset;
1406 return block->host + (addr - block->offset);
1407 }
1408 }
1409
1410 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1411 abort();
1412 }
1413}
1414
1415/* Some of the softmmu routines need to translate from a host pointer
1416 (typically a TLB entry) back to a ram offset. */
1417MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1418{
1419 RAMBlock *block;
1420 uint8_t *host = ptr;
1421
1422 if (xen_enabled()) {
1423 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1424 return qemu_get_ram_block(*ram_addr)->mr;
1425 }
1426
1427 block = ram_list.mru_block;
1428 if (block && block->host && host - block->host < block->length) {
1429 goto found;
1430 }
1431
1432 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1433 /* This case append when the block is not mapped. */
1434 if (block->host == NULL) {
1435 continue;
1436 }
1437 if (host - block->host < block->length) {
1438 goto found;
1439 }
1440 }
1441
1442 return NULL;
1443
1444found:
1445 *ram_addr = block->offset + (host - block->host);
1446 return block->mr;
1447}
1448
1449static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1450 uint64_t val, unsigned size)
1451{
1452 int dirty_flags;
1453 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1454 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1455 tb_invalidate_phys_page_fast(ram_addr, size);
1456 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1457 }
1458 switch (size) {
1459 case 1:
1460 stb_p(qemu_get_ram_ptr(ram_addr), val);
1461 break;
1462 case 2:
1463 stw_p(qemu_get_ram_ptr(ram_addr), val);
1464 break;
1465 case 4:
1466 stl_p(qemu_get_ram_ptr(ram_addr), val);
1467 break;
1468 default:
1469 abort();
1470 }
1471 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1472 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1473 /* we remove the notdirty callback only if the code has been
1474 flushed */
1475 if (dirty_flags == 0xff) {
1476 CPUArchState *env = current_cpu->env_ptr;
1477 tlb_set_dirty(env, env->mem_io_vaddr);
1478 }
1479}
1480
1481static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1482 unsigned size, bool is_write)
1483{
1484 return is_write;
1485}
1486
1487static const MemoryRegionOps notdirty_mem_ops = {
1488 .write = notdirty_mem_write,
1489 .valid.accepts = notdirty_mem_accepts,
1490 .endianness = DEVICE_NATIVE_ENDIAN,
1491};
1492
1493/* Generate a debug exception if a watchpoint has been hit. */
1494static void check_watchpoint(int offset, int len_mask, int flags)
1495{
1496 CPUArchState *env = current_cpu->env_ptr;
1497 target_ulong pc, cs_base;
1498 target_ulong vaddr;
1499 CPUWatchpoint *wp;
1500 int cpu_flags;
1501
1502 if (env->watchpoint_hit) {
1503 /* We re-entered the check after replacing the TB. Now raise
1504 * the debug interrupt so that is will trigger after the
1505 * current instruction. */
1506 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1507 return;
1508 }
1509 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1510 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1511 if ((vaddr == (wp->vaddr & len_mask) ||
1512 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1513 wp->flags |= BP_WATCHPOINT_HIT;
1514 if (!env->watchpoint_hit) {
1515 env->watchpoint_hit = wp;
1516 tb_check_watchpoint(env);
1517 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1518 env->exception_index = EXCP_DEBUG;
1519 cpu_loop_exit(env);
1520 } else {
1521 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1522 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1523 cpu_resume_from_signal(env, NULL);
1524 }
1525 }
1526 } else {
1527 wp->flags &= ~BP_WATCHPOINT_HIT;
1528 }
1529 }
1530}
1531
1532/* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1533 so these check for a hit then pass through to the normal out-of-line
1534 phys routines. */
1535static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1536 unsigned size)
1537{
1538 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1539 switch (size) {
1540 case 1: return ldub_phys(addr);
1541 case 2: return lduw_phys(addr);
1542 case 4: return ldl_phys(addr);
1543 default: abort();
1544 }
1545}
1546
1547static void watch_mem_write(void *opaque, hwaddr addr,
1548 uint64_t val, unsigned size)
1549{
1550 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1551 switch (size) {
1552 case 1:
1553 stb_phys(addr, val);
1554 break;
1555 case 2:
1556 stw_phys(addr, val);
1557 break;
1558 case 4:
1559 stl_phys(addr, val);
1560 break;
1561 default: abort();
1562 }
1563}
1564
1565static const MemoryRegionOps watch_mem_ops = {
1566 .read = watch_mem_read,
1567 .write = watch_mem_write,
1568 .endianness = DEVICE_NATIVE_ENDIAN,
1569};
1570
1571static uint64_t subpage_read(void *opaque, hwaddr addr,
1572 unsigned len)
1573{
1574 subpage_t *subpage = opaque;
1575 uint8_t buf[4];
1576
1577#if defined(DEBUG_SUBPAGE)
1578 printf("%s: subpage %p len %d addr " TARGET_FMT_plx "\n", __func__,
1579 subpage, len, addr);
1580#endif
1581 address_space_read(subpage->as, addr + subpage->base, buf, len);
1582 switch (len) {
1583 case 1:
1584 return ldub_p(buf);
1585 case 2:
1586 return lduw_p(buf);
1587 case 4:
1588 return ldl_p(buf);
1589 default:
1590 abort();
1591 }
1592}
1593
1594static void subpage_write(void *opaque, hwaddr addr,
1595 uint64_t value, unsigned len)
1596{
1597 subpage_t *subpage = opaque;
1598 uint8_t buf[4];
1599
1600#if defined(DEBUG_SUBPAGE)
1601 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1602 " value %"PRIx64"\n",
1603 __func__, subpage, len, addr, value);
1604#endif
1605 switch (len) {
1606 case 1:
1607 stb_p(buf, value);
1608 break;
1609 case 2:
1610 stw_p(buf, value);
1611 break;
1612 case 4:
1613 stl_p(buf, value);
1614 break;
1615 default:
1616 abort();
1617 }
1618 address_space_write(subpage->as, addr + subpage->base, buf, len);
1619}
1620
1621static bool subpage_accepts(void *opaque, hwaddr addr,
1622 unsigned size, bool is_write)
1623{
1624 subpage_t *subpage = opaque;
1625#if defined(DEBUG_SUBPAGE)
1626 printf("%s: subpage %p %c len %d addr " TARGET_FMT_plx "\n",
1627 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1628#endif
1629
1630 return address_space_access_valid(subpage->as, addr + subpage->base,
1631 size, is_write);
1632}
1633
1634static const MemoryRegionOps subpage_ops = {
1635 .read = subpage_read,
1636 .write = subpage_write,
1637 .valid.accepts = subpage_accepts,
1638 .endianness = DEVICE_NATIVE_ENDIAN,
1639};
1640
1641static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1642 uint16_t section)
1643{
1644 int idx, eidx;
1645
1646 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1647 return -1;
1648 idx = SUBPAGE_IDX(start);
1649 eidx = SUBPAGE_IDX(end);
1650#if defined(DEBUG_SUBPAGE)
1651 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1652 mmio, start, end, idx, eidx, memory);
1653#endif
1654 for (; idx <= eidx; idx++) {
1655 mmio->sub_section[idx] = section;
1656 }
1657
1658 return 0;
1659}
1660
1661static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1662{
1663 subpage_t *mmio;
1664
1665 mmio = g_malloc0(sizeof(subpage_t));
1666
1667 mmio->as = as;
1668 mmio->base = base;
1669 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1670 "subpage", TARGET_PAGE_SIZE);
1671 mmio->iomem.subpage = true;
1672#if defined(DEBUG_SUBPAGE)
1673 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1674 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1675#endif
1676 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1677
1678 return mmio;
1679}
1680
1681static uint16_t dummy_section(MemoryRegion *mr)
1682{
1683 MemoryRegionSection section = {
1684 .mr = mr,
1685 .offset_within_address_space = 0,
1686 .offset_within_region = 0,
1687 .size = int128_2_64(),
1688 };
1689
1690 return phys_section_add(&section);
1691}
1692
1693MemoryRegion *iotlb_to_region(hwaddr index)
1694{
1695 return address_space_memory.dispatch->sections[index & ~TARGET_PAGE_MASK].mr;
1696}
1697
1698static void io_mem_init(void)
1699{
1700 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1701 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1702 "unassigned", UINT64_MAX);
1703 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1704 "notdirty", UINT64_MAX);
1705 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1706 "watch", UINT64_MAX);
1707}
1708
1709static void mem_begin(MemoryListener *listener)
1710{
1711 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1712 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1713
1714 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1715 d->as = as;
1716 as->next_dispatch = d;
1717}
1718
1719static void mem_commit(MemoryListener *listener)
1720{
1721 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1722 AddressSpaceDispatch *cur = as->dispatch;
1723 AddressSpaceDispatch *next = as->next_dispatch;
1724
1725 next->nodes = next_map.nodes;
1726 next->sections = next_map.sections;
1727
1728 as->dispatch = next;
1729 g_free(cur);
1730}
1731
1732static void core_begin(MemoryListener *listener)
1733{
1734 uint16_t n;
1735
1736 prev_map = g_new(PhysPageMap, 1);
1737 *prev_map = next_map;
1738
1739 memset(&next_map, 0, sizeof(next_map));
1740 n = dummy_section(&io_mem_unassigned);
1741 assert(n == PHYS_SECTION_UNASSIGNED);
1742 n = dummy_section(&io_mem_notdirty);
1743 assert(n == PHYS_SECTION_NOTDIRTY);
1744 n = dummy_section(&io_mem_rom);
1745 assert(n == PHYS_SECTION_ROM);
1746 n = dummy_section(&io_mem_watch);
1747 assert(n == PHYS_SECTION_WATCH);
1748}
1749
1750/* This listener's commit run after the other AddressSpaceDispatch listeners'.
1751 * All AddressSpaceDispatch instances have switched to the next map.
1752 */
1753static void core_commit(MemoryListener *listener)
1754{
1755 phys_sections_free(prev_map);
1756}
1757
1758static void tcg_commit(MemoryListener *listener)
1759{
1760 CPUState *cpu;
1761
1762 /* since each CPU stores ram addresses in its TLB cache, we must
1763 reset the modified entries */
1764 /* XXX: slow ! */
1765 for (cpu = first_cpu; cpu != NULL; cpu = cpu->next_cpu) {
1766 CPUArchState *env = cpu->env_ptr;
1767
1768 tlb_flush(env, 1);
1769 }
1770}
1771
1772static void core_log_global_start(MemoryListener *listener)
1773{
1774 cpu_physical_memory_set_dirty_tracking(1);
1775}
1776
1777static void core_log_global_stop(MemoryListener *listener)
1778{
1779 cpu_physical_memory_set_dirty_tracking(0);
1780}
1781
1782static MemoryListener core_memory_listener = {
1783 .begin = core_begin,
1784 .commit = core_commit,
1785 .log_global_start = core_log_global_start,
1786 .log_global_stop = core_log_global_stop,
1787 .priority = 1,
1788};
1789
1790static MemoryListener tcg_memory_listener = {
1791 .commit = tcg_commit,
1792};
1793
1794void address_space_init_dispatch(AddressSpace *as)
1795{
1796 as->dispatch = NULL;
1797 as->dispatch_listener = (MemoryListener) {
1798 .begin = mem_begin,
1799 .commit = mem_commit,
1800 .region_add = mem_add,
1801 .region_nop = mem_add,
1802 .priority = 0,
1803 };
1804 memory_listener_register(&as->dispatch_listener, as);
1805}
1806
1807void address_space_destroy_dispatch(AddressSpace *as)
1808{
1809 AddressSpaceDispatch *d = as->dispatch;
1810
1811 memory_listener_unregister(&as->dispatch_listener);
1812 g_free(d);
1813 as->dispatch = NULL;
1814}
1815
1816static void memory_map_init(void)
1817{
1818 system_memory = g_malloc(sizeof(*system_memory));
1819 memory_region_init(system_memory, NULL, "system", INT64_MAX);
1820 address_space_init(&address_space_memory, system_memory, "memory");
1821
1822 system_io = g_malloc(sizeof(*system_io));
1823 memory_region_init(system_io, NULL, "io", 65536);
1824 address_space_init(&address_space_io, system_io, "I/O");
1825
1826 memory_listener_register(&core_memory_listener, &address_space_memory);
1827 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1828}
1829
1830MemoryRegion *get_system_memory(void)
1831{
1832 return system_memory;
1833}
1834
1835MemoryRegion *get_system_io(void)
1836{
1837 return system_io;
1838}
1839
1840#endif /* !defined(CONFIG_USER_ONLY) */
1841
1842/* physical memory access (slow version, mainly for debug) */
1843#if defined(CONFIG_USER_ONLY)
1844int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1845 uint8_t *buf, int len, int is_write)
1846{
1847 int l, flags;
1848 target_ulong page;
1849 void * p;
1850
1851 while (len > 0) {
1852 page = addr & TARGET_PAGE_MASK;
1853 l = (page + TARGET_PAGE_SIZE) - addr;
1854 if (l > len)
1855 l = len;
1856 flags = page_get_flags(page);
1857 if (!(flags & PAGE_VALID))
1858 return -1;
1859 if (is_write) {
1860 if (!(flags & PAGE_WRITE))
1861 return -1;
1862 /* XXX: this code should not depend on lock_user */
1863 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1864 return -1;
1865 memcpy(p, buf, l);
1866 unlock_user(p, addr, l);
1867 } else {
1868 if (!(flags & PAGE_READ))
1869 return -1;
1870 /* XXX: this code should not depend on lock_user */
1871 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1872 return -1;
1873 memcpy(buf, p, l);
1874 unlock_user(p, addr, 0);
1875 }
1876 len -= l;
1877 buf += l;
1878 addr += l;
1879 }
1880 return 0;
1881}
1882
1883#else
1884
1885static void invalidate_and_set_dirty(hwaddr addr,
1886 hwaddr length)
1887{
1888 if (!cpu_physical_memory_is_dirty(addr)) {
1889 /* invalidate code */
1890 tb_invalidate_phys_page_range(addr, addr + length, 0);
1891 /* set dirty bit */
1892 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1893 }
1894 xen_modified_memory(addr, length);
1895}
1896
1897static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1898{
1899 if (memory_region_is_ram(mr)) {
1900 return !(is_write && mr->readonly);
1901 }
1902 if (memory_region_is_romd(mr)) {
1903 return !is_write;
1904 }
1905
1906 return false;
1907}
1908
1909static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1910{
1911 unsigned access_size_max = mr->ops->valid.max_access_size;
1912
1913 /* Regions are assumed to support 1-4 byte accesses unless
1914 otherwise specified. */
1915 if (access_size_max == 0) {
1916 access_size_max = 4;
1917 }
1918
1919 /* Bound the maximum access by the alignment of the address. */
1920 if (!mr->ops->impl.unaligned) {
1921 unsigned align_size_max = addr & -addr;
1922 if (align_size_max != 0 && align_size_max < access_size_max) {
1923 access_size_max = align_size_max;
1924 }
1925 }
1926
1927 /* Don't attempt accesses larger than the maximum. */
1928 if (l > access_size_max) {
1929 l = access_size_max;
1930 }
1931
1932 return l;
1933}
1934
1935bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1936 int len, bool is_write)
1937{
1938 hwaddr l;
1939 uint8_t *ptr;
1940 uint64_t val;
1941 hwaddr addr1;
1942 MemoryRegion *mr;
1943 bool error = false;
1944
1945 while (len > 0) {
1946 l = len;
1947 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1948
1949 if (is_write) {
1950 if (!memory_access_is_direct(mr, is_write)) {
1951 l = memory_access_size(mr, l, addr1);
1952 /* XXX: could force current_cpu to NULL to avoid
1953 potential bugs */
1954 switch (l) {
1955 case 8:
1956 /* 64 bit write access */
1957 val = ldq_p(buf);
1958 error |= io_mem_write(mr, addr1, val, 8);
1959 break;
1960 case 4:
1961 /* 32 bit write access */
1962 val = ldl_p(buf);
1963 error |= io_mem_write(mr, addr1, val, 4);
1964 break;
1965 case 2:
1966 /* 16 bit write access */
1967 val = lduw_p(buf);
1968 error |= io_mem_write(mr, addr1, val, 2);
1969 break;
1970 case 1:
1971 /* 8 bit write access */
1972 val = ldub_p(buf);
1973 error |= io_mem_write(mr, addr1, val, 1);
1974 break;
1975 default:
1976 abort();
1977 }
1978 } else {
1979 addr1 += memory_region_get_ram_addr(mr);
1980 /* RAM case */
1981 ptr = qemu_get_ram_ptr(addr1);
1982 memcpy(ptr, buf, l);
1983 invalidate_and_set_dirty(addr1, l);
1984 }
1985 } else {
1986 if (!memory_access_is_direct(mr, is_write)) {
1987 /* I/O case */
1988 l = memory_access_size(mr, l, addr1);
1989 switch (l) {
1990 case 8:
1991 /* 64 bit read access */
1992 error |= io_mem_read(mr, addr1, &val, 8);
1993 stq_p(buf, val);
1994 break;
1995 case 4:
1996 /* 32 bit read access */
1997 error |= io_mem_read(mr, addr1, &val, 4);
1998 stl_p(buf, val);
1999 break;
2000 case 2:
2001 /* 16 bit read access */
2002 error |= io_mem_read(mr, addr1, &val, 2);
2003 stw_p(buf, val);
2004 break;
2005 case 1:
2006 /* 8 bit read access */
2007 error |= io_mem_read(mr, addr1, &val, 1);
2008 stb_p(buf, val);
2009 break;
2010 default:
2011 abort();
2012 }
2013 } else {
2014 /* RAM case */
2015 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2016 memcpy(buf, ptr, l);
2017 }
2018 }
2019 len -= l;
2020 buf += l;
2021 addr += l;
2022 }
2023
2024 return error;
2025}
2026
2027bool address_space_write(AddressSpace *as, hwaddr addr,
2028 const uint8_t *buf, int len)
2029{
2030 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2031}
2032
2033bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2034{
2035 return address_space_rw(as, addr, buf, len, false);
2036}
2037
2038
2039void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2040 int len, int is_write)
2041{
2042 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2043}
2044
2045/* used for ROM loading : can write in RAM and ROM */
2046void cpu_physical_memory_write_rom(hwaddr addr,
2047 const uint8_t *buf, int len)
2048{
2049 hwaddr l;
2050 uint8_t *ptr;
2051 hwaddr addr1;
2052 MemoryRegion *mr;
2053
2054 while (len > 0) {
2055 l = len;
2056 mr = address_space_translate(&address_space_memory,
2057 addr, &addr1, &l, true);
2058
2059 if (!(memory_region_is_ram(mr) ||
2060 memory_region_is_romd(mr))) {
2061 /* do nothing */
2062 } else {
2063 addr1 += memory_region_get_ram_addr(mr);
2064 /* ROM/RAM case */
2065 ptr = qemu_get_ram_ptr(addr1);
2066 memcpy(ptr, buf, l);
2067 invalidate_and_set_dirty(addr1, l);
2068 }
2069 len -= l;
2070 buf += l;
2071 addr += l;
2072 }
2073}
2074
2075typedef struct {
2076 MemoryRegion *mr;
2077 void *buffer;
2078 hwaddr addr;
2079 hwaddr len;
2080} BounceBuffer;
2081
2082static BounceBuffer bounce;
2083
2084typedef struct MapClient {
2085 void *opaque;
2086 void (*callback)(void *opaque);
2087 QLIST_ENTRY(MapClient) link;
2088} MapClient;
2089
2090static QLIST_HEAD(map_client_list, MapClient) map_client_list
2091 = QLIST_HEAD_INITIALIZER(map_client_list);
2092
2093void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2094{
2095 MapClient *client = g_malloc(sizeof(*client));
2096
2097 client->opaque = opaque;
2098 client->callback = callback;
2099 QLIST_INSERT_HEAD(&map_client_list, client, link);
2100 return client;
2101}
2102
2103static void cpu_unregister_map_client(void *_client)
2104{
2105 MapClient *client = (MapClient *)_client;
2106
2107 QLIST_REMOVE(client, link);
2108 g_free(client);
2109}
2110
2111static void cpu_notify_map_clients(void)
2112{
2113 MapClient *client;
2114
2115 while (!QLIST_EMPTY(&map_client_list)) {
2116 client = QLIST_FIRST(&map_client_list);
2117 client->callback(client->opaque);
2118 cpu_unregister_map_client(client);
2119 }
2120}
2121
2122bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2123{
2124 MemoryRegion *mr;
2125 hwaddr l, xlat;
2126
2127 while (len > 0) {
2128 l = len;
2129 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2130 if (!memory_access_is_direct(mr, is_write)) {
2131 l = memory_access_size(mr, l, addr);
2132 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2133 return false;
2134 }
2135 }
2136
2137 len -= l;
2138 addr += l;
2139 }
2140 return true;
2141}
2142
2143/* Map a physical memory region into a host virtual address.
2144 * May map a subset of the requested range, given by and returned in *plen.
2145 * May return NULL if resources needed to perform the mapping are exhausted.
2146 * Use only for reads OR writes - not for read-modify-write operations.
2147 * Use cpu_register_map_client() to know when retrying the map operation is
2148 * likely to succeed.
2149 */
2150void *address_space_map(AddressSpace *as,
2151 hwaddr addr,
2152 hwaddr *plen,
2153 bool is_write)
2154{
2155 hwaddr len = *plen;
2156 hwaddr done = 0;
2157 hwaddr l, xlat, base;
2158 MemoryRegion *mr, *this_mr;
2159 ram_addr_t raddr;
2160
2161 if (len == 0) {
2162 return NULL;
2163 }
2164
2165 l = len;
2166 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2167 if (!memory_access_is_direct(mr, is_write)) {
2168 if (bounce.buffer) {
2169 return NULL;
2170 }
2171 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2172 bounce.addr = addr;
2173 bounce.len = l;
2174
2175 memory_region_ref(mr);
2176 bounce.mr = mr;
2177 if (!is_write) {
2178 address_space_read(as, addr, bounce.buffer, l);
2179 }
2180
2181 *plen = l;
2182 return bounce.buffer;
2183 }
2184
2185 base = xlat;
2186 raddr = memory_region_get_ram_addr(mr);
2187
2188 for (;;) {
2189 len -= l;
2190 addr += l;
2191 done += l;
2192 if (len == 0) {
2193 break;
2194 }
2195
2196 l = len;
2197 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2198 if (this_mr != mr || xlat != base + done) {
2199 break;
2200 }
2201 }
2202
2203 memory_region_ref(mr);
2204 *plen = done;
2205 return qemu_ram_ptr_length(raddr + base, plen);
2206}
2207
2208/* Unmaps a memory region previously mapped by address_space_map().
2209 * Will also mark the memory as dirty if is_write == 1. access_len gives
2210 * the amount of memory that was actually read or written by the caller.
2211 */
2212void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2213 int is_write, hwaddr access_len)
2214{
2215 if (buffer != bounce.buffer) {
2216 MemoryRegion *mr;
2217 ram_addr_t addr1;
2218
2219 mr = qemu_ram_addr_from_host(buffer, &addr1);
2220 assert(mr != NULL);
2221 if (is_write) {
2222 while (access_len) {
2223 unsigned l;
2224 l = TARGET_PAGE_SIZE;
2225 if (l > access_len)
2226 l = access_len;
2227 invalidate_and_set_dirty(addr1, l);
2228 addr1 += l;
2229 access_len -= l;
2230 }
2231 }
2232 if (xen_enabled()) {
2233 xen_invalidate_map_cache_entry(buffer);
2234 }
2235 memory_region_unref(mr);
2236 return;
2237 }
2238 if (is_write) {
2239 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2240 }
2241 qemu_vfree(bounce.buffer);
2242 bounce.buffer = NULL;
2243 memory_region_unref(bounce.mr);
2244 cpu_notify_map_clients();
2245}
2246
2247void *cpu_physical_memory_map(hwaddr addr,
2248 hwaddr *plen,
2249 int is_write)
2250{
2251 return address_space_map(&address_space_memory, addr, plen, is_write);
2252}
2253
2254void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2255 int is_write, hwaddr access_len)
2256{
2257 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2258}
2259
2260/* warning: addr must be aligned */
2261static inline uint32_t ldl_phys_internal(hwaddr addr,
2262 enum device_endian endian)
2263{
2264 uint8_t *ptr;
2265 uint64_t val;
2266 MemoryRegion *mr;
2267 hwaddr l = 4;
2268 hwaddr addr1;
2269
2270 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2271 false);
2272 if (l < 4 || !memory_access_is_direct(mr, false)) {
2273 /* I/O case */
2274 io_mem_read(mr, addr1, &val, 4);
2275#if defined(TARGET_WORDS_BIGENDIAN)
2276 if (endian == DEVICE_LITTLE_ENDIAN) {
2277 val = bswap32(val);
2278 }
2279#else
2280 if (endian == DEVICE_BIG_ENDIAN) {
2281 val = bswap32(val);
2282 }
2283#endif
2284 } else {
2285 /* RAM case */
2286 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2287 & TARGET_PAGE_MASK)
2288 + addr1);
2289 switch (endian) {
2290 case DEVICE_LITTLE_ENDIAN:
2291 val = ldl_le_p(ptr);
2292 break;
2293 case DEVICE_BIG_ENDIAN:
2294 val = ldl_be_p(ptr);
2295 break;
2296 default:
2297 val = ldl_p(ptr);
2298 break;
2299 }
2300 }
2301 return val;
2302}
2303
2304uint32_t ldl_phys(hwaddr addr)
2305{
2306 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2307}
2308
2309uint32_t ldl_le_phys(hwaddr addr)
2310{
2311 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2312}
2313
2314uint32_t ldl_be_phys(hwaddr addr)
2315{
2316 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2317}
2318
2319/* warning: addr must be aligned */
2320static inline uint64_t ldq_phys_internal(hwaddr addr,
2321 enum device_endian endian)
2322{
2323 uint8_t *ptr;
2324 uint64_t val;
2325 MemoryRegion *mr;
2326 hwaddr l = 8;
2327 hwaddr addr1;
2328
2329 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2330 false);
2331 if (l < 8 || !memory_access_is_direct(mr, false)) {
2332 /* I/O case */
2333 io_mem_read(mr, addr1, &val, 8);
2334#if defined(TARGET_WORDS_BIGENDIAN)
2335 if (endian == DEVICE_LITTLE_ENDIAN) {
2336 val = bswap64(val);
2337 }
2338#else
2339 if (endian == DEVICE_BIG_ENDIAN) {
2340 val = bswap64(val);
2341 }
2342#endif
2343 } else {
2344 /* RAM case */
2345 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2346 & TARGET_PAGE_MASK)
2347 + addr1);
2348 switch (endian) {
2349 case DEVICE_LITTLE_ENDIAN:
2350 val = ldq_le_p(ptr);
2351 break;
2352 case DEVICE_BIG_ENDIAN:
2353 val = ldq_be_p(ptr);
2354 break;
2355 default:
2356 val = ldq_p(ptr);
2357 break;
2358 }
2359 }
2360 return val;
2361}
2362
2363uint64_t ldq_phys(hwaddr addr)
2364{
2365 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2366}
2367
2368uint64_t ldq_le_phys(hwaddr addr)
2369{
2370 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2371}
2372
2373uint64_t ldq_be_phys(hwaddr addr)
2374{
2375 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2376}
2377
2378/* XXX: optimize */
2379uint32_t ldub_phys(hwaddr addr)
2380{
2381 uint8_t val;
2382 cpu_physical_memory_read(addr, &val, 1);
2383 return val;
2384}
2385
2386/* warning: addr must be aligned */
2387static inline uint32_t lduw_phys_internal(hwaddr addr,
2388 enum device_endian endian)
2389{
2390 uint8_t *ptr;
2391 uint64_t val;
2392 MemoryRegion *mr;
2393 hwaddr l = 2;
2394 hwaddr addr1;
2395
2396 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2397 false);
2398 if (l < 2 || !memory_access_is_direct(mr, false)) {
2399 /* I/O case */
2400 io_mem_read(mr, addr1, &val, 2);
2401#if defined(TARGET_WORDS_BIGENDIAN)
2402 if (endian == DEVICE_LITTLE_ENDIAN) {
2403 val = bswap16(val);
2404 }
2405#else
2406 if (endian == DEVICE_BIG_ENDIAN) {
2407 val = bswap16(val);
2408 }
2409#endif
2410 } else {
2411 /* RAM case */
2412 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2413 & TARGET_PAGE_MASK)
2414 + addr1);
2415 switch (endian) {
2416 case DEVICE_LITTLE_ENDIAN:
2417 val = lduw_le_p(ptr);
2418 break;
2419 case DEVICE_BIG_ENDIAN:
2420 val = lduw_be_p(ptr);
2421 break;
2422 default:
2423 val = lduw_p(ptr);
2424 break;
2425 }
2426 }
2427 return val;
2428}
2429
2430uint32_t lduw_phys(hwaddr addr)
2431{
2432 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2433}
2434
2435uint32_t lduw_le_phys(hwaddr addr)
2436{
2437 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2438}
2439
2440uint32_t lduw_be_phys(hwaddr addr)
2441{
2442 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2443}
2444
2445/* warning: addr must be aligned. The ram page is not masked as dirty
2446 and the code inside is not invalidated. It is useful if the dirty
2447 bits are used to track modified PTEs */
2448void stl_phys_notdirty(hwaddr addr, uint32_t val)
2449{
2450 uint8_t *ptr;
2451 MemoryRegion *mr;
2452 hwaddr l = 4;
2453 hwaddr addr1;
2454
2455 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2456 true);
2457 if (l < 4 || !memory_access_is_direct(mr, true)) {
2458 io_mem_write(mr, addr1, val, 4);
2459 } else {
2460 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2461 ptr = qemu_get_ram_ptr(addr1);
2462 stl_p(ptr, val);
2463
2464 if (unlikely(in_migration)) {
2465 if (!cpu_physical_memory_is_dirty(addr1)) {
2466 /* invalidate code */
2467 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2468 /* set dirty bit */
2469 cpu_physical_memory_set_dirty_flags(
2470 addr1, (0xff & ~CODE_DIRTY_FLAG));
2471 }
2472 }
2473 }
2474}
2475
2476/* warning: addr must be aligned */
2477static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2478 enum device_endian endian)
2479{
2480 uint8_t *ptr;
2481 MemoryRegion *mr;
2482 hwaddr l = 4;
2483 hwaddr addr1;
2484
2485 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2486 true);
2487 if (l < 4 || !memory_access_is_direct(mr, true)) {
2488#if defined(TARGET_WORDS_BIGENDIAN)
2489 if (endian == DEVICE_LITTLE_ENDIAN) {
2490 val = bswap32(val);
2491 }
2492#else
2493 if (endian == DEVICE_BIG_ENDIAN) {
2494 val = bswap32(val);
2495 }
2496#endif
2497 io_mem_write(mr, addr1, val, 4);
2498 } else {
2499 /* RAM case */
2500 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2501 ptr = qemu_get_ram_ptr(addr1);
2502 switch (endian) {
2503 case DEVICE_LITTLE_ENDIAN:
2504 stl_le_p(ptr, val);
2505 break;
2506 case DEVICE_BIG_ENDIAN:
2507 stl_be_p(ptr, val);
2508 break;
2509 default:
2510 stl_p(ptr, val);
2511 break;
2512 }
2513 invalidate_and_set_dirty(addr1, 4);
2514 }
2515}
2516
2517void stl_phys(hwaddr addr, uint32_t val)
2518{
2519 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2520}
2521
2522void stl_le_phys(hwaddr addr, uint32_t val)
2523{
2524 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2525}
2526
2527void stl_be_phys(hwaddr addr, uint32_t val)
2528{
2529 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2530}
2531
2532/* XXX: optimize */
2533void stb_phys(hwaddr addr, uint32_t val)
2534{
2535 uint8_t v = val;
2536 cpu_physical_memory_write(addr, &v, 1);
2537}
2538
2539/* warning: addr must be aligned */
2540static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2541 enum device_endian endian)
2542{
2543 uint8_t *ptr;
2544 MemoryRegion *mr;
2545 hwaddr l = 2;
2546 hwaddr addr1;
2547
2548 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2549 true);
2550 if (l < 2 || !memory_access_is_direct(mr, true)) {
2551#if defined(TARGET_WORDS_BIGENDIAN)
2552 if (endian == DEVICE_LITTLE_ENDIAN) {
2553 val = bswap16(val);
2554 }
2555#else
2556 if (endian == DEVICE_BIG_ENDIAN) {
2557 val = bswap16(val);
2558 }
2559#endif
2560 io_mem_write(mr, addr1, val, 2);
2561 } else {
2562 /* RAM case */
2563 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2564 ptr = qemu_get_ram_ptr(addr1);
2565 switch (endian) {
2566 case DEVICE_LITTLE_ENDIAN:
2567 stw_le_p(ptr, val);
2568 break;
2569 case DEVICE_BIG_ENDIAN:
2570 stw_be_p(ptr, val);
2571 break;
2572 default:
2573 stw_p(ptr, val);
2574 break;
2575 }
2576 invalidate_and_set_dirty(addr1, 2);
2577 }
2578}
2579
2580void stw_phys(hwaddr addr, uint32_t val)
2581{
2582 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2583}
2584
2585void stw_le_phys(hwaddr addr, uint32_t val)
2586{
2587 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2588}
2589
2590void stw_be_phys(hwaddr addr, uint32_t val)
2591{
2592 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2593}
2594
2595/* XXX: optimize */
2596void stq_phys(hwaddr addr, uint64_t val)
2597{
2598 val = tswap64(val);
2599 cpu_physical_memory_write(addr, &val, 8);
2600}
2601
2602void stq_le_phys(hwaddr addr, uint64_t val)
2603{
2604 val = cpu_to_le64(val);
2605 cpu_physical_memory_write(addr, &val, 8);
2606}
2607
2608void stq_be_phys(hwaddr addr, uint64_t val)
2609{
2610 val = cpu_to_be64(val);
2611 cpu_physical_memory_write(addr, &val, 8);
2612}
2613
2614/* virtual memory access for debug (includes writing to ROM) */
2615int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2616 uint8_t *buf, int len, int is_write)
2617{
2618 int l;
2619 hwaddr phys_addr;
2620 target_ulong page;
2621
2622 while (len > 0) {
2623 page = addr & TARGET_PAGE_MASK;
2624 phys_addr = cpu_get_phys_page_debug(cpu, page);
2625 /* if no physical page mapped, return an error */
2626 if (phys_addr == -1)
2627 return -1;
2628 l = (page + TARGET_PAGE_SIZE) - addr;
2629 if (l > len)
2630 l = len;
2631 phys_addr += (addr & ~TARGET_PAGE_MASK);
2632 if (is_write)
2633 cpu_physical_memory_write_rom(phys_addr, buf, l);
2634 else
2635 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2636 len -= l;
2637 buf += l;
2638 addr += l;
2639 }
2640 return 0;
2641}
2642#endif
2643
2644#if !defined(CONFIG_USER_ONLY)
2645
2646/*
2647 * A helper function for the _utterly broken_ virtio device model to find out if
2648 * it's running on a big endian machine. Don't do this at home kids!
2649 */
2650bool virtio_is_big_endian(void);
2651bool virtio_is_big_endian(void)
2652{
2653#if defined(TARGET_WORDS_BIGENDIAN)
2654 return true;
2655#else
2656 return false;
2657#endif
2658}
2659
2660#endif
2661
2662#ifndef CONFIG_USER_ONLY
2663bool cpu_physical_memory_is_io(hwaddr phys_addr)
2664{
2665 MemoryRegion*mr;
2666 hwaddr l = 1;
2667
2668 mr = address_space_translate(&address_space_memory,
2669 phys_addr, &phys_addr, &l, false);
2670
2671 return !(memory_region_is_ram(mr) ||
2672 memory_region_is_romd(mr));
2673}
2674
2675void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2676{
2677 RAMBlock *block;
2678
2679 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2680 func(block->host, block->offset, block->length, opaque);
2681 }
2682}
2683#endif
QEMU mirror