4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
21 #include <sys/types.h>
25 #include "qemu-common.h"
30 #include "qemu/osdep.h"
31 #include "sysemu/kvm.h"
32 #include "sysemu/sysemu.h"
33 #include "hw/xen/xen.h"
34 #include "qemu/timer.h"
35 #include "qemu/config-file.h"
36 #include "qemu/error-report.h"
37 #include "exec/memory.h"
38 #include "sysemu/dma.h"
39 #include "exec/address-spaces.h"
40 #if defined(CONFIG_USER_ONLY)
42 #else /* !CONFIG_USER_ONLY */
43 #include "sysemu/xen-mapcache.h"
46 #include "exec/cpu-all.h"
48 #include "exec/cputlb.h"
49 #include "translate-all.h"
51 #include "exec/memory-internal.h"
52 #include "exec/ram_addr.h"
54 #include "qemu/range.h"
56 //#define DEBUG_SUBPAGE
58 #if !defined(CONFIG_USER_ONLY)
59 static bool in_migration
;
61 RAMList ram_list
= { .blocks
= QTAILQ_HEAD_INITIALIZER(ram_list
.blocks
) };
63 static MemoryRegion
*system_memory
;
64 static MemoryRegion
*system_io
;
66 AddressSpace address_space_io
;
67 AddressSpace address_space_memory
;
69 MemoryRegion io_mem_rom
, io_mem_notdirty
;
70 static MemoryRegion io_mem_unassigned
;
72 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
73 #define RAM_PREALLOC (1 << 0)
75 /* RAM is mmap-ed with MAP_SHARED */
76 #define RAM_SHARED (1 << 1)
78 /* Only a portion of RAM (used_length) is actually used, and migrated.
79 * This used_length size can change across reboots.
81 #define RAM_RESIZEABLE (1 << 2)
85 struct CPUTailQ cpus
= QTAILQ_HEAD_INITIALIZER(cpus
);
86 /* current CPU in the current thread. It is only valid inside
88 DEFINE_TLS(CPUState
*, current_cpu
);
89 /* 0 = Do not count executed instructions.
90 1 = Precise instruction counting.
91 2 = Adaptive rate instruction counting. */
94 #if !defined(CONFIG_USER_ONLY)
96 typedef struct PhysPageEntry PhysPageEntry
;
98 struct PhysPageEntry
{
99 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
101 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
105 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
107 /* Size of the L2 (and L3, etc) page tables. */
108 #define ADDR_SPACE_BITS 64
111 #define P_L2_SIZE (1 << P_L2_BITS)
113 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
115 typedef PhysPageEntry Node
[P_L2_SIZE
];
117 typedef struct PhysPageMap
{
120 unsigned sections_nb
;
121 unsigned sections_nb_alloc
;
123 unsigned nodes_nb_alloc
;
125 MemoryRegionSection
*sections
;
128 struct AddressSpaceDispatch
{
131 /* This is a multi-level map on the physical address space.
132 * The bottom level has pointers to MemoryRegionSections.
134 PhysPageEntry phys_map
;
139 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
140 typedef struct subpage_t
{
144 uint16_t sub_section
[TARGET_PAGE_SIZE
];
147 #define PHYS_SECTION_UNASSIGNED 0
148 #define PHYS_SECTION_NOTDIRTY 1
149 #define PHYS_SECTION_ROM 2
150 #define PHYS_SECTION_WATCH 3
152 static void io_mem_init(void);
153 static void memory_map_init(void);
154 static void tcg_commit(MemoryListener
*listener
);
156 static MemoryRegion io_mem_watch
;
159 #if !defined(CONFIG_USER_ONLY)
161 static void phys_map_node_reserve(PhysPageMap
*map
, unsigned nodes
)
163 if (map
->nodes_nb
+ nodes
> map
->nodes_nb_alloc
) {
164 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
* 2, 16);
165 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
, map
->nodes_nb
+ nodes
);
166 map
->nodes
= g_renew(Node
, map
->nodes
, map
->nodes_nb_alloc
);
170 static uint32_t phys_map_node_alloc(PhysPageMap
*map
)
175 ret
= map
->nodes_nb
++;
176 assert(ret
!= PHYS_MAP_NODE_NIL
);
177 assert(ret
!= map
->nodes_nb_alloc
);
178 for (i
= 0; i
< P_L2_SIZE
; ++i
) {
179 map
->nodes
[ret
][i
].skip
= 1;
180 map
->nodes
[ret
][i
].ptr
= PHYS_MAP_NODE_NIL
;
185 static void phys_page_set_level(PhysPageMap
*map
, PhysPageEntry
*lp
,
186 hwaddr
*index
, hwaddr
*nb
, uint16_t leaf
,
191 hwaddr step
= (hwaddr
)1 << (level
* P_L2_BITS
);
193 if (lp
->skip
&& lp
->ptr
== PHYS_MAP_NODE_NIL
) {
194 lp
->ptr
= phys_map_node_alloc(map
);
195 p
= map
->nodes
[lp
->ptr
];
197 for (i
= 0; i
< P_L2_SIZE
; i
++) {
199 p
[i
].ptr
= PHYS_SECTION_UNASSIGNED
;
203 p
= map
->nodes
[lp
->ptr
];
205 lp
= &p
[(*index
>> (level
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
207 while (*nb
&& lp
< &p
[P_L2_SIZE
]) {
208 if ((*index
& (step
- 1)) == 0 && *nb
>= step
) {
214 phys_page_set_level(map
, lp
, index
, nb
, leaf
, level
- 1);
220 static void phys_page_set(AddressSpaceDispatch
*d
,
221 hwaddr index
, hwaddr nb
,
224 /* Wildly overreserve - it doesn't matter much. */
225 phys_map_node_reserve(&d
->map
, 3 * P_L2_LEVELS
);
227 phys_page_set_level(&d
->map
, &d
->phys_map
, &index
, &nb
, leaf
, P_L2_LEVELS
- 1);
230 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
231 * and update our entry so we can skip it and go directly to the destination.
233 static void phys_page_compact(PhysPageEntry
*lp
, Node
*nodes
, unsigned long *compacted
)
235 unsigned valid_ptr
= P_L2_SIZE
;
240 if (lp
->ptr
== PHYS_MAP_NODE_NIL
) {
245 for (i
= 0; i
< P_L2_SIZE
; i
++) {
246 if (p
[i
].ptr
== PHYS_MAP_NODE_NIL
) {
253 phys_page_compact(&p
[i
], nodes
, compacted
);
257 /* We can only compress if there's only one child. */
262 assert(valid_ptr
< P_L2_SIZE
);
264 /* Don't compress if it won't fit in the # of bits we have. */
265 if (lp
->skip
+ p
[valid_ptr
].skip
>= (1 << 3)) {
269 lp
->ptr
= p
[valid_ptr
].ptr
;
270 if (!p
[valid_ptr
].skip
) {
271 /* If our only child is a leaf, make this a leaf. */
272 /* By design, we should have made this node a leaf to begin with so we
273 * should never reach here.
274 * But since it's so simple to handle this, let's do it just in case we
279 lp
->skip
+= p
[valid_ptr
].skip
;
283 static void phys_page_compact_all(AddressSpaceDispatch
*d
, int nodes_nb
)
285 DECLARE_BITMAP(compacted
, nodes_nb
);
287 if (d
->phys_map
.skip
) {
288 phys_page_compact(&d
->phys_map
, d
->map
.nodes
, compacted
);
292 static MemoryRegionSection
*phys_page_find(PhysPageEntry lp
, hwaddr addr
,
293 Node
*nodes
, MemoryRegionSection
*sections
)
296 hwaddr index
= addr
>> TARGET_PAGE_BITS
;
299 for (i
= P_L2_LEVELS
; lp
.skip
&& (i
-= lp
.skip
) >= 0;) {
300 if (lp
.ptr
== PHYS_MAP_NODE_NIL
) {
301 return §ions
[PHYS_SECTION_UNASSIGNED
];
304 lp
= p
[(index
>> (i
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
307 if (sections
[lp
.ptr
].size
.hi
||
308 range_covers_byte(sections
[lp
.ptr
].offset_within_address_space
,
309 sections
[lp
.ptr
].size
.lo
, addr
)) {
310 return §ions
[lp
.ptr
];
312 return §ions
[PHYS_SECTION_UNASSIGNED
];
316 bool memory_region_is_unassigned(MemoryRegion
*mr
)
318 return mr
!= &io_mem_rom
&& mr
!= &io_mem_notdirty
&& !mr
->rom_device
319 && mr
!= &io_mem_watch
;
322 /* Called from RCU critical section */
323 static MemoryRegionSection
*address_space_lookup_region(AddressSpaceDispatch
*d
,
325 bool resolve_subpage
)
327 MemoryRegionSection
*section
;
330 section
= phys_page_find(d
->phys_map
, addr
, d
->map
.nodes
, d
->map
.sections
);
331 if (resolve_subpage
&& section
->mr
->subpage
) {
332 subpage
= container_of(section
->mr
, subpage_t
, iomem
);
333 section
= &d
->map
.sections
[subpage
->sub_section
[SUBPAGE_IDX(addr
)]];
338 /* Called from RCU critical section */
339 static MemoryRegionSection
*
340 address_space_translate_internal(AddressSpaceDispatch
*d
, hwaddr addr
, hwaddr
*xlat
,
341 hwaddr
*plen
, bool resolve_subpage
)
343 MemoryRegionSection
*section
;
346 section
= address_space_lookup_region(d
, addr
, resolve_subpage
);
347 /* Compute offset within MemoryRegionSection */
348 addr
-= section
->offset_within_address_space
;
350 /* Compute offset within MemoryRegion */
351 *xlat
= addr
+ section
->offset_within_region
;
353 diff
= int128_sub(section
->mr
->size
, int128_make64(addr
));
354 *plen
= int128_get64(int128_min(diff
, int128_make64(*plen
)));
358 static inline bool memory_access_is_direct(MemoryRegion
*mr
, bool is_write
)
360 if (memory_region_is_ram(mr
)) {
361 return !(is_write
&& mr
->readonly
);
363 if (memory_region_is_romd(mr
)) {
370 MemoryRegion
*address_space_translate(AddressSpace
*as
, hwaddr addr
,
371 hwaddr
*xlat
, hwaddr
*plen
,
375 MemoryRegionSection
*section
;
381 AddressSpaceDispatch
*d
= atomic_rcu_read(&as
->dispatch
);
382 section
= address_space_translate_internal(d
, addr
, &addr
, plen
, true);
385 if (!mr
->iommu_ops
) {
389 iotlb
= mr
->iommu_ops
->translate(mr
, addr
, is_write
);
390 addr
= ((iotlb
.translated_addr
& ~iotlb
.addr_mask
)
391 | (addr
& iotlb
.addr_mask
));
392 len
= MIN(len
, (addr
| iotlb
.addr_mask
) - addr
+ 1);
393 if (!(iotlb
.perm
& (1 << is_write
))) {
394 mr
= &io_mem_unassigned
;
398 as
= iotlb
.target_as
;
401 if (xen_enabled() && memory_access_is_direct(mr
, is_write
)) {
402 hwaddr page
= ((addr
& TARGET_PAGE_MASK
) + TARGET_PAGE_SIZE
) - addr
;
403 len
= MIN(page
, len
);
412 /* Called from RCU critical section */
413 MemoryRegionSection
*
414 address_space_translate_for_iotlb(CPUState
*cpu
, hwaddr addr
,
415 hwaddr
*xlat
, hwaddr
*plen
)
417 MemoryRegionSection
*section
;
418 section
= address_space_translate_internal(cpu
->memory_dispatch
,
419 addr
, xlat
, plen
, false);
421 assert(!section
->mr
->iommu_ops
);
426 void cpu_exec_init_all(void)
428 #if !defined(CONFIG_USER_ONLY)
429 qemu_mutex_init(&ram_list
.mutex
);
435 #if !defined(CONFIG_USER_ONLY)
437 static int cpu_common_post_load(void *opaque
, int version_id
)
439 CPUState
*cpu
= opaque
;
441 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
442 version_id is increased. */
443 cpu
->interrupt_request
&= ~0x01;
449 static int cpu_common_pre_load(void *opaque
)
451 CPUState
*cpu
= opaque
;
453 cpu
->exception_index
= -1;
458 static bool cpu_common_exception_index_needed(void *opaque
)
460 CPUState
*cpu
= opaque
;
462 return tcg_enabled() && cpu
->exception_index
!= -1;
465 static const VMStateDescription vmstate_cpu_common_exception_index
= {
466 .name
= "cpu_common/exception_index",
468 .minimum_version_id
= 1,
469 .fields
= (VMStateField
[]) {
470 VMSTATE_INT32(exception_index
, CPUState
),
471 VMSTATE_END_OF_LIST()
475 const VMStateDescription vmstate_cpu_common
= {
476 .name
= "cpu_common",
478 .minimum_version_id
= 1,
479 .pre_load
= cpu_common_pre_load
,
480 .post_load
= cpu_common_post_load
,
481 .fields
= (VMStateField
[]) {
482 VMSTATE_UINT32(halted
, CPUState
),
483 VMSTATE_UINT32(interrupt_request
, CPUState
),
484 VMSTATE_END_OF_LIST()
486 .subsections
= (VMStateSubsection
[]) {
488 .vmsd
= &vmstate_cpu_common_exception_index
,
489 .needed
= cpu_common_exception_index_needed
,
498 CPUState
*qemu_get_cpu(int index
)
503 if (cpu
->cpu_index
== index
) {
511 #if !defined(CONFIG_USER_ONLY)
512 void tcg_cpu_address_space_init(CPUState
*cpu
, AddressSpace
*as
)
514 /* We only support one address space per cpu at the moment. */
515 assert(cpu
->as
== as
);
517 if (cpu
->tcg_as_listener
) {
518 memory_listener_unregister(cpu
->tcg_as_listener
);
520 cpu
->tcg_as_listener
= g_new0(MemoryListener
, 1);
522 cpu
->tcg_as_listener
->commit
= tcg_commit
;
523 memory_listener_register(cpu
->tcg_as_listener
, as
);
527 void cpu_exec_init(CPUArchState
*env
)
529 CPUState
*cpu
= ENV_GET_CPU(env
);
530 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
534 #if defined(CONFIG_USER_ONLY)
538 CPU_FOREACH(some_cpu
) {
541 cpu
->cpu_index
= cpu_index
;
543 QTAILQ_INIT(&cpu
->breakpoints
);
544 QTAILQ_INIT(&cpu
->watchpoints
);
545 #ifndef CONFIG_USER_ONLY
546 cpu
->as
= &address_space_memory
;
547 cpu
->thread_id
= qemu_get_thread_id();
549 QTAILQ_INSERT_TAIL(&cpus
, cpu
, node
);
550 #if defined(CONFIG_USER_ONLY)
553 if (qdev_get_vmsd(DEVICE(cpu
)) == NULL
) {
554 vmstate_register(NULL
, cpu_index
, &vmstate_cpu_common
, cpu
);
556 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
557 register_savevm(NULL
, "cpu", cpu_index
, CPU_SAVE_VERSION
,
558 cpu_save
, cpu_load
, env
);
559 assert(cc
->vmsd
== NULL
);
560 assert(qdev_get_vmsd(DEVICE(cpu
)) == NULL
);
562 if (cc
->vmsd
!= NULL
) {
563 vmstate_register(NULL
, cpu_index
, cc
->vmsd
, cpu
);
567 #if defined(CONFIG_USER_ONLY)
568 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
570 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
573 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
575 hwaddr phys
= cpu_get_phys_page_debug(cpu
, pc
);
577 tb_invalidate_phys_addr(cpu
->as
,
578 phys
| (pc
& ~TARGET_PAGE_MASK
));
583 #if defined(CONFIG_USER_ONLY)
584 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
589 int cpu_watchpoint_remove(CPUState
*cpu
, vaddr addr
, vaddr len
,
595 void cpu_watchpoint_remove_by_ref(CPUState
*cpu
, CPUWatchpoint
*watchpoint
)
599 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
600 int flags
, CPUWatchpoint
**watchpoint
)
605 /* Add a watchpoint. */
606 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
607 int flags
, CPUWatchpoint
**watchpoint
)
611 /* forbid ranges which are empty or run off the end of the address space */
612 if (len
== 0 || (addr
+ len
- 1) < addr
) {
613 error_report("tried to set invalid watchpoint at %"
614 VADDR_PRIx
", len=%" VADDR_PRIu
, addr
, len
);
617 wp
= g_malloc(sizeof(*wp
));
623 /* keep all GDB-injected watchpoints in front */
624 if (flags
& BP_GDB
) {
625 QTAILQ_INSERT_HEAD(&cpu
->watchpoints
, wp
, entry
);
627 QTAILQ_INSERT_TAIL(&cpu
->watchpoints
, wp
, entry
);
630 tlb_flush_page(cpu
, addr
);
637 /* Remove a specific watchpoint. */
638 int cpu_watchpoint_remove(CPUState
*cpu
, vaddr addr
, vaddr len
,
643 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
644 if (addr
== wp
->vaddr
&& len
== wp
->len
645 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
646 cpu_watchpoint_remove_by_ref(cpu
, wp
);
653 /* Remove a specific watchpoint by reference. */
654 void cpu_watchpoint_remove_by_ref(CPUState
*cpu
, CPUWatchpoint
*watchpoint
)
656 QTAILQ_REMOVE(&cpu
->watchpoints
, watchpoint
, entry
);
658 tlb_flush_page(cpu
, watchpoint
->vaddr
);
663 /* Remove all matching watchpoints. */
664 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
666 CPUWatchpoint
*wp
, *next
;
668 QTAILQ_FOREACH_SAFE(wp
, &cpu
->watchpoints
, entry
, next
) {
669 if (wp
->flags
& mask
) {
670 cpu_watchpoint_remove_by_ref(cpu
, wp
);
675 /* Return true if this watchpoint address matches the specified
676 * access (ie the address range covered by the watchpoint overlaps
677 * partially or completely with the address range covered by the
680 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint
*wp
,
684 /* We know the lengths are non-zero, but a little caution is
685 * required to avoid errors in the case where the range ends
686 * exactly at the top of the address space and so addr + len
687 * wraps round to zero.
689 vaddr wpend
= wp
->vaddr
+ wp
->len
- 1;
690 vaddr addrend
= addr
+ len
- 1;
692 return !(addr
> wpend
|| wp
->vaddr
> addrend
);
697 /* Add a breakpoint. */
698 int cpu_breakpoint_insert(CPUState
*cpu
, vaddr pc
, int flags
,
699 CPUBreakpoint
**breakpoint
)
703 bp
= g_malloc(sizeof(*bp
));
708 /* keep all GDB-injected breakpoints in front */
709 if (flags
& BP_GDB
) {
710 QTAILQ_INSERT_HEAD(&cpu
->breakpoints
, bp
, entry
);
712 QTAILQ_INSERT_TAIL(&cpu
->breakpoints
, bp
, entry
);
715 breakpoint_invalidate(cpu
, pc
);
723 /* Remove a specific breakpoint. */
724 int cpu_breakpoint_remove(CPUState
*cpu
, vaddr pc
, int flags
)
728 QTAILQ_FOREACH(bp
, &cpu
->breakpoints
, entry
) {
729 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
730 cpu_breakpoint_remove_by_ref(cpu
, bp
);
737 /* Remove a specific breakpoint by reference. */
738 void cpu_breakpoint_remove_by_ref(CPUState
*cpu
, CPUBreakpoint
*breakpoint
)
740 QTAILQ_REMOVE(&cpu
->breakpoints
, breakpoint
, entry
);
742 breakpoint_invalidate(cpu
, breakpoint
->pc
);
747 /* Remove all matching breakpoints. */
748 void cpu_breakpoint_remove_all(CPUState
*cpu
, int mask
)
750 CPUBreakpoint
*bp
, *next
;
752 QTAILQ_FOREACH_SAFE(bp
, &cpu
->breakpoints
, entry
, next
) {
753 if (bp
->flags
& mask
) {
754 cpu_breakpoint_remove_by_ref(cpu
, bp
);
759 /* enable or disable single step mode. EXCP_DEBUG is returned by the
760 CPU loop after each instruction */
761 void cpu_single_step(CPUState
*cpu
, int enabled
)
763 if (cpu
->singlestep_enabled
!= enabled
) {
764 cpu
->singlestep_enabled
= enabled
;
766 kvm_update_guest_debug(cpu
, 0);
768 /* must flush all the translated code to avoid inconsistencies */
769 /* XXX: only flush what is necessary */
770 CPUArchState
*env
= cpu
->env_ptr
;
776 void cpu_abort(CPUState
*cpu
, const char *fmt
, ...)
783 fprintf(stderr
, "qemu: fatal: ");
784 vfprintf(stderr
, fmt
, ap
);
785 fprintf(stderr
, "\n");
786 cpu_dump_state(cpu
, stderr
, fprintf
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
787 if (qemu_log_enabled()) {
788 qemu_log("qemu: fatal: ");
789 qemu_log_vprintf(fmt
, ap2
);
791 log_cpu_state(cpu
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
797 #if defined(CONFIG_USER_ONLY)
799 struct sigaction act
;
800 sigfillset(&act
.sa_mask
);
801 act
.sa_handler
= SIG_DFL
;
802 sigaction(SIGABRT
, &act
, NULL
);
808 #if !defined(CONFIG_USER_ONLY)
809 static RAMBlock
*qemu_get_ram_block(ram_addr_t addr
)
813 /* The list is protected by the iothread lock here. */
814 block
= atomic_rcu_read(&ram_list
.mru_block
);
815 if (block
&& addr
- block
->offset
< block
->max_length
) {
818 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
819 if (addr
- block
->offset
< block
->max_length
) {
824 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
828 /* It is safe to write mru_block outside the iothread lock. This
833 * xxx removed from list
837 * call_rcu(reclaim_ramblock, xxx);
840 * atomic_rcu_set is not needed here. The block was already published
841 * when it was placed into the list. Here we're just making an extra
842 * copy of the pointer.
844 ram_list
.mru_block
= block
;
848 static void tlb_reset_dirty_range_all(ram_addr_t start
, ram_addr_t length
)
854 end
= TARGET_PAGE_ALIGN(start
+ length
);
855 start
&= TARGET_PAGE_MASK
;
857 block
= qemu_get_ram_block(start
);
858 assert(block
== qemu_get_ram_block(end
- 1));
859 start1
= (uintptr_t)ramblock_ptr(block
, start
- block
->offset
);
860 cpu_tlb_reset_dirty_all(start1
, length
);
863 /* Note: start and end must be within the same ram block. */
864 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t length
,
869 cpu_physical_memory_clear_dirty_range_type(start
, length
, client
);
872 tlb_reset_dirty_range_all(start
, length
);
876 static void cpu_physical_memory_set_dirty_tracking(bool enable
)
878 in_migration
= enable
;
881 /* Called from RCU critical section */
882 hwaddr
memory_region_section_get_iotlb(CPUState
*cpu
,
883 MemoryRegionSection
*section
,
885 hwaddr paddr
, hwaddr xlat
,
887 target_ulong
*address
)
892 if (memory_region_is_ram(section
->mr
)) {
894 iotlb
= (memory_region_get_ram_addr(section
->mr
) & TARGET_PAGE_MASK
)
896 if (!section
->readonly
) {
897 iotlb
|= PHYS_SECTION_NOTDIRTY
;
899 iotlb
|= PHYS_SECTION_ROM
;
902 iotlb
= section
- section
->address_space
->dispatch
->map
.sections
;
906 /* Make accesses to pages with watchpoints go via the
907 watchpoint trap routines. */
908 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
909 if (cpu_watchpoint_address_matches(wp
, vaddr
, TARGET_PAGE_SIZE
)) {
910 /* Avoid trapping reads of pages with a write breakpoint. */
911 if ((prot
& PAGE_WRITE
) || (wp
->flags
& BP_MEM_READ
)) {
912 iotlb
= PHYS_SECTION_WATCH
+ paddr
;
913 *address
|= TLB_MMIO
;
921 #endif /* defined(CONFIG_USER_ONLY) */
923 #if !defined(CONFIG_USER_ONLY)
925 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
927 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
);
929 static void *(*phys_mem_alloc
)(size_t size
, uint64_t *align
) =
933 * Set a custom physical guest memory alloator.
934 * Accelerators with unusual needs may need this. Hopefully, we can
935 * get rid of it eventually.
937 void phys_mem_set_alloc(void *(*alloc
)(size_t, uint64_t *align
))
939 phys_mem_alloc
= alloc
;
942 static uint16_t phys_section_add(PhysPageMap
*map
,
943 MemoryRegionSection
*section
)
945 /* The physical section number is ORed with a page-aligned
946 * pointer to produce the iotlb entries. Thus it should
947 * never overflow into the page-aligned value.
949 assert(map
->sections_nb
< TARGET_PAGE_SIZE
);
951 if (map
->sections_nb
== map
->sections_nb_alloc
) {
952 map
->sections_nb_alloc
= MAX(map
->sections_nb_alloc
* 2, 16);
953 map
->sections
= g_renew(MemoryRegionSection
, map
->sections
,
954 map
->sections_nb_alloc
);
956 map
->sections
[map
->sections_nb
] = *section
;
957 memory_region_ref(section
->mr
);
958 return map
->sections_nb
++;
961 static void phys_section_destroy(MemoryRegion
*mr
)
963 memory_region_unref(mr
);
966 subpage_t
*subpage
= container_of(mr
, subpage_t
, iomem
);
967 object_unref(OBJECT(&subpage
->iomem
));
972 static void phys_sections_free(PhysPageMap
*map
)
974 while (map
->sections_nb
> 0) {
975 MemoryRegionSection
*section
= &map
->sections
[--map
->sections_nb
];
976 phys_section_destroy(section
->mr
);
978 g_free(map
->sections
);
982 static void register_subpage(AddressSpaceDispatch
*d
, MemoryRegionSection
*section
)
985 hwaddr base
= section
->offset_within_address_space
987 MemoryRegionSection
*existing
= phys_page_find(d
->phys_map
, base
,
988 d
->map
.nodes
, d
->map
.sections
);
989 MemoryRegionSection subsection
= {
990 .offset_within_address_space
= base
,
991 .size
= int128_make64(TARGET_PAGE_SIZE
),
995 assert(existing
->mr
->subpage
|| existing
->mr
== &io_mem_unassigned
);
997 if (!(existing
->mr
->subpage
)) {
998 subpage
= subpage_init(d
->as
, base
);
999 subsection
.address_space
= d
->as
;
1000 subsection
.mr
= &subpage
->iomem
;
1001 phys_page_set(d
, base
>> TARGET_PAGE_BITS
, 1,
1002 phys_section_add(&d
->map
, &subsection
));
1004 subpage
= container_of(existing
->mr
, subpage_t
, iomem
);
1006 start
= section
->offset_within_address_space
& ~TARGET_PAGE_MASK
;
1007 end
= start
+ int128_get64(section
->size
) - 1;
1008 subpage_register(subpage
, start
, end
,
1009 phys_section_add(&d
->map
, section
));
1013 static void register_multipage(AddressSpaceDispatch
*d
,
1014 MemoryRegionSection
*section
)
1016 hwaddr start_addr
= section
->offset_within_address_space
;
1017 uint16_t section_index
= phys_section_add(&d
->map
, section
);
1018 uint64_t num_pages
= int128_get64(int128_rshift(section
->size
,
1022 phys_page_set(d
, start_addr
>> TARGET_PAGE_BITS
, num_pages
, section_index
);
1025 static void mem_add(MemoryListener
*listener
, MemoryRegionSection
*section
)
1027 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
1028 AddressSpaceDispatch
*d
= as
->next_dispatch
;
1029 MemoryRegionSection now
= *section
, remain
= *section
;
1030 Int128 page_size
= int128_make64(TARGET_PAGE_SIZE
);
1032 if (now
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
1033 uint64_t left
= TARGET_PAGE_ALIGN(now
.offset_within_address_space
)
1034 - now
.offset_within_address_space
;
1036 now
.size
= int128_min(int128_make64(left
), now
.size
);
1037 register_subpage(d
, &now
);
1039 now
.size
= int128_zero();
1041 while (int128_ne(remain
.size
, now
.size
)) {
1042 remain
.size
= int128_sub(remain
.size
, now
.size
);
1043 remain
.offset_within_address_space
+= int128_get64(now
.size
);
1044 remain
.offset_within_region
+= int128_get64(now
.size
);
1046 if (int128_lt(remain
.size
, page_size
)) {
1047 register_subpage(d
, &now
);
1048 } else if (remain
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
1049 now
.size
= page_size
;
1050 register_subpage(d
, &now
);
1052 now
.size
= int128_and(now
.size
, int128_neg(page_size
));
1053 register_multipage(d
, &now
);
1058 void qemu_flush_coalesced_mmio_buffer(void)
1061 kvm_flush_coalesced_mmio_buffer();
1064 void qemu_mutex_lock_ramlist(void)
1066 qemu_mutex_lock(&ram_list
.mutex
);
1069 void qemu_mutex_unlock_ramlist(void)
1071 qemu_mutex_unlock(&ram_list
.mutex
);
1076 #include <sys/vfs.h>
1078 #define HUGETLBFS_MAGIC 0x958458f6
1080 static long gethugepagesize(const char *path
, Error
**errp
)
1086 ret
= statfs(path
, &fs
);
1087 } while (ret
!= 0 && errno
== EINTR
);
1090 error_setg_errno(errp
, errno
, "failed to get page size of file %s",
1095 if (fs
.f_type
!= HUGETLBFS_MAGIC
)
1096 fprintf(stderr
, "Warning: path not on HugeTLBFS: %s\n", path
);
1101 static void *file_ram_alloc(RAMBlock
*block
,
1107 char *sanitized_name
;
1112 Error
*local_err
= NULL
;
1114 hpagesize
= gethugepagesize(path
, &local_err
);
1116 error_propagate(errp
, local_err
);
1119 block
->mr
->align
= hpagesize
;
1121 if (memory
< hpagesize
) {
1122 error_setg(errp
, "memory size 0x" RAM_ADDR_FMT
" must be equal to "
1123 "or larger than huge page size 0x%" PRIx64
,
1128 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1130 "host lacks kvm mmu notifiers, -mem-path unsupported");
1134 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1135 sanitized_name
= g_strdup(memory_region_name(block
->mr
));
1136 for (c
= sanitized_name
; *c
!= '\0'; c
++) {
1141 filename
= g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path
,
1143 g_free(sanitized_name
);
1145 fd
= mkstemp(filename
);
1147 error_setg_errno(errp
, errno
,
1148 "unable to create backing store for hugepages");
1155 memory
= (memory
+hpagesize
-1) & ~(hpagesize
-1);
1158 * ftruncate is not supported by hugetlbfs in older
1159 * hosts, so don't bother bailing out on errors.
1160 * If anything goes wrong with it under other filesystems,
1163 if (ftruncate(fd
, memory
)) {
1164 perror("ftruncate");
1167 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
,
1168 (block
->flags
& RAM_SHARED
? MAP_SHARED
: MAP_PRIVATE
),
1170 if (area
== MAP_FAILED
) {
1171 error_setg_errno(errp
, errno
,
1172 "unable to map backing store for hugepages");
1178 os_mem_prealloc(fd
, area
, memory
);
1186 error_report("%s\n", error_get_pretty(*errp
));
1193 static ram_addr_t
find_ram_offset(ram_addr_t size
)
1195 RAMBlock
*block
, *next_block
;
1196 ram_addr_t offset
= RAM_ADDR_MAX
, mingap
= RAM_ADDR_MAX
;
1198 assert(size
!= 0); /* it would hand out same offset multiple times */
1200 if (QTAILQ_EMPTY(&ram_list
.blocks
))
1203 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1204 ram_addr_t end
, next
= RAM_ADDR_MAX
;
1206 end
= block
->offset
+ block
->max_length
;
1208 QTAILQ_FOREACH(next_block
, &ram_list
.blocks
, next
) {
1209 if (next_block
->offset
>= end
) {
1210 next
= MIN(next
, next_block
->offset
);
1213 if (next
- end
>= size
&& next
- end
< mingap
) {
1215 mingap
= next
- end
;
1219 if (offset
== RAM_ADDR_MAX
) {
1220 fprintf(stderr
, "Failed to find gap of requested size: %" PRIu64
"\n",
1228 ram_addr_t
last_ram_offset(void)
1231 ram_addr_t last
= 0;
1233 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
)
1234 last
= MAX(last
, block
->offset
+ block
->max_length
);
1239 static void qemu_ram_setup_dump(void *addr
, ram_addr_t size
)
1243 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1244 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1245 "dump-guest-core", true)) {
1246 ret
= qemu_madvise(addr
, size
, QEMU_MADV_DONTDUMP
);
1248 perror("qemu_madvise");
1249 fprintf(stderr
, "madvise doesn't support MADV_DONTDUMP, "
1250 "but dump_guest_core=off specified\n");
1255 static RAMBlock
*find_ram_block(ram_addr_t addr
)
1259 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1260 if (block
->offset
== addr
) {
1268 /* Called with iothread lock held. */
1269 void qemu_ram_set_idstr(ram_addr_t addr
, const char *name
, DeviceState
*dev
)
1271 RAMBlock
*new_block
, *block
;
1273 new_block
= find_ram_block(addr
);
1275 assert(!new_block
->idstr
[0]);
1278 char *id
= qdev_get_dev_path(dev
);
1280 snprintf(new_block
->idstr
, sizeof(new_block
->idstr
), "%s/", id
);
1284 pstrcat(new_block
->idstr
, sizeof(new_block
->idstr
), name
);
1286 qemu_mutex_lock_ramlist();
1287 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1288 if (block
!= new_block
&& !strcmp(block
->idstr
, new_block
->idstr
)) {
1289 fprintf(stderr
, "RAMBlock \"%s\" already registered, abort!\n",
1294 qemu_mutex_unlock_ramlist();
1297 /* Called with iothread lock held. */
1298 void qemu_ram_unset_idstr(ram_addr_t addr
)
1302 /* FIXME: arch_init.c assumes that this is not called throughout
1303 * migration. Ignore the problem since hot-unplug during migration
1304 * does not work anyway.
1307 block
= find_ram_block(addr
);
1309 memset(block
->idstr
, 0, sizeof(block
->idstr
));
1313 static int memory_try_enable_merging(void *addr
, size_t len
)
1315 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1316 /* disabled by the user */
1320 return qemu_madvise(addr
, len
, QEMU_MADV_MERGEABLE
);
1323 /* Only legal before guest might have detected the memory size: e.g. on
1324 * incoming migration, or right after reset.
1326 * As memory core doesn't know how is memory accessed, it is up to
1327 * resize callback to update device state and/or add assertions to detect
1328 * misuse, if necessary.
1330 int qemu_ram_resize(ram_addr_t base
, ram_addr_t newsize
, Error
**errp
)
1332 RAMBlock
*block
= find_ram_block(base
);
1336 if (block
->used_length
== newsize
) {
1340 if (!(block
->flags
& RAM_RESIZEABLE
)) {
1341 error_setg_errno(errp
, EINVAL
,
1342 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1343 " in != 0x" RAM_ADDR_FMT
, block
->idstr
,
1344 newsize
, block
->used_length
);
1348 if (block
->max_length
< newsize
) {
1349 error_setg_errno(errp
, EINVAL
,
1350 "Length too large: %s: 0x" RAM_ADDR_FMT
1351 " > 0x" RAM_ADDR_FMT
, block
->idstr
,
1352 newsize
, block
->max_length
);
1356 cpu_physical_memory_clear_dirty_range(block
->offset
, block
->used_length
);
1357 block
->used_length
= newsize
;
1358 cpu_physical_memory_set_dirty_range(block
->offset
, block
->used_length
);
1359 memory_region_set_size(block
->mr
, newsize
);
1360 if (block
->resized
) {
1361 block
->resized(block
->idstr
, newsize
, block
->host
);
1366 static ram_addr_t
ram_block_add(RAMBlock
*new_block
, Error
**errp
)
1369 ram_addr_t old_ram_size
, new_ram_size
;
1371 old_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1373 /* This assumes the iothread lock is taken here too. */
1374 qemu_mutex_lock_ramlist();
1375 new_block
->offset
= find_ram_offset(new_block
->max_length
);
1377 if (!new_block
->host
) {
1378 if (xen_enabled()) {
1379 xen_ram_alloc(new_block
->offset
, new_block
->max_length
,
1382 new_block
->host
= phys_mem_alloc(new_block
->max_length
,
1383 &new_block
->mr
->align
);
1384 if (!new_block
->host
) {
1385 error_setg_errno(errp
, errno
,
1386 "cannot set up guest memory '%s'",
1387 memory_region_name(new_block
->mr
));
1388 qemu_mutex_unlock_ramlist();
1391 memory_try_enable_merging(new_block
->host
, new_block
->max_length
);
1395 /* Keep the list sorted from biggest to smallest block. */
1396 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1397 if (block
->max_length
< new_block
->max_length
) {
1402 QTAILQ_INSERT_BEFORE(block
, new_block
, next
);
1404 QTAILQ_INSERT_TAIL(&ram_list
.blocks
, new_block
, next
);
1406 ram_list
.mru_block
= NULL
;
1409 qemu_mutex_unlock_ramlist();
1411 new_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1413 if (new_ram_size
> old_ram_size
) {
1416 /* ram_list.dirty_memory[] is protected by the iothread lock. */
1417 for (i
= 0; i
< DIRTY_MEMORY_NUM
; i
++) {
1418 ram_list
.dirty_memory
[i
] =
1419 bitmap_zero_extend(ram_list
.dirty_memory
[i
],
1420 old_ram_size
, new_ram_size
);
1423 cpu_physical_memory_set_dirty_range(new_block
->offset
,
1424 new_block
->used_length
);
1426 if (new_block
->host
) {
1427 qemu_ram_setup_dump(new_block
->host
, new_block
->max_length
);
1428 qemu_madvise(new_block
->host
, new_block
->max_length
, QEMU_MADV_HUGEPAGE
);
1429 qemu_madvise(new_block
->host
, new_block
->max_length
, QEMU_MADV_DONTFORK
);
1430 if (kvm_enabled()) {
1431 kvm_setup_guest_memory(new_block
->host
, new_block
->max_length
);
1435 return new_block
->offset
;
1439 ram_addr_t
qemu_ram_alloc_from_file(ram_addr_t size
, MemoryRegion
*mr
,
1440 bool share
, const char *mem_path
,
1443 RAMBlock
*new_block
;
1445 Error
*local_err
= NULL
;
1447 if (xen_enabled()) {
1448 error_setg(errp
, "-mem-path not supported with Xen");
1452 if (phys_mem_alloc
!= qemu_anon_ram_alloc
) {
1454 * file_ram_alloc() needs to allocate just like
1455 * phys_mem_alloc, but we haven't bothered to provide
1459 "-mem-path not supported with this accelerator");
1463 size
= TARGET_PAGE_ALIGN(size
);
1464 new_block
= g_malloc0(sizeof(*new_block
));
1466 new_block
->used_length
= size
;
1467 new_block
->max_length
= size
;
1468 new_block
->flags
= share
? RAM_SHARED
: 0;
1469 new_block
->host
= file_ram_alloc(new_block
, size
,
1471 if (!new_block
->host
) {
1476 addr
= ram_block_add(new_block
, &local_err
);
1479 error_propagate(errp
, local_err
);
1487 ram_addr_t
qemu_ram_alloc_internal(ram_addr_t size
, ram_addr_t max_size
,
1488 void (*resized
)(const char*,
1491 void *host
, bool resizeable
,
1492 MemoryRegion
*mr
, Error
**errp
)
1494 RAMBlock
*new_block
;
1496 Error
*local_err
= NULL
;
1498 size
= TARGET_PAGE_ALIGN(size
);
1499 max_size
= TARGET_PAGE_ALIGN(max_size
);
1500 new_block
= g_malloc0(sizeof(*new_block
));
1502 new_block
->resized
= resized
;
1503 new_block
->used_length
= size
;
1504 new_block
->max_length
= max_size
;
1505 assert(max_size
>= size
);
1507 new_block
->host
= host
;
1509 new_block
->flags
|= RAM_PREALLOC
;
1512 new_block
->flags
|= RAM_RESIZEABLE
;
1514 addr
= ram_block_add(new_block
, &local_err
);
1517 error_propagate(errp
, local_err
);
1523 ram_addr_t
qemu_ram_alloc_from_ptr(ram_addr_t size
, void *host
,
1524 MemoryRegion
*mr
, Error
**errp
)
1526 return qemu_ram_alloc_internal(size
, size
, NULL
, host
, false, mr
, errp
);
1529 ram_addr_t
qemu_ram_alloc(ram_addr_t size
, MemoryRegion
*mr
, Error
**errp
)
1531 return qemu_ram_alloc_internal(size
, size
, NULL
, NULL
, false, mr
, errp
);
1534 ram_addr_t
qemu_ram_alloc_resizeable(ram_addr_t size
, ram_addr_t maxsz
,
1535 void (*resized
)(const char*,
1538 MemoryRegion
*mr
, Error
**errp
)
1540 return qemu_ram_alloc_internal(size
, maxsz
, resized
, NULL
, true, mr
, errp
);
1543 void qemu_ram_free_from_ptr(ram_addr_t addr
)
1547 /* This assumes the iothread lock is taken here too. */
1548 qemu_mutex_lock_ramlist();
1549 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1550 if (addr
== block
->offset
) {
1551 QTAILQ_REMOVE(&ram_list
.blocks
, block
, next
);
1552 ram_list
.mru_block
= NULL
;
1554 g_free_rcu(block
, rcu
);
1558 qemu_mutex_unlock_ramlist();
1561 static void reclaim_ramblock(RAMBlock
*block
)
1563 if (block
->flags
& RAM_PREALLOC
) {
1565 } else if (xen_enabled()) {
1566 xen_invalidate_map_cache_entry(block
->host
);
1568 } else if (block
->fd
>= 0) {
1569 munmap(block
->host
, block
->max_length
);
1573 qemu_anon_ram_free(block
->host
, block
->max_length
);
1578 /* Called with the iothread lock held */
1579 void qemu_ram_free(ram_addr_t addr
)
1583 /* This assumes the iothread lock is taken here too. */
1584 qemu_mutex_lock_ramlist();
1585 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1586 if (addr
== block
->offset
) {
1587 QTAILQ_REMOVE(&ram_list
.blocks
, block
, next
);
1588 ram_list
.mru_block
= NULL
;
1590 call_rcu(block
, reclaim_ramblock
, rcu
);
1594 qemu_mutex_unlock_ramlist();
1598 void qemu_ram_remap(ram_addr_t addr
, ram_addr_t length
)
1605 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1606 offset
= addr
- block
->offset
;
1607 if (offset
< block
->max_length
) {
1608 vaddr
= ramblock_ptr(block
, offset
);
1609 if (block
->flags
& RAM_PREALLOC
) {
1611 } else if (xen_enabled()) {
1615 munmap(vaddr
, length
);
1616 if (block
->fd
>= 0) {
1617 flags
|= (block
->flags
& RAM_SHARED
?
1618 MAP_SHARED
: MAP_PRIVATE
);
1619 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1620 flags
, block
->fd
, offset
);
1623 * Remap needs to match alloc. Accelerators that
1624 * set phys_mem_alloc never remap. If they did,
1625 * we'd need a remap hook here.
1627 assert(phys_mem_alloc
== qemu_anon_ram_alloc
);
1629 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
1630 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1633 if (area
!= vaddr
) {
1634 fprintf(stderr
, "Could not remap addr: "
1635 RAM_ADDR_FMT
"@" RAM_ADDR_FMT
"\n",
1639 memory_try_enable_merging(vaddr
, length
);
1640 qemu_ram_setup_dump(vaddr
, length
);
1645 #endif /* !_WIN32 */
1647 int qemu_get_ram_fd(ram_addr_t addr
)
1652 block
= qemu_get_ram_block(addr
);
1657 void *qemu_get_ram_block_host_ptr(ram_addr_t addr
)
1662 block
= qemu_get_ram_block(addr
);
1663 ptr
= ramblock_ptr(block
, 0);
1667 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1668 * This should not be used for general purpose DMA. Use address_space_map
1669 * or address_space_rw instead. For local memory (e.g. video ram) that the
1670 * device owns, use memory_region_get_ram_ptr.
1672 void *qemu_get_ram_ptr(ram_addr_t addr
)
1677 block
= qemu_get_ram_block(addr
);
1679 if (xen_enabled() && block
->host
== NULL
) {
1680 /* We need to check if the requested address is in the RAM
1681 * because we don't want to map the entire memory in QEMU.
1682 * In that case just map until the end of the page.
1684 if (block
->offset
== 0) {
1685 ptr
= xen_map_cache(addr
, 0, 0);
1689 block
->host
= xen_map_cache(block
->offset
, block
->max_length
, 1);
1691 ptr
= ramblock_ptr(block
, addr
- block
->offset
);
1697 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1698 * but takes a size argument.
1700 static void *qemu_ram_ptr_length(ram_addr_t addr
, hwaddr
*size
)
1706 if (xen_enabled()) {
1707 return xen_map_cache(addr
, *size
, 1);
1710 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1711 if (addr
- block
->offset
< block
->max_length
) {
1712 if (addr
- block
->offset
+ *size
> block
->max_length
)
1713 *size
= block
->max_length
- addr
+ block
->offset
;
1714 ptr
= ramblock_ptr(block
, addr
- block
->offset
);
1719 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
1724 /* Some of the softmmu routines need to translate from a host pointer
1725 * (typically a TLB entry) back to a ram offset.
1727 * By the time this function returns, the returned pointer is not protected
1728 * by RCU anymore. If the caller is not within an RCU critical section and
1729 * does not hold the iothread lock, it must have other means of protecting the
1730 * pointer, such as a reference to the region that includes the incoming
1733 MemoryRegion
*qemu_ram_addr_from_host(void *ptr
, ram_addr_t
*ram_addr
)
1736 uint8_t *host
= ptr
;
1739 if (xen_enabled()) {
1740 *ram_addr
= xen_ram_addr_from_mapcache(ptr
);
1741 mr
= qemu_get_ram_block(*ram_addr
)->mr
;
1745 block
= ram_list
.mru_block
;
1746 if (block
&& block
->host
&& host
- block
->host
< block
->max_length
) {
1750 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
1751 /* This case append when the block is not mapped. */
1752 if (block
->host
== NULL
) {
1755 if (host
- block
->host
< block
->max_length
) {
1763 *ram_addr
= block
->offset
+ (host
- block
->host
);
1768 static void notdirty_mem_write(void *opaque
, hwaddr ram_addr
,
1769 uint64_t val
, unsigned size
)
1771 if (!cpu_physical_memory_get_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
)) {
1772 tb_invalidate_phys_page_fast(ram_addr
, size
);
1776 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
1779 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
1782 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
1787 cpu_physical_memory_set_dirty_range_nocode(ram_addr
, size
);
1788 /* we remove the notdirty callback only if the code has been
1790 if (!cpu_physical_memory_is_clean(ram_addr
)) {
1791 CPUArchState
*env
= current_cpu
->env_ptr
;
1792 tlb_set_dirty(env
, current_cpu
->mem_io_vaddr
);
1796 static bool notdirty_mem_accepts(void *opaque
, hwaddr addr
,
1797 unsigned size
, bool is_write
)
1802 static const MemoryRegionOps notdirty_mem_ops
= {
1803 .write
= notdirty_mem_write
,
1804 .valid
.accepts
= notdirty_mem_accepts
,
1805 .endianness
= DEVICE_NATIVE_ENDIAN
,
1808 /* Generate a debug exception if a watchpoint has been hit. */
1809 static void check_watchpoint(int offset
, int len
, int flags
)
1811 CPUState
*cpu
= current_cpu
;
1812 CPUArchState
*env
= cpu
->env_ptr
;
1813 target_ulong pc
, cs_base
;
1818 if (cpu
->watchpoint_hit
) {
1819 /* We re-entered the check after replacing the TB. Now raise
1820 * the debug interrupt so that is will trigger after the
1821 * current instruction. */
1822 cpu_interrupt(cpu
, CPU_INTERRUPT_DEBUG
);
1825 vaddr
= (cpu
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
1826 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
1827 if (cpu_watchpoint_address_matches(wp
, vaddr
, len
)
1828 && (wp
->flags
& flags
)) {
1829 if (flags
== BP_MEM_READ
) {
1830 wp
->flags
|= BP_WATCHPOINT_HIT_READ
;
1832 wp
->flags
|= BP_WATCHPOINT_HIT_WRITE
;
1834 wp
->hitaddr
= vaddr
;
1835 if (!cpu
->watchpoint_hit
) {
1836 cpu
->watchpoint_hit
= wp
;
1837 tb_check_watchpoint(cpu
);
1838 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
1839 cpu
->exception_index
= EXCP_DEBUG
;
1842 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
1843 tb_gen_code(cpu
, pc
, cs_base
, cpu_flags
, 1);
1844 cpu_resume_from_signal(cpu
, NULL
);
1848 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
1853 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1854 so these check for a hit then pass through to the normal out-of-line
1856 static uint64_t watch_mem_read(void *opaque
, hwaddr addr
,
1859 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, size
, BP_MEM_READ
);
1861 case 1: return ldub_phys(&address_space_memory
, addr
);
1862 case 2: return lduw_phys(&address_space_memory
, addr
);
1863 case 4: return ldl_phys(&address_space_memory
, addr
);
1868 static void watch_mem_write(void *opaque
, hwaddr addr
,
1869 uint64_t val
, unsigned size
)
1871 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, size
, BP_MEM_WRITE
);
1874 stb_phys(&address_space_memory
, addr
, val
);
1877 stw_phys(&address_space_memory
, addr
, val
);
1880 stl_phys(&address_space_memory
, addr
, val
);
1886 static const MemoryRegionOps watch_mem_ops
= {
1887 .read
= watch_mem_read
,
1888 .write
= watch_mem_write
,
1889 .endianness
= DEVICE_NATIVE_ENDIAN
,
1892 static uint64_t subpage_read(void *opaque
, hwaddr addr
,
1895 subpage_t
*subpage
= opaque
;
1898 #if defined(DEBUG_SUBPAGE)
1899 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
"\n", __func__
,
1900 subpage
, len
, addr
);
1902 address_space_read(subpage
->as
, addr
+ subpage
->base
, buf
, len
);
1917 static void subpage_write(void *opaque
, hwaddr addr
,
1918 uint64_t value
, unsigned len
)
1920 subpage_t
*subpage
= opaque
;
1923 #if defined(DEBUG_SUBPAGE)
1924 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
1925 " value %"PRIx64
"\n",
1926 __func__
, subpage
, len
, addr
, value
);
1944 address_space_write(subpage
->as
, addr
+ subpage
->base
, buf
, len
);
1947 static bool subpage_accepts(void *opaque
, hwaddr addr
,
1948 unsigned len
, bool is_write
)
1950 subpage_t
*subpage
= opaque
;
1951 #if defined(DEBUG_SUBPAGE)
1952 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx
"\n",
1953 __func__
, subpage
, is_write
? 'w' : 'r', len
, addr
);
1956 return address_space_access_valid(subpage
->as
, addr
+ subpage
->base
,
1960 static const MemoryRegionOps subpage_ops
= {
1961 .read
= subpage_read
,
1962 .write
= subpage_write
,
1963 .impl
.min_access_size
= 1,
1964 .impl
.max_access_size
= 8,
1965 .valid
.min_access_size
= 1,
1966 .valid
.max_access_size
= 8,
1967 .valid
.accepts
= subpage_accepts
,
1968 .endianness
= DEVICE_NATIVE_ENDIAN
,
1971 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
1976 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
1978 idx
= SUBPAGE_IDX(start
);
1979 eidx
= SUBPAGE_IDX(end
);
1980 #if defined(DEBUG_SUBPAGE)
1981 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
1982 __func__
, mmio
, start
, end
, idx
, eidx
, section
);
1984 for (; idx
<= eidx
; idx
++) {
1985 mmio
->sub_section
[idx
] = section
;
1991 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
)
1995 mmio
= g_malloc0(sizeof(subpage_t
));
1999 memory_region_init_io(&mmio
->iomem
, NULL
, &subpage_ops
, mmio
,
2000 NULL
, TARGET_PAGE_SIZE
);
2001 mmio
->iomem
.subpage
= true;
2002 #if defined(DEBUG_SUBPAGE)
2003 printf("%s: %p base " TARGET_FMT_plx
" len %08x\n", __func__
,
2004 mmio
, base
, TARGET_PAGE_SIZE
);
2006 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
-1, PHYS_SECTION_UNASSIGNED
);
2011 static uint16_t dummy_section(PhysPageMap
*map
, AddressSpace
*as
,
2015 MemoryRegionSection section
= {
2016 .address_space
= as
,
2018 .offset_within_address_space
= 0,
2019 .offset_within_region
= 0,
2020 .size
= int128_2_64(),
2023 return phys_section_add(map
, §ion
);
2026 MemoryRegion
*iotlb_to_region(CPUState
*cpu
, hwaddr index
)
2028 AddressSpaceDispatch
*d
= atomic_rcu_read(&cpu
->memory_dispatch
);
2029 MemoryRegionSection
*sections
= d
->map
.sections
;
2031 return sections
[index
& ~TARGET_PAGE_MASK
].mr
;
2034 static void io_mem_init(void)
2036 memory_region_init_io(&io_mem_rom
, NULL
, &unassigned_mem_ops
, NULL
, NULL
, UINT64_MAX
);
2037 memory_region_init_io(&io_mem_unassigned
, NULL
, &unassigned_mem_ops
, NULL
,
2039 memory_region_init_io(&io_mem_notdirty
, NULL
, ¬dirty_mem_ops
, NULL
,
2041 memory_region_init_io(&io_mem_watch
, NULL
, &watch_mem_ops
, NULL
,
2045 static void mem_begin(MemoryListener
*listener
)
2047 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
2048 AddressSpaceDispatch
*d
= g_new0(AddressSpaceDispatch
, 1);
2051 n
= dummy_section(&d
->map
, as
, &io_mem_unassigned
);
2052 assert(n
== PHYS_SECTION_UNASSIGNED
);
2053 n
= dummy_section(&d
->map
, as
, &io_mem_notdirty
);
2054 assert(n
== PHYS_SECTION_NOTDIRTY
);
2055 n
= dummy_section(&d
->map
, as
, &io_mem_rom
);
2056 assert(n
== PHYS_SECTION_ROM
);
2057 n
= dummy_section(&d
->map
, as
, &io_mem_watch
);
2058 assert(n
== PHYS_SECTION_WATCH
);
2060 d
->phys_map
= (PhysPageEntry
) { .ptr
= PHYS_MAP_NODE_NIL
, .skip
= 1 };
2062 as
->next_dispatch
= d
;
2065 static void address_space_dispatch_free(AddressSpaceDispatch
*d
)
2067 phys_sections_free(&d
->map
);
2071 static void mem_commit(MemoryListener
*listener
)
2073 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
2074 AddressSpaceDispatch
*cur
= as
->dispatch
;
2075 AddressSpaceDispatch
*next
= as
->next_dispatch
;
2077 phys_page_compact_all(next
, next
->map
.nodes_nb
);
2079 atomic_rcu_set(&as
->dispatch
, next
);
2081 call_rcu(cur
, address_space_dispatch_free
, rcu
);
2085 static void tcg_commit(MemoryListener
*listener
)
2089 /* since each CPU stores ram addresses in its TLB cache, we must
2090 reset the modified entries */
2093 /* FIXME: Disentangle the cpu.h circular files deps so we can
2094 directly get the right CPU from listener. */
2095 if (cpu
->tcg_as_listener
!= listener
) {
2098 cpu_reload_memory_map(cpu
);
2102 static void core_log_global_start(MemoryListener
*listener
)
2104 cpu_physical_memory_set_dirty_tracking(true);
2107 static void core_log_global_stop(MemoryListener
*listener
)
2109 cpu_physical_memory_set_dirty_tracking(false);
2112 static MemoryListener core_memory_listener
= {
2113 .log_global_start
= core_log_global_start
,
2114 .log_global_stop
= core_log_global_stop
,
2118 void address_space_init_dispatch(AddressSpace
*as
)
2120 as
->dispatch
= NULL
;
2121 as
->dispatch_listener
= (MemoryListener
) {
2123 .commit
= mem_commit
,
2124 .region_add
= mem_add
,
2125 .region_nop
= mem_add
,
2128 memory_listener_register(&as
->dispatch_listener
, as
);
2131 void address_space_unregister(AddressSpace
*as
)
2133 memory_listener_unregister(&as
->dispatch_listener
);
2136 void address_space_destroy_dispatch(AddressSpace
*as
)
2138 AddressSpaceDispatch
*d
= as
->dispatch
;
2140 atomic_rcu_set(&as
->dispatch
, NULL
);
2142 call_rcu(d
, address_space_dispatch_free
, rcu
);
2146 static void memory_map_init(void)
2148 system_memory
= g_malloc(sizeof(*system_memory
));
2150 memory_region_init(system_memory
, NULL
, "system", UINT64_MAX
);
2151 address_space_init(&address_space_memory
, system_memory
, "memory");
2153 system_io
= g_malloc(sizeof(*system_io
));
2154 memory_region_init_io(system_io
, NULL
, &unassigned_io_ops
, NULL
, "io",
2156 address_space_init(&address_space_io
, system_io
, "I/O");
2158 memory_listener_register(&core_memory_listener
, &address_space_memory
);
2161 MemoryRegion
*get_system_memory(void)
2163 return system_memory
;
2166 MemoryRegion
*get_system_io(void)
2171 #endif /* !defined(CONFIG_USER_ONLY) */
2173 /* physical memory access (slow version, mainly for debug) */
2174 #if defined(CONFIG_USER_ONLY)
2175 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
2176 uint8_t *buf
, int len
, int is_write
)
2183 page
= addr
& TARGET_PAGE_MASK
;
2184 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2187 flags
= page_get_flags(page
);
2188 if (!(flags
& PAGE_VALID
))
2191 if (!(flags
& PAGE_WRITE
))
2193 /* XXX: this code should not depend on lock_user */
2194 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
2197 unlock_user(p
, addr
, l
);
2199 if (!(flags
& PAGE_READ
))
2201 /* XXX: this code should not depend on lock_user */
2202 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
2205 unlock_user(p
, addr
, 0);
2216 static void invalidate_and_set_dirty(hwaddr addr
,
2219 if (cpu_physical_memory_range_includes_clean(addr
, length
)) {
2220 tb_invalidate_phys_range(addr
, addr
+ length
, 0);
2221 cpu_physical_memory_set_dirty_range_nocode(addr
, length
);
2223 xen_modified_memory(addr
, length
);
2226 static int memory_access_size(MemoryRegion
*mr
, unsigned l
, hwaddr addr
)
2228 unsigned access_size_max
= mr
->ops
->valid
.max_access_size
;
2230 /* Regions are assumed to support 1-4 byte accesses unless
2231 otherwise specified. */
2232 if (access_size_max
== 0) {
2233 access_size_max
= 4;
2236 /* Bound the maximum access by the alignment of the address. */
2237 if (!mr
->ops
->impl
.unaligned
) {
2238 unsigned align_size_max
= addr
& -addr
;
2239 if (align_size_max
!= 0 && align_size_max
< access_size_max
) {
2240 access_size_max
= align_size_max
;
2244 /* Don't attempt accesses larger than the maximum. */
2245 if (l
> access_size_max
) {
2246 l
= access_size_max
;
2249 l
= 1 << (qemu_fls(l
) - 1);
2255 bool address_space_rw(AddressSpace
*as
, hwaddr addr
, uint8_t *buf
,
2256 int len
, bool is_write
)
2267 mr
= address_space_translate(as
, addr
, &addr1
, &l
, is_write
);
2270 if (!memory_access_is_direct(mr
, is_write
)) {
2271 l
= memory_access_size(mr
, l
, addr1
);
2272 /* XXX: could force current_cpu to NULL to avoid
2276 /* 64 bit write access */
2278 error
|= io_mem_write(mr
, addr1
, val
, 8);
2281 /* 32 bit write access */
2283 error
|= io_mem_write(mr
, addr1
, val
, 4);
2286 /* 16 bit write access */
2288 error
|= io_mem_write(mr
, addr1
, val
, 2);
2291 /* 8 bit write access */
2293 error
|= io_mem_write(mr
, addr1
, val
, 1);
2299 addr1
+= memory_region_get_ram_addr(mr
);
2301 ptr
= qemu_get_ram_ptr(addr1
);
2302 memcpy(ptr
, buf
, l
);
2303 invalidate_and_set_dirty(addr1
, l
);
2306 if (!memory_access_is_direct(mr
, is_write
)) {
2308 l
= memory_access_size(mr
, l
, addr1
);
2311 /* 64 bit read access */
2312 error
|= io_mem_read(mr
, addr1
, &val
, 8);
2316 /* 32 bit read access */
2317 error
|= io_mem_read(mr
, addr1
, &val
, 4);
2321 /* 16 bit read access */
2322 error
|= io_mem_read(mr
, addr1
, &val
, 2);
2326 /* 8 bit read access */
2327 error
|= io_mem_read(mr
, addr1
, &val
, 1);
2335 ptr
= qemu_get_ram_ptr(mr
->ram_addr
+ addr1
);
2336 memcpy(buf
, ptr
, l
);
2347 bool address_space_write(AddressSpace
*as
, hwaddr addr
,
2348 const uint8_t *buf
, int len
)
2350 return address_space_rw(as
, addr
, (uint8_t *)buf
, len
, true);
2353 bool address_space_read(AddressSpace
*as
, hwaddr addr
, uint8_t *buf
, int len
)
2355 return address_space_rw(as
, addr
, buf
, len
, false);
2359 void cpu_physical_memory_rw(hwaddr addr
, uint8_t *buf
,
2360 int len
, int is_write
)
2362 address_space_rw(&address_space_memory
, addr
, buf
, len
, is_write
);
2365 enum write_rom_type
{
2370 static inline void cpu_physical_memory_write_rom_internal(AddressSpace
*as
,
2371 hwaddr addr
, const uint8_t *buf
, int len
, enum write_rom_type type
)
2380 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
2382 if (!(memory_region_is_ram(mr
) ||
2383 memory_region_is_romd(mr
))) {
2386 addr1
+= memory_region_get_ram_addr(mr
);
2388 ptr
= qemu_get_ram_ptr(addr1
);
2391 memcpy(ptr
, buf
, l
);
2392 invalidate_and_set_dirty(addr1
, l
);
2395 flush_icache_range((uintptr_t)ptr
, (uintptr_t)ptr
+ l
);
2405 /* used for ROM loading : can write in RAM and ROM */
2406 void cpu_physical_memory_write_rom(AddressSpace
*as
, hwaddr addr
,
2407 const uint8_t *buf
, int len
)
2409 cpu_physical_memory_write_rom_internal(as
, addr
, buf
, len
, WRITE_DATA
);
2412 void cpu_flush_icache_range(hwaddr start
, int len
)
2415 * This function should do the same thing as an icache flush that was
2416 * triggered from within the guest. For TCG we are always cache coherent,
2417 * so there is no need to flush anything. For KVM / Xen we need to flush
2418 * the host's instruction cache at least.
2420 if (tcg_enabled()) {
2424 cpu_physical_memory_write_rom_internal(&address_space_memory
,
2425 start
, NULL
, len
, FLUSH_CACHE
);
2435 static BounceBuffer bounce
;
2437 typedef struct MapClient
{
2439 void (*callback
)(void *opaque
);
2440 QLIST_ENTRY(MapClient
) link
;
2443 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
2444 = QLIST_HEAD_INITIALIZER(map_client_list
);
2446 void *cpu_register_map_client(void *opaque
, void (*callback
)(void *opaque
))
2448 MapClient
*client
= g_malloc(sizeof(*client
));
2450 client
->opaque
= opaque
;
2451 client
->callback
= callback
;
2452 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
2456 static void cpu_unregister_map_client(void *_client
)
2458 MapClient
*client
= (MapClient
*)_client
;
2460 QLIST_REMOVE(client
, link
);
2464 static void cpu_notify_map_clients(void)
2468 while (!QLIST_EMPTY(&map_client_list
)) {
2469 client
= QLIST_FIRST(&map_client_list
);
2470 client
->callback(client
->opaque
);
2471 cpu_unregister_map_client(client
);
2475 bool address_space_access_valid(AddressSpace
*as
, hwaddr addr
, int len
, bool is_write
)
2482 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2483 if (!memory_access_is_direct(mr
, is_write
)) {
2484 l
= memory_access_size(mr
, l
, addr
);
2485 if (!memory_region_access_valid(mr
, xlat
, l
, is_write
)) {
2496 /* Map a physical memory region into a host virtual address.
2497 * May map a subset of the requested range, given by and returned in *plen.
2498 * May return NULL if resources needed to perform the mapping are exhausted.
2499 * Use only for reads OR writes - not for read-modify-write operations.
2500 * Use cpu_register_map_client() to know when retrying the map operation is
2501 * likely to succeed.
2503 void *address_space_map(AddressSpace
*as
,
2510 hwaddr l
, xlat
, base
;
2511 MemoryRegion
*mr
, *this_mr
;
2519 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2520 if (!memory_access_is_direct(mr
, is_write
)) {
2521 if (bounce
.buffer
) {
2524 /* Avoid unbounded allocations */
2525 l
= MIN(l
, TARGET_PAGE_SIZE
);
2526 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, l
);
2530 memory_region_ref(mr
);
2533 address_space_read(as
, addr
, bounce
.buffer
, l
);
2537 return bounce
.buffer
;
2541 raddr
= memory_region_get_ram_addr(mr
);
2552 this_mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2553 if (this_mr
!= mr
|| xlat
!= base
+ done
) {
2558 memory_region_ref(mr
);
2560 return qemu_ram_ptr_length(raddr
+ base
, plen
);
2563 /* Unmaps a memory region previously mapped by address_space_map().
2564 * Will also mark the memory as dirty if is_write == 1. access_len gives
2565 * the amount of memory that was actually read or written by the caller.
2567 void address_space_unmap(AddressSpace
*as
, void *buffer
, hwaddr len
,
2568 int is_write
, hwaddr access_len
)
2570 if (buffer
!= bounce
.buffer
) {
2574 mr
= qemu_ram_addr_from_host(buffer
, &addr1
);
2577 invalidate_and_set_dirty(addr1
, access_len
);
2579 if (xen_enabled()) {
2580 xen_invalidate_map_cache_entry(buffer
);
2582 memory_region_unref(mr
);
2586 address_space_write(as
, bounce
.addr
, bounce
.buffer
, access_len
);
2588 qemu_vfree(bounce
.buffer
);
2589 bounce
.buffer
= NULL
;
2590 memory_region_unref(bounce
.mr
);
2591 cpu_notify_map_clients();
2594 void *cpu_physical_memory_map(hwaddr addr
,
2598 return address_space_map(&address_space_memory
, addr
, plen
, is_write
);
2601 void cpu_physical_memory_unmap(void *buffer
, hwaddr len
,
2602 int is_write
, hwaddr access_len
)
2604 return address_space_unmap(&address_space_memory
, buffer
, len
, is_write
, access_len
);
2607 /* warning: addr must be aligned */
2608 static inline uint32_t ldl_phys_internal(AddressSpace
*as
, hwaddr addr
,
2609 enum device_endian endian
)
2617 mr
= address_space_translate(as
, addr
, &addr1
, &l
, false);
2618 if (l
< 4 || !memory_access_is_direct(mr
, false)) {
2620 io_mem_read(mr
, addr1
, &val
, 4);
2621 #if defined(TARGET_WORDS_BIGENDIAN)
2622 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2626 if (endian
== DEVICE_BIG_ENDIAN
) {
2632 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2636 case DEVICE_LITTLE_ENDIAN
:
2637 val
= ldl_le_p(ptr
);
2639 case DEVICE_BIG_ENDIAN
:
2640 val
= ldl_be_p(ptr
);
2650 uint32_t ldl_phys(AddressSpace
*as
, hwaddr addr
)
2652 return ldl_phys_internal(as
, addr
, DEVICE_NATIVE_ENDIAN
);
2655 uint32_t ldl_le_phys(AddressSpace
*as
, hwaddr addr
)
2657 return ldl_phys_internal(as
, addr
, DEVICE_LITTLE_ENDIAN
);
2660 uint32_t ldl_be_phys(AddressSpace
*as
, hwaddr addr
)
2662 return ldl_phys_internal(as
, addr
, DEVICE_BIG_ENDIAN
);
2665 /* warning: addr must be aligned */
2666 static inline uint64_t ldq_phys_internal(AddressSpace
*as
, hwaddr addr
,
2667 enum device_endian endian
)
2675 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2677 if (l
< 8 || !memory_access_is_direct(mr
, false)) {
2679 io_mem_read(mr
, addr1
, &val
, 8);
2680 #if defined(TARGET_WORDS_BIGENDIAN)
2681 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2685 if (endian
== DEVICE_BIG_ENDIAN
) {
2691 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2695 case DEVICE_LITTLE_ENDIAN
:
2696 val
= ldq_le_p(ptr
);
2698 case DEVICE_BIG_ENDIAN
:
2699 val
= ldq_be_p(ptr
);
2709 uint64_t ldq_phys(AddressSpace
*as
, hwaddr addr
)
2711 return ldq_phys_internal(as
, addr
, DEVICE_NATIVE_ENDIAN
);
2714 uint64_t ldq_le_phys(AddressSpace
*as
, hwaddr addr
)
2716 return ldq_phys_internal(as
, addr
, DEVICE_LITTLE_ENDIAN
);
2719 uint64_t ldq_be_phys(AddressSpace
*as
, hwaddr addr
)
2721 return ldq_phys_internal(as
, addr
, DEVICE_BIG_ENDIAN
);
2725 uint32_t ldub_phys(AddressSpace
*as
, hwaddr addr
)
2728 address_space_rw(as
, addr
, &val
, 1, 0);
2732 /* warning: addr must be aligned */
2733 static inline uint32_t lduw_phys_internal(AddressSpace
*as
, hwaddr addr
,
2734 enum device_endian endian
)
2742 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2744 if (l
< 2 || !memory_access_is_direct(mr
, false)) {
2746 io_mem_read(mr
, addr1
, &val
, 2);
2747 #if defined(TARGET_WORDS_BIGENDIAN)
2748 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2752 if (endian
== DEVICE_BIG_ENDIAN
) {
2758 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2762 case DEVICE_LITTLE_ENDIAN
:
2763 val
= lduw_le_p(ptr
);
2765 case DEVICE_BIG_ENDIAN
:
2766 val
= lduw_be_p(ptr
);
2776 uint32_t lduw_phys(AddressSpace
*as
, hwaddr addr
)
2778 return lduw_phys_internal(as
, addr
, DEVICE_NATIVE_ENDIAN
);
2781 uint32_t lduw_le_phys(AddressSpace
*as
, hwaddr addr
)
2783 return lduw_phys_internal(as
, addr
, DEVICE_LITTLE_ENDIAN
);
2786 uint32_t lduw_be_phys(AddressSpace
*as
, hwaddr addr
)
2788 return lduw_phys_internal(as
, addr
, DEVICE_BIG_ENDIAN
);
2791 /* warning: addr must be aligned. The ram page is not masked as dirty
2792 and the code inside is not invalidated. It is useful if the dirty
2793 bits are used to track modified PTEs */
2794 void stl_phys_notdirty(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2801 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2803 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
2804 io_mem_write(mr
, addr1
, val
, 4);
2806 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
2807 ptr
= qemu_get_ram_ptr(addr1
);
2810 if (unlikely(in_migration
)) {
2811 if (cpu_physical_memory_is_clean(addr1
)) {
2812 /* invalidate code */
2813 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
2815 cpu_physical_memory_set_dirty_range_nocode(addr1
, 4);
2821 /* warning: addr must be aligned */
2822 static inline void stl_phys_internal(AddressSpace
*as
,
2823 hwaddr addr
, uint32_t val
,
2824 enum device_endian endian
)
2831 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2833 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
2834 #if defined(TARGET_WORDS_BIGENDIAN)
2835 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2839 if (endian
== DEVICE_BIG_ENDIAN
) {
2843 io_mem_write(mr
, addr1
, val
, 4);
2846 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
2847 ptr
= qemu_get_ram_ptr(addr1
);
2849 case DEVICE_LITTLE_ENDIAN
:
2852 case DEVICE_BIG_ENDIAN
:
2859 invalidate_and_set_dirty(addr1
, 4);
2863 void stl_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2865 stl_phys_internal(as
, addr
, val
, DEVICE_NATIVE_ENDIAN
);
2868 void stl_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2870 stl_phys_internal(as
, addr
, val
, DEVICE_LITTLE_ENDIAN
);
2873 void stl_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2875 stl_phys_internal(as
, addr
, val
, DEVICE_BIG_ENDIAN
);
2879 void stb_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2882 address_space_rw(as
, addr
, &v
, 1, 1);
2885 /* warning: addr must be aligned */
2886 static inline void stw_phys_internal(AddressSpace
*as
,
2887 hwaddr addr
, uint32_t val
,
2888 enum device_endian endian
)
2895 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
2896 if (l
< 2 || !memory_access_is_direct(mr
, true)) {
2897 #if defined(TARGET_WORDS_BIGENDIAN)
2898 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2902 if (endian
== DEVICE_BIG_ENDIAN
) {
2906 io_mem_write(mr
, addr1
, val
, 2);
2909 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
2910 ptr
= qemu_get_ram_ptr(addr1
);
2912 case DEVICE_LITTLE_ENDIAN
:
2915 case DEVICE_BIG_ENDIAN
:
2922 invalidate_and_set_dirty(addr1
, 2);
2926 void stw_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2928 stw_phys_internal(as
, addr
, val
, DEVICE_NATIVE_ENDIAN
);
2931 void stw_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2933 stw_phys_internal(as
, addr
, val
, DEVICE_LITTLE_ENDIAN
);
2936 void stw_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
2938 stw_phys_internal(as
, addr
, val
, DEVICE_BIG_ENDIAN
);
2942 void stq_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
2945 address_space_rw(as
, addr
, (void *) &val
, 8, 1);
2948 void stq_le_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
2950 val
= cpu_to_le64(val
);
2951 address_space_rw(as
, addr
, (void *) &val
, 8, 1);
2954 void stq_be_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
2956 val
= cpu_to_be64(val
);
2957 address_space_rw(as
, addr
, (void *) &val
, 8, 1);
2960 /* virtual memory access for debug (includes writing to ROM) */
2961 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
2962 uint8_t *buf
, int len
, int is_write
)
2969 page
= addr
& TARGET_PAGE_MASK
;
2970 phys_addr
= cpu_get_phys_page_debug(cpu
, page
);
2971 /* if no physical page mapped, return an error */
2972 if (phys_addr
== -1)
2974 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2977 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
2979 cpu_physical_memory_write_rom(cpu
->as
, phys_addr
, buf
, l
);
2981 address_space_rw(cpu
->as
, phys_addr
, buf
, l
, 0);
2992 * A helper function for the _utterly broken_ virtio device model to find out if
2993 * it's running on a big endian machine. Don't do this at home kids!
2995 bool target_words_bigendian(void);
2996 bool target_words_bigendian(void)
2998 #if defined(TARGET_WORDS_BIGENDIAN)
3005 #ifndef CONFIG_USER_ONLY
3006 bool cpu_physical_memory_is_io(hwaddr phys_addr
)
3011 mr
= address_space_translate(&address_space_memory
,
3012 phys_addr
, &phys_addr
, &l
, false);
3014 return !(memory_region_is_ram(mr
) ||
3015 memory_region_is_romd(mr
));
3018 void qemu_ram_foreach_block(RAMBlockIterFunc func
, void *opaque
)
3022 QTAILQ_FOREACH(block
, &ram_list
.blocks
, next
) {
3023 func(block
->host
, block
->offset
, block
->used_length
, opaque
);