4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
21 #include <sys/types.h>
25 #include "qemu-common.h"
29 #if !defined(CONFIG_USER_ONLY)
30 #include "hw/boards.h"
33 #include "qemu/osdep.h"
34 #include "sysemu/kvm.h"
35 #include "sysemu/sysemu.h"
36 #include "hw/xen/xen.h"
37 #include "qemu/timer.h"
38 #include "qemu/config-file.h"
39 #include "qemu/error-report.h"
40 #include "exec/memory.h"
41 #include "sysemu/dma.h"
42 #include "exec/address-spaces.h"
43 #if defined(CONFIG_USER_ONLY)
45 #else /* !CONFIG_USER_ONLY */
46 #include "sysemu/xen-mapcache.h"
49 #include "exec/cpu-all.h"
50 #include "qemu/rcu_queue.h"
51 #include "qemu/main-loop.h"
52 #include "exec/cputlb.h"
53 #include "translate-all.h"
55 #include "exec/memory-internal.h"
56 #include "exec/ram_addr.h"
58 #include "qemu/range.h"
60 //#define DEBUG_SUBPAGE
62 #if !defined(CONFIG_USER_ONLY)
63 /* ram_list is read under rcu_read_lock()/rcu_read_unlock(). Writes
64 * are protected by the ramlist lock.
66 RAMList ram_list
= { .blocks
= QLIST_HEAD_INITIALIZER(ram_list
.blocks
) };
68 static MemoryRegion
*system_memory
;
69 static MemoryRegion
*system_io
;
71 AddressSpace address_space_io
;
72 AddressSpace address_space_memory
;
74 MemoryRegion io_mem_rom
, io_mem_notdirty
;
75 static MemoryRegion io_mem_unassigned
;
77 /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
78 #define RAM_PREALLOC (1 << 0)
80 /* RAM is mmap-ed with MAP_SHARED */
81 #define RAM_SHARED (1 << 1)
83 /* Only a portion of RAM (used_length) is actually used, and migrated.
84 * This used_length size can change across reboots.
86 #define RAM_RESIZEABLE (1 << 2)
90 struct CPUTailQ cpus
= QTAILQ_HEAD_INITIALIZER(cpus
);
91 /* current CPU in the current thread. It is only valid inside
93 DEFINE_TLS(CPUState
*, current_cpu
);
94 /* 0 = Do not count executed instructions.
95 1 = Precise instruction counting.
96 2 = Adaptive rate instruction counting. */
99 #if !defined(CONFIG_USER_ONLY)
101 typedef struct PhysPageEntry PhysPageEntry
;
103 struct PhysPageEntry
{
104 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
106 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
110 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
112 /* Size of the L2 (and L3, etc) page tables. */
113 #define ADDR_SPACE_BITS 64
116 #define P_L2_SIZE (1 << P_L2_BITS)
118 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
120 typedef PhysPageEntry Node
[P_L2_SIZE
];
122 typedef struct PhysPageMap
{
125 unsigned sections_nb
;
126 unsigned sections_nb_alloc
;
128 unsigned nodes_nb_alloc
;
130 MemoryRegionSection
*sections
;
133 struct AddressSpaceDispatch
{
136 /* This is a multi-level map on the physical address space.
137 * The bottom level has pointers to MemoryRegionSections.
139 PhysPageEntry phys_map
;
144 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
145 typedef struct subpage_t
{
149 uint16_t sub_section
[TARGET_PAGE_SIZE
];
152 #define PHYS_SECTION_UNASSIGNED 0
153 #define PHYS_SECTION_NOTDIRTY 1
154 #define PHYS_SECTION_ROM 2
155 #define PHYS_SECTION_WATCH 3
157 static void io_mem_init(void);
158 static void memory_map_init(void);
159 static void tcg_commit(MemoryListener
*listener
);
161 static MemoryRegion io_mem_watch
;
164 #if !defined(CONFIG_USER_ONLY)
166 static void phys_map_node_reserve(PhysPageMap
*map
, unsigned nodes
)
168 if (map
->nodes_nb
+ nodes
> map
->nodes_nb_alloc
) {
169 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
* 2, 16);
170 map
->nodes_nb_alloc
= MAX(map
->nodes_nb_alloc
, map
->nodes_nb
+ nodes
);
171 map
->nodes
= g_renew(Node
, map
->nodes
, map
->nodes_nb_alloc
);
175 static uint32_t phys_map_node_alloc(PhysPageMap
*map
, bool leaf
)
182 ret
= map
->nodes_nb
++;
184 assert(ret
!= PHYS_MAP_NODE_NIL
);
185 assert(ret
!= map
->nodes_nb_alloc
);
187 e
.skip
= leaf
? 0 : 1;
188 e
.ptr
= leaf
? PHYS_SECTION_UNASSIGNED
: PHYS_MAP_NODE_NIL
;
189 for (i
= 0; i
< P_L2_SIZE
; ++i
) {
190 memcpy(&p
[i
], &e
, sizeof(e
));
195 static void phys_page_set_level(PhysPageMap
*map
, PhysPageEntry
*lp
,
196 hwaddr
*index
, hwaddr
*nb
, uint16_t leaf
,
200 hwaddr step
= (hwaddr
)1 << (level
* P_L2_BITS
);
202 if (lp
->skip
&& lp
->ptr
== PHYS_MAP_NODE_NIL
) {
203 lp
->ptr
= phys_map_node_alloc(map
, level
== 0);
205 p
= map
->nodes
[lp
->ptr
];
206 lp
= &p
[(*index
>> (level
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
208 while (*nb
&& lp
< &p
[P_L2_SIZE
]) {
209 if ((*index
& (step
- 1)) == 0 && *nb
>= step
) {
215 phys_page_set_level(map
, lp
, index
, nb
, leaf
, level
- 1);
221 static void phys_page_set(AddressSpaceDispatch
*d
,
222 hwaddr index
, hwaddr nb
,
225 /* Wildly overreserve - it doesn't matter much. */
226 phys_map_node_reserve(&d
->map
, 3 * P_L2_LEVELS
);
228 phys_page_set_level(&d
->map
, &d
->phys_map
, &index
, &nb
, leaf
, P_L2_LEVELS
- 1);
231 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
232 * and update our entry so we can skip it and go directly to the destination.
234 static void phys_page_compact(PhysPageEntry
*lp
, Node
*nodes
, unsigned long *compacted
)
236 unsigned valid_ptr
= P_L2_SIZE
;
241 if (lp
->ptr
== PHYS_MAP_NODE_NIL
) {
246 for (i
= 0; i
< P_L2_SIZE
; i
++) {
247 if (p
[i
].ptr
== PHYS_MAP_NODE_NIL
) {
254 phys_page_compact(&p
[i
], nodes
, compacted
);
258 /* We can only compress if there's only one child. */
263 assert(valid_ptr
< P_L2_SIZE
);
265 /* Don't compress if it won't fit in the # of bits we have. */
266 if (lp
->skip
+ p
[valid_ptr
].skip
>= (1 << 3)) {
270 lp
->ptr
= p
[valid_ptr
].ptr
;
271 if (!p
[valid_ptr
].skip
) {
272 /* If our only child is a leaf, make this a leaf. */
273 /* By design, we should have made this node a leaf to begin with so we
274 * should never reach here.
275 * But since it's so simple to handle this, let's do it just in case we
280 lp
->skip
+= p
[valid_ptr
].skip
;
284 static void phys_page_compact_all(AddressSpaceDispatch
*d
, int nodes_nb
)
286 DECLARE_BITMAP(compacted
, nodes_nb
);
288 if (d
->phys_map
.skip
) {
289 phys_page_compact(&d
->phys_map
, d
->map
.nodes
, compacted
);
293 static MemoryRegionSection
*phys_page_find(PhysPageEntry lp
, hwaddr addr
,
294 Node
*nodes
, MemoryRegionSection
*sections
)
297 hwaddr index
= addr
>> TARGET_PAGE_BITS
;
300 for (i
= P_L2_LEVELS
; lp
.skip
&& (i
-= lp
.skip
) >= 0;) {
301 if (lp
.ptr
== PHYS_MAP_NODE_NIL
) {
302 return §ions
[PHYS_SECTION_UNASSIGNED
];
305 lp
= p
[(index
>> (i
* P_L2_BITS
)) & (P_L2_SIZE
- 1)];
308 if (sections
[lp
.ptr
].size
.hi
||
309 range_covers_byte(sections
[lp
.ptr
].offset_within_address_space
,
310 sections
[lp
.ptr
].size
.lo
, addr
)) {
311 return §ions
[lp
.ptr
];
313 return §ions
[PHYS_SECTION_UNASSIGNED
];
317 bool memory_region_is_unassigned(MemoryRegion
*mr
)
319 return mr
!= &io_mem_rom
&& mr
!= &io_mem_notdirty
&& !mr
->rom_device
320 && mr
!= &io_mem_watch
;
323 /* Called from RCU critical section */
324 static MemoryRegionSection
*address_space_lookup_region(AddressSpaceDispatch
*d
,
326 bool resolve_subpage
)
328 MemoryRegionSection
*section
;
331 section
= phys_page_find(d
->phys_map
, addr
, d
->map
.nodes
, d
->map
.sections
);
332 if (resolve_subpage
&& section
->mr
->subpage
) {
333 subpage
= container_of(section
->mr
, subpage_t
, iomem
);
334 section
= &d
->map
.sections
[subpage
->sub_section
[SUBPAGE_IDX(addr
)]];
339 /* Called from RCU critical section */
340 static MemoryRegionSection
*
341 address_space_translate_internal(AddressSpaceDispatch
*d
, hwaddr addr
, hwaddr
*xlat
,
342 hwaddr
*plen
, bool resolve_subpage
)
344 MemoryRegionSection
*section
;
348 section
= address_space_lookup_region(d
, addr
, resolve_subpage
);
349 /* Compute offset within MemoryRegionSection */
350 addr
-= section
->offset_within_address_space
;
352 /* Compute offset within MemoryRegion */
353 *xlat
= addr
+ section
->offset_within_region
;
357 /* MMIO registers can be expected to perform full-width accesses based only
358 * on their address, without considering adjacent registers that could
359 * decode to completely different MemoryRegions. When such registers
360 * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
361 * regions overlap wildly. For this reason we cannot clamp the accesses
364 * If the length is small (as is the case for address_space_ldl/stl),
365 * everything works fine. If the incoming length is large, however,
366 * the caller really has to do the clamping through memory_access_size.
368 if (memory_region_is_ram(mr
)) {
369 diff
= int128_sub(section
->size
, int128_make64(addr
));
370 *plen
= int128_get64(int128_min(diff
, int128_make64(*plen
)));
375 static inline bool memory_access_is_direct(MemoryRegion
*mr
, bool is_write
)
377 if (memory_region_is_ram(mr
)) {
378 return !(is_write
&& mr
->readonly
);
380 if (memory_region_is_romd(mr
)) {
387 /* Called from RCU critical section */
388 MemoryRegion
*address_space_translate(AddressSpace
*as
, hwaddr addr
,
389 hwaddr
*xlat
, hwaddr
*plen
,
393 MemoryRegionSection
*section
;
397 AddressSpaceDispatch
*d
= atomic_rcu_read(&as
->dispatch
);
398 section
= address_space_translate_internal(d
, addr
, &addr
, plen
, true);
401 if (!mr
->iommu_ops
) {
405 iotlb
= mr
->iommu_ops
->translate(mr
, addr
, is_write
);
406 addr
= ((iotlb
.translated_addr
& ~iotlb
.addr_mask
)
407 | (addr
& iotlb
.addr_mask
));
408 *plen
= MIN(*plen
, (addr
| iotlb
.addr_mask
) - addr
+ 1);
409 if (!(iotlb
.perm
& (1 << is_write
))) {
410 mr
= &io_mem_unassigned
;
414 as
= iotlb
.target_as
;
417 if (xen_enabled() && memory_access_is_direct(mr
, is_write
)) {
418 hwaddr page
= ((addr
& TARGET_PAGE_MASK
) + TARGET_PAGE_SIZE
) - addr
;
419 *plen
= MIN(page
, *plen
);
426 /* Called from RCU critical section */
427 MemoryRegionSection
*
428 address_space_translate_for_iotlb(CPUState
*cpu
, hwaddr addr
,
429 hwaddr
*xlat
, hwaddr
*plen
)
431 MemoryRegionSection
*section
;
432 section
= address_space_translate_internal(cpu
->memory_dispatch
,
433 addr
, xlat
, plen
, false);
435 assert(!section
->mr
->iommu_ops
);
440 #if !defined(CONFIG_USER_ONLY)
442 static int cpu_common_post_load(void *opaque
, int version_id
)
444 CPUState
*cpu
= opaque
;
446 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
447 version_id is increased. */
448 cpu
->interrupt_request
&= ~0x01;
454 static int cpu_common_pre_load(void *opaque
)
456 CPUState
*cpu
= opaque
;
458 cpu
->exception_index
= -1;
463 static bool cpu_common_exception_index_needed(void *opaque
)
465 CPUState
*cpu
= opaque
;
467 return tcg_enabled() && cpu
->exception_index
!= -1;
470 static const VMStateDescription vmstate_cpu_common_exception_index
= {
471 .name
= "cpu_common/exception_index",
473 .minimum_version_id
= 1,
474 .needed
= cpu_common_exception_index_needed
,
475 .fields
= (VMStateField
[]) {
476 VMSTATE_INT32(exception_index
, CPUState
),
477 VMSTATE_END_OF_LIST()
481 const VMStateDescription vmstate_cpu_common
= {
482 .name
= "cpu_common",
484 .minimum_version_id
= 1,
485 .pre_load
= cpu_common_pre_load
,
486 .post_load
= cpu_common_post_load
,
487 .fields
= (VMStateField
[]) {
488 VMSTATE_UINT32(halted
, CPUState
),
489 VMSTATE_UINT32(interrupt_request
, CPUState
),
490 VMSTATE_END_OF_LIST()
492 .subsections
= (const VMStateDescription
*[]) {
493 &vmstate_cpu_common_exception_index
,
500 CPUState
*qemu_get_cpu(int index
)
505 if (cpu
->cpu_index
== index
) {
513 #if !defined(CONFIG_USER_ONLY)
514 void tcg_cpu_address_space_init(CPUState
*cpu
, AddressSpace
*as
)
516 /* We only support one address space per cpu at the moment. */
517 assert(cpu
->as
== as
);
519 if (cpu
->tcg_as_listener
) {
520 memory_listener_unregister(cpu
->tcg_as_listener
);
522 cpu
->tcg_as_listener
= g_new0(MemoryListener
, 1);
524 cpu
->tcg_as_listener
->commit
= tcg_commit
;
525 memory_listener_register(cpu
->tcg_as_listener
, as
);
529 void cpu_exec_init(CPUArchState
*env
)
531 CPUState
*cpu
= ENV_GET_CPU(env
);
532 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
536 #if defined(CONFIG_USER_ONLY)
540 CPU_FOREACH(some_cpu
) {
543 cpu
->cpu_index
= cpu_index
;
544 QTAILQ_INIT(&cpu
->breakpoints
);
545 QTAILQ_INIT(&cpu
->watchpoints
);
546 #ifndef CONFIG_USER_ONLY
547 cpu
->as
= &address_space_memory
;
548 cpu
->thread_id
= qemu_get_thread_id();
549 cpu_reload_memory_map(cpu
);
551 QTAILQ_INSERT_TAIL(&cpus
, cpu
, node
);
552 #if defined(CONFIG_USER_ONLY)
555 if (qdev_get_vmsd(DEVICE(cpu
)) == NULL
) {
556 vmstate_register(NULL
, cpu_index
, &vmstate_cpu_common
, cpu
);
558 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
559 register_savevm(NULL
, "cpu", cpu_index
, CPU_SAVE_VERSION
,
560 cpu_save
, cpu_load
, env
);
561 assert(cc
->vmsd
== NULL
);
562 assert(qdev_get_vmsd(DEVICE(cpu
)) == NULL
);
564 if (cc
->vmsd
!= NULL
) {
565 vmstate_register(NULL
, cpu_index
, cc
->vmsd
, cpu
);
569 #if defined(CONFIG_USER_ONLY)
570 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
572 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
575 static void breakpoint_invalidate(CPUState
*cpu
, target_ulong pc
)
577 hwaddr phys
= cpu_get_phys_page_debug(cpu
, pc
);
579 tb_invalidate_phys_addr(cpu
->as
,
580 phys
| (pc
& ~TARGET_PAGE_MASK
));
585 #if defined(CONFIG_USER_ONLY)
586 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
591 int cpu_watchpoint_remove(CPUState
*cpu
, vaddr addr
, vaddr len
,
597 void cpu_watchpoint_remove_by_ref(CPUState
*cpu
, CPUWatchpoint
*watchpoint
)
601 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
602 int flags
, CPUWatchpoint
**watchpoint
)
607 /* Add a watchpoint. */
608 int cpu_watchpoint_insert(CPUState
*cpu
, vaddr addr
, vaddr len
,
609 int flags
, CPUWatchpoint
**watchpoint
)
613 /* forbid ranges which are empty or run off the end of the address space */
614 if (len
== 0 || (addr
+ len
- 1) < addr
) {
615 error_report("tried to set invalid watchpoint at %"
616 VADDR_PRIx
", len=%" VADDR_PRIu
, addr
, len
);
619 wp
= g_malloc(sizeof(*wp
));
625 /* keep all GDB-injected watchpoints in front */
626 if (flags
& BP_GDB
) {
627 QTAILQ_INSERT_HEAD(&cpu
->watchpoints
, wp
, entry
);
629 QTAILQ_INSERT_TAIL(&cpu
->watchpoints
, wp
, entry
);
632 tlb_flush_page(cpu
, addr
);
639 /* Remove a specific watchpoint. */
640 int cpu_watchpoint_remove(CPUState
*cpu
, vaddr addr
, vaddr len
,
645 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
646 if (addr
== wp
->vaddr
&& len
== wp
->len
647 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
648 cpu_watchpoint_remove_by_ref(cpu
, wp
);
655 /* Remove a specific watchpoint by reference. */
656 void cpu_watchpoint_remove_by_ref(CPUState
*cpu
, CPUWatchpoint
*watchpoint
)
658 QTAILQ_REMOVE(&cpu
->watchpoints
, watchpoint
, entry
);
660 tlb_flush_page(cpu
, watchpoint
->vaddr
);
665 /* Remove all matching watchpoints. */
666 void cpu_watchpoint_remove_all(CPUState
*cpu
, int mask
)
668 CPUWatchpoint
*wp
, *next
;
670 QTAILQ_FOREACH_SAFE(wp
, &cpu
->watchpoints
, entry
, next
) {
671 if (wp
->flags
& mask
) {
672 cpu_watchpoint_remove_by_ref(cpu
, wp
);
677 /* Return true if this watchpoint address matches the specified
678 * access (ie the address range covered by the watchpoint overlaps
679 * partially or completely with the address range covered by the
682 static inline bool cpu_watchpoint_address_matches(CPUWatchpoint
*wp
,
686 /* We know the lengths are non-zero, but a little caution is
687 * required to avoid errors in the case where the range ends
688 * exactly at the top of the address space and so addr + len
689 * wraps round to zero.
691 vaddr wpend
= wp
->vaddr
+ wp
->len
- 1;
692 vaddr addrend
= addr
+ len
- 1;
694 return !(addr
> wpend
|| wp
->vaddr
> addrend
);
699 /* Add a breakpoint. */
700 int cpu_breakpoint_insert(CPUState
*cpu
, vaddr pc
, int flags
,
701 CPUBreakpoint
**breakpoint
)
705 bp
= g_malloc(sizeof(*bp
));
710 /* keep all GDB-injected breakpoints in front */
711 if (flags
& BP_GDB
) {
712 QTAILQ_INSERT_HEAD(&cpu
->breakpoints
, bp
, entry
);
714 QTAILQ_INSERT_TAIL(&cpu
->breakpoints
, bp
, entry
);
717 breakpoint_invalidate(cpu
, pc
);
725 /* Remove a specific breakpoint. */
726 int cpu_breakpoint_remove(CPUState
*cpu
, vaddr pc
, int flags
)
730 QTAILQ_FOREACH(bp
, &cpu
->breakpoints
, entry
) {
731 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
732 cpu_breakpoint_remove_by_ref(cpu
, bp
);
739 /* Remove a specific breakpoint by reference. */
740 void cpu_breakpoint_remove_by_ref(CPUState
*cpu
, CPUBreakpoint
*breakpoint
)
742 QTAILQ_REMOVE(&cpu
->breakpoints
, breakpoint
, entry
);
744 breakpoint_invalidate(cpu
, breakpoint
->pc
);
749 /* Remove all matching breakpoints. */
750 void cpu_breakpoint_remove_all(CPUState
*cpu
, int mask
)
752 CPUBreakpoint
*bp
, *next
;
754 QTAILQ_FOREACH_SAFE(bp
, &cpu
->breakpoints
, entry
, next
) {
755 if (bp
->flags
& mask
) {
756 cpu_breakpoint_remove_by_ref(cpu
, bp
);
761 /* enable or disable single step mode. EXCP_DEBUG is returned by the
762 CPU loop after each instruction */
763 void cpu_single_step(CPUState
*cpu
, int enabled
)
765 if (cpu
->singlestep_enabled
!= enabled
) {
766 cpu
->singlestep_enabled
= enabled
;
768 kvm_update_guest_debug(cpu
, 0);
770 /* must flush all the translated code to avoid inconsistencies */
771 /* XXX: only flush what is necessary */
772 CPUArchState
*env
= cpu
->env_ptr
;
778 void cpu_abort(CPUState
*cpu
, const char *fmt
, ...)
785 fprintf(stderr
, "qemu: fatal: ");
786 vfprintf(stderr
, fmt
, ap
);
787 fprintf(stderr
, "\n");
788 cpu_dump_state(cpu
, stderr
, fprintf
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
789 if (qemu_log_enabled()) {
790 qemu_log("qemu: fatal: ");
791 qemu_log_vprintf(fmt
, ap2
);
793 log_cpu_state(cpu
, CPU_DUMP_FPU
| CPU_DUMP_CCOP
);
799 #if defined(CONFIG_USER_ONLY)
801 struct sigaction act
;
802 sigfillset(&act
.sa_mask
);
803 act
.sa_handler
= SIG_DFL
;
804 sigaction(SIGABRT
, &act
, NULL
);
810 #if !defined(CONFIG_USER_ONLY)
811 /* Called from RCU critical section */
812 static RAMBlock
*qemu_get_ram_block(ram_addr_t addr
)
816 block
= atomic_rcu_read(&ram_list
.mru_block
);
817 if (block
&& addr
- block
->offset
< block
->max_length
) {
820 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
821 if (addr
- block
->offset
< block
->max_length
) {
826 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
830 /* It is safe to write mru_block outside the iothread lock. This
835 * xxx removed from list
839 * call_rcu(reclaim_ramblock, xxx);
842 * atomic_rcu_set is not needed here. The block was already published
843 * when it was placed into the list. Here we're just making an extra
844 * copy of the pointer.
846 ram_list
.mru_block
= block
;
850 static void tlb_reset_dirty_range_all(ram_addr_t start
, ram_addr_t length
)
856 end
= TARGET_PAGE_ALIGN(start
+ length
);
857 start
&= TARGET_PAGE_MASK
;
860 block
= qemu_get_ram_block(start
);
861 assert(block
== qemu_get_ram_block(end
- 1));
862 start1
= (uintptr_t)ramblock_ptr(block
, start
- block
->offset
);
863 cpu_tlb_reset_dirty_all(start1
, length
);
867 /* Note: start and end must be within the same ram block. */
868 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start
,
872 unsigned long end
, page
;
879 end
= TARGET_PAGE_ALIGN(start
+ length
) >> TARGET_PAGE_BITS
;
880 page
= start
>> TARGET_PAGE_BITS
;
881 dirty
= bitmap_test_and_clear_atomic(ram_list
.dirty_memory
[client
],
884 if (dirty
&& tcg_enabled()) {
885 tlb_reset_dirty_range_all(start
, length
);
891 /* Called from RCU critical section */
892 hwaddr
memory_region_section_get_iotlb(CPUState
*cpu
,
893 MemoryRegionSection
*section
,
895 hwaddr paddr
, hwaddr xlat
,
897 target_ulong
*address
)
902 if (memory_region_is_ram(section
->mr
)) {
904 iotlb
= (memory_region_get_ram_addr(section
->mr
) & TARGET_PAGE_MASK
)
906 if (!section
->readonly
) {
907 iotlb
|= PHYS_SECTION_NOTDIRTY
;
909 iotlb
|= PHYS_SECTION_ROM
;
912 iotlb
= section
- section
->address_space
->dispatch
->map
.sections
;
916 /* Make accesses to pages with watchpoints go via the
917 watchpoint trap routines. */
918 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
919 if (cpu_watchpoint_address_matches(wp
, vaddr
, TARGET_PAGE_SIZE
)) {
920 /* Avoid trapping reads of pages with a write breakpoint. */
921 if ((prot
& PAGE_WRITE
) || (wp
->flags
& BP_MEM_READ
)) {
922 iotlb
= PHYS_SECTION_WATCH
+ paddr
;
923 *address
|= TLB_MMIO
;
931 #endif /* defined(CONFIG_USER_ONLY) */
933 #if !defined(CONFIG_USER_ONLY)
935 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
937 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
);
939 static void *(*phys_mem_alloc
)(size_t size
, uint64_t *align
) =
943 * Set a custom physical guest memory alloator.
944 * Accelerators with unusual needs may need this. Hopefully, we can
945 * get rid of it eventually.
947 void phys_mem_set_alloc(void *(*alloc
)(size_t, uint64_t *align
))
949 phys_mem_alloc
= alloc
;
952 static uint16_t phys_section_add(PhysPageMap
*map
,
953 MemoryRegionSection
*section
)
955 /* The physical section number is ORed with a page-aligned
956 * pointer to produce the iotlb entries. Thus it should
957 * never overflow into the page-aligned value.
959 assert(map
->sections_nb
< TARGET_PAGE_SIZE
);
961 if (map
->sections_nb
== map
->sections_nb_alloc
) {
962 map
->sections_nb_alloc
= MAX(map
->sections_nb_alloc
* 2, 16);
963 map
->sections
= g_renew(MemoryRegionSection
, map
->sections
,
964 map
->sections_nb_alloc
);
966 map
->sections
[map
->sections_nb
] = *section
;
967 memory_region_ref(section
->mr
);
968 return map
->sections_nb
++;
971 static void phys_section_destroy(MemoryRegion
*mr
)
973 memory_region_unref(mr
);
976 subpage_t
*subpage
= container_of(mr
, subpage_t
, iomem
);
977 object_unref(OBJECT(&subpage
->iomem
));
982 static void phys_sections_free(PhysPageMap
*map
)
984 while (map
->sections_nb
> 0) {
985 MemoryRegionSection
*section
= &map
->sections
[--map
->sections_nb
];
986 phys_section_destroy(section
->mr
);
988 g_free(map
->sections
);
992 static void register_subpage(AddressSpaceDispatch
*d
, MemoryRegionSection
*section
)
995 hwaddr base
= section
->offset_within_address_space
997 MemoryRegionSection
*existing
= phys_page_find(d
->phys_map
, base
,
998 d
->map
.nodes
, d
->map
.sections
);
999 MemoryRegionSection subsection
= {
1000 .offset_within_address_space
= base
,
1001 .size
= int128_make64(TARGET_PAGE_SIZE
),
1005 assert(existing
->mr
->subpage
|| existing
->mr
== &io_mem_unassigned
);
1007 if (!(existing
->mr
->subpage
)) {
1008 subpage
= subpage_init(d
->as
, base
);
1009 subsection
.address_space
= d
->as
;
1010 subsection
.mr
= &subpage
->iomem
;
1011 phys_page_set(d
, base
>> TARGET_PAGE_BITS
, 1,
1012 phys_section_add(&d
->map
, &subsection
));
1014 subpage
= container_of(existing
->mr
, subpage_t
, iomem
);
1016 start
= section
->offset_within_address_space
& ~TARGET_PAGE_MASK
;
1017 end
= start
+ int128_get64(section
->size
) - 1;
1018 subpage_register(subpage
, start
, end
,
1019 phys_section_add(&d
->map
, section
));
1023 static void register_multipage(AddressSpaceDispatch
*d
,
1024 MemoryRegionSection
*section
)
1026 hwaddr start_addr
= section
->offset_within_address_space
;
1027 uint16_t section_index
= phys_section_add(&d
->map
, section
);
1028 uint64_t num_pages
= int128_get64(int128_rshift(section
->size
,
1032 phys_page_set(d
, start_addr
>> TARGET_PAGE_BITS
, num_pages
, section_index
);
1035 static void mem_add(MemoryListener
*listener
, MemoryRegionSection
*section
)
1037 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
1038 AddressSpaceDispatch
*d
= as
->next_dispatch
;
1039 MemoryRegionSection now
= *section
, remain
= *section
;
1040 Int128 page_size
= int128_make64(TARGET_PAGE_SIZE
);
1042 if (now
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
1043 uint64_t left
= TARGET_PAGE_ALIGN(now
.offset_within_address_space
)
1044 - now
.offset_within_address_space
;
1046 now
.size
= int128_min(int128_make64(left
), now
.size
);
1047 register_subpage(d
, &now
);
1049 now
.size
= int128_zero();
1051 while (int128_ne(remain
.size
, now
.size
)) {
1052 remain
.size
= int128_sub(remain
.size
, now
.size
);
1053 remain
.offset_within_address_space
+= int128_get64(now
.size
);
1054 remain
.offset_within_region
+= int128_get64(now
.size
);
1056 if (int128_lt(remain
.size
, page_size
)) {
1057 register_subpage(d
, &now
);
1058 } else if (remain
.offset_within_address_space
& ~TARGET_PAGE_MASK
) {
1059 now
.size
= page_size
;
1060 register_subpage(d
, &now
);
1062 now
.size
= int128_and(now
.size
, int128_neg(page_size
));
1063 register_multipage(d
, &now
);
1068 void qemu_flush_coalesced_mmio_buffer(void)
1071 kvm_flush_coalesced_mmio_buffer();
1074 void qemu_mutex_lock_ramlist(void)
1076 qemu_mutex_lock(&ram_list
.mutex
);
1079 void qemu_mutex_unlock_ramlist(void)
1081 qemu_mutex_unlock(&ram_list
.mutex
);
1086 #include <sys/vfs.h>
1088 #define HUGETLBFS_MAGIC 0x958458f6
1090 static long gethugepagesize(const char *path
, Error
**errp
)
1096 ret
= statfs(path
, &fs
);
1097 } while (ret
!= 0 && errno
== EINTR
);
1100 error_setg_errno(errp
, errno
, "failed to get page size of file %s",
1105 if (fs
.f_type
!= HUGETLBFS_MAGIC
)
1106 fprintf(stderr
, "Warning: path not on HugeTLBFS: %s\n", path
);
1111 static void *file_ram_alloc(RAMBlock
*block
,
1117 char *sanitized_name
;
1122 Error
*local_err
= NULL
;
1124 hpagesize
= gethugepagesize(path
, &local_err
);
1126 error_propagate(errp
, local_err
);
1129 block
->mr
->align
= hpagesize
;
1131 if (memory
< hpagesize
) {
1132 error_setg(errp
, "memory size 0x" RAM_ADDR_FMT
" must be equal to "
1133 "or larger than huge page size 0x%" PRIx64
,
1138 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1140 "host lacks kvm mmu notifiers, -mem-path unsupported");
1144 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1145 sanitized_name
= g_strdup(memory_region_name(block
->mr
));
1146 for (c
= sanitized_name
; *c
!= '\0'; c
++) {
1151 filename
= g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path
,
1153 g_free(sanitized_name
);
1155 fd
= mkstemp(filename
);
1157 error_setg_errno(errp
, errno
,
1158 "unable to create backing store for hugepages");
1165 memory
= (memory
+hpagesize
-1) & ~(hpagesize
-1);
1168 * ftruncate is not supported by hugetlbfs in older
1169 * hosts, so don't bother bailing out on errors.
1170 * If anything goes wrong with it under other filesystems,
1173 if (ftruncate(fd
, memory
)) {
1174 perror("ftruncate");
1177 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
,
1178 (block
->flags
& RAM_SHARED
? MAP_SHARED
: MAP_PRIVATE
),
1180 if (area
== MAP_FAILED
) {
1181 error_setg_errno(errp
, errno
,
1182 "unable to map backing store for hugepages");
1188 os_mem_prealloc(fd
, area
, memory
);
1196 error_report("%s", error_get_pretty(*errp
));
1203 /* Called with the ramlist lock held. */
1204 static ram_addr_t
find_ram_offset(ram_addr_t size
)
1206 RAMBlock
*block
, *next_block
;
1207 ram_addr_t offset
= RAM_ADDR_MAX
, mingap
= RAM_ADDR_MAX
;
1209 assert(size
!= 0); /* it would hand out same offset multiple times */
1211 if (QLIST_EMPTY_RCU(&ram_list
.blocks
)) {
1215 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1216 ram_addr_t end
, next
= RAM_ADDR_MAX
;
1218 end
= block
->offset
+ block
->max_length
;
1220 QLIST_FOREACH_RCU(next_block
, &ram_list
.blocks
, next
) {
1221 if (next_block
->offset
>= end
) {
1222 next
= MIN(next
, next_block
->offset
);
1225 if (next
- end
>= size
&& next
- end
< mingap
) {
1227 mingap
= next
- end
;
1231 if (offset
== RAM_ADDR_MAX
) {
1232 fprintf(stderr
, "Failed to find gap of requested size: %" PRIu64
"\n",
1240 ram_addr_t
last_ram_offset(void)
1243 ram_addr_t last
= 0;
1246 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1247 last
= MAX(last
, block
->offset
+ block
->max_length
);
1253 static void qemu_ram_setup_dump(void *addr
, ram_addr_t size
)
1257 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1258 if (!machine_dump_guest_core(current_machine
)) {
1259 ret
= qemu_madvise(addr
, size
, QEMU_MADV_DONTDUMP
);
1261 perror("qemu_madvise");
1262 fprintf(stderr
, "madvise doesn't support MADV_DONTDUMP, "
1263 "but dump_guest_core=off specified\n");
1268 /* Called within an RCU critical section, or while the ramlist lock
1271 static RAMBlock
*find_ram_block(ram_addr_t addr
)
1275 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1276 if (block
->offset
== addr
) {
1284 /* Called with iothread lock held. */
1285 void qemu_ram_set_idstr(ram_addr_t addr
, const char *name
, DeviceState
*dev
)
1287 RAMBlock
*new_block
, *block
;
1290 new_block
= find_ram_block(addr
);
1292 assert(!new_block
->idstr
[0]);
1295 char *id
= qdev_get_dev_path(dev
);
1297 snprintf(new_block
->idstr
, sizeof(new_block
->idstr
), "%s/", id
);
1301 pstrcat(new_block
->idstr
, sizeof(new_block
->idstr
), name
);
1303 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1304 if (block
!= new_block
&& !strcmp(block
->idstr
, new_block
->idstr
)) {
1305 fprintf(stderr
, "RAMBlock \"%s\" already registered, abort!\n",
1313 /* Called with iothread lock held. */
1314 void qemu_ram_unset_idstr(ram_addr_t addr
)
1318 /* FIXME: arch_init.c assumes that this is not called throughout
1319 * migration. Ignore the problem since hot-unplug during migration
1320 * does not work anyway.
1324 block
= find_ram_block(addr
);
1326 memset(block
->idstr
, 0, sizeof(block
->idstr
));
1331 static int memory_try_enable_merging(void *addr
, size_t len
)
1333 if (!machine_mem_merge(current_machine
)) {
1334 /* disabled by the user */
1338 return qemu_madvise(addr
, len
, QEMU_MADV_MERGEABLE
);
1341 /* Only legal before guest might have detected the memory size: e.g. on
1342 * incoming migration, or right after reset.
1344 * As memory core doesn't know how is memory accessed, it is up to
1345 * resize callback to update device state and/or add assertions to detect
1346 * misuse, if necessary.
1348 int qemu_ram_resize(ram_addr_t base
, ram_addr_t newsize
, Error
**errp
)
1350 RAMBlock
*block
= find_ram_block(base
);
1354 newsize
= TARGET_PAGE_ALIGN(newsize
);
1356 if (block
->used_length
== newsize
) {
1360 if (!(block
->flags
& RAM_RESIZEABLE
)) {
1361 error_setg_errno(errp
, EINVAL
,
1362 "Length mismatch: %s: 0x" RAM_ADDR_FMT
1363 " in != 0x" RAM_ADDR_FMT
, block
->idstr
,
1364 newsize
, block
->used_length
);
1368 if (block
->max_length
< newsize
) {
1369 error_setg_errno(errp
, EINVAL
,
1370 "Length too large: %s: 0x" RAM_ADDR_FMT
1371 " > 0x" RAM_ADDR_FMT
, block
->idstr
,
1372 newsize
, block
->max_length
);
1376 cpu_physical_memory_clear_dirty_range(block
->offset
, block
->used_length
);
1377 block
->used_length
= newsize
;
1378 cpu_physical_memory_set_dirty_range(block
->offset
, block
->used_length
,
1380 memory_region_set_size(block
->mr
, newsize
);
1381 if (block
->resized
) {
1382 block
->resized(block
->idstr
, newsize
, block
->host
);
1387 static ram_addr_t
ram_block_add(RAMBlock
*new_block
, Error
**errp
)
1390 RAMBlock
*last_block
= NULL
;
1391 ram_addr_t old_ram_size
, new_ram_size
;
1393 old_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1395 qemu_mutex_lock_ramlist();
1396 new_block
->offset
= find_ram_offset(new_block
->max_length
);
1398 if (!new_block
->host
) {
1399 if (xen_enabled()) {
1400 xen_ram_alloc(new_block
->offset
, new_block
->max_length
,
1403 new_block
->host
= phys_mem_alloc(new_block
->max_length
,
1404 &new_block
->mr
->align
);
1405 if (!new_block
->host
) {
1406 error_setg_errno(errp
, errno
,
1407 "cannot set up guest memory '%s'",
1408 memory_region_name(new_block
->mr
));
1409 qemu_mutex_unlock_ramlist();
1412 memory_try_enable_merging(new_block
->host
, new_block
->max_length
);
1416 new_ram_size
= MAX(old_ram_size
,
1417 (new_block
->offset
+ new_block
->max_length
) >> TARGET_PAGE_BITS
);
1418 if (new_ram_size
> old_ram_size
) {
1419 migration_bitmap_extend(old_ram_size
, new_ram_size
);
1421 /* Keep the list sorted from biggest to smallest block. Unlike QTAILQ,
1422 * QLIST (which has an RCU-friendly variant) does not have insertion at
1423 * tail, so save the last element in last_block.
1425 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1427 if (block
->max_length
< new_block
->max_length
) {
1432 QLIST_INSERT_BEFORE_RCU(block
, new_block
, next
);
1433 } else if (last_block
) {
1434 QLIST_INSERT_AFTER_RCU(last_block
, new_block
, next
);
1435 } else { /* list is empty */
1436 QLIST_INSERT_HEAD_RCU(&ram_list
.blocks
, new_block
, next
);
1438 ram_list
.mru_block
= NULL
;
1440 /* Write list before version */
1443 qemu_mutex_unlock_ramlist();
1445 new_ram_size
= last_ram_offset() >> TARGET_PAGE_BITS
;
1447 if (new_ram_size
> old_ram_size
) {
1450 /* ram_list.dirty_memory[] is protected by the iothread lock. */
1451 for (i
= 0; i
< DIRTY_MEMORY_NUM
; i
++) {
1452 ram_list
.dirty_memory
[i
] =
1453 bitmap_zero_extend(ram_list
.dirty_memory
[i
],
1454 old_ram_size
, new_ram_size
);
1457 cpu_physical_memory_set_dirty_range(new_block
->offset
,
1458 new_block
->used_length
,
1461 if (new_block
->host
) {
1462 qemu_ram_setup_dump(new_block
->host
, new_block
->max_length
);
1463 qemu_madvise(new_block
->host
, new_block
->max_length
, QEMU_MADV_HUGEPAGE
);
1464 qemu_madvise(new_block
->host
, new_block
->max_length
, QEMU_MADV_DONTFORK
);
1465 if (kvm_enabled()) {
1466 kvm_setup_guest_memory(new_block
->host
, new_block
->max_length
);
1470 return new_block
->offset
;
1474 ram_addr_t
qemu_ram_alloc_from_file(ram_addr_t size
, MemoryRegion
*mr
,
1475 bool share
, const char *mem_path
,
1478 RAMBlock
*new_block
;
1480 Error
*local_err
= NULL
;
1482 if (xen_enabled()) {
1483 error_setg(errp
, "-mem-path not supported with Xen");
1487 if (phys_mem_alloc
!= qemu_anon_ram_alloc
) {
1489 * file_ram_alloc() needs to allocate just like
1490 * phys_mem_alloc, but we haven't bothered to provide
1494 "-mem-path not supported with this accelerator");
1498 size
= TARGET_PAGE_ALIGN(size
);
1499 new_block
= g_malloc0(sizeof(*new_block
));
1501 new_block
->used_length
= size
;
1502 new_block
->max_length
= size
;
1503 new_block
->flags
= share
? RAM_SHARED
: 0;
1504 new_block
->host
= file_ram_alloc(new_block
, size
,
1506 if (!new_block
->host
) {
1511 addr
= ram_block_add(new_block
, &local_err
);
1514 error_propagate(errp
, local_err
);
1522 ram_addr_t
qemu_ram_alloc_internal(ram_addr_t size
, ram_addr_t max_size
,
1523 void (*resized
)(const char*,
1526 void *host
, bool resizeable
,
1527 MemoryRegion
*mr
, Error
**errp
)
1529 RAMBlock
*new_block
;
1531 Error
*local_err
= NULL
;
1533 size
= TARGET_PAGE_ALIGN(size
);
1534 max_size
= TARGET_PAGE_ALIGN(max_size
);
1535 new_block
= g_malloc0(sizeof(*new_block
));
1537 new_block
->resized
= resized
;
1538 new_block
->used_length
= size
;
1539 new_block
->max_length
= max_size
;
1540 assert(max_size
>= size
);
1542 new_block
->host
= host
;
1544 new_block
->flags
|= RAM_PREALLOC
;
1547 new_block
->flags
|= RAM_RESIZEABLE
;
1549 addr
= ram_block_add(new_block
, &local_err
);
1552 error_propagate(errp
, local_err
);
1558 ram_addr_t
qemu_ram_alloc_from_ptr(ram_addr_t size
, void *host
,
1559 MemoryRegion
*mr
, Error
**errp
)
1561 return qemu_ram_alloc_internal(size
, size
, NULL
, host
, false, mr
, errp
);
1564 ram_addr_t
qemu_ram_alloc(ram_addr_t size
, MemoryRegion
*mr
, Error
**errp
)
1566 return qemu_ram_alloc_internal(size
, size
, NULL
, NULL
, false, mr
, errp
);
1569 ram_addr_t
qemu_ram_alloc_resizeable(ram_addr_t size
, ram_addr_t maxsz
,
1570 void (*resized
)(const char*,
1573 MemoryRegion
*mr
, Error
**errp
)
1575 return qemu_ram_alloc_internal(size
, maxsz
, resized
, NULL
, true, mr
, errp
);
1578 void qemu_ram_free_from_ptr(ram_addr_t addr
)
1582 qemu_mutex_lock_ramlist();
1583 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1584 if (addr
== block
->offset
) {
1585 QLIST_REMOVE_RCU(block
, next
);
1586 ram_list
.mru_block
= NULL
;
1587 /* Write list before version */
1590 g_free_rcu(block
, rcu
);
1594 qemu_mutex_unlock_ramlist();
1597 static void reclaim_ramblock(RAMBlock
*block
)
1599 if (block
->flags
& RAM_PREALLOC
) {
1601 } else if (xen_enabled()) {
1602 xen_invalidate_map_cache_entry(block
->host
);
1604 } else if (block
->fd
>= 0) {
1605 munmap(block
->host
, block
->max_length
);
1609 qemu_anon_ram_free(block
->host
, block
->max_length
);
1614 void qemu_ram_free(ram_addr_t addr
)
1618 qemu_mutex_lock_ramlist();
1619 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1620 if (addr
== block
->offset
) {
1621 QLIST_REMOVE_RCU(block
, next
);
1622 ram_list
.mru_block
= NULL
;
1623 /* Write list before version */
1626 call_rcu(block
, reclaim_ramblock
, rcu
);
1630 qemu_mutex_unlock_ramlist();
1634 void qemu_ram_remap(ram_addr_t addr
, ram_addr_t length
)
1641 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1642 offset
= addr
- block
->offset
;
1643 if (offset
< block
->max_length
) {
1644 vaddr
= ramblock_ptr(block
, offset
);
1645 if (block
->flags
& RAM_PREALLOC
) {
1647 } else if (xen_enabled()) {
1651 if (block
->fd
>= 0) {
1652 flags
|= (block
->flags
& RAM_SHARED
?
1653 MAP_SHARED
: MAP_PRIVATE
);
1654 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1655 flags
, block
->fd
, offset
);
1658 * Remap needs to match alloc. Accelerators that
1659 * set phys_mem_alloc never remap. If they did,
1660 * we'd need a remap hook here.
1662 assert(phys_mem_alloc
== qemu_anon_ram_alloc
);
1664 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
1665 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
1668 if (area
!= vaddr
) {
1669 fprintf(stderr
, "Could not remap addr: "
1670 RAM_ADDR_FMT
"@" RAM_ADDR_FMT
"\n",
1674 memory_try_enable_merging(vaddr
, length
);
1675 qemu_ram_setup_dump(vaddr
, length
);
1680 #endif /* !_WIN32 */
1682 int qemu_get_ram_fd(ram_addr_t addr
)
1688 block
= qemu_get_ram_block(addr
);
1694 void *qemu_get_ram_block_host_ptr(ram_addr_t addr
)
1700 block
= qemu_get_ram_block(addr
);
1701 ptr
= ramblock_ptr(block
, 0);
1706 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1707 * This should not be used for general purpose DMA. Use address_space_map
1708 * or address_space_rw instead. For local memory (e.g. video ram) that the
1709 * device owns, use memory_region_get_ram_ptr.
1711 * By the time this function returns, the returned pointer is not protected
1712 * by RCU anymore. If the caller is not within an RCU critical section and
1713 * does not hold the iothread lock, it must have other means of protecting the
1714 * pointer, such as a reference to the region that includes the incoming
1717 void *qemu_get_ram_ptr(ram_addr_t addr
)
1723 block
= qemu_get_ram_block(addr
);
1725 if (xen_enabled() && block
->host
== NULL
) {
1726 /* We need to check if the requested address is in the RAM
1727 * because we don't want to map the entire memory in QEMU.
1728 * In that case just map until the end of the page.
1730 if (block
->offset
== 0) {
1731 ptr
= xen_map_cache(addr
, 0, 0);
1735 block
->host
= xen_map_cache(block
->offset
, block
->max_length
, 1);
1737 ptr
= ramblock_ptr(block
, addr
- block
->offset
);
1744 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1745 * but takes a size argument.
1747 * By the time this function returns, the returned pointer is not protected
1748 * by RCU anymore. If the caller is not within an RCU critical section and
1749 * does not hold the iothread lock, it must have other means of protecting the
1750 * pointer, such as a reference to the region that includes the incoming
1753 static void *qemu_ram_ptr_length(ram_addr_t addr
, hwaddr
*size
)
1759 if (xen_enabled()) {
1760 return xen_map_cache(addr
, *size
, 1);
1764 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1765 if (addr
- block
->offset
< block
->max_length
) {
1766 if (addr
- block
->offset
+ *size
> block
->max_length
)
1767 *size
= block
->max_length
- addr
+ block
->offset
;
1768 ptr
= ramblock_ptr(block
, addr
- block
->offset
);
1774 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
1779 /* Some of the softmmu routines need to translate from a host pointer
1780 * (typically a TLB entry) back to a ram offset.
1782 * By the time this function returns, the returned pointer is not protected
1783 * by RCU anymore. If the caller is not within an RCU critical section and
1784 * does not hold the iothread lock, it must have other means of protecting the
1785 * pointer, such as a reference to the region that includes the incoming
1788 MemoryRegion
*qemu_ram_addr_from_host(void *ptr
, ram_addr_t
*ram_addr
)
1791 uint8_t *host
= ptr
;
1794 if (xen_enabled()) {
1796 *ram_addr
= xen_ram_addr_from_mapcache(ptr
);
1797 mr
= qemu_get_ram_block(*ram_addr
)->mr
;
1803 block
= atomic_rcu_read(&ram_list
.mru_block
);
1804 if (block
&& block
->host
&& host
- block
->host
< block
->max_length
) {
1808 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
1809 /* This case append when the block is not mapped. */
1810 if (block
->host
== NULL
) {
1813 if (host
- block
->host
< block
->max_length
) {
1822 *ram_addr
= block
->offset
+ (host
- block
->host
);
1828 static void notdirty_mem_write(void *opaque
, hwaddr ram_addr
,
1829 uint64_t val
, unsigned size
)
1831 if (!cpu_physical_memory_get_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
)) {
1832 tb_invalidate_phys_page_fast(ram_addr
, size
);
1836 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
1839 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
1842 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
1847 /* Set both VGA and migration bits for simplicity and to remove
1848 * the notdirty callback faster.
1850 cpu_physical_memory_set_dirty_range(ram_addr
, size
,
1851 DIRTY_CLIENTS_NOCODE
);
1852 /* we remove the notdirty callback only if the code has been
1854 if (!cpu_physical_memory_is_clean(ram_addr
)) {
1855 CPUArchState
*env
= current_cpu
->env_ptr
;
1856 tlb_set_dirty(env
, current_cpu
->mem_io_vaddr
);
1860 static bool notdirty_mem_accepts(void *opaque
, hwaddr addr
,
1861 unsigned size
, bool is_write
)
1866 static const MemoryRegionOps notdirty_mem_ops
= {
1867 .write
= notdirty_mem_write
,
1868 .valid
.accepts
= notdirty_mem_accepts
,
1869 .endianness
= DEVICE_NATIVE_ENDIAN
,
1872 /* Generate a debug exception if a watchpoint has been hit. */
1873 static void check_watchpoint(int offset
, int len
, MemTxAttrs attrs
, int flags
)
1875 CPUState
*cpu
= current_cpu
;
1876 CPUArchState
*env
= cpu
->env_ptr
;
1877 target_ulong pc
, cs_base
;
1882 if (cpu
->watchpoint_hit
) {
1883 /* We re-entered the check after replacing the TB. Now raise
1884 * the debug interrupt so that is will trigger after the
1885 * current instruction. */
1886 cpu_interrupt(cpu
, CPU_INTERRUPT_DEBUG
);
1889 vaddr
= (cpu
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
1890 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
1891 if (cpu_watchpoint_address_matches(wp
, vaddr
, len
)
1892 && (wp
->flags
& flags
)) {
1893 if (flags
== BP_MEM_READ
) {
1894 wp
->flags
|= BP_WATCHPOINT_HIT_READ
;
1896 wp
->flags
|= BP_WATCHPOINT_HIT_WRITE
;
1898 wp
->hitaddr
= vaddr
;
1899 wp
->hitattrs
= attrs
;
1900 if (!cpu
->watchpoint_hit
) {
1901 cpu
->watchpoint_hit
= wp
;
1902 tb_check_watchpoint(cpu
);
1903 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
1904 cpu
->exception_index
= EXCP_DEBUG
;
1907 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
1908 tb_gen_code(cpu
, pc
, cs_base
, cpu_flags
, 1);
1909 cpu_resume_from_signal(cpu
, NULL
);
1913 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
1918 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1919 so these check for a hit then pass through to the normal out-of-line
1921 static MemTxResult
watch_mem_read(void *opaque
, hwaddr addr
, uint64_t *pdata
,
1922 unsigned size
, MemTxAttrs attrs
)
1927 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, size
, attrs
, BP_MEM_READ
);
1930 data
= address_space_ldub(&address_space_memory
, addr
, attrs
, &res
);
1933 data
= address_space_lduw(&address_space_memory
, addr
, attrs
, &res
);
1936 data
= address_space_ldl(&address_space_memory
, addr
, attrs
, &res
);
1944 static MemTxResult
watch_mem_write(void *opaque
, hwaddr addr
,
1945 uint64_t val
, unsigned size
,
1950 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, size
, attrs
, BP_MEM_WRITE
);
1953 address_space_stb(&address_space_memory
, addr
, val
, attrs
, &res
);
1956 address_space_stw(&address_space_memory
, addr
, val
, attrs
, &res
);
1959 address_space_stl(&address_space_memory
, addr
, val
, attrs
, &res
);
1966 static const MemoryRegionOps watch_mem_ops
= {
1967 .read_with_attrs
= watch_mem_read
,
1968 .write_with_attrs
= watch_mem_write
,
1969 .endianness
= DEVICE_NATIVE_ENDIAN
,
1972 static MemTxResult
subpage_read(void *opaque
, hwaddr addr
, uint64_t *data
,
1973 unsigned len
, MemTxAttrs attrs
)
1975 subpage_t
*subpage
= opaque
;
1979 #if defined(DEBUG_SUBPAGE)
1980 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
"\n", __func__
,
1981 subpage
, len
, addr
);
1983 res
= address_space_read(subpage
->as
, addr
+ subpage
->base
,
1990 *data
= ldub_p(buf
);
1993 *data
= lduw_p(buf
);
2006 static MemTxResult
subpage_write(void *opaque
, hwaddr addr
,
2007 uint64_t value
, unsigned len
, MemTxAttrs attrs
)
2009 subpage_t
*subpage
= opaque
;
2012 #if defined(DEBUG_SUBPAGE)
2013 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
2014 " value %"PRIx64
"\n",
2015 __func__
, subpage
, len
, addr
, value
);
2033 return address_space_write(subpage
->as
, addr
+ subpage
->base
,
2037 static bool subpage_accepts(void *opaque
, hwaddr addr
,
2038 unsigned len
, bool is_write
)
2040 subpage_t
*subpage
= opaque
;
2041 #if defined(DEBUG_SUBPAGE)
2042 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx
"\n",
2043 __func__
, subpage
, is_write
? 'w' : 'r', len
, addr
);
2046 return address_space_access_valid(subpage
->as
, addr
+ subpage
->base
,
2050 static const MemoryRegionOps subpage_ops
= {
2051 .read_with_attrs
= subpage_read
,
2052 .write_with_attrs
= subpage_write
,
2053 .impl
.min_access_size
= 1,
2054 .impl
.max_access_size
= 8,
2055 .valid
.min_access_size
= 1,
2056 .valid
.max_access_size
= 8,
2057 .valid
.accepts
= subpage_accepts
,
2058 .endianness
= DEVICE_NATIVE_ENDIAN
,
2061 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2066 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
2068 idx
= SUBPAGE_IDX(start
);
2069 eidx
= SUBPAGE_IDX(end
);
2070 #if defined(DEBUG_SUBPAGE)
2071 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2072 __func__
, mmio
, start
, end
, idx
, eidx
, section
);
2074 for (; idx
<= eidx
; idx
++) {
2075 mmio
->sub_section
[idx
] = section
;
2081 static subpage_t
*subpage_init(AddressSpace
*as
, hwaddr base
)
2085 mmio
= g_malloc0(sizeof(subpage_t
));
2089 memory_region_init_io(&mmio
->iomem
, NULL
, &subpage_ops
, mmio
,
2090 NULL
, TARGET_PAGE_SIZE
);
2091 mmio
->iomem
.subpage
= true;
2092 #if defined(DEBUG_SUBPAGE)
2093 printf("%s: %p base " TARGET_FMT_plx
" len %08x\n", __func__
,
2094 mmio
, base
, TARGET_PAGE_SIZE
);
2096 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
-1, PHYS_SECTION_UNASSIGNED
);
2101 static uint16_t dummy_section(PhysPageMap
*map
, AddressSpace
*as
,
2105 MemoryRegionSection section
= {
2106 .address_space
= as
,
2108 .offset_within_address_space
= 0,
2109 .offset_within_region
= 0,
2110 .size
= int128_2_64(),
2113 return phys_section_add(map
, §ion
);
2116 MemoryRegion
*iotlb_to_region(CPUState
*cpu
, hwaddr index
)
2118 AddressSpaceDispatch
*d
= atomic_rcu_read(&cpu
->memory_dispatch
);
2119 MemoryRegionSection
*sections
= d
->map
.sections
;
2121 return sections
[index
& ~TARGET_PAGE_MASK
].mr
;
2124 static void io_mem_init(void)
2126 memory_region_init_io(&io_mem_rom
, NULL
, &unassigned_mem_ops
, NULL
, NULL
, UINT64_MAX
);
2127 memory_region_init_io(&io_mem_unassigned
, NULL
, &unassigned_mem_ops
, NULL
,
2129 memory_region_init_io(&io_mem_notdirty
, NULL
, ¬dirty_mem_ops
, NULL
,
2131 memory_region_init_io(&io_mem_watch
, NULL
, &watch_mem_ops
, NULL
,
2135 static void mem_begin(MemoryListener
*listener
)
2137 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
2138 AddressSpaceDispatch
*d
= g_new0(AddressSpaceDispatch
, 1);
2141 n
= dummy_section(&d
->map
, as
, &io_mem_unassigned
);
2142 assert(n
== PHYS_SECTION_UNASSIGNED
);
2143 n
= dummy_section(&d
->map
, as
, &io_mem_notdirty
);
2144 assert(n
== PHYS_SECTION_NOTDIRTY
);
2145 n
= dummy_section(&d
->map
, as
, &io_mem_rom
);
2146 assert(n
== PHYS_SECTION_ROM
);
2147 n
= dummy_section(&d
->map
, as
, &io_mem_watch
);
2148 assert(n
== PHYS_SECTION_WATCH
);
2150 d
->phys_map
= (PhysPageEntry
) { .ptr
= PHYS_MAP_NODE_NIL
, .skip
= 1 };
2152 as
->next_dispatch
= d
;
2155 static void address_space_dispatch_free(AddressSpaceDispatch
*d
)
2157 phys_sections_free(&d
->map
);
2161 static void mem_commit(MemoryListener
*listener
)
2163 AddressSpace
*as
= container_of(listener
, AddressSpace
, dispatch_listener
);
2164 AddressSpaceDispatch
*cur
= as
->dispatch
;
2165 AddressSpaceDispatch
*next
= as
->next_dispatch
;
2167 phys_page_compact_all(next
, next
->map
.nodes_nb
);
2169 atomic_rcu_set(&as
->dispatch
, next
);
2171 call_rcu(cur
, address_space_dispatch_free
, rcu
);
2175 static void tcg_commit(MemoryListener
*listener
)
2179 /* since each CPU stores ram addresses in its TLB cache, we must
2180 reset the modified entries */
2183 /* FIXME: Disentangle the cpu.h circular files deps so we can
2184 directly get the right CPU from listener. */
2185 if (cpu
->tcg_as_listener
!= listener
) {
2188 cpu_reload_memory_map(cpu
);
2192 void address_space_init_dispatch(AddressSpace
*as
)
2194 as
->dispatch
= NULL
;
2195 as
->dispatch_listener
= (MemoryListener
) {
2197 .commit
= mem_commit
,
2198 .region_add
= mem_add
,
2199 .region_nop
= mem_add
,
2202 memory_listener_register(&as
->dispatch_listener
, as
);
2205 void address_space_unregister(AddressSpace
*as
)
2207 memory_listener_unregister(&as
->dispatch_listener
);
2210 void address_space_destroy_dispatch(AddressSpace
*as
)
2212 AddressSpaceDispatch
*d
= as
->dispatch
;
2214 atomic_rcu_set(&as
->dispatch
, NULL
);
2216 call_rcu(d
, address_space_dispatch_free
, rcu
);
2220 static void memory_map_init(void)
2222 system_memory
= g_malloc(sizeof(*system_memory
));
2224 memory_region_init(system_memory
, NULL
, "system", UINT64_MAX
);
2225 address_space_init(&address_space_memory
, system_memory
, "memory");
2227 system_io
= g_malloc(sizeof(*system_io
));
2228 memory_region_init_io(system_io
, NULL
, &unassigned_io_ops
, NULL
, "io",
2230 address_space_init(&address_space_io
, system_io
, "I/O");
2233 MemoryRegion
*get_system_memory(void)
2235 return system_memory
;
2238 MemoryRegion
*get_system_io(void)
2243 #endif /* !defined(CONFIG_USER_ONLY) */
2245 /* physical memory access (slow version, mainly for debug) */
2246 #if defined(CONFIG_USER_ONLY)
2247 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
2248 uint8_t *buf
, int len
, int is_write
)
2255 page
= addr
& TARGET_PAGE_MASK
;
2256 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2259 flags
= page_get_flags(page
);
2260 if (!(flags
& PAGE_VALID
))
2263 if (!(flags
& PAGE_WRITE
))
2265 /* XXX: this code should not depend on lock_user */
2266 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
2269 unlock_user(p
, addr
, l
);
2271 if (!(flags
& PAGE_READ
))
2273 /* XXX: this code should not depend on lock_user */
2274 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
2277 unlock_user(p
, addr
, 0);
2288 static void invalidate_and_set_dirty(MemoryRegion
*mr
, hwaddr addr
,
2291 uint8_t dirty_log_mask
= memory_region_get_dirty_log_mask(mr
);
2292 /* No early return if dirty_log_mask is or becomes 0, because
2293 * cpu_physical_memory_set_dirty_range will still call
2294 * xen_modified_memory.
2296 if (dirty_log_mask
) {
2298 cpu_physical_memory_range_includes_clean(addr
, length
, dirty_log_mask
);
2300 if (dirty_log_mask
& (1 << DIRTY_MEMORY_CODE
)) {
2301 tb_invalidate_phys_range(addr
, addr
+ length
);
2302 dirty_log_mask
&= ~(1 << DIRTY_MEMORY_CODE
);
2304 cpu_physical_memory_set_dirty_range(addr
, length
, dirty_log_mask
);
2307 static int memory_access_size(MemoryRegion
*mr
, unsigned l
, hwaddr addr
)
2309 unsigned access_size_max
= mr
->ops
->valid
.max_access_size
;
2311 /* Regions are assumed to support 1-4 byte accesses unless
2312 otherwise specified. */
2313 if (access_size_max
== 0) {
2314 access_size_max
= 4;
2317 /* Bound the maximum access by the alignment of the address. */
2318 if (!mr
->ops
->impl
.unaligned
) {
2319 unsigned align_size_max
= addr
& -addr
;
2320 if (align_size_max
!= 0 && align_size_max
< access_size_max
) {
2321 access_size_max
= align_size_max
;
2325 /* Don't attempt accesses larger than the maximum. */
2326 if (l
> access_size_max
) {
2327 l
= access_size_max
;
2330 l
= 1 << (qemu_fls(l
) - 1);
2336 static bool prepare_mmio_access(MemoryRegion
*mr
)
2338 bool unlocked
= !qemu_mutex_iothread_locked();
2339 bool release_lock
= false;
2341 if (unlocked
&& mr
->global_locking
) {
2342 qemu_mutex_lock_iothread();
2344 release_lock
= true;
2346 if (mr
->flush_coalesced_mmio
) {
2348 qemu_mutex_lock_iothread();
2350 qemu_flush_coalesced_mmio_buffer();
2352 qemu_mutex_unlock_iothread();
2356 return release_lock
;
2359 MemTxResult
address_space_rw(AddressSpace
*as
, hwaddr addr
, MemTxAttrs attrs
,
2360 uint8_t *buf
, int len
, bool is_write
)
2367 MemTxResult result
= MEMTX_OK
;
2368 bool release_lock
= false;
2373 mr
= address_space_translate(as
, addr
, &addr1
, &l
, is_write
);
2376 if (!memory_access_is_direct(mr
, is_write
)) {
2377 release_lock
|= prepare_mmio_access(mr
);
2378 l
= memory_access_size(mr
, l
, addr1
);
2379 /* XXX: could force current_cpu to NULL to avoid
2383 /* 64 bit write access */
2385 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 8,
2389 /* 32 bit write access */
2391 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 4,
2395 /* 16 bit write access */
2397 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 2,
2401 /* 8 bit write access */
2403 result
|= memory_region_dispatch_write(mr
, addr1
, val
, 1,
2410 addr1
+= memory_region_get_ram_addr(mr
);
2412 ptr
= qemu_get_ram_ptr(addr1
);
2413 memcpy(ptr
, buf
, l
);
2414 invalidate_and_set_dirty(mr
, addr1
, l
);
2417 if (!memory_access_is_direct(mr
, is_write
)) {
2419 release_lock
|= prepare_mmio_access(mr
);
2420 l
= memory_access_size(mr
, l
, addr1
);
2423 /* 64 bit read access */
2424 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 8,
2429 /* 32 bit read access */
2430 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 4,
2435 /* 16 bit read access */
2436 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 2,
2441 /* 8 bit read access */
2442 result
|= memory_region_dispatch_read(mr
, addr1
, &val
, 1,
2451 ptr
= qemu_get_ram_ptr(mr
->ram_addr
+ addr1
);
2452 memcpy(buf
, ptr
, l
);
2457 qemu_mutex_unlock_iothread();
2458 release_lock
= false;
2470 MemTxResult
address_space_write(AddressSpace
*as
, hwaddr addr
, MemTxAttrs attrs
,
2471 const uint8_t *buf
, int len
)
2473 return address_space_rw(as
, addr
, attrs
, (uint8_t *)buf
, len
, true);
2476 MemTxResult
address_space_read(AddressSpace
*as
, hwaddr addr
, MemTxAttrs attrs
,
2477 uint8_t *buf
, int len
)
2479 return address_space_rw(as
, addr
, attrs
, buf
, len
, false);
2483 void cpu_physical_memory_rw(hwaddr addr
, uint8_t *buf
,
2484 int len
, int is_write
)
2486 address_space_rw(&address_space_memory
, addr
, MEMTXATTRS_UNSPECIFIED
,
2487 buf
, len
, is_write
);
2490 enum write_rom_type
{
2495 static inline void cpu_physical_memory_write_rom_internal(AddressSpace
*as
,
2496 hwaddr addr
, const uint8_t *buf
, int len
, enum write_rom_type type
)
2506 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
2508 if (!(memory_region_is_ram(mr
) ||
2509 memory_region_is_romd(mr
))) {
2510 l
= memory_access_size(mr
, l
, addr1
);
2512 addr1
+= memory_region_get_ram_addr(mr
);
2514 ptr
= qemu_get_ram_ptr(addr1
);
2517 memcpy(ptr
, buf
, l
);
2518 invalidate_and_set_dirty(mr
, addr1
, l
);
2521 flush_icache_range((uintptr_t)ptr
, (uintptr_t)ptr
+ l
);
2532 /* used for ROM loading : can write in RAM and ROM */
2533 void cpu_physical_memory_write_rom(AddressSpace
*as
, hwaddr addr
,
2534 const uint8_t *buf
, int len
)
2536 cpu_physical_memory_write_rom_internal(as
, addr
, buf
, len
, WRITE_DATA
);
2539 void cpu_flush_icache_range(hwaddr start
, int len
)
2542 * This function should do the same thing as an icache flush that was
2543 * triggered from within the guest. For TCG we are always cache coherent,
2544 * so there is no need to flush anything. For KVM / Xen we need to flush
2545 * the host's instruction cache at least.
2547 if (tcg_enabled()) {
2551 cpu_physical_memory_write_rom_internal(&address_space_memory
,
2552 start
, NULL
, len
, FLUSH_CACHE
);
2563 static BounceBuffer bounce
;
2565 typedef struct MapClient
{
2567 QLIST_ENTRY(MapClient
) link
;
2570 QemuMutex map_client_list_lock
;
2571 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
2572 = QLIST_HEAD_INITIALIZER(map_client_list
);
2574 static void cpu_unregister_map_client_do(MapClient
*client
)
2576 QLIST_REMOVE(client
, link
);
2580 static void cpu_notify_map_clients_locked(void)
2584 while (!QLIST_EMPTY(&map_client_list
)) {
2585 client
= QLIST_FIRST(&map_client_list
);
2586 qemu_bh_schedule(client
->bh
);
2587 cpu_unregister_map_client_do(client
);
2591 void cpu_register_map_client(QEMUBH
*bh
)
2593 MapClient
*client
= g_malloc(sizeof(*client
));
2595 qemu_mutex_lock(&map_client_list_lock
);
2597 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
2598 if (!atomic_read(&bounce
.in_use
)) {
2599 cpu_notify_map_clients_locked();
2601 qemu_mutex_unlock(&map_client_list_lock
);
2604 void cpu_exec_init_all(void)
2606 qemu_mutex_init(&ram_list
.mutex
);
2609 qemu_mutex_init(&map_client_list_lock
);
2612 void cpu_unregister_map_client(QEMUBH
*bh
)
2616 qemu_mutex_lock(&map_client_list_lock
);
2617 QLIST_FOREACH(client
, &map_client_list
, link
) {
2618 if (client
->bh
== bh
) {
2619 cpu_unregister_map_client_do(client
);
2623 qemu_mutex_unlock(&map_client_list_lock
);
2626 static void cpu_notify_map_clients(void)
2628 qemu_mutex_lock(&map_client_list_lock
);
2629 cpu_notify_map_clients_locked();
2630 qemu_mutex_unlock(&map_client_list_lock
);
2633 bool address_space_access_valid(AddressSpace
*as
, hwaddr addr
, int len
, bool is_write
)
2641 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2642 if (!memory_access_is_direct(mr
, is_write
)) {
2643 l
= memory_access_size(mr
, l
, addr
);
2644 if (!memory_region_access_valid(mr
, xlat
, l
, is_write
)) {
2656 /* Map a physical memory region into a host virtual address.
2657 * May map a subset of the requested range, given by and returned in *plen.
2658 * May return NULL if resources needed to perform the mapping are exhausted.
2659 * Use only for reads OR writes - not for read-modify-write operations.
2660 * Use cpu_register_map_client() to know when retrying the map operation is
2661 * likely to succeed.
2663 void *address_space_map(AddressSpace
*as
,
2670 hwaddr l
, xlat
, base
;
2671 MemoryRegion
*mr
, *this_mr
;
2680 mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2682 if (!memory_access_is_direct(mr
, is_write
)) {
2683 if (atomic_xchg(&bounce
.in_use
, true)) {
2687 /* Avoid unbounded allocations */
2688 l
= MIN(l
, TARGET_PAGE_SIZE
);
2689 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, l
);
2693 memory_region_ref(mr
);
2696 address_space_read(as
, addr
, MEMTXATTRS_UNSPECIFIED
,
2702 return bounce
.buffer
;
2706 raddr
= memory_region_get_ram_addr(mr
);
2717 this_mr
= address_space_translate(as
, addr
, &xlat
, &l
, is_write
);
2718 if (this_mr
!= mr
|| xlat
!= base
+ done
) {
2723 memory_region_ref(mr
);
2726 return qemu_ram_ptr_length(raddr
+ base
, plen
);
2729 /* Unmaps a memory region previously mapped by address_space_map().
2730 * Will also mark the memory as dirty if is_write == 1. access_len gives
2731 * the amount of memory that was actually read or written by the caller.
2733 void address_space_unmap(AddressSpace
*as
, void *buffer
, hwaddr len
,
2734 int is_write
, hwaddr access_len
)
2736 if (buffer
!= bounce
.buffer
) {
2740 mr
= qemu_ram_addr_from_host(buffer
, &addr1
);
2743 invalidate_and_set_dirty(mr
, addr1
, access_len
);
2745 if (xen_enabled()) {
2746 xen_invalidate_map_cache_entry(buffer
);
2748 memory_region_unref(mr
);
2752 address_space_write(as
, bounce
.addr
, MEMTXATTRS_UNSPECIFIED
,
2753 bounce
.buffer
, access_len
);
2755 qemu_vfree(bounce
.buffer
);
2756 bounce
.buffer
= NULL
;
2757 memory_region_unref(bounce
.mr
);
2758 atomic_mb_set(&bounce
.in_use
, false);
2759 cpu_notify_map_clients();
2762 void *cpu_physical_memory_map(hwaddr addr
,
2766 return address_space_map(&address_space_memory
, addr
, plen
, is_write
);
2769 void cpu_physical_memory_unmap(void *buffer
, hwaddr len
,
2770 int is_write
, hwaddr access_len
)
2772 return address_space_unmap(&address_space_memory
, buffer
, len
, is_write
, access_len
);
2775 /* warning: addr must be aligned */
2776 static inline uint32_t address_space_ldl_internal(AddressSpace
*as
, hwaddr addr
,
2778 MemTxResult
*result
,
2779 enum device_endian endian
)
2787 bool release_lock
= false;
2790 mr
= address_space_translate(as
, addr
, &addr1
, &l
, false);
2791 if (l
< 4 || !memory_access_is_direct(mr
, false)) {
2792 release_lock
|= prepare_mmio_access(mr
);
2795 r
= memory_region_dispatch_read(mr
, addr1
, &val
, 4, attrs
);
2796 #if defined(TARGET_WORDS_BIGENDIAN)
2797 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2801 if (endian
== DEVICE_BIG_ENDIAN
) {
2807 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2811 case DEVICE_LITTLE_ENDIAN
:
2812 val
= ldl_le_p(ptr
);
2814 case DEVICE_BIG_ENDIAN
:
2815 val
= ldl_be_p(ptr
);
2827 qemu_mutex_unlock_iothread();
2833 uint32_t address_space_ldl(AddressSpace
*as
, hwaddr addr
,
2834 MemTxAttrs attrs
, MemTxResult
*result
)
2836 return address_space_ldl_internal(as
, addr
, attrs
, result
,
2837 DEVICE_NATIVE_ENDIAN
);
2840 uint32_t address_space_ldl_le(AddressSpace
*as
, hwaddr addr
,
2841 MemTxAttrs attrs
, MemTxResult
*result
)
2843 return address_space_ldl_internal(as
, addr
, attrs
, result
,
2844 DEVICE_LITTLE_ENDIAN
);
2847 uint32_t address_space_ldl_be(AddressSpace
*as
, hwaddr addr
,
2848 MemTxAttrs attrs
, MemTxResult
*result
)
2850 return address_space_ldl_internal(as
, addr
, attrs
, result
,
2854 uint32_t ldl_phys(AddressSpace
*as
, hwaddr addr
)
2856 return address_space_ldl(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2859 uint32_t ldl_le_phys(AddressSpace
*as
, hwaddr addr
)
2861 return address_space_ldl_le(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2864 uint32_t ldl_be_phys(AddressSpace
*as
, hwaddr addr
)
2866 return address_space_ldl_be(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2869 /* warning: addr must be aligned */
2870 static inline uint64_t address_space_ldq_internal(AddressSpace
*as
, hwaddr addr
,
2872 MemTxResult
*result
,
2873 enum device_endian endian
)
2881 bool release_lock
= false;
2884 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
2886 if (l
< 8 || !memory_access_is_direct(mr
, false)) {
2887 release_lock
|= prepare_mmio_access(mr
);
2890 r
= memory_region_dispatch_read(mr
, addr1
, &val
, 8, attrs
);
2891 #if defined(TARGET_WORDS_BIGENDIAN)
2892 if (endian
== DEVICE_LITTLE_ENDIAN
) {
2896 if (endian
== DEVICE_BIG_ENDIAN
) {
2902 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
2906 case DEVICE_LITTLE_ENDIAN
:
2907 val
= ldq_le_p(ptr
);
2909 case DEVICE_BIG_ENDIAN
:
2910 val
= ldq_be_p(ptr
);
2922 qemu_mutex_unlock_iothread();
2928 uint64_t address_space_ldq(AddressSpace
*as
, hwaddr addr
,
2929 MemTxAttrs attrs
, MemTxResult
*result
)
2931 return address_space_ldq_internal(as
, addr
, attrs
, result
,
2932 DEVICE_NATIVE_ENDIAN
);
2935 uint64_t address_space_ldq_le(AddressSpace
*as
, hwaddr addr
,
2936 MemTxAttrs attrs
, MemTxResult
*result
)
2938 return address_space_ldq_internal(as
, addr
, attrs
, result
,
2939 DEVICE_LITTLE_ENDIAN
);
2942 uint64_t address_space_ldq_be(AddressSpace
*as
, hwaddr addr
,
2943 MemTxAttrs attrs
, MemTxResult
*result
)
2945 return address_space_ldq_internal(as
, addr
, attrs
, result
,
2949 uint64_t ldq_phys(AddressSpace
*as
, hwaddr addr
)
2951 return address_space_ldq(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2954 uint64_t ldq_le_phys(AddressSpace
*as
, hwaddr addr
)
2956 return address_space_ldq_le(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2959 uint64_t ldq_be_phys(AddressSpace
*as
, hwaddr addr
)
2961 return address_space_ldq_be(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2965 uint32_t address_space_ldub(AddressSpace
*as
, hwaddr addr
,
2966 MemTxAttrs attrs
, MemTxResult
*result
)
2971 r
= address_space_rw(as
, addr
, attrs
, &val
, 1, 0);
2978 uint32_t ldub_phys(AddressSpace
*as
, hwaddr addr
)
2980 return address_space_ldub(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
2983 /* warning: addr must be aligned */
2984 static inline uint32_t address_space_lduw_internal(AddressSpace
*as
,
2987 MemTxResult
*result
,
2988 enum device_endian endian
)
2996 bool release_lock
= false;
2999 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
3001 if (l
< 2 || !memory_access_is_direct(mr
, false)) {
3002 release_lock
|= prepare_mmio_access(mr
);
3005 r
= memory_region_dispatch_read(mr
, addr1
, &val
, 2, attrs
);
3006 #if defined(TARGET_WORDS_BIGENDIAN)
3007 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3011 if (endian
== DEVICE_BIG_ENDIAN
) {
3017 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(mr
)
3021 case DEVICE_LITTLE_ENDIAN
:
3022 val
= lduw_le_p(ptr
);
3024 case DEVICE_BIG_ENDIAN
:
3025 val
= lduw_be_p(ptr
);
3037 qemu_mutex_unlock_iothread();
3043 uint32_t address_space_lduw(AddressSpace
*as
, hwaddr addr
,
3044 MemTxAttrs attrs
, MemTxResult
*result
)
3046 return address_space_lduw_internal(as
, addr
, attrs
, result
,
3047 DEVICE_NATIVE_ENDIAN
);
3050 uint32_t address_space_lduw_le(AddressSpace
*as
, hwaddr addr
,
3051 MemTxAttrs attrs
, MemTxResult
*result
)
3053 return address_space_lduw_internal(as
, addr
, attrs
, result
,
3054 DEVICE_LITTLE_ENDIAN
);
3057 uint32_t address_space_lduw_be(AddressSpace
*as
, hwaddr addr
,
3058 MemTxAttrs attrs
, MemTxResult
*result
)
3060 return address_space_lduw_internal(as
, addr
, attrs
, result
,
3064 uint32_t lduw_phys(AddressSpace
*as
, hwaddr addr
)
3066 return address_space_lduw(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3069 uint32_t lduw_le_phys(AddressSpace
*as
, hwaddr addr
)
3071 return address_space_lduw_le(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3074 uint32_t lduw_be_phys(AddressSpace
*as
, hwaddr addr
)
3076 return address_space_lduw_be(as
, addr
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3079 /* warning: addr must be aligned. The ram page is not masked as dirty
3080 and the code inside is not invalidated. It is useful if the dirty
3081 bits are used to track modified PTEs */
3082 void address_space_stl_notdirty(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3083 MemTxAttrs attrs
, MemTxResult
*result
)
3090 uint8_t dirty_log_mask
;
3091 bool release_lock
= false;
3094 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
3096 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
3097 release_lock
|= prepare_mmio_access(mr
);
3099 r
= memory_region_dispatch_write(mr
, addr1
, val
, 4, attrs
);
3101 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
3102 ptr
= qemu_get_ram_ptr(addr1
);
3105 dirty_log_mask
= memory_region_get_dirty_log_mask(mr
);
3106 dirty_log_mask
&= ~(1 << DIRTY_MEMORY_CODE
);
3107 cpu_physical_memory_set_dirty_range(addr1
, 4, dirty_log_mask
);
3114 qemu_mutex_unlock_iothread();
3119 void stl_phys_notdirty(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3121 address_space_stl_notdirty(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3124 /* warning: addr must be aligned */
3125 static inline void address_space_stl_internal(AddressSpace
*as
,
3126 hwaddr addr
, uint32_t val
,
3128 MemTxResult
*result
,
3129 enum device_endian endian
)
3136 bool release_lock
= false;
3139 mr
= address_space_translate(as
, addr
, &addr1
, &l
,
3141 if (l
< 4 || !memory_access_is_direct(mr
, true)) {
3142 release_lock
|= prepare_mmio_access(mr
);
3144 #if defined(TARGET_WORDS_BIGENDIAN)
3145 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3149 if (endian
== DEVICE_BIG_ENDIAN
) {
3153 r
= memory_region_dispatch_write(mr
, addr1
, val
, 4, attrs
);
3156 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
3157 ptr
= qemu_get_ram_ptr(addr1
);
3159 case DEVICE_LITTLE_ENDIAN
:
3162 case DEVICE_BIG_ENDIAN
:
3169 invalidate_and_set_dirty(mr
, addr1
, 4);
3176 qemu_mutex_unlock_iothread();
3181 void address_space_stl(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3182 MemTxAttrs attrs
, MemTxResult
*result
)
3184 address_space_stl_internal(as
, addr
, val
, attrs
, result
,
3185 DEVICE_NATIVE_ENDIAN
);
3188 void address_space_stl_le(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3189 MemTxAttrs attrs
, MemTxResult
*result
)
3191 address_space_stl_internal(as
, addr
, val
, attrs
, result
,
3192 DEVICE_LITTLE_ENDIAN
);
3195 void address_space_stl_be(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3196 MemTxAttrs attrs
, MemTxResult
*result
)
3198 address_space_stl_internal(as
, addr
, val
, attrs
, result
,
3202 void stl_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3204 address_space_stl(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3207 void stl_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3209 address_space_stl_le(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3212 void stl_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3214 address_space_stl_be(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3218 void address_space_stb(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3219 MemTxAttrs attrs
, MemTxResult
*result
)
3224 r
= address_space_rw(as
, addr
, attrs
, &v
, 1, 1);
3230 void stb_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3232 address_space_stb(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3235 /* warning: addr must be aligned */
3236 static inline void address_space_stw_internal(AddressSpace
*as
,
3237 hwaddr addr
, uint32_t val
,
3239 MemTxResult
*result
,
3240 enum device_endian endian
)
3247 bool release_lock
= false;
3250 mr
= address_space_translate(as
, addr
, &addr1
, &l
, true);
3251 if (l
< 2 || !memory_access_is_direct(mr
, true)) {
3252 release_lock
|= prepare_mmio_access(mr
);
3254 #if defined(TARGET_WORDS_BIGENDIAN)
3255 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3259 if (endian
== DEVICE_BIG_ENDIAN
) {
3263 r
= memory_region_dispatch_write(mr
, addr1
, val
, 2, attrs
);
3266 addr1
+= memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
;
3267 ptr
= qemu_get_ram_ptr(addr1
);
3269 case DEVICE_LITTLE_ENDIAN
:
3272 case DEVICE_BIG_ENDIAN
:
3279 invalidate_and_set_dirty(mr
, addr1
, 2);
3286 qemu_mutex_unlock_iothread();
3291 void address_space_stw(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3292 MemTxAttrs attrs
, MemTxResult
*result
)
3294 address_space_stw_internal(as
, addr
, val
, attrs
, result
,
3295 DEVICE_NATIVE_ENDIAN
);
3298 void address_space_stw_le(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3299 MemTxAttrs attrs
, MemTxResult
*result
)
3301 address_space_stw_internal(as
, addr
, val
, attrs
, result
,
3302 DEVICE_LITTLE_ENDIAN
);
3305 void address_space_stw_be(AddressSpace
*as
, hwaddr addr
, uint32_t val
,
3306 MemTxAttrs attrs
, MemTxResult
*result
)
3308 address_space_stw_internal(as
, addr
, val
, attrs
, result
,
3312 void stw_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3314 address_space_stw(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3317 void stw_le_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3319 address_space_stw_le(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3322 void stw_be_phys(AddressSpace
*as
, hwaddr addr
, uint32_t val
)
3324 address_space_stw_be(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3328 void address_space_stq(AddressSpace
*as
, hwaddr addr
, uint64_t val
,
3329 MemTxAttrs attrs
, MemTxResult
*result
)
3333 r
= address_space_rw(as
, addr
, attrs
, (void *) &val
, 8, 1);
3339 void address_space_stq_le(AddressSpace
*as
, hwaddr addr
, uint64_t val
,
3340 MemTxAttrs attrs
, MemTxResult
*result
)
3343 val
= cpu_to_le64(val
);
3344 r
= address_space_rw(as
, addr
, attrs
, (void *) &val
, 8, 1);
3349 void address_space_stq_be(AddressSpace
*as
, hwaddr addr
, uint64_t val
,
3350 MemTxAttrs attrs
, MemTxResult
*result
)
3353 val
= cpu_to_be64(val
);
3354 r
= address_space_rw(as
, addr
, attrs
, (void *) &val
, 8, 1);
3360 void stq_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3362 address_space_stq(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3365 void stq_le_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3367 address_space_stq_le(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3370 void stq_be_phys(AddressSpace
*as
, hwaddr addr
, uint64_t val
)
3372 address_space_stq_be(as
, addr
, val
, MEMTXATTRS_UNSPECIFIED
, NULL
);
3375 /* virtual memory access for debug (includes writing to ROM) */
3376 int cpu_memory_rw_debug(CPUState
*cpu
, target_ulong addr
,
3377 uint8_t *buf
, int len
, int is_write
)
3384 page
= addr
& TARGET_PAGE_MASK
;
3385 phys_addr
= cpu_get_phys_page_debug(cpu
, page
);
3386 /* if no physical page mapped, return an error */
3387 if (phys_addr
== -1)
3389 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3392 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
3394 cpu_physical_memory_write_rom(cpu
->as
, phys_addr
, buf
, l
);
3396 address_space_rw(cpu
->as
, phys_addr
, MEMTXATTRS_UNSPECIFIED
,
3408 * A helper function for the _utterly broken_ virtio device model to find out if
3409 * it's running on a big endian machine. Don't do this at home kids!
3411 bool target_words_bigendian(void);
3412 bool target_words_bigendian(void)
3414 #if defined(TARGET_WORDS_BIGENDIAN)
3421 #ifndef CONFIG_USER_ONLY
3422 bool cpu_physical_memory_is_io(hwaddr phys_addr
)
3429 mr
= address_space_translate(&address_space_memory
,
3430 phys_addr
, &phys_addr
, &l
, false);
3432 res
= !(memory_region_is_ram(mr
) || memory_region_is_romd(mr
));
3437 int qemu_ram_foreach_block(RAMBlockIterFunc func
, void *opaque
)
3443 QLIST_FOREACH_RCU(block
, &ram_list
.blocks
, next
) {
3444 ret
= func(block
->idstr
, block
->host
, block
->offset
,
3445 block
->used_length
, opaque
);