2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
23 #include <sys/types.h>
27 #include "qemu-common.h"
35 #include "qemu-timer.h"
37 #include "exec-memory.h"
38 #if defined(CONFIG_USER_ONLY)
40 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
41 #include <sys/param.h>
42 #if __FreeBSD_version >= 700104
43 #define HAVE_KINFO_GETVMMAP
44 #define sigqueue sigqueue_freebsd /* avoid redefinition */
47 #include <machine/profile.h>
55 #else /* !CONFIG_USER_ONLY */
56 #include "xen-mapcache.h"
62 #define WANT_EXEC_OBSOLETE
63 #include "exec-obsolete.h"
65 //#define DEBUG_TB_INVALIDATE
67 //#define DEBUG_UNASSIGNED
69 /* make various TB consistency checks */
70 //#define DEBUG_TB_CHECK
72 //#define DEBUG_IOPORT
73 //#define DEBUG_SUBPAGE
75 #if !defined(CONFIG_USER_ONLY)
76 /* TB consistency checks only implemented for usermode emulation. */
80 #define SMC_BITMAP_USE_THRESHOLD 10
82 static TranslationBlock
*tbs
;
83 static int code_gen_max_blocks
;
84 TranslationBlock
*tb_phys_hash
[CODE_GEN_PHYS_HASH_SIZE
];
86 /* any access to the tbs or the page table must use this lock */
87 spinlock_t tb_lock
= SPIN_LOCK_UNLOCKED
;
89 #if defined(__arm__) || defined(__sparc__)
90 /* The prologue must be reachable with a direct jump. ARM and Sparc64
91 have limited branch ranges (possibly also PPC) so place it in a
92 section close to code segment. */
93 #define code_gen_section \
94 __attribute__((__section__(".gen_code"))) \
95 __attribute__((aligned (32)))
96 #elif defined(_WIN32) && !defined(_WIN64)
97 #define code_gen_section \
98 __attribute__((aligned (16)))
100 #define code_gen_section \
101 __attribute__((aligned (32)))
104 uint8_t code_gen_prologue
[1024] code_gen_section
;
105 static uint8_t *code_gen_buffer
;
106 static unsigned long code_gen_buffer_size
;
107 /* threshold to flush the translated code buffer */
108 static unsigned long code_gen_buffer_max_size
;
109 static uint8_t *code_gen_ptr
;
111 #if !defined(CONFIG_USER_ONLY)
113 static int in_migration
;
115 RAMList ram_list
= { .blocks
= QLIST_HEAD_INITIALIZER(ram_list
.blocks
) };
117 static MemoryRegion
*system_memory
;
118 static MemoryRegion
*system_io
;
120 MemoryRegion io_mem_ram
, io_mem_rom
, io_mem_unassigned
, io_mem_notdirty
;
121 static MemoryRegion io_mem_subpage_ram
;
125 CPUArchState
*first_cpu
;
126 /* current CPU in the current thread. It is only valid inside
128 DEFINE_TLS(CPUArchState
*,cpu_single_env
);
129 /* 0 = Do not count executed instructions.
130 1 = Precise instruction counting.
131 2 = Adaptive rate instruction counting. */
134 typedef struct PageDesc
{
135 /* list of TBs intersecting this ram page */
136 TranslationBlock
*first_tb
;
137 /* in order to optimize self modifying code, we count the number
138 of lookups we do to a given page to use a bitmap */
139 unsigned int code_write_count
;
140 uint8_t *code_bitmap
;
141 #if defined(CONFIG_USER_ONLY)
146 /* In system mode we want L1_MAP to be based on ram offsets,
147 while in user mode we want it to be based on virtual addresses. */
148 #if !defined(CONFIG_USER_ONLY)
149 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
150 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
152 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
155 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
158 /* Size of the L2 (and L3, etc) page tables. */
160 #define L2_SIZE (1 << L2_BITS)
162 #define P_L2_LEVELS \
163 (((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / L2_BITS) + 1)
165 /* The bits remaining after N lower levels of page tables. */
166 #define V_L1_BITS_REM \
167 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
169 #if V_L1_BITS_REM < 4
170 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
172 #define V_L1_BITS V_L1_BITS_REM
175 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
177 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
179 uintptr_t qemu_real_host_page_size
;
180 uintptr_t qemu_host_page_size
;
181 uintptr_t qemu_host_page_mask
;
183 /* This is a multi-level map on the virtual address space.
184 The bottom level has pointers to PageDesc. */
185 static void *l1_map
[V_L1_SIZE
];
187 #if !defined(CONFIG_USER_ONLY)
188 typedef struct PhysPageEntry PhysPageEntry
;
190 static MemoryRegionSection
*phys_sections
;
191 static unsigned phys_sections_nb
, phys_sections_nb_alloc
;
192 static uint16_t phys_section_unassigned
;
193 static uint16_t phys_section_notdirty
;
194 static uint16_t phys_section_rom
;
195 static uint16_t phys_section_watch
;
197 struct PhysPageEntry
{
198 uint16_t is_leaf
: 1;
199 /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
203 /* Simple allocator for PhysPageEntry nodes */
204 static PhysPageEntry (*phys_map_nodes
)[L2_SIZE
];
205 static unsigned phys_map_nodes_nb
, phys_map_nodes_nb_alloc
;
207 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
209 /* This is a multi-level map on the physical address space.
210 The bottom level has pointers to MemoryRegionSections. */
211 static PhysPageEntry phys_map
= { .ptr
= PHYS_MAP_NODE_NIL
, .is_leaf
= 0 };
213 static void io_mem_init(void);
214 static void memory_map_init(void);
216 static MemoryRegion io_mem_watch
;
220 static int tb_flush_count
;
221 static int tb_phys_invalidate_count
;
224 static void map_exec(void *addr
, long size
)
227 VirtualProtect(addr
, size
,
228 PAGE_EXECUTE_READWRITE
, &old_protect
);
232 static void map_exec(void *addr
, long size
)
234 unsigned long start
, end
, page_size
;
236 page_size
= getpagesize();
237 start
= (unsigned long)addr
;
238 start
&= ~(page_size
- 1);
240 end
= (unsigned long)addr
+ size
;
241 end
+= page_size
- 1;
242 end
&= ~(page_size
- 1);
244 mprotect((void *)start
, end
- start
,
245 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
249 static void page_init(void)
251 /* NOTE: we can always suppose that qemu_host_page_size >=
255 SYSTEM_INFO system_info
;
257 GetSystemInfo(&system_info
);
258 qemu_real_host_page_size
= system_info
.dwPageSize
;
261 qemu_real_host_page_size
= getpagesize();
263 if (qemu_host_page_size
== 0)
264 qemu_host_page_size
= qemu_real_host_page_size
;
265 if (qemu_host_page_size
< TARGET_PAGE_SIZE
)
266 qemu_host_page_size
= TARGET_PAGE_SIZE
;
267 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
269 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
271 #ifdef HAVE_KINFO_GETVMMAP
272 struct kinfo_vmentry
*freep
;
275 freep
= kinfo_getvmmap(getpid(), &cnt
);
278 for (i
= 0; i
< cnt
; i
++) {
279 unsigned long startaddr
, endaddr
;
281 startaddr
= freep
[i
].kve_start
;
282 endaddr
= freep
[i
].kve_end
;
283 if (h2g_valid(startaddr
)) {
284 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
286 if (h2g_valid(endaddr
)) {
287 endaddr
= h2g(endaddr
);
288 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
290 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
292 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
303 last_brk
= (unsigned long)sbrk(0);
305 f
= fopen("/compat/linux/proc/self/maps", "r");
310 unsigned long startaddr
, endaddr
;
313 n
= fscanf (f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
315 if (n
== 2 && h2g_valid(startaddr
)) {
316 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
318 if (h2g_valid(endaddr
)) {
319 endaddr
= h2g(endaddr
);
323 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
335 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
341 #if defined(CONFIG_USER_ONLY)
342 /* We can't use g_malloc because it may recurse into a locked mutex. */
343 # define ALLOC(P, SIZE) \
345 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
346 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
349 # define ALLOC(P, SIZE) \
350 do { P = g_malloc0(SIZE); } while (0)
353 /* Level 1. Always allocated. */
354 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
357 for (i
= V_L1_SHIFT
/ L2_BITS
- 1; i
> 0; i
--) {
364 ALLOC(p
, sizeof(void *) * L2_SIZE
);
368 lp
= p
+ ((index
>> (i
* L2_BITS
)) & (L2_SIZE
- 1));
376 ALLOC(pd
, sizeof(PageDesc
) * L2_SIZE
);
382 return pd
+ (index
& (L2_SIZE
- 1));
385 static inline PageDesc
*page_find(tb_page_addr_t index
)
387 return page_find_alloc(index
, 0);
390 #if !defined(CONFIG_USER_ONLY)
392 static void phys_map_node_reserve(unsigned nodes
)
394 if (phys_map_nodes_nb
+ nodes
> phys_map_nodes_nb_alloc
) {
395 typedef PhysPageEntry Node
[L2_SIZE
];
396 phys_map_nodes_nb_alloc
= MAX(phys_map_nodes_nb_alloc
* 2, 16);
397 phys_map_nodes_nb_alloc
= MAX(phys_map_nodes_nb_alloc
,
398 phys_map_nodes_nb
+ nodes
);
399 phys_map_nodes
= g_renew(Node
, phys_map_nodes
,
400 phys_map_nodes_nb_alloc
);
404 static uint16_t phys_map_node_alloc(void)
409 ret
= phys_map_nodes_nb
++;
410 assert(ret
!= PHYS_MAP_NODE_NIL
);
411 assert(ret
!= phys_map_nodes_nb_alloc
);
412 for (i
= 0; i
< L2_SIZE
; ++i
) {
413 phys_map_nodes
[ret
][i
].is_leaf
= 0;
414 phys_map_nodes
[ret
][i
].ptr
= PHYS_MAP_NODE_NIL
;
419 static void phys_map_nodes_reset(void)
421 phys_map_nodes_nb
= 0;
425 static void phys_page_set_level(PhysPageEntry
*lp
, target_phys_addr_t
*index
,
426 target_phys_addr_t
*nb
, uint16_t leaf
,
431 target_phys_addr_t step
= (target_phys_addr_t
)1 << (level
* L2_BITS
);
433 if (!lp
->is_leaf
&& lp
->ptr
== PHYS_MAP_NODE_NIL
) {
434 lp
->ptr
= phys_map_node_alloc();
435 p
= phys_map_nodes
[lp
->ptr
];
437 for (i
= 0; i
< L2_SIZE
; i
++) {
439 p
[i
].ptr
= phys_section_unassigned
;
443 p
= phys_map_nodes
[lp
->ptr
];
445 lp
= &p
[(*index
>> (level
* L2_BITS
)) & (L2_SIZE
- 1)];
447 while (*nb
&& lp
< &p
[L2_SIZE
]) {
448 if ((*index
& (step
- 1)) == 0 && *nb
>= step
) {
454 phys_page_set_level(lp
, index
, nb
, leaf
, level
- 1);
460 static void phys_page_set(target_phys_addr_t index
, target_phys_addr_t nb
,
463 /* Wildly overreserve - it doesn't matter much. */
464 phys_map_node_reserve(3 * P_L2_LEVELS
);
466 phys_page_set_level(&phys_map
, &index
, &nb
, leaf
, P_L2_LEVELS
- 1);
469 MemoryRegionSection
*phys_page_find(target_phys_addr_t index
)
471 PhysPageEntry lp
= phys_map
;
474 uint16_t s_index
= phys_section_unassigned
;
476 for (i
= P_L2_LEVELS
- 1; i
>= 0 && !lp
.is_leaf
; i
--) {
477 if (lp
.ptr
== PHYS_MAP_NODE_NIL
) {
480 p
= phys_map_nodes
[lp
.ptr
];
481 lp
= p
[(index
>> (i
* L2_BITS
)) & (L2_SIZE
- 1)];
486 return &phys_sections
[s_index
];
489 bool memory_region_is_unassigned(MemoryRegion
*mr
)
491 return mr
!= &io_mem_ram
&& mr
!= &io_mem_rom
492 && mr
!= &io_mem_notdirty
&& !mr
->rom_device
493 && mr
!= &io_mem_watch
;
496 #define mmap_lock() do { } while(0)
497 #define mmap_unlock() do { } while(0)
500 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
502 #if defined(CONFIG_USER_ONLY)
503 /* Currently it is not recommended to allocate big chunks of data in
504 user mode. It will change when a dedicated libc will be used */
505 #define USE_STATIC_CODE_GEN_BUFFER
508 #ifdef USE_STATIC_CODE_GEN_BUFFER
509 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
510 __attribute__((aligned (CODE_GEN_ALIGN
)));
513 static void code_gen_alloc(unsigned long tb_size
)
515 #ifdef USE_STATIC_CODE_GEN_BUFFER
516 code_gen_buffer
= static_code_gen_buffer
;
517 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
518 map_exec(code_gen_buffer
, code_gen_buffer_size
);
520 code_gen_buffer_size
= tb_size
;
521 if (code_gen_buffer_size
== 0) {
522 #if defined(CONFIG_USER_ONLY)
523 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
525 /* XXX: needs adjustments */
526 code_gen_buffer_size
= (unsigned long)(ram_size
/ 4);
529 if (code_gen_buffer_size
< MIN_CODE_GEN_BUFFER_SIZE
)
530 code_gen_buffer_size
= MIN_CODE_GEN_BUFFER_SIZE
;
531 /* The code gen buffer location may have constraints depending on
532 the host cpu and OS */
533 #if defined(__linux__)
538 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
539 #if defined(__x86_64__)
541 /* Cannot map more than that */
542 if (code_gen_buffer_size
> (800 * 1024 * 1024))
543 code_gen_buffer_size
= (800 * 1024 * 1024);
544 #elif defined(__sparc__) && HOST_LONG_BITS == 64
545 // Map the buffer below 2G, so we can use direct calls and branches
546 start
= (void *) 0x40000000UL
;
547 if (code_gen_buffer_size
> (512 * 1024 * 1024))
548 code_gen_buffer_size
= (512 * 1024 * 1024);
549 #elif defined(__arm__)
550 /* Keep the buffer no bigger than 16MB to branch between blocks */
551 if (code_gen_buffer_size
> 16 * 1024 * 1024)
552 code_gen_buffer_size
= 16 * 1024 * 1024;
553 #elif defined(__s390x__)
554 /* Map the buffer so that we can use direct calls and branches. */
555 /* We have a +- 4GB range on the branches; leave some slop. */
556 if (code_gen_buffer_size
> (3ul * 1024 * 1024 * 1024)) {
557 code_gen_buffer_size
= 3ul * 1024 * 1024 * 1024;
559 start
= (void *)0x90000000UL
;
561 code_gen_buffer
= mmap(start
, code_gen_buffer_size
,
562 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
564 if (code_gen_buffer
== MAP_FAILED
) {
565 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
569 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
570 || defined(__DragonFly__) || defined(__OpenBSD__) \
571 || defined(__NetBSD__)
575 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
576 #if defined(__x86_64__)
577 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
578 * 0x40000000 is free */
580 addr
= (void *)0x40000000;
581 /* Cannot map more than that */
582 if (code_gen_buffer_size
> (800 * 1024 * 1024))
583 code_gen_buffer_size
= (800 * 1024 * 1024);
584 #elif defined(__sparc__) && HOST_LONG_BITS == 64
585 // Map the buffer below 2G, so we can use direct calls and branches
586 addr
= (void *) 0x40000000UL
;
587 if (code_gen_buffer_size
> (512 * 1024 * 1024)) {
588 code_gen_buffer_size
= (512 * 1024 * 1024);
591 code_gen_buffer
= mmap(addr
, code_gen_buffer_size
,
592 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
594 if (code_gen_buffer
== MAP_FAILED
) {
595 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
600 code_gen_buffer
= g_malloc(code_gen_buffer_size
);
601 map_exec(code_gen_buffer
, code_gen_buffer_size
);
603 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
604 map_exec(code_gen_prologue
, sizeof(code_gen_prologue
));
605 code_gen_buffer_max_size
= code_gen_buffer_size
-
606 (TCG_MAX_OP_SIZE
* OPC_BUF_SIZE
);
607 code_gen_max_blocks
= code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
608 tbs
= g_malloc(code_gen_max_blocks
* sizeof(TranslationBlock
));
611 /* Must be called before using the QEMU cpus. 'tb_size' is the size
612 (in bytes) allocated to the translation buffer. Zero means default
614 void tcg_exec_init(unsigned long tb_size
)
617 code_gen_alloc(tb_size
);
618 code_gen_ptr
= code_gen_buffer
;
619 tcg_register_jit(code_gen_buffer
, code_gen_buffer_size
);
621 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
622 /* There's no guest base to take into account, so go ahead and
623 initialize the prologue now. */
624 tcg_prologue_init(&tcg_ctx
);
628 bool tcg_enabled(void)
630 return code_gen_buffer
!= NULL
;
633 void cpu_exec_init_all(void)
635 #if !defined(CONFIG_USER_ONLY)
641 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
643 static int cpu_common_post_load(void *opaque
, int version_id
)
645 CPUArchState
*env
= opaque
;
647 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
648 version_id is increased. */
649 env
->interrupt_request
&= ~0x01;
655 static const VMStateDescription vmstate_cpu_common
= {
656 .name
= "cpu_common",
658 .minimum_version_id
= 1,
659 .minimum_version_id_old
= 1,
660 .post_load
= cpu_common_post_load
,
661 .fields
= (VMStateField
[]) {
662 VMSTATE_UINT32(halted
, CPUArchState
),
663 VMSTATE_UINT32(interrupt_request
, CPUArchState
),
664 VMSTATE_END_OF_LIST()
669 CPUArchState
*qemu_get_cpu(int cpu
)
671 CPUArchState
*env
= first_cpu
;
674 if (env
->cpu_index
== cpu
)
682 void cpu_exec_init(CPUArchState
*env
)
687 #if defined(CONFIG_USER_ONLY)
690 env
->next_cpu
= NULL
;
693 while (*penv
!= NULL
) {
694 penv
= &(*penv
)->next_cpu
;
697 env
->cpu_index
= cpu_index
;
699 QTAILQ_INIT(&env
->breakpoints
);
700 QTAILQ_INIT(&env
->watchpoints
);
701 #ifndef CONFIG_USER_ONLY
702 env
->thread_id
= qemu_get_thread_id();
705 #if defined(CONFIG_USER_ONLY)
708 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
709 vmstate_register(NULL
, cpu_index
, &vmstate_cpu_common
, env
);
710 register_savevm(NULL
, "cpu", cpu_index
, CPU_SAVE_VERSION
,
711 cpu_save
, cpu_load
, env
);
715 /* Allocate a new translation block. Flush the translation buffer if
716 too many translation blocks or too much generated code. */
717 static TranslationBlock
*tb_alloc(target_ulong pc
)
719 TranslationBlock
*tb
;
721 if (nb_tbs
>= code_gen_max_blocks
||
722 (code_gen_ptr
- code_gen_buffer
) >= code_gen_buffer_max_size
)
730 void tb_free(TranslationBlock
*tb
)
732 /* In practice this is mostly used for single use temporary TB
733 Ignore the hard cases and just back up if this TB happens to
734 be the last one generated. */
735 if (nb_tbs
> 0 && tb
== &tbs
[nb_tbs
- 1]) {
736 code_gen_ptr
= tb
->tc_ptr
;
741 static inline void invalidate_page_bitmap(PageDesc
*p
)
743 if (p
->code_bitmap
) {
744 g_free(p
->code_bitmap
);
745 p
->code_bitmap
= NULL
;
747 p
->code_write_count
= 0;
750 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
752 static void page_flush_tb_1 (int level
, void **lp
)
761 for (i
= 0; i
< L2_SIZE
; ++i
) {
762 pd
[i
].first_tb
= NULL
;
763 invalidate_page_bitmap(pd
+ i
);
767 for (i
= 0; i
< L2_SIZE
; ++i
) {
768 page_flush_tb_1 (level
- 1, pp
+ i
);
773 static void page_flush_tb(void)
776 for (i
= 0; i
< V_L1_SIZE
; i
++) {
777 page_flush_tb_1(V_L1_SHIFT
/ L2_BITS
- 1, l1_map
+ i
);
781 /* flush all the translation blocks */
782 /* XXX: tb_flush is currently not thread safe */
783 void tb_flush(CPUArchState
*env1
)
786 #if defined(DEBUG_FLUSH)
787 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
788 (unsigned long)(code_gen_ptr
- code_gen_buffer
),
790 ((unsigned long)(code_gen_ptr
- code_gen_buffer
)) / nb_tbs
: 0);
792 if ((unsigned long)(code_gen_ptr
- code_gen_buffer
) > code_gen_buffer_size
)
793 cpu_abort(env1
, "Internal error: code buffer overflow\n");
797 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
798 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
801 memset (tb_phys_hash
, 0, CODE_GEN_PHYS_HASH_SIZE
* sizeof (void *));
804 code_gen_ptr
= code_gen_buffer
;
805 /* XXX: flush processor icache at this point if cache flush is
810 #ifdef DEBUG_TB_CHECK
812 static void tb_invalidate_check(target_ulong address
)
814 TranslationBlock
*tb
;
816 address
&= TARGET_PAGE_MASK
;
817 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
818 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
819 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
820 address
>= tb
->pc
+ tb
->size
)) {
821 printf("ERROR invalidate: address=" TARGET_FMT_lx
822 " PC=%08lx size=%04x\n",
823 address
, (long)tb
->pc
, tb
->size
);
829 /* verify that all the pages have correct rights for code */
830 static void tb_page_check(void)
832 TranslationBlock
*tb
;
833 int i
, flags1
, flags2
;
835 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
836 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
837 flags1
= page_get_flags(tb
->pc
);
838 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
839 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
840 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
841 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
849 /* invalidate one TB */
850 static inline void tb_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
,
853 TranslationBlock
*tb1
;
857 *ptb
= *(TranslationBlock
**)((char *)tb1
+ next_offset
);
860 ptb
= (TranslationBlock
**)((char *)tb1
+ next_offset
);
864 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
866 TranslationBlock
*tb1
;
871 n1
= (uintptr_t)tb1
& 3;
872 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
874 *ptb
= tb1
->page_next
[n1
];
877 ptb
= &tb1
->page_next
[n1
];
881 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
883 TranslationBlock
*tb1
, **ptb
;
886 ptb
= &tb
->jmp_next
[n
];
889 /* find tb(n) in circular list */
892 n1
= (uintptr_t)tb1
& 3;
893 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
894 if (n1
== n
&& tb1
== tb
)
897 ptb
= &tb1
->jmp_first
;
899 ptb
= &tb1
->jmp_next
[n1
];
902 /* now we can suppress tb(n) from the list */
903 *ptb
= tb
->jmp_next
[n
];
905 tb
->jmp_next
[n
] = NULL
;
909 /* reset the jump entry 'n' of a TB so that it is not chained to
911 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
913 tb_set_jmp_target(tb
, n
, (uintptr_t)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
916 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
921 tb_page_addr_t phys_pc
;
922 TranslationBlock
*tb1
, *tb2
;
924 /* remove the TB from the hash list */
925 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
926 h
= tb_phys_hash_func(phys_pc
);
927 tb_remove(&tb_phys_hash
[h
], tb
,
928 offsetof(TranslationBlock
, phys_hash_next
));
930 /* remove the TB from the page list */
931 if (tb
->page_addr
[0] != page_addr
) {
932 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
933 tb_page_remove(&p
->first_tb
, tb
);
934 invalidate_page_bitmap(p
);
936 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
937 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
938 tb_page_remove(&p
->first_tb
, tb
);
939 invalidate_page_bitmap(p
);
942 tb_invalidated_flag
= 1;
944 /* remove the TB from the hash list */
945 h
= tb_jmp_cache_hash_func(tb
->pc
);
946 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
947 if (env
->tb_jmp_cache
[h
] == tb
)
948 env
->tb_jmp_cache
[h
] = NULL
;
951 /* suppress this TB from the two jump lists */
952 tb_jmp_remove(tb
, 0);
953 tb_jmp_remove(tb
, 1);
955 /* suppress any remaining jumps to this TB */
958 n1
= (uintptr_t)tb1
& 3;
961 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
962 tb2
= tb1
->jmp_next
[n1
];
963 tb_reset_jump(tb1
, n1
);
964 tb1
->jmp_next
[n1
] = NULL
;
967 tb
->jmp_first
= (TranslationBlock
*)((uintptr_t)tb
| 2); /* fail safe */
969 tb_phys_invalidate_count
++;
972 static inline void set_bits(uint8_t *tab
, int start
, int len
)
978 mask
= 0xff << (start
& 7);
979 if ((start
& ~7) == (end
& ~7)) {
981 mask
&= ~(0xff << (end
& 7));
986 start
= (start
+ 8) & ~7;
988 while (start
< end1
) {
993 mask
= ~(0xff << (end
& 7));
999 static void build_page_bitmap(PageDesc
*p
)
1001 int n
, tb_start
, tb_end
;
1002 TranslationBlock
*tb
;
1004 p
->code_bitmap
= g_malloc0(TARGET_PAGE_SIZE
/ 8);
1007 while (tb
!= NULL
) {
1008 n
= (uintptr_t)tb
& 3;
1009 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1010 /* NOTE: this is subtle as a TB may span two physical pages */
1012 /* NOTE: tb_end may be after the end of the page, but
1013 it is not a problem */
1014 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
1015 tb_end
= tb_start
+ tb
->size
;
1016 if (tb_end
> TARGET_PAGE_SIZE
)
1017 tb_end
= TARGET_PAGE_SIZE
;
1020 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1022 set_bits(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
1023 tb
= tb
->page_next
[n
];
1027 TranslationBlock
*tb_gen_code(CPUArchState
*env
,
1028 target_ulong pc
, target_ulong cs_base
,
1029 int flags
, int cflags
)
1031 TranslationBlock
*tb
;
1033 tb_page_addr_t phys_pc
, phys_page2
;
1034 target_ulong virt_page2
;
1037 phys_pc
= get_page_addr_code(env
, pc
);
1040 /* flush must be done */
1042 /* cannot fail at this point */
1044 /* Don't forget to invalidate previous TB info. */
1045 tb_invalidated_flag
= 1;
1047 tc_ptr
= code_gen_ptr
;
1048 tb
->tc_ptr
= tc_ptr
;
1049 tb
->cs_base
= cs_base
;
1051 tb
->cflags
= cflags
;
1052 cpu_gen_code(env
, tb
, &code_gen_size
);
1053 code_gen_ptr
= (void *)(((uintptr_t)code_gen_ptr
+ code_gen_size
+
1054 CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
1056 /* check next page if needed */
1057 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1059 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1060 phys_page2
= get_page_addr_code(env
, virt_page2
);
1062 tb_link_page(tb
, phys_pc
, phys_page2
);
1067 * Invalidate all TBs which intersect with the target physical address range
1068 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1069 * 'is_cpu_write_access' should be true if called from a real cpu write
1070 * access: the virtual CPU will exit the current TB if code is modified inside
1073 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
,
1074 int is_cpu_write_access
)
1076 while (start
< end
) {
1077 tb_invalidate_phys_page_range(start
, end
, is_cpu_write_access
);
1078 start
&= TARGET_PAGE_MASK
;
1079 start
+= TARGET_PAGE_SIZE
;
1084 * Invalidate all TBs which intersect with the target physical address range
1085 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1086 * 'is_cpu_write_access' should be true if called from a real cpu write
1087 * access: the virtual CPU will exit the current TB if code is modified inside
1090 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1091 int is_cpu_write_access
)
1093 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
1094 CPUArchState
*env
= cpu_single_env
;
1095 tb_page_addr_t tb_start
, tb_end
;
1098 #ifdef TARGET_HAS_PRECISE_SMC
1099 int current_tb_not_found
= is_cpu_write_access
;
1100 TranslationBlock
*current_tb
= NULL
;
1101 int current_tb_modified
= 0;
1102 target_ulong current_pc
= 0;
1103 target_ulong current_cs_base
= 0;
1104 int current_flags
= 0;
1105 #endif /* TARGET_HAS_PRECISE_SMC */
1107 p
= page_find(start
>> TARGET_PAGE_BITS
);
1110 if (!p
->code_bitmap
&&
1111 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
&&
1112 is_cpu_write_access
) {
1113 /* build code bitmap */
1114 build_page_bitmap(p
);
1117 /* we remove all the TBs in the range [start, end[ */
1118 /* XXX: see if in some cases it could be faster to invalidate all the code */
1120 while (tb
!= NULL
) {
1121 n
= (uintptr_t)tb
& 3;
1122 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1123 tb_next
= tb
->page_next
[n
];
1124 /* NOTE: this is subtle as a TB may span two physical pages */
1126 /* NOTE: tb_end may be after the end of the page, but
1127 it is not a problem */
1128 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1129 tb_end
= tb_start
+ tb
->size
;
1131 tb_start
= tb
->page_addr
[1];
1132 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1134 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1135 #ifdef TARGET_HAS_PRECISE_SMC
1136 if (current_tb_not_found
) {
1137 current_tb_not_found
= 0;
1139 if (env
->mem_io_pc
) {
1140 /* now we have a real cpu fault */
1141 current_tb
= tb_find_pc(env
->mem_io_pc
);
1144 if (current_tb
== tb
&&
1145 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1146 /* If we are modifying the current TB, we must stop
1147 its execution. We could be more precise by checking
1148 that the modification is after the current PC, but it
1149 would require a specialized function to partially
1150 restore the CPU state */
1152 current_tb_modified
= 1;
1153 cpu_restore_state(current_tb
, env
, env
->mem_io_pc
);
1154 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1157 #endif /* TARGET_HAS_PRECISE_SMC */
1158 /* we need to do that to handle the case where a signal
1159 occurs while doing tb_phys_invalidate() */
1162 saved_tb
= env
->current_tb
;
1163 env
->current_tb
= NULL
;
1165 tb_phys_invalidate(tb
, -1);
1167 env
->current_tb
= saved_tb
;
1168 if (env
->interrupt_request
&& env
->current_tb
)
1169 cpu_interrupt(env
, env
->interrupt_request
);
1174 #if !defined(CONFIG_USER_ONLY)
1175 /* if no code remaining, no need to continue to use slow writes */
1177 invalidate_page_bitmap(p
);
1178 if (is_cpu_write_access
) {
1179 tlb_unprotect_code_phys(env
, start
, env
->mem_io_vaddr
);
1183 #ifdef TARGET_HAS_PRECISE_SMC
1184 if (current_tb_modified
) {
1185 /* we generate a block containing just the instruction
1186 modifying the memory. It will ensure that it cannot modify
1188 env
->current_tb
= NULL
;
1189 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1190 cpu_resume_from_signal(env
, NULL
);
1195 /* len must be <= 8 and start must be a multiple of len */
1196 static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1202 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1203 cpu_single_env
->mem_io_vaddr
, len
,
1204 cpu_single_env
->eip
,
1205 cpu_single_env
->eip
+
1206 (intptr_t)cpu_single_env
->segs
[R_CS
].base
);
1209 p
= page_find(start
>> TARGET_PAGE_BITS
);
1212 if (p
->code_bitmap
) {
1213 offset
= start
& ~TARGET_PAGE_MASK
;
1214 b
= p
->code_bitmap
[offset
>> 3] >> (offset
& 7);
1215 if (b
& ((1 << len
) - 1))
1219 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1223 #if !defined(CONFIG_SOFTMMU)
1224 static void tb_invalidate_phys_page(tb_page_addr_t addr
,
1225 uintptr_t pc
, void *puc
)
1227 TranslationBlock
*tb
;
1230 #ifdef TARGET_HAS_PRECISE_SMC
1231 TranslationBlock
*current_tb
= NULL
;
1232 CPUArchState
*env
= cpu_single_env
;
1233 int current_tb_modified
= 0;
1234 target_ulong current_pc
= 0;
1235 target_ulong current_cs_base
= 0;
1236 int current_flags
= 0;
1239 addr
&= TARGET_PAGE_MASK
;
1240 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1244 #ifdef TARGET_HAS_PRECISE_SMC
1245 if (tb
&& pc
!= 0) {
1246 current_tb
= tb_find_pc(pc
);
1249 while (tb
!= NULL
) {
1250 n
= (uintptr_t)tb
& 3;
1251 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1252 #ifdef TARGET_HAS_PRECISE_SMC
1253 if (current_tb
== tb
&&
1254 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1255 /* If we are modifying the current TB, we must stop
1256 its execution. We could be more precise by checking
1257 that the modification is after the current PC, but it
1258 would require a specialized function to partially
1259 restore the CPU state */
1261 current_tb_modified
= 1;
1262 cpu_restore_state(current_tb
, env
, pc
);
1263 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1266 #endif /* TARGET_HAS_PRECISE_SMC */
1267 tb_phys_invalidate(tb
, addr
);
1268 tb
= tb
->page_next
[n
];
1271 #ifdef TARGET_HAS_PRECISE_SMC
1272 if (current_tb_modified
) {
1273 /* we generate a block containing just the instruction
1274 modifying the memory. It will ensure that it cannot modify
1276 env
->current_tb
= NULL
;
1277 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1278 cpu_resume_from_signal(env
, puc
);
1284 /* add the tb in the target page and protect it if necessary */
1285 static inline void tb_alloc_page(TranslationBlock
*tb
,
1286 unsigned int n
, tb_page_addr_t page_addr
)
1289 #ifndef CONFIG_USER_ONLY
1290 bool page_already_protected
;
1293 tb
->page_addr
[n
] = page_addr
;
1294 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1295 tb
->page_next
[n
] = p
->first_tb
;
1296 #ifndef CONFIG_USER_ONLY
1297 page_already_protected
= p
->first_tb
!= NULL
;
1299 p
->first_tb
= (TranslationBlock
*)((uintptr_t)tb
| n
);
1300 invalidate_page_bitmap(p
);
1302 #if defined(TARGET_HAS_SMC) || 1
1304 #if defined(CONFIG_USER_ONLY)
1305 if (p
->flags
& PAGE_WRITE
) {
1310 /* force the host page as non writable (writes will have a
1311 page fault + mprotect overhead) */
1312 page_addr
&= qemu_host_page_mask
;
1314 for(addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1315 addr
+= TARGET_PAGE_SIZE
) {
1317 p2
= page_find (addr
>> TARGET_PAGE_BITS
);
1321 p2
->flags
&= ~PAGE_WRITE
;
1323 mprotect(g2h(page_addr
), qemu_host_page_size
,
1324 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1325 #ifdef DEBUG_TB_INVALIDATE
1326 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1331 /* if some code is already present, then the pages are already
1332 protected. So we handle the case where only the first TB is
1333 allocated in a physical page */
1334 if (!page_already_protected
) {
1335 tlb_protect_code(page_addr
);
1339 #endif /* TARGET_HAS_SMC */
1342 /* add a new TB and link it to the physical page tables. phys_page2 is
1343 (-1) to indicate that only one page contains the TB. */
1344 void tb_link_page(TranslationBlock
*tb
,
1345 tb_page_addr_t phys_pc
, tb_page_addr_t phys_page2
)
1348 TranslationBlock
**ptb
;
1350 /* Grab the mmap lock to stop another thread invalidating this TB
1351 before we are done. */
1353 /* add in the physical hash table */
1354 h
= tb_phys_hash_func(phys_pc
);
1355 ptb
= &tb_phys_hash
[h
];
1356 tb
->phys_hash_next
= *ptb
;
1359 /* add in the page list */
1360 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1361 if (phys_page2
!= -1)
1362 tb_alloc_page(tb
, 1, phys_page2
);
1364 tb
->page_addr
[1] = -1;
1366 tb
->jmp_first
= (TranslationBlock
*)((uintptr_t)tb
| 2);
1367 tb
->jmp_next
[0] = NULL
;
1368 tb
->jmp_next
[1] = NULL
;
1370 /* init original jump addresses */
1371 if (tb
->tb_next_offset
[0] != 0xffff)
1372 tb_reset_jump(tb
, 0);
1373 if (tb
->tb_next_offset
[1] != 0xffff)
1374 tb_reset_jump(tb
, 1);
1376 #ifdef DEBUG_TB_CHECK
1382 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1383 tb[1].tc_ptr. Return NULL if not found */
1384 TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
)
1386 int m_min
, m_max
, m
;
1388 TranslationBlock
*tb
;
1392 if (tc_ptr
< (uintptr_t)code_gen_buffer
||
1393 tc_ptr
>= (uintptr_t)code_gen_ptr
) {
1396 /* binary search (cf Knuth) */
1399 while (m_min
<= m_max
) {
1400 m
= (m_min
+ m_max
) >> 1;
1402 v
= (uintptr_t)tb
->tc_ptr
;
1405 else if (tc_ptr
< v
) {
1414 static void tb_reset_jump_recursive(TranslationBlock
*tb
);
1416 static inline void tb_reset_jump_recursive2(TranslationBlock
*tb
, int n
)
1418 TranslationBlock
*tb1
, *tb_next
, **ptb
;
1421 tb1
= tb
->jmp_next
[n
];
1423 /* find head of list */
1425 n1
= (uintptr_t)tb1
& 3;
1426 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
1429 tb1
= tb1
->jmp_next
[n1
];
1431 /* we are now sure now that tb jumps to tb1 */
1434 /* remove tb from the jmp_first list */
1435 ptb
= &tb_next
->jmp_first
;
1438 n1
= (uintptr_t)tb1
& 3;
1439 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
1440 if (n1
== n
&& tb1
== tb
)
1442 ptb
= &tb1
->jmp_next
[n1
];
1444 *ptb
= tb
->jmp_next
[n
];
1445 tb
->jmp_next
[n
] = NULL
;
1447 /* suppress the jump to next tb in generated code */
1448 tb_reset_jump(tb
, n
);
1450 /* suppress jumps in the tb on which we could have jumped */
1451 tb_reset_jump_recursive(tb_next
);
1455 static void tb_reset_jump_recursive(TranslationBlock
*tb
)
1457 tb_reset_jump_recursive2(tb
, 0);
1458 tb_reset_jump_recursive2(tb
, 1);
1461 #if defined(TARGET_HAS_ICE)
1462 #if defined(CONFIG_USER_ONLY)
1463 static void breakpoint_invalidate(CPUArchState
*env
, target_ulong pc
)
1465 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
1468 void tb_invalidate_phys_addr(target_phys_addr_t addr
)
1470 ram_addr_t ram_addr
;
1471 MemoryRegionSection
*section
;
1473 section
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
1474 if (!(memory_region_is_ram(section
->mr
)
1475 || (section
->mr
->rom_device
&& section
->mr
->readable
))) {
1478 ram_addr
= (memory_region_get_ram_addr(section
->mr
) & TARGET_PAGE_MASK
)
1479 + memory_region_section_addr(section
, addr
);
1480 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1483 static void breakpoint_invalidate(CPUArchState
*env
, target_ulong pc
)
1485 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env
, pc
) |
1486 (pc
& ~TARGET_PAGE_MASK
));
1489 #endif /* TARGET_HAS_ICE */
1491 #if defined(CONFIG_USER_ONLY)
1492 void cpu_watchpoint_remove_all(CPUArchState
*env
, int mask
)
1497 int cpu_watchpoint_insert(CPUArchState
*env
, target_ulong addr
, target_ulong len
,
1498 int flags
, CPUWatchpoint
**watchpoint
)
1503 /* Add a watchpoint. */
1504 int cpu_watchpoint_insert(CPUArchState
*env
, target_ulong addr
, target_ulong len
,
1505 int flags
, CPUWatchpoint
**watchpoint
)
1507 target_ulong len_mask
= ~(len
- 1);
1510 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1511 if ((len
& (len
- 1)) || (addr
& ~len_mask
) ||
1512 len
== 0 || len
> TARGET_PAGE_SIZE
) {
1513 fprintf(stderr
, "qemu: tried to set invalid watchpoint at "
1514 TARGET_FMT_lx
", len=" TARGET_FMT_lu
"\n", addr
, len
);
1517 wp
= g_malloc(sizeof(*wp
));
1520 wp
->len_mask
= len_mask
;
1523 /* keep all GDB-injected watchpoints in front */
1525 QTAILQ_INSERT_HEAD(&env
->watchpoints
, wp
, entry
);
1527 QTAILQ_INSERT_TAIL(&env
->watchpoints
, wp
, entry
);
1529 tlb_flush_page(env
, addr
);
1536 /* Remove a specific watchpoint. */
1537 int cpu_watchpoint_remove(CPUArchState
*env
, target_ulong addr
, target_ulong len
,
1540 target_ulong len_mask
= ~(len
- 1);
1543 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1544 if (addr
== wp
->vaddr
&& len_mask
== wp
->len_mask
1545 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
1546 cpu_watchpoint_remove_by_ref(env
, wp
);
1553 /* Remove a specific watchpoint by reference. */
1554 void cpu_watchpoint_remove_by_ref(CPUArchState
*env
, CPUWatchpoint
*watchpoint
)
1556 QTAILQ_REMOVE(&env
->watchpoints
, watchpoint
, entry
);
1558 tlb_flush_page(env
, watchpoint
->vaddr
);
1563 /* Remove all matching watchpoints. */
1564 void cpu_watchpoint_remove_all(CPUArchState
*env
, int mask
)
1566 CPUWatchpoint
*wp
, *next
;
1568 QTAILQ_FOREACH_SAFE(wp
, &env
->watchpoints
, entry
, next
) {
1569 if (wp
->flags
& mask
)
1570 cpu_watchpoint_remove_by_ref(env
, wp
);
1575 /* Add a breakpoint. */
1576 int cpu_breakpoint_insert(CPUArchState
*env
, target_ulong pc
, int flags
,
1577 CPUBreakpoint
**breakpoint
)
1579 #if defined(TARGET_HAS_ICE)
1582 bp
= g_malloc(sizeof(*bp
));
1587 /* keep all GDB-injected breakpoints in front */
1589 QTAILQ_INSERT_HEAD(&env
->breakpoints
, bp
, entry
);
1591 QTAILQ_INSERT_TAIL(&env
->breakpoints
, bp
, entry
);
1593 breakpoint_invalidate(env
, pc
);
1603 /* Remove a specific breakpoint. */
1604 int cpu_breakpoint_remove(CPUArchState
*env
, target_ulong pc
, int flags
)
1606 #if defined(TARGET_HAS_ICE)
1609 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1610 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
1611 cpu_breakpoint_remove_by_ref(env
, bp
);
1621 /* Remove a specific breakpoint by reference. */
1622 void cpu_breakpoint_remove_by_ref(CPUArchState
*env
, CPUBreakpoint
*breakpoint
)
1624 #if defined(TARGET_HAS_ICE)
1625 QTAILQ_REMOVE(&env
->breakpoints
, breakpoint
, entry
);
1627 breakpoint_invalidate(env
, breakpoint
->pc
);
1633 /* Remove all matching breakpoints. */
1634 void cpu_breakpoint_remove_all(CPUArchState
*env
, int mask
)
1636 #if defined(TARGET_HAS_ICE)
1637 CPUBreakpoint
*bp
, *next
;
1639 QTAILQ_FOREACH_SAFE(bp
, &env
->breakpoints
, entry
, next
) {
1640 if (bp
->flags
& mask
)
1641 cpu_breakpoint_remove_by_ref(env
, bp
);
1646 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1647 CPU loop after each instruction */
1648 void cpu_single_step(CPUArchState
*env
, int enabled
)
1650 #if defined(TARGET_HAS_ICE)
1651 if (env
->singlestep_enabled
!= enabled
) {
1652 env
->singlestep_enabled
= enabled
;
1654 kvm_update_guest_debug(env
, 0);
1656 /* must flush all the translated code to avoid inconsistencies */
1657 /* XXX: only flush what is necessary */
1664 static void cpu_unlink_tb(CPUArchState
*env
)
1666 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1667 problem and hope the cpu will stop of its own accord. For userspace
1668 emulation this often isn't actually as bad as it sounds. Often
1669 signals are used primarily to interrupt blocking syscalls. */
1670 TranslationBlock
*tb
;
1671 static spinlock_t interrupt_lock
= SPIN_LOCK_UNLOCKED
;
1673 spin_lock(&interrupt_lock
);
1674 tb
= env
->current_tb
;
1675 /* if the cpu is currently executing code, we must unlink it and
1676 all the potentially executing TB */
1678 env
->current_tb
= NULL
;
1679 tb_reset_jump_recursive(tb
);
1681 spin_unlock(&interrupt_lock
);
1684 #ifndef CONFIG_USER_ONLY
1685 /* mask must never be zero, except for A20 change call */
1686 static void tcg_handle_interrupt(CPUArchState
*env
, int mask
)
1690 old_mask
= env
->interrupt_request
;
1691 env
->interrupt_request
|= mask
;
1694 * If called from iothread context, wake the target cpu in
1697 if (!qemu_cpu_is_self(env
)) {
1703 env
->icount_decr
.u16
.high
= 0xffff;
1705 && (mask
& ~old_mask
) != 0) {
1706 cpu_abort(env
, "Raised interrupt while not in I/O function");
1713 CPUInterruptHandler cpu_interrupt_handler
= tcg_handle_interrupt
;
1715 #else /* CONFIG_USER_ONLY */
1717 void cpu_interrupt(CPUArchState
*env
, int mask
)
1719 env
->interrupt_request
|= mask
;
1722 #endif /* CONFIG_USER_ONLY */
1724 void cpu_reset_interrupt(CPUArchState
*env
, int mask
)
1726 env
->interrupt_request
&= ~mask
;
1729 void cpu_exit(CPUArchState
*env
)
1731 env
->exit_request
= 1;
1735 void cpu_abort(CPUArchState
*env
, const char *fmt
, ...)
1742 fprintf(stderr
, "qemu: fatal: ");
1743 vfprintf(stderr
, fmt
, ap
);
1744 fprintf(stderr
, "\n");
1746 cpu_dump_state(env
, stderr
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1748 cpu_dump_state(env
, stderr
, fprintf
, 0);
1750 if (qemu_log_enabled()) {
1751 qemu_log("qemu: fatal: ");
1752 qemu_log_vprintf(fmt
, ap2
);
1755 log_cpu_state(env
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1757 log_cpu_state(env
, 0);
1764 #if defined(CONFIG_USER_ONLY)
1766 struct sigaction act
;
1767 sigfillset(&act
.sa_mask
);
1768 act
.sa_handler
= SIG_DFL
;
1769 sigaction(SIGABRT
, &act
, NULL
);
1775 CPUArchState
*cpu_copy(CPUArchState
*env
)
1777 CPUArchState
*new_env
= cpu_init(env
->cpu_model_str
);
1778 CPUArchState
*next_cpu
= new_env
->next_cpu
;
1779 int cpu_index
= new_env
->cpu_index
;
1780 #if defined(TARGET_HAS_ICE)
1785 memcpy(new_env
, env
, sizeof(CPUArchState
));
1787 /* Preserve chaining and index. */
1788 new_env
->next_cpu
= next_cpu
;
1789 new_env
->cpu_index
= cpu_index
;
1791 /* Clone all break/watchpoints.
1792 Note: Once we support ptrace with hw-debug register access, make sure
1793 BP_CPU break/watchpoints are handled correctly on clone. */
1794 QTAILQ_INIT(&env
->breakpoints
);
1795 QTAILQ_INIT(&env
->watchpoints
);
1796 #if defined(TARGET_HAS_ICE)
1797 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1798 cpu_breakpoint_insert(new_env
, bp
->pc
, bp
->flags
, NULL
);
1800 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1801 cpu_watchpoint_insert(new_env
, wp
->vaddr
, (~wp
->len_mask
) + 1,
1809 #if !defined(CONFIG_USER_ONLY)
1810 void tb_flush_jmp_cache(CPUArchState
*env
, target_ulong addr
)
1814 /* Discard jump cache entries for any tb which might potentially
1815 overlap the flushed page. */
1816 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1817 memset (&env
->tb_jmp_cache
[i
], 0,
1818 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1820 i
= tb_jmp_cache_hash_page(addr
);
1821 memset (&env
->tb_jmp_cache
[i
], 0,
1822 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1825 static void tlb_reset_dirty_range_all(ram_addr_t start
, ram_addr_t end
,
1830 /* we modify the TLB cache so that the dirty bit will be set again
1831 when accessing the range */
1832 start1
= (uintptr_t)qemu_safe_ram_ptr(start
);
1833 /* Check that we don't span multiple blocks - this breaks the
1834 address comparisons below. */
1835 if ((uintptr_t)qemu_safe_ram_ptr(end
- 1) - start1
1836 != (end
- 1) - start
) {
1839 cpu_tlb_reset_dirty_all(start1
, length
);
1843 /* Note: start and end must be within the same ram block. */
1844 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t end
,
1849 start
&= TARGET_PAGE_MASK
;
1850 end
= TARGET_PAGE_ALIGN(end
);
1852 length
= end
- start
;
1855 cpu_physical_memory_mask_dirty_range(start
, length
, dirty_flags
);
1857 if (tcg_enabled()) {
1858 tlb_reset_dirty_range_all(start
, end
, length
);
1862 int cpu_physical_memory_set_dirty_tracking(int enable
)
1865 in_migration
= enable
;
1869 target_phys_addr_t
memory_region_section_get_iotlb(CPUArchState
*env
,
1870 MemoryRegionSection
*section
,
1872 target_phys_addr_t paddr
,
1874 target_ulong
*address
)
1876 target_phys_addr_t iotlb
;
1879 if (memory_region_is_ram(section
->mr
)) {
1881 iotlb
= (memory_region_get_ram_addr(section
->mr
) & TARGET_PAGE_MASK
)
1882 + memory_region_section_addr(section
, paddr
);
1883 if (!section
->readonly
) {
1884 iotlb
|= phys_section_notdirty
;
1886 iotlb
|= phys_section_rom
;
1889 /* IO handlers are currently passed a physical address.
1890 It would be nice to pass an offset from the base address
1891 of that region. This would avoid having to special case RAM,
1892 and avoid full address decoding in every device.
1893 We can't use the high bits of pd for this because
1894 IO_MEM_ROMD uses these as a ram address. */
1895 iotlb
= section
- phys_sections
;
1896 iotlb
+= memory_region_section_addr(section
, paddr
);
1899 /* Make accesses to pages with watchpoints go via the
1900 watchpoint trap routines. */
1901 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1902 if (vaddr
== (wp
->vaddr
& TARGET_PAGE_MASK
)) {
1903 /* Avoid trapping reads of pages with a write breakpoint. */
1904 if ((prot
& PAGE_WRITE
) || (wp
->flags
& BP_MEM_READ
)) {
1905 iotlb
= phys_section_watch
+ paddr
;
1906 *address
|= TLB_MMIO
;
1917 * Walks guest process memory "regions" one by one
1918 * and calls callback function 'fn' for each region.
1921 struct walk_memory_regions_data
1923 walk_memory_regions_fn fn
;
1929 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
1930 abi_ulong end
, int new_prot
)
1932 if (data
->start
!= -1ul) {
1933 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
1939 data
->start
= (new_prot
? end
: -1ul);
1940 data
->prot
= new_prot
;
1945 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
1946 abi_ulong base
, int level
, void **lp
)
1952 return walk_memory_regions_end(data
, base
, 0);
1957 for (i
= 0; i
< L2_SIZE
; ++i
) {
1958 int prot
= pd
[i
].flags
;
1960 pa
= base
| (i
<< TARGET_PAGE_BITS
);
1961 if (prot
!= data
->prot
) {
1962 rc
= walk_memory_regions_end(data
, pa
, prot
);
1970 for (i
= 0; i
< L2_SIZE
; ++i
) {
1971 pa
= base
| ((abi_ulong
)i
<<
1972 (TARGET_PAGE_BITS
+ L2_BITS
* level
));
1973 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
1983 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
1985 struct walk_memory_regions_data data
;
1993 for (i
= 0; i
< V_L1_SIZE
; i
++) {
1994 int rc
= walk_memory_regions_1(&data
, (abi_ulong
)i
<< V_L1_SHIFT
,
1995 V_L1_SHIFT
/ L2_BITS
- 1, l1_map
+ i
);
2001 return walk_memory_regions_end(&data
, 0, 0);
2004 static int dump_region(void *priv
, abi_ulong start
,
2005 abi_ulong end
, unsigned long prot
)
2007 FILE *f
= (FILE *)priv
;
2009 (void) fprintf(f
, TARGET_ABI_FMT_lx
"-"TARGET_ABI_FMT_lx
2010 " "TARGET_ABI_FMT_lx
" %c%c%c\n",
2011 start
, end
, end
- start
,
2012 ((prot
& PAGE_READ
) ? 'r' : '-'),
2013 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
2014 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
2019 /* dump memory mappings */
2020 void page_dump(FILE *f
)
2022 (void) fprintf(f
, "%-8s %-8s %-8s %s\n",
2023 "start", "end", "size", "prot");
2024 walk_memory_regions(f
, dump_region
);
2027 int page_get_flags(target_ulong address
)
2031 p
= page_find(address
>> TARGET_PAGE_BITS
);
2037 /* Modify the flags of a page and invalidate the code if necessary.
2038 The flag PAGE_WRITE_ORG is positioned automatically depending
2039 on PAGE_WRITE. The mmap_lock should already be held. */
2040 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2042 target_ulong addr
, len
;
2044 /* This function should never be called with addresses outside the
2045 guest address space. If this assert fires, it probably indicates
2046 a missing call to h2g_valid. */
2047 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2048 assert(end
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2050 assert(start
< end
);
2052 start
= start
& TARGET_PAGE_MASK
;
2053 end
= TARGET_PAGE_ALIGN(end
);
2055 if (flags
& PAGE_WRITE
) {
2056 flags
|= PAGE_WRITE_ORG
;
2059 for (addr
= start
, len
= end
- start
;
2061 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2062 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2064 /* If the write protection bit is set, then we invalidate
2066 if (!(p
->flags
& PAGE_WRITE
) &&
2067 (flags
& PAGE_WRITE
) &&
2069 tb_invalidate_phys_page(addr
, 0, NULL
);
2075 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2081 /* This function should never be called with addresses outside the
2082 guest address space. If this assert fires, it probably indicates
2083 a missing call to h2g_valid. */
2084 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2085 assert(start
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2091 if (start
+ len
- 1 < start
) {
2092 /* We've wrapped around. */
2096 end
= TARGET_PAGE_ALIGN(start
+len
); /* must do before we loose bits in the next step */
2097 start
= start
& TARGET_PAGE_MASK
;
2099 for (addr
= start
, len
= end
- start
;
2101 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2102 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2105 if( !(p
->flags
& PAGE_VALID
) )
2108 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
))
2110 if (flags
& PAGE_WRITE
) {
2111 if (!(p
->flags
& PAGE_WRITE_ORG
))
2113 /* unprotect the page if it was put read-only because it
2114 contains translated code */
2115 if (!(p
->flags
& PAGE_WRITE
)) {
2116 if (!page_unprotect(addr
, 0, NULL
))
2125 /* called from signal handler: invalidate the code and unprotect the
2126 page. Return TRUE if the fault was successfully handled. */
2127 int page_unprotect(target_ulong address
, uintptr_t pc
, void *puc
)
2131 target_ulong host_start
, host_end
, addr
;
2133 /* Technically this isn't safe inside a signal handler. However we
2134 know this only ever happens in a synchronous SEGV handler, so in
2135 practice it seems to be ok. */
2138 p
= page_find(address
>> TARGET_PAGE_BITS
);
2144 /* if the page was really writable, then we change its
2145 protection back to writable */
2146 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
2147 host_start
= address
& qemu_host_page_mask
;
2148 host_end
= host_start
+ qemu_host_page_size
;
2151 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2152 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2153 p
->flags
|= PAGE_WRITE
;
2156 /* and since the content will be modified, we must invalidate
2157 the corresponding translated code. */
2158 tb_invalidate_phys_page(addr
, pc
, puc
);
2159 #ifdef DEBUG_TB_CHECK
2160 tb_invalidate_check(addr
);
2163 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2172 #endif /* defined(CONFIG_USER_ONLY) */
2174 #if !defined(CONFIG_USER_ONLY)
2176 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2177 typedef struct subpage_t
{
2179 target_phys_addr_t base
;
2180 uint16_t sub_section
[TARGET_PAGE_SIZE
];
2183 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2185 static subpage_t
*subpage_init(target_phys_addr_t base
);
2186 static void destroy_page_desc(uint16_t section_index
)
2188 MemoryRegionSection
*section
= &phys_sections
[section_index
];
2189 MemoryRegion
*mr
= section
->mr
;
2192 subpage_t
*subpage
= container_of(mr
, subpage_t
, iomem
);
2193 memory_region_destroy(&subpage
->iomem
);
2198 static void destroy_l2_mapping(PhysPageEntry
*lp
, unsigned level
)
2203 if (lp
->ptr
== PHYS_MAP_NODE_NIL
) {
2207 p
= phys_map_nodes
[lp
->ptr
];
2208 for (i
= 0; i
< L2_SIZE
; ++i
) {
2209 if (!p
[i
].is_leaf
) {
2210 destroy_l2_mapping(&p
[i
], level
- 1);
2212 destroy_page_desc(p
[i
].ptr
);
2216 lp
->ptr
= PHYS_MAP_NODE_NIL
;
2219 static void destroy_all_mappings(void)
2221 destroy_l2_mapping(&phys_map
, P_L2_LEVELS
- 1);
2222 phys_map_nodes_reset();
2225 static uint16_t phys_section_add(MemoryRegionSection
*section
)
2227 if (phys_sections_nb
== phys_sections_nb_alloc
) {
2228 phys_sections_nb_alloc
= MAX(phys_sections_nb_alloc
* 2, 16);
2229 phys_sections
= g_renew(MemoryRegionSection
, phys_sections
,
2230 phys_sections_nb_alloc
);
2232 phys_sections
[phys_sections_nb
] = *section
;
2233 return phys_sections_nb
++;
2236 static void phys_sections_clear(void)
2238 phys_sections_nb
= 0;
2241 static void register_subpage(MemoryRegionSection
*section
)
2244 target_phys_addr_t base
= section
->offset_within_address_space
2246 MemoryRegionSection
*existing
= phys_page_find(base
>> TARGET_PAGE_BITS
);
2247 MemoryRegionSection subsection
= {
2248 .offset_within_address_space
= base
,
2249 .size
= TARGET_PAGE_SIZE
,
2251 target_phys_addr_t start
, end
;
2253 assert(existing
->mr
->subpage
|| existing
->mr
== &io_mem_unassigned
);
2255 if (!(existing
->mr
->subpage
)) {
2256 subpage
= subpage_init(base
);
2257 subsection
.mr
= &subpage
->iomem
;
2258 phys_page_set(base
>> TARGET_PAGE_BITS
, 1,
2259 phys_section_add(&subsection
));
2261 subpage
= container_of(existing
->mr
, subpage_t
, iomem
);
2263 start
= section
->offset_within_address_space
& ~TARGET_PAGE_MASK
;
2264 end
= start
+ section
->size
- 1;
2265 subpage_register(subpage
, start
, end
, phys_section_add(section
));
2269 static void register_multipage(MemoryRegionSection
*section
)
2271 target_phys_addr_t start_addr
= section
->offset_within_address_space
;
2272 ram_addr_t size
= section
->size
;
2273 target_phys_addr_t addr
;
2274 uint16_t section_index
= phys_section_add(section
);
2279 phys_page_set(addr
>> TARGET_PAGE_BITS
, size
>> TARGET_PAGE_BITS
,
2283 void cpu_register_physical_memory_log(MemoryRegionSection
*section
,
2286 MemoryRegionSection now
= *section
, remain
= *section
;
2288 if ((now
.offset_within_address_space
& ~TARGET_PAGE_MASK
)
2289 || (now
.size
< TARGET_PAGE_SIZE
)) {
2290 now
.size
= MIN(TARGET_PAGE_ALIGN(now
.offset_within_address_space
)
2291 - now
.offset_within_address_space
,
2293 register_subpage(&now
);
2294 remain
.size
-= now
.size
;
2295 remain
.offset_within_address_space
+= now
.size
;
2296 remain
.offset_within_region
+= now
.size
;
2298 while (remain
.size
>= TARGET_PAGE_SIZE
) {
2300 if (remain
.offset_within_region
& ~TARGET_PAGE_MASK
) {
2301 now
.size
= TARGET_PAGE_SIZE
;
2302 register_subpage(&now
);
2304 now
.size
&= TARGET_PAGE_MASK
;
2305 register_multipage(&now
);
2307 remain
.size
-= now
.size
;
2308 remain
.offset_within_address_space
+= now
.size
;
2309 remain
.offset_within_region
+= now
.size
;
2313 register_subpage(&now
);
2318 void qemu_register_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2321 kvm_coalesce_mmio_region(addr
, size
);
2324 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2327 kvm_uncoalesce_mmio_region(addr
, size
);
2330 void qemu_flush_coalesced_mmio_buffer(void)
2333 kvm_flush_coalesced_mmio_buffer();
2336 #if defined(__linux__) && !defined(TARGET_S390X)
2338 #include <sys/vfs.h>
2340 #define HUGETLBFS_MAGIC 0x958458f6
2342 static long gethugepagesize(const char *path
)
2348 ret
= statfs(path
, &fs
);
2349 } while (ret
!= 0 && errno
== EINTR
);
2356 if (fs
.f_type
!= HUGETLBFS_MAGIC
)
2357 fprintf(stderr
, "Warning: path not on HugeTLBFS: %s\n", path
);
2362 static void *file_ram_alloc(RAMBlock
*block
,
2372 unsigned long hpagesize
;
2374 hpagesize
= gethugepagesize(path
);
2379 if (memory
< hpagesize
) {
2383 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2384 fprintf(stderr
, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2388 if (asprintf(&filename
, "%s/qemu_back_mem.XXXXXX", path
) == -1) {
2392 fd
= mkstemp(filename
);
2394 perror("unable to create backing store for hugepages");
2401 memory
= (memory
+hpagesize
-1) & ~(hpagesize
-1);
2404 * ftruncate is not supported by hugetlbfs in older
2405 * hosts, so don't bother bailing out on errors.
2406 * If anything goes wrong with it under other filesystems,
2409 if (ftruncate(fd
, memory
))
2410 perror("ftruncate");
2413 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2414 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2415 * to sidestep this quirk.
2417 flags
= mem_prealloc
? MAP_POPULATE
| MAP_SHARED
: MAP_PRIVATE
;
2418 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, flags
, fd
, 0);
2420 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, MAP_PRIVATE
, fd
, 0);
2422 if (area
== MAP_FAILED
) {
2423 perror("file_ram_alloc: can't mmap RAM pages");
2432 static ram_addr_t
find_ram_offset(ram_addr_t size
)
2434 RAMBlock
*block
, *next_block
;
2435 ram_addr_t offset
= RAM_ADDR_MAX
, mingap
= RAM_ADDR_MAX
;
2437 if (QLIST_EMPTY(&ram_list
.blocks
))
2440 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2441 ram_addr_t end
, next
= RAM_ADDR_MAX
;
2443 end
= block
->offset
+ block
->length
;
2445 QLIST_FOREACH(next_block
, &ram_list
.blocks
, next
) {
2446 if (next_block
->offset
>= end
) {
2447 next
= MIN(next
, next_block
->offset
);
2450 if (next
- end
>= size
&& next
- end
< mingap
) {
2452 mingap
= next
- end
;
2456 if (offset
== RAM_ADDR_MAX
) {
2457 fprintf(stderr
, "Failed to find gap of requested size: %" PRIu64
"\n",
2465 static ram_addr_t
last_ram_offset(void)
2468 ram_addr_t last
= 0;
2470 QLIST_FOREACH(block
, &ram_list
.blocks
, next
)
2471 last
= MAX(last
, block
->offset
+ block
->length
);
2476 static void qemu_ram_setup_dump(void *addr
, ram_addr_t size
)
2479 QemuOpts
*machine_opts
;
2481 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
2482 machine_opts
= qemu_opts_find(qemu_find_opts("machine"), 0);
2484 !qemu_opt_get_bool(machine_opts
, "dump-guest-core", true)) {
2485 ret
= qemu_madvise(addr
, size
, QEMU_MADV_DONTDUMP
);
2487 perror("qemu_madvise");
2488 fprintf(stderr
, "madvise doesn't support MADV_DONTDUMP, "
2489 "but dump_guest_core=off specified\n");
2494 void qemu_ram_set_idstr(ram_addr_t addr
, const char *name
, DeviceState
*dev
)
2496 RAMBlock
*new_block
, *block
;
2499 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2500 if (block
->offset
== addr
) {
2506 assert(!new_block
->idstr
[0]);
2509 char *id
= qdev_get_dev_path(dev
);
2511 snprintf(new_block
->idstr
, sizeof(new_block
->idstr
), "%s/", id
);
2515 pstrcat(new_block
->idstr
, sizeof(new_block
->idstr
), name
);
2517 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2518 if (block
!= new_block
&& !strcmp(block
->idstr
, new_block
->idstr
)) {
2519 fprintf(stderr
, "RAMBlock \"%s\" already registered, abort!\n",
2526 ram_addr_t
qemu_ram_alloc_from_ptr(ram_addr_t size
, void *host
,
2529 RAMBlock
*new_block
;
2531 size
= TARGET_PAGE_ALIGN(size
);
2532 new_block
= g_malloc0(sizeof(*new_block
));
2535 new_block
->offset
= find_ram_offset(size
);
2537 new_block
->host
= host
;
2538 new_block
->flags
|= RAM_PREALLOC_MASK
;
2541 #if defined (__linux__) && !defined(TARGET_S390X)
2542 new_block
->host
= file_ram_alloc(new_block
, size
, mem_path
);
2543 if (!new_block
->host
) {
2544 new_block
->host
= qemu_vmalloc(size
);
2545 qemu_madvise(new_block
->host
, size
, QEMU_MADV_MERGEABLE
);
2548 fprintf(stderr
, "-mem-path option unsupported\n");
2552 if (xen_enabled()) {
2553 xen_ram_alloc(new_block
->offset
, size
, mr
);
2554 } else if (kvm_enabled()) {
2555 /* some s390/kvm configurations have special constraints */
2556 new_block
->host
= kvm_vmalloc(size
);
2558 new_block
->host
= qemu_vmalloc(size
);
2560 qemu_madvise(new_block
->host
, size
, QEMU_MADV_MERGEABLE
);
2563 new_block
->length
= size
;
2565 QLIST_INSERT_HEAD(&ram_list
.blocks
, new_block
, next
);
2567 ram_list
.phys_dirty
= g_realloc(ram_list
.phys_dirty
,
2568 last_ram_offset() >> TARGET_PAGE_BITS
);
2569 memset(ram_list
.phys_dirty
+ (new_block
->offset
>> TARGET_PAGE_BITS
),
2570 0, size
>> TARGET_PAGE_BITS
);
2571 cpu_physical_memory_set_dirty_range(new_block
->offset
, size
, 0xff);
2573 qemu_ram_setup_dump(new_block
->host
, size
);
2576 kvm_setup_guest_memory(new_block
->host
, size
);
2578 return new_block
->offset
;
2581 ram_addr_t
qemu_ram_alloc(ram_addr_t size
, MemoryRegion
*mr
)
2583 return qemu_ram_alloc_from_ptr(size
, NULL
, mr
);
2586 void qemu_ram_free_from_ptr(ram_addr_t addr
)
2590 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2591 if (addr
== block
->offset
) {
2592 QLIST_REMOVE(block
, next
);
2599 void qemu_ram_free(ram_addr_t addr
)
2603 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2604 if (addr
== block
->offset
) {
2605 QLIST_REMOVE(block
, next
);
2606 if (block
->flags
& RAM_PREALLOC_MASK
) {
2608 } else if (mem_path
) {
2609 #if defined (__linux__) && !defined(TARGET_S390X)
2611 munmap(block
->host
, block
->length
);
2614 qemu_vfree(block
->host
);
2620 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2621 munmap(block
->host
, block
->length
);
2623 if (xen_enabled()) {
2624 xen_invalidate_map_cache_entry(block
->host
);
2626 qemu_vfree(block
->host
);
2638 void qemu_ram_remap(ram_addr_t addr
, ram_addr_t length
)
2645 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2646 offset
= addr
- block
->offset
;
2647 if (offset
< block
->length
) {
2648 vaddr
= block
->host
+ offset
;
2649 if (block
->flags
& RAM_PREALLOC_MASK
) {
2653 munmap(vaddr
, length
);
2655 #if defined(__linux__) && !defined(TARGET_S390X)
2658 flags
|= mem_prealloc
? MAP_POPULATE
| MAP_SHARED
:
2661 flags
|= MAP_PRIVATE
;
2663 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
2664 flags
, block
->fd
, offset
);
2666 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
2667 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
2674 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2675 flags
|= MAP_SHARED
| MAP_ANONYMOUS
;
2676 area
= mmap(vaddr
, length
, PROT_EXEC
|PROT_READ
|PROT_WRITE
,
2679 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
2680 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
2684 if (area
!= vaddr
) {
2685 fprintf(stderr
, "Could not remap addr: "
2686 RAM_ADDR_FMT
"@" RAM_ADDR_FMT
"\n",
2690 qemu_madvise(vaddr
, length
, QEMU_MADV_MERGEABLE
);
2691 qemu_ram_setup_dump(vaddr
, length
);
2697 #endif /* !_WIN32 */
2699 /* Return a host pointer to ram allocated with qemu_ram_alloc.
2700 With the exception of the softmmu code in this file, this should
2701 only be used for local memory (e.g. video ram) that the device owns,
2702 and knows it isn't going to access beyond the end of the block.
2704 It should not be used for general purpose DMA.
2705 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2707 void *qemu_get_ram_ptr(ram_addr_t addr
)
2711 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2712 if (addr
- block
->offset
< block
->length
) {
2713 /* Move this entry to to start of the list. */
2714 if (block
!= QLIST_FIRST(&ram_list
.blocks
)) {
2715 QLIST_REMOVE(block
, next
);
2716 QLIST_INSERT_HEAD(&ram_list
.blocks
, block
, next
);
2718 if (xen_enabled()) {
2719 /* We need to check if the requested address is in the RAM
2720 * because we don't want to map the entire memory in QEMU.
2721 * In that case just map until the end of the page.
2723 if (block
->offset
== 0) {
2724 return xen_map_cache(addr
, 0, 0);
2725 } else if (block
->host
== NULL
) {
2727 xen_map_cache(block
->offset
, block
->length
, 1);
2730 return block
->host
+ (addr
- block
->offset
);
2734 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
2740 /* Return a host pointer to ram allocated with qemu_ram_alloc.
2741 * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
2743 void *qemu_safe_ram_ptr(ram_addr_t addr
)
2747 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2748 if (addr
- block
->offset
< block
->length
) {
2749 if (xen_enabled()) {
2750 /* We need to check if the requested address is in the RAM
2751 * because we don't want to map the entire memory in QEMU.
2752 * In that case just map until the end of the page.
2754 if (block
->offset
== 0) {
2755 return xen_map_cache(addr
, 0, 0);
2756 } else if (block
->host
== NULL
) {
2758 xen_map_cache(block
->offset
, block
->length
, 1);
2761 return block
->host
+ (addr
- block
->offset
);
2765 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
2771 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
2772 * but takes a size argument */
2773 void *qemu_ram_ptr_length(ram_addr_t addr
, ram_addr_t
*size
)
2778 if (xen_enabled()) {
2779 return xen_map_cache(addr
, *size
, 1);
2783 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2784 if (addr
- block
->offset
< block
->length
) {
2785 if (addr
- block
->offset
+ *size
> block
->length
)
2786 *size
= block
->length
- addr
+ block
->offset
;
2787 return block
->host
+ (addr
- block
->offset
);
2791 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
2796 void qemu_put_ram_ptr(void *addr
)
2798 trace_qemu_put_ram_ptr(addr
);
2801 int qemu_ram_addr_from_host(void *ptr
, ram_addr_t
*ram_addr
)
2804 uint8_t *host
= ptr
;
2806 if (xen_enabled()) {
2807 *ram_addr
= xen_ram_addr_from_mapcache(ptr
);
2811 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2812 /* This case append when the block is not mapped. */
2813 if (block
->host
== NULL
) {
2816 if (host
- block
->host
< block
->length
) {
2817 *ram_addr
= block
->offset
+ (host
- block
->host
);
2825 /* Some of the softmmu routines need to translate from a host pointer
2826 (typically a TLB entry) back to a ram offset. */
2827 ram_addr_t
qemu_ram_addr_from_host_nofail(void *ptr
)
2829 ram_addr_t ram_addr
;
2831 if (qemu_ram_addr_from_host(ptr
, &ram_addr
)) {
2832 fprintf(stderr
, "Bad ram pointer %p\n", ptr
);
2838 static uint64_t unassigned_mem_read(void *opaque
, target_phys_addr_t addr
,
2841 #ifdef DEBUG_UNASSIGNED
2842 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2844 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2845 cpu_unassigned_access(cpu_single_env
, addr
, 0, 0, 0, size
);
2850 static void unassigned_mem_write(void *opaque
, target_phys_addr_t addr
,
2851 uint64_t val
, unsigned size
)
2853 #ifdef DEBUG_UNASSIGNED
2854 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%"PRIx64
"\n", addr
, val
);
2856 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2857 cpu_unassigned_access(cpu_single_env
, addr
, 1, 0, 0, size
);
2861 static const MemoryRegionOps unassigned_mem_ops
= {
2862 .read
= unassigned_mem_read
,
2863 .write
= unassigned_mem_write
,
2864 .endianness
= DEVICE_NATIVE_ENDIAN
,
2867 static uint64_t error_mem_read(void *opaque
, target_phys_addr_t addr
,
2873 static void error_mem_write(void *opaque
, target_phys_addr_t addr
,
2874 uint64_t value
, unsigned size
)
2879 static const MemoryRegionOps error_mem_ops
= {
2880 .read
= error_mem_read
,
2881 .write
= error_mem_write
,
2882 .endianness
= DEVICE_NATIVE_ENDIAN
,
2885 static const MemoryRegionOps rom_mem_ops
= {
2886 .read
= error_mem_read
,
2887 .write
= unassigned_mem_write
,
2888 .endianness
= DEVICE_NATIVE_ENDIAN
,
2891 static void notdirty_mem_write(void *opaque
, target_phys_addr_t ram_addr
,
2892 uint64_t val
, unsigned size
)
2895 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
2896 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2897 #if !defined(CONFIG_USER_ONLY)
2898 tb_invalidate_phys_page_fast(ram_addr
, size
);
2899 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
2904 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
2907 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
2910 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
2915 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2916 cpu_physical_memory_set_dirty_flags(ram_addr
, dirty_flags
);
2917 /* we remove the notdirty callback only if the code has been
2919 if (dirty_flags
== 0xff)
2920 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2923 static const MemoryRegionOps notdirty_mem_ops
= {
2924 .read
= error_mem_read
,
2925 .write
= notdirty_mem_write
,
2926 .endianness
= DEVICE_NATIVE_ENDIAN
,
2929 /* Generate a debug exception if a watchpoint has been hit. */
2930 static void check_watchpoint(int offset
, int len_mask
, int flags
)
2932 CPUArchState
*env
= cpu_single_env
;
2933 target_ulong pc
, cs_base
;
2934 TranslationBlock
*tb
;
2939 if (env
->watchpoint_hit
) {
2940 /* We re-entered the check after replacing the TB. Now raise
2941 * the debug interrupt so that is will trigger after the
2942 * current instruction. */
2943 cpu_interrupt(env
, CPU_INTERRUPT_DEBUG
);
2946 vaddr
= (env
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
2947 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
2948 if ((vaddr
== (wp
->vaddr
& len_mask
) ||
2949 (vaddr
& wp
->len_mask
) == wp
->vaddr
) && (wp
->flags
& flags
)) {
2950 wp
->flags
|= BP_WATCHPOINT_HIT
;
2951 if (!env
->watchpoint_hit
) {
2952 env
->watchpoint_hit
= wp
;
2953 tb
= tb_find_pc(env
->mem_io_pc
);
2955 cpu_abort(env
, "check_watchpoint: could not find TB for "
2956 "pc=%p", (void *)env
->mem_io_pc
);
2958 cpu_restore_state(tb
, env
, env
->mem_io_pc
);
2959 tb_phys_invalidate(tb
, -1);
2960 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
2961 env
->exception_index
= EXCP_DEBUG
;
2964 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
2965 tb_gen_code(env
, pc
, cs_base
, cpu_flags
, 1);
2966 cpu_resume_from_signal(env
, NULL
);
2970 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
2975 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2976 so these check for a hit then pass through to the normal out-of-line
2978 static uint64_t watch_mem_read(void *opaque
, target_phys_addr_t addr
,
2981 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~(size
- 1), BP_MEM_READ
);
2983 case 1: return ldub_phys(addr
);
2984 case 2: return lduw_phys(addr
);
2985 case 4: return ldl_phys(addr
);
2990 static void watch_mem_write(void *opaque
, target_phys_addr_t addr
,
2991 uint64_t val
, unsigned size
)
2993 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~(size
- 1), BP_MEM_WRITE
);
2996 stb_phys(addr
, val
);
2999 stw_phys(addr
, val
);
3002 stl_phys(addr
, val
);
3008 static const MemoryRegionOps watch_mem_ops
= {
3009 .read
= watch_mem_read
,
3010 .write
= watch_mem_write
,
3011 .endianness
= DEVICE_NATIVE_ENDIAN
,
3014 static uint64_t subpage_read(void *opaque
, target_phys_addr_t addr
,
3017 subpage_t
*mmio
= opaque
;
3018 unsigned int idx
= SUBPAGE_IDX(addr
);
3019 MemoryRegionSection
*section
;
3020 #if defined(DEBUG_SUBPAGE)
3021 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d\n", __func__
,
3022 mmio
, len
, addr
, idx
);
3025 section
= &phys_sections
[mmio
->sub_section
[idx
]];
3027 addr
-= section
->offset_within_address_space
;
3028 addr
+= section
->offset_within_region
;
3029 return io_mem_read(section
->mr
, addr
, len
);
3032 static void subpage_write(void *opaque
, target_phys_addr_t addr
,
3033 uint64_t value
, unsigned len
)
3035 subpage_t
*mmio
= opaque
;
3036 unsigned int idx
= SUBPAGE_IDX(addr
);
3037 MemoryRegionSection
*section
;
3038 #if defined(DEBUG_SUBPAGE)
3039 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
3040 " idx %d value %"PRIx64
"\n",
3041 __func__
, mmio
, len
, addr
, idx
, value
);
3044 section
= &phys_sections
[mmio
->sub_section
[idx
]];
3046 addr
-= section
->offset_within_address_space
;
3047 addr
+= section
->offset_within_region
;
3048 io_mem_write(section
->mr
, addr
, value
, len
);
3051 static const MemoryRegionOps subpage_ops
= {
3052 .read
= subpage_read
,
3053 .write
= subpage_write
,
3054 .endianness
= DEVICE_NATIVE_ENDIAN
,
3057 static uint64_t subpage_ram_read(void *opaque
, target_phys_addr_t addr
,
3060 ram_addr_t raddr
= addr
;
3061 void *ptr
= qemu_get_ram_ptr(raddr
);
3063 case 1: return ldub_p(ptr
);
3064 case 2: return lduw_p(ptr
);
3065 case 4: return ldl_p(ptr
);
3070 static void subpage_ram_write(void *opaque
, target_phys_addr_t addr
,
3071 uint64_t value
, unsigned size
)
3073 ram_addr_t raddr
= addr
;
3074 void *ptr
= qemu_get_ram_ptr(raddr
);
3076 case 1: return stb_p(ptr
, value
);
3077 case 2: return stw_p(ptr
, value
);
3078 case 4: return stl_p(ptr
, value
);
3083 static const MemoryRegionOps subpage_ram_ops
= {
3084 .read
= subpage_ram_read
,
3085 .write
= subpage_ram_write
,
3086 .endianness
= DEVICE_NATIVE_ENDIAN
,
3089 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
3094 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
3096 idx
= SUBPAGE_IDX(start
);
3097 eidx
= SUBPAGE_IDX(end
);
3098 #if defined(DEBUG_SUBPAGE)
3099 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__
,
3100 mmio
, start
, end
, idx
, eidx
, memory
);
3102 if (memory_region_is_ram(phys_sections
[section
].mr
)) {
3103 MemoryRegionSection new_section
= phys_sections
[section
];
3104 new_section
.mr
= &io_mem_subpage_ram
;
3105 section
= phys_section_add(&new_section
);
3107 for (; idx
<= eidx
; idx
++) {
3108 mmio
->sub_section
[idx
] = section
;
3114 static subpage_t
*subpage_init(target_phys_addr_t base
)
3118 mmio
= g_malloc0(sizeof(subpage_t
));
3121 memory_region_init_io(&mmio
->iomem
, &subpage_ops
, mmio
,
3122 "subpage", TARGET_PAGE_SIZE
);
3123 mmio
->iomem
.subpage
= true;
3124 #if defined(DEBUG_SUBPAGE)
3125 printf("%s: %p base " TARGET_FMT_plx
" len %08x %d\n", __func__
,
3126 mmio
, base
, TARGET_PAGE_SIZE
, subpage_memory
);
3128 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
-1, phys_section_unassigned
);
3133 static uint16_t dummy_section(MemoryRegion
*mr
)
3135 MemoryRegionSection section
= {
3137 .offset_within_address_space
= 0,
3138 .offset_within_region
= 0,
3142 return phys_section_add(§ion
);
3145 MemoryRegion
*iotlb_to_region(target_phys_addr_t index
)
3147 return phys_sections
[index
& ~TARGET_PAGE_MASK
].mr
;
3150 static void io_mem_init(void)
3152 memory_region_init_io(&io_mem_ram
, &error_mem_ops
, NULL
, "ram", UINT64_MAX
);
3153 memory_region_init_io(&io_mem_rom
, &rom_mem_ops
, NULL
, "rom", UINT64_MAX
);
3154 memory_region_init_io(&io_mem_unassigned
, &unassigned_mem_ops
, NULL
,
3155 "unassigned", UINT64_MAX
);
3156 memory_region_init_io(&io_mem_notdirty
, ¬dirty_mem_ops
, NULL
,
3157 "notdirty", UINT64_MAX
);
3158 memory_region_init_io(&io_mem_subpage_ram
, &subpage_ram_ops
, NULL
,
3159 "subpage-ram", UINT64_MAX
);
3160 memory_region_init_io(&io_mem_watch
, &watch_mem_ops
, NULL
,
3161 "watch", UINT64_MAX
);
3164 static void core_begin(MemoryListener
*listener
)
3166 destroy_all_mappings();
3167 phys_sections_clear();
3168 phys_map
.ptr
= PHYS_MAP_NODE_NIL
;
3169 phys_section_unassigned
= dummy_section(&io_mem_unassigned
);
3170 phys_section_notdirty
= dummy_section(&io_mem_notdirty
);
3171 phys_section_rom
= dummy_section(&io_mem_rom
);
3172 phys_section_watch
= dummy_section(&io_mem_watch
);
3175 static void core_commit(MemoryListener
*listener
)
3179 /* since each CPU stores ram addresses in its TLB cache, we must
3180 reset the modified entries */
3182 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
3187 static void core_region_add(MemoryListener
*listener
,
3188 MemoryRegionSection
*section
)
3190 cpu_register_physical_memory_log(section
, section
->readonly
);
3193 static void core_region_del(MemoryListener
*listener
,
3194 MemoryRegionSection
*section
)
3198 static void core_region_nop(MemoryListener
*listener
,
3199 MemoryRegionSection
*section
)
3201 cpu_register_physical_memory_log(section
, section
->readonly
);
3204 static void core_log_start(MemoryListener
*listener
,
3205 MemoryRegionSection
*section
)
3209 static void core_log_stop(MemoryListener
*listener
,
3210 MemoryRegionSection
*section
)
3214 static void core_log_sync(MemoryListener
*listener
,
3215 MemoryRegionSection
*section
)
3219 static void core_log_global_start(MemoryListener
*listener
)
3221 cpu_physical_memory_set_dirty_tracking(1);
3224 static void core_log_global_stop(MemoryListener
*listener
)
3226 cpu_physical_memory_set_dirty_tracking(0);
3229 static void core_eventfd_add(MemoryListener
*listener
,
3230 MemoryRegionSection
*section
,
3231 bool match_data
, uint64_t data
, EventNotifier
*e
)
3235 static void core_eventfd_del(MemoryListener
*listener
,
3236 MemoryRegionSection
*section
,
3237 bool match_data
, uint64_t data
, EventNotifier
*e
)
3241 static void io_begin(MemoryListener
*listener
)
3245 static void io_commit(MemoryListener
*listener
)
3249 static void io_region_add(MemoryListener
*listener
,
3250 MemoryRegionSection
*section
)
3252 MemoryRegionIORange
*mrio
= g_new(MemoryRegionIORange
, 1);
3254 mrio
->mr
= section
->mr
;
3255 mrio
->offset
= section
->offset_within_region
;
3256 iorange_init(&mrio
->iorange
, &memory_region_iorange_ops
,
3257 section
->offset_within_address_space
, section
->size
);
3258 ioport_register(&mrio
->iorange
);
3261 static void io_region_del(MemoryListener
*listener
,
3262 MemoryRegionSection
*section
)
3264 isa_unassign_ioport(section
->offset_within_address_space
, section
->size
);
3267 static void io_region_nop(MemoryListener
*listener
,
3268 MemoryRegionSection
*section
)
3272 static void io_log_start(MemoryListener
*listener
,
3273 MemoryRegionSection
*section
)
3277 static void io_log_stop(MemoryListener
*listener
,
3278 MemoryRegionSection
*section
)
3282 static void io_log_sync(MemoryListener
*listener
,
3283 MemoryRegionSection
*section
)
3287 static void io_log_global_start(MemoryListener
*listener
)
3291 static void io_log_global_stop(MemoryListener
*listener
)
3295 static void io_eventfd_add(MemoryListener
*listener
,
3296 MemoryRegionSection
*section
,
3297 bool match_data
, uint64_t data
, EventNotifier
*e
)
3301 static void io_eventfd_del(MemoryListener
*listener
,
3302 MemoryRegionSection
*section
,
3303 bool match_data
, uint64_t data
, EventNotifier
*e
)
3307 static MemoryListener core_memory_listener
= {
3308 .begin
= core_begin
,
3309 .commit
= core_commit
,
3310 .region_add
= core_region_add
,
3311 .region_del
= core_region_del
,
3312 .region_nop
= core_region_nop
,
3313 .log_start
= core_log_start
,
3314 .log_stop
= core_log_stop
,
3315 .log_sync
= core_log_sync
,
3316 .log_global_start
= core_log_global_start
,
3317 .log_global_stop
= core_log_global_stop
,
3318 .eventfd_add
= core_eventfd_add
,
3319 .eventfd_del
= core_eventfd_del
,
3323 static MemoryListener io_memory_listener
= {
3325 .commit
= io_commit
,
3326 .region_add
= io_region_add
,
3327 .region_del
= io_region_del
,
3328 .region_nop
= io_region_nop
,
3329 .log_start
= io_log_start
,
3330 .log_stop
= io_log_stop
,
3331 .log_sync
= io_log_sync
,
3332 .log_global_start
= io_log_global_start
,
3333 .log_global_stop
= io_log_global_stop
,
3334 .eventfd_add
= io_eventfd_add
,
3335 .eventfd_del
= io_eventfd_del
,
3339 static void memory_map_init(void)
3341 system_memory
= g_malloc(sizeof(*system_memory
));
3342 memory_region_init(system_memory
, "system", INT64_MAX
);
3343 set_system_memory_map(system_memory
);
3345 system_io
= g_malloc(sizeof(*system_io
));
3346 memory_region_init(system_io
, "io", 65536);
3347 set_system_io_map(system_io
);
3349 memory_listener_register(&core_memory_listener
, system_memory
);
3350 memory_listener_register(&io_memory_listener
, system_io
);
3353 MemoryRegion
*get_system_memory(void)
3355 return system_memory
;
3358 MemoryRegion
*get_system_io(void)
3363 #endif /* !defined(CONFIG_USER_ONLY) */
3365 /* physical memory access (slow version, mainly for debug) */
3366 #if defined(CONFIG_USER_ONLY)
3367 int cpu_memory_rw_debug(CPUArchState
*env
, target_ulong addr
,
3368 uint8_t *buf
, int len
, int is_write
)
3375 page
= addr
& TARGET_PAGE_MASK
;
3376 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3379 flags
= page_get_flags(page
);
3380 if (!(flags
& PAGE_VALID
))
3383 if (!(flags
& PAGE_WRITE
))
3385 /* XXX: this code should not depend on lock_user */
3386 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
3389 unlock_user(p
, addr
, l
);
3391 if (!(flags
& PAGE_READ
))
3393 /* XXX: this code should not depend on lock_user */
3394 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
3397 unlock_user(p
, addr
, 0);
3407 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
3408 int len
, int is_write
)
3413 target_phys_addr_t page
;
3414 MemoryRegionSection
*section
;
3417 page
= addr
& TARGET_PAGE_MASK
;
3418 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3421 section
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3424 if (!memory_region_is_ram(section
->mr
)) {
3425 target_phys_addr_t addr1
;
3426 addr1
= memory_region_section_addr(section
, addr
);
3427 /* XXX: could force cpu_single_env to NULL to avoid
3429 if (l
>= 4 && ((addr1
& 3) == 0)) {
3430 /* 32 bit write access */
3432 io_mem_write(section
->mr
, addr1
, val
, 4);
3434 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3435 /* 16 bit write access */
3437 io_mem_write(section
->mr
, addr1
, val
, 2);
3440 /* 8 bit write access */
3442 io_mem_write(section
->mr
, addr1
, val
, 1);
3445 } else if (!section
->readonly
) {
3447 addr1
= memory_region_get_ram_addr(section
->mr
)
3448 + memory_region_section_addr(section
, addr
);
3450 ptr
= qemu_get_ram_ptr(addr1
);
3451 memcpy(ptr
, buf
, l
);
3452 if (!cpu_physical_memory_is_dirty(addr1
)) {
3453 /* invalidate code */
3454 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3456 cpu_physical_memory_set_dirty_flags(
3457 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
3459 qemu_put_ram_ptr(ptr
);
3462 if (!(memory_region_is_ram(section
->mr
) ||
3463 memory_region_is_romd(section
->mr
))) {
3464 target_phys_addr_t addr1
;
3466 addr1
= memory_region_section_addr(section
, addr
);
3467 if (l
>= 4 && ((addr1
& 3) == 0)) {
3468 /* 32 bit read access */
3469 val
= io_mem_read(section
->mr
, addr1
, 4);
3472 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3473 /* 16 bit read access */
3474 val
= io_mem_read(section
->mr
, addr1
, 2);
3478 /* 8 bit read access */
3479 val
= io_mem_read(section
->mr
, addr1
, 1);
3485 ptr
= qemu_get_ram_ptr(section
->mr
->ram_addr
3486 + memory_region_section_addr(section
,
3488 memcpy(buf
, ptr
, l
);
3489 qemu_put_ram_ptr(ptr
);
3498 /* used for ROM loading : can write in RAM and ROM */
3499 void cpu_physical_memory_write_rom(target_phys_addr_t addr
,
3500 const uint8_t *buf
, int len
)
3504 target_phys_addr_t page
;
3505 MemoryRegionSection
*section
;
3508 page
= addr
& TARGET_PAGE_MASK
;
3509 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3512 section
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3514 if (!(memory_region_is_ram(section
->mr
) ||
3515 memory_region_is_romd(section
->mr
))) {
3518 unsigned long addr1
;
3519 addr1
= memory_region_get_ram_addr(section
->mr
)
3520 + memory_region_section_addr(section
, addr
);
3522 ptr
= qemu_get_ram_ptr(addr1
);
3523 memcpy(ptr
, buf
, l
);
3524 if (!cpu_physical_memory_is_dirty(addr1
)) {
3525 /* invalidate code */
3526 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3528 cpu_physical_memory_set_dirty_flags(
3529 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
3531 qemu_put_ram_ptr(ptr
);
3541 target_phys_addr_t addr
;
3542 target_phys_addr_t len
;
3545 static BounceBuffer bounce
;
3547 typedef struct MapClient
{
3549 void (*callback
)(void *opaque
);
3550 QLIST_ENTRY(MapClient
) link
;
3553 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
3554 = QLIST_HEAD_INITIALIZER(map_client_list
);
3556 void *cpu_register_map_client(void *opaque
, void (*callback
)(void *opaque
))
3558 MapClient
*client
= g_malloc(sizeof(*client
));
3560 client
->opaque
= opaque
;
3561 client
->callback
= callback
;
3562 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
3566 void cpu_unregister_map_client(void *_client
)
3568 MapClient
*client
= (MapClient
*)_client
;
3570 QLIST_REMOVE(client
, link
);
3574 static void cpu_notify_map_clients(void)
3578 while (!QLIST_EMPTY(&map_client_list
)) {
3579 client
= QLIST_FIRST(&map_client_list
);
3580 client
->callback(client
->opaque
);
3581 cpu_unregister_map_client(client
);
3585 /* Map a physical memory region into a host virtual address.
3586 * May map a subset of the requested range, given by and returned in *plen.
3587 * May return NULL if resources needed to perform the mapping are exhausted.
3588 * Use only for reads OR writes - not for read-modify-write operations.
3589 * Use cpu_register_map_client() to know when retrying the map operation is
3590 * likely to succeed.
3592 void *cpu_physical_memory_map(target_phys_addr_t addr
,
3593 target_phys_addr_t
*plen
,
3596 target_phys_addr_t len
= *plen
;
3597 target_phys_addr_t todo
= 0;
3599 target_phys_addr_t page
;
3600 MemoryRegionSection
*section
;
3601 ram_addr_t raddr
= RAM_ADDR_MAX
;
3606 page
= addr
& TARGET_PAGE_MASK
;
3607 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3610 section
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3612 if (!(memory_region_is_ram(section
->mr
) && !section
->readonly
)) {
3613 if (todo
|| bounce
.buffer
) {
3616 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, TARGET_PAGE_SIZE
);
3620 cpu_physical_memory_read(addr
, bounce
.buffer
, l
);
3624 return bounce
.buffer
;
3627 raddr
= memory_region_get_ram_addr(section
->mr
)
3628 + memory_region_section_addr(section
, addr
);
3636 ret
= qemu_ram_ptr_length(raddr
, &rlen
);
3641 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3642 * Will also mark the memory as dirty if is_write == 1. access_len gives
3643 * the amount of memory that was actually read or written by the caller.
3645 void cpu_physical_memory_unmap(void *buffer
, target_phys_addr_t len
,
3646 int is_write
, target_phys_addr_t access_len
)
3648 if (buffer
!= bounce
.buffer
) {
3650 ram_addr_t addr1
= qemu_ram_addr_from_host_nofail(buffer
);
3651 while (access_len
) {
3653 l
= TARGET_PAGE_SIZE
;
3656 if (!cpu_physical_memory_is_dirty(addr1
)) {
3657 /* invalidate code */
3658 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3660 cpu_physical_memory_set_dirty_flags(
3661 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
3667 if (xen_enabled()) {
3668 xen_invalidate_map_cache_entry(buffer
);
3673 cpu_physical_memory_write(bounce
.addr
, bounce
.buffer
, access_len
);
3675 qemu_vfree(bounce
.buffer
);
3676 bounce
.buffer
= NULL
;
3677 cpu_notify_map_clients();
3680 /* warning: addr must be aligned */
3681 static inline uint32_t ldl_phys_internal(target_phys_addr_t addr
,
3682 enum device_endian endian
)
3686 MemoryRegionSection
*section
;
3688 section
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3690 if (!(memory_region_is_ram(section
->mr
) ||
3691 memory_region_is_romd(section
->mr
))) {
3693 addr
= memory_region_section_addr(section
, addr
);
3694 val
= io_mem_read(section
->mr
, addr
, 4);
3695 #if defined(TARGET_WORDS_BIGENDIAN)
3696 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3700 if (endian
== DEVICE_BIG_ENDIAN
) {
3706 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(section
->mr
)
3708 + memory_region_section_addr(section
, addr
));
3710 case DEVICE_LITTLE_ENDIAN
:
3711 val
= ldl_le_p(ptr
);
3713 case DEVICE_BIG_ENDIAN
:
3714 val
= ldl_be_p(ptr
);
3724 uint32_t ldl_phys(target_phys_addr_t addr
)
3726 return ldl_phys_internal(addr
, DEVICE_NATIVE_ENDIAN
);
3729 uint32_t ldl_le_phys(target_phys_addr_t addr
)
3731 return ldl_phys_internal(addr
, DEVICE_LITTLE_ENDIAN
);
3734 uint32_t ldl_be_phys(target_phys_addr_t addr
)
3736 return ldl_phys_internal(addr
, DEVICE_BIG_ENDIAN
);
3739 /* warning: addr must be aligned */
3740 static inline uint64_t ldq_phys_internal(target_phys_addr_t addr
,
3741 enum device_endian endian
)
3745 MemoryRegionSection
*section
;
3747 section
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3749 if (!(memory_region_is_ram(section
->mr
) ||
3750 memory_region_is_romd(section
->mr
))) {
3752 addr
= memory_region_section_addr(section
, addr
);
3754 /* XXX This is broken when device endian != cpu endian.
3755 Fix and add "endian" variable check */
3756 #ifdef TARGET_WORDS_BIGENDIAN
3757 val
= io_mem_read(section
->mr
, addr
, 4) << 32;
3758 val
|= io_mem_read(section
->mr
, addr
+ 4, 4);
3760 val
= io_mem_read(section
->mr
, addr
, 4);
3761 val
|= io_mem_read(section
->mr
, addr
+ 4, 4) << 32;
3765 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(section
->mr
)
3767 + memory_region_section_addr(section
, addr
));
3769 case DEVICE_LITTLE_ENDIAN
:
3770 val
= ldq_le_p(ptr
);
3772 case DEVICE_BIG_ENDIAN
:
3773 val
= ldq_be_p(ptr
);
3783 uint64_t ldq_phys(target_phys_addr_t addr
)
3785 return ldq_phys_internal(addr
, DEVICE_NATIVE_ENDIAN
);
3788 uint64_t ldq_le_phys(target_phys_addr_t addr
)
3790 return ldq_phys_internal(addr
, DEVICE_LITTLE_ENDIAN
);
3793 uint64_t ldq_be_phys(target_phys_addr_t addr
)
3795 return ldq_phys_internal(addr
, DEVICE_BIG_ENDIAN
);
3799 uint32_t ldub_phys(target_phys_addr_t addr
)
3802 cpu_physical_memory_read(addr
, &val
, 1);
3806 /* warning: addr must be aligned */
3807 static inline uint32_t lduw_phys_internal(target_phys_addr_t addr
,
3808 enum device_endian endian
)
3812 MemoryRegionSection
*section
;
3814 section
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3816 if (!(memory_region_is_ram(section
->mr
) ||
3817 memory_region_is_romd(section
->mr
))) {
3819 addr
= memory_region_section_addr(section
, addr
);
3820 val
= io_mem_read(section
->mr
, addr
, 2);
3821 #if defined(TARGET_WORDS_BIGENDIAN)
3822 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3826 if (endian
== DEVICE_BIG_ENDIAN
) {
3832 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(section
->mr
)
3834 + memory_region_section_addr(section
, addr
));
3836 case DEVICE_LITTLE_ENDIAN
:
3837 val
= lduw_le_p(ptr
);
3839 case DEVICE_BIG_ENDIAN
:
3840 val
= lduw_be_p(ptr
);
3850 uint32_t lduw_phys(target_phys_addr_t addr
)
3852 return lduw_phys_internal(addr
, DEVICE_NATIVE_ENDIAN
);
3855 uint32_t lduw_le_phys(target_phys_addr_t addr
)
3857 return lduw_phys_internal(addr
, DEVICE_LITTLE_ENDIAN
);
3860 uint32_t lduw_be_phys(target_phys_addr_t addr
)
3862 return lduw_phys_internal(addr
, DEVICE_BIG_ENDIAN
);
3865 /* warning: addr must be aligned. The ram page is not masked as dirty
3866 and the code inside is not invalidated. It is useful if the dirty
3867 bits are used to track modified PTEs */
3868 void stl_phys_notdirty(target_phys_addr_t addr
, uint32_t val
)
3871 MemoryRegionSection
*section
;
3873 section
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3875 if (!memory_region_is_ram(section
->mr
) || section
->readonly
) {
3876 addr
= memory_region_section_addr(section
, addr
);
3877 if (memory_region_is_ram(section
->mr
)) {
3878 section
= &phys_sections
[phys_section_rom
];
3880 io_mem_write(section
->mr
, addr
, val
, 4);
3882 unsigned long addr1
= (memory_region_get_ram_addr(section
->mr
)
3884 + memory_region_section_addr(section
, addr
);
3885 ptr
= qemu_get_ram_ptr(addr1
);
3888 if (unlikely(in_migration
)) {
3889 if (!cpu_physical_memory_is_dirty(addr1
)) {
3890 /* invalidate code */
3891 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3893 cpu_physical_memory_set_dirty_flags(
3894 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
3900 void stq_phys_notdirty(target_phys_addr_t addr
, uint64_t val
)
3903 MemoryRegionSection
*section
;
3905 section
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3907 if (!memory_region_is_ram(section
->mr
) || section
->readonly
) {
3908 addr
= memory_region_section_addr(section
, addr
);
3909 if (memory_region_is_ram(section
->mr
)) {
3910 section
= &phys_sections
[phys_section_rom
];
3912 #ifdef TARGET_WORDS_BIGENDIAN
3913 io_mem_write(section
->mr
, addr
, val
>> 32, 4);
3914 io_mem_write(section
->mr
, addr
+ 4, (uint32_t)val
, 4);
3916 io_mem_write(section
->mr
, addr
, (uint32_t)val
, 4);
3917 io_mem_write(section
->mr
, addr
+ 4, val
>> 32, 4);
3920 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(section
->mr
)
3922 + memory_region_section_addr(section
, addr
));
3927 /* warning: addr must be aligned */
3928 static inline void stl_phys_internal(target_phys_addr_t addr
, uint32_t val
,
3929 enum device_endian endian
)
3932 MemoryRegionSection
*section
;
3934 section
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3936 if (!memory_region_is_ram(section
->mr
) || section
->readonly
) {
3937 addr
= memory_region_section_addr(section
, addr
);
3938 if (memory_region_is_ram(section
->mr
)) {
3939 section
= &phys_sections
[phys_section_rom
];
3941 #if defined(TARGET_WORDS_BIGENDIAN)
3942 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3946 if (endian
== DEVICE_BIG_ENDIAN
) {
3950 io_mem_write(section
->mr
, addr
, val
, 4);
3952 unsigned long addr1
;
3953 addr1
= (memory_region_get_ram_addr(section
->mr
) & TARGET_PAGE_MASK
)
3954 + memory_region_section_addr(section
, addr
);
3956 ptr
= qemu_get_ram_ptr(addr1
);
3958 case DEVICE_LITTLE_ENDIAN
:
3961 case DEVICE_BIG_ENDIAN
:
3968 if (!cpu_physical_memory_is_dirty(addr1
)) {
3969 /* invalidate code */
3970 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3972 cpu_physical_memory_set_dirty_flags(addr1
,
3973 (0xff & ~CODE_DIRTY_FLAG
));
3978 void stl_phys(target_phys_addr_t addr
, uint32_t val
)
3980 stl_phys_internal(addr
, val
, DEVICE_NATIVE_ENDIAN
);
3983 void stl_le_phys(target_phys_addr_t addr
, uint32_t val
)
3985 stl_phys_internal(addr
, val
, DEVICE_LITTLE_ENDIAN
);
3988 void stl_be_phys(target_phys_addr_t addr
, uint32_t val
)
3990 stl_phys_internal(addr
, val
, DEVICE_BIG_ENDIAN
);
3994 void stb_phys(target_phys_addr_t addr
, uint32_t val
)
3997 cpu_physical_memory_write(addr
, &v
, 1);
4000 /* warning: addr must be aligned */
4001 static inline void stw_phys_internal(target_phys_addr_t addr
, uint32_t val
,
4002 enum device_endian endian
)
4005 MemoryRegionSection
*section
;
4007 section
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4009 if (!memory_region_is_ram(section
->mr
) || section
->readonly
) {
4010 addr
= memory_region_section_addr(section
, addr
);
4011 if (memory_region_is_ram(section
->mr
)) {
4012 section
= &phys_sections
[phys_section_rom
];
4014 #if defined(TARGET_WORDS_BIGENDIAN)
4015 if (endian
== DEVICE_LITTLE_ENDIAN
) {
4019 if (endian
== DEVICE_BIG_ENDIAN
) {
4023 io_mem_write(section
->mr
, addr
, val
, 2);
4025 unsigned long addr1
;
4026 addr1
= (memory_region_get_ram_addr(section
->mr
) & TARGET_PAGE_MASK
)
4027 + memory_region_section_addr(section
, addr
);
4029 ptr
= qemu_get_ram_ptr(addr1
);
4031 case DEVICE_LITTLE_ENDIAN
:
4034 case DEVICE_BIG_ENDIAN
:
4041 if (!cpu_physical_memory_is_dirty(addr1
)) {
4042 /* invalidate code */
4043 tb_invalidate_phys_page_range(addr1
, addr1
+ 2, 0);
4045 cpu_physical_memory_set_dirty_flags(addr1
,
4046 (0xff & ~CODE_DIRTY_FLAG
));
4051 void stw_phys(target_phys_addr_t addr
, uint32_t val
)
4053 stw_phys_internal(addr
, val
, DEVICE_NATIVE_ENDIAN
);
4056 void stw_le_phys(target_phys_addr_t addr
, uint32_t val
)
4058 stw_phys_internal(addr
, val
, DEVICE_LITTLE_ENDIAN
);
4061 void stw_be_phys(target_phys_addr_t addr
, uint32_t val
)
4063 stw_phys_internal(addr
, val
, DEVICE_BIG_ENDIAN
);
4067 void stq_phys(target_phys_addr_t addr
, uint64_t val
)
4070 cpu_physical_memory_write(addr
, &val
, 8);
4073 void stq_le_phys(target_phys_addr_t addr
, uint64_t val
)
4075 val
= cpu_to_le64(val
);
4076 cpu_physical_memory_write(addr
, &val
, 8);
4079 void stq_be_phys(target_phys_addr_t addr
, uint64_t val
)
4081 val
= cpu_to_be64(val
);
4082 cpu_physical_memory_write(addr
, &val
, 8);
4085 /* virtual memory access for debug (includes writing to ROM) */
4086 int cpu_memory_rw_debug(CPUArchState
*env
, target_ulong addr
,
4087 uint8_t *buf
, int len
, int is_write
)
4090 target_phys_addr_t phys_addr
;
4094 page
= addr
& TARGET_PAGE_MASK
;
4095 phys_addr
= cpu_get_phys_page_debug(env
, page
);
4096 /* if no physical page mapped, return an error */
4097 if (phys_addr
== -1)
4099 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
4102 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
4104 cpu_physical_memory_write_rom(phys_addr
, buf
, l
);
4106 cpu_physical_memory_rw(phys_addr
, buf
, l
, is_write
);
4115 /* in deterministic execution mode, instructions doing device I/Os
4116 must be at the end of the TB */
4117 void cpu_io_recompile(CPUArchState
*env
, uintptr_t retaddr
)
4119 TranslationBlock
*tb
;
4121 target_ulong pc
, cs_base
;
4124 tb
= tb_find_pc(retaddr
);
4126 cpu_abort(env
, "cpu_io_recompile: could not find TB for pc=%p",
4129 n
= env
->icount_decr
.u16
.low
+ tb
->icount
;
4130 cpu_restore_state(tb
, env
, retaddr
);
4131 /* Calculate how many instructions had been executed before the fault
4133 n
= n
- env
->icount_decr
.u16
.low
;
4134 /* Generate a new TB ending on the I/O insn. */
4136 /* On MIPS and SH, delay slot instructions can only be restarted if
4137 they were already the first instruction in the TB. If this is not
4138 the first instruction in a TB then re-execute the preceding
4140 #if defined(TARGET_MIPS)
4141 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
4142 env
->active_tc
.PC
-= 4;
4143 env
->icount_decr
.u16
.low
++;
4144 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
4146 #elif defined(TARGET_SH4)
4147 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
4150 env
->icount_decr
.u16
.low
++;
4151 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
4154 /* This should never happen. */
4155 if (n
> CF_COUNT_MASK
)
4156 cpu_abort(env
, "TB too big during recompile");
4158 cflags
= n
| CF_LAST_IO
;
4160 cs_base
= tb
->cs_base
;
4162 tb_phys_invalidate(tb
, -1);
4163 /* FIXME: In theory this could raise an exception. In practice
4164 we have already translated the block once so it's probably ok. */
4165 tb_gen_code(env
, pc
, cs_base
, flags
, cflags
);
4166 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
4167 the first in the TB) then we end up generating a whole new TB and
4168 repeating the fault, which is horribly inefficient.
4169 Better would be to execute just this insn uncached, or generate a
4171 cpu_resume_from_signal(env
, NULL
);
4174 #if !defined(CONFIG_USER_ONLY)
4176 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
4178 int i
, target_code_size
, max_target_code_size
;
4179 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
4180 TranslationBlock
*tb
;
4182 target_code_size
= 0;
4183 max_target_code_size
= 0;
4185 direct_jmp_count
= 0;
4186 direct_jmp2_count
= 0;
4187 for(i
= 0; i
< nb_tbs
; i
++) {
4189 target_code_size
+= tb
->size
;
4190 if (tb
->size
> max_target_code_size
)
4191 max_target_code_size
= tb
->size
;
4192 if (tb
->page_addr
[1] != -1)
4194 if (tb
->tb_next_offset
[0] != 0xffff) {
4196 if (tb
->tb_next_offset
[1] != 0xffff) {
4197 direct_jmp2_count
++;
4201 /* XXX: avoid using doubles ? */
4202 cpu_fprintf(f
, "Translation buffer state:\n");
4203 cpu_fprintf(f
, "gen code size %td/%ld\n",
4204 code_gen_ptr
- code_gen_buffer
, code_gen_buffer_max_size
);
4205 cpu_fprintf(f
, "TB count %d/%d\n",
4206 nb_tbs
, code_gen_max_blocks
);
4207 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
4208 nb_tbs
? target_code_size
/ nb_tbs
: 0,
4209 max_target_code_size
);
4210 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
4211 nb_tbs
? (code_gen_ptr
- code_gen_buffer
) / nb_tbs
: 0,
4212 target_code_size
? (double) (code_gen_ptr
- code_gen_buffer
) / target_code_size
: 0);
4213 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n",
4215 nb_tbs
? (cross_page
* 100) / nb_tbs
: 0);
4216 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
4218 nb_tbs
? (direct_jmp_count
* 100) / nb_tbs
: 0,
4220 nb_tbs
? (direct_jmp2_count
* 100) / nb_tbs
: 0);
4221 cpu_fprintf(f
, "\nStatistics:\n");
4222 cpu_fprintf(f
, "TB flush count %d\n", tb_flush_count
);
4223 cpu_fprintf(f
, "TB invalidate count %d\n", tb_phys_invalidate_count
);
4224 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
4225 tcg_dump_info(f
, cpu_fprintf
);
4229 * A helper function for the _utterly broken_ virtio device model to find out if
4230 * it's running on a big endian machine. Don't do this at home kids!
4232 bool virtio_is_big_endian(void);
4233 bool virtio_is_big_endian(void)
4235 #if defined(TARGET_WORDS_BIGENDIAN)
4244 #ifndef CONFIG_USER_ONLY
4245 bool cpu_physical_memory_is_io(target_phys_addr_t phys_addr
)
4247 MemoryRegionSection
*section
;
4249 section
= phys_page_find(phys_addr
>> TARGET_PAGE_BITS
);
4251 return !(memory_region_is_ram(section
->mr
) ||
4252 memory_region_is_romd(section
->mr
));