2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
23 #include <sys/types.h>
27 #include "qemu-common.h"
36 #include "qemu-timer.h"
37 #if defined(CONFIG_USER_ONLY)
39 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
40 #include <sys/param.h>
41 #if __FreeBSD_version >= 700104
42 #define HAVE_KINFO_GETVMMAP
43 #define sigqueue sigqueue_freebsd /* avoid redefinition */
46 #include <machine/profile.h>
54 #else /* !CONFIG_USER_ONLY */
55 #include "xen-mapcache.h"
59 //#define DEBUG_TB_INVALIDATE
62 //#define DEBUG_UNASSIGNED
64 /* make various TB consistency checks */
65 //#define DEBUG_TB_CHECK
66 //#define DEBUG_TLB_CHECK
68 //#define DEBUG_IOPORT
69 //#define DEBUG_SUBPAGE
71 #if !defined(CONFIG_USER_ONLY)
72 /* TB consistency checks only implemented for usermode emulation. */
76 #define SMC_BITMAP_USE_THRESHOLD 10
78 static TranslationBlock
*tbs
;
79 static int code_gen_max_blocks
;
80 TranslationBlock
*tb_phys_hash
[CODE_GEN_PHYS_HASH_SIZE
];
82 /* any access to the tbs or the page table must use this lock */
83 spinlock_t tb_lock
= SPIN_LOCK_UNLOCKED
;
85 #if defined(__arm__) || defined(__sparc_v9__)
86 /* The prologue must be reachable with a direct jump. ARM and Sparc64
87 have limited branch ranges (possibly also PPC) so place it in a
88 section close to code segment. */
89 #define code_gen_section \
90 __attribute__((__section__(".gen_code"))) \
91 __attribute__((aligned (32)))
93 /* Maximum alignment for Win32 is 16. */
94 #define code_gen_section \
95 __attribute__((aligned (16)))
97 #define code_gen_section \
98 __attribute__((aligned (32)))
101 uint8_t code_gen_prologue
[1024] code_gen_section
;
102 static uint8_t *code_gen_buffer
;
103 static unsigned long code_gen_buffer_size
;
104 /* threshold to flush the translated code buffer */
105 static unsigned long code_gen_buffer_max_size
;
106 static uint8_t *code_gen_ptr
;
108 #if !defined(CONFIG_USER_ONLY)
110 static int in_migration
;
112 RAMList ram_list
= { .blocks
= QLIST_HEAD_INITIALIZER(ram_list
) };
116 /* current CPU in the current thread. It is only valid inside
118 CPUState
*cpu_single_env
;
119 /* 0 = Do not count executed instructions.
120 1 = Precise instruction counting.
121 2 = Adaptive rate instruction counting. */
123 /* Current instruction counter. While executing translated code this may
124 include some instructions that have not yet been executed. */
127 typedef struct PageDesc
{
128 /* list of TBs intersecting this ram page */
129 TranslationBlock
*first_tb
;
130 /* in order to optimize self modifying code, we count the number
131 of lookups we do to a given page to use a bitmap */
132 unsigned int code_write_count
;
133 uint8_t *code_bitmap
;
134 #if defined(CONFIG_USER_ONLY)
139 /* In system mode we want L1_MAP to be based on ram offsets,
140 while in user mode we want it to be based on virtual addresses. */
141 #if !defined(CONFIG_USER_ONLY)
142 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
143 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
145 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
148 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
151 /* Size of the L2 (and L3, etc) page tables. */
153 #define L2_SIZE (1 << L2_BITS)
155 /* The bits remaining after N lower levels of page tables. */
156 #define P_L1_BITS_REM \
157 ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
158 #define V_L1_BITS_REM \
159 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
161 /* Size of the L1 page table. Avoid silly small sizes. */
162 #if P_L1_BITS_REM < 4
163 #define P_L1_BITS (P_L1_BITS_REM + L2_BITS)
165 #define P_L1_BITS P_L1_BITS_REM
168 #if V_L1_BITS_REM < 4
169 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
171 #define V_L1_BITS V_L1_BITS_REM
174 #define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS)
175 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
177 #define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
178 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
180 unsigned long qemu_real_host_page_size
;
181 unsigned long qemu_host_page_bits
;
182 unsigned long qemu_host_page_size
;
183 unsigned long qemu_host_page_mask
;
185 /* This is a multi-level map on the virtual address space.
186 The bottom level has pointers to PageDesc. */
187 static void *l1_map
[V_L1_SIZE
];
189 #if !defined(CONFIG_USER_ONLY)
190 typedef struct PhysPageDesc
{
191 /* offset in host memory of the page + io_index in the low bits */
192 ram_addr_t phys_offset
;
193 ram_addr_t region_offset
;
196 /* This is a multi-level map on the physical address space.
197 The bottom level has pointers to PhysPageDesc. */
198 static void *l1_phys_map
[P_L1_SIZE
];
200 static void io_mem_init(void);
202 /* io memory support */
203 CPUWriteMemoryFunc
*io_mem_write
[IO_MEM_NB_ENTRIES
][4];
204 CPUReadMemoryFunc
*io_mem_read
[IO_MEM_NB_ENTRIES
][4];
205 void *io_mem_opaque
[IO_MEM_NB_ENTRIES
];
206 static char io_mem_used
[IO_MEM_NB_ENTRIES
];
207 static int io_mem_watch
;
212 static const char *logfilename
= "qemu.log";
214 static const char *logfilename
= "/tmp/qemu.log";
218 static int log_append
= 0;
221 #if !defined(CONFIG_USER_ONLY)
222 static int tlb_flush_count
;
224 static int tb_flush_count
;
225 static int tb_phys_invalidate_count
;
228 static void map_exec(void *addr
, long size
)
231 VirtualProtect(addr
, size
,
232 PAGE_EXECUTE_READWRITE
, &old_protect
);
236 static void map_exec(void *addr
, long size
)
238 unsigned long start
, end
, page_size
;
240 page_size
= getpagesize();
241 start
= (unsigned long)addr
;
242 start
&= ~(page_size
- 1);
244 end
= (unsigned long)addr
+ size
;
245 end
+= page_size
- 1;
246 end
&= ~(page_size
- 1);
248 mprotect((void *)start
, end
- start
,
249 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
253 static void page_init(void)
255 /* NOTE: we can always suppose that qemu_host_page_size >=
259 SYSTEM_INFO system_info
;
261 GetSystemInfo(&system_info
);
262 qemu_real_host_page_size
= system_info
.dwPageSize
;
265 qemu_real_host_page_size
= getpagesize();
267 if (qemu_host_page_size
== 0)
268 qemu_host_page_size
= qemu_real_host_page_size
;
269 if (qemu_host_page_size
< TARGET_PAGE_SIZE
)
270 qemu_host_page_size
= TARGET_PAGE_SIZE
;
271 qemu_host_page_bits
= 0;
272 while ((1 << qemu_host_page_bits
) < qemu_host_page_size
)
273 qemu_host_page_bits
++;
274 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
276 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
278 #ifdef HAVE_KINFO_GETVMMAP
279 struct kinfo_vmentry
*freep
;
282 freep
= kinfo_getvmmap(getpid(), &cnt
);
285 for (i
= 0; i
< cnt
; i
++) {
286 unsigned long startaddr
, endaddr
;
288 startaddr
= freep
[i
].kve_start
;
289 endaddr
= freep
[i
].kve_end
;
290 if (h2g_valid(startaddr
)) {
291 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
293 if (h2g_valid(endaddr
)) {
294 endaddr
= h2g(endaddr
);
295 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
297 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
299 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
310 last_brk
= (unsigned long)sbrk(0);
312 f
= fopen("/compat/linux/proc/self/maps", "r");
317 unsigned long startaddr
, endaddr
;
320 n
= fscanf (f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
322 if (n
== 2 && h2g_valid(startaddr
)) {
323 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
325 if (h2g_valid(endaddr
)) {
326 endaddr
= h2g(endaddr
);
330 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
342 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
348 #if defined(CONFIG_USER_ONLY)
349 /* We can't use qemu_malloc because it may recurse into a locked mutex. */
350 # define ALLOC(P, SIZE) \
352 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
353 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
356 # define ALLOC(P, SIZE) \
357 do { P = qemu_mallocz(SIZE); } while (0)
360 /* Level 1. Always allocated. */
361 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
364 for (i
= V_L1_SHIFT
/ L2_BITS
- 1; i
> 0; i
--) {
371 ALLOC(p
, sizeof(void *) * L2_SIZE
);
375 lp
= p
+ ((index
>> (i
* L2_BITS
)) & (L2_SIZE
- 1));
383 ALLOC(pd
, sizeof(PageDesc
) * L2_SIZE
);
389 return pd
+ (index
& (L2_SIZE
- 1));
392 static inline PageDesc
*page_find(tb_page_addr_t index
)
394 return page_find_alloc(index
, 0);
397 #if !defined(CONFIG_USER_ONLY)
398 static PhysPageDesc
*phys_page_find_alloc(target_phys_addr_t index
, int alloc
)
404 /* Level 1. Always allocated. */
405 lp
= l1_phys_map
+ ((index
>> P_L1_SHIFT
) & (P_L1_SIZE
- 1));
408 for (i
= P_L1_SHIFT
/ L2_BITS
- 1; i
> 0; i
--) {
414 *lp
= p
= qemu_mallocz(sizeof(void *) * L2_SIZE
);
416 lp
= p
+ ((index
>> (i
* L2_BITS
)) & (L2_SIZE
- 1));
427 *lp
= pd
= qemu_malloc(sizeof(PhysPageDesc
) * L2_SIZE
);
429 for (i
= 0; i
< L2_SIZE
; i
++) {
430 pd
[i
].phys_offset
= IO_MEM_UNASSIGNED
;
431 pd
[i
].region_offset
= (index
+ i
) << TARGET_PAGE_BITS
;
435 return pd
+ (index
& (L2_SIZE
- 1));
438 static inline PhysPageDesc
*phys_page_find(target_phys_addr_t index
)
440 return phys_page_find_alloc(index
, 0);
443 static void tlb_protect_code(ram_addr_t ram_addr
);
444 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
446 #define mmap_lock() do { } while(0)
447 #define mmap_unlock() do { } while(0)
450 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
452 #if defined(CONFIG_USER_ONLY)
453 /* Currently it is not recommended to allocate big chunks of data in
454 user mode. It will change when a dedicated libc will be used */
455 #define USE_STATIC_CODE_GEN_BUFFER
458 #ifdef USE_STATIC_CODE_GEN_BUFFER
459 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
460 __attribute__((aligned (CODE_GEN_ALIGN
)));
463 static void code_gen_alloc(unsigned long tb_size
)
465 #ifdef USE_STATIC_CODE_GEN_BUFFER
466 code_gen_buffer
= static_code_gen_buffer
;
467 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
468 map_exec(code_gen_buffer
, code_gen_buffer_size
);
470 code_gen_buffer_size
= tb_size
;
471 if (code_gen_buffer_size
== 0) {
472 #if defined(CONFIG_USER_ONLY)
473 /* in user mode, phys_ram_size is not meaningful */
474 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
476 /* XXX: needs adjustments */
477 code_gen_buffer_size
= (unsigned long)(ram_size
/ 4);
480 if (code_gen_buffer_size
< MIN_CODE_GEN_BUFFER_SIZE
)
481 code_gen_buffer_size
= MIN_CODE_GEN_BUFFER_SIZE
;
482 /* The code gen buffer location may have constraints depending on
483 the host cpu and OS */
484 #if defined(__linux__)
489 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
490 #if defined(__x86_64__)
492 /* Cannot map more than that */
493 if (code_gen_buffer_size
> (800 * 1024 * 1024))
494 code_gen_buffer_size
= (800 * 1024 * 1024);
495 #elif defined(__sparc_v9__)
496 // Map the buffer below 2G, so we can use direct calls and branches
498 start
= (void *) 0x60000000UL
;
499 if (code_gen_buffer_size
> (512 * 1024 * 1024))
500 code_gen_buffer_size
= (512 * 1024 * 1024);
501 #elif defined(__arm__)
502 /* Map the buffer below 32M, so we can use direct calls and branches */
504 start
= (void *) 0x01000000UL
;
505 if (code_gen_buffer_size
> 16 * 1024 * 1024)
506 code_gen_buffer_size
= 16 * 1024 * 1024;
507 #elif defined(__s390x__)
508 /* Map the buffer so that we can use direct calls and branches. */
509 /* We have a +- 4GB range on the branches; leave some slop. */
510 if (code_gen_buffer_size
> (3ul * 1024 * 1024 * 1024)) {
511 code_gen_buffer_size
= 3ul * 1024 * 1024 * 1024;
513 start
= (void *)0x90000000UL
;
515 code_gen_buffer
= mmap(start
, code_gen_buffer_size
,
516 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
518 if (code_gen_buffer
== MAP_FAILED
) {
519 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
523 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
524 || defined(__DragonFly__) || defined(__OpenBSD__)
528 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
529 #if defined(__x86_64__)
530 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
531 * 0x40000000 is free */
533 addr
= (void *)0x40000000;
534 /* Cannot map more than that */
535 if (code_gen_buffer_size
> (800 * 1024 * 1024))
536 code_gen_buffer_size
= (800 * 1024 * 1024);
537 #elif defined(__sparc_v9__)
538 // Map the buffer below 2G, so we can use direct calls and branches
540 addr
= (void *) 0x60000000UL
;
541 if (code_gen_buffer_size
> (512 * 1024 * 1024)) {
542 code_gen_buffer_size
= (512 * 1024 * 1024);
545 code_gen_buffer
= mmap(addr
, code_gen_buffer_size
,
546 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
548 if (code_gen_buffer
== MAP_FAILED
) {
549 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
554 code_gen_buffer
= qemu_malloc(code_gen_buffer_size
);
555 map_exec(code_gen_buffer
, code_gen_buffer_size
);
557 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
558 map_exec(code_gen_prologue
, sizeof(code_gen_prologue
));
559 code_gen_buffer_max_size
= code_gen_buffer_size
-
560 (TCG_MAX_OP_SIZE
* OPC_MAX_SIZE
);
561 code_gen_max_blocks
= code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
562 tbs
= qemu_malloc(code_gen_max_blocks
* sizeof(TranslationBlock
));
565 /* Must be called before using the QEMU cpus. 'tb_size' is the size
566 (in bytes) allocated to the translation buffer. Zero means default
568 void cpu_exec_init_all(unsigned long tb_size
)
571 code_gen_alloc(tb_size
);
572 code_gen_ptr
= code_gen_buffer
;
574 #if !defined(CONFIG_USER_ONLY)
577 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
578 /* There's no guest base to take into account, so go ahead and
579 initialize the prologue now. */
580 tcg_prologue_init(&tcg_ctx
);
584 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
586 static int cpu_common_post_load(void *opaque
, int version_id
)
588 CPUState
*env
= opaque
;
590 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
591 version_id is increased. */
592 env
->interrupt_request
&= ~0x01;
598 static const VMStateDescription vmstate_cpu_common
= {
599 .name
= "cpu_common",
601 .minimum_version_id
= 1,
602 .minimum_version_id_old
= 1,
603 .post_load
= cpu_common_post_load
,
604 .fields
= (VMStateField
[]) {
605 VMSTATE_UINT32(halted
, CPUState
),
606 VMSTATE_UINT32(interrupt_request
, CPUState
),
607 VMSTATE_END_OF_LIST()
612 CPUState
*qemu_get_cpu(int cpu
)
614 CPUState
*env
= first_cpu
;
617 if (env
->cpu_index
== cpu
)
625 void cpu_exec_init(CPUState
*env
)
630 #if defined(CONFIG_USER_ONLY)
633 env
->next_cpu
= NULL
;
636 while (*penv
!= NULL
) {
637 penv
= &(*penv
)->next_cpu
;
640 env
->cpu_index
= cpu_index
;
642 QTAILQ_INIT(&env
->breakpoints
);
643 QTAILQ_INIT(&env
->watchpoints
);
644 #ifndef CONFIG_USER_ONLY
645 env
->thread_id
= qemu_get_thread_id();
648 #if defined(CONFIG_USER_ONLY)
651 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
652 vmstate_register(NULL
, cpu_index
, &vmstate_cpu_common
, env
);
653 register_savevm(NULL
, "cpu", cpu_index
, CPU_SAVE_VERSION
,
654 cpu_save
, cpu_load
, env
);
658 /* Allocate a new translation block. Flush the translation buffer if
659 too many translation blocks or too much generated code. */
660 static TranslationBlock
*tb_alloc(target_ulong pc
)
662 TranslationBlock
*tb
;
664 if (nb_tbs
>= code_gen_max_blocks
||
665 (code_gen_ptr
- code_gen_buffer
) >= code_gen_buffer_max_size
)
673 void tb_free(TranslationBlock
*tb
)
675 /* In practice this is mostly used for single use temporary TB
676 Ignore the hard cases and just back up if this TB happens to
677 be the last one generated. */
678 if (nb_tbs
> 0 && tb
== &tbs
[nb_tbs
- 1]) {
679 code_gen_ptr
= tb
->tc_ptr
;
684 static inline void invalidate_page_bitmap(PageDesc
*p
)
686 if (p
->code_bitmap
) {
687 qemu_free(p
->code_bitmap
);
688 p
->code_bitmap
= NULL
;
690 p
->code_write_count
= 0;
693 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
695 static void page_flush_tb_1 (int level
, void **lp
)
704 for (i
= 0; i
< L2_SIZE
; ++i
) {
705 pd
[i
].first_tb
= NULL
;
706 invalidate_page_bitmap(pd
+ i
);
710 for (i
= 0; i
< L2_SIZE
; ++i
) {
711 page_flush_tb_1 (level
- 1, pp
+ i
);
716 static void page_flush_tb(void)
719 for (i
= 0; i
< V_L1_SIZE
; i
++) {
720 page_flush_tb_1(V_L1_SHIFT
/ L2_BITS
- 1, l1_map
+ i
);
724 /* flush all the translation blocks */
725 /* XXX: tb_flush is currently not thread safe */
726 void tb_flush(CPUState
*env1
)
729 #if defined(DEBUG_FLUSH)
730 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
731 (unsigned long)(code_gen_ptr
- code_gen_buffer
),
733 ((unsigned long)(code_gen_ptr
- code_gen_buffer
)) / nb_tbs
: 0);
735 if ((unsigned long)(code_gen_ptr
- code_gen_buffer
) > code_gen_buffer_size
)
736 cpu_abort(env1
, "Internal error: code buffer overflow\n");
740 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
741 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
744 memset (tb_phys_hash
, 0, CODE_GEN_PHYS_HASH_SIZE
* sizeof (void *));
747 code_gen_ptr
= code_gen_buffer
;
748 /* XXX: flush processor icache at this point if cache flush is
753 #ifdef DEBUG_TB_CHECK
755 static void tb_invalidate_check(target_ulong address
)
757 TranslationBlock
*tb
;
759 address
&= TARGET_PAGE_MASK
;
760 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
761 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
762 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
763 address
>= tb
->pc
+ tb
->size
)) {
764 printf("ERROR invalidate: address=" TARGET_FMT_lx
765 " PC=%08lx size=%04x\n",
766 address
, (long)tb
->pc
, tb
->size
);
772 /* verify that all the pages have correct rights for code */
773 static void tb_page_check(void)
775 TranslationBlock
*tb
;
776 int i
, flags1
, flags2
;
778 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
779 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
780 flags1
= page_get_flags(tb
->pc
);
781 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
782 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
783 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
784 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
792 /* invalidate one TB */
793 static inline void tb_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
,
796 TranslationBlock
*tb1
;
800 *ptb
= *(TranslationBlock
**)((char *)tb1
+ next_offset
);
803 ptb
= (TranslationBlock
**)((char *)tb1
+ next_offset
);
807 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
809 TranslationBlock
*tb1
;
815 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
817 *ptb
= tb1
->page_next
[n1
];
820 ptb
= &tb1
->page_next
[n1
];
824 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
826 TranslationBlock
*tb1
, **ptb
;
829 ptb
= &tb
->jmp_next
[n
];
832 /* find tb(n) in circular list */
836 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
837 if (n1
== n
&& tb1
== tb
)
840 ptb
= &tb1
->jmp_first
;
842 ptb
= &tb1
->jmp_next
[n1
];
845 /* now we can suppress tb(n) from the list */
846 *ptb
= tb
->jmp_next
[n
];
848 tb
->jmp_next
[n
] = NULL
;
852 /* reset the jump entry 'n' of a TB so that it is not chained to
854 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
856 tb_set_jmp_target(tb
, n
, (unsigned long)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
859 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
864 tb_page_addr_t phys_pc
;
865 TranslationBlock
*tb1
, *tb2
;
867 /* remove the TB from the hash list */
868 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
869 h
= tb_phys_hash_func(phys_pc
);
870 tb_remove(&tb_phys_hash
[h
], tb
,
871 offsetof(TranslationBlock
, phys_hash_next
));
873 /* remove the TB from the page list */
874 if (tb
->page_addr
[0] != page_addr
) {
875 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
876 tb_page_remove(&p
->first_tb
, tb
);
877 invalidate_page_bitmap(p
);
879 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
880 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
881 tb_page_remove(&p
->first_tb
, tb
);
882 invalidate_page_bitmap(p
);
885 tb_invalidated_flag
= 1;
887 /* remove the TB from the hash list */
888 h
= tb_jmp_cache_hash_func(tb
->pc
);
889 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
890 if (env
->tb_jmp_cache
[h
] == tb
)
891 env
->tb_jmp_cache
[h
] = NULL
;
894 /* suppress this TB from the two jump lists */
895 tb_jmp_remove(tb
, 0);
896 tb_jmp_remove(tb
, 1);
898 /* suppress any remaining jumps to this TB */
904 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
905 tb2
= tb1
->jmp_next
[n1
];
906 tb_reset_jump(tb1
, n1
);
907 tb1
->jmp_next
[n1
] = NULL
;
910 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2); /* fail safe */
912 tb_phys_invalidate_count
++;
915 static inline void set_bits(uint8_t *tab
, int start
, int len
)
921 mask
= 0xff << (start
& 7);
922 if ((start
& ~7) == (end
& ~7)) {
924 mask
&= ~(0xff << (end
& 7));
929 start
= (start
+ 8) & ~7;
931 while (start
< end1
) {
936 mask
= ~(0xff << (end
& 7));
942 static void build_page_bitmap(PageDesc
*p
)
944 int n
, tb_start
, tb_end
;
945 TranslationBlock
*tb
;
947 p
->code_bitmap
= qemu_mallocz(TARGET_PAGE_SIZE
/ 8);
952 tb
= (TranslationBlock
*)((long)tb
& ~3);
953 /* NOTE: this is subtle as a TB may span two physical pages */
955 /* NOTE: tb_end may be after the end of the page, but
956 it is not a problem */
957 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
958 tb_end
= tb_start
+ tb
->size
;
959 if (tb_end
> TARGET_PAGE_SIZE
)
960 tb_end
= TARGET_PAGE_SIZE
;
963 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
965 set_bits(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
966 tb
= tb
->page_next
[n
];
970 TranslationBlock
*tb_gen_code(CPUState
*env
,
971 target_ulong pc
, target_ulong cs_base
,
972 int flags
, int cflags
)
974 TranslationBlock
*tb
;
976 tb_page_addr_t phys_pc
, phys_page2
;
977 target_ulong virt_page2
;
980 phys_pc
= get_page_addr_code(env
, pc
);
983 /* flush must be done */
985 /* cannot fail at this point */
987 /* Don't forget to invalidate previous TB info. */
988 tb_invalidated_flag
= 1;
990 tc_ptr
= code_gen_ptr
;
992 tb
->cs_base
= cs_base
;
995 cpu_gen_code(env
, tb
, &code_gen_size
);
996 code_gen_ptr
= (void *)(((unsigned long)code_gen_ptr
+ code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
998 /* check next page if needed */
999 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1001 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1002 phys_page2
= get_page_addr_code(env
, virt_page2
);
1004 tb_link_page(tb
, phys_pc
, phys_page2
);
1008 /* invalidate all TBs which intersect with the target physical page
1009 starting in range [start;end[. NOTE: start and end must refer to
1010 the same physical page. 'is_cpu_write_access' should be true if called
1011 from a real cpu write access: the virtual CPU will exit the current
1012 TB if code is modified inside this TB. */
1013 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1014 int is_cpu_write_access
)
1016 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
1017 CPUState
*env
= cpu_single_env
;
1018 tb_page_addr_t tb_start
, tb_end
;
1021 #ifdef TARGET_HAS_PRECISE_SMC
1022 int current_tb_not_found
= is_cpu_write_access
;
1023 TranslationBlock
*current_tb
= NULL
;
1024 int current_tb_modified
= 0;
1025 target_ulong current_pc
= 0;
1026 target_ulong current_cs_base
= 0;
1027 int current_flags
= 0;
1028 #endif /* TARGET_HAS_PRECISE_SMC */
1030 p
= page_find(start
>> TARGET_PAGE_BITS
);
1033 if (!p
->code_bitmap
&&
1034 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
&&
1035 is_cpu_write_access
) {
1036 /* build code bitmap */
1037 build_page_bitmap(p
);
1040 /* we remove all the TBs in the range [start, end[ */
1041 /* XXX: see if in some cases it could be faster to invalidate all the code */
1043 while (tb
!= NULL
) {
1045 tb
= (TranslationBlock
*)((long)tb
& ~3);
1046 tb_next
= tb
->page_next
[n
];
1047 /* NOTE: this is subtle as a TB may span two physical pages */
1049 /* NOTE: tb_end may be after the end of the page, but
1050 it is not a problem */
1051 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1052 tb_end
= tb_start
+ tb
->size
;
1054 tb_start
= tb
->page_addr
[1];
1055 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1057 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1058 #ifdef TARGET_HAS_PRECISE_SMC
1059 if (current_tb_not_found
) {
1060 current_tb_not_found
= 0;
1062 if (env
->mem_io_pc
) {
1063 /* now we have a real cpu fault */
1064 current_tb
= tb_find_pc(env
->mem_io_pc
);
1067 if (current_tb
== tb
&&
1068 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1069 /* If we are modifying the current TB, we must stop
1070 its execution. We could be more precise by checking
1071 that the modification is after the current PC, but it
1072 would require a specialized function to partially
1073 restore the CPU state */
1075 current_tb_modified
= 1;
1076 cpu_restore_state(current_tb
, env
, env
->mem_io_pc
);
1077 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1080 #endif /* TARGET_HAS_PRECISE_SMC */
1081 /* we need to do that to handle the case where a signal
1082 occurs while doing tb_phys_invalidate() */
1085 saved_tb
= env
->current_tb
;
1086 env
->current_tb
= NULL
;
1088 tb_phys_invalidate(tb
, -1);
1090 env
->current_tb
= saved_tb
;
1091 if (env
->interrupt_request
&& env
->current_tb
)
1092 cpu_interrupt(env
, env
->interrupt_request
);
1097 #if !defined(CONFIG_USER_ONLY)
1098 /* if no code remaining, no need to continue to use slow writes */
1100 invalidate_page_bitmap(p
);
1101 if (is_cpu_write_access
) {
1102 tlb_unprotect_code_phys(env
, start
, env
->mem_io_vaddr
);
1106 #ifdef TARGET_HAS_PRECISE_SMC
1107 if (current_tb_modified
) {
1108 /* we generate a block containing just the instruction
1109 modifying the memory. It will ensure that it cannot modify
1111 env
->current_tb
= NULL
;
1112 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1113 cpu_resume_from_signal(env
, NULL
);
1118 /* len must be <= 8 and start must be a multiple of len */
1119 static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1125 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1126 cpu_single_env
->mem_io_vaddr
, len
,
1127 cpu_single_env
->eip
,
1128 cpu_single_env
->eip
+ (long)cpu_single_env
->segs
[R_CS
].base
);
1131 p
= page_find(start
>> TARGET_PAGE_BITS
);
1134 if (p
->code_bitmap
) {
1135 offset
= start
& ~TARGET_PAGE_MASK
;
1136 b
= p
->code_bitmap
[offset
>> 3] >> (offset
& 7);
1137 if (b
& ((1 << len
) - 1))
1141 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1145 #if !defined(CONFIG_SOFTMMU)
1146 static void tb_invalidate_phys_page(tb_page_addr_t addr
,
1147 unsigned long pc
, void *puc
)
1149 TranslationBlock
*tb
;
1152 #ifdef TARGET_HAS_PRECISE_SMC
1153 TranslationBlock
*current_tb
= NULL
;
1154 CPUState
*env
= cpu_single_env
;
1155 int current_tb_modified
= 0;
1156 target_ulong current_pc
= 0;
1157 target_ulong current_cs_base
= 0;
1158 int current_flags
= 0;
1161 addr
&= TARGET_PAGE_MASK
;
1162 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1166 #ifdef TARGET_HAS_PRECISE_SMC
1167 if (tb
&& pc
!= 0) {
1168 current_tb
= tb_find_pc(pc
);
1171 while (tb
!= NULL
) {
1173 tb
= (TranslationBlock
*)((long)tb
& ~3);
1174 #ifdef TARGET_HAS_PRECISE_SMC
1175 if (current_tb
== tb
&&
1176 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1177 /* If we are modifying the current TB, we must stop
1178 its execution. We could be more precise by checking
1179 that the modification is after the current PC, but it
1180 would require a specialized function to partially
1181 restore the CPU state */
1183 current_tb_modified
= 1;
1184 cpu_restore_state(current_tb
, env
, pc
);
1185 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1188 #endif /* TARGET_HAS_PRECISE_SMC */
1189 tb_phys_invalidate(tb
, addr
);
1190 tb
= tb
->page_next
[n
];
1193 #ifdef TARGET_HAS_PRECISE_SMC
1194 if (current_tb_modified
) {
1195 /* we generate a block containing just the instruction
1196 modifying the memory. It will ensure that it cannot modify
1198 env
->current_tb
= NULL
;
1199 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1200 cpu_resume_from_signal(env
, puc
);
1206 /* add the tb in the target page and protect it if necessary */
1207 static inline void tb_alloc_page(TranslationBlock
*tb
,
1208 unsigned int n
, tb_page_addr_t page_addr
)
1211 TranslationBlock
*last_first_tb
;
1213 tb
->page_addr
[n
] = page_addr
;
1214 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1215 tb
->page_next
[n
] = p
->first_tb
;
1216 last_first_tb
= p
->first_tb
;
1217 p
->first_tb
= (TranslationBlock
*)((long)tb
| n
);
1218 invalidate_page_bitmap(p
);
1220 #if defined(TARGET_HAS_SMC) || 1
1222 #if defined(CONFIG_USER_ONLY)
1223 if (p
->flags
& PAGE_WRITE
) {
1228 /* force the host page as non writable (writes will have a
1229 page fault + mprotect overhead) */
1230 page_addr
&= qemu_host_page_mask
;
1232 for(addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1233 addr
+= TARGET_PAGE_SIZE
) {
1235 p2
= page_find (addr
>> TARGET_PAGE_BITS
);
1239 p2
->flags
&= ~PAGE_WRITE
;
1241 mprotect(g2h(page_addr
), qemu_host_page_size
,
1242 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1243 #ifdef DEBUG_TB_INVALIDATE
1244 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1249 /* if some code is already present, then the pages are already
1250 protected. So we handle the case where only the first TB is
1251 allocated in a physical page */
1252 if (!last_first_tb
) {
1253 tlb_protect_code(page_addr
);
1257 #endif /* TARGET_HAS_SMC */
1260 /* add a new TB and link it to the physical page tables. phys_page2 is
1261 (-1) to indicate that only one page contains the TB. */
1262 void tb_link_page(TranslationBlock
*tb
,
1263 tb_page_addr_t phys_pc
, tb_page_addr_t phys_page2
)
1266 TranslationBlock
**ptb
;
1268 /* Grab the mmap lock to stop another thread invalidating this TB
1269 before we are done. */
1271 /* add in the physical hash table */
1272 h
= tb_phys_hash_func(phys_pc
);
1273 ptb
= &tb_phys_hash
[h
];
1274 tb
->phys_hash_next
= *ptb
;
1277 /* add in the page list */
1278 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1279 if (phys_page2
!= -1)
1280 tb_alloc_page(tb
, 1, phys_page2
);
1282 tb
->page_addr
[1] = -1;
1284 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2);
1285 tb
->jmp_next
[0] = NULL
;
1286 tb
->jmp_next
[1] = NULL
;
1288 /* init original jump addresses */
1289 if (tb
->tb_next_offset
[0] != 0xffff)
1290 tb_reset_jump(tb
, 0);
1291 if (tb
->tb_next_offset
[1] != 0xffff)
1292 tb_reset_jump(tb
, 1);
1294 #ifdef DEBUG_TB_CHECK
1300 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1301 tb[1].tc_ptr. Return NULL if not found */
1302 TranslationBlock
*tb_find_pc(unsigned long tc_ptr
)
1304 int m_min
, m_max
, m
;
1306 TranslationBlock
*tb
;
1310 if (tc_ptr
< (unsigned long)code_gen_buffer
||
1311 tc_ptr
>= (unsigned long)code_gen_ptr
)
1313 /* binary search (cf Knuth) */
1316 while (m_min
<= m_max
) {
1317 m
= (m_min
+ m_max
) >> 1;
1319 v
= (unsigned long)tb
->tc_ptr
;
1322 else if (tc_ptr
< v
) {
1331 static void tb_reset_jump_recursive(TranslationBlock
*tb
);
1333 static inline void tb_reset_jump_recursive2(TranslationBlock
*tb
, int n
)
1335 TranslationBlock
*tb1
, *tb_next
, **ptb
;
1338 tb1
= tb
->jmp_next
[n
];
1340 /* find head of list */
1343 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1346 tb1
= tb1
->jmp_next
[n1
];
1348 /* we are now sure now that tb jumps to tb1 */
1351 /* remove tb from the jmp_first list */
1352 ptb
= &tb_next
->jmp_first
;
1356 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1357 if (n1
== n
&& tb1
== tb
)
1359 ptb
= &tb1
->jmp_next
[n1
];
1361 *ptb
= tb
->jmp_next
[n
];
1362 tb
->jmp_next
[n
] = NULL
;
1364 /* suppress the jump to next tb in generated code */
1365 tb_reset_jump(tb
, n
);
1367 /* suppress jumps in the tb on which we could have jumped */
1368 tb_reset_jump_recursive(tb_next
);
1372 static void tb_reset_jump_recursive(TranslationBlock
*tb
)
1374 tb_reset_jump_recursive2(tb
, 0);
1375 tb_reset_jump_recursive2(tb
, 1);
1378 #if defined(TARGET_HAS_ICE)
1379 #if defined(CONFIG_USER_ONLY)
1380 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1382 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
1385 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1387 target_phys_addr_t addr
;
1389 ram_addr_t ram_addr
;
1392 addr
= cpu_get_phys_page_debug(env
, pc
);
1393 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
1395 pd
= IO_MEM_UNASSIGNED
;
1397 pd
= p
->phys_offset
;
1399 ram_addr
= (pd
& TARGET_PAGE_MASK
) | (pc
& ~TARGET_PAGE_MASK
);
1400 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1403 #endif /* TARGET_HAS_ICE */
1405 #if defined(CONFIG_USER_ONLY)
1406 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1411 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1412 int flags
, CPUWatchpoint
**watchpoint
)
1417 /* Add a watchpoint. */
1418 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1419 int flags
, CPUWatchpoint
**watchpoint
)
1421 target_ulong len_mask
= ~(len
- 1);
1424 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1425 if ((len
!= 1 && len
!= 2 && len
!= 4 && len
!= 8) || (addr
& ~len_mask
)) {
1426 fprintf(stderr
, "qemu: tried to set invalid watchpoint at "
1427 TARGET_FMT_lx
", len=" TARGET_FMT_lu
"\n", addr
, len
);
1430 wp
= qemu_malloc(sizeof(*wp
));
1433 wp
->len_mask
= len_mask
;
1436 /* keep all GDB-injected watchpoints in front */
1438 QTAILQ_INSERT_HEAD(&env
->watchpoints
, wp
, entry
);
1440 QTAILQ_INSERT_TAIL(&env
->watchpoints
, wp
, entry
);
1442 tlb_flush_page(env
, addr
);
1449 /* Remove a specific watchpoint. */
1450 int cpu_watchpoint_remove(CPUState
*env
, target_ulong addr
, target_ulong len
,
1453 target_ulong len_mask
= ~(len
- 1);
1456 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1457 if (addr
== wp
->vaddr
&& len_mask
== wp
->len_mask
1458 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
1459 cpu_watchpoint_remove_by_ref(env
, wp
);
1466 /* Remove a specific watchpoint by reference. */
1467 void cpu_watchpoint_remove_by_ref(CPUState
*env
, CPUWatchpoint
*watchpoint
)
1469 QTAILQ_REMOVE(&env
->watchpoints
, watchpoint
, entry
);
1471 tlb_flush_page(env
, watchpoint
->vaddr
);
1473 qemu_free(watchpoint
);
1476 /* Remove all matching watchpoints. */
1477 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1479 CPUWatchpoint
*wp
, *next
;
1481 QTAILQ_FOREACH_SAFE(wp
, &env
->watchpoints
, entry
, next
) {
1482 if (wp
->flags
& mask
)
1483 cpu_watchpoint_remove_by_ref(env
, wp
);
1488 /* Add a breakpoint. */
1489 int cpu_breakpoint_insert(CPUState
*env
, target_ulong pc
, int flags
,
1490 CPUBreakpoint
**breakpoint
)
1492 #if defined(TARGET_HAS_ICE)
1495 bp
= qemu_malloc(sizeof(*bp
));
1500 /* keep all GDB-injected breakpoints in front */
1502 QTAILQ_INSERT_HEAD(&env
->breakpoints
, bp
, entry
);
1504 QTAILQ_INSERT_TAIL(&env
->breakpoints
, bp
, entry
);
1506 breakpoint_invalidate(env
, pc
);
1516 /* Remove a specific breakpoint. */
1517 int cpu_breakpoint_remove(CPUState
*env
, target_ulong pc
, int flags
)
1519 #if defined(TARGET_HAS_ICE)
1522 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1523 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
1524 cpu_breakpoint_remove_by_ref(env
, bp
);
1534 /* Remove a specific breakpoint by reference. */
1535 void cpu_breakpoint_remove_by_ref(CPUState
*env
, CPUBreakpoint
*breakpoint
)
1537 #if defined(TARGET_HAS_ICE)
1538 QTAILQ_REMOVE(&env
->breakpoints
, breakpoint
, entry
);
1540 breakpoint_invalidate(env
, breakpoint
->pc
);
1542 qemu_free(breakpoint
);
1546 /* Remove all matching breakpoints. */
1547 void cpu_breakpoint_remove_all(CPUState
*env
, int mask
)
1549 #if defined(TARGET_HAS_ICE)
1550 CPUBreakpoint
*bp
, *next
;
1552 QTAILQ_FOREACH_SAFE(bp
, &env
->breakpoints
, entry
, next
) {
1553 if (bp
->flags
& mask
)
1554 cpu_breakpoint_remove_by_ref(env
, bp
);
1559 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1560 CPU loop after each instruction */
1561 void cpu_single_step(CPUState
*env
, int enabled
)
1563 #if defined(TARGET_HAS_ICE)
1564 if (env
->singlestep_enabled
!= enabled
) {
1565 env
->singlestep_enabled
= enabled
;
1567 kvm_update_guest_debug(env
, 0);
1569 /* must flush all the translated code to avoid inconsistencies */
1570 /* XXX: only flush what is necessary */
1577 /* enable or disable low levels log */
1578 void cpu_set_log(int log_flags
)
1580 loglevel
= log_flags
;
1581 if (loglevel
&& !logfile
) {
1582 logfile
= fopen(logfilename
, log_append
? "a" : "w");
1584 perror(logfilename
);
1587 #if !defined(CONFIG_SOFTMMU)
1588 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1590 static char logfile_buf
[4096];
1591 setvbuf(logfile
, logfile_buf
, _IOLBF
, sizeof(logfile_buf
));
1593 #elif !defined(_WIN32)
1594 /* Win32 doesn't support line-buffering and requires size >= 2 */
1595 setvbuf(logfile
, NULL
, _IOLBF
, 0);
1599 if (!loglevel
&& logfile
) {
1605 void cpu_set_log_filename(const char *filename
)
1607 logfilename
= strdup(filename
);
1612 cpu_set_log(loglevel
);
1615 static void cpu_unlink_tb(CPUState
*env
)
1617 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1618 problem and hope the cpu will stop of its own accord. For userspace
1619 emulation this often isn't actually as bad as it sounds. Often
1620 signals are used primarily to interrupt blocking syscalls. */
1621 TranslationBlock
*tb
;
1622 static spinlock_t interrupt_lock
= SPIN_LOCK_UNLOCKED
;
1624 spin_lock(&interrupt_lock
);
1625 tb
= env
->current_tb
;
1626 /* if the cpu is currently executing code, we must unlink it and
1627 all the potentially executing TB */
1629 env
->current_tb
= NULL
;
1630 tb_reset_jump_recursive(tb
);
1632 spin_unlock(&interrupt_lock
);
1635 #ifndef CONFIG_USER_ONLY
1636 /* mask must never be zero, except for A20 change call */
1637 static void tcg_handle_interrupt(CPUState
*env
, int mask
)
1641 old_mask
= env
->interrupt_request
;
1642 env
->interrupt_request
|= mask
;
1645 * If called from iothread context, wake the target cpu in
1648 if (!qemu_cpu_is_self(env
)) {
1654 env
->icount_decr
.u16
.high
= 0xffff;
1656 && (mask
& ~old_mask
) != 0) {
1657 cpu_abort(env
, "Raised interrupt while not in I/O function");
1664 CPUInterruptHandler cpu_interrupt_handler
= tcg_handle_interrupt
;
1666 #else /* CONFIG_USER_ONLY */
1668 void cpu_interrupt(CPUState
*env
, int mask
)
1670 env
->interrupt_request
|= mask
;
1673 #endif /* CONFIG_USER_ONLY */
1675 void cpu_reset_interrupt(CPUState
*env
, int mask
)
1677 env
->interrupt_request
&= ~mask
;
1680 void cpu_exit(CPUState
*env
)
1682 env
->exit_request
= 1;
1686 const CPULogItem cpu_log_items
[] = {
1687 { CPU_LOG_TB_OUT_ASM
, "out_asm",
1688 "show generated host assembly code for each compiled TB" },
1689 { CPU_LOG_TB_IN_ASM
, "in_asm",
1690 "show target assembly code for each compiled TB" },
1691 { CPU_LOG_TB_OP
, "op",
1692 "show micro ops for each compiled TB" },
1693 { CPU_LOG_TB_OP_OPT
, "op_opt",
1696 "before eflags optimization and "
1698 "after liveness analysis" },
1699 { CPU_LOG_INT
, "int",
1700 "show interrupts/exceptions in short format" },
1701 { CPU_LOG_EXEC
, "exec",
1702 "show trace before each executed TB (lots of logs)" },
1703 { CPU_LOG_TB_CPU
, "cpu",
1704 "show CPU state before block translation" },
1706 { CPU_LOG_PCALL
, "pcall",
1707 "show protected mode far calls/returns/exceptions" },
1708 { CPU_LOG_RESET
, "cpu_reset",
1709 "show CPU state before CPU resets" },
1712 { CPU_LOG_IOPORT
, "ioport",
1713 "show all i/o ports accesses" },
1718 #ifndef CONFIG_USER_ONLY
1719 static QLIST_HEAD(memory_client_list
, CPUPhysMemoryClient
) memory_client_list
1720 = QLIST_HEAD_INITIALIZER(memory_client_list
);
1722 static void cpu_notify_set_memory(target_phys_addr_t start_addr
,
1724 ram_addr_t phys_offset
,
1727 CPUPhysMemoryClient
*client
;
1728 QLIST_FOREACH(client
, &memory_client_list
, list
) {
1729 client
->set_memory(client
, start_addr
, size
, phys_offset
, log_dirty
);
1733 static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start
,
1734 target_phys_addr_t end
)
1736 CPUPhysMemoryClient
*client
;
1737 QLIST_FOREACH(client
, &memory_client_list
, list
) {
1738 int r
= client
->sync_dirty_bitmap(client
, start
, end
);
1745 static int cpu_notify_migration_log(int enable
)
1747 CPUPhysMemoryClient
*client
;
1748 QLIST_FOREACH(client
, &memory_client_list
, list
) {
1749 int r
= client
->migration_log(client
, enable
);
1756 /* The l1_phys_map provides the upper P_L1_BITs of the guest physical
1757 * address. Each intermediate table provides the next L2_BITs of guest
1758 * physical address space. The number of levels vary based on host and
1759 * guest configuration, making it efficient to build the final guest
1760 * physical address by seeding the L1 offset and shifting and adding in
1761 * each L2 offset as we recurse through them. */
1762 static void phys_page_for_each_1(CPUPhysMemoryClient
*client
,
1763 int level
, void **lp
, target_phys_addr_t addr
)
1771 PhysPageDesc
*pd
= *lp
;
1772 addr
<<= L2_BITS
+ TARGET_PAGE_BITS
;
1773 for (i
= 0; i
< L2_SIZE
; ++i
) {
1774 if (pd
[i
].phys_offset
!= IO_MEM_UNASSIGNED
) {
1775 client
->set_memory(client
, addr
| i
<< TARGET_PAGE_BITS
,
1776 TARGET_PAGE_SIZE
, pd
[i
].phys_offset
, false);
1781 for (i
= 0; i
< L2_SIZE
; ++i
) {
1782 phys_page_for_each_1(client
, level
- 1, pp
+ i
,
1783 (addr
<< L2_BITS
) | i
);
1788 static void phys_page_for_each(CPUPhysMemoryClient
*client
)
1791 for (i
= 0; i
< P_L1_SIZE
; ++i
) {
1792 phys_page_for_each_1(client
, P_L1_SHIFT
/ L2_BITS
- 1,
1793 l1_phys_map
+ i
, i
);
1797 void cpu_register_phys_memory_client(CPUPhysMemoryClient
*client
)
1799 QLIST_INSERT_HEAD(&memory_client_list
, client
, list
);
1800 phys_page_for_each(client
);
1803 void cpu_unregister_phys_memory_client(CPUPhysMemoryClient
*client
)
1805 QLIST_REMOVE(client
, list
);
1809 static int cmp1(const char *s1
, int n
, const char *s2
)
1811 if (strlen(s2
) != n
)
1813 return memcmp(s1
, s2
, n
) == 0;
1816 /* takes a comma separated list of log masks. Return 0 if error. */
1817 int cpu_str_to_log_mask(const char *str
)
1819 const CPULogItem
*item
;
1826 p1
= strchr(p
, ',');
1829 if(cmp1(p
,p1
-p
,"all")) {
1830 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1834 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1835 if (cmp1(p
, p1
- p
, item
->name
))
1849 void cpu_abort(CPUState
*env
, const char *fmt
, ...)
1856 fprintf(stderr
, "qemu: fatal: ");
1857 vfprintf(stderr
, fmt
, ap
);
1858 fprintf(stderr
, "\n");
1860 cpu_dump_state(env
, stderr
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1862 cpu_dump_state(env
, stderr
, fprintf
, 0);
1864 if (qemu_log_enabled()) {
1865 qemu_log("qemu: fatal: ");
1866 qemu_log_vprintf(fmt
, ap2
);
1869 log_cpu_state(env
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1871 log_cpu_state(env
, 0);
1878 #if defined(CONFIG_USER_ONLY)
1880 struct sigaction act
;
1881 sigfillset(&act
.sa_mask
);
1882 act
.sa_handler
= SIG_DFL
;
1883 sigaction(SIGABRT
, &act
, NULL
);
1889 CPUState
*cpu_copy(CPUState
*env
)
1891 CPUState
*new_env
= cpu_init(env
->cpu_model_str
);
1892 CPUState
*next_cpu
= new_env
->next_cpu
;
1893 int cpu_index
= new_env
->cpu_index
;
1894 #if defined(TARGET_HAS_ICE)
1899 memcpy(new_env
, env
, sizeof(CPUState
));
1901 /* Preserve chaining and index. */
1902 new_env
->next_cpu
= next_cpu
;
1903 new_env
->cpu_index
= cpu_index
;
1905 /* Clone all break/watchpoints.
1906 Note: Once we support ptrace with hw-debug register access, make sure
1907 BP_CPU break/watchpoints are handled correctly on clone. */
1908 QTAILQ_INIT(&env
->breakpoints
);
1909 QTAILQ_INIT(&env
->watchpoints
);
1910 #if defined(TARGET_HAS_ICE)
1911 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1912 cpu_breakpoint_insert(new_env
, bp
->pc
, bp
->flags
, NULL
);
1914 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1915 cpu_watchpoint_insert(new_env
, wp
->vaddr
, (~wp
->len_mask
) + 1,
1923 #if !defined(CONFIG_USER_ONLY)
1925 static inline void tlb_flush_jmp_cache(CPUState
*env
, target_ulong addr
)
1929 /* Discard jump cache entries for any tb which might potentially
1930 overlap the flushed page. */
1931 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1932 memset (&env
->tb_jmp_cache
[i
], 0,
1933 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1935 i
= tb_jmp_cache_hash_page(addr
);
1936 memset (&env
->tb_jmp_cache
[i
], 0,
1937 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1940 static CPUTLBEntry s_cputlb_empty_entry
= {
1947 /* NOTE: if flush_global is true, also flush global entries (not
1949 void tlb_flush(CPUState
*env
, int flush_global
)
1953 #if defined(DEBUG_TLB)
1954 printf("tlb_flush:\n");
1956 /* must reset current TB so that interrupts cannot modify the
1957 links while we are modifying them */
1958 env
->current_tb
= NULL
;
1960 for(i
= 0; i
< CPU_TLB_SIZE
; i
++) {
1962 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1963 env
->tlb_table
[mmu_idx
][i
] = s_cputlb_empty_entry
;
1967 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
1969 env
->tlb_flush_addr
= -1;
1970 env
->tlb_flush_mask
= 0;
1974 static inline void tlb_flush_entry(CPUTLBEntry
*tlb_entry
, target_ulong addr
)
1976 if (addr
== (tlb_entry
->addr_read
&
1977 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1978 addr
== (tlb_entry
->addr_write
&
1979 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1980 addr
== (tlb_entry
->addr_code
&
1981 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
))) {
1982 *tlb_entry
= s_cputlb_empty_entry
;
1986 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
1991 #if defined(DEBUG_TLB)
1992 printf("tlb_flush_page: " TARGET_FMT_lx
"\n", addr
);
1994 /* Check if we need to flush due to large pages. */
1995 if ((addr
& env
->tlb_flush_mask
) == env
->tlb_flush_addr
) {
1996 #if defined(DEBUG_TLB)
1997 printf("tlb_flush_page: forced full flush ("
1998 TARGET_FMT_lx
"/" TARGET_FMT_lx
")\n",
1999 env
->tlb_flush_addr
, env
->tlb_flush_mask
);
2004 /* must reset current TB so that interrupts cannot modify the
2005 links while we are modifying them */
2006 env
->current_tb
= NULL
;
2008 addr
&= TARGET_PAGE_MASK
;
2009 i
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2010 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++)
2011 tlb_flush_entry(&env
->tlb_table
[mmu_idx
][i
], addr
);
2013 tlb_flush_jmp_cache(env
, addr
);
2016 /* update the TLBs so that writes to code in the virtual page 'addr'
2018 static void tlb_protect_code(ram_addr_t ram_addr
)
2020 cpu_physical_memory_reset_dirty(ram_addr
,
2021 ram_addr
+ TARGET_PAGE_SIZE
,
2025 /* update the TLB so that writes in physical page 'phys_addr' are no longer
2026 tested for self modifying code */
2027 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
2030 cpu_physical_memory_set_dirty_flags(ram_addr
, CODE_DIRTY_FLAG
);
2033 static inline void tlb_reset_dirty_range(CPUTLBEntry
*tlb_entry
,
2034 unsigned long start
, unsigned long length
)
2037 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
2038 addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) + tlb_entry
->addend
;
2039 if ((addr
- start
) < length
) {
2040 tlb_entry
->addr_write
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) | TLB_NOTDIRTY
;
2045 /* Note: start and end must be within the same ram block. */
2046 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t end
,
2050 unsigned long length
, start1
;
2053 start
&= TARGET_PAGE_MASK
;
2054 end
= TARGET_PAGE_ALIGN(end
);
2056 length
= end
- start
;
2059 cpu_physical_memory_mask_dirty_range(start
, length
, dirty_flags
);
2061 /* we modify the TLB cache so that the dirty bit will be set again
2062 when accessing the range */
2063 start1
= (unsigned long)qemu_safe_ram_ptr(start
);
2064 /* Check that we don't span multiple blocks - this breaks the
2065 address comparisons below. */
2066 if ((unsigned long)qemu_safe_ram_ptr(end
- 1) - start1
2067 != (end
- 1) - start
) {
2071 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2073 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
2074 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
2075 tlb_reset_dirty_range(&env
->tlb_table
[mmu_idx
][i
],
2081 int cpu_physical_memory_set_dirty_tracking(int enable
)
2084 in_migration
= enable
;
2085 ret
= cpu_notify_migration_log(!!enable
);
2089 int cpu_physical_memory_get_dirty_tracking(void)
2091 return in_migration
;
2094 int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr
,
2095 target_phys_addr_t end_addr
)
2099 ret
= cpu_notify_sync_dirty_bitmap(start_addr
, end_addr
);
2103 int cpu_physical_log_start(target_phys_addr_t start_addr
,
2106 CPUPhysMemoryClient
*client
;
2107 QLIST_FOREACH(client
, &memory_client_list
, list
) {
2108 if (client
->log_start
) {
2109 int r
= client
->log_start(client
, start_addr
, size
);
2118 int cpu_physical_log_stop(target_phys_addr_t start_addr
,
2121 CPUPhysMemoryClient
*client
;
2122 QLIST_FOREACH(client
, &memory_client_list
, list
) {
2123 if (client
->log_stop
) {
2124 int r
= client
->log_stop(client
, start_addr
, size
);
2133 static inline void tlb_update_dirty(CPUTLBEntry
*tlb_entry
)
2135 ram_addr_t ram_addr
;
2138 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
2139 p
= (void *)(unsigned long)((tlb_entry
->addr_write
& TARGET_PAGE_MASK
)
2140 + tlb_entry
->addend
);
2141 ram_addr
= qemu_ram_addr_from_host_nofail(p
);
2142 if (!cpu_physical_memory_is_dirty(ram_addr
)) {
2143 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
2148 /* update the TLB according to the current state of the dirty bits */
2149 void cpu_tlb_update_dirty(CPUState
*env
)
2153 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
2154 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
2155 tlb_update_dirty(&env
->tlb_table
[mmu_idx
][i
]);
2159 static inline void tlb_set_dirty1(CPUTLBEntry
*tlb_entry
, target_ulong vaddr
)
2161 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
))
2162 tlb_entry
->addr_write
= vaddr
;
2165 /* update the TLB corresponding to virtual page vaddr
2166 so that it is no longer dirty */
2167 static inline void tlb_set_dirty(CPUState
*env
, target_ulong vaddr
)
2172 vaddr
&= TARGET_PAGE_MASK
;
2173 i
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2174 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++)
2175 tlb_set_dirty1(&env
->tlb_table
[mmu_idx
][i
], vaddr
);
2178 /* Our TLB does not support large pages, so remember the area covered by
2179 large pages and trigger a full TLB flush if these are invalidated. */
2180 static void tlb_add_large_page(CPUState
*env
, target_ulong vaddr
,
2183 target_ulong mask
= ~(size
- 1);
2185 if (env
->tlb_flush_addr
== (target_ulong
)-1) {
2186 env
->tlb_flush_addr
= vaddr
& mask
;
2187 env
->tlb_flush_mask
= mask
;
2190 /* Extend the existing region to include the new page.
2191 This is a compromise between unnecessary flushes and the cost
2192 of maintaining a full variable size TLB. */
2193 mask
&= env
->tlb_flush_mask
;
2194 while (((env
->tlb_flush_addr
^ vaddr
) & mask
) != 0) {
2197 env
->tlb_flush_addr
&= mask
;
2198 env
->tlb_flush_mask
= mask
;
2201 /* Add a new TLB entry. At most one entry for a given virtual address
2202 is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2203 supplied size is only used by tlb_flush_page. */
2204 void tlb_set_page(CPUState
*env
, target_ulong vaddr
,
2205 target_phys_addr_t paddr
, int prot
,
2206 int mmu_idx
, target_ulong size
)
2211 target_ulong address
;
2212 target_ulong code_address
;
2213 unsigned long addend
;
2216 target_phys_addr_t iotlb
;
2218 assert(size
>= TARGET_PAGE_SIZE
);
2219 if (size
!= TARGET_PAGE_SIZE
) {
2220 tlb_add_large_page(env
, vaddr
, size
);
2222 p
= phys_page_find(paddr
>> TARGET_PAGE_BITS
);
2224 pd
= IO_MEM_UNASSIGNED
;
2226 pd
= p
->phys_offset
;
2228 #if defined(DEBUG_TLB)
2229 printf("tlb_set_page: vaddr=" TARGET_FMT_lx
" paddr=0x" TARGET_FMT_plx
2230 " prot=%x idx=%d pd=0x%08lx\n",
2231 vaddr
, paddr
, prot
, mmu_idx
, pd
);
2235 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&& !(pd
& IO_MEM_ROMD
)) {
2236 /* IO memory case (romd handled later) */
2237 address
|= TLB_MMIO
;
2239 addend
= (unsigned long)qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
);
2240 if ((pd
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
) {
2242 iotlb
= pd
& TARGET_PAGE_MASK
;
2243 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
)
2244 iotlb
|= IO_MEM_NOTDIRTY
;
2246 iotlb
|= IO_MEM_ROM
;
2248 /* IO handlers are currently passed a physical address.
2249 It would be nice to pass an offset from the base address
2250 of that region. This would avoid having to special case RAM,
2251 and avoid full address decoding in every device.
2252 We can't use the high bits of pd for this because
2253 IO_MEM_ROMD uses these as a ram address. */
2254 iotlb
= (pd
& ~TARGET_PAGE_MASK
);
2256 iotlb
+= p
->region_offset
;
2262 code_address
= address
;
2263 /* Make accesses to pages with watchpoints go via the
2264 watchpoint trap routines. */
2265 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
2266 if (vaddr
== (wp
->vaddr
& TARGET_PAGE_MASK
)) {
2267 /* Avoid trapping reads of pages with a write breakpoint. */
2268 if ((prot
& PAGE_WRITE
) || (wp
->flags
& BP_MEM_READ
)) {
2269 iotlb
= io_mem_watch
+ paddr
;
2270 address
|= TLB_MMIO
;
2276 index
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2277 env
->iotlb
[mmu_idx
][index
] = iotlb
- vaddr
;
2278 te
= &env
->tlb_table
[mmu_idx
][index
];
2279 te
->addend
= addend
- vaddr
;
2280 if (prot
& PAGE_READ
) {
2281 te
->addr_read
= address
;
2286 if (prot
& PAGE_EXEC
) {
2287 te
->addr_code
= code_address
;
2291 if (prot
& PAGE_WRITE
) {
2292 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_ROM
||
2293 (pd
& IO_MEM_ROMD
)) {
2294 /* Write access calls the I/O callback. */
2295 te
->addr_write
= address
| TLB_MMIO
;
2296 } else if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
&&
2297 !cpu_physical_memory_is_dirty(pd
)) {
2298 te
->addr_write
= address
| TLB_NOTDIRTY
;
2300 te
->addr_write
= address
;
2303 te
->addr_write
= -1;
2309 void tlb_flush(CPUState
*env
, int flush_global
)
2313 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
2318 * Walks guest process memory "regions" one by one
2319 * and calls callback function 'fn' for each region.
2322 struct walk_memory_regions_data
2324 walk_memory_regions_fn fn
;
2326 unsigned long start
;
2330 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
2331 abi_ulong end
, int new_prot
)
2333 if (data
->start
!= -1ul) {
2334 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
2340 data
->start
= (new_prot
? end
: -1ul);
2341 data
->prot
= new_prot
;
2346 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
2347 abi_ulong base
, int level
, void **lp
)
2353 return walk_memory_regions_end(data
, base
, 0);
2358 for (i
= 0; i
< L2_SIZE
; ++i
) {
2359 int prot
= pd
[i
].flags
;
2361 pa
= base
| (i
<< TARGET_PAGE_BITS
);
2362 if (prot
!= data
->prot
) {
2363 rc
= walk_memory_regions_end(data
, pa
, prot
);
2371 for (i
= 0; i
< L2_SIZE
; ++i
) {
2372 pa
= base
| ((abi_ulong
)i
<<
2373 (TARGET_PAGE_BITS
+ L2_BITS
* level
));
2374 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
2384 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
2386 struct walk_memory_regions_data data
;
2394 for (i
= 0; i
< V_L1_SIZE
; i
++) {
2395 int rc
= walk_memory_regions_1(&data
, (abi_ulong
)i
<< V_L1_SHIFT
,
2396 V_L1_SHIFT
/ L2_BITS
- 1, l1_map
+ i
);
2402 return walk_memory_regions_end(&data
, 0, 0);
2405 static int dump_region(void *priv
, abi_ulong start
,
2406 abi_ulong end
, unsigned long prot
)
2408 FILE *f
= (FILE *)priv
;
2410 (void) fprintf(f
, TARGET_ABI_FMT_lx
"-"TARGET_ABI_FMT_lx
2411 " "TARGET_ABI_FMT_lx
" %c%c%c\n",
2412 start
, end
, end
- start
,
2413 ((prot
& PAGE_READ
) ? 'r' : '-'),
2414 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
2415 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
2420 /* dump memory mappings */
2421 void page_dump(FILE *f
)
2423 (void) fprintf(f
, "%-8s %-8s %-8s %s\n",
2424 "start", "end", "size", "prot");
2425 walk_memory_regions(f
, dump_region
);
2428 int page_get_flags(target_ulong address
)
2432 p
= page_find(address
>> TARGET_PAGE_BITS
);
2438 /* Modify the flags of a page and invalidate the code if necessary.
2439 The flag PAGE_WRITE_ORG is positioned automatically depending
2440 on PAGE_WRITE. The mmap_lock should already be held. */
2441 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2443 target_ulong addr
, len
;
2445 /* This function should never be called with addresses outside the
2446 guest address space. If this assert fires, it probably indicates
2447 a missing call to h2g_valid. */
2448 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2449 assert(end
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2451 assert(start
< end
);
2453 start
= start
& TARGET_PAGE_MASK
;
2454 end
= TARGET_PAGE_ALIGN(end
);
2456 if (flags
& PAGE_WRITE
) {
2457 flags
|= PAGE_WRITE_ORG
;
2460 for (addr
= start
, len
= end
- start
;
2462 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2463 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2465 /* If the write protection bit is set, then we invalidate
2467 if (!(p
->flags
& PAGE_WRITE
) &&
2468 (flags
& PAGE_WRITE
) &&
2470 tb_invalidate_phys_page(addr
, 0, NULL
);
2476 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2482 /* This function should never be called with addresses outside the
2483 guest address space. If this assert fires, it probably indicates
2484 a missing call to h2g_valid. */
2485 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2486 assert(start
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2492 if (start
+ len
- 1 < start
) {
2493 /* We've wrapped around. */
2497 end
= TARGET_PAGE_ALIGN(start
+len
); /* must do before we loose bits in the next step */
2498 start
= start
& TARGET_PAGE_MASK
;
2500 for (addr
= start
, len
= end
- start
;
2502 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2503 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2506 if( !(p
->flags
& PAGE_VALID
) )
2509 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
))
2511 if (flags
& PAGE_WRITE
) {
2512 if (!(p
->flags
& PAGE_WRITE_ORG
))
2514 /* unprotect the page if it was put read-only because it
2515 contains translated code */
2516 if (!(p
->flags
& PAGE_WRITE
)) {
2517 if (!page_unprotect(addr
, 0, NULL
))
2526 /* called from signal handler: invalidate the code and unprotect the
2527 page. Return TRUE if the fault was successfully handled. */
2528 int page_unprotect(target_ulong address
, unsigned long pc
, void *puc
)
2532 target_ulong host_start
, host_end
, addr
;
2534 /* Technically this isn't safe inside a signal handler. However we
2535 know this only ever happens in a synchronous SEGV handler, so in
2536 practice it seems to be ok. */
2539 p
= page_find(address
>> TARGET_PAGE_BITS
);
2545 /* if the page was really writable, then we change its
2546 protection back to writable */
2547 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
2548 host_start
= address
& qemu_host_page_mask
;
2549 host_end
= host_start
+ qemu_host_page_size
;
2552 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2553 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2554 p
->flags
|= PAGE_WRITE
;
2557 /* and since the content will be modified, we must invalidate
2558 the corresponding translated code. */
2559 tb_invalidate_phys_page(addr
, pc
, puc
);
2560 #ifdef DEBUG_TB_CHECK
2561 tb_invalidate_check(addr
);
2564 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2574 static inline void tlb_set_dirty(CPUState
*env
,
2575 unsigned long addr
, target_ulong vaddr
)
2578 #endif /* defined(CONFIG_USER_ONLY) */
2580 #if !defined(CONFIG_USER_ONLY)
2582 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2583 typedef struct subpage_t
{
2584 target_phys_addr_t base
;
2585 ram_addr_t sub_io_index
[TARGET_PAGE_SIZE
];
2586 ram_addr_t region_offset
[TARGET_PAGE_SIZE
];
2589 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2590 ram_addr_t memory
, ram_addr_t region_offset
);
2591 static subpage_t
*subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
2592 ram_addr_t orig_memory
,
2593 ram_addr_t region_offset
);
2594 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2597 if (addr > start_addr) \
2600 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2601 if (start_addr2 > 0) \
2605 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2606 end_addr2 = TARGET_PAGE_SIZE - 1; \
2608 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2609 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2614 /* register physical memory.
2615 For RAM, 'size' must be a multiple of the target page size.
2616 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2617 io memory page. The address used when calling the IO function is
2618 the offset from the start of the region, plus region_offset. Both
2619 start_addr and region_offset are rounded down to a page boundary
2620 before calculating this offset. This should not be a problem unless
2621 the low bits of start_addr and region_offset differ. */
2622 void cpu_register_physical_memory_log(target_phys_addr_t start_addr
,
2624 ram_addr_t phys_offset
,
2625 ram_addr_t region_offset
,
2628 target_phys_addr_t addr
, end_addr
;
2631 ram_addr_t orig_size
= size
;
2635 cpu_notify_set_memory(start_addr
, size
, phys_offset
, log_dirty
);
2637 if (phys_offset
== IO_MEM_UNASSIGNED
) {
2638 region_offset
= start_addr
;
2640 region_offset
&= TARGET_PAGE_MASK
;
2641 size
= (size
+ TARGET_PAGE_SIZE
- 1) & TARGET_PAGE_MASK
;
2642 end_addr
= start_addr
+ (target_phys_addr_t
)size
;
2646 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2647 if (p
&& p
->phys_offset
!= IO_MEM_UNASSIGNED
) {
2648 ram_addr_t orig_memory
= p
->phys_offset
;
2649 target_phys_addr_t start_addr2
, end_addr2
;
2650 int need_subpage
= 0;
2652 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
, end_addr2
,
2655 if (!(orig_memory
& IO_MEM_SUBPAGE
)) {
2656 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2657 &p
->phys_offset
, orig_memory
,
2660 subpage
= io_mem_opaque
[(orig_memory
& ~TARGET_PAGE_MASK
)
2663 subpage_register(subpage
, start_addr2
, end_addr2
, phys_offset
,
2665 p
->region_offset
= 0;
2667 p
->phys_offset
= phys_offset
;
2668 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2669 (phys_offset
& IO_MEM_ROMD
))
2670 phys_offset
+= TARGET_PAGE_SIZE
;
2673 p
= phys_page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2674 p
->phys_offset
= phys_offset
;
2675 p
->region_offset
= region_offset
;
2676 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2677 (phys_offset
& IO_MEM_ROMD
)) {
2678 phys_offset
+= TARGET_PAGE_SIZE
;
2680 target_phys_addr_t start_addr2
, end_addr2
;
2681 int need_subpage
= 0;
2683 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
,
2684 end_addr2
, need_subpage
);
2687 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2688 &p
->phys_offset
, IO_MEM_UNASSIGNED
,
2689 addr
& TARGET_PAGE_MASK
);
2690 subpage_register(subpage
, start_addr2
, end_addr2
,
2691 phys_offset
, region_offset
);
2692 p
->region_offset
= 0;
2696 region_offset
+= TARGET_PAGE_SIZE
;
2697 addr
+= TARGET_PAGE_SIZE
;
2698 } while (addr
!= end_addr
);
2700 /* since each CPU stores ram addresses in its TLB cache, we must
2701 reset the modified entries */
2703 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2708 /* XXX: temporary until new memory mapping API */
2709 ram_addr_t
cpu_get_physical_page_desc(target_phys_addr_t addr
)
2713 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2715 return IO_MEM_UNASSIGNED
;
2716 return p
->phys_offset
;
2719 void qemu_register_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2722 kvm_coalesce_mmio_region(addr
, size
);
2725 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2728 kvm_uncoalesce_mmio_region(addr
, size
);
2731 void qemu_flush_coalesced_mmio_buffer(void)
2734 kvm_flush_coalesced_mmio_buffer();
2737 #if defined(__linux__) && !defined(TARGET_S390X)
2739 #include <sys/vfs.h>
2741 #define HUGETLBFS_MAGIC 0x958458f6
2743 static long gethugepagesize(const char *path
)
2749 ret
= statfs(path
, &fs
);
2750 } while (ret
!= 0 && errno
== EINTR
);
2757 if (fs
.f_type
!= HUGETLBFS_MAGIC
)
2758 fprintf(stderr
, "Warning: path not on HugeTLBFS: %s\n", path
);
2763 static void *file_ram_alloc(RAMBlock
*block
,
2773 unsigned long hpagesize
;
2775 hpagesize
= gethugepagesize(path
);
2780 if (memory
< hpagesize
) {
2784 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2785 fprintf(stderr
, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2789 if (asprintf(&filename
, "%s/qemu_back_mem.XXXXXX", path
) == -1) {
2793 fd
= mkstemp(filename
);
2795 perror("unable to create backing store for hugepages");
2802 memory
= (memory
+hpagesize
-1) & ~(hpagesize
-1);
2805 * ftruncate is not supported by hugetlbfs in older
2806 * hosts, so don't bother bailing out on errors.
2807 * If anything goes wrong with it under other filesystems,
2810 if (ftruncate(fd
, memory
))
2811 perror("ftruncate");
2814 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2815 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2816 * to sidestep this quirk.
2818 flags
= mem_prealloc
? MAP_POPULATE
| MAP_SHARED
: MAP_PRIVATE
;
2819 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, flags
, fd
, 0);
2821 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, MAP_PRIVATE
, fd
, 0);
2823 if (area
== MAP_FAILED
) {
2824 perror("file_ram_alloc: can't mmap RAM pages");
2833 static ram_addr_t
find_ram_offset(ram_addr_t size
)
2835 RAMBlock
*block
, *next_block
;
2836 ram_addr_t offset
= 0, mingap
= ULONG_MAX
;
2838 if (QLIST_EMPTY(&ram_list
.blocks
))
2841 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2842 ram_addr_t end
, next
= ULONG_MAX
;
2844 end
= block
->offset
+ block
->length
;
2846 QLIST_FOREACH(next_block
, &ram_list
.blocks
, next
) {
2847 if (next_block
->offset
>= end
) {
2848 next
= MIN(next
, next_block
->offset
);
2851 if (next
- end
>= size
&& next
- end
< mingap
) {
2853 mingap
= next
- end
;
2859 static ram_addr_t
last_ram_offset(void)
2862 ram_addr_t last
= 0;
2864 QLIST_FOREACH(block
, &ram_list
.blocks
, next
)
2865 last
= MAX(last
, block
->offset
+ block
->length
);
2870 ram_addr_t
qemu_ram_alloc_from_ptr(DeviceState
*dev
, const char *name
,
2871 ram_addr_t size
, void *host
)
2873 RAMBlock
*new_block
, *block
;
2875 size
= TARGET_PAGE_ALIGN(size
);
2876 new_block
= qemu_mallocz(sizeof(*new_block
));
2878 if (dev
&& dev
->parent_bus
&& dev
->parent_bus
->info
->get_dev_path
) {
2879 char *id
= dev
->parent_bus
->info
->get_dev_path(dev
);
2881 snprintf(new_block
->idstr
, sizeof(new_block
->idstr
), "%s/", id
);
2885 pstrcat(new_block
->idstr
, sizeof(new_block
->idstr
), name
);
2887 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2888 if (!strcmp(block
->idstr
, new_block
->idstr
)) {
2889 fprintf(stderr
, "RAMBlock \"%s\" already registered, abort!\n",
2895 new_block
->offset
= find_ram_offset(size
);
2897 new_block
->host
= host
;
2898 new_block
->flags
|= RAM_PREALLOC_MASK
;
2901 #if defined (__linux__) && !defined(TARGET_S390X)
2902 new_block
->host
= file_ram_alloc(new_block
, size
, mem_path
);
2903 if (!new_block
->host
) {
2904 new_block
->host
= qemu_vmalloc(size
);
2905 qemu_madvise(new_block
->host
, size
, QEMU_MADV_MERGEABLE
);
2908 fprintf(stderr
, "-mem-path option unsupported\n");
2912 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2913 /* S390 KVM requires the topmost vma of the RAM to be smaller than
2914 an system defined value, which is at least 256GB. Larger systems
2915 have larger values. We put the guest between the end of data
2916 segment (system break) and this value. We use 32GB as a base to
2917 have enough room for the system break to grow. */
2918 new_block
->host
= mmap((void*)0x800000000, size
,
2919 PROT_EXEC
|PROT_READ
|PROT_WRITE
,
2920 MAP_SHARED
| MAP_ANONYMOUS
| MAP_FIXED
, -1, 0);
2921 if (new_block
->host
== MAP_FAILED
) {
2922 fprintf(stderr
, "Allocating RAM failed\n");
2926 if (xen_mapcache_enabled()) {
2927 xen_ram_alloc(new_block
->offset
, size
);
2929 new_block
->host
= qemu_vmalloc(size
);
2932 qemu_madvise(new_block
->host
, size
, QEMU_MADV_MERGEABLE
);
2935 new_block
->length
= size
;
2937 QLIST_INSERT_HEAD(&ram_list
.blocks
, new_block
, next
);
2939 ram_list
.phys_dirty
= qemu_realloc(ram_list
.phys_dirty
,
2940 last_ram_offset() >> TARGET_PAGE_BITS
);
2941 memset(ram_list
.phys_dirty
+ (new_block
->offset
>> TARGET_PAGE_BITS
),
2942 0xff, size
>> TARGET_PAGE_BITS
);
2945 kvm_setup_guest_memory(new_block
->host
, size
);
2947 return new_block
->offset
;
2950 ram_addr_t
qemu_ram_alloc(DeviceState
*dev
, const char *name
, ram_addr_t size
)
2952 return qemu_ram_alloc_from_ptr(dev
, name
, size
, NULL
);
2955 void qemu_ram_free_from_ptr(ram_addr_t addr
)
2959 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2960 if (addr
== block
->offset
) {
2961 QLIST_REMOVE(block
, next
);
2968 void qemu_ram_free(ram_addr_t addr
)
2972 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2973 if (addr
== block
->offset
) {
2974 QLIST_REMOVE(block
, next
);
2975 if (block
->flags
& RAM_PREALLOC_MASK
) {
2977 } else if (mem_path
) {
2978 #if defined (__linux__) && !defined(TARGET_S390X)
2980 munmap(block
->host
, block
->length
);
2983 qemu_vfree(block
->host
);
2989 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2990 munmap(block
->host
, block
->length
);
2992 if (xen_mapcache_enabled()) {
2993 qemu_invalidate_entry(block
->host
);
2995 qemu_vfree(block
->host
);
3007 void qemu_ram_remap(ram_addr_t addr
, ram_addr_t length
)
3014 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
3015 offset
= addr
- block
->offset
;
3016 if (offset
< block
->length
) {
3017 vaddr
= block
->host
+ offset
;
3018 if (block
->flags
& RAM_PREALLOC_MASK
) {
3022 munmap(vaddr
, length
);
3024 #if defined(__linux__) && !defined(TARGET_S390X)
3027 flags
|= mem_prealloc
? MAP_POPULATE
| MAP_SHARED
:
3030 flags
|= MAP_PRIVATE
;
3032 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
3033 flags
, block
->fd
, offset
);
3035 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
3036 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
3043 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
3044 flags
|= MAP_SHARED
| MAP_ANONYMOUS
;
3045 area
= mmap(vaddr
, length
, PROT_EXEC
|PROT_READ
|PROT_WRITE
,
3048 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
3049 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
3053 if (area
!= vaddr
) {
3054 fprintf(stderr
, "Could not remap addr: %lx@%lx\n",
3058 qemu_madvise(vaddr
, length
, QEMU_MADV_MERGEABLE
);
3064 #endif /* !_WIN32 */
3066 /* Return a host pointer to ram allocated with qemu_ram_alloc.
3067 With the exception of the softmmu code in this file, this should
3068 only be used for local memory (e.g. video ram) that the device owns,
3069 and knows it isn't going to access beyond the end of the block.
3071 It should not be used for general purpose DMA.
3072 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
3074 void *qemu_get_ram_ptr(ram_addr_t addr
)
3078 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
3079 if (addr
- block
->offset
< block
->length
) {
3080 /* Move this entry to to start of the list. */
3081 if (block
!= QLIST_FIRST(&ram_list
.blocks
)) {
3082 QLIST_REMOVE(block
, next
);
3083 QLIST_INSERT_HEAD(&ram_list
.blocks
, block
, next
);
3085 if (xen_mapcache_enabled()) {
3086 /* We need to check if the requested address is in the RAM
3087 * because we don't want to map the entire memory in QEMU.
3089 if (block
->offset
== 0) {
3090 return qemu_map_cache(addr
, 0, 1);
3091 } else if (block
->host
== NULL
) {
3092 block
->host
= qemu_map_cache(block
->offset
, block
->length
, 1);
3095 return block
->host
+ (addr
- block
->offset
);
3099 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
3105 /* Return a host pointer to ram allocated with qemu_ram_alloc.
3106 * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
3108 void *qemu_safe_ram_ptr(ram_addr_t addr
)
3112 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
3113 if (addr
- block
->offset
< block
->length
) {
3114 if (xen_mapcache_enabled()) {
3115 /* We need to check if the requested address is in the RAM
3116 * because we don't want to map the entire memory in QEMU.
3118 if (block
->offset
== 0) {
3119 return qemu_map_cache(addr
, 0, 1);
3120 } else if (block
->host
== NULL
) {
3121 block
->host
= qemu_map_cache(block
->offset
, block
->length
, 1);
3124 return block
->host
+ (addr
- block
->offset
);
3128 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
3134 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
3135 * but takes a size argument */
3136 void *qemu_ram_ptr_length(target_phys_addr_t addr
, target_phys_addr_t
*size
)
3138 if (xen_mapcache_enabled())
3139 return qemu_map_cache(addr
, *size
, 1);
3143 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
3144 if (addr
- block
->offset
< block
->length
) {
3145 if (addr
- block
->offset
+ *size
> block
->length
)
3146 *size
= block
->length
- addr
+ block
->offset
;
3147 return block
->host
+ (addr
- block
->offset
);
3151 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
3159 void qemu_put_ram_ptr(void *addr
)
3161 trace_qemu_put_ram_ptr(addr
);
3163 if (xen_mapcache_enabled()) {
3164 qemu_invalidate_entry(block
->host
);
3168 int qemu_ram_addr_from_host(void *ptr
, ram_addr_t
*ram_addr
)
3171 uint8_t *host
= ptr
;
3173 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
3174 /* This case append when the block is not mapped. */
3175 if (block
->host
== NULL
) {
3178 if (host
- block
->host
< block
->length
) {
3179 *ram_addr
= block
->offset
+ (host
- block
->host
);
3184 if (xen_mapcache_enabled()) {
3185 *ram_addr
= qemu_ram_addr_from_mapcache(ptr
);
3192 /* Some of the softmmu routines need to translate from a host pointer
3193 (typically a TLB entry) back to a ram offset. */
3194 ram_addr_t
qemu_ram_addr_from_host_nofail(void *ptr
)
3196 ram_addr_t ram_addr
;
3198 if (qemu_ram_addr_from_host(ptr
, &ram_addr
)) {
3199 fprintf(stderr
, "Bad ram pointer %p\n", ptr
);
3205 static uint32_t unassigned_mem_readb(void *opaque
, target_phys_addr_t addr
)
3207 #ifdef DEBUG_UNASSIGNED
3208 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
3210 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3211 do_unassigned_access(addr
, 0, 0, 0, 1);
3216 static uint32_t unassigned_mem_readw(void *opaque
, target_phys_addr_t addr
)
3218 #ifdef DEBUG_UNASSIGNED
3219 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
3221 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3222 do_unassigned_access(addr
, 0, 0, 0, 2);
3227 static uint32_t unassigned_mem_readl(void *opaque
, target_phys_addr_t addr
)
3229 #ifdef DEBUG_UNASSIGNED
3230 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
3232 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3233 do_unassigned_access(addr
, 0, 0, 0, 4);
3238 static void unassigned_mem_writeb(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
3240 #ifdef DEBUG_UNASSIGNED
3241 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
3243 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3244 do_unassigned_access(addr
, 1, 0, 0, 1);
3248 static void unassigned_mem_writew(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
3250 #ifdef DEBUG_UNASSIGNED
3251 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
3253 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3254 do_unassigned_access(addr
, 1, 0, 0, 2);
3258 static void unassigned_mem_writel(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
3260 #ifdef DEBUG_UNASSIGNED
3261 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
3263 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3264 do_unassigned_access(addr
, 1, 0, 0, 4);
3268 static CPUReadMemoryFunc
* const unassigned_mem_read
[3] = {
3269 unassigned_mem_readb
,
3270 unassigned_mem_readw
,
3271 unassigned_mem_readl
,
3274 static CPUWriteMemoryFunc
* const unassigned_mem_write
[3] = {
3275 unassigned_mem_writeb
,
3276 unassigned_mem_writew
,
3277 unassigned_mem_writel
,
3280 static void notdirty_mem_writeb(void *opaque
, target_phys_addr_t ram_addr
,
3284 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3285 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
3286 #if !defined(CONFIG_USER_ONLY)
3287 tb_invalidate_phys_page_fast(ram_addr
, 1);
3288 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3291 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
3292 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
3293 cpu_physical_memory_set_dirty_flags(ram_addr
, dirty_flags
);
3294 /* we remove the notdirty callback only if the code has been
3296 if (dirty_flags
== 0xff)
3297 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
3300 static void notdirty_mem_writew(void *opaque
, target_phys_addr_t ram_addr
,
3304 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3305 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
3306 #if !defined(CONFIG_USER_ONLY)
3307 tb_invalidate_phys_page_fast(ram_addr
, 2);
3308 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3311 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
3312 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
3313 cpu_physical_memory_set_dirty_flags(ram_addr
, dirty_flags
);
3314 /* we remove the notdirty callback only if the code has been
3316 if (dirty_flags
== 0xff)
3317 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
3320 static void notdirty_mem_writel(void *opaque
, target_phys_addr_t ram_addr
,
3324 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3325 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
3326 #if !defined(CONFIG_USER_ONLY)
3327 tb_invalidate_phys_page_fast(ram_addr
, 4);
3328 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3331 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
3332 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
3333 cpu_physical_memory_set_dirty_flags(ram_addr
, dirty_flags
);
3334 /* we remove the notdirty callback only if the code has been
3336 if (dirty_flags
== 0xff)
3337 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
3340 static CPUReadMemoryFunc
* const error_mem_read
[3] = {
3341 NULL
, /* never used */
3342 NULL
, /* never used */
3343 NULL
, /* never used */
3346 static CPUWriteMemoryFunc
* const notdirty_mem_write
[3] = {
3347 notdirty_mem_writeb
,
3348 notdirty_mem_writew
,
3349 notdirty_mem_writel
,
3352 /* Generate a debug exception if a watchpoint has been hit. */
3353 static void check_watchpoint(int offset
, int len_mask
, int flags
)
3355 CPUState
*env
= cpu_single_env
;
3356 target_ulong pc
, cs_base
;
3357 TranslationBlock
*tb
;
3362 if (env
->watchpoint_hit
) {
3363 /* We re-entered the check after replacing the TB. Now raise
3364 * the debug interrupt so that is will trigger after the
3365 * current instruction. */
3366 cpu_interrupt(env
, CPU_INTERRUPT_DEBUG
);
3369 vaddr
= (env
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
3370 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
3371 if ((vaddr
== (wp
->vaddr
& len_mask
) ||
3372 (vaddr
& wp
->len_mask
) == wp
->vaddr
) && (wp
->flags
& flags
)) {
3373 wp
->flags
|= BP_WATCHPOINT_HIT
;
3374 if (!env
->watchpoint_hit
) {
3375 env
->watchpoint_hit
= wp
;
3376 tb
= tb_find_pc(env
->mem_io_pc
);
3378 cpu_abort(env
, "check_watchpoint: could not find TB for "
3379 "pc=%p", (void *)env
->mem_io_pc
);
3381 cpu_restore_state(tb
, env
, env
->mem_io_pc
);
3382 tb_phys_invalidate(tb
, -1);
3383 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
3384 env
->exception_index
= EXCP_DEBUG
;
3386 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
3387 tb_gen_code(env
, pc
, cs_base
, cpu_flags
, 1);
3389 cpu_resume_from_signal(env
, NULL
);
3392 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
3397 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3398 so these check for a hit then pass through to the normal out-of-line
3400 static uint32_t watch_mem_readb(void *opaque
, target_phys_addr_t addr
)
3402 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_READ
);
3403 return ldub_phys(addr
);
3406 static uint32_t watch_mem_readw(void *opaque
, target_phys_addr_t addr
)
3408 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_READ
);
3409 return lduw_phys(addr
);
3412 static uint32_t watch_mem_readl(void *opaque
, target_phys_addr_t addr
)
3414 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_READ
);
3415 return ldl_phys(addr
);
3418 static void watch_mem_writeb(void *opaque
, target_phys_addr_t addr
,
3421 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_WRITE
);
3422 stb_phys(addr
, val
);
3425 static void watch_mem_writew(void *opaque
, target_phys_addr_t addr
,
3428 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_WRITE
);
3429 stw_phys(addr
, val
);
3432 static void watch_mem_writel(void *opaque
, target_phys_addr_t addr
,
3435 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_WRITE
);
3436 stl_phys(addr
, val
);
3439 static CPUReadMemoryFunc
* const watch_mem_read
[3] = {
3445 static CPUWriteMemoryFunc
* const watch_mem_write
[3] = {
3451 static inline uint32_t subpage_readlen (subpage_t
*mmio
,
3452 target_phys_addr_t addr
,
3455 unsigned int idx
= SUBPAGE_IDX(addr
);
3456 #if defined(DEBUG_SUBPAGE)
3457 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d\n", __func__
,
3458 mmio
, len
, addr
, idx
);
3461 addr
+= mmio
->region_offset
[idx
];
3462 idx
= mmio
->sub_io_index
[idx
];
3463 return io_mem_read
[idx
][len
](io_mem_opaque
[idx
], addr
);
3466 static inline void subpage_writelen (subpage_t
*mmio
, target_phys_addr_t addr
,
3467 uint32_t value
, unsigned int len
)
3469 unsigned int idx
= SUBPAGE_IDX(addr
);
3470 #if defined(DEBUG_SUBPAGE)
3471 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d value %08x\n",
3472 __func__
, mmio
, len
, addr
, idx
, value
);
3475 addr
+= mmio
->region_offset
[idx
];
3476 idx
= mmio
->sub_io_index
[idx
];
3477 io_mem_write
[idx
][len
](io_mem_opaque
[idx
], addr
, value
);
3480 static uint32_t subpage_readb (void *opaque
, target_phys_addr_t addr
)
3482 return subpage_readlen(opaque
, addr
, 0);
3485 static void subpage_writeb (void *opaque
, target_phys_addr_t addr
,
3488 subpage_writelen(opaque
, addr
, value
, 0);
3491 static uint32_t subpage_readw (void *opaque
, target_phys_addr_t addr
)
3493 return subpage_readlen(opaque
, addr
, 1);
3496 static void subpage_writew (void *opaque
, target_phys_addr_t addr
,
3499 subpage_writelen(opaque
, addr
, value
, 1);
3502 static uint32_t subpage_readl (void *opaque
, target_phys_addr_t addr
)
3504 return subpage_readlen(opaque
, addr
, 2);
3507 static void subpage_writel (void *opaque
, target_phys_addr_t addr
,
3510 subpage_writelen(opaque
, addr
, value
, 2);
3513 static CPUReadMemoryFunc
* const subpage_read
[] = {
3519 static CPUWriteMemoryFunc
* const subpage_write
[] = {
3525 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
3526 ram_addr_t memory
, ram_addr_t region_offset
)
3530 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
3532 idx
= SUBPAGE_IDX(start
);
3533 eidx
= SUBPAGE_IDX(end
);
3534 #if defined(DEBUG_SUBPAGE)
3535 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__
,
3536 mmio
, start
, end
, idx
, eidx
, memory
);
3538 if ((memory
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
)
3539 memory
= IO_MEM_UNASSIGNED
;
3540 memory
= (memory
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3541 for (; idx
<= eidx
; idx
++) {
3542 mmio
->sub_io_index
[idx
] = memory
;
3543 mmio
->region_offset
[idx
] = region_offset
;
3549 static subpage_t
*subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
3550 ram_addr_t orig_memory
,
3551 ram_addr_t region_offset
)
3556 mmio
= qemu_mallocz(sizeof(subpage_t
));
3559 subpage_memory
= cpu_register_io_memory(subpage_read
, subpage_write
, mmio
,
3560 DEVICE_NATIVE_ENDIAN
);
3561 #if defined(DEBUG_SUBPAGE)
3562 printf("%s: %p base " TARGET_FMT_plx
" len %08x %d\n", __func__
,
3563 mmio
, base
, TARGET_PAGE_SIZE
, subpage_memory
);
3565 *phys
= subpage_memory
| IO_MEM_SUBPAGE
;
3566 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
-1, orig_memory
, region_offset
);
3571 static int get_free_io_mem_idx(void)
3575 for (i
= 0; i
<IO_MEM_NB_ENTRIES
; i
++)
3576 if (!io_mem_used
[i
]) {
3580 fprintf(stderr
, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES
);
3585 * Usually, devices operate in little endian mode. There are devices out
3586 * there that operate in big endian too. Each device gets byte swapped
3587 * mmio if plugged onto a CPU that does the other endianness.
3597 typedef struct SwapEndianContainer
{
3598 CPUReadMemoryFunc
*read
[3];
3599 CPUWriteMemoryFunc
*write
[3];
3601 } SwapEndianContainer
;
3603 static uint32_t swapendian_mem_readb (void *opaque
, target_phys_addr_t addr
)
3606 SwapEndianContainer
*c
= opaque
;
3607 val
= c
->read
[0](c
->opaque
, addr
);
3611 static uint32_t swapendian_mem_readw(void *opaque
, target_phys_addr_t addr
)
3614 SwapEndianContainer
*c
= opaque
;
3615 val
= bswap16(c
->read
[1](c
->opaque
, addr
));
3619 static uint32_t swapendian_mem_readl(void *opaque
, target_phys_addr_t addr
)
3622 SwapEndianContainer
*c
= opaque
;
3623 val
= bswap32(c
->read
[2](c
->opaque
, addr
));
3627 static CPUReadMemoryFunc
* const swapendian_readfn
[3]={
3628 swapendian_mem_readb
,
3629 swapendian_mem_readw
,
3630 swapendian_mem_readl
3633 static void swapendian_mem_writeb(void *opaque
, target_phys_addr_t addr
,
3636 SwapEndianContainer
*c
= opaque
;
3637 c
->write
[0](c
->opaque
, addr
, val
);
3640 static void swapendian_mem_writew(void *opaque
, target_phys_addr_t addr
,
3643 SwapEndianContainer
*c
= opaque
;
3644 c
->write
[1](c
->opaque
, addr
, bswap16(val
));
3647 static void swapendian_mem_writel(void *opaque
, target_phys_addr_t addr
,
3650 SwapEndianContainer
*c
= opaque
;
3651 c
->write
[2](c
->opaque
, addr
, bswap32(val
));
3654 static CPUWriteMemoryFunc
* const swapendian_writefn
[3]={
3655 swapendian_mem_writeb
,
3656 swapendian_mem_writew
,
3657 swapendian_mem_writel
3660 static void swapendian_init(int io_index
)
3662 SwapEndianContainer
*c
= qemu_malloc(sizeof(SwapEndianContainer
));
3665 /* Swap mmio for big endian targets */
3666 c
->opaque
= io_mem_opaque
[io_index
];
3667 for (i
= 0; i
< 3; i
++) {
3668 c
->read
[i
] = io_mem_read
[io_index
][i
];
3669 c
->write
[i
] = io_mem_write
[io_index
][i
];
3671 io_mem_read
[io_index
][i
] = swapendian_readfn
[i
];
3672 io_mem_write
[io_index
][i
] = swapendian_writefn
[i
];
3674 io_mem_opaque
[io_index
] = c
;
3677 static void swapendian_del(int io_index
)
3679 if (io_mem_read
[io_index
][0] == swapendian_readfn
[0]) {
3680 qemu_free(io_mem_opaque
[io_index
]);
3684 /* mem_read and mem_write are arrays of functions containing the
3685 function to access byte (index 0), word (index 1) and dword (index
3686 2). Functions can be omitted with a NULL function pointer.
3687 If io_index is non zero, the corresponding io zone is
3688 modified. If it is zero, a new io zone is allocated. The return
3689 value can be used with cpu_register_physical_memory(). (-1) is
3690 returned if error. */
3691 static int cpu_register_io_memory_fixed(int io_index
,
3692 CPUReadMemoryFunc
* const *mem_read
,
3693 CPUWriteMemoryFunc
* const *mem_write
,
3694 void *opaque
, enum device_endian endian
)
3698 if (io_index
<= 0) {
3699 io_index
= get_free_io_mem_idx();
3703 io_index
>>= IO_MEM_SHIFT
;
3704 if (io_index
>= IO_MEM_NB_ENTRIES
)
3708 for (i
= 0; i
< 3; ++i
) {
3709 io_mem_read
[io_index
][i
]
3710 = (mem_read
[i
] ? mem_read
[i
] : unassigned_mem_read
[i
]);
3712 for (i
= 0; i
< 3; ++i
) {
3713 io_mem_write
[io_index
][i
]
3714 = (mem_write
[i
] ? mem_write
[i
] : unassigned_mem_write
[i
]);
3716 io_mem_opaque
[io_index
] = opaque
;
3719 case DEVICE_BIG_ENDIAN
:
3720 #ifndef TARGET_WORDS_BIGENDIAN
3721 swapendian_init(io_index
);
3724 case DEVICE_LITTLE_ENDIAN
:
3725 #ifdef TARGET_WORDS_BIGENDIAN
3726 swapendian_init(io_index
);
3729 case DEVICE_NATIVE_ENDIAN
:
3734 return (io_index
<< IO_MEM_SHIFT
);
3737 int cpu_register_io_memory(CPUReadMemoryFunc
* const *mem_read
,
3738 CPUWriteMemoryFunc
* const *mem_write
,
3739 void *opaque
, enum device_endian endian
)
3741 return cpu_register_io_memory_fixed(0, mem_read
, mem_write
, opaque
, endian
);
3744 void cpu_unregister_io_memory(int io_table_address
)
3747 int io_index
= io_table_address
>> IO_MEM_SHIFT
;
3749 swapendian_del(io_index
);
3751 for (i
=0;i
< 3; i
++) {
3752 io_mem_read
[io_index
][i
] = unassigned_mem_read
[i
];
3753 io_mem_write
[io_index
][i
] = unassigned_mem_write
[i
];
3755 io_mem_opaque
[io_index
] = NULL
;
3756 io_mem_used
[io_index
] = 0;
3759 static void io_mem_init(void)
3763 cpu_register_io_memory_fixed(IO_MEM_ROM
, error_mem_read
,
3764 unassigned_mem_write
, NULL
,
3765 DEVICE_NATIVE_ENDIAN
);
3766 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED
, unassigned_mem_read
,
3767 unassigned_mem_write
, NULL
,
3768 DEVICE_NATIVE_ENDIAN
);
3769 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY
, error_mem_read
,
3770 notdirty_mem_write
, NULL
,
3771 DEVICE_NATIVE_ENDIAN
);
3775 io_mem_watch
= cpu_register_io_memory(watch_mem_read
,
3776 watch_mem_write
, NULL
,
3777 DEVICE_NATIVE_ENDIAN
);
3780 #endif /* !defined(CONFIG_USER_ONLY) */
3782 /* physical memory access (slow version, mainly for debug) */
3783 #if defined(CONFIG_USER_ONLY)
3784 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
3785 uint8_t *buf
, int len
, int is_write
)
3792 page
= addr
& TARGET_PAGE_MASK
;
3793 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3796 flags
= page_get_flags(page
);
3797 if (!(flags
& PAGE_VALID
))
3800 if (!(flags
& PAGE_WRITE
))
3802 /* XXX: this code should not depend on lock_user */
3803 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
3806 unlock_user(p
, addr
, l
);
3808 if (!(flags
& PAGE_READ
))
3810 /* XXX: this code should not depend on lock_user */
3811 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
3814 unlock_user(p
, addr
, 0);
3824 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
3825 int len
, int is_write
)
3830 target_phys_addr_t page
;
3835 page
= addr
& TARGET_PAGE_MASK
;
3836 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3839 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3841 pd
= IO_MEM_UNASSIGNED
;
3843 pd
= p
->phys_offset
;
3847 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3848 target_phys_addr_t addr1
= addr
;
3849 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3851 addr1
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3852 /* XXX: could force cpu_single_env to NULL to avoid
3854 if (l
>= 4 && ((addr1
& 3) == 0)) {
3855 /* 32 bit write access */
3857 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr1
, val
);
3859 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3860 /* 16 bit write access */
3862 io_mem_write
[io_index
][1](io_mem_opaque
[io_index
], addr1
, val
);
3865 /* 8 bit write access */
3867 io_mem_write
[io_index
][0](io_mem_opaque
[io_index
], addr1
, val
);
3871 unsigned long addr1
;
3872 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3874 ptr
= qemu_get_ram_ptr(addr1
);
3875 memcpy(ptr
, buf
, l
);
3876 if (!cpu_physical_memory_is_dirty(addr1
)) {
3877 /* invalidate code */
3878 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3880 cpu_physical_memory_set_dirty_flags(
3881 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
3883 qemu_put_ram_ptr(ptr
);
3886 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3887 !(pd
& IO_MEM_ROMD
)) {
3888 target_phys_addr_t addr1
= addr
;
3890 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3892 addr1
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3893 if (l
>= 4 && ((addr1
& 3) == 0)) {
3894 /* 32 bit read access */
3895 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr1
);
3898 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3899 /* 16 bit read access */
3900 val
= io_mem_read
[io_index
][1](io_mem_opaque
[io_index
], addr1
);
3904 /* 8 bit read access */
3905 val
= io_mem_read
[io_index
][0](io_mem_opaque
[io_index
], addr1
);
3911 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
);
3912 memcpy(buf
, ptr
+ (addr
& ~TARGET_PAGE_MASK
), l
);
3913 qemu_put_ram_ptr(ptr
);
3922 /* used for ROM loading : can write in RAM and ROM */
3923 void cpu_physical_memory_write_rom(target_phys_addr_t addr
,
3924 const uint8_t *buf
, int len
)
3928 target_phys_addr_t page
;
3933 page
= addr
& TARGET_PAGE_MASK
;
3934 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3937 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3939 pd
= IO_MEM_UNASSIGNED
;
3941 pd
= p
->phys_offset
;
3944 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
&&
3945 (pd
& ~TARGET_PAGE_MASK
) != IO_MEM_ROM
&&
3946 !(pd
& IO_MEM_ROMD
)) {
3949 unsigned long addr1
;
3950 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3952 ptr
= qemu_get_ram_ptr(addr1
);
3953 memcpy(ptr
, buf
, l
);
3954 qemu_put_ram_ptr(ptr
);
3964 target_phys_addr_t addr
;
3965 target_phys_addr_t len
;
3968 static BounceBuffer bounce
;
3970 typedef struct MapClient
{
3972 void (*callback
)(void *opaque
);
3973 QLIST_ENTRY(MapClient
) link
;
3976 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
3977 = QLIST_HEAD_INITIALIZER(map_client_list
);
3979 void *cpu_register_map_client(void *opaque
, void (*callback
)(void *opaque
))
3981 MapClient
*client
= qemu_malloc(sizeof(*client
));
3983 client
->opaque
= opaque
;
3984 client
->callback
= callback
;
3985 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
3989 void cpu_unregister_map_client(void *_client
)
3991 MapClient
*client
= (MapClient
*)_client
;
3993 QLIST_REMOVE(client
, link
);
3997 static void cpu_notify_map_clients(void)
4001 while (!QLIST_EMPTY(&map_client_list
)) {
4002 client
= QLIST_FIRST(&map_client_list
);
4003 client
->callback(client
->opaque
);
4004 cpu_unregister_map_client(client
);
4008 /* Map a physical memory region into a host virtual address.
4009 * May map a subset of the requested range, given by and returned in *plen.
4010 * May return NULL if resources needed to perform the mapping are exhausted.
4011 * Use only for reads OR writes - not for read-modify-write operations.
4012 * Use cpu_register_map_client() to know when retrying the map operation is
4013 * likely to succeed.
4015 void *cpu_physical_memory_map(target_phys_addr_t addr
,
4016 target_phys_addr_t
*plen
,
4019 target_phys_addr_t len
= *plen
;
4020 target_phys_addr_t todo
= 0;
4022 target_phys_addr_t page
;
4025 target_phys_addr_t addr1
= addr
;
4028 page
= addr
& TARGET_PAGE_MASK
;
4029 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
4032 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
4034 pd
= IO_MEM_UNASSIGNED
;
4036 pd
= p
->phys_offset
;
4039 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
4040 if (todo
|| bounce
.buffer
) {
4043 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, TARGET_PAGE_SIZE
);
4047 cpu_physical_memory_read(addr
, bounce
.buffer
, l
);
4051 return bounce
.buffer
;
4059 return qemu_ram_ptr_length(addr1
, plen
);
4062 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
4063 * Will also mark the memory as dirty if is_write == 1. access_len gives
4064 * the amount of memory that was actually read or written by the caller.
4066 void cpu_physical_memory_unmap(void *buffer
, target_phys_addr_t len
,
4067 int is_write
, target_phys_addr_t access_len
)
4069 if (buffer
!= bounce
.buffer
) {
4071 ram_addr_t addr1
= qemu_ram_addr_from_host_nofail(buffer
);
4072 while (access_len
) {
4074 l
= TARGET_PAGE_SIZE
;
4077 if (!cpu_physical_memory_is_dirty(addr1
)) {
4078 /* invalidate code */
4079 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
4081 cpu_physical_memory_set_dirty_flags(
4082 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
4088 if (xen_mapcache_enabled()) {
4089 uint8_t *buffer1
= buffer
;
4090 uint8_t *end_buffer
= buffer
+ len
;
4092 while (buffer1
< end_buffer
) {
4093 qemu_put_ram_ptr(buffer1
);
4094 buffer1
+= TARGET_PAGE_SIZE
;
4100 cpu_physical_memory_write(bounce
.addr
, bounce
.buffer
, access_len
);
4102 qemu_vfree(bounce
.buffer
);
4103 bounce
.buffer
= NULL
;
4104 cpu_notify_map_clients();
4107 /* warning: addr must be aligned */
4108 uint32_t ldl_phys(target_phys_addr_t addr
)
4116 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4118 pd
= IO_MEM_UNASSIGNED
;
4120 pd
= p
->phys_offset
;
4123 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
4124 !(pd
& IO_MEM_ROMD
)) {
4126 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
4128 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
4129 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
4132 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
4133 (addr
& ~TARGET_PAGE_MASK
);
4139 /* warning: addr must be aligned */
4140 uint64_t ldq_phys(target_phys_addr_t addr
)
4148 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4150 pd
= IO_MEM_UNASSIGNED
;
4152 pd
= p
->phys_offset
;
4155 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
4156 !(pd
& IO_MEM_ROMD
)) {
4158 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
4160 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
4161 #ifdef TARGET_WORDS_BIGENDIAN
4162 val
= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
) << 32;
4163 val
|= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4);
4165 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
4166 val
|= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4) << 32;
4170 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
4171 (addr
& ~TARGET_PAGE_MASK
);
4178 uint32_t ldub_phys(target_phys_addr_t addr
)
4181 cpu_physical_memory_read(addr
, &val
, 1);
4185 /* warning: addr must be aligned */
4186 uint32_t lduw_phys(target_phys_addr_t addr
)
4194 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4196 pd
= IO_MEM_UNASSIGNED
;
4198 pd
= p
->phys_offset
;
4201 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
4202 !(pd
& IO_MEM_ROMD
)) {
4204 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
4206 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
4207 val
= io_mem_read
[io_index
][1](io_mem_opaque
[io_index
], addr
);
4210 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
4211 (addr
& ~TARGET_PAGE_MASK
);
4217 /* warning: addr must be aligned. The ram page is not masked as dirty
4218 and the code inside is not invalidated. It is useful if the dirty
4219 bits are used to track modified PTEs */
4220 void stl_phys_notdirty(target_phys_addr_t addr
, uint32_t val
)
4227 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4229 pd
= IO_MEM_UNASSIGNED
;
4231 pd
= p
->phys_offset
;
4234 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
4235 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
4237 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
4238 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
4240 unsigned long addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
4241 ptr
= qemu_get_ram_ptr(addr1
);
4244 if (unlikely(in_migration
)) {
4245 if (!cpu_physical_memory_is_dirty(addr1
)) {
4246 /* invalidate code */
4247 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
4249 cpu_physical_memory_set_dirty_flags(
4250 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
4256 void stq_phys_notdirty(target_phys_addr_t addr
, uint64_t val
)
4263 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4265 pd
= IO_MEM_UNASSIGNED
;
4267 pd
= p
->phys_offset
;
4270 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
4271 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
4273 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
4274 #ifdef TARGET_WORDS_BIGENDIAN
4275 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
>> 32);
4276 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
);
4278 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
4279 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
>> 32);
4282 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
4283 (addr
& ~TARGET_PAGE_MASK
);
4288 /* warning: addr must be aligned */
4289 void stl_phys(target_phys_addr_t addr
, uint32_t val
)
4296 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4298 pd
= IO_MEM_UNASSIGNED
;
4300 pd
= p
->phys_offset
;
4303 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
4304 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
4306 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
4307 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
4309 unsigned long addr1
;
4310 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
4312 ptr
= qemu_get_ram_ptr(addr1
);
4314 if (!cpu_physical_memory_is_dirty(addr1
)) {
4315 /* invalidate code */
4316 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
4318 cpu_physical_memory_set_dirty_flags(addr1
,
4319 (0xff & ~CODE_DIRTY_FLAG
));
4325 void stb_phys(target_phys_addr_t addr
, uint32_t val
)
4328 cpu_physical_memory_write(addr
, &v
, 1);
4331 /* warning: addr must be aligned */
4332 void stw_phys(target_phys_addr_t addr
, uint32_t val
)
4339 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4341 pd
= IO_MEM_UNASSIGNED
;
4343 pd
= p
->phys_offset
;
4346 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
4347 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
4349 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
4350 io_mem_write
[io_index
][1](io_mem_opaque
[io_index
], addr
, val
);
4352 unsigned long addr1
;
4353 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
4355 ptr
= qemu_get_ram_ptr(addr1
);
4357 if (!cpu_physical_memory_is_dirty(addr1
)) {
4358 /* invalidate code */
4359 tb_invalidate_phys_page_range(addr1
, addr1
+ 2, 0);
4361 cpu_physical_memory_set_dirty_flags(addr1
,
4362 (0xff & ~CODE_DIRTY_FLAG
));
4368 void stq_phys(target_phys_addr_t addr
, uint64_t val
)
4371 cpu_physical_memory_write(addr
, &val
, 8);
4374 /* virtual memory access for debug (includes writing to ROM) */
4375 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
4376 uint8_t *buf
, int len
, int is_write
)
4379 target_phys_addr_t phys_addr
;
4383 page
= addr
& TARGET_PAGE_MASK
;
4384 phys_addr
= cpu_get_phys_page_debug(env
, page
);
4385 /* if no physical page mapped, return an error */
4386 if (phys_addr
== -1)
4388 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
4391 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
4393 cpu_physical_memory_write_rom(phys_addr
, buf
, l
);
4395 cpu_physical_memory_rw(phys_addr
, buf
, l
, is_write
);
4404 /* in deterministic execution mode, instructions doing device I/Os
4405 must be at the end of the TB */
4406 void cpu_io_recompile(CPUState
*env
, void *retaddr
)
4408 TranslationBlock
*tb
;
4410 target_ulong pc
, cs_base
;
4413 tb
= tb_find_pc((unsigned long)retaddr
);
4415 cpu_abort(env
, "cpu_io_recompile: could not find TB for pc=%p",
4418 n
= env
->icount_decr
.u16
.low
+ tb
->icount
;
4419 cpu_restore_state(tb
, env
, (unsigned long)retaddr
);
4420 /* Calculate how many instructions had been executed before the fault
4422 n
= n
- env
->icount_decr
.u16
.low
;
4423 /* Generate a new TB ending on the I/O insn. */
4425 /* On MIPS and SH, delay slot instructions can only be restarted if
4426 they were already the first instruction in the TB. If this is not
4427 the first instruction in a TB then re-execute the preceding
4429 #if defined(TARGET_MIPS)
4430 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
4431 env
->active_tc
.PC
-= 4;
4432 env
->icount_decr
.u16
.low
++;
4433 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
4435 #elif defined(TARGET_SH4)
4436 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
4439 env
->icount_decr
.u16
.low
++;
4440 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
4443 /* This should never happen. */
4444 if (n
> CF_COUNT_MASK
)
4445 cpu_abort(env
, "TB too big during recompile");
4447 cflags
= n
| CF_LAST_IO
;
4449 cs_base
= tb
->cs_base
;
4451 tb_phys_invalidate(tb
, -1);
4452 /* FIXME: In theory this could raise an exception. In practice
4453 we have already translated the block once so it's probably ok. */
4454 tb_gen_code(env
, pc
, cs_base
, flags
, cflags
);
4455 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
4456 the first in the TB) then we end up generating a whole new TB and
4457 repeating the fault, which is horribly inefficient.
4458 Better would be to execute just this insn uncached, or generate a
4460 cpu_resume_from_signal(env
, NULL
);
4463 #if !defined(CONFIG_USER_ONLY)
4465 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
4467 int i
, target_code_size
, max_target_code_size
;
4468 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
4469 TranslationBlock
*tb
;
4471 target_code_size
= 0;
4472 max_target_code_size
= 0;
4474 direct_jmp_count
= 0;
4475 direct_jmp2_count
= 0;
4476 for(i
= 0; i
< nb_tbs
; i
++) {
4478 target_code_size
+= tb
->size
;
4479 if (tb
->size
> max_target_code_size
)
4480 max_target_code_size
= tb
->size
;
4481 if (tb
->page_addr
[1] != -1)
4483 if (tb
->tb_next_offset
[0] != 0xffff) {
4485 if (tb
->tb_next_offset
[1] != 0xffff) {
4486 direct_jmp2_count
++;
4490 /* XXX: avoid using doubles ? */
4491 cpu_fprintf(f
, "Translation buffer state:\n");
4492 cpu_fprintf(f
, "gen code size %td/%ld\n",
4493 code_gen_ptr
- code_gen_buffer
, code_gen_buffer_max_size
);
4494 cpu_fprintf(f
, "TB count %d/%d\n",
4495 nb_tbs
, code_gen_max_blocks
);
4496 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
4497 nb_tbs
? target_code_size
/ nb_tbs
: 0,
4498 max_target_code_size
);
4499 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
4500 nb_tbs
? (code_gen_ptr
- code_gen_buffer
) / nb_tbs
: 0,
4501 target_code_size
? (double) (code_gen_ptr
- code_gen_buffer
) / target_code_size
: 0);
4502 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n",
4504 nb_tbs
? (cross_page
* 100) / nb_tbs
: 0);
4505 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
4507 nb_tbs
? (direct_jmp_count
* 100) / nb_tbs
: 0,
4509 nb_tbs
? (direct_jmp2_count
* 100) / nb_tbs
: 0);
4510 cpu_fprintf(f
, "\nStatistics:\n");
4511 cpu_fprintf(f
, "TB flush count %d\n", tb_flush_count
);
4512 cpu_fprintf(f
, "TB invalidate count %d\n", tb_phys_invalidate_count
);
4513 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
4514 tcg_dump_info(f
, cpu_fprintf
);
4517 #define MMUSUFFIX _cmmu
4518 #define GETPC() NULL
4519 #define env cpu_single_env
4520 #define SOFTMMU_CODE_ACCESS
4523 #include "softmmu_template.h"
4526 #include "softmmu_template.h"
4529 #include "softmmu_template.h"
4532 #include "softmmu_template.h"