2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
23 #include <sys/types.h>
36 #include "qemu-common.h"
41 #include "qemu-timer.h"
42 #if defined(CONFIG_USER_ONLY)
45 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
46 #include <sys/param.h>
47 #if __FreeBSD_version >= 700104
48 #define HAVE_KINFO_GETVMMAP
49 #define sigqueue sigqueue_freebsd /* avoid redefinition */
52 #include <machine/profile.h>
62 //#define DEBUG_TB_INVALIDATE
65 //#define DEBUG_UNASSIGNED
67 /* make various TB consistency checks */
68 //#define DEBUG_TB_CHECK
69 //#define DEBUG_TLB_CHECK
71 //#define DEBUG_IOPORT
72 //#define DEBUG_SUBPAGE
74 #if !defined(CONFIG_USER_ONLY)
75 /* TB consistency checks only implemented for usermode emulation. */
79 #define SMC_BITMAP_USE_THRESHOLD 10
81 static TranslationBlock
*tbs
;
82 int code_gen_max_blocks
;
83 TranslationBlock
*tb_phys_hash
[CODE_GEN_PHYS_HASH_SIZE
];
85 /* any access to the tbs or the page table must use this lock */
86 spinlock_t tb_lock
= SPIN_LOCK_UNLOCKED
;
88 #if defined(__arm__) || defined(__sparc_v9__)
89 /* The prologue must be reachable with a direct jump. ARM and Sparc64
90 have limited branch ranges (possibly also PPC) so place it in a
91 section close to code segment. */
92 #define code_gen_section \
93 __attribute__((__section__(".gen_code"))) \
94 __attribute__((aligned (32)))
96 /* Maximum alignment for Win32 is 16. */
97 #define code_gen_section \
98 __attribute__((aligned (16)))
100 #define code_gen_section \
101 __attribute__((aligned (32)))
104 uint8_t code_gen_prologue
[1024] code_gen_section
;
105 static uint8_t *code_gen_buffer
;
106 static unsigned long code_gen_buffer_size
;
107 /* threshold to flush the translated code buffer */
108 static unsigned long code_gen_buffer_max_size
;
109 uint8_t *code_gen_ptr
;
111 #if !defined(CONFIG_USER_ONLY)
113 uint8_t *phys_ram_dirty
;
114 static int in_migration
;
116 typedef struct RAMBlock
{
120 struct RAMBlock
*next
;
123 static RAMBlock
*ram_blocks
;
124 /* TODO: When we implement (and use) ram deallocation (e.g. for hotplug)
125 then we can no longer assume contiguous ram offsets, and external uses
126 of this variable will break. */
127 ram_addr_t last_ram_offset
;
131 /* current CPU in the current thread. It is only valid inside
133 CPUState
*cpu_single_env
;
134 /* 0 = Do not count executed instructions.
135 1 = Precise instruction counting.
136 2 = Adaptive rate instruction counting. */
138 /* Current instruction counter. While executing translated code this may
139 include some instructions that have not yet been executed. */
142 typedef struct PageDesc
{
143 /* list of TBs intersecting this ram page */
144 TranslationBlock
*first_tb
;
145 /* in order to optimize self modifying code, we count the number
146 of lookups we do to a given page to use a bitmap */
147 unsigned int code_write_count
;
148 uint8_t *code_bitmap
;
149 #if defined(CONFIG_USER_ONLY)
154 /* In system mode we want L1_MAP to be based on ram offsets,
155 while in user mode we want it to be based on virtual addresses. */
156 #if !defined(CONFIG_USER_ONLY)
157 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
158 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
160 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
163 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
166 /* Size of the L2 (and L3, etc) page tables. */
168 #define L2_SIZE (1 << L2_BITS)
170 /* The bits remaining after N lower levels of page tables. */
171 #define P_L1_BITS_REM \
172 ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
173 #define V_L1_BITS_REM \
174 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
176 /* Size of the L1 page table. Avoid silly small sizes. */
177 #if P_L1_BITS_REM < 4
178 #define P_L1_BITS (P_L1_BITS_REM + L2_BITS)
180 #define P_L1_BITS P_L1_BITS_REM
183 #if V_L1_BITS_REM < 4
184 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
186 #define V_L1_BITS V_L1_BITS_REM
189 #define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS)
190 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
192 #define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
193 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
195 unsigned long qemu_real_host_page_size
;
196 unsigned long qemu_host_page_bits
;
197 unsigned long qemu_host_page_size
;
198 unsigned long qemu_host_page_mask
;
200 /* This is a multi-level map on the virtual address space.
201 The bottom level has pointers to PageDesc. */
202 static void *l1_map
[V_L1_SIZE
];
204 #if !defined(CONFIG_USER_ONLY)
205 typedef struct PhysPageDesc
{
206 /* offset in host memory of the page + io_index in the low bits */
207 ram_addr_t phys_offset
;
208 ram_addr_t region_offset
;
211 /* This is a multi-level map on the physical address space.
212 The bottom level has pointers to PhysPageDesc. */
213 static void *l1_phys_map
[P_L1_SIZE
];
215 static void io_mem_init(void);
217 /* io memory support */
218 CPUWriteMemoryFunc
*io_mem_write
[IO_MEM_NB_ENTRIES
][4];
219 CPUReadMemoryFunc
*io_mem_read
[IO_MEM_NB_ENTRIES
][4];
220 void *io_mem_opaque
[IO_MEM_NB_ENTRIES
];
221 static char io_mem_used
[IO_MEM_NB_ENTRIES
];
222 static int io_mem_watch
;
227 static const char *logfilename
= "qemu.log";
229 static const char *logfilename
= "/tmp/qemu.log";
233 static int log_append
= 0;
236 #if !defined(CONFIG_USER_ONLY)
237 static int tlb_flush_count
;
239 static int tb_flush_count
;
240 static int tb_phys_invalidate_count
;
243 static void map_exec(void *addr
, long size
)
246 VirtualProtect(addr
, size
,
247 PAGE_EXECUTE_READWRITE
, &old_protect
);
251 static void map_exec(void *addr
, long size
)
253 unsigned long start
, end
, page_size
;
255 page_size
= getpagesize();
256 start
= (unsigned long)addr
;
257 start
&= ~(page_size
- 1);
259 end
= (unsigned long)addr
+ size
;
260 end
+= page_size
- 1;
261 end
&= ~(page_size
- 1);
263 mprotect((void *)start
, end
- start
,
264 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
268 static void page_init(void)
270 /* NOTE: we can always suppose that qemu_host_page_size >=
274 SYSTEM_INFO system_info
;
276 GetSystemInfo(&system_info
);
277 qemu_real_host_page_size
= system_info
.dwPageSize
;
280 qemu_real_host_page_size
= getpagesize();
282 if (qemu_host_page_size
== 0)
283 qemu_host_page_size
= qemu_real_host_page_size
;
284 if (qemu_host_page_size
< TARGET_PAGE_SIZE
)
285 qemu_host_page_size
= TARGET_PAGE_SIZE
;
286 qemu_host_page_bits
= 0;
287 while ((1 << qemu_host_page_bits
) < qemu_host_page_size
)
288 qemu_host_page_bits
++;
289 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
291 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
293 #ifdef HAVE_KINFO_GETVMMAP
294 struct kinfo_vmentry
*freep
;
297 freep
= kinfo_getvmmap(getpid(), &cnt
);
300 for (i
= 0; i
< cnt
; i
++) {
301 unsigned long startaddr
, endaddr
;
303 startaddr
= freep
[i
].kve_start
;
304 endaddr
= freep
[i
].kve_end
;
305 if (h2g_valid(startaddr
)) {
306 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
308 if (h2g_valid(endaddr
)) {
309 endaddr
= h2g(endaddr
);
310 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
312 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
314 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
325 last_brk
= (unsigned long)sbrk(0);
327 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
328 f
= fopen("/compat/linux/proc/self/maps", "r");
330 f
= fopen("/proc/self/maps", "r");
336 unsigned long startaddr
, endaddr
;
339 n
= fscanf (f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
341 if (n
== 2 && h2g_valid(startaddr
)) {
342 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
344 if (h2g_valid(endaddr
)) {
345 endaddr
= h2g(endaddr
);
349 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
361 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
367 #if defined(CONFIG_USER_ONLY)
368 /* We can't use qemu_malloc because it may recurse into a locked mutex.
369 Neither can we record the new pages we reserve while allocating a
370 given page because that may recurse into an unallocated page table
371 entry. Stuff the allocations we do make into a queue and process
372 them after having completed one entire page table allocation. */
374 unsigned long reserve
[2 * (V_L1_SHIFT
/ L2_BITS
)];
377 # define ALLOC(P, SIZE) \
379 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
380 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
381 if (h2g_valid(P)) { \
382 reserve[reserve_idx] = h2g(P); \
383 reserve[reserve_idx + 1] = SIZE; \
388 # define ALLOC(P, SIZE) \
389 do { P = qemu_mallocz(SIZE); } while (0)
392 /* Level 1. Always allocated. */
393 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
396 for (i
= V_L1_SHIFT
/ L2_BITS
- 1; i
> 0; i
--) {
403 ALLOC(p
, sizeof(void *) * L2_SIZE
);
407 lp
= p
+ ((index
>> (i
* L2_BITS
)) & (L2_SIZE
- 1));
415 ALLOC(pd
, sizeof(PageDesc
) * L2_SIZE
);
420 #if defined(CONFIG_USER_ONLY)
421 for (i
= 0; i
< reserve_idx
; i
+= 2) {
422 unsigned long addr
= reserve
[i
];
423 unsigned long len
= reserve
[i
+ 1];
425 page_set_flags(addr
& TARGET_PAGE_MASK
,
426 TARGET_PAGE_ALIGN(addr
+ len
),
431 return pd
+ (index
& (L2_SIZE
- 1));
434 static inline PageDesc
*page_find(tb_page_addr_t index
)
436 return page_find_alloc(index
, 0);
439 #if !defined(CONFIG_USER_ONLY)
440 static PhysPageDesc
*phys_page_find_alloc(target_phys_addr_t index
, int alloc
)
446 /* Level 1. Always allocated. */
447 lp
= l1_phys_map
+ ((index
>> P_L1_SHIFT
) & (P_L1_SIZE
- 1));
450 for (i
= P_L1_SHIFT
/ L2_BITS
- 1; i
> 0; i
--) {
456 *lp
= p
= qemu_mallocz(sizeof(void *) * L2_SIZE
);
458 lp
= p
+ ((index
>> (i
* L2_BITS
)) & (L2_SIZE
- 1));
469 *lp
= pd
= qemu_malloc(sizeof(PhysPageDesc
) * L2_SIZE
);
471 for (i
= 0; i
< L2_SIZE
; i
++) {
472 pd
[i
].phys_offset
= IO_MEM_UNASSIGNED
;
473 pd
[i
].region_offset
= (index
+ i
) << TARGET_PAGE_BITS
;
477 return pd
+ (index
& (L2_SIZE
- 1));
480 static inline PhysPageDesc
*phys_page_find(target_phys_addr_t index
)
482 return phys_page_find_alloc(index
, 0);
485 static void tlb_protect_code(ram_addr_t ram_addr
);
486 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
488 #define mmap_lock() do { } while(0)
489 #define mmap_unlock() do { } while(0)
492 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
494 #if defined(CONFIG_USER_ONLY)
495 /* Currently it is not recommended to allocate big chunks of data in
496 user mode. It will change when a dedicated libc will be used */
497 #define USE_STATIC_CODE_GEN_BUFFER
500 #ifdef USE_STATIC_CODE_GEN_BUFFER
501 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
];
504 static void code_gen_alloc(unsigned long tb_size
)
506 #ifdef USE_STATIC_CODE_GEN_BUFFER
507 code_gen_buffer
= static_code_gen_buffer
;
508 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
509 map_exec(code_gen_buffer
, code_gen_buffer_size
);
511 code_gen_buffer_size
= tb_size
;
512 if (code_gen_buffer_size
== 0) {
513 #if defined(CONFIG_USER_ONLY)
514 /* in user mode, phys_ram_size is not meaningful */
515 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
517 /* XXX: needs adjustments */
518 code_gen_buffer_size
= (unsigned long)(ram_size
/ 4);
521 if (code_gen_buffer_size
< MIN_CODE_GEN_BUFFER_SIZE
)
522 code_gen_buffer_size
= MIN_CODE_GEN_BUFFER_SIZE
;
523 /* The code gen buffer location may have constraints depending on
524 the host cpu and OS */
525 #if defined(__linux__)
530 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
531 #if defined(__x86_64__)
533 /* Cannot map more than that */
534 if (code_gen_buffer_size
> (800 * 1024 * 1024))
535 code_gen_buffer_size
= (800 * 1024 * 1024);
536 #elif defined(__sparc_v9__)
537 // Map the buffer below 2G, so we can use direct calls and branches
539 start
= (void *) 0x60000000UL
;
540 if (code_gen_buffer_size
> (512 * 1024 * 1024))
541 code_gen_buffer_size
= (512 * 1024 * 1024);
542 #elif defined(__arm__)
543 /* Map the buffer below 32M, so we can use direct calls and branches */
545 start
= (void *) 0x01000000UL
;
546 if (code_gen_buffer_size
> 16 * 1024 * 1024)
547 code_gen_buffer_size
= 16 * 1024 * 1024;
549 code_gen_buffer
= mmap(start
, code_gen_buffer_size
,
550 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
552 if (code_gen_buffer
== MAP_FAILED
) {
553 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
557 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
561 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
562 #if defined(__x86_64__)
563 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
564 * 0x40000000 is free */
566 addr
= (void *)0x40000000;
567 /* Cannot map more than that */
568 if (code_gen_buffer_size
> (800 * 1024 * 1024))
569 code_gen_buffer_size
= (800 * 1024 * 1024);
571 code_gen_buffer
= mmap(addr
, code_gen_buffer_size
,
572 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
574 if (code_gen_buffer
== MAP_FAILED
) {
575 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
580 code_gen_buffer
= qemu_malloc(code_gen_buffer_size
);
581 map_exec(code_gen_buffer
, code_gen_buffer_size
);
583 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
584 map_exec(code_gen_prologue
, sizeof(code_gen_prologue
));
585 code_gen_buffer_max_size
= code_gen_buffer_size
-
586 code_gen_max_block_size();
587 code_gen_max_blocks
= code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
588 tbs
= qemu_malloc(code_gen_max_blocks
* sizeof(TranslationBlock
));
591 /* Must be called before using the QEMU cpus. 'tb_size' is the size
592 (in bytes) allocated to the translation buffer. Zero means default
594 void cpu_exec_init_all(unsigned long tb_size
)
597 code_gen_alloc(tb_size
);
598 code_gen_ptr
= code_gen_buffer
;
600 #if !defined(CONFIG_USER_ONLY)
605 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
607 static int cpu_common_post_load(void *opaque
, int version_id
)
609 CPUState
*env
= opaque
;
611 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
612 version_id is increased. */
613 env
->interrupt_request
&= ~0x01;
619 static const VMStateDescription vmstate_cpu_common
= {
620 .name
= "cpu_common",
622 .minimum_version_id
= 1,
623 .minimum_version_id_old
= 1,
624 .post_load
= cpu_common_post_load
,
625 .fields
= (VMStateField
[]) {
626 VMSTATE_UINT32(halted
, CPUState
),
627 VMSTATE_UINT32(interrupt_request
, CPUState
),
628 VMSTATE_END_OF_LIST()
633 CPUState
*qemu_get_cpu(int cpu
)
635 CPUState
*env
= first_cpu
;
638 if (env
->cpu_index
== cpu
)
646 void cpu_exec_init(CPUState
*env
)
651 #if defined(CONFIG_USER_ONLY)
654 env
->next_cpu
= NULL
;
657 while (*penv
!= NULL
) {
658 penv
= &(*penv
)->next_cpu
;
661 env
->cpu_index
= cpu_index
;
663 QTAILQ_INIT(&env
->breakpoints
);
664 QTAILQ_INIT(&env
->watchpoints
);
666 #if defined(CONFIG_USER_ONLY)
669 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
670 vmstate_register(cpu_index
, &vmstate_cpu_common
, env
);
671 register_savevm("cpu", cpu_index
, CPU_SAVE_VERSION
,
672 cpu_save
, cpu_load
, env
);
676 static inline void invalidate_page_bitmap(PageDesc
*p
)
678 if (p
->code_bitmap
) {
679 qemu_free(p
->code_bitmap
);
680 p
->code_bitmap
= NULL
;
682 p
->code_write_count
= 0;
685 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
687 static void page_flush_tb_1 (int level
, void **lp
)
696 for (i
= 0; i
< L2_SIZE
; ++i
) {
697 pd
[i
].first_tb
= NULL
;
698 invalidate_page_bitmap(pd
+ i
);
702 for (i
= 0; i
< L2_SIZE
; ++i
) {
703 page_flush_tb_1 (level
- 1, pp
+ i
);
708 static void page_flush_tb(void)
711 for (i
= 0; i
< V_L1_SIZE
; i
++) {
712 page_flush_tb_1(V_L1_SHIFT
/ L2_BITS
- 1, l1_map
+ i
);
716 /* flush all the translation blocks */
717 /* XXX: tb_flush is currently not thread safe */
718 void tb_flush(CPUState
*env1
)
721 #if defined(DEBUG_FLUSH)
722 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
723 (unsigned long)(code_gen_ptr
- code_gen_buffer
),
725 ((unsigned long)(code_gen_ptr
- code_gen_buffer
)) / nb_tbs
: 0);
727 if ((unsigned long)(code_gen_ptr
- code_gen_buffer
) > code_gen_buffer_size
)
728 cpu_abort(env1
, "Internal error: code buffer overflow\n");
732 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
733 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
736 memset (tb_phys_hash
, 0, CODE_GEN_PHYS_HASH_SIZE
* sizeof (void *));
739 code_gen_ptr
= code_gen_buffer
;
740 /* XXX: flush processor icache at this point if cache flush is
745 #ifdef DEBUG_TB_CHECK
747 static void tb_invalidate_check(target_ulong address
)
749 TranslationBlock
*tb
;
751 address
&= TARGET_PAGE_MASK
;
752 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
753 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
754 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
755 address
>= tb
->pc
+ tb
->size
)) {
756 printf("ERROR invalidate: address=" TARGET_FMT_lx
757 " PC=%08lx size=%04x\n",
758 address
, (long)tb
->pc
, tb
->size
);
764 /* verify that all the pages have correct rights for code */
765 static void tb_page_check(void)
767 TranslationBlock
*tb
;
768 int i
, flags1
, flags2
;
770 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
771 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
772 flags1
= page_get_flags(tb
->pc
);
773 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
774 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
775 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
776 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
784 /* invalidate one TB */
785 static inline void tb_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
,
788 TranslationBlock
*tb1
;
792 *ptb
= *(TranslationBlock
**)((char *)tb1
+ next_offset
);
795 ptb
= (TranslationBlock
**)((char *)tb1
+ next_offset
);
799 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
801 TranslationBlock
*tb1
;
807 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
809 *ptb
= tb1
->page_next
[n1
];
812 ptb
= &tb1
->page_next
[n1
];
816 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
818 TranslationBlock
*tb1
, **ptb
;
821 ptb
= &tb
->jmp_next
[n
];
824 /* find tb(n) in circular list */
828 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
829 if (n1
== n
&& tb1
== tb
)
832 ptb
= &tb1
->jmp_first
;
834 ptb
= &tb1
->jmp_next
[n1
];
837 /* now we can suppress tb(n) from the list */
838 *ptb
= tb
->jmp_next
[n
];
840 tb
->jmp_next
[n
] = NULL
;
844 /* reset the jump entry 'n' of a TB so that it is not chained to
846 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
848 tb_set_jmp_target(tb
, n
, (unsigned long)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
851 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
856 tb_page_addr_t phys_pc
;
857 TranslationBlock
*tb1
, *tb2
;
859 /* remove the TB from the hash list */
860 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
861 h
= tb_phys_hash_func(phys_pc
);
862 tb_remove(&tb_phys_hash
[h
], tb
,
863 offsetof(TranslationBlock
, phys_hash_next
));
865 /* remove the TB from the page list */
866 if (tb
->page_addr
[0] != page_addr
) {
867 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
868 tb_page_remove(&p
->first_tb
, tb
);
869 invalidate_page_bitmap(p
);
871 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
872 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
873 tb_page_remove(&p
->first_tb
, tb
);
874 invalidate_page_bitmap(p
);
877 tb_invalidated_flag
= 1;
879 /* remove the TB from the hash list */
880 h
= tb_jmp_cache_hash_func(tb
->pc
);
881 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
882 if (env
->tb_jmp_cache
[h
] == tb
)
883 env
->tb_jmp_cache
[h
] = NULL
;
886 /* suppress this TB from the two jump lists */
887 tb_jmp_remove(tb
, 0);
888 tb_jmp_remove(tb
, 1);
890 /* suppress any remaining jumps to this TB */
896 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
897 tb2
= tb1
->jmp_next
[n1
];
898 tb_reset_jump(tb1
, n1
);
899 tb1
->jmp_next
[n1
] = NULL
;
902 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2); /* fail safe */
904 tb_phys_invalidate_count
++;
907 static inline void set_bits(uint8_t *tab
, int start
, int len
)
913 mask
= 0xff << (start
& 7);
914 if ((start
& ~7) == (end
& ~7)) {
916 mask
&= ~(0xff << (end
& 7));
921 start
= (start
+ 8) & ~7;
923 while (start
< end1
) {
928 mask
= ~(0xff << (end
& 7));
934 static void build_page_bitmap(PageDesc
*p
)
936 int n
, tb_start
, tb_end
;
937 TranslationBlock
*tb
;
939 p
->code_bitmap
= qemu_mallocz(TARGET_PAGE_SIZE
/ 8);
944 tb
= (TranslationBlock
*)((long)tb
& ~3);
945 /* NOTE: this is subtle as a TB may span two physical pages */
947 /* NOTE: tb_end may be after the end of the page, but
948 it is not a problem */
949 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
950 tb_end
= tb_start
+ tb
->size
;
951 if (tb_end
> TARGET_PAGE_SIZE
)
952 tb_end
= TARGET_PAGE_SIZE
;
955 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
957 set_bits(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
958 tb
= tb
->page_next
[n
];
962 TranslationBlock
*tb_gen_code(CPUState
*env
,
963 target_ulong pc
, target_ulong cs_base
,
964 int flags
, int cflags
)
966 TranslationBlock
*tb
;
968 tb_page_addr_t phys_pc
, phys_page2
;
969 target_ulong virt_page2
;
972 phys_pc
= get_page_addr_code(env
, pc
);
975 /* flush must be done */
977 /* cannot fail at this point */
979 /* Don't forget to invalidate previous TB info. */
980 tb_invalidated_flag
= 1;
982 tc_ptr
= code_gen_ptr
;
984 tb
->cs_base
= cs_base
;
987 cpu_gen_code(env
, tb
, &code_gen_size
);
988 code_gen_ptr
= (void *)(((unsigned long)code_gen_ptr
+ code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
990 /* check next page if needed */
991 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
993 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
994 phys_page2
= get_page_addr_code(env
, virt_page2
);
996 tb_link_page(tb
, phys_pc
, phys_page2
);
1000 /* invalidate all TBs which intersect with the target physical page
1001 starting in range [start;end[. NOTE: start and end must refer to
1002 the same physical page. 'is_cpu_write_access' should be true if called
1003 from a real cpu write access: the virtual CPU will exit the current
1004 TB if code is modified inside this TB. */
1005 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1006 int is_cpu_write_access
)
1008 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
1009 CPUState
*env
= cpu_single_env
;
1010 tb_page_addr_t tb_start
, tb_end
;
1013 #ifdef TARGET_HAS_PRECISE_SMC
1014 int current_tb_not_found
= is_cpu_write_access
;
1015 TranslationBlock
*current_tb
= NULL
;
1016 int current_tb_modified
= 0;
1017 target_ulong current_pc
= 0;
1018 target_ulong current_cs_base
= 0;
1019 int current_flags
= 0;
1020 #endif /* TARGET_HAS_PRECISE_SMC */
1022 p
= page_find(start
>> TARGET_PAGE_BITS
);
1025 if (!p
->code_bitmap
&&
1026 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
&&
1027 is_cpu_write_access
) {
1028 /* build code bitmap */
1029 build_page_bitmap(p
);
1032 /* we remove all the TBs in the range [start, end[ */
1033 /* XXX: see if in some cases it could be faster to invalidate all the code */
1035 while (tb
!= NULL
) {
1037 tb
= (TranslationBlock
*)((long)tb
& ~3);
1038 tb_next
= tb
->page_next
[n
];
1039 /* NOTE: this is subtle as a TB may span two physical pages */
1041 /* NOTE: tb_end may be after the end of the page, but
1042 it is not a problem */
1043 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1044 tb_end
= tb_start
+ tb
->size
;
1046 tb_start
= tb
->page_addr
[1];
1047 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1049 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1050 #ifdef TARGET_HAS_PRECISE_SMC
1051 if (current_tb_not_found
) {
1052 current_tb_not_found
= 0;
1054 if (env
->mem_io_pc
) {
1055 /* now we have a real cpu fault */
1056 current_tb
= tb_find_pc(env
->mem_io_pc
);
1059 if (current_tb
== tb
&&
1060 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1061 /* If we are modifying the current TB, we must stop
1062 its execution. We could be more precise by checking
1063 that the modification is after the current PC, but it
1064 would require a specialized function to partially
1065 restore the CPU state */
1067 current_tb_modified
= 1;
1068 cpu_restore_state(current_tb
, env
,
1069 env
->mem_io_pc
, NULL
);
1070 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1073 #endif /* TARGET_HAS_PRECISE_SMC */
1074 /* we need to do that to handle the case where a signal
1075 occurs while doing tb_phys_invalidate() */
1078 saved_tb
= env
->current_tb
;
1079 env
->current_tb
= NULL
;
1081 tb_phys_invalidate(tb
, -1);
1083 env
->current_tb
= saved_tb
;
1084 if (env
->interrupt_request
&& env
->current_tb
)
1085 cpu_interrupt(env
, env
->interrupt_request
);
1090 #if !defined(CONFIG_USER_ONLY)
1091 /* if no code remaining, no need to continue to use slow writes */
1093 invalidate_page_bitmap(p
);
1094 if (is_cpu_write_access
) {
1095 tlb_unprotect_code_phys(env
, start
, env
->mem_io_vaddr
);
1099 #ifdef TARGET_HAS_PRECISE_SMC
1100 if (current_tb_modified
) {
1101 /* we generate a block containing just the instruction
1102 modifying the memory. It will ensure that it cannot modify
1104 env
->current_tb
= NULL
;
1105 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1106 cpu_resume_from_signal(env
, NULL
);
1111 /* len must be <= 8 and start must be a multiple of len */
1112 static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1118 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1119 cpu_single_env
->mem_io_vaddr
, len
,
1120 cpu_single_env
->eip
,
1121 cpu_single_env
->eip
+ (long)cpu_single_env
->segs
[R_CS
].base
);
1124 p
= page_find(start
>> TARGET_PAGE_BITS
);
1127 if (p
->code_bitmap
) {
1128 offset
= start
& ~TARGET_PAGE_MASK
;
1129 b
= p
->code_bitmap
[offset
>> 3] >> (offset
& 7);
1130 if (b
& ((1 << len
) - 1))
1134 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1138 #if !defined(CONFIG_SOFTMMU)
1139 static void tb_invalidate_phys_page(tb_page_addr_t addr
,
1140 unsigned long pc
, void *puc
)
1142 TranslationBlock
*tb
;
1145 #ifdef TARGET_HAS_PRECISE_SMC
1146 TranslationBlock
*current_tb
= NULL
;
1147 CPUState
*env
= cpu_single_env
;
1148 int current_tb_modified
= 0;
1149 target_ulong current_pc
= 0;
1150 target_ulong current_cs_base
= 0;
1151 int current_flags
= 0;
1154 addr
&= TARGET_PAGE_MASK
;
1155 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1159 #ifdef TARGET_HAS_PRECISE_SMC
1160 if (tb
&& pc
!= 0) {
1161 current_tb
= tb_find_pc(pc
);
1164 while (tb
!= NULL
) {
1166 tb
= (TranslationBlock
*)((long)tb
& ~3);
1167 #ifdef TARGET_HAS_PRECISE_SMC
1168 if (current_tb
== tb
&&
1169 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1170 /* If we are modifying the current TB, we must stop
1171 its execution. We could be more precise by checking
1172 that the modification is after the current PC, but it
1173 would require a specialized function to partially
1174 restore the CPU state */
1176 current_tb_modified
= 1;
1177 cpu_restore_state(current_tb
, env
, pc
, puc
);
1178 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1181 #endif /* TARGET_HAS_PRECISE_SMC */
1182 tb_phys_invalidate(tb
, addr
);
1183 tb
= tb
->page_next
[n
];
1186 #ifdef TARGET_HAS_PRECISE_SMC
1187 if (current_tb_modified
) {
1188 /* we generate a block containing just the instruction
1189 modifying the memory. It will ensure that it cannot modify
1191 env
->current_tb
= NULL
;
1192 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1193 cpu_resume_from_signal(env
, puc
);
1199 /* add the tb in the target page and protect it if necessary */
1200 static inline void tb_alloc_page(TranslationBlock
*tb
,
1201 unsigned int n
, tb_page_addr_t page_addr
)
1204 TranslationBlock
*last_first_tb
;
1206 tb
->page_addr
[n
] = page_addr
;
1207 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1208 tb
->page_next
[n
] = p
->first_tb
;
1209 last_first_tb
= p
->first_tb
;
1210 p
->first_tb
= (TranslationBlock
*)((long)tb
| n
);
1211 invalidate_page_bitmap(p
);
1213 #if defined(TARGET_HAS_SMC) || 1
1215 #if defined(CONFIG_USER_ONLY)
1216 if (p
->flags
& PAGE_WRITE
) {
1221 /* force the host page as non writable (writes will have a
1222 page fault + mprotect overhead) */
1223 page_addr
&= qemu_host_page_mask
;
1225 for(addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1226 addr
+= TARGET_PAGE_SIZE
) {
1228 p2
= page_find (addr
>> TARGET_PAGE_BITS
);
1232 p2
->flags
&= ~PAGE_WRITE
;
1234 mprotect(g2h(page_addr
), qemu_host_page_size
,
1235 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1236 #ifdef DEBUG_TB_INVALIDATE
1237 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1242 /* if some code is already present, then the pages are already
1243 protected. So we handle the case where only the first TB is
1244 allocated in a physical page */
1245 if (!last_first_tb
) {
1246 tlb_protect_code(page_addr
);
1250 #endif /* TARGET_HAS_SMC */
1253 /* Allocate a new translation block. Flush the translation buffer if
1254 too many translation blocks or too much generated code. */
1255 TranslationBlock
*tb_alloc(target_ulong pc
)
1257 TranslationBlock
*tb
;
1259 if (nb_tbs
>= code_gen_max_blocks
||
1260 (code_gen_ptr
- code_gen_buffer
) >= code_gen_buffer_max_size
)
1262 tb
= &tbs
[nb_tbs
++];
1268 void tb_free(TranslationBlock
*tb
)
1270 /* In practice this is mostly used for single use temporary TB
1271 Ignore the hard cases and just back up if this TB happens to
1272 be the last one generated. */
1273 if (nb_tbs
> 0 && tb
== &tbs
[nb_tbs
- 1]) {
1274 code_gen_ptr
= tb
->tc_ptr
;
1279 /* add a new TB and link it to the physical page tables. phys_page2 is
1280 (-1) to indicate that only one page contains the TB. */
1281 void tb_link_page(TranslationBlock
*tb
,
1282 tb_page_addr_t phys_pc
, tb_page_addr_t phys_page2
)
1285 TranslationBlock
**ptb
;
1287 /* Grab the mmap lock to stop another thread invalidating this TB
1288 before we are done. */
1290 /* add in the physical hash table */
1291 h
= tb_phys_hash_func(phys_pc
);
1292 ptb
= &tb_phys_hash
[h
];
1293 tb
->phys_hash_next
= *ptb
;
1296 /* add in the page list */
1297 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1298 if (phys_page2
!= -1)
1299 tb_alloc_page(tb
, 1, phys_page2
);
1301 tb
->page_addr
[1] = -1;
1303 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2);
1304 tb
->jmp_next
[0] = NULL
;
1305 tb
->jmp_next
[1] = NULL
;
1307 /* init original jump addresses */
1308 if (tb
->tb_next_offset
[0] != 0xffff)
1309 tb_reset_jump(tb
, 0);
1310 if (tb
->tb_next_offset
[1] != 0xffff)
1311 tb_reset_jump(tb
, 1);
1313 #ifdef DEBUG_TB_CHECK
1319 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1320 tb[1].tc_ptr. Return NULL if not found */
1321 TranslationBlock
*tb_find_pc(unsigned long tc_ptr
)
1323 int m_min
, m_max
, m
;
1325 TranslationBlock
*tb
;
1329 if (tc_ptr
< (unsigned long)code_gen_buffer
||
1330 tc_ptr
>= (unsigned long)code_gen_ptr
)
1332 /* binary search (cf Knuth) */
1335 while (m_min
<= m_max
) {
1336 m
= (m_min
+ m_max
) >> 1;
1338 v
= (unsigned long)tb
->tc_ptr
;
1341 else if (tc_ptr
< v
) {
1350 static void tb_reset_jump_recursive(TranslationBlock
*tb
);
1352 static inline void tb_reset_jump_recursive2(TranslationBlock
*tb
, int n
)
1354 TranslationBlock
*tb1
, *tb_next
, **ptb
;
1357 tb1
= tb
->jmp_next
[n
];
1359 /* find head of list */
1362 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1365 tb1
= tb1
->jmp_next
[n1
];
1367 /* we are now sure now that tb jumps to tb1 */
1370 /* remove tb from the jmp_first list */
1371 ptb
= &tb_next
->jmp_first
;
1375 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1376 if (n1
== n
&& tb1
== tb
)
1378 ptb
= &tb1
->jmp_next
[n1
];
1380 *ptb
= tb
->jmp_next
[n
];
1381 tb
->jmp_next
[n
] = NULL
;
1383 /* suppress the jump to next tb in generated code */
1384 tb_reset_jump(tb
, n
);
1386 /* suppress jumps in the tb on which we could have jumped */
1387 tb_reset_jump_recursive(tb_next
);
1391 static void tb_reset_jump_recursive(TranslationBlock
*tb
)
1393 tb_reset_jump_recursive2(tb
, 0);
1394 tb_reset_jump_recursive2(tb
, 1);
1397 #if defined(TARGET_HAS_ICE)
1398 #if defined(CONFIG_USER_ONLY)
1399 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1401 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
1404 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1406 target_phys_addr_t addr
;
1408 ram_addr_t ram_addr
;
1411 addr
= cpu_get_phys_page_debug(env
, pc
);
1412 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
1414 pd
= IO_MEM_UNASSIGNED
;
1416 pd
= p
->phys_offset
;
1418 ram_addr
= (pd
& TARGET_PAGE_MASK
) | (pc
& ~TARGET_PAGE_MASK
);
1419 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1422 #endif /* TARGET_HAS_ICE */
1424 #if defined(CONFIG_USER_ONLY)
1425 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1430 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1431 int flags
, CPUWatchpoint
**watchpoint
)
1436 /* Add a watchpoint. */
1437 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1438 int flags
, CPUWatchpoint
**watchpoint
)
1440 target_ulong len_mask
= ~(len
- 1);
1443 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1444 if ((len
!= 1 && len
!= 2 && len
!= 4 && len
!= 8) || (addr
& ~len_mask
)) {
1445 fprintf(stderr
, "qemu: tried to set invalid watchpoint at "
1446 TARGET_FMT_lx
", len=" TARGET_FMT_lu
"\n", addr
, len
);
1449 wp
= qemu_malloc(sizeof(*wp
));
1452 wp
->len_mask
= len_mask
;
1455 /* keep all GDB-injected watchpoints in front */
1457 QTAILQ_INSERT_HEAD(&env
->watchpoints
, wp
, entry
);
1459 QTAILQ_INSERT_TAIL(&env
->watchpoints
, wp
, entry
);
1461 tlb_flush_page(env
, addr
);
1468 /* Remove a specific watchpoint. */
1469 int cpu_watchpoint_remove(CPUState
*env
, target_ulong addr
, target_ulong len
,
1472 target_ulong len_mask
= ~(len
- 1);
1475 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1476 if (addr
== wp
->vaddr
&& len_mask
== wp
->len_mask
1477 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
1478 cpu_watchpoint_remove_by_ref(env
, wp
);
1485 /* Remove a specific watchpoint by reference. */
1486 void cpu_watchpoint_remove_by_ref(CPUState
*env
, CPUWatchpoint
*watchpoint
)
1488 QTAILQ_REMOVE(&env
->watchpoints
, watchpoint
, entry
);
1490 tlb_flush_page(env
, watchpoint
->vaddr
);
1492 qemu_free(watchpoint
);
1495 /* Remove all matching watchpoints. */
1496 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1498 CPUWatchpoint
*wp
, *next
;
1500 QTAILQ_FOREACH_SAFE(wp
, &env
->watchpoints
, entry
, next
) {
1501 if (wp
->flags
& mask
)
1502 cpu_watchpoint_remove_by_ref(env
, wp
);
1507 /* Add a breakpoint. */
1508 int cpu_breakpoint_insert(CPUState
*env
, target_ulong pc
, int flags
,
1509 CPUBreakpoint
**breakpoint
)
1511 #if defined(TARGET_HAS_ICE)
1514 bp
= qemu_malloc(sizeof(*bp
));
1519 /* keep all GDB-injected breakpoints in front */
1521 QTAILQ_INSERT_HEAD(&env
->breakpoints
, bp
, entry
);
1523 QTAILQ_INSERT_TAIL(&env
->breakpoints
, bp
, entry
);
1525 breakpoint_invalidate(env
, pc
);
1535 /* Remove a specific breakpoint. */
1536 int cpu_breakpoint_remove(CPUState
*env
, target_ulong pc
, int flags
)
1538 #if defined(TARGET_HAS_ICE)
1541 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1542 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
1543 cpu_breakpoint_remove_by_ref(env
, bp
);
1553 /* Remove a specific breakpoint by reference. */
1554 void cpu_breakpoint_remove_by_ref(CPUState
*env
, CPUBreakpoint
*breakpoint
)
1556 #if defined(TARGET_HAS_ICE)
1557 QTAILQ_REMOVE(&env
->breakpoints
, breakpoint
, entry
);
1559 breakpoint_invalidate(env
, breakpoint
->pc
);
1561 qemu_free(breakpoint
);
1565 /* Remove all matching breakpoints. */
1566 void cpu_breakpoint_remove_all(CPUState
*env
, int mask
)
1568 #if defined(TARGET_HAS_ICE)
1569 CPUBreakpoint
*bp
, *next
;
1571 QTAILQ_FOREACH_SAFE(bp
, &env
->breakpoints
, entry
, next
) {
1572 if (bp
->flags
& mask
)
1573 cpu_breakpoint_remove_by_ref(env
, bp
);
1578 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1579 CPU loop after each instruction */
1580 void cpu_single_step(CPUState
*env
, int enabled
)
1582 #if defined(TARGET_HAS_ICE)
1583 if (env
->singlestep_enabled
!= enabled
) {
1584 env
->singlestep_enabled
= enabled
;
1586 kvm_update_guest_debug(env
, 0);
1588 /* must flush all the translated code to avoid inconsistencies */
1589 /* XXX: only flush what is necessary */
1596 /* enable or disable low levels log */
1597 void cpu_set_log(int log_flags
)
1599 loglevel
= log_flags
;
1600 if (loglevel
&& !logfile
) {
1601 logfile
= fopen(logfilename
, log_append
? "a" : "w");
1603 perror(logfilename
);
1606 #if !defined(CONFIG_SOFTMMU)
1607 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1609 static char logfile_buf
[4096];
1610 setvbuf(logfile
, logfile_buf
, _IOLBF
, sizeof(logfile_buf
));
1612 #elif !defined(_WIN32)
1613 /* Win32 doesn't support line-buffering and requires size >= 2 */
1614 setvbuf(logfile
, NULL
, _IOLBF
, 0);
1618 if (!loglevel
&& logfile
) {
1624 void cpu_set_log_filename(const char *filename
)
1626 logfilename
= strdup(filename
);
1631 cpu_set_log(loglevel
);
1634 static void cpu_unlink_tb(CPUState
*env
)
1636 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1637 problem and hope the cpu will stop of its own accord. For userspace
1638 emulation this often isn't actually as bad as it sounds. Often
1639 signals are used primarily to interrupt blocking syscalls. */
1640 TranslationBlock
*tb
;
1641 static spinlock_t interrupt_lock
= SPIN_LOCK_UNLOCKED
;
1643 spin_lock(&interrupt_lock
);
1644 tb
= env
->current_tb
;
1645 /* if the cpu is currently executing code, we must unlink it and
1646 all the potentially executing TB */
1648 env
->current_tb
= NULL
;
1649 tb_reset_jump_recursive(tb
);
1651 spin_unlock(&interrupt_lock
);
1654 /* mask must never be zero, except for A20 change call */
1655 void cpu_interrupt(CPUState
*env
, int mask
)
1659 old_mask
= env
->interrupt_request
;
1660 env
->interrupt_request
|= mask
;
1662 #ifndef CONFIG_USER_ONLY
1664 * If called from iothread context, wake the target cpu in
1667 if (!qemu_cpu_self(env
)) {
1674 env
->icount_decr
.u16
.high
= 0xffff;
1675 #ifndef CONFIG_USER_ONLY
1677 && (mask
& ~old_mask
) != 0) {
1678 cpu_abort(env
, "Raised interrupt while not in I/O function");
1686 void cpu_reset_interrupt(CPUState
*env
, int mask
)
1688 env
->interrupt_request
&= ~mask
;
1691 void cpu_exit(CPUState
*env
)
1693 env
->exit_request
= 1;
1697 const CPULogItem cpu_log_items
[] = {
1698 { CPU_LOG_TB_OUT_ASM
, "out_asm",
1699 "show generated host assembly code for each compiled TB" },
1700 { CPU_LOG_TB_IN_ASM
, "in_asm",
1701 "show target assembly code for each compiled TB" },
1702 { CPU_LOG_TB_OP
, "op",
1703 "show micro ops for each compiled TB" },
1704 { CPU_LOG_TB_OP_OPT
, "op_opt",
1707 "before eflags optimization and "
1709 "after liveness analysis" },
1710 { CPU_LOG_INT
, "int",
1711 "show interrupts/exceptions in short format" },
1712 { CPU_LOG_EXEC
, "exec",
1713 "show trace before each executed TB (lots of logs)" },
1714 { CPU_LOG_TB_CPU
, "cpu",
1715 "show CPU state before block translation" },
1717 { CPU_LOG_PCALL
, "pcall",
1718 "show protected mode far calls/returns/exceptions" },
1719 { CPU_LOG_RESET
, "cpu_reset",
1720 "show CPU state before CPU resets" },
1723 { CPU_LOG_IOPORT
, "ioport",
1724 "show all i/o ports accesses" },
1729 #ifndef CONFIG_USER_ONLY
1730 static QLIST_HEAD(memory_client_list
, CPUPhysMemoryClient
) memory_client_list
1731 = QLIST_HEAD_INITIALIZER(memory_client_list
);
1733 static void cpu_notify_set_memory(target_phys_addr_t start_addr
,
1735 ram_addr_t phys_offset
)
1737 CPUPhysMemoryClient
*client
;
1738 QLIST_FOREACH(client
, &memory_client_list
, list
) {
1739 client
->set_memory(client
, start_addr
, size
, phys_offset
);
1743 static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start
,
1744 target_phys_addr_t end
)
1746 CPUPhysMemoryClient
*client
;
1747 QLIST_FOREACH(client
, &memory_client_list
, list
) {
1748 int r
= client
->sync_dirty_bitmap(client
, start
, end
);
1755 static int cpu_notify_migration_log(int enable
)
1757 CPUPhysMemoryClient
*client
;
1758 QLIST_FOREACH(client
, &memory_client_list
, list
) {
1759 int r
= client
->migration_log(client
, enable
);
1766 static void phys_page_for_each_1(CPUPhysMemoryClient
*client
,
1767 int level
, void **lp
)
1775 PhysPageDesc
*pd
= *lp
;
1776 for (i
= 0; i
< L2_SIZE
; ++i
) {
1777 if (pd
[i
].phys_offset
!= IO_MEM_UNASSIGNED
) {
1778 client
->set_memory(client
, pd
[i
].region_offset
,
1779 TARGET_PAGE_SIZE
, pd
[i
].phys_offset
);
1784 for (i
= 0; i
< L2_SIZE
; ++i
) {
1785 phys_page_for_each_1(client
, level
- 1, pp
+ i
);
1790 static void phys_page_for_each(CPUPhysMemoryClient
*client
)
1793 for (i
= 0; i
< P_L1_SIZE
; ++i
) {
1794 phys_page_for_each_1(client
, P_L1_SHIFT
/ L2_BITS
- 1,
1799 void cpu_register_phys_memory_client(CPUPhysMemoryClient
*client
)
1801 QLIST_INSERT_HEAD(&memory_client_list
, client
, list
);
1802 phys_page_for_each(client
);
1805 void cpu_unregister_phys_memory_client(CPUPhysMemoryClient
*client
)
1807 QLIST_REMOVE(client
, list
);
1811 static int cmp1(const char *s1
, int n
, const char *s2
)
1813 if (strlen(s2
) != n
)
1815 return memcmp(s1
, s2
, n
) == 0;
1818 /* takes a comma separated list of log masks. Return 0 if error. */
1819 int cpu_str_to_log_mask(const char *str
)
1821 const CPULogItem
*item
;
1828 p1
= strchr(p
, ',');
1831 if(cmp1(p
,p1
-p
,"all")) {
1832 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1836 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1837 if (cmp1(p
, p1
- p
, item
->name
))
1851 void cpu_abort(CPUState
*env
, const char *fmt
, ...)
1858 fprintf(stderr
, "qemu: fatal: ");
1859 vfprintf(stderr
, fmt
, ap
);
1860 fprintf(stderr
, "\n");
1862 cpu_dump_state(env
, stderr
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1864 cpu_dump_state(env
, stderr
, fprintf
, 0);
1866 if (qemu_log_enabled()) {
1867 qemu_log("qemu: fatal: ");
1868 qemu_log_vprintf(fmt
, ap2
);
1871 log_cpu_state(env
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1873 log_cpu_state(env
, 0);
1880 #if defined(CONFIG_USER_ONLY)
1882 struct sigaction act
;
1883 sigfillset(&act
.sa_mask
);
1884 act
.sa_handler
= SIG_DFL
;
1885 sigaction(SIGABRT
, &act
, NULL
);
1891 CPUState
*cpu_copy(CPUState
*env
)
1893 CPUState
*new_env
= cpu_init(env
->cpu_model_str
);
1894 CPUState
*next_cpu
= new_env
->next_cpu
;
1895 int cpu_index
= new_env
->cpu_index
;
1896 #if defined(TARGET_HAS_ICE)
1901 memcpy(new_env
, env
, sizeof(CPUState
));
1903 /* Preserve chaining and index. */
1904 new_env
->next_cpu
= next_cpu
;
1905 new_env
->cpu_index
= cpu_index
;
1907 /* Clone all break/watchpoints.
1908 Note: Once we support ptrace with hw-debug register access, make sure
1909 BP_CPU break/watchpoints are handled correctly on clone. */
1910 QTAILQ_INIT(&env
->breakpoints
);
1911 QTAILQ_INIT(&env
->watchpoints
);
1912 #if defined(TARGET_HAS_ICE)
1913 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1914 cpu_breakpoint_insert(new_env
, bp
->pc
, bp
->flags
, NULL
);
1916 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1917 cpu_watchpoint_insert(new_env
, wp
->vaddr
, (~wp
->len_mask
) + 1,
1925 #if !defined(CONFIG_USER_ONLY)
1927 static inline void tlb_flush_jmp_cache(CPUState
*env
, target_ulong addr
)
1931 /* Discard jump cache entries for any tb which might potentially
1932 overlap the flushed page. */
1933 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1934 memset (&env
->tb_jmp_cache
[i
], 0,
1935 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1937 i
= tb_jmp_cache_hash_page(addr
);
1938 memset (&env
->tb_jmp_cache
[i
], 0,
1939 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1942 static CPUTLBEntry s_cputlb_empty_entry
= {
1949 /* NOTE: if flush_global is true, also flush global entries (not
1951 void tlb_flush(CPUState
*env
, int flush_global
)
1955 #if defined(DEBUG_TLB)
1956 printf("tlb_flush:\n");
1958 /* must reset current TB so that interrupts cannot modify the
1959 links while we are modifying them */
1960 env
->current_tb
= NULL
;
1962 for(i
= 0; i
< CPU_TLB_SIZE
; i
++) {
1964 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1965 env
->tlb_table
[mmu_idx
][i
] = s_cputlb_empty_entry
;
1969 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
1971 env
->tlb_flush_addr
= -1;
1972 env
->tlb_flush_mask
= 0;
1976 static inline void tlb_flush_entry(CPUTLBEntry
*tlb_entry
, target_ulong addr
)
1978 if (addr
== (tlb_entry
->addr_read
&
1979 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1980 addr
== (tlb_entry
->addr_write
&
1981 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1982 addr
== (tlb_entry
->addr_code
&
1983 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
))) {
1984 *tlb_entry
= s_cputlb_empty_entry
;
1988 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
1993 #if defined(DEBUG_TLB)
1994 printf("tlb_flush_page: " TARGET_FMT_lx
"\n", addr
);
1996 /* Check if we need to flush due to large pages. */
1997 if ((addr
& env
->tlb_flush_mask
) == env
->tlb_flush_addr
) {
1998 #if defined(DEBUG_TLB)
1999 printf("tlb_flush_page: forced full flush ("
2000 TARGET_FMT_lx
"/" TARGET_FMT_lx
")\n",
2001 env
->tlb_flush_addr
, env
->tlb_flush_mask
);
2006 /* must reset current TB so that interrupts cannot modify the
2007 links while we are modifying them */
2008 env
->current_tb
= NULL
;
2010 addr
&= TARGET_PAGE_MASK
;
2011 i
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2012 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++)
2013 tlb_flush_entry(&env
->tlb_table
[mmu_idx
][i
], addr
);
2015 tlb_flush_jmp_cache(env
, addr
);
2018 /* update the TLBs so that writes to code in the virtual page 'addr'
2020 static void tlb_protect_code(ram_addr_t ram_addr
)
2022 cpu_physical_memory_reset_dirty(ram_addr
,
2023 ram_addr
+ TARGET_PAGE_SIZE
,
2027 /* update the TLB so that writes in physical page 'phys_addr' are no longer
2028 tested for self modifying code */
2029 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
2032 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] |= CODE_DIRTY_FLAG
;
2035 static inline void tlb_reset_dirty_range(CPUTLBEntry
*tlb_entry
,
2036 unsigned long start
, unsigned long length
)
2039 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
2040 addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) + tlb_entry
->addend
;
2041 if ((addr
- start
) < length
) {
2042 tlb_entry
->addr_write
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) | TLB_NOTDIRTY
;
2047 /* Note: start and end must be within the same ram block. */
2048 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t end
,
2052 unsigned long length
, start1
;
2056 start
&= TARGET_PAGE_MASK
;
2057 end
= TARGET_PAGE_ALIGN(end
);
2059 length
= end
- start
;
2062 len
= length
>> TARGET_PAGE_BITS
;
2063 mask
= ~dirty_flags
;
2064 p
= phys_ram_dirty
+ (start
>> TARGET_PAGE_BITS
);
2065 for(i
= 0; i
< len
; i
++)
2068 /* we modify the TLB cache so that the dirty bit will be set again
2069 when accessing the range */
2070 start1
= (unsigned long)qemu_get_ram_ptr(start
);
2071 /* Chek that we don't span multiple blocks - this breaks the
2072 address comparisons below. */
2073 if ((unsigned long)qemu_get_ram_ptr(end
- 1) - start1
2074 != (end
- 1) - start
) {
2078 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2080 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
2081 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
2082 tlb_reset_dirty_range(&env
->tlb_table
[mmu_idx
][i
],
2088 int cpu_physical_memory_set_dirty_tracking(int enable
)
2091 in_migration
= enable
;
2092 ret
= cpu_notify_migration_log(!!enable
);
2096 int cpu_physical_memory_get_dirty_tracking(void)
2098 return in_migration
;
2101 int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr
,
2102 target_phys_addr_t end_addr
)
2106 ret
= cpu_notify_sync_dirty_bitmap(start_addr
, end_addr
);
2110 static inline void tlb_update_dirty(CPUTLBEntry
*tlb_entry
)
2112 ram_addr_t ram_addr
;
2115 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
2116 p
= (void *)(unsigned long)((tlb_entry
->addr_write
& TARGET_PAGE_MASK
)
2117 + tlb_entry
->addend
);
2118 ram_addr
= qemu_ram_addr_from_host(p
);
2119 if (!cpu_physical_memory_is_dirty(ram_addr
)) {
2120 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
2125 /* update the TLB according to the current state of the dirty bits */
2126 void cpu_tlb_update_dirty(CPUState
*env
)
2130 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
2131 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
2132 tlb_update_dirty(&env
->tlb_table
[mmu_idx
][i
]);
2136 static inline void tlb_set_dirty1(CPUTLBEntry
*tlb_entry
, target_ulong vaddr
)
2138 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
))
2139 tlb_entry
->addr_write
= vaddr
;
2142 /* update the TLB corresponding to virtual page vaddr
2143 so that it is no longer dirty */
2144 static inline void tlb_set_dirty(CPUState
*env
, target_ulong vaddr
)
2149 vaddr
&= TARGET_PAGE_MASK
;
2150 i
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2151 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++)
2152 tlb_set_dirty1(&env
->tlb_table
[mmu_idx
][i
], vaddr
);
2155 /* Our TLB does not support large pages, so remember the area covered by
2156 large pages and trigger a full TLB flush if these are invalidated. */
2157 static void tlb_add_large_page(CPUState
*env
, target_ulong vaddr
,
2160 target_ulong mask
= ~(size
- 1);
2162 if (env
->tlb_flush_addr
== (target_ulong
)-1) {
2163 env
->tlb_flush_addr
= vaddr
& mask
;
2164 env
->tlb_flush_mask
= mask
;
2167 /* Extend the existing region to include the new page.
2168 This is a compromise between unnecessary flushes and the cost
2169 of maintaining a full variable size TLB. */
2170 mask
&= env
->tlb_flush_mask
;
2171 while (((env
->tlb_flush_addr
^ vaddr
) & mask
) != 0) {
2174 env
->tlb_flush_addr
&= mask
;
2175 env
->tlb_flush_mask
= mask
;
2178 /* Add a new TLB entry. At most one entry for a given virtual address
2179 is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2180 supplied size is only used by tlb_flush_page. */
2181 void tlb_set_page(CPUState
*env
, target_ulong vaddr
,
2182 target_phys_addr_t paddr
, int prot
,
2183 int mmu_idx
, target_ulong size
)
2188 target_ulong address
;
2189 target_ulong code_address
;
2190 target_phys_addr_t addend
;
2193 target_phys_addr_t iotlb
;
2195 assert(size
>= TARGET_PAGE_SIZE
);
2196 if (size
!= TARGET_PAGE_SIZE
) {
2197 tlb_add_large_page(env
, vaddr
, size
);
2199 p
= phys_page_find(paddr
>> TARGET_PAGE_BITS
);
2201 pd
= IO_MEM_UNASSIGNED
;
2203 pd
= p
->phys_offset
;
2205 #if defined(DEBUG_TLB)
2206 printf("tlb_set_page: vaddr=" TARGET_FMT_lx
" paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
2207 vaddr
, (int)paddr
, prot
, mmu_idx
, is_softmmu
, pd
);
2211 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&& !(pd
& IO_MEM_ROMD
)) {
2212 /* IO memory case (romd handled later) */
2213 address
|= TLB_MMIO
;
2215 addend
= (unsigned long)qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
);
2216 if ((pd
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
) {
2218 iotlb
= pd
& TARGET_PAGE_MASK
;
2219 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
)
2220 iotlb
|= IO_MEM_NOTDIRTY
;
2222 iotlb
|= IO_MEM_ROM
;
2224 /* IO handlers are currently passed a physical address.
2225 It would be nice to pass an offset from the base address
2226 of that region. This would avoid having to special case RAM,
2227 and avoid full address decoding in every device.
2228 We can't use the high bits of pd for this because
2229 IO_MEM_ROMD uses these as a ram address. */
2230 iotlb
= (pd
& ~TARGET_PAGE_MASK
);
2232 iotlb
+= p
->region_offset
;
2238 code_address
= address
;
2239 /* Make accesses to pages with watchpoints go via the
2240 watchpoint trap routines. */
2241 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
2242 if (vaddr
== (wp
->vaddr
& TARGET_PAGE_MASK
)) {
2243 iotlb
= io_mem_watch
+ paddr
;
2244 /* TODO: The memory case can be optimized by not trapping
2245 reads of pages with a write breakpoint. */
2246 address
|= TLB_MMIO
;
2250 index
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2251 env
->iotlb
[mmu_idx
][index
] = iotlb
- vaddr
;
2252 te
= &env
->tlb_table
[mmu_idx
][index
];
2253 te
->addend
= addend
- vaddr
;
2254 if (prot
& PAGE_READ
) {
2255 te
->addr_read
= address
;
2260 if (prot
& PAGE_EXEC
) {
2261 te
->addr_code
= code_address
;
2265 if (prot
& PAGE_WRITE
) {
2266 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_ROM
||
2267 (pd
& IO_MEM_ROMD
)) {
2268 /* Write access calls the I/O callback. */
2269 te
->addr_write
= address
| TLB_MMIO
;
2270 } else if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
&&
2271 !cpu_physical_memory_is_dirty(pd
)) {
2272 te
->addr_write
= address
| TLB_NOTDIRTY
;
2274 te
->addr_write
= address
;
2277 te
->addr_write
= -1;
2283 void tlb_flush(CPUState
*env
, int flush_global
)
2287 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
2292 * Walks guest process memory "regions" one by one
2293 * and calls callback function 'fn' for each region.
2296 struct walk_memory_regions_data
2298 walk_memory_regions_fn fn
;
2300 unsigned long start
;
2304 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
2305 abi_ulong end
, int new_prot
)
2307 if (data
->start
!= -1ul) {
2308 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
2314 data
->start
= (new_prot
? end
: -1ul);
2315 data
->prot
= new_prot
;
2320 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
2321 abi_ulong base
, int level
, void **lp
)
2327 return walk_memory_regions_end(data
, base
, 0);
2332 for (i
= 0; i
< L2_SIZE
; ++i
) {
2333 int prot
= pd
[i
].flags
;
2335 pa
= base
| (i
<< TARGET_PAGE_BITS
);
2336 if (prot
!= data
->prot
) {
2337 rc
= walk_memory_regions_end(data
, pa
, prot
);
2345 for (i
= 0; i
< L2_SIZE
; ++i
) {
2346 pa
= base
| ((abi_ulong
)i
<<
2347 (TARGET_PAGE_BITS
+ L2_BITS
* level
));
2348 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
2358 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
2360 struct walk_memory_regions_data data
;
2368 for (i
= 0; i
< V_L1_SIZE
; i
++) {
2369 int rc
= walk_memory_regions_1(&data
, (abi_ulong
)i
<< V_L1_SHIFT
,
2370 V_L1_SHIFT
/ L2_BITS
- 1, l1_map
+ i
);
2376 return walk_memory_regions_end(&data
, 0, 0);
2379 static int dump_region(void *priv
, abi_ulong start
,
2380 abi_ulong end
, unsigned long prot
)
2382 FILE *f
= (FILE *)priv
;
2384 (void) fprintf(f
, TARGET_ABI_FMT_lx
"-"TARGET_ABI_FMT_lx
2385 " "TARGET_ABI_FMT_lx
" %c%c%c\n",
2386 start
, end
, end
- start
,
2387 ((prot
& PAGE_READ
) ? 'r' : '-'),
2388 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
2389 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
2394 /* dump memory mappings */
2395 void page_dump(FILE *f
)
2397 (void) fprintf(f
, "%-8s %-8s %-8s %s\n",
2398 "start", "end", "size", "prot");
2399 walk_memory_regions(f
, dump_region
);
2402 int page_get_flags(target_ulong address
)
2406 p
= page_find(address
>> TARGET_PAGE_BITS
);
2412 /* Modify the flags of a page and invalidate the code if necessary.
2413 The flag PAGE_WRITE_ORG is positioned automatically depending
2414 on PAGE_WRITE. The mmap_lock should already be held. */
2415 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2417 target_ulong addr
, len
;
2419 /* This function should never be called with addresses outside the
2420 guest address space. If this assert fires, it probably indicates
2421 a missing call to h2g_valid. */
2422 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2423 assert(end
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2425 assert(start
< end
);
2427 start
= start
& TARGET_PAGE_MASK
;
2428 end
= TARGET_PAGE_ALIGN(end
);
2430 if (flags
& PAGE_WRITE
) {
2431 flags
|= PAGE_WRITE_ORG
;
2434 for (addr
= start
, len
= end
- start
;
2436 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2437 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2439 /* If the write protection bit is set, then we invalidate
2441 if (!(p
->flags
& PAGE_WRITE
) &&
2442 (flags
& PAGE_WRITE
) &&
2444 tb_invalidate_phys_page(addr
, 0, NULL
);
2450 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2456 /* This function should never be called with addresses outside the
2457 guest address space. If this assert fires, it probably indicates
2458 a missing call to h2g_valid. */
2459 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2460 assert(start
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2463 if (start
+ len
- 1 < start
) {
2464 /* We've wrapped around. */
2468 end
= TARGET_PAGE_ALIGN(start
+len
); /* must do before we loose bits in the next step */
2469 start
= start
& TARGET_PAGE_MASK
;
2471 for (addr
= start
, len
= end
- start
;
2473 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2474 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2477 if( !(p
->flags
& PAGE_VALID
) )
2480 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
))
2482 if (flags
& PAGE_WRITE
) {
2483 if (!(p
->flags
& PAGE_WRITE_ORG
))
2485 /* unprotect the page if it was put read-only because it
2486 contains translated code */
2487 if (!(p
->flags
& PAGE_WRITE
)) {
2488 if (!page_unprotect(addr
, 0, NULL
))
2497 /* called from signal handler: invalidate the code and unprotect the
2498 page. Return TRUE if the fault was successfully handled. */
2499 int page_unprotect(target_ulong address
, unsigned long pc
, void *puc
)
2503 target_ulong host_start
, host_end
, addr
;
2505 /* Technically this isn't safe inside a signal handler. However we
2506 know this only ever happens in a synchronous SEGV handler, so in
2507 practice it seems to be ok. */
2510 p
= page_find(address
>> TARGET_PAGE_BITS
);
2516 /* if the page was really writable, then we change its
2517 protection back to writable */
2518 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
2519 host_start
= address
& qemu_host_page_mask
;
2520 host_end
= host_start
+ qemu_host_page_size
;
2523 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2524 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2525 p
->flags
|= PAGE_WRITE
;
2528 /* and since the content will be modified, we must invalidate
2529 the corresponding translated code. */
2530 tb_invalidate_phys_page(addr
, pc
, puc
);
2531 #ifdef DEBUG_TB_CHECK
2532 tb_invalidate_check(addr
);
2535 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2545 static inline void tlb_set_dirty(CPUState
*env
,
2546 unsigned long addr
, target_ulong vaddr
)
2549 #endif /* defined(CONFIG_USER_ONLY) */
2551 #if !defined(CONFIG_USER_ONLY)
2553 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2554 typedef struct subpage_t
{
2555 target_phys_addr_t base
;
2556 CPUReadMemoryFunc
* const *mem_read
[TARGET_PAGE_SIZE
][4];
2557 CPUWriteMemoryFunc
* const *mem_write
[TARGET_PAGE_SIZE
][4];
2558 void *opaque
[TARGET_PAGE_SIZE
][2][4];
2559 ram_addr_t region_offset
[TARGET_PAGE_SIZE
][2][4];
2562 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2563 ram_addr_t memory
, ram_addr_t region_offset
);
2564 static void *subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
2565 ram_addr_t orig_memory
, ram_addr_t region_offset
);
2566 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2569 if (addr > start_addr) \
2572 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2573 if (start_addr2 > 0) \
2577 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2578 end_addr2 = TARGET_PAGE_SIZE - 1; \
2580 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2581 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2586 /* register physical memory.
2587 For RAM, 'size' must be a multiple of the target page size.
2588 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2589 io memory page. The address used when calling the IO function is
2590 the offset from the start of the region, plus region_offset. Both
2591 start_addr and region_offset are rounded down to a page boundary
2592 before calculating this offset. This should not be a problem unless
2593 the low bits of start_addr and region_offset differ. */
2594 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr
,
2596 ram_addr_t phys_offset
,
2597 ram_addr_t region_offset
)
2599 target_phys_addr_t addr
, end_addr
;
2602 ram_addr_t orig_size
= size
;
2605 cpu_notify_set_memory(start_addr
, size
, phys_offset
);
2607 if (phys_offset
== IO_MEM_UNASSIGNED
) {
2608 region_offset
= start_addr
;
2610 region_offset
&= TARGET_PAGE_MASK
;
2611 size
= (size
+ TARGET_PAGE_SIZE
- 1) & TARGET_PAGE_MASK
;
2612 end_addr
= start_addr
+ (target_phys_addr_t
)size
;
2613 for(addr
= start_addr
; addr
!= end_addr
; addr
+= TARGET_PAGE_SIZE
) {
2614 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2615 if (p
&& p
->phys_offset
!= IO_MEM_UNASSIGNED
) {
2616 ram_addr_t orig_memory
= p
->phys_offset
;
2617 target_phys_addr_t start_addr2
, end_addr2
;
2618 int need_subpage
= 0;
2620 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
, end_addr2
,
2622 if (need_subpage
|| phys_offset
& IO_MEM_SUBWIDTH
) {
2623 if (!(orig_memory
& IO_MEM_SUBPAGE
)) {
2624 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2625 &p
->phys_offset
, orig_memory
,
2628 subpage
= io_mem_opaque
[(orig_memory
& ~TARGET_PAGE_MASK
)
2631 subpage_register(subpage
, start_addr2
, end_addr2
, phys_offset
,
2633 p
->region_offset
= 0;
2635 p
->phys_offset
= phys_offset
;
2636 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2637 (phys_offset
& IO_MEM_ROMD
))
2638 phys_offset
+= TARGET_PAGE_SIZE
;
2641 p
= phys_page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2642 p
->phys_offset
= phys_offset
;
2643 p
->region_offset
= region_offset
;
2644 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2645 (phys_offset
& IO_MEM_ROMD
)) {
2646 phys_offset
+= TARGET_PAGE_SIZE
;
2648 target_phys_addr_t start_addr2
, end_addr2
;
2649 int need_subpage
= 0;
2651 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
,
2652 end_addr2
, need_subpage
);
2654 if (need_subpage
|| phys_offset
& IO_MEM_SUBWIDTH
) {
2655 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2656 &p
->phys_offset
, IO_MEM_UNASSIGNED
,
2657 addr
& TARGET_PAGE_MASK
);
2658 subpage_register(subpage
, start_addr2
, end_addr2
,
2659 phys_offset
, region_offset
);
2660 p
->region_offset
= 0;
2664 region_offset
+= TARGET_PAGE_SIZE
;
2667 /* since each CPU stores ram addresses in its TLB cache, we must
2668 reset the modified entries */
2670 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2675 /* XXX: temporary until new memory mapping API */
2676 ram_addr_t
cpu_get_physical_page_desc(target_phys_addr_t addr
)
2680 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2682 return IO_MEM_UNASSIGNED
;
2683 return p
->phys_offset
;
2686 void qemu_register_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2689 kvm_coalesce_mmio_region(addr
, size
);
2692 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2695 kvm_uncoalesce_mmio_region(addr
, size
);
2698 void qemu_flush_coalesced_mmio_buffer(void)
2701 kvm_flush_coalesced_mmio_buffer();
2704 #if defined(__linux__) && !defined(TARGET_S390X)
2706 #include <sys/vfs.h>
2708 #define HUGETLBFS_MAGIC 0x958458f6
2710 static long gethugepagesize(const char *path
)
2716 ret
= statfs(path
, &fs
);
2717 } while (ret
!= 0 && errno
== EINTR
);
2724 if (fs
.f_type
!= HUGETLBFS_MAGIC
)
2725 fprintf(stderr
, "Warning: path not on HugeTLBFS: %s\n", path
);
2730 static void *file_ram_alloc(ram_addr_t memory
, const char *path
)
2738 unsigned long hpagesize
;
2740 hpagesize
= gethugepagesize(path
);
2745 if (memory
< hpagesize
) {
2749 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2750 fprintf(stderr
, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2754 if (asprintf(&filename
, "%s/qemu_back_mem.XXXXXX", path
) == -1) {
2758 fd
= mkstemp(filename
);
2760 perror("unable to create backing store for hugepages");
2767 memory
= (memory
+hpagesize
-1) & ~(hpagesize
-1);
2770 * ftruncate is not supported by hugetlbfs in older
2771 * hosts, so don't bother bailing out on errors.
2772 * If anything goes wrong with it under other filesystems,
2775 if (ftruncate(fd
, memory
))
2776 perror("ftruncate");
2779 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2780 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2781 * to sidestep this quirk.
2783 flags
= mem_prealloc
? MAP_POPULATE
| MAP_SHARED
: MAP_PRIVATE
;
2784 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, flags
, fd
, 0);
2786 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, MAP_PRIVATE
, fd
, 0);
2788 if (area
== MAP_FAILED
) {
2789 perror("file_ram_alloc: can't mmap RAM pages");
2797 ram_addr_t
qemu_ram_alloc(ram_addr_t size
)
2799 RAMBlock
*new_block
;
2801 size
= TARGET_PAGE_ALIGN(size
);
2802 new_block
= qemu_malloc(sizeof(*new_block
));
2805 #if defined (__linux__) && !defined(TARGET_S390X)
2806 new_block
->host
= file_ram_alloc(size
, mem_path
);
2807 if (!new_block
->host
)
2810 fprintf(stderr
, "-mem-path option unsupported\n");
2814 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2815 /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
2816 new_block
->host
= mmap((void*)0x1000000, size
,
2817 PROT_EXEC
|PROT_READ
|PROT_WRITE
,
2818 MAP_SHARED
| MAP_ANONYMOUS
, -1, 0);
2820 new_block
->host
= qemu_vmalloc(size
);
2822 #ifdef MADV_MERGEABLE
2823 madvise(new_block
->host
, size
, MADV_MERGEABLE
);
2826 new_block
->offset
= last_ram_offset
;
2827 new_block
->length
= size
;
2829 new_block
->next
= ram_blocks
;
2830 ram_blocks
= new_block
;
2832 phys_ram_dirty
= qemu_realloc(phys_ram_dirty
,
2833 (last_ram_offset
+ size
) >> TARGET_PAGE_BITS
);
2834 memset(phys_ram_dirty
+ (last_ram_offset
>> TARGET_PAGE_BITS
),
2835 0xff, size
>> TARGET_PAGE_BITS
);
2837 last_ram_offset
+= size
;
2840 kvm_setup_guest_memory(new_block
->host
, size
);
2842 return new_block
->offset
;
2845 void qemu_ram_free(ram_addr_t addr
)
2847 /* TODO: implement this. */
2850 /* Return a host pointer to ram allocated with qemu_ram_alloc.
2851 With the exception of the softmmu code in this file, this should
2852 only be used for local memory (e.g. video ram) that the device owns,
2853 and knows it isn't going to access beyond the end of the block.
2855 It should not be used for general purpose DMA.
2856 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2858 void *qemu_get_ram_ptr(ram_addr_t addr
)
2865 prevp
= &ram_blocks
;
2867 while (block
&& (block
->offset
> addr
2868 || block
->offset
+ block
->length
<= addr
)) {
2870 prevp
= &prev
->next
;
2872 block
= block
->next
;
2875 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
2878 /* Move this entry to to start of the list. */
2880 prev
->next
= block
->next
;
2881 block
->next
= *prevp
;
2884 return block
->host
+ (addr
- block
->offset
);
2887 /* Some of the softmmu routines need to translate from a host pointer
2888 (typically a TLB entry) back to a ram offset. */
2889 ram_addr_t
qemu_ram_addr_from_host(void *ptr
)
2893 uint8_t *host
= ptr
;
2897 while (block
&& (block
->host
> host
2898 || block
->host
+ block
->length
<= host
)) {
2900 block
= block
->next
;
2903 fprintf(stderr
, "Bad ram pointer %p\n", ptr
);
2906 return block
->offset
+ (host
- block
->host
);
2909 static uint32_t unassigned_mem_readb(void *opaque
, target_phys_addr_t addr
)
2911 #ifdef DEBUG_UNASSIGNED
2912 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2914 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2915 do_unassigned_access(addr
, 0, 0, 0, 1);
2920 static uint32_t unassigned_mem_readw(void *opaque
, target_phys_addr_t addr
)
2922 #ifdef DEBUG_UNASSIGNED
2923 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2925 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2926 do_unassigned_access(addr
, 0, 0, 0, 2);
2931 static uint32_t unassigned_mem_readl(void *opaque
, target_phys_addr_t addr
)
2933 #ifdef DEBUG_UNASSIGNED
2934 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2936 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2937 do_unassigned_access(addr
, 0, 0, 0, 4);
2942 static void unassigned_mem_writeb(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2944 #ifdef DEBUG_UNASSIGNED
2945 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2947 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2948 do_unassigned_access(addr
, 1, 0, 0, 1);
2952 static void unassigned_mem_writew(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2954 #ifdef DEBUG_UNASSIGNED
2955 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2957 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2958 do_unassigned_access(addr
, 1, 0, 0, 2);
2962 static void unassigned_mem_writel(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2964 #ifdef DEBUG_UNASSIGNED
2965 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2967 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2968 do_unassigned_access(addr
, 1, 0, 0, 4);
2972 static CPUReadMemoryFunc
* const unassigned_mem_read
[3] = {
2973 unassigned_mem_readb
,
2974 unassigned_mem_readw
,
2975 unassigned_mem_readl
,
2978 static CPUWriteMemoryFunc
* const unassigned_mem_write
[3] = {
2979 unassigned_mem_writeb
,
2980 unassigned_mem_writew
,
2981 unassigned_mem_writel
,
2984 static void notdirty_mem_writeb(void *opaque
, target_phys_addr_t ram_addr
,
2988 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2989 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2990 #if !defined(CONFIG_USER_ONLY)
2991 tb_invalidate_phys_page_fast(ram_addr
, 1);
2992 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2995 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
2996 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2997 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2998 /* we remove the notdirty callback only if the code has been
3000 if (dirty_flags
== 0xff)
3001 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
3004 static void notdirty_mem_writew(void *opaque
, target_phys_addr_t ram_addr
,
3008 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
3009 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
3010 #if !defined(CONFIG_USER_ONLY)
3011 tb_invalidate_phys_page_fast(ram_addr
, 2);
3012 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
3015 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
3016 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
3017 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
3018 /* we remove the notdirty callback only if the code has been
3020 if (dirty_flags
== 0xff)
3021 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
3024 static void notdirty_mem_writel(void *opaque
, target_phys_addr_t ram_addr
,
3028 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
3029 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
3030 #if !defined(CONFIG_USER_ONLY)
3031 tb_invalidate_phys_page_fast(ram_addr
, 4);
3032 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
3035 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
3036 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
3037 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
3038 /* we remove the notdirty callback only if the code has been
3040 if (dirty_flags
== 0xff)
3041 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
3044 static CPUReadMemoryFunc
* const error_mem_read
[3] = {
3045 NULL
, /* never used */
3046 NULL
, /* never used */
3047 NULL
, /* never used */
3050 static CPUWriteMemoryFunc
* const notdirty_mem_write
[3] = {
3051 notdirty_mem_writeb
,
3052 notdirty_mem_writew
,
3053 notdirty_mem_writel
,
3056 /* Generate a debug exception if a watchpoint has been hit. */
3057 static void check_watchpoint(int offset
, int len_mask
, int flags
)
3059 CPUState
*env
= cpu_single_env
;
3060 target_ulong pc
, cs_base
;
3061 TranslationBlock
*tb
;
3066 if (env
->watchpoint_hit
) {
3067 /* We re-entered the check after replacing the TB. Now raise
3068 * the debug interrupt so that is will trigger after the
3069 * current instruction. */
3070 cpu_interrupt(env
, CPU_INTERRUPT_DEBUG
);
3073 vaddr
= (env
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
3074 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
3075 if ((vaddr
== (wp
->vaddr
& len_mask
) ||
3076 (vaddr
& wp
->len_mask
) == wp
->vaddr
) && (wp
->flags
& flags
)) {
3077 wp
->flags
|= BP_WATCHPOINT_HIT
;
3078 if (!env
->watchpoint_hit
) {
3079 env
->watchpoint_hit
= wp
;
3080 tb
= tb_find_pc(env
->mem_io_pc
);
3082 cpu_abort(env
, "check_watchpoint: could not find TB for "
3083 "pc=%p", (void *)env
->mem_io_pc
);
3085 cpu_restore_state(tb
, env
, env
->mem_io_pc
, NULL
);
3086 tb_phys_invalidate(tb
, -1);
3087 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
3088 env
->exception_index
= EXCP_DEBUG
;
3090 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
3091 tb_gen_code(env
, pc
, cs_base
, cpu_flags
, 1);
3093 cpu_resume_from_signal(env
, NULL
);
3096 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
3101 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3102 so these check for a hit then pass through to the normal out-of-line
3104 static uint32_t watch_mem_readb(void *opaque
, target_phys_addr_t addr
)
3106 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_READ
);
3107 return ldub_phys(addr
);
3110 static uint32_t watch_mem_readw(void *opaque
, target_phys_addr_t addr
)
3112 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_READ
);
3113 return lduw_phys(addr
);
3116 static uint32_t watch_mem_readl(void *opaque
, target_phys_addr_t addr
)
3118 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_READ
);
3119 return ldl_phys(addr
);
3122 static void watch_mem_writeb(void *opaque
, target_phys_addr_t addr
,
3125 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_WRITE
);
3126 stb_phys(addr
, val
);
3129 static void watch_mem_writew(void *opaque
, target_phys_addr_t addr
,
3132 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_WRITE
);
3133 stw_phys(addr
, val
);
3136 static void watch_mem_writel(void *opaque
, target_phys_addr_t addr
,
3139 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_WRITE
);
3140 stl_phys(addr
, val
);
3143 static CPUReadMemoryFunc
* const watch_mem_read
[3] = {
3149 static CPUWriteMemoryFunc
* const watch_mem_write
[3] = {
3155 static inline uint32_t subpage_readlen (subpage_t
*mmio
, target_phys_addr_t addr
,
3161 idx
= SUBPAGE_IDX(addr
);
3162 #if defined(DEBUG_SUBPAGE)
3163 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d\n", __func__
,
3164 mmio
, len
, addr
, idx
);
3166 ret
= (**mmio
->mem_read
[idx
][len
])(mmio
->opaque
[idx
][0][len
],
3167 addr
+ mmio
->region_offset
[idx
][0][len
]);
3172 static inline void subpage_writelen (subpage_t
*mmio
, target_phys_addr_t addr
,
3173 uint32_t value
, unsigned int len
)
3177 idx
= SUBPAGE_IDX(addr
);
3178 #if defined(DEBUG_SUBPAGE)
3179 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d value %08x\n", __func__
,
3180 mmio
, len
, addr
, idx
, value
);
3182 (**mmio
->mem_write
[idx
][len
])(mmio
->opaque
[idx
][1][len
],
3183 addr
+ mmio
->region_offset
[idx
][1][len
],
3187 static uint32_t subpage_readb (void *opaque
, target_phys_addr_t addr
)
3189 #if defined(DEBUG_SUBPAGE)
3190 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
3193 return subpage_readlen(opaque
, addr
, 0);
3196 static void subpage_writeb (void *opaque
, target_phys_addr_t addr
,
3199 #if defined(DEBUG_SUBPAGE)
3200 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
3202 subpage_writelen(opaque
, addr
, value
, 0);
3205 static uint32_t subpage_readw (void *opaque
, target_phys_addr_t addr
)
3207 #if defined(DEBUG_SUBPAGE)
3208 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
3211 return subpage_readlen(opaque
, addr
, 1);
3214 static void subpage_writew (void *opaque
, target_phys_addr_t addr
,
3217 #if defined(DEBUG_SUBPAGE)
3218 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
3220 subpage_writelen(opaque
, addr
, value
, 1);
3223 static uint32_t subpage_readl (void *opaque
, target_phys_addr_t addr
)
3225 #if defined(DEBUG_SUBPAGE)
3226 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
3229 return subpage_readlen(opaque
, addr
, 2);
3232 static void subpage_writel (void *opaque
,
3233 target_phys_addr_t addr
, uint32_t value
)
3235 #if defined(DEBUG_SUBPAGE)
3236 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
3238 subpage_writelen(opaque
, addr
, value
, 2);
3241 static CPUReadMemoryFunc
* const subpage_read
[] = {
3247 static CPUWriteMemoryFunc
* const subpage_write
[] = {
3253 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
3254 ram_addr_t memory
, ram_addr_t region_offset
)
3259 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
3261 idx
= SUBPAGE_IDX(start
);
3262 eidx
= SUBPAGE_IDX(end
);
3263 #if defined(DEBUG_SUBPAGE)
3264 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__
,
3265 mmio
, start
, end
, idx
, eidx
, memory
);
3267 memory
>>= IO_MEM_SHIFT
;
3268 for (; idx
<= eidx
; idx
++) {
3269 for (i
= 0; i
< 4; i
++) {
3270 if (io_mem_read
[memory
][i
]) {
3271 mmio
->mem_read
[idx
][i
] = &io_mem_read
[memory
][i
];
3272 mmio
->opaque
[idx
][0][i
] = io_mem_opaque
[memory
];
3273 mmio
->region_offset
[idx
][0][i
] = region_offset
;
3275 if (io_mem_write
[memory
][i
]) {
3276 mmio
->mem_write
[idx
][i
] = &io_mem_write
[memory
][i
];
3277 mmio
->opaque
[idx
][1][i
] = io_mem_opaque
[memory
];
3278 mmio
->region_offset
[idx
][1][i
] = region_offset
;
3286 static void *subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
3287 ram_addr_t orig_memory
, ram_addr_t region_offset
)
3292 mmio
= qemu_mallocz(sizeof(subpage_t
));
3295 subpage_memory
= cpu_register_io_memory(subpage_read
, subpage_write
, mmio
);
3296 #if defined(DEBUG_SUBPAGE)
3297 printf("%s: %p base " TARGET_FMT_plx
" len %08x %d\n", __func__
,
3298 mmio
, base
, TARGET_PAGE_SIZE
, subpage_memory
);
3300 *phys
= subpage_memory
| IO_MEM_SUBPAGE
;
3301 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
- 1, orig_memory
,
3307 static int get_free_io_mem_idx(void)
3311 for (i
= 0; i
<IO_MEM_NB_ENTRIES
; i
++)
3312 if (!io_mem_used
[i
]) {
3316 fprintf(stderr
, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES
);
3320 /* mem_read and mem_write are arrays of functions containing the
3321 function to access byte (index 0), word (index 1) and dword (index
3322 2). Functions can be omitted with a NULL function pointer.
3323 If io_index is non zero, the corresponding io zone is
3324 modified. If it is zero, a new io zone is allocated. The return
3325 value can be used with cpu_register_physical_memory(). (-1) is
3326 returned if error. */
3327 static int cpu_register_io_memory_fixed(int io_index
,
3328 CPUReadMemoryFunc
* const *mem_read
,
3329 CPUWriteMemoryFunc
* const *mem_write
,
3332 int i
, subwidth
= 0;
3334 if (io_index
<= 0) {
3335 io_index
= get_free_io_mem_idx();
3339 io_index
>>= IO_MEM_SHIFT
;
3340 if (io_index
>= IO_MEM_NB_ENTRIES
)
3344 for(i
= 0;i
< 3; i
++) {
3345 if (!mem_read
[i
] || !mem_write
[i
])
3346 subwidth
= IO_MEM_SUBWIDTH
;
3347 io_mem_read
[io_index
][i
] = mem_read
[i
];
3348 io_mem_write
[io_index
][i
] = mem_write
[i
];
3350 io_mem_opaque
[io_index
] = opaque
;
3351 return (io_index
<< IO_MEM_SHIFT
) | subwidth
;
3354 int cpu_register_io_memory(CPUReadMemoryFunc
* const *mem_read
,
3355 CPUWriteMemoryFunc
* const *mem_write
,
3358 return cpu_register_io_memory_fixed(0, mem_read
, mem_write
, opaque
);
3361 void cpu_unregister_io_memory(int io_table_address
)
3364 int io_index
= io_table_address
>> IO_MEM_SHIFT
;
3366 for (i
=0;i
< 3; i
++) {
3367 io_mem_read
[io_index
][i
] = unassigned_mem_read
[i
];
3368 io_mem_write
[io_index
][i
] = unassigned_mem_write
[i
];
3370 io_mem_opaque
[io_index
] = NULL
;
3371 io_mem_used
[io_index
] = 0;
3374 static void io_mem_init(void)
3378 cpu_register_io_memory_fixed(IO_MEM_ROM
, error_mem_read
, unassigned_mem_write
, NULL
);
3379 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED
, unassigned_mem_read
, unassigned_mem_write
, NULL
);
3380 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY
, error_mem_read
, notdirty_mem_write
, NULL
);
3384 io_mem_watch
= cpu_register_io_memory(watch_mem_read
,
3385 watch_mem_write
, NULL
);
3388 #endif /* !defined(CONFIG_USER_ONLY) */
3390 /* physical memory access (slow version, mainly for debug) */
3391 #if defined(CONFIG_USER_ONLY)
3392 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
3393 uint8_t *buf
, int len
, int is_write
)
3400 page
= addr
& TARGET_PAGE_MASK
;
3401 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3404 flags
= page_get_flags(page
);
3405 if (!(flags
& PAGE_VALID
))
3408 if (!(flags
& PAGE_WRITE
))
3410 /* XXX: this code should not depend on lock_user */
3411 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
3414 unlock_user(p
, addr
, l
);
3416 if (!(flags
& PAGE_READ
))
3418 /* XXX: this code should not depend on lock_user */
3419 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
3422 unlock_user(p
, addr
, 0);
3432 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
3433 int len
, int is_write
)
3438 target_phys_addr_t page
;
3443 page
= addr
& TARGET_PAGE_MASK
;
3444 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3447 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3449 pd
= IO_MEM_UNASSIGNED
;
3451 pd
= p
->phys_offset
;
3455 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3456 target_phys_addr_t addr1
= addr
;
3457 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3459 addr1
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3460 /* XXX: could force cpu_single_env to NULL to avoid
3462 if (l
>= 4 && ((addr1
& 3) == 0)) {
3463 /* 32 bit write access */
3465 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr1
, val
);
3467 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3468 /* 16 bit write access */
3470 io_mem_write
[io_index
][1](io_mem_opaque
[io_index
], addr1
, val
);
3473 /* 8 bit write access */
3475 io_mem_write
[io_index
][0](io_mem_opaque
[io_index
], addr1
, val
);
3479 unsigned long addr1
;
3480 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3482 ptr
= qemu_get_ram_ptr(addr1
);
3483 memcpy(ptr
, buf
, l
);
3484 if (!cpu_physical_memory_is_dirty(addr1
)) {
3485 /* invalidate code */
3486 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3488 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3489 (0xff & ~CODE_DIRTY_FLAG
);
3493 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3494 !(pd
& IO_MEM_ROMD
)) {
3495 target_phys_addr_t addr1
= addr
;
3497 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3499 addr1
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3500 if (l
>= 4 && ((addr1
& 3) == 0)) {
3501 /* 32 bit read access */
3502 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr1
);
3505 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3506 /* 16 bit read access */
3507 val
= io_mem_read
[io_index
][1](io_mem_opaque
[io_index
], addr1
);
3511 /* 8 bit read access */
3512 val
= io_mem_read
[io_index
][0](io_mem_opaque
[io_index
], addr1
);
3518 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3519 (addr
& ~TARGET_PAGE_MASK
);
3520 memcpy(buf
, ptr
, l
);
3529 /* used for ROM loading : can write in RAM and ROM */
3530 void cpu_physical_memory_write_rom(target_phys_addr_t addr
,
3531 const uint8_t *buf
, int len
)
3535 target_phys_addr_t page
;
3540 page
= addr
& TARGET_PAGE_MASK
;
3541 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3544 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3546 pd
= IO_MEM_UNASSIGNED
;
3548 pd
= p
->phys_offset
;
3551 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
&&
3552 (pd
& ~TARGET_PAGE_MASK
) != IO_MEM_ROM
&&
3553 !(pd
& IO_MEM_ROMD
)) {
3556 unsigned long addr1
;
3557 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3559 ptr
= qemu_get_ram_ptr(addr1
);
3560 memcpy(ptr
, buf
, l
);
3570 target_phys_addr_t addr
;
3571 target_phys_addr_t len
;
3574 static BounceBuffer bounce
;
3576 typedef struct MapClient
{
3578 void (*callback
)(void *opaque
);
3579 QLIST_ENTRY(MapClient
) link
;
3582 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
3583 = QLIST_HEAD_INITIALIZER(map_client_list
);
3585 void *cpu_register_map_client(void *opaque
, void (*callback
)(void *opaque
))
3587 MapClient
*client
= qemu_malloc(sizeof(*client
));
3589 client
->opaque
= opaque
;
3590 client
->callback
= callback
;
3591 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
3595 void cpu_unregister_map_client(void *_client
)
3597 MapClient
*client
= (MapClient
*)_client
;
3599 QLIST_REMOVE(client
, link
);
3603 static void cpu_notify_map_clients(void)
3607 while (!QLIST_EMPTY(&map_client_list
)) {
3608 client
= QLIST_FIRST(&map_client_list
);
3609 client
->callback(client
->opaque
);
3610 cpu_unregister_map_client(client
);
3614 /* Map a physical memory region into a host virtual address.
3615 * May map a subset of the requested range, given by and returned in *plen.
3616 * May return NULL if resources needed to perform the mapping are exhausted.
3617 * Use only for reads OR writes - not for read-modify-write operations.
3618 * Use cpu_register_map_client() to know when retrying the map operation is
3619 * likely to succeed.
3621 void *cpu_physical_memory_map(target_phys_addr_t addr
,
3622 target_phys_addr_t
*plen
,
3625 target_phys_addr_t len
= *plen
;
3626 target_phys_addr_t done
= 0;
3628 uint8_t *ret
= NULL
;
3630 target_phys_addr_t page
;
3633 unsigned long addr1
;
3636 page
= addr
& TARGET_PAGE_MASK
;
3637 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3640 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3642 pd
= IO_MEM_UNASSIGNED
;
3644 pd
= p
->phys_offset
;
3647 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3648 if (done
|| bounce
.buffer
) {
3651 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, TARGET_PAGE_SIZE
);
3655 cpu_physical_memory_rw(addr
, bounce
.buffer
, l
, 0);
3657 ptr
= bounce
.buffer
;
3659 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3660 ptr
= qemu_get_ram_ptr(addr1
);
3664 } else if (ret
+ done
!= ptr
) {
3676 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3677 * Will also mark the memory as dirty if is_write == 1. access_len gives
3678 * the amount of memory that was actually read or written by the caller.
3680 void cpu_physical_memory_unmap(void *buffer
, target_phys_addr_t len
,
3681 int is_write
, target_phys_addr_t access_len
)
3683 if (buffer
!= bounce
.buffer
) {
3685 ram_addr_t addr1
= qemu_ram_addr_from_host(buffer
);
3686 while (access_len
) {
3688 l
= TARGET_PAGE_SIZE
;
3691 if (!cpu_physical_memory_is_dirty(addr1
)) {
3692 /* invalidate code */
3693 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3695 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3696 (0xff & ~CODE_DIRTY_FLAG
);
3705 cpu_physical_memory_write(bounce
.addr
, bounce
.buffer
, access_len
);
3707 qemu_vfree(bounce
.buffer
);
3708 bounce
.buffer
= NULL
;
3709 cpu_notify_map_clients();
3712 /* warning: addr must be aligned */
3713 uint32_t ldl_phys(target_phys_addr_t addr
)
3721 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3723 pd
= IO_MEM_UNASSIGNED
;
3725 pd
= p
->phys_offset
;
3728 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3729 !(pd
& IO_MEM_ROMD
)) {
3731 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3733 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3734 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
3737 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3738 (addr
& ~TARGET_PAGE_MASK
);
3744 /* warning: addr must be aligned */
3745 uint64_t ldq_phys(target_phys_addr_t addr
)
3753 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3755 pd
= IO_MEM_UNASSIGNED
;
3757 pd
= p
->phys_offset
;
3760 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3761 !(pd
& IO_MEM_ROMD
)) {
3763 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3765 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3766 #ifdef TARGET_WORDS_BIGENDIAN
3767 val
= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
) << 32;
3768 val
|= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4);
3770 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
3771 val
|= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4) << 32;
3775 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3776 (addr
& ~TARGET_PAGE_MASK
);
3783 uint32_t ldub_phys(target_phys_addr_t addr
)
3786 cpu_physical_memory_read(addr
, &val
, 1);
3791 uint32_t lduw_phys(target_phys_addr_t addr
)
3794 cpu_physical_memory_read(addr
, (uint8_t *)&val
, 2);
3795 return tswap16(val
);
3798 /* warning: addr must be aligned. The ram page is not masked as dirty
3799 and the code inside is not invalidated. It is useful if the dirty
3800 bits are used to track modified PTEs */
3801 void stl_phys_notdirty(target_phys_addr_t addr
, uint32_t val
)
3808 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3810 pd
= IO_MEM_UNASSIGNED
;
3812 pd
= p
->phys_offset
;
3815 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3816 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3818 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3819 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3821 unsigned long addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3822 ptr
= qemu_get_ram_ptr(addr1
);
3825 if (unlikely(in_migration
)) {
3826 if (!cpu_physical_memory_is_dirty(addr1
)) {
3827 /* invalidate code */
3828 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3830 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3831 (0xff & ~CODE_DIRTY_FLAG
);
3837 void stq_phys_notdirty(target_phys_addr_t addr
, uint64_t val
)
3844 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3846 pd
= IO_MEM_UNASSIGNED
;
3848 pd
= p
->phys_offset
;
3851 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3852 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3854 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3855 #ifdef TARGET_WORDS_BIGENDIAN
3856 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
>> 32);
3857 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
);
3859 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3860 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
>> 32);
3863 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3864 (addr
& ~TARGET_PAGE_MASK
);
3869 /* warning: addr must be aligned */
3870 void stl_phys(target_phys_addr_t addr
, uint32_t val
)
3877 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3879 pd
= IO_MEM_UNASSIGNED
;
3881 pd
= p
->phys_offset
;
3884 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3885 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3887 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3888 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3890 unsigned long addr1
;
3891 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3893 ptr
= qemu_get_ram_ptr(addr1
);
3895 if (!cpu_physical_memory_is_dirty(addr1
)) {
3896 /* invalidate code */
3897 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3899 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3900 (0xff & ~CODE_DIRTY_FLAG
);
3906 void stb_phys(target_phys_addr_t addr
, uint32_t val
)
3909 cpu_physical_memory_write(addr
, &v
, 1);
3913 void stw_phys(target_phys_addr_t addr
, uint32_t val
)
3915 uint16_t v
= tswap16(val
);
3916 cpu_physical_memory_write(addr
, (const uint8_t *)&v
, 2);
3920 void stq_phys(target_phys_addr_t addr
, uint64_t val
)
3923 cpu_physical_memory_write(addr
, (const uint8_t *)&val
, 8);
3926 /* virtual memory access for debug (includes writing to ROM) */
3927 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
3928 uint8_t *buf
, int len
, int is_write
)
3931 target_phys_addr_t phys_addr
;
3935 page
= addr
& TARGET_PAGE_MASK
;
3936 phys_addr
= cpu_get_phys_page_debug(env
, page
);
3937 /* if no physical page mapped, return an error */
3938 if (phys_addr
== -1)
3940 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3943 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
3945 cpu_physical_memory_write_rom(phys_addr
, buf
, l
);
3947 cpu_physical_memory_rw(phys_addr
, buf
, l
, is_write
);
3956 /* in deterministic execution mode, instructions doing device I/Os
3957 must be at the end of the TB */
3958 void cpu_io_recompile(CPUState
*env
, void *retaddr
)
3960 TranslationBlock
*tb
;
3962 target_ulong pc
, cs_base
;
3965 tb
= tb_find_pc((unsigned long)retaddr
);
3967 cpu_abort(env
, "cpu_io_recompile: could not find TB for pc=%p",
3970 n
= env
->icount_decr
.u16
.low
+ tb
->icount
;
3971 cpu_restore_state(tb
, env
, (unsigned long)retaddr
, NULL
);
3972 /* Calculate how many instructions had been executed before the fault
3974 n
= n
- env
->icount_decr
.u16
.low
;
3975 /* Generate a new TB ending on the I/O insn. */
3977 /* On MIPS and SH, delay slot instructions can only be restarted if
3978 they were already the first instruction in the TB. If this is not
3979 the first instruction in a TB then re-execute the preceding
3981 #if defined(TARGET_MIPS)
3982 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
3983 env
->active_tc
.PC
-= 4;
3984 env
->icount_decr
.u16
.low
++;
3985 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
3987 #elif defined(TARGET_SH4)
3988 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
3991 env
->icount_decr
.u16
.low
++;
3992 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
3995 /* This should never happen. */
3996 if (n
> CF_COUNT_MASK
)
3997 cpu_abort(env
, "TB too big during recompile");
3999 cflags
= n
| CF_LAST_IO
;
4001 cs_base
= tb
->cs_base
;
4003 tb_phys_invalidate(tb
, -1);
4004 /* FIXME: In theory this could raise an exception. In practice
4005 we have already translated the block once so it's probably ok. */
4006 tb_gen_code(env
, pc
, cs_base
, flags
, cflags
);
4007 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
4008 the first in the TB) then we end up generating a whole new TB and
4009 repeating the fault, which is horribly inefficient.
4010 Better would be to execute just this insn uncached, or generate a
4012 cpu_resume_from_signal(env
, NULL
);
4015 #if !defined(CONFIG_USER_ONLY)
4017 void dump_exec_info(FILE *f
,
4018 int (*cpu_fprintf
)(FILE *f
, const char *fmt
, ...))
4020 int i
, target_code_size
, max_target_code_size
;
4021 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
4022 TranslationBlock
*tb
;
4024 target_code_size
= 0;
4025 max_target_code_size
= 0;
4027 direct_jmp_count
= 0;
4028 direct_jmp2_count
= 0;
4029 for(i
= 0; i
< nb_tbs
; i
++) {
4031 target_code_size
+= tb
->size
;
4032 if (tb
->size
> max_target_code_size
)
4033 max_target_code_size
= tb
->size
;
4034 if (tb
->page_addr
[1] != -1)
4036 if (tb
->tb_next_offset
[0] != 0xffff) {
4038 if (tb
->tb_next_offset
[1] != 0xffff) {
4039 direct_jmp2_count
++;
4043 /* XXX: avoid using doubles ? */
4044 cpu_fprintf(f
, "Translation buffer state:\n");
4045 cpu_fprintf(f
, "gen code size %ld/%ld\n",
4046 code_gen_ptr
- code_gen_buffer
, code_gen_buffer_max_size
);
4047 cpu_fprintf(f
, "TB count %d/%d\n",
4048 nb_tbs
, code_gen_max_blocks
);
4049 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
4050 nb_tbs
? target_code_size
/ nb_tbs
: 0,
4051 max_target_code_size
);
4052 cpu_fprintf(f
, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
4053 nb_tbs
? (code_gen_ptr
- code_gen_buffer
) / nb_tbs
: 0,
4054 target_code_size
? (double) (code_gen_ptr
- code_gen_buffer
) / target_code_size
: 0);
4055 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n",
4057 nb_tbs
? (cross_page
* 100) / nb_tbs
: 0);
4058 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
4060 nb_tbs
? (direct_jmp_count
* 100) / nb_tbs
: 0,
4062 nb_tbs
? (direct_jmp2_count
* 100) / nb_tbs
: 0);
4063 cpu_fprintf(f
, "\nStatistics:\n");
4064 cpu_fprintf(f
, "TB flush count %d\n", tb_flush_count
);
4065 cpu_fprintf(f
, "TB invalidate count %d\n", tb_phys_invalidate_count
);
4066 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
4067 tcg_dump_info(f
, cpu_fprintf
);
4070 #define MMUSUFFIX _cmmu
4071 #define GETPC() NULL
4072 #define env cpu_single_env
4073 #define SOFTMMU_CODE_ACCESS
4076 #include "softmmu_template.h"
4079 #include "softmmu_template.h"
4082 #include "softmmu_template.h"
4085 #include "softmmu_template.h"