2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301 USA
22 #define WIN32_LEAN_AND_MEAN
25 #include <sys/types.h>
38 #include "qemu-common.h"
43 #if defined(CONFIG_USER_ONLY)
47 //#define DEBUG_TB_INVALIDATE
50 //#define DEBUG_UNASSIGNED
52 /* make various TB consistency checks */
53 //#define DEBUG_TB_CHECK
54 //#define DEBUG_TLB_CHECK
56 //#define DEBUG_IOPORT
57 //#define DEBUG_SUBPAGE
59 #if !defined(CONFIG_USER_ONLY)
60 /* TB consistency checks only implemented for usermode emulation. */
64 #define SMC_BITMAP_USE_THRESHOLD 10
66 #define MMAP_AREA_START 0x00000000
67 #define MMAP_AREA_END 0xa8000000
69 #if defined(TARGET_SPARC64)
70 #define TARGET_PHYS_ADDR_SPACE_BITS 41
71 #elif defined(TARGET_SPARC)
72 #define TARGET_PHYS_ADDR_SPACE_BITS 36
73 #elif defined(TARGET_ALPHA)
74 #define TARGET_PHYS_ADDR_SPACE_BITS 42
75 #define TARGET_VIRT_ADDR_SPACE_BITS 42
76 #elif defined(TARGET_PPC64)
77 #define TARGET_PHYS_ADDR_SPACE_BITS 42
78 #elif defined(TARGET_X86_64) && !defined(USE_KQEMU)
79 #define TARGET_PHYS_ADDR_SPACE_BITS 42
80 #elif defined(TARGET_I386) && !defined(USE_KQEMU)
81 #define TARGET_PHYS_ADDR_SPACE_BITS 36
83 /* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
84 #define TARGET_PHYS_ADDR_SPACE_BITS 32
87 static TranslationBlock
*tbs
;
88 int code_gen_max_blocks
;
89 TranslationBlock
*tb_phys_hash
[CODE_GEN_PHYS_HASH_SIZE
];
91 /* any access to the tbs or the page table must use this lock */
92 spinlock_t tb_lock
= SPIN_LOCK_UNLOCKED
;
94 #if defined(__arm__) || defined(__sparc_v9__)
95 /* The prologue must be reachable with a direct jump. ARM and Sparc64
96 have limited branch ranges (possibly also PPC) so place it in a
97 section close to code segment. */
98 #define code_gen_section \
99 __attribute__((__section__(".gen_code"))) \
100 __attribute__((aligned (32)))
102 #define code_gen_section \
103 __attribute__((aligned (32)))
106 uint8_t code_gen_prologue
[1024] code_gen_section
;
107 static uint8_t *code_gen_buffer
;
108 static unsigned long code_gen_buffer_size
;
109 /* threshold to flush the translated code buffer */
110 static unsigned long code_gen_buffer_max_size
;
111 uint8_t *code_gen_ptr
;
113 #if !defined(CONFIG_USER_ONLY)
114 ram_addr_t phys_ram_size
;
116 uint8_t *phys_ram_base
;
117 uint8_t *phys_ram_dirty
;
118 static int in_migration
;
119 static ram_addr_t phys_ram_alloc_offset
= 0;
123 /* current CPU in the current thread. It is only valid inside
125 CPUState
*cpu_single_env
;
126 /* 0 = Do not count executed instructions.
127 1 = Precise instruction counting.
128 2 = Adaptive rate instruction counting. */
130 /* Current instruction counter. While executing translated code this may
131 include some instructions that have not yet been executed. */
134 typedef struct PageDesc
{
135 /* list of TBs intersecting this ram page */
136 TranslationBlock
*first_tb
;
137 /* in order to optimize self modifying code, we count the number
138 of lookups we do to a given page to use a bitmap */
139 unsigned int code_write_count
;
140 uint8_t *code_bitmap
;
141 #if defined(CONFIG_USER_ONLY)
146 typedef struct PhysPageDesc
{
147 /* offset in host memory of the page + io_index in the low bits */
148 ram_addr_t phys_offset
;
149 ram_addr_t region_offset
;
153 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
154 /* XXX: this is a temporary hack for alpha target.
155 * In the future, this is to be replaced by a multi-level table
156 * to actually be able to handle the complete 64 bits address space.
158 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
160 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
163 #define L1_SIZE (1 << L1_BITS)
164 #define L2_SIZE (1 << L2_BITS)
166 unsigned long qemu_real_host_page_size
;
167 unsigned long qemu_host_page_bits
;
168 unsigned long qemu_host_page_size
;
169 unsigned long qemu_host_page_mask
;
171 /* XXX: for system emulation, it could just be an array */
172 static PageDesc
*l1_map
[L1_SIZE
];
173 static PhysPageDesc
**l1_phys_map
;
175 #if !defined(CONFIG_USER_ONLY)
176 static void io_mem_init(void);
178 /* io memory support */
179 CPUWriteMemoryFunc
*io_mem_write
[IO_MEM_NB_ENTRIES
][4];
180 CPUReadMemoryFunc
*io_mem_read
[IO_MEM_NB_ENTRIES
][4];
181 void *io_mem_opaque
[IO_MEM_NB_ENTRIES
];
182 static int io_mem_nb
;
183 static int io_mem_watch
;
187 static const char *logfilename
= "/tmp/qemu.log";
190 static int log_append
= 0;
193 static int tlb_flush_count
;
194 static int tb_flush_count
;
195 static int tb_phys_invalidate_count
;
197 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
198 typedef struct subpage_t
{
199 target_phys_addr_t base
;
200 CPUReadMemoryFunc
**mem_read
[TARGET_PAGE_SIZE
][4];
201 CPUWriteMemoryFunc
**mem_write
[TARGET_PAGE_SIZE
][4];
202 void *opaque
[TARGET_PAGE_SIZE
][2][4];
203 ram_addr_t region_offset
[TARGET_PAGE_SIZE
][2][4];
207 static void map_exec(void *addr
, long size
)
210 VirtualProtect(addr
, size
,
211 PAGE_EXECUTE_READWRITE
, &old_protect
);
215 static void map_exec(void *addr
, long size
)
217 unsigned long start
, end
, page_size
;
219 page_size
= getpagesize();
220 start
= (unsigned long)addr
;
221 start
&= ~(page_size
- 1);
223 end
= (unsigned long)addr
+ size
;
224 end
+= page_size
- 1;
225 end
&= ~(page_size
- 1);
227 mprotect((void *)start
, end
- start
,
228 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
232 static void page_init(void)
234 /* NOTE: we can always suppose that qemu_host_page_size >=
238 SYSTEM_INFO system_info
;
240 GetSystemInfo(&system_info
);
241 qemu_real_host_page_size
= system_info
.dwPageSize
;
244 qemu_real_host_page_size
= getpagesize();
246 if (qemu_host_page_size
== 0)
247 qemu_host_page_size
= qemu_real_host_page_size
;
248 if (qemu_host_page_size
< TARGET_PAGE_SIZE
)
249 qemu_host_page_size
= TARGET_PAGE_SIZE
;
250 qemu_host_page_bits
= 0;
251 while ((1 << qemu_host_page_bits
) < qemu_host_page_size
)
252 qemu_host_page_bits
++;
253 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
254 l1_phys_map
= qemu_vmalloc(L1_SIZE
* sizeof(void *));
255 memset(l1_phys_map
, 0, L1_SIZE
* sizeof(void *));
257 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
259 long long startaddr
, endaddr
;
264 last_brk
= (unsigned long)sbrk(0);
265 f
= fopen("/proc/self/maps", "r");
268 n
= fscanf (f
, "%llx-%llx %*[^\n]\n", &startaddr
, &endaddr
);
270 startaddr
= MIN(startaddr
,
271 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS
) - 1);
272 endaddr
= MIN(endaddr
,
273 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS
) - 1);
274 page_set_flags(startaddr
& TARGET_PAGE_MASK
,
275 TARGET_PAGE_ALIGN(endaddr
),
286 static inline PageDesc
**page_l1_map(target_ulong index
)
288 #if TARGET_LONG_BITS > 32
289 /* Host memory outside guest VM. For 32-bit targets we have already
290 excluded high addresses. */
291 if (index
> ((target_ulong
)L2_SIZE
* L1_SIZE
))
294 return &l1_map
[index
>> L2_BITS
];
297 static inline PageDesc
*page_find_alloc(target_ulong index
)
300 lp
= page_l1_map(index
);
306 /* allocate if not found */
307 #if defined(CONFIG_USER_ONLY)
308 size_t len
= sizeof(PageDesc
) * L2_SIZE
;
309 /* Don't use qemu_malloc because it may recurse. */
310 p
= mmap(0, len
, PROT_READ
| PROT_WRITE
,
311 MAP_PRIVATE
| MAP_ANONYMOUS
, -1, 0);
314 unsigned long addr
= h2g(p
);
315 page_set_flags(addr
& TARGET_PAGE_MASK
,
316 TARGET_PAGE_ALIGN(addr
+ len
),
320 p
= qemu_mallocz(sizeof(PageDesc
) * L2_SIZE
);
324 return p
+ (index
& (L2_SIZE
- 1));
327 static inline PageDesc
*page_find(target_ulong index
)
330 lp
= page_l1_map(index
);
337 return p
+ (index
& (L2_SIZE
- 1));
340 static PhysPageDesc
*phys_page_find_alloc(target_phys_addr_t index
, int alloc
)
345 p
= (void **)l1_phys_map
;
346 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
348 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
349 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
351 lp
= p
+ ((index
>> (L1_BITS
+ L2_BITS
)) & (L1_SIZE
- 1));
354 /* allocate if not found */
357 p
= qemu_vmalloc(sizeof(void *) * L1_SIZE
);
358 memset(p
, 0, sizeof(void *) * L1_SIZE
);
362 lp
= p
+ ((index
>> L2_BITS
) & (L1_SIZE
- 1));
366 /* allocate if not found */
369 pd
= qemu_vmalloc(sizeof(PhysPageDesc
) * L2_SIZE
);
371 for (i
= 0; i
< L2_SIZE
; i
++)
372 pd
[i
].phys_offset
= IO_MEM_UNASSIGNED
;
374 return ((PhysPageDesc
*)pd
) + (index
& (L2_SIZE
- 1));
377 static inline PhysPageDesc
*phys_page_find(target_phys_addr_t index
)
379 return phys_page_find_alloc(index
, 0);
382 #if !defined(CONFIG_USER_ONLY)
383 static void tlb_protect_code(ram_addr_t ram_addr
);
384 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
386 #define mmap_lock() do { } while(0)
387 #define mmap_unlock() do { } while(0)
390 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
392 #if defined(CONFIG_USER_ONLY)
393 /* Currently it is not recommanded to allocate big chunks of data in
394 user mode. It will change when a dedicated libc will be used */
395 #define USE_STATIC_CODE_GEN_BUFFER
398 #ifdef USE_STATIC_CODE_GEN_BUFFER
399 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
];
402 static void code_gen_alloc(unsigned long tb_size
)
404 #ifdef USE_STATIC_CODE_GEN_BUFFER
405 code_gen_buffer
= static_code_gen_buffer
;
406 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
407 map_exec(code_gen_buffer
, code_gen_buffer_size
);
409 code_gen_buffer_size
= tb_size
;
410 if (code_gen_buffer_size
== 0) {
411 #if defined(CONFIG_USER_ONLY)
412 /* in user mode, phys_ram_size is not meaningful */
413 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
415 /* XXX: needs ajustments */
416 code_gen_buffer_size
= (unsigned long)(phys_ram_size
/ 4);
419 if (code_gen_buffer_size
< MIN_CODE_GEN_BUFFER_SIZE
)
420 code_gen_buffer_size
= MIN_CODE_GEN_BUFFER_SIZE
;
421 /* The code gen buffer location may have constraints depending on
422 the host cpu and OS */
423 #if defined(__linux__)
428 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
429 #if defined(__x86_64__)
431 /* Cannot map more than that */
432 if (code_gen_buffer_size
> (800 * 1024 * 1024))
433 code_gen_buffer_size
= (800 * 1024 * 1024);
434 #elif defined(__sparc_v9__)
435 // Map the buffer below 2G, so we can use direct calls and branches
437 start
= (void *) 0x60000000UL
;
438 if (code_gen_buffer_size
> (512 * 1024 * 1024))
439 code_gen_buffer_size
= (512 * 1024 * 1024);
440 #elif defined(__arm__)
441 /* Map the buffer below 32M, so we can use direct calls and branches */
443 start
= (void *) 0x01000000UL
;
444 if (code_gen_buffer_size
> 16 * 1024 * 1024)
445 code_gen_buffer_size
= 16 * 1024 * 1024;
447 code_gen_buffer
= mmap(start
, code_gen_buffer_size
,
448 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
450 if (code_gen_buffer
== MAP_FAILED
) {
451 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
455 #elif defined(__FreeBSD__)
459 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
460 #if defined(__x86_64__)
461 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
462 * 0x40000000 is free */
464 addr
= (void *)0x40000000;
465 /* Cannot map more than that */
466 if (code_gen_buffer_size
> (800 * 1024 * 1024))
467 code_gen_buffer_size
= (800 * 1024 * 1024);
469 code_gen_buffer
= mmap(addr
, code_gen_buffer_size
,
470 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
472 if (code_gen_buffer
== MAP_FAILED
) {
473 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
478 code_gen_buffer
= qemu_malloc(code_gen_buffer_size
);
479 if (!code_gen_buffer
) {
480 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
483 map_exec(code_gen_buffer
, code_gen_buffer_size
);
485 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
486 map_exec(code_gen_prologue
, sizeof(code_gen_prologue
));
487 code_gen_buffer_max_size
= code_gen_buffer_size
-
488 code_gen_max_block_size();
489 code_gen_max_blocks
= code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
490 tbs
= qemu_malloc(code_gen_max_blocks
* sizeof(TranslationBlock
));
493 /* Must be called before using the QEMU cpus. 'tb_size' is the size
494 (in bytes) allocated to the translation buffer. Zero means default
496 void cpu_exec_init_all(unsigned long tb_size
)
499 code_gen_alloc(tb_size
);
500 code_gen_ptr
= code_gen_buffer
;
502 #if !defined(CONFIG_USER_ONLY)
507 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
509 #define CPU_COMMON_SAVE_VERSION 1
511 static void cpu_common_save(QEMUFile
*f
, void *opaque
)
513 CPUState
*env
= opaque
;
515 qemu_put_be32s(f
, &env
->halted
);
516 qemu_put_be32s(f
, &env
->interrupt_request
);
519 static int cpu_common_load(QEMUFile
*f
, void *opaque
, int version_id
)
521 CPUState
*env
= opaque
;
523 if (version_id
!= CPU_COMMON_SAVE_VERSION
)
526 qemu_get_be32s(f
, &env
->halted
);
527 qemu_get_be32s(f
, &env
->interrupt_request
);
534 void cpu_exec_init(CPUState
*env
)
539 env
->next_cpu
= NULL
;
542 while (*penv
!= NULL
) {
543 penv
= (CPUState
**)&(*penv
)->next_cpu
;
546 env
->cpu_index
= cpu_index
;
547 TAILQ_INIT(&env
->breakpoints
);
548 TAILQ_INIT(&env
->watchpoints
);
550 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
551 register_savevm("cpu_common", cpu_index
, CPU_COMMON_SAVE_VERSION
,
552 cpu_common_save
, cpu_common_load
, env
);
553 register_savevm("cpu", cpu_index
, CPU_SAVE_VERSION
,
554 cpu_save
, cpu_load
, env
);
558 static inline void invalidate_page_bitmap(PageDesc
*p
)
560 if (p
->code_bitmap
) {
561 qemu_free(p
->code_bitmap
);
562 p
->code_bitmap
= NULL
;
564 p
->code_write_count
= 0;
567 /* set to NULL all the 'first_tb' fields in all PageDescs */
568 static void page_flush_tb(void)
573 for(i
= 0; i
< L1_SIZE
; i
++) {
576 for(j
= 0; j
< L2_SIZE
; j
++) {
578 invalidate_page_bitmap(p
);
585 /* flush all the translation blocks */
586 /* XXX: tb_flush is currently not thread safe */
587 void tb_flush(CPUState
*env1
)
590 #if defined(DEBUG_FLUSH)
591 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
592 (unsigned long)(code_gen_ptr
- code_gen_buffer
),
594 ((unsigned long)(code_gen_ptr
- code_gen_buffer
)) / nb_tbs
: 0);
596 if ((unsigned long)(code_gen_ptr
- code_gen_buffer
) > code_gen_buffer_size
)
597 cpu_abort(env1
, "Internal error: code buffer overflow\n");
601 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
602 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
605 memset (tb_phys_hash
, 0, CODE_GEN_PHYS_HASH_SIZE
* sizeof (void *));
608 code_gen_ptr
= code_gen_buffer
;
609 /* XXX: flush processor icache at this point if cache flush is
614 #ifdef DEBUG_TB_CHECK
616 static void tb_invalidate_check(target_ulong address
)
618 TranslationBlock
*tb
;
620 address
&= TARGET_PAGE_MASK
;
621 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
622 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
623 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
624 address
>= tb
->pc
+ tb
->size
)) {
625 printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
626 address
, (long)tb
->pc
, tb
->size
);
632 /* verify that all the pages have correct rights for code */
633 static void tb_page_check(void)
635 TranslationBlock
*tb
;
636 int i
, flags1
, flags2
;
638 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
639 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
640 flags1
= page_get_flags(tb
->pc
);
641 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
642 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
643 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
644 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
650 static void tb_jmp_check(TranslationBlock
*tb
)
652 TranslationBlock
*tb1
;
655 /* suppress any remaining jumps to this TB */
659 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
662 tb1
= tb1
->jmp_next
[n1
];
664 /* check end of list */
666 printf("ERROR: jmp_list from 0x%08lx\n", (long)tb
);
672 /* invalidate one TB */
673 static inline void tb_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
,
676 TranslationBlock
*tb1
;
680 *ptb
= *(TranslationBlock
**)((char *)tb1
+ next_offset
);
683 ptb
= (TranslationBlock
**)((char *)tb1
+ next_offset
);
687 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
689 TranslationBlock
*tb1
;
695 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
697 *ptb
= tb1
->page_next
[n1
];
700 ptb
= &tb1
->page_next
[n1
];
704 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
706 TranslationBlock
*tb1
, **ptb
;
709 ptb
= &tb
->jmp_next
[n
];
712 /* find tb(n) in circular list */
716 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
717 if (n1
== n
&& tb1
== tb
)
720 ptb
= &tb1
->jmp_first
;
722 ptb
= &tb1
->jmp_next
[n1
];
725 /* now we can suppress tb(n) from the list */
726 *ptb
= tb
->jmp_next
[n
];
728 tb
->jmp_next
[n
] = NULL
;
732 /* reset the jump entry 'n' of a TB so that it is not chained to
734 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
736 tb_set_jmp_target(tb
, n
, (unsigned long)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
739 void tb_phys_invalidate(TranslationBlock
*tb
, target_ulong page_addr
)
744 target_phys_addr_t phys_pc
;
745 TranslationBlock
*tb1
, *tb2
;
747 /* remove the TB from the hash list */
748 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
749 h
= tb_phys_hash_func(phys_pc
);
750 tb_remove(&tb_phys_hash
[h
], tb
,
751 offsetof(TranslationBlock
, phys_hash_next
));
753 /* remove the TB from the page list */
754 if (tb
->page_addr
[0] != page_addr
) {
755 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
756 tb_page_remove(&p
->first_tb
, tb
);
757 invalidate_page_bitmap(p
);
759 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
760 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
761 tb_page_remove(&p
->first_tb
, tb
);
762 invalidate_page_bitmap(p
);
765 tb_invalidated_flag
= 1;
767 /* remove the TB from the hash list */
768 h
= tb_jmp_cache_hash_func(tb
->pc
);
769 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
770 if (env
->tb_jmp_cache
[h
] == tb
)
771 env
->tb_jmp_cache
[h
] = NULL
;
774 /* suppress this TB from the two jump lists */
775 tb_jmp_remove(tb
, 0);
776 tb_jmp_remove(tb
, 1);
778 /* suppress any remaining jumps to this TB */
784 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
785 tb2
= tb1
->jmp_next
[n1
];
786 tb_reset_jump(tb1
, n1
);
787 tb1
->jmp_next
[n1
] = NULL
;
790 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2); /* fail safe */
792 tb_phys_invalidate_count
++;
795 static inline void set_bits(uint8_t *tab
, int start
, int len
)
801 mask
= 0xff << (start
& 7);
802 if ((start
& ~7) == (end
& ~7)) {
804 mask
&= ~(0xff << (end
& 7));
809 start
= (start
+ 8) & ~7;
811 while (start
< end1
) {
816 mask
= ~(0xff << (end
& 7));
822 static void build_page_bitmap(PageDesc
*p
)
824 int n
, tb_start
, tb_end
;
825 TranslationBlock
*tb
;
827 p
->code_bitmap
= qemu_mallocz(TARGET_PAGE_SIZE
/ 8);
834 tb
= (TranslationBlock
*)((long)tb
& ~3);
835 /* NOTE: this is subtle as a TB may span two physical pages */
837 /* NOTE: tb_end may be after the end of the page, but
838 it is not a problem */
839 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
840 tb_end
= tb_start
+ tb
->size
;
841 if (tb_end
> TARGET_PAGE_SIZE
)
842 tb_end
= TARGET_PAGE_SIZE
;
845 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
847 set_bits(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
848 tb
= tb
->page_next
[n
];
852 TranslationBlock
*tb_gen_code(CPUState
*env
,
853 target_ulong pc
, target_ulong cs_base
,
854 int flags
, int cflags
)
856 TranslationBlock
*tb
;
858 target_ulong phys_pc
, phys_page2
, virt_page2
;
861 phys_pc
= get_phys_addr_code(env
, pc
);
864 /* flush must be done */
866 /* cannot fail at this point */
868 /* Don't forget to invalidate previous TB info. */
869 tb_invalidated_flag
= 1;
871 tc_ptr
= code_gen_ptr
;
873 tb
->cs_base
= cs_base
;
876 cpu_gen_code(env
, tb
, &code_gen_size
);
877 code_gen_ptr
= (void *)(((unsigned long)code_gen_ptr
+ code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
879 /* check next page if needed */
880 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
882 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
883 phys_page2
= get_phys_addr_code(env
, virt_page2
);
885 tb_link_phys(tb
, phys_pc
, phys_page2
);
889 /* invalidate all TBs which intersect with the target physical page
890 starting in range [start;end[. NOTE: start and end must refer to
891 the same physical page. 'is_cpu_write_access' should be true if called
892 from a real cpu write access: the virtual CPU will exit the current
893 TB if code is modified inside this TB. */
894 void tb_invalidate_phys_page_range(target_phys_addr_t start
, target_phys_addr_t end
,
895 int is_cpu_write_access
)
897 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
898 CPUState
*env
= cpu_single_env
;
899 target_ulong tb_start
, tb_end
;
902 #ifdef TARGET_HAS_PRECISE_SMC
903 int current_tb_not_found
= is_cpu_write_access
;
904 TranslationBlock
*current_tb
= NULL
;
905 int current_tb_modified
= 0;
906 target_ulong current_pc
= 0;
907 target_ulong current_cs_base
= 0;
908 int current_flags
= 0;
909 #endif /* TARGET_HAS_PRECISE_SMC */
911 p
= page_find(start
>> TARGET_PAGE_BITS
);
914 if (!p
->code_bitmap
&&
915 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
&&
916 is_cpu_write_access
) {
917 /* build code bitmap */
918 build_page_bitmap(p
);
921 /* we remove all the TBs in the range [start, end[ */
922 /* XXX: see if in some cases it could be faster to invalidate all the code */
926 tb
= (TranslationBlock
*)((long)tb
& ~3);
927 tb_next
= tb
->page_next
[n
];
928 /* NOTE: this is subtle as a TB may span two physical pages */
930 /* NOTE: tb_end may be after the end of the page, but
931 it is not a problem */
932 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
933 tb_end
= tb_start
+ tb
->size
;
935 tb_start
= tb
->page_addr
[1];
936 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
938 if (!(tb_end
<= start
|| tb_start
>= end
)) {
939 #ifdef TARGET_HAS_PRECISE_SMC
940 if (current_tb_not_found
) {
941 current_tb_not_found
= 0;
943 if (env
->mem_io_pc
) {
944 /* now we have a real cpu fault */
945 current_tb
= tb_find_pc(env
->mem_io_pc
);
948 if (current_tb
== tb
&&
949 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
950 /* If we are modifying the current TB, we must stop
951 its execution. We could be more precise by checking
952 that the modification is after the current PC, but it
953 would require a specialized function to partially
954 restore the CPU state */
956 current_tb_modified
= 1;
957 cpu_restore_state(current_tb
, env
,
958 env
->mem_io_pc
, NULL
);
959 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
962 #endif /* TARGET_HAS_PRECISE_SMC */
963 /* we need to do that to handle the case where a signal
964 occurs while doing tb_phys_invalidate() */
967 saved_tb
= env
->current_tb
;
968 env
->current_tb
= NULL
;
970 tb_phys_invalidate(tb
, -1);
972 env
->current_tb
= saved_tb
;
973 if (env
->interrupt_request
&& env
->current_tb
)
974 cpu_interrupt(env
, env
->interrupt_request
);
979 #if !defined(CONFIG_USER_ONLY)
980 /* if no code remaining, no need to continue to use slow writes */
982 invalidate_page_bitmap(p
);
983 if (is_cpu_write_access
) {
984 tlb_unprotect_code_phys(env
, start
, env
->mem_io_vaddr
);
988 #ifdef TARGET_HAS_PRECISE_SMC
989 if (current_tb_modified
) {
990 /* we generate a block containing just the instruction
991 modifying the memory. It will ensure that it cannot modify
993 env
->current_tb
= NULL
;
994 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
995 cpu_resume_from_signal(env
, NULL
);
1000 /* len must be <= 8 and start must be a multiple of len */
1001 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start
, int len
)
1008 fprintf(logfile
, "modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1009 cpu_single_env
->mem_io_vaddr
, len
,
1010 cpu_single_env
->eip
,
1011 cpu_single_env
->eip
+ (long)cpu_single_env
->segs
[R_CS
].base
);
1015 p
= page_find(start
>> TARGET_PAGE_BITS
);
1018 if (p
->code_bitmap
) {
1019 offset
= start
& ~TARGET_PAGE_MASK
;
1020 b
= p
->code_bitmap
[offset
>> 3] >> (offset
& 7);
1021 if (b
& ((1 << len
) - 1))
1025 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1029 #if !defined(CONFIG_SOFTMMU)
1030 static void tb_invalidate_phys_page(target_phys_addr_t addr
,
1031 unsigned long pc
, void *puc
)
1033 TranslationBlock
*tb
;
1036 #ifdef TARGET_HAS_PRECISE_SMC
1037 TranslationBlock
*current_tb
= NULL
;
1038 CPUState
*env
= cpu_single_env
;
1039 int current_tb_modified
= 0;
1040 target_ulong current_pc
= 0;
1041 target_ulong current_cs_base
= 0;
1042 int current_flags
= 0;
1045 addr
&= TARGET_PAGE_MASK
;
1046 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1050 #ifdef TARGET_HAS_PRECISE_SMC
1051 if (tb
&& pc
!= 0) {
1052 current_tb
= tb_find_pc(pc
);
1055 while (tb
!= NULL
) {
1057 tb
= (TranslationBlock
*)((long)tb
& ~3);
1058 #ifdef TARGET_HAS_PRECISE_SMC
1059 if (current_tb
== tb
&&
1060 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1061 /* If we are modifying the current TB, we must stop
1062 its execution. We could be more precise by checking
1063 that the modification is after the current PC, but it
1064 would require a specialized function to partially
1065 restore the CPU state */
1067 current_tb_modified
= 1;
1068 cpu_restore_state(current_tb
, env
, pc
, puc
);
1069 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1072 #endif /* TARGET_HAS_PRECISE_SMC */
1073 tb_phys_invalidate(tb
, addr
);
1074 tb
= tb
->page_next
[n
];
1077 #ifdef TARGET_HAS_PRECISE_SMC
1078 if (current_tb_modified
) {
1079 /* we generate a block containing just the instruction
1080 modifying the memory. It will ensure that it cannot modify
1082 env
->current_tb
= NULL
;
1083 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1084 cpu_resume_from_signal(env
, puc
);
1090 /* add the tb in the target page and protect it if necessary */
1091 static inline void tb_alloc_page(TranslationBlock
*tb
,
1092 unsigned int n
, target_ulong page_addr
)
1095 TranslationBlock
*last_first_tb
;
1097 tb
->page_addr
[n
] = page_addr
;
1098 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
);
1099 tb
->page_next
[n
] = p
->first_tb
;
1100 last_first_tb
= p
->first_tb
;
1101 p
->first_tb
= (TranslationBlock
*)((long)tb
| n
);
1102 invalidate_page_bitmap(p
);
1104 #if defined(TARGET_HAS_SMC) || 1
1106 #if defined(CONFIG_USER_ONLY)
1107 if (p
->flags
& PAGE_WRITE
) {
1112 /* force the host page as non writable (writes will have a
1113 page fault + mprotect overhead) */
1114 page_addr
&= qemu_host_page_mask
;
1116 for(addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1117 addr
+= TARGET_PAGE_SIZE
) {
1119 p2
= page_find (addr
>> TARGET_PAGE_BITS
);
1123 p2
->flags
&= ~PAGE_WRITE
;
1124 page_get_flags(addr
);
1126 mprotect(g2h(page_addr
), qemu_host_page_size
,
1127 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1128 #ifdef DEBUG_TB_INVALIDATE
1129 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1134 /* if some code is already present, then the pages are already
1135 protected. So we handle the case where only the first TB is
1136 allocated in a physical page */
1137 if (!last_first_tb
) {
1138 tlb_protect_code(page_addr
);
1142 #endif /* TARGET_HAS_SMC */
1145 /* Allocate a new translation block. Flush the translation buffer if
1146 too many translation blocks or too much generated code. */
1147 TranslationBlock
*tb_alloc(target_ulong pc
)
1149 TranslationBlock
*tb
;
1151 if (nb_tbs
>= code_gen_max_blocks
||
1152 (code_gen_ptr
- code_gen_buffer
) >= code_gen_buffer_max_size
)
1154 tb
= &tbs
[nb_tbs
++];
1160 void tb_free(TranslationBlock
*tb
)
1162 /* In practice this is mostly used for single use temporary TB
1163 Ignore the hard cases and just back up if this TB happens to
1164 be the last one generated. */
1165 if (nb_tbs
> 0 && tb
== &tbs
[nb_tbs
- 1]) {
1166 code_gen_ptr
= tb
->tc_ptr
;
1171 /* add a new TB and link it to the physical page tables. phys_page2 is
1172 (-1) to indicate that only one page contains the TB. */
1173 void tb_link_phys(TranslationBlock
*tb
,
1174 target_ulong phys_pc
, target_ulong phys_page2
)
1177 TranslationBlock
**ptb
;
1179 /* Grab the mmap lock to stop another thread invalidating this TB
1180 before we are done. */
1182 /* add in the physical hash table */
1183 h
= tb_phys_hash_func(phys_pc
);
1184 ptb
= &tb_phys_hash
[h
];
1185 tb
->phys_hash_next
= *ptb
;
1188 /* add in the page list */
1189 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1190 if (phys_page2
!= -1)
1191 tb_alloc_page(tb
, 1, phys_page2
);
1193 tb
->page_addr
[1] = -1;
1195 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2);
1196 tb
->jmp_next
[0] = NULL
;
1197 tb
->jmp_next
[1] = NULL
;
1199 /* init original jump addresses */
1200 if (tb
->tb_next_offset
[0] != 0xffff)
1201 tb_reset_jump(tb
, 0);
1202 if (tb
->tb_next_offset
[1] != 0xffff)
1203 tb_reset_jump(tb
, 1);
1205 #ifdef DEBUG_TB_CHECK
1211 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1212 tb[1].tc_ptr. Return NULL if not found */
1213 TranslationBlock
*tb_find_pc(unsigned long tc_ptr
)
1215 int m_min
, m_max
, m
;
1217 TranslationBlock
*tb
;
1221 if (tc_ptr
< (unsigned long)code_gen_buffer
||
1222 tc_ptr
>= (unsigned long)code_gen_ptr
)
1224 /* binary search (cf Knuth) */
1227 while (m_min
<= m_max
) {
1228 m
= (m_min
+ m_max
) >> 1;
1230 v
= (unsigned long)tb
->tc_ptr
;
1233 else if (tc_ptr
< v
) {
1242 static void tb_reset_jump_recursive(TranslationBlock
*tb
);
1244 static inline void tb_reset_jump_recursive2(TranslationBlock
*tb
, int n
)
1246 TranslationBlock
*tb1
, *tb_next
, **ptb
;
1249 tb1
= tb
->jmp_next
[n
];
1251 /* find head of list */
1254 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1257 tb1
= tb1
->jmp_next
[n1
];
1259 /* we are now sure now that tb jumps to tb1 */
1262 /* remove tb from the jmp_first list */
1263 ptb
= &tb_next
->jmp_first
;
1267 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1268 if (n1
== n
&& tb1
== tb
)
1270 ptb
= &tb1
->jmp_next
[n1
];
1272 *ptb
= tb
->jmp_next
[n
];
1273 tb
->jmp_next
[n
] = NULL
;
1275 /* suppress the jump to next tb in generated code */
1276 tb_reset_jump(tb
, n
);
1278 /* suppress jumps in the tb on which we could have jumped */
1279 tb_reset_jump_recursive(tb_next
);
1283 static void tb_reset_jump_recursive(TranslationBlock
*tb
)
1285 tb_reset_jump_recursive2(tb
, 0);
1286 tb_reset_jump_recursive2(tb
, 1);
1289 #if defined(TARGET_HAS_ICE)
1290 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1292 target_phys_addr_t addr
;
1294 ram_addr_t ram_addr
;
1297 addr
= cpu_get_phys_page_debug(env
, pc
);
1298 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
1300 pd
= IO_MEM_UNASSIGNED
;
1302 pd
= p
->phys_offset
;
1304 ram_addr
= (pd
& TARGET_PAGE_MASK
) | (pc
& ~TARGET_PAGE_MASK
);
1305 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1309 /* Add a watchpoint. */
1310 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1311 int flags
, CPUWatchpoint
**watchpoint
)
1313 target_ulong len_mask
= ~(len
- 1);
1316 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1317 if ((len
!= 1 && len
!= 2 && len
!= 4 && len
!= 8) || (addr
& ~len_mask
)) {
1318 fprintf(stderr
, "qemu: tried to set invalid watchpoint at "
1319 TARGET_FMT_lx
", len=" TARGET_FMT_lu
"\n", addr
, len
);
1322 wp
= qemu_malloc(sizeof(*wp
));
1327 wp
->len_mask
= len_mask
;
1330 /* keep all GDB-injected watchpoints in front */
1332 TAILQ_INSERT_HEAD(&env
->watchpoints
, wp
, entry
);
1334 TAILQ_INSERT_TAIL(&env
->watchpoints
, wp
, entry
);
1336 tlb_flush_page(env
, addr
);
1343 /* Remove a specific watchpoint. */
1344 int cpu_watchpoint_remove(CPUState
*env
, target_ulong addr
, target_ulong len
,
1347 target_ulong len_mask
= ~(len
- 1);
1350 TAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1351 if (addr
== wp
->vaddr
&& len_mask
== wp
->len_mask
1352 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
1353 cpu_watchpoint_remove_by_ref(env
, wp
);
1360 /* Remove a specific watchpoint by reference. */
1361 void cpu_watchpoint_remove_by_ref(CPUState
*env
, CPUWatchpoint
*watchpoint
)
1363 TAILQ_REMOVE(&env
->watchpoints
, watchpoint
, entry
);
1365 tlb_flush_page(env
, watchpoint
->vaddr
);
1367 qemu_free(watchpoint
);
1370 /* Remove all matching watchpoints. */
1371 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1373 CPUWatchpoint
*wp
, *next
;
1375 TAILQ_FOREACH_SAFE(wp
, &env
->watchpoints
, entry
, next
) {
1376 if (wp
->flags
& mask
)
1377 cpu_watchpoint_remove_by_ref(env
, wp
);
1381 /* Add a breakpoint. */
1382 int cpu_breakpoint_insert(CPUState
*env
, target_ulong pc
, int flags
,
1383 CPUBreakpoint
**breakpoint
)
1385 #if defined(TARGET_HAS_ICE)
1388 bp
= qemu_malloc(sizeof(*bp
));
1395 /* keep all GDB-injected breakpoints in front */
1397 TAILQ_INSERT_HEAD(&env
->breakpoints
, bp
, entry
);
1399 TAILQ_INSERT_TAIL(&env
->breakpoints
, bp
, entry
);
1401 breakpoint_invalidate(env
, pc
);
1411 /* Remove a specific breakpoint. */
1412 int cpu_breakpoint_remove(CPUState
*env
, target_ulong pc
, int flags
)
1414 #if defined(TARGET_HAS_ICE)
1417 TAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1418 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
1419 cpu_breakpoint_remove_by_ref(env
, bp
);
1429 /* Remove a specific breakpoint by reference. */
1430 void cpu_breakpoint_remove_by_ref(CPUState
*env
, CPUBreakpoint
*breakpoint
)
1432 #if defined(TARGET_HAS_ICE)
1433 TAILQ_REMOVE(&env
->breakpoints
, breakpoint
, entry
);
1435 breakpoint_invalidate(env
, breakpoint
->pc
);
1437 qemu_free(breakpoint
);
1441 /* Remove all matching breakpoints. */
1442 void cpu_breakpoint_remove_all(CPUState
*env
, int mask
)
1444 #if defined(TARGET_HAS_ICE)
1445 CPUBreakpoint
*bp
, *next
;
1447 TAILQ_FOREACH_SAFE(bp
, &env
->breakpoints
, entry
, next
) {
1448 if (bp
->flags
& mask
)
1449 cpu_breakpoint_remove_by_ref(env
, bp
);
1454 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1455 CPU loop after each instruction */
1456 void cpu_single_step(CPUState
*env
, int enabled
)
1458 #if defined(TARGET_HAS_ICE)
1459 if (env
->singlestep_enabled
!= enabled
) {
1460 env
->singlestep_enabled
= enabled
;
1461 /* must flush all the translated code to avoid inconsistancies */
1462 /* XXX: only flush what is necessary */
1468 /* enable or disable low levels log */
1469 void cpu_set_log(int log_flags
)
1471 loglevel
= log_flags
;
1472 if (loglevel
&& !logfile
) {
1473 logfile
= fopen(logfilename
, log_append
? "a" : "w");
1475 perror(logfilename
);
1478 #if !defined(CONFIG_SOFTMMU)
1479 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1481 static char logfile_buf
[4096];
1482 setvbuf(logfile
, logfile_buf
, _IOLBF
, sizeof(logfile_buf
));
1485 setvbuf(logfile
, NULL
, _IOLBF
, 0);
1489 if (!loglevel
&& logfile
) {
1495 void cpu_set_log_filename(const char *filename
)
1497 logfilename
= strdup(filename
);
1502 cpu_set_log(loglevel
);
1505 /* mask must never be zero, except for A20 change call */
1506 void cpu_interrupt(CPUState
*env
, int mask
)
1508 #if !defined(USE_NPTL)
1509 TranslationBlock
*tb
;
1510 static spinlock_t interrupt_lock
= SPIN_LOCK_UNLOCKED
;
1514 old_mask
= env
->interrupt_request
;
1515 /* FIXME: This is probably not threadsafe. A different thread could
1516 be in the middle of a read-modify-write operation. */
1517 env
->interrupt_request
|= mask
;
1518 #if defined(USE_NPTL)
1519 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1520 problem and hope the cpu will stop of its own accord. For userspace
1521 emulation this often isn't actually as bad as it sounds. Often
1522 signals are used primarily to interrupt blocking syscalls. */
1525 env
->icount_decr
.u16
.high
= 0xffff;
1526 #ifndef CONFIG_USER_ONLY
1527 /* CPU_INTERRUPT_EXIT isn't a real interrupt. It just means
1528 an async event happened and we need to process it. */
1530 && (mask
& ~(old_mask
| CPU_INTERRUPT_EXIT
)) != 0) {
1531 cpu_abort(env
, "Raised interrupt while not in I/O function");
1535 tb
= env
->current_tb
;
1536 /* if the cpu is currently executing code, we must unlink it and
1537 all the potentially executing TB */
1538 if (tb
&& !testandset(&interrupt_lock
)) {
1539 env
->current_tb
= NULL
;
1540 tb_reset_jump_recursive(tb
);
1541 resetlock(&interrupt_lock
);
1547 void cpu_reset_interrupt(CPUState
*env
, int mask
)
1549 env
->interrupt_request
&= ~mask
;
1552 const CPULogItem cpu_log_items
[] = {
1553 { CPU_LOG_TB_OUT_ASM
, "out_asm",
1554 "show generated host assembly code for each compiled TB" },
1555 { CPU_LOG_TB_IN_ASM
, "in_asm",
1556 "show target assembly code for each compiled TB" },
1557 { CPU_LOG_TB_OP
, "op",
1558 "show micro ops for each compiled TB" },
1559 { CPU_LOG_TB_OP_OPT
, "op_opt",
1562 "before eflags optimization and "
1564 "after liveness analysis" },
1565 { CPU_LOG_INT
, "int",
1566 "show interrupts/exceptions in short format" },
1567 { CPU_LOG_EXEC
, "exec",
1568 "show trace before each executed TB (lots of logs)" },
1569 { CPU_LOG_TB_CPU
, "cpu",
1570 "show CPU state before block translation" },
1572 { CPU_LOG_PCALL
, "pcall",
1573 "show protected mode far calls/returns/exceptions" },
1576 { CPU_LOG_IOPORT
, "ioport",
1577 "show all i/o ports accesses" },
1582 static int cmp1(const char *s1
, int n
, const char *s2
)
1584 if (strlen(s2
) != n
)
1586 return memcmp(s1
, s2
, n
) == 0;
1589 /* takes a comma separated list of log masks. Return 0 if error. */
1590 int cpu_str_to_log_mask(const char *str
)
1592 const CPULogItem
*item
;
1599 p1
= strchr(p
, ',');
1602 if(cmp1(p
,p1
-p
,"all")) {
1603 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1607 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1608 if (cmp1(p
, p1
- p
, item
->name
))
1622 void cpu_abort(CPUState
*env
, const char *fmt
, ...)
1629 fprintf(stderr
, "qemu: fatal: ");
1630 vfprintf(stderr
, fmt
, ap
);
1631 fprintf(stderr
, "\n");
1633 cpu_dump_state(env
, stderr
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1635 cpu_dump_state(env
, stderr
, fprintf
, 0);
1638 fprintf(logfile
, "qemu: fatal: ");
1639 vfprintf(logfile
, fmt
, ap2
);
1640 fprintf(logfile
, "\n");
1642 cpu_dump_state(env
, logfile
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1644 cpu_dump_state(env
, logfile
, fprintf
, 0);
1654 CPUState
*cpu_copy(CPUState
*env
)
1656 CPUState
*new_env
= cpu_init(env
->cpu_model_str
);
1657 CPUState
*next_cpu
= new_env
->next_cpu
;
1658 int cpu_index
= new_env
->cpu_index
;
1659 #if defined(TARGET_HAS_ICE)
1664 memcpy(new_env
, env
, sizeof(CPUState
));
1666 /* Preserve chaining and index. */
1667 new_env
->next_cpu
= next_cpu
;
1668 new_env
->cpu_index
= cpu_index
;
1670 /* Clone all break/watchpoints.
1671 Note: Once we support ptrace with hw-debug register access, make sure
1672 BP_CPU break/watchpoints are handled correctly on clone. */
1673 TAILQ_INIT(&env
->breakpoints
);
1674 TAILQ_INIT(&env
->watchpoints
);
1675 #if defined(TARGET_HAS_ICE)
1676 TAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1677 cpu_breakpoint_insert(new_env
, bp
->pc
, bp
->flags
, NULL
);
1679 TAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1680 cpu_watchpoint_insert(new_env
, wp
->vaddr
, (~wp
->len_mask
) + 1,
1688 #if !defined(CONFIG_USER_ONLY)
1690 static inline void tlb_flush_jmp_cache(CPUState
*env
, target_ulong addr
)
1694 /* Discard jump cache entries for any tb which might potentially
1695 overlap the flushed page. */
1696 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1697 memset (&env
->tb_jmp_cache
[i
], 0,
1698 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1700 i
= tb_jmp_cache_hash_page(addr
);
1701 memset (&env
->tb_jmp_cache
[i
], 0,
1702 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1705 /* NOTE: if flush_global is true, also flush global entries (not
1707 void tlb_flush(CPUState
*env
, int flush_global
)
1711 #if defined(DEBUG_TLB)
1712 printf("tlb_flush:\n");
1714 /* must reset current TB so that interrupts cannot modify the
1715 links while we are modifying them */
1716 env
->current_tb
= NULL
;
1718 for(i
= 0; i
< CPU_TLB_SIZE
; i
++) {
1719 env
->tlb_table
[0][i
].addr_read
= -1;
1720 env
->tlb_table
[0][i
].addr_write
= -1;
1721 env
->tlb_table
[0][i
].addr_code
= -1;
1722 env
->tlb_table
[1][i
].addr_read
= -1;
1723 env
->tlb_table
[1][i
].addr_write
= -1;
1724 env
->tlb_table
[1][i
].addr_code
= -1;
1725 #if (NB_MMU_MODES >= 3)
1726 env
->tlb_table
[2][i
].addr_read
= -1;
1727 env
->tlb_table
[2][i
].addr_write
= -1;
1728 env
->tlb_table
[2][i
].addr_code
= -1;
1729 #if (NB_MMU_MODES == 4)
1730 env
->tlb_table
[3][i
].addr_read
= -1;
1731 env
->tlb_table
[3][i
].addr_write
= -1;
1732 env
->tlb_table
[3][i
].addr_code
= -1;
1737 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
1740 if (env
->kqemu_enabled
) {
1741 kqemu_flush(env
, flush_global
);
1747 static inline void tlb_flush_entry(CPUTLBEntry
*tlb_entry
, target_ulong addr
)
1749 if (addr
== (tlb_entry
->addr_read
&
1750 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1751 addr
== (tlb_entry
->addr_write
&
1752 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1753 addr
== (tlb_entry
->addr_code
&
1754 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
))) {
1755 tlb_entry
->addr_read
= -1;
1756 tlb_entry
->addr_write
= -1;
1757 tlb_entry
->addr_code
= -1;
1761 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
1765 #if defined(DEBUG_TLB)
1766 printf("tlb_flush_page: " TARGET_FMT_lx
"\n", addr
);
1768 /* must reset current TB so that interrupts cannot modify the
1769 links while we are modifying them */
1770 env
->current_tb
= NULL
;
1772 addr
&= TARGET_PAGE_MASK
;
1773 i
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1774 tlb_flush_entry(&env
->tlb_table
[0][i
], addr
);
1775 tlb_flush_entry(&env
->tlb_table
[1][i
], addr
);
1776 #if (NB_MMU_MODES >= 3)
1777 tlb_flush_entry(&env
->tlb_table
[2][i
], addr
);
1778 #if (NB_MMU_MODES == 4)
1779 tlb_flush_entry(&env
->tlb_table
[3][i
], addr
);
1783 tlb_flush_jmp_cache(env
, addr
);
1786 if (env
->kqemu_enabled
) {
1787 kqemu_flush_page(env
, addr
);
1792 /* update the TLBs so that writes to code in the virtual page 'addr'
1794 static void tlb_protect_code(ram_addr_t ram_addr
)
1796 cpu_physical_memory_reset_dirty(ram_addr
,
1797 ram_addr
+ TARGET_PAGE_SIZE
,
1801 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1802 tested for self modifying code */
1803 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
1806 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] |= CODE_DIRTY_FLAG
;
1809 static inline void tlb_reset_dirty_range(CPUTLBEntry
*tlb_entry
,
1810 unsigned long start
, unsigned long length
)
1813 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
1814 addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) + tlb_entry
->addend
;
1815 if ((addr
- start
) < length
) {
1816 tlb_entry
->addr_write
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) | TLB_NOTDIRTY
;
1821 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t end
,
1825 unsigned long length
, start1
;
1829 start
&= TARGET_PAGE_MASK
;
1830 end
= TARGET_PAGE_ALIGN(end
);
1832 length
= end
- start
;
1835 len
= length
>> TARGET_PAGE_BITS
;
1837 /* XXX: should not depend on cpu context */
1839 if (env
->kqemu_enabled
) {
1842 for(i
= 0; i
< len
; i
++) {
1843 kqemu_set_notdirty(env
, addr
);
1844 addr
+= TARGET_PAGE_SIZE
;
1848 mask
= ~dirty_flags
;
1849 p
= phys_ram_dirty
+ (start
>> TARGET_PAGE_BITS
);
1850 for(i
= 0; i
< len
; i
++)
1853 /* we modify the TLB cache so that the dirty bit will be set again
1854 when accessing the range */
1855 start1
= start
+ (unsigned long)phys_ram_base
;
1856 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
1857 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1858 tlb_reset_dirty_range(&env
->tlb_table
[0][i
], start1
, length
);
1859 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1860 tlb_reset_dirty_range(&env
->tlb_table
[1][i
], start1
, length
);
1861 #if (NB_MMU_MODES >= 3)
1862 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1863 tlb_reset_dirty_range(&env
->tlb_table
[2][i
], start1
, length
);
1864 #if (NB_MMU_MODES == 4)
1865 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1866 tlb_reset_dirty_range(&env
->tlb_table
[3][i
], start1
, length
);
1872 int cpu_physical_memory_set_dirty_tracking(int enable
)
1874 in_migration
= enable
;
1878 int cpu_physical_memory_get_dirty_tracking(void)
1880 return in_migration
;
1883 void cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr
, target_phys_addr_t end_addr
)
1886 kvm_physical_sync_dirty_bitmap(start_addr
, end_addr
);
1889 static inline void tlb_update_dirty(CPUTLBEntry
*tlb_entry
)
1891 ram_addr_t ram_addr
;
1893 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
1894 ram_addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) +
1895 tlb_entry
->addend
- (unsigned long)phys_ram_base
;
1896 if (!cpu_physical_memory_is_dirty(ram_addr
)) {
1897 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
1902 /* update the TLB according to the current state of the dirty bits */
1903 void cpu_tlb_update_dirty(CPUState
*env
)
1906 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1907 tlb_update_dirty(&env
->tlb_table
[0][i
]);
1908 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1909 tlb_update_dirty(&env
->tlb_table
[1][i
]);
1910 #if (NB_MMU_MODES >= 3)
1911 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1912 tlb_update_dirty(&env
->tlb_table
[2][i
]);
1913 #if (NB_MMU_MODES == 4)
1914 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1915 tlb_update_dirty(&env
->tlb_table
[3][i
]);
1920 static inline void tlb_set_dirty1(CPUTLBEntry
*tlb_entry
, target_ulong vaddr
)
1922 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
))
1923 tlb_entry
->addr_write
= vaddr
;
1926 /* update the TLB corresponding to virtual page vaddr
1927 so that it is no longer dirty */
1928 static inline void tlb_set_dirty(CPUState
*env
, target_ulong vaddr
)
1932 vaddr
&= TARGET_PAGE_MASK
;
1933 i
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1934 tlb_set_dirty1(&env
->tlb_table
[0][i
], vaddr
);
1935 tlb_set_dirty1(&env
->tlb_table
[1][i
], vaddr
);
1936 #if (NB_MMU_MODES >= 3)
1937 tlb_set_dirty1(&env
->tlb_table
[2][i
], vaddr
);
1938 #if (NB_MMU_MODES == 4)
1939 tlb_set_dirty1(&env
->tlb_table
[3][i
], vaddr
);
1944 /* add a new TLB entry. At most one entry for a given virtual address
1945 is permitted. Return 0 if OK or 2 if the page could not be mapped
1946 (can only happen in non SOFTMMU mode for I/O pages or pages
1947 conflicting with the host address space). */
1948 int tlb_set_page_exec(CPUState
*env
, target_ulong vaddr
,
1949 target_phys_addr_t paddr
, int prot
,
1950 int mmu_idx
, int is_softmmu
)
1955 target_ulong address
;
1956 target_ulong code_address
;
1957 target_phys_addr_t addend
;
1961 target_phys_addr_t iotlb
;
1963 p
= phys_page_find(paddr
>> TARGET_PAGE_BITS
);
1965 pd
= IO_MEM_UNASSIGNED
;
1967 pd
= p
->phys_offset
;
1969 #if defined(DEBUG_TLB)
1970 printf("tlb_set_page: vaddr=" TARGET_FMT_lx
" paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1971 vaddr
, (int)paddr
, prot
, mmu_idx
, is_softmmu
, pd
);
1976 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&& !(pd
& IO_MEM_ROMD
)) {
1977 /* IO memory case (romd handled later) */
1978 address
|= TLB_MMIO
;
1980 addend
= (unsigned long)phys_ram_base
+ (pd
& TARGET_PAGE_MASK
);
1981 if ((pd
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
) {
1983 iotlb
= pd
& TARGET_PAGE_MASK
;
1984 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
)
1985 iotlb
|= IO_MEM_NOTDIRTY
;
1987 iotlb
|= IO_MEM_ROM
;
1989 /* IO handlers are currently passed a phsical address.
1990 It would be nice to pass an offset from the base address
1991 of that region. This would avoid having to special case RAM,
1992 and avoid full address decoding in every device.
1993 We can't use the high bits of pd for this because
1994 IO_MEM_ROMD uses these as a ram address. */
1995 iotlb
= (pd
& ~TARGET_PAGE_MASK
);
1997 iotlb
+= p
->region_offset
;
2003 code_address
= address
;
2004 /* Make accesses to pages with watchpoints go via the
2005 watchpoint trap routines. */
2006 TAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
2007 if (vaddr
== (wp
->vaddr
& TARGET_PAGE_MASK
)) {
2008 iotlb
= io_mem_watch
+ paddr
;
2009 /* TODO: The memory case can be optimized by not trapping
2010 reads of pages with a write breakpoint. */
2011 address
|= TLB_MMIO
;
2015 index
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2016 env
->iotlb
[mmu_idx
][index
] = iotlb
- vaddr
;
2017 te
= &env
->tlb_table
[mmu_idx
][index
];
2018 te
->addend
= addend
- vaddr
;
2019 if (prot
& PAGE_READ
) {
2020 te
->addr_read
= address
;
2025 if (prot
& PAGE_EXEC
) {
2026 te
->addr_code
= code_address
;
2030 if (prot
& PAGE_WRITE
) {
2031 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_ROM
||
2032 (pd
& IO_MEM_ROMD
)) {
2033 /* Write access calls the I/O callback. */
2034 te
->addr_write
= address
| TLB_MMIO
;
2035 } else if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
&&
2036 !cpu_physical_memory_is_dirty(pd
)) {
2037 te
->addr_write
= address
| TLB_NOTDIRTY
;
2039 te
->addr_write
= address
;
2042 te
->addr_write
= -1;
2049 void tlb_flush(CPUState
*env
, int flush_global
)
2053 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
2057 int tlb_set_page_exec(CPUState
*env
, target_ulong vaddr
,
2058 target_phys_addr_t paddr
, int prot
,
2059 int mmu_idx
, int is_softmmu
)
2064 /* dump memory mappings */
2065 void page_dump(FILE *f
)
2067 unsigned long start
, end
;
2068 int i
, j
, prot
, prot1
;
2071 fprintf(f
, "%-8s %-8s %-8s %s\n",
2072 "start", "end", "size", "prot");
2076 for(i
= 0; i
<= L1_SIZE
; i
++) {
2081 for(j
= 0;j
< L2_SIZE
; j
++) {
2086 if (prot1
!= prot
) {
2087 end
= (i
<< (32 - L1_BITS
)) | (j
<< TARGET_PAGE_BITS
);
2089 fprintf(f
, "%08lx-%08lx %08lx %c%c%c\n",
2090 start
, end
, end
- start
,
2091 prot
& PAGE_READ
? 'r' : '-',
2092 prot
& PAGE_WRITE
? 'w' : '-',
2093 prot
& PAGE_EXEC
? 'x' : '-');
2107 int page_get_flags(target_ulong address
)
2111 p
= page_find(address
>> TARGET_PAGE_BITS
);
2117 /* modify the flags of a page and invalidate the code if
2118 necessary. The flag PAGE_WRITE_ORG is positionned automatically
2119 depending on PAGE_WRITE */
2120 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2125 /* mmap_lock should already be held. */
2126 start
= start
& TARGET_PAGE_MASK
;
2127 end
= TARGET_PAGE_ALIGN(end
);
2128 if (flags
& PAGE_WRITE
)
2129 flags
|= PAGE_WRITE_ORG
;
2130 for(addr
= start
; addr
< end
; addr
+= TARGET_PAGE_SIZE
) {
2131 p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
);
2132 /* We may be called for host regions that are outside guest
2136 /* if the write protection is set, then we invalidate the code
2138 if (!(p
->flags
& PAGE_WRITE
) &&
2139 (flags
& PAGE_WRITE
) &&
2141 tb_invalidate_phys_page(addr
, 0, NULL
);
2147 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2153 if (start
+ len
< start
)
2154 /* we've wrapped around */
2157 end
= TARGET_PAGE_ALIGN(start
+len
); /* must do before we loose bits in the next step */
2158 start
= start
& TARGET_PAGE_MASK
;
2160 for(addr
= start
; addr
< end
; addr
+= TARGET_PAGE_SIZE
) {
2161 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2164 if( !(p
->flags
& PAGE_VALID
) )
2167 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
))
2169 if (flags
& PAGE_WRITE
) {
2170 if (!(p
->flags
& PAGE_WRITE_ORG
))
2172 /* unprotect the page if it was put read-only because it
2173 contains translated code */
2174 if (!(p
->flags
& PAGE_WRITE
)) {
2175 if (!page_unprotect(addr
, 0, NULL
))
2184 /* called from signal handler: invalidate the code and unprotect the
2185 page. Return TRUE if the fault was succesfully handled. */
2186 int page_unprotect(target_ulong address
, unsigned long pc
, void *puc
)
2188 unsigned int page_index
, prot
, pindex
;
2190 target_ulong host_start
, host_end
, addr
;
2192 /* Technically this isn't safe inside a signal handler. However we
2193 know this only ever happens in a synchronous SEGV handler, so in
2194 practice it seems to be ok. */
2197 host_start
= address
& qemu_host_page_mask
;
2198 page_index
= host_start
>> TARGET_PAGE_BITS
;
2199 p1
= page_find(page_index
);
2204 host_end
= host_start
+ qemu_host_page_size
;
2207 for(addr
= host_start
;addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2211 /* if the page was really writable, then we change its
2212 protection back to writable */
2213 if (prot
& PAGE_WRITE_ORG
) {
2214 pindex
= (address
- host_start
) >> TARGET_PAGE_BITS
;
2215 if (!(p1
[pindex
].flags
& PAGE_WRITE
)) {
2216 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2217 (prot
& PAGE_BITS
) | PAGE_WRITE
);
2218 p1
[pindex
].flags
|= PAGE_WRITE
;
2219 /* and since the content will be modified, we must invalidate
2220 the corresponding translated code. */
2221 tb_invalidate_phys_page(address
, pc
, puc
);
2222 #ifdef DEBUG_TB_CHECK
2223 tb_invalidate_check(address
);
2233 static inline void tlb_set_dirty(CPUState
*env
,
2234 unsigned long addr
, target_ulong vaddr
)
2237 #endif /* defined(CONFIG_USER_ONLY) */
2239 #if !defined(CONFIG_USER_ONLY)
2241 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2242 ram_addr_t memory
, ram_addr_t region_offset
);
2243 static void *subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
2244 ram_addr_t orig_memory
, ram_addr_t region_offset
);
2245 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2248 if (addr > start_addr) \
2251 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2252 if (start_addr2 > 0) \
2256 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2257 end_addr2 = TARGET_PAGE_SIZE - 1; \
2259 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2260 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2265 /* register physical memory. 'size' must be a multiple of the target
2266 page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2267 io memory page. The address used when calling the IO function is
2268 the offset from the start of the region, plus region_offset. Both
2269 start_region and regon_offset are rounded down to a page boundary
2270 before calculating this offset. This should not be a problem unless
2271 the low bits of start_addr and region_offset differ. */
2272 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr
,
2274 ram_addr_t phys_offset
,
2275 ram_addr_t region_offset
)
2277 target_phys_addr_t addr
, end_addr
;
2280 ram_addr_t orig_size
= size
;
2284 /* XXX: should not depend on cpu context */
2286 if (env
->kqemu_enabled
) {
2287 kqemu_set_phys_mem(start_addr
, size
, phys_offset
);
2291 kvm_set_phys_mem(start_addr
, size
, phys_offset
);
2293 region_offset
&= TARGET_PAGE_MASK
;
2294 size
= (size
+ TARGET_PAGE_SIZE
- 1) & TARGET_PAGE_MASK
;
2295 end_addr
= start_addr
+ (target_phys_addr_t
)size
;
2296 for(addr
= start_addr
; addr
!= end_addr
; addr
+= TARGET_PAGE_SIZE
) {
2297 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2298 if (p
&& p
->phys_offset
!= IO_MEM_UNASSIGNED
) {
2299 ram_addr_t orig_memory
= p
->phys_offset
;
2300 target_phys_addr_t start_addr2
, end_addr2
;
2301 int need_subpage
= 0;
2303 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
, end_addr2
,
2305 if (need_subpage
|| phys_offset
& IO_MEM_SUBWIDTH
) {
2306 if (!(orig_memory
& IO_MEM_SUBPAGE
)) {
2307 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2308 &p
->phys_offset
, orig_memory
,
2311 subpage
= io_mem_opaque
[(orig_memory
& ~TARGET_PAGE_MASK
)
2314 subpage_register(subpage
, start_addr2
, end_addr2
, phys_offset
,
2316 p
->region_offset
= 0;
2318 p
->phys_offset
= phys_offset
;
2319 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2320 (phys_offset
& IO_MEM_ROMD
))
2321 phys_offset
+= TARGET_PAGE_SIZE
;
2324 p
= phys_page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2325 p
->phys_offset
= phys_offset
;
2326 p
->region_offset
= region_offset
;
2327 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2328 (phys_offset
& IO_MEM_ROMD
)) {
2329 phys_offset
+= TARGET_PAGE_SIZE
;
2331 target_phys_addr_t start_addr2
, end_addr2
;
2332 int need_subpage
= 0;
2334 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
,
2335 end_addr2
, need_subpage
);
2337 if (need_subpage
|| phys_offset
& IO_MEM_SUBWIDTH
) {
2338 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2339 &p
->phys_offset
, IO_MEM_UNASSIGNED
,
2341 subpage_register(subpage
, start_addr2
, end_addr2
,
2342 phys_offset
, region_offset
);
2343 p
->region_offset
= 0;
2347 region_offset
+= TARGET_PAGE_SIZE
;
2350 /* since each CPU stores ram addresses in its TLB cache, we must
2351 reset the modified entries */
2353 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2358 /* XXX: temporary until new memory mapping API */
2359 ram_addr_t
cpu_get_physical_page_desc(target_phys_addr_t addr
)
2363 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2365 return IO_MEM_UNASSIGNED
;
2366 return p
->phys_offset
;
2369 void qemu_register_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2372 kvm_coalesce_mmio_region(addr
, size
);
2375 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2378 kvm_uncoalesce_mmio_region(addr
, size
);
2381 /* XXX: better than nothing */
2382 ram_addr_t
qemu_ram_alloc(ram_addr_t size
)
2385 if ((phys_ram_alloc_offset
+ size
) > phys_ram_size
) {
2386 fprintf(stderr
, "Not enough memory (requested_size = %" PRIu64
", max memory = %" PRIu64
")\n",
2387 (uint64_t)size
, (uint64_t)phys_ram_size
);
2390 addr
= phys_ram_alloc_offset
;
2391 phys_ram_alloc_offset
= TARGET_PAGE_ALIGN(phys_ram_alloc_offset
+ size
);
2395 void qemu_ram_free(ram_addr_t addr
)
2399 static uint32_t unassigned_mem_readb(void *opaque
, target_phys_addr_t addr
)
2401 #ifdef DEBUG_UNASSIGNED
2402 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2404 #if defined(TARGET_SPARC)
2405 do_unassigned_access(addr
, 0, 0, 0, 1);
2410 static uint32_t unassigned_mem_readw(void *opaque
, target_phys_addr_t addr
)
2412 #ifdef DEBUG_UNASSIGNED
2413 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2415 #if defined(TARGET_SPARC)
2416 do_unassigned_access(addr
, 0, 0, 0, 2);
2421 static uint32_t unassigned_mem_readl(void *opaque
, target_phys_addr_t addr
)
2423 #ifdef DEBUG_UNASSIGNED
2424 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2426 #if defined(TARGET_SPARC)
2427 do_unassigned_access(addr
, 0, 0, 0, 4);
2432 static void unassigned_mem_writeb(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2434 #ifdef DEBUG_UNASSIGNED
2435 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2437 #if defined(TARGET_SPARC)
2438 do_unassigned_access(addr
, 1, 0, 0, 1);
2442 static void unassigned_mem_writew(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2444 #ifdef DEBUG_UNASSIGNED
2445 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2447 #if defined(TARGET_SPARC)
2448 do_unassigned_access(addr
, 1, 0, 0, 2);
2452 static void unassigned_mem_writel(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2454 #ifdef DEBUG_UNASSIGNED
2455 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2457 #if defined(TARGET_SPARC)
2458 do_unassigned_access(addr
, 1, 0, 0, 4);
2462 static CPUReadMemoryFunc
*unassigned_mem_read
[3] = {
2463 unassigned_mem_readb
,
2464 unassigned_mem_readw
,
2465 unassigned_mem_readl
,
2468 static CPUWriteMemoryFunc
*unassigned_mem_write
[3] = {
2469 unassigned_mem_writeb
,
2470 unassigned_mem_writew
,
2471 unassigned_mem_writel
,
2474 static void notdirty_mem_writeb(void *opaque
, target_phys_addr_t ram_addr
,
2478 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2479 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2480 #if !defined(CONFIG_USER_ONLY)
2481 tb_invalidate_phys_page_fast(ram_addr
, 1);
2482 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2485 stb_p(phys_ram_base
+ ram_addr
, val
);
2487 if (cpu_single_env
->kqemu_enabled
&&
2488 (dirty_flags
& KQEMU_MODIFY_PAGE_MASK
) != KQEMU_MODIFY_PAGE_MASK
)
2489 kqemu_modify_page(cpu_single_env
, ram_addr
);
2491 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2492 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2493 /* we remove the notdirty callback only if the code has been
2495 if (dirty_flags
== 0xff)
2496 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2499 static void notdirty_mem_writew(void *opaque
, target_phys_addr_t ram_addr
,
2503 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2504 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2505 #if !defined(CONFIG_USER_ONLY)
2506 tb_invalidate_phys_page_fast(ram_addr
, 2);
2507 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2510 stw_p(phys_ram_base
+ ram_addr
, val
);
2512 if (cpu_single_env
->kqemu_enabled
&&
2513 (dirty_flags
& KQEMU_MODIFY_PAGE_MASK
) != KQEMU_MODIFY_PAGE_MASK
)
2514 kqemu_modify_page(cpu_single_env
, ram_addr
);
2516 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2517 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2518 /* we remove the notdirty callback only if the code has been
2520 if (dirty_flags
== 0xff)
2521 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2524 static void notdirty_mem_writel(void *opaque
, target_phys_addr_t ram_addr
,
2528 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2529 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2530 #if !defined(CONFIG_USER_ONLY)
2531 tb_invalidate_phys_page_fast(ram_addr
, 4);
2532 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2535 stl_p(phys_ram_base
+ ram_addr
, val
);
2537 if (cpu_single_env
->kqemu_enabled
&&
2538 (dirty_flags
& KQEMU_MODIFY_PAGE_MASK
) != KQEMU_MODIFY_PAGE_MASK
)
2539 kqemu_modify_page(cpu_single_env
, ram_addr
);
2541 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2542 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2543 /* we remove the notdirty callback only if the code has been
2545 if (dirty_flags
== 0xff)
2546 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2549 static CPUReadMemoryFunc
*error_mem_read
[3] = {
2550 NULL
, /* never used */
2551 NULL
, /* never used */
2552 NULL
, /* never used */
2555 static CPUWriteMemoryFunc
*notdirty_mem_write
[3] = {
2556 notdirty_mem_writeb
,
2557 notdirty_mem_writew
,
2558 notdirty_mem_writel
,
2561 /* Generate a debug exception if a watchpoint has been hit. */
2562 static void check_watchpoint(int offset
, int len_mask
, int flags
)
2564 CPUState
*env
= cpu_single_env
;
2565 target_ulong pc
, cs_base
;
2566 TranslationBlock
*tb
;
2571 if (env
->watchpoint_hit
) {
2572 /* We re-entered the check after replacing the TB. Now raise
2573 * the debug interrupt so that is will trigger after the
2574 * current instruction. */
2575 cpu_interrupt(env
, CPU_INTERRUPT_DEBUG
);
2578 vaddr
= (env
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
2579 TAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
2580 if ((vaddr
== (wp
->vaddr
& len_mask
) ||
2581 (vaddr
& wp
->len_mask
) == wp
->vaddr
) && (wp
->flags
& flags
)) {
2582 wp
->flags
|= BP_WATCHPOINT_HIT
;
2583 if (!env
->watchpoint_hit
) {
2584 env
->watchpoint_hit
= wp
;
2585 tb
= tb_find_pc(env
->mem_io_pc
);
2587 cpu_abort(env
, "check_watchpoint: could not find TB for "
2588 "pc=%p", (void *)env
->mem_io_pc
);
2590 cpu_restore_state(tb
, env
, env
->mem_io_pc
, NULL
);
2591 tb_phys_invalidate(tb
, -1);
2592 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
2593 env
->exception_index
= EXCP_DEBUG
;
2595 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
2596 tb_gen_code(env
, pc
, cs_base
, cpu_flags
, 1);
2598 cpu_resume_from_signal(env
, NULL
);
2601 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
2606 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2607 so these check for a hit then pass through to the normal out-of-line
2609 static uint32_t watch_mem_readb(void *opaque
, target_phys_addr_t addr
)
2611 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_READ
);
2612 return ldub_phys(addr
);
2615 static uint32_t watch_mem_readw(void *opaque
, target_phys_addr_t addr
)
2617 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_READ
);
2618 return lduw_phys(addr
);
2621 static uint32_t watch_mem_readl(void *opaque
, target_phys_addr_t addr
)
2623 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_READ
);
2624 return ldl_phys(addr
);
2627 static void watch_mem_writeb(void *opaque
, target_phys_addr_t addr
,
2630 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_WRITE
);
2631 stb_phys(addr
, val
);
2634 static void watch_mem_writew(void *opaque
, target_phys_addr_t addr
,
2637 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_WRITE
);
2638 stw_phys(addr
, val
);
2641 static void watch_mem_writel(void *opaque
, target_phys_addr_t addr
,
2644 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_WRITE
);
2645 stl_phys(addr
, val
);
2648 static CPUReadMemoryFunc
*watch_mem_read
[3] = {
2654 static CPUWriteMemoryFunc
*watch_mem_write
[3] = {
2660 static inline uint32_t subpage_readlen (subpage_t
*mmio
, target_phys_addr_t addr
,
2666 idx
= SUBPAGE_IDX(addr
);
2667 #if defined(DEBUG_SUBPAGE)
2668 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d\n", __func__
,
2669 mmio
, len
, addr
, idx
);
2671 ret
= (**mmio
->mem_read
[idx
][len
])(mmio
->opaque
[idx
][0][len
],
2672 addr
+ mmio
->region_offset
[idx
][0][len
]);
2677 static inline void subpage_writelen (subpage_t
*mmio
, target_phys_addr_t addr
,
2678 uint32_t value
, unsigned int len
)
2682 idx
= SUBPAGE_IDX(addr
);
2683 #if defined(DEBUG_SUBPAGE)
2684 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d value %08x\n", __func__
,
2685 mmio
, len
, addr
, idx
, value
);
2687 (**mmio
->mem_write
[idx
][len
])(mmio
->opaque
[idx
][1][len
],
2688 addr
+ mmio
->region_offset
[idx
][1][len
],
2692 static uint32_t subpage_readb (void *opaque
, target_phys_addr_t addr
)
2694 #if defined(DEBUG_SUBPAGE)
2695 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2698 return subpage_readlen(opaque
, addr
, 0);
2701 static void subpage_writeb (void *opaque
, target_phys_addr_t addr
,
2704 #if defined(DEBUG_SUBPAGE)
2705 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2707 subpage_writelen(opaque
, addr
, value
, 0);
2710 static uint32_t subpage_readw (void *opaque
, target_phys_addr_t addr
)
2712 #if defined(DEBUG_SUBPAGE)
2713 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2716 return subpage_readlen(opaque
, addr
, 1);
2719 static void subpage_writew (void *opaque
, target_phys_addr_t addr
,
2722 #if defined(DEBUG_SUBPAGE)
2723 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2725 subpage_writelen(opaque
, addr
, value
, 1);
2728 static uint32_t subpage_readl (void *opaque
, target_phys_addr_t addr
)
2730 #if defined(DEBUG_SUBPAGE)
2731 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2734 return subpage_readlen(opaque
, addr
, 2);
2737 static void subpage_writel (void *opaque
,
2738 target_phys_addr_t addr
, uint32_t value
)
2740 #if defined(DEBUG_SUBPAGE)
2741 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2743 subpage_writelen(opaque
, addr
, value
, 2);
2746 static CPUReadMemoryFunc
*subpage_read
[] = {
2752 static CPUWriteMemoryFunc
*subpage_write
[] = {
2758 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2759 ram_addr_t memory
, ram_addr_t region_offset
)
2764 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
2766 idx
= SUBPAGE_IDX(start
);
2767 eidx
= SUBPAGE_IDX(end
);
2768 #if defined(DEBUG_SUBPAGE)
2769 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %d\n", __func__
,
2770 mmio
, start
, end
, idx
, eidx
, memory
);
2772 memory
>>= IO_MEM_SHIFT
;
2773 for (; idx
<= eidx
; idx
++) {
2774 for (i
= 0; i
< 4; i
++) {
2775 if (io_mem_read
[memory
][i
]) {
2776 mmio
->mem_read
[idx
][i
] = &io_mem_read
[memory
][i
];
2777 mmio
->opaque
[idx
][0][i
] = io_mem_opaque
[memory
];
2778 mmio
->region_offset
[idx
][0][i
] = region_offset
;
2780 if (io_mem_write
[memory
][i
]) {
2781 mmio
->mem_write
[idx
][i
] = &io_mem_write
[memory
][i
];
2782 mmio
->opaque
[idx
][1][i
] = io_mem_opaque
[memory
];
2783 mmio
->region_offset
[idx
][1][i
] = region_offset
;
2791 static void *subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
2792 ram_addr_t orig_memory
, ram_addr_t region_offset
)
2797 mmio
= qemu_mallocz(sizeof(subpage_t
));
2800 subpage_memory
= cpu_register_io_memory(0, subpage_read
, subpage_write
, mmio
);
2801 #if defined(DEBUG_SUBPAGE)
2802 printf("%s: %p base " TARGET_FMT_plx
" len %08x %d\n", __func__
,
2803 mmio
, base
, TARGET_PAGE_SIZE
, subpage_memory
);
2805 *phys
= subpage_memory
| IO_MEM_SUBPAGE
;
2806 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
- 1, orig_memory
,
2813 static void io_mem_init(void)
2815 cpu_register_io_memory(IO_MEM_ROM
>> IO_MEM_SHIFT
, error_mem_read
, unassigned_mem_write
, NULL
);
2816 cpu_register_io_memory(IO_MEM_UNASSIGNED
>> IO_MEM_SHIFT
, unassigned_mem_read
, unassigned_mem_write
, NULL
);
2817 cpu_register_io_memory(IO_MEM_NOTDIRTY
>> IO_MEM_SHIFT
, error_mem_read
, notdirty_mem_write
, NULL
);
2820 io_mem_watch
= cpu_register_io_memory(0, watch_mem_read
,
2821 watch_mem_write
, NULL
);
2822 /* alloc dirty bits array */
2823 phys_ram_dirty
= qemu_vmalloc(phys_ram_size
>> TARGET_PAGE_BITS
);
2824 memset(phys_ram_dirty
, 0xff, phys_ram_size
>> TARGET_PAGE_BITS
);
2827 /* mem_read and mem_write are arrays of functions containing the
2828 function to access byte (index 0), word (index 1) and dword (index
2829 2). Functions can be omitted with a NULL function pointer. The
2830 registered functions may be modified dynamically later.
2831 If io_index is non zero, the corresponding io zone is
2832 modified. If it is zero, a new io zone is allocated. The return
2833 value can be used with cpu_register_physical_memory(). (-1) is
2834 returned if error. */
2835 int cpu_register_io_memory(int io_index
,
2836 CPUReadMemoryFunc
**mem_read
,
2837 CPUWriteMemoryFunc
**mem_write
,
2840 int i
, subwidth
= 0;
2842 if (io_index
<= 0) {
2843 if (io_mem_nb
>= IO_MEM_NB_ENTRIES
)
2845 io_index
= io_mem_nb
++;
2847 if (io_index
>= IO_MEM_NB_ENTRIES
)
2851 for(i
= 0;i
< 3; i
++) {
2852 if (!mem_read
[i
] || !mem_write
[i
])
2853 subwidth
= IO_MEM_SUBWIDTH
;
2854 io_mem_read
[io_index
][i
] = mem_read
[i
];
2855 io_mem_write
[io_index
][i
] = mem_write
[i
];
2857 io_mem_opaque
[io_index
] = opaque
;
2858 return (io_index
<< IO_MEM_SHIFT
) | subwidth
;
2861 CPUWriteMemoryFunc
**cpu_get_io_memory_write(int io_index
)
2863 return io_mem_write
[io_index
>> IO_MEM_SHIFT
];
2866 CPUReadMemoryFunc
**cpu_get_io_memory_read(int io_index
)
2868 return io_mem_read
[io_index
>> IO_MEM_SHIFT
];
2871 #endif /* !defined(CONFIG_USER_ONLY) */
2873 /* physical memory access (slow version, mainly for debug) */
2874 #if defined(CONFIG_USER_ONLY)
2875 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
2876 int len
, int is_write
)
2883 page
= addr
& TARGET_PAGE_MASK
;
2884 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2887 flags
= page_get_flags(page
);
2888 if (!(flags
& PAGE_VALID
))
2891 if (!(flags
& PAGE_WRITE
))
2893 /* XXX: this code should not depend on lock_user */
2894 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
2895 /* FIXME - should this return an error rather than just fail? */
2898 unlock_user(p
, addr
, l
);
2900 if (!(flags
& PAGE_READ
))
2902 /* XXX: this code should not depend on lock_user */
2903 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
2904 /* FIXME - should this return an error rather than just fail? */
2907 unlock_user(p
, addr
, 0);
2916 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
2917 int len
, int is_write
)
2922 target_phys_addr_t page
;
2927 page
= addr
& TARGET_PAGE_MASK
;
2928 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2931 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
2933 pd
= IO_MEM_UNASSIGNED
;
2935 pd
= p
->phys_offset
;
2939 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
2940 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
2942 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
2943 /* XXX: could force cpu_single_env to NULL to avoid
2945 if (l
>= 4 && ((addr
& 3) == 0)) {
2946 /* 32 bit write access */
2948 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
2950 } else if (l
>= 2 && ((addr
& 1) == 0)) {
2951 /* 16 bit write access */
2953 io_mem_write
[io_index
][1](io_mem_opaque
[io_index
], addr
, val
);
2956 /* 8 bit write access */
2958 io_mem_write
[io_index
][0](io_mem_opaque
[io_index
], addr
, val
);
2962 unsigned long addr1
;
2963 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
2965 ptr
= phys_ram_base
+ addr1
;
2966 memcpy(ptr
, buf
, l
);
2967 if (!cpu_physical_memory_is_dirty(addr1
)) {
2968 /* invalidate code */
2969 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
2971 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
2972 (0xff & ~CODE_DIRTY_FLAG
);
2976 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
2977 !(pd
& IO_MEM_ROMD
)) {
2979 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
2981 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
2982 if (l
>= 4 && ((addr
& 3) == 0)) {
2983 /* 32 bit read access */
2984 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
2987 } else if (l
>= 2 && ((addr
& 1) == 0)) {
2988 /* 16 bit read access */
2989 val
= io_mem_read
[io_index
][1](io_mem_opaque
[io_index
], addr
);
2993 /* 8 bit read access */
2994 val
= io_mem_read
[io_index
][0](io_mem_opaque
[io_index
], addr
);
3000 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
3001 (addr
& ~TARGET_PAGE_MASK
);
3002 memcpy(buf
, ptr
, l
);
3011 /* used for ROM loading : can write in RAM and ROM */
3012 void cpu_physical_memory_write_rom(target_phys_addr_t addr
,
3013 const uint8_t *buf
, int len
)
3017 target_phys_addr_t page
;
3022 page
= addr
& TARGET_PAGE_MASK
;
3023 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3026 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3028 pd
= IO_MEM_UNASSIGNED
;
3030 pd
= p
->phys_offset
;
3033 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
&&
3034 (pd
& ~TARGET_PAGE_MASK
) != IO_MEM_ROM
&&
3035 !(pd
& IO_MEM_ROMD
)) {
3038 unsigned long addr1
;
3039 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3041 ptr
= phys_ram_base
+ addr1
;
3042 memcpy(ptr
, buf
, l
);
3051 /* warning: addr must be aligned */
3052 uint32_t ldl_phys(target_phys_addr_t addr
)
3060 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3062 pd
= IO_MEM_UNASSIGNED
;
3064 pd
= p
->phys_offset
;
3067 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3068 !(pd
& IO_MEM_ROMD
)) {
3070 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3072 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3073 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
3076 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
3077 (addr
& ~TARGET_PAGE_MASK
);
3083 /* warning: addr must be aligned */
3084 uint64_t ldq_phys(target_phys_addr_t addr
)
3092 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3094 pd
= IO_MEM_UNASSIGNED
;
3096 pd
= p
->phys_offset
;
3099 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3100 !(pd
& IO_MEM_ROMD
)) {
3102 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3104 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3105 #ifdef TARGET_WORDS_BIGENDIAN
3106 val
= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
) << 32;
3107 val
|= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4);
3109 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
3110 val
|= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4) << 32;
3114 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
3115 (addr
& ~TARGET_PAGE_MASK
);
3122 uint32_t ldub_phys(target_phys_addr_t addr
)
3125 cpu_physical_memory_read(addr
, &val
, 1);
3130 uint32_t lduw_phys(target_phys_addr_t addr
)
3133 cpu_physical_memory_read(addr
, (uint8_t *)&val
, 2);
3134 return tswap16(val
);
3137 /* warning: addr must be aligned. The ram page is not masked as dirty
3138 and the code inside is not invalidated. It is useful if the dirty
3139 bits are used to track modified PTEs */
3140 void stl_phys_notdirty(target_phys_addr_t addr
, uint32_t val
)
3147 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3149 pd
= IO_MEM_UNASSIGNED
;
3151 pd
= p
->phys_offset
;
3154 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3155 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3157 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3158 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3160 unsigned long addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3161 ptr
= phys_ram_base
+ addr1
;
3164 if (unlikely(in_migration
)) {
3165 if (!cpu_physical_memory_is_dirty(addr1
)) {
3166 /* invalidate code */
3167 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3169 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3170 (0xff & ~CODE_DIRTY_FLAG
);
3176 void stq_phys_notdirty(target_phys_addr_t addr
, uint64_t val
)
3183 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3185 pd
= IO_MEM_UNASSIGNED
;
3187 pd
= p
->phys_offset
;
3190 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3191 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3193 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3194 #ifdef TARGET_WORDS_BIGENDIAN
3195 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
>> 32);
3196 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
);
3198 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3199 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
>> 32);
3202 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
3203 (addr
& ~TARGET_PAGE_MASK
);
3208 /* warning: addr must be aligned */
3209 void stl_phys(target_phys_addr_t addr
, uint32_t val
)
3216 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3218 pd
= IO_MEM_UNASSIGNED
;
3220 pd
= p
->phys_offset
;
3223 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3224 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3226 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3227 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3229 unsigned long addr1
;
3230 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3232 ptr
= phys_ram_base
+ addr1
;
3234 if (!cpu_physical_memory_is_dirty(addr1
)) {
3235 /* invalidate code */
3236 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3238 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3239 (0xff & ~CODE_DIRTY_FLAG
);
3245 void stb_phys(target_phys_addr_t addr
, uint32_t val
)
3248 cpu_physical_memory_write(addr
, &v
, 1);
3252 void stw_phys(target_phys_addr_t addr
, uint32_t val
)
3254 uint16_t v
= tswap16(val
);
3255 cpu_physical_memory_write(addr
, (const uint8_t *)&v
, 2);
3259 void stq_phys(target_phys_addr_t addr
, uint64_t val
)
3262 cpu_physical_memory_write(addr
, (const uint8_t *)&val
, 8);
3267 /* virtual memory access for debug */
3268 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
3269 uint8_t *buf
, int len
, int is_write
)
3272 target_phys_addr_t phys_addr
;
3276 page
= addr
& TARGET_PAGE_MASK
;
3277 phys_addr
= cpu_get_phys_page_debug(env
, page
);
3278 /* if no physical page mapped, return an error */
3279 if (phys_addr
== -1)
3281 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3284 cpu_physical_memory_rw(phys_addr
+ (addr
& ~TARGET_PAGE_MASK
),
3293 /* in deterministic execution mode, instructions doing device I/Os
3294 must be at the end of the TB */
3295 void cpu_io_recompile(CPUState
*env
, void *retaddr
)
3297 TranslationBlock
*tb
;
3299 target_ulong pc
, cs_base
;
3302 tb
= tb_find_pc((unsigned long)retaddr
);
3304 cpu_abort(env
, "cpu_io_recompile: could not find TB for pc=%p",
3307 n
= env
->icount_decr
.u16
.low
+ tb
->icount
;
3308 cpu_restore_state(tb
, env
, (unsigned long)retaddr
, NULL
);
3309 /* Calculate how many instructions had been executed before the fault
3311 n
= n
- env
->icount_decr
.u16
.low
;
3312 /* Generate a new TB ending on the I/O insn. */
3314 /* On MIPS and SH, delay slot instructions can only be restarted if
3315 they were already the first instruction in the TB. If this is not
3316 the first instruction in a TB then re-execute the preceding
3318 #if defined(TARGET_MIPS)
3319 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
3320 env
->active_tc
.PC
-= 4;
3321 env
->icount_decr
.u16
.low
++;
3322 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
3324 #elif defined(TARGET_SH4)
3325 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
3328 env
->icount_decr
.u16
.low
++;
3329 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
3332 /* This should never happen. */
3333 if (n
> CF_COUNT_MASK
)
3334 cpu_abort(env
, "TB too big during recompile");
3336 cflags
= n
| CF_LAST_IO
;
3338 cs_base
= tb
->cs_base
;
3340 tb_phys_invalidate(tb
, -1);
3341 /* FIXME: In theory this could raise an exception. In practice
3342 we have already translated the block once so it's probably ok. */
3343 tb_gen_code(env
, pc
, cs_base
, flags
, cflags
);
3344 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3345 the first in the TB) then we end up generating a whole new TB and
3346 repeating the fault, which is horribly inefficient.
3347 Better would be to execute just this insn uncached, or generate a
3349 cpu_resume_from_signal(env
, NULL
);
3352 void dump_exec_info(FILE *f
,
3353 int (*cpu_fprintf
)(FILE *f
, const char *fmt
, ...))
3355 int i
, target_code_size
, max_target_code_size
;
3356 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
3357 TranslationBlock
*tb
;
3359 target_code_size
= 0;
3360 max_target_code_size
= 0;
3362 direct_jmp_count
= 0;
3363 direct_jmp2_count
= 0;
3364 for(i
= 0; i
< nb_tbs
; i
++) {
3366 target_code_size
+= tb
->size
;
3367 if (tb
->size
> max_target_code_size
)
3368 max_target_code_size
= tb
->size
;
3369 if (tb
->page_addr
[1] != -1)
3371 if (tb
->tb_next_offset
[0] != 0xffff) {
3373 if (tb
->tb_next_offset
[1] != 0xffff) {
3374 direct_jmp2_count
++;
3378 /* XXX: avoid using doubles ? */
3379 cpu_fprintf(f
, "Translation buffer state:\n");
3380 cpu_fprintf(f
, "gen code size %ld/%ld\n",
3381 code_gen_ptr
- code_gen_buffer
, code_gen_buffer_max_size
);
3382 cpu_fprintf(f
, "TB count %d/%d\n",
3383 nb_tbs
, code_gen_max_blocks
);
3384 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
3385 nb_tbs
? target_code_size
/ nb_tbs
: 0,
3386 max_target_code_size
);
3387 cpu_fprintf(f
, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3388 nb_tbs
? (code_gen_ptr
- code_gen_buffer
) / nb_tbs
: 0,
3389 target_code_size
? (double) (code_gen_ptr
- code_gen_buffer
) / target_code_size
: 0);
3390 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n",
3392 nb_tbs
? (cross_page
* 100) / nb_tbs
: 0);
3393 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3395 nb_tbs
? (direct_jmp_count
* 100) / nb_tbs
: 0,
3397 nb_tbs
? (direct_jmp2_count
* 100) / nb_tbs
: 0);
3398 cpu_fprintf(f
, "\nStatistics:\n");
3399 cpu_fprintf(f
, "TB flush count %d\n", tb_flush_count
);
3400 cpu_fprintf(f
, "TB invalidate count %d\n", tb_phys_invalidate_count
);
3401 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
3402 tcg_dump_info(f
, cpu_fprintf
);
3405 #if !defined(CONFIG_USER_ONLY)
3407 #define MMUSUFFIX _cmmu
3408 #define GETPC() NULL
3409 #define env cpu_single_env
3410 #define SOFTMMU_CODE_ACCESS
3413 #include "softmmu_template.h"
3416 #include "softmmu_template.h"
3419 #include "softmmu_template.h"
3422 #include "softmmu_template.h"