2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 #define WIN32_LEAN_AND_MEAN
25 #include <sys/types.h>
38 #include "qemu-common.h"
43 #if defined(CONFIG_USER_ONLY)
47 //#define DEBUG_TB_INVALIDATE
50 //#define DEBUG_UNASSIGNED
52 /* make various TB consistency checks */
53 //#define DEBUG_TB_CHECK
54 //#define DEBUG_TLB_CHECK
56 //#define DEBUG_IOPORT
57 //#define DEBUG_SUBPAGE
59 #if !defined(CONFIG_USER_ONLY)
60 /* TB consistency checks only implemented for usermode emulation. */
64 #define SMC_BITMAP_USE_THRESHOLD 10
66 #define MMAP_AREA_START 0x00000000
67 #define MMAP_AREA_END 0xa8000000
69 #if defined(TARGET_SPARC64)
70 #define TARGET_PHYS_ADDR_SPACE_BITS 41
71 #elif defined(TARGET_SPARC)
72 #define TARGET_PHYS_ADDR_SPACE_BITS 36
73 #elif defined(TARGET_ALPHA)
74 #define TARGET_PHYS_ADDR_SPACE_BITS 42
75 #define TARGET_VIRT_ADDR_SPACE_BITS 42
76 #elif defined(TARGET_PPC64)
77 #define TARGET_PHYS_ADDR_SPACE_BITS 42
78 #elif defined(TARGET_X86_64) && !defined(USE_KQEMU)
79 #define TARGET_PHYS_ADDR_SPACE_BITS 42
80 #elif defined(TARGET_I386) && !defined(USE_KQEMU)
81 #define TARGET_PHYS_ADDR_SPACE_BITS 36
83 /* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
84 #define TARGET_PHYS_ADDR_SPACE_BITS 32
87 static TranslationBlock
*tbs
;
88 int code_gen_max_blocks
;
89 TranslationBlock
*tb_phys_hash
[CODE_GEN_PHYS_HASH_SIZE
];
91 /* any access to the tbs or the page table must use this lock */
92 spinlock_t tb_lock
= SPIN_LOCK_UNLOCKED
;
94 #if defined(__arm__) || defined(__sparc_v9__)
95 /* The prologue must be reachable with a direct jump. ARM and Sparc64
96 have limited branch ranges (possibly also PPC) so place it in a
97 section close to code segment. */
98 #define code_gen_section \
99 __attribute__((__section__(".gen_code"))) \
100 __attribute__((aligned (32)))
102 #define code_gen_section \
103 __attribute__((aligned (32)))
106 uint8_t code_gen_prologue
[1024] code_gen_section
;
107 static uint8_t *code_gen_buffer
;
108 static unsigned long code_gen_buffer_size
;
109 /* threshold to flush the translated code buffer */
110 static unsigned long code_gen_buffer_max_size
;
111 uint8_t *code_gen_ptr
;
113 #if !defined(CONFIG_USER_ONLY)
114 ram_addr_t phys_ram_size
;
116 uint8_t *phys_ram_base
;
117 uint8_t *phys_ram_dirty
;
118 static int in_migration
;
119 static ram_addr_t phys_ram_alloc_offset
= 0;
123 /* current CPU in the current thread. It is only valid inside
125 CPUState
*cpu_single_env
;
126 /* 0 = Do not count executed instructions.
127 1 = Precise instruction counting.
128 2 = Adaptive rate instruction counting. */
130 /* Current instruction counter. While executing translated code this may
131 include some instructions that have not yet been executed. */
134 typedef struct PageDesc
{
135 /* list of TBs intersecting this ram page */
136 TranslationBlock
*first_tb
;
137 /* in order to optimize self modifying code, we count the number
138 of lookups we do to a given page to use a bitmap */
139 unsigned int code_write_count
;
140 uint8_t *code_bitmap
;
141 #if defined(CONFIG_USER_ONLY)
146 typedef struct PhysPageDesc
{
147 /* offset in host memory of the page + io_index in the low bits */
148 ram_addr_t phys_offset
;
152 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
153 /* XXX: this is a temporary hack for alpha target.
154 * In the future, this is to be replaced by a multi-level table
155 * to actually be able to handle the complete 64 bits address space.
157 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
159 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
162 #define L1_SIZE (1 << L1_BITS)
163 #define L2_SIZE (1 << L2_BITS)
165 unsigned long qemu_real_host_page_size
;
166 unsigned long qemu_host_page_bits
;
167 unsigned long qemu_host_page_size
;
168 unsigned long qemu_host_page_mask
;
170 /* XXX: for system emulation, it could just be an array */
171 static PageDesc
*l1_map
[L1_SIZE
];
172 static PhysPageDesc
**l1_phys_map
;
174 #if !defined(CONFIG_USER_ONLY)
175 static void io_mem_init(void);
177 /* io memory support */
178 CPUWriteMemoryFunc
*io_mem_write
[IO_MEM_NB_ENTRIES
][4];
179 CPUReadMemoryFunc
*io_mem_read
[IO_MEM_NB_ENTRIES
][4];
180 void *io_mem_opaque
[IO_MEM_NB_ENTRIES
];
181 static int io_mem_nb
;
182 static int io_mem_watch
;
186 static const char *logfilename
= "/tmp/qemu.log";
189 static int log_append
= 0;
192 static int tlb_flush_count
;
193 static int tb_flush_count
;
194 static int tb_phys_invalidate_count
;
196 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
197 typedef struct subpage_t
{
198 target_phys_addr_t base
;
199 CPUReadMemoryFunc
**mem_read
[TARGET_PAGE_SIZE
][4];
200 CPUWriteMemoryFunc
**mem_write
[TARGET_PAGE_SIZE
][4];
201 void *opaque
[TARGET_PAGE_SIZE
][2][4];
205 static void map_exec(void *addr
, long size
)
208 VirtualProtect(addr
, size
,
209 PAGE_EXECUTE_READWRITE
, &old_protect
);
213 static void map_exec(void *addr
, long size
)
215 unsigned long start
, end
, page_size
;
217 page_size
= getpagesize();
218 start
= (unsigned long)addr
;
219 start
&= ~(page_size
- 1);
221 end
= (unsigned long)addr
+ size
;
222 end
+= page_size
- 1;
223 end
&= ~(page_size
- 1);
225 mprotect((void *)start
, end
- start
,
226 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
230 static void page_init(void)
232 /* NOTE: we can always suppose that qemu_host_page_size >=
236 SYSTEM_INFO system_info
;
238 GetSystemInfo(&system_info
);
239 qemu_real_host_page_size
= system_info
.dwPageSize
;
242 qemu_real_host_page_size
= getpagesize();
244 if (qemu_host_page_size
== 0)
245 qemu_host_page_size
= qemu_real_host_page_size
;
246 if (qemu_host_page_size
< TARGET_PAGE_SIZE
)
247 qemu_host_page_size
= TARGET_PAGE_SIZE
;
248 qemu_host_page_bits
= 0;
249 while ((1 << qemu_host_page_bits
) < qemu_host_page_size
)
250 qemu_host_page_bits
++;
251 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
252 l1_phys_map
= qemu_vmalloc(L1_SIZE
* sizeof(void *));
253 memset(l1_phys_map
, 0, L1_SIZE
* sizeof(void *));
255 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
257 long long startaddr
, endaddr
;
262 last_brk
= (unsigned long)sbrk(0);
263 f
= fopen("/proc/self/maps", "r");
266 n
= fscanf (f
, "%llx-%llx %*[^\n]\n", &startaddr
, &endaddr
);
268 startaddr
= MIN(startaddr
,
269 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS
) - 1);
270 endaddr
= MIN(endaddr
,
271 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS
) - 1);
272 page_set_flags(startaddr
& TARGET_PAGE_MASK
,
273 TARGET_PAGE_ALIGN(endaddr
),
284 static inline PageDesc
**page_l1_map(target_ulong index
)
286 #if TARGET_LONG_BITS > 32
287 /* Host memory outside guest VM. For 32-bit targets we have already
288 excluded high addresses. */
289 if (index
> ((target_ulong
)L2_SIZE
* L1_SIZE
))
292 return &l1_map
[index
>> L2_BITS
];
295 static inline PageDesc
*page_find_alloc(target_ulong index
)
298 lp
= page_l1_map(index
);
304 /* allocate if not found */
305 #if defined(CONFIG_USER_ONLY)
307 size_t len
= sizeof(PageDesc
) * L2_SIZE
;
308 /* Don't use qemu_malloc because it may recurse. */
309 p
= mmap(0, len
, PROT_READ
| PROT_WRITE
,
310 MAP_PRIVATE
| MAP_ANONYMOUS
, -1, 0);
313 if (addr
== (target_ulong
)addr
) {
314 page_set_flags(addr
& TARGET_PAGE_MASK
,
315 TARGET_PAGE_ALIGN(addr
+ len
),
319 p
= qemu_mallocz(sizeof(PageDesc
) * L2_SIZE
);
323 return p
+ (index
& (L2_SIZE
- 1));
326 static inline PageDesc
*page_find(target_ulong index
)
329 lp
= page_l1_map(index
);
336 return p
+ (index
& (L2_SIZE
- 1));
339 static PhysPageDesc
*phys_page_find_alloc(target_phys_addr_t index
, int alloc
)
344 p
= (void **)l1_phys_map
;
345 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
347 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
348 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
350 lp
= p
+ ((index
>> (L1_BITS
+ L2_BITS
)) & (L1_SIZE
- 1));
353 /* allocate if not found */
356 p
= qemu_vmalloc(sizeof(void *) * L1_SIZE
);
357 memset(p
, 0, sizeof(void *) * L1_SIZE
);
361 lp
= p
+ ((index
>> L2_BITS
) & (L1_SIZE
- 1));
365 /* allocate if not found */
368 pd
= qemu_vmalloc(sizeof(PhysPageDesc
) * L2_SIZE
);
370 for (i
= 0; i
< L2_SIZE
; i
++)
371 pd
[i
].phys_offset
= IO_MEM_UNASSIGNED
;
373 return ((PhysPageDesc
*)pd
) + (index
& (L2_SIZE
- 1));
376 static inline PhysPageDesc
*phys_page_find(target_phys_addr_t index
)
378 return phys_page_find_alloc(index
, 0);
381 #if !defined(CONFIG_USER_ONLY)
382 static void tlb_protect_code(ram_addr_t ram_addr
);
383 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
385 #define mmap_lock() do { } while(0)
386 #define mmap_unlock() do { } while(0)
389 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
391 #if defined(CONFIG_USER_ONLY)
392 /* Currently it is not recommanded to allocate big chunks of data in
393 user mode. It will change when a dedicated libc will be used */
394 #define USE_STATIC_CODE_GEN_BUFFER
397 #ifdef USE_STATIC_CODE_GEN_BUFFER
398 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
];
401 static void code_gen_alloc(unsigned long tb_size
)
403 #ifdef USE_STATIC_CODE_GEN_BUFFER
404 code_gen_buffer
= static_code_gen_buffer
;
405 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
406 map_exec(code_gen_buffer
, code_gen_buffer_size
);
408 code_gen_buffer_size
= tb_size
;
409 if (code_gen_buffer_size
== 0) {
410 #if defined(CONFIG_USER_ONLY)
411 /* in user mode, phys_ram_size is not meaningful */
412 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
414 /* XXX: needs ajustments */
415 code_gen_buffer_size
= (unsigned long)(phys_ram_size
/ 4);
418 if (code_gen_buffer_size
< MIN_CODE_GEN_BUFFER_SIZE
)
419 code_gen_buffer_size
= MIN_CODE_GEN_BUFFER_SIZE
;
420 /* The code gen buffer location may have constraints depending on
421 the host cpu and OS */
422 #if defined(__linux__)
427 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
428 #if defined(__x86_64__)
430 /* Cannot map more than that */
431 if (code_gen_buffer_size
> (800 * 1024 * 1024))
432 code_gen_buffer_size
= (800 * 1024 * 1024);
433 #elif defined(__sparc_v9__)
434 // Map the buffer below 2G, so we can use direct calls and branches
436 start
= (void *) 0x60000000UL
;
437 if (code_gen_buffer_size
> (512 * 1024 * 1024))
438 code_gen_buffer_size
= (512 * 1024 * 1024);
440 code_gen_buffer
= mmap(start
, code_gen_buffer_size
,
441 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
443 if (code_gen_buffer
== MAP_FAILED
) {
444 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
448 #elif defined(__FreeBSD__)
452 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
453 #if defined(__x86_64__)
454 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
455 * 0x40000000 is free */
457 addr
= (void *)0x40000000;
458 /* Cannot map more than that */
459 if (code_gen_buffer_size
> (800 * 1024 * 1024))
460 code_gen_buffer_size
= (800 * 1024 * 1024);
462 code_gen_buffer
= mmap(addr
, code_gen_buffer_size
,
463 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
465 if (code_gen_buffer
== MAP_FAILED
) {
466 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
471 code_gen_buffer
= qemu_malloc(code_gen_buffer_size
);
472 if (!code_gen_buffer
) {
473 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
476 map_exec(code_gen_buffer
, code_gen_buffer_size
);
478 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
479 map_exec(code_gen_prologue
, sizeof(code_gen_prologue
));
480 code_gen_buffer_max_size
= code_gen_buffer_size
-
481 code_gen_max_block_size();
482 code_gen_max_blocks
= code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
483 tbs
= qemu_malloc(code_gen_max_blocks
* sizeof(TranslationBlock
));
486 /* Must be called before using the QEMU cpus. 'tb_size' is the size
487 (in bytes) allocated to the translation buffer. Zero means default
489 void cpu_exec_init_all(unsigned long tb_size
)
492 code_gen_alloc(tb_size
);
493 code_gen_ptr
= code_gen_buffer
;
495 #if !defined(CONFIG_USER_ONLY)
500 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
502 #define CPU_COMMON_SAVE_VERSION 1
504 static void cpu_common_save(QEMUFile
*f
, void *opaque
)
506 CPUState
*env
= opaque
;
508 qemu_put_be32s(f
, &env
->halted
);
509 qemu_put_be32s(f
, &env
->interrupt_request
);
512 static int cpu_common_load(QEMUFile
*f
, void *opaque
, int version_id
)
514 CPUState
*env
= opaque
;
516 if (version_id
!= CPU_COMMON_SAVE_VERSION
)
519 qemu_get_be32s(f
, &env
->halted
);
520 qemu_get_be32s(f
, &env
->interrupt_request
);
527 void cpu_exec_init(CPUState
*env
)
532 env
->next_cpu
= NULL
;
535 while (*penv
!= NULL
) {
536 penv
= (CPUState
**)&(*penv
)->next_cpu
;
539 env
->cpu_index
= cpu_index
;
541 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
542 register_savevm("cpu_common", cpu_index
, CPU_COMMON_SAVE_VERSION
,
543 cpu_common_save
, cpu_common_load
, env
);
544 register_savevm("cpu", cpu_index
, CPU_SAVE_VERSION
,
545 cpu_save
, cpu_load
, env
);
549 static inline void invalidate_page_bitmap(PageDesc
*p
)
551 if (p
->code_bitmap
) {
552 qemu_free(p
->code_bitmap
);
553 p
->code_bitmap
= NULL
;
555 p
->code_write_count
= 0;
558 /* set to NULL all the 'first_tb' fields in all PageDescs */
559 static void page_flush_tb(void)
564 for(i
= 0; i
< L1_SIZE
; i
++) {
567 for(j
= 0; j
< L2_SIZE
; j
++) {
569 invalidate_page_bitmap(p
);
576 /* flush all the translation blocks */
577 /* XXX: tb_flush is currently not thread safe */
578 void tb_flush(CPUState
*env1
)
581 #if defined(DEBUG_FLUSH)
582 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
583 (unsigned long)(code_gen_ptr
- code_gen_buffer
),
585 ((unsigned long)(code_gen_ptr
- code_gen_buffer
)) / nb_tbs
: 0);
587 if ((unsigned long)(code_gen_ptr
- code_gen_buffer
) > code_gen_buffer_size
)
588 cpu_abort(env1
, "Internal error: code buffer overflow\n");
592 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
593 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
596 memset (tb_phys_hash
, 0, CODE_GEN_PHYS_HASH_SIZE
* sizeof (void *));
599 code_gen_ptr
= code_gen_buffer
;
600 /* XXX: flush processor icache at this point if cache flush is
605 #ifdef DEBUG_TB_CHECK
607 static void tb_invalidate_check(target_ulong address
)
609 TranslationBlock
*tb
;
611 address
&= TARGET_PAGE_MASK
;
612 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
613 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
614 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
615 address
>= tb
->pc
+ tb
->size
)) {
616 printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
617 address
, (long)tb
->pc
, tb
->size
);
623 /* verify that all the pages have correct rights for code */
624 static void tb_page_check(void)
626 TranslationBlock
*tb
;
627 int i
, flags1
, flags2
;
629 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
630 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
631 flags1
= page_get_flags(tb
->pc
);
632 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
633 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
634 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
635 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
641 static void tb_jmp_check(TranslationBlock
*tb
)
643 TranslationBlock
*tb1
;
646 /* suppress any remaining jumps to this TB */
650 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
653 tb1
= tb1
->jmp_next
[n1
];
655 /* check end of list */
657 printf("ERROR: jmp_list from 0x%08lx\n", (long)tb
);
663 /* invalidate one TB */
664 static inline void tb_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
,
667 TranslationBlock
*tb1
;
671 *ptb
= *(TranslationBlock
**)((char *)tb1
+ next_offset
);
674 ptb
= (TranslationBlock
**)((char *)tb1
+ next_offset
);
678 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
680 TranslationBlock
*tb1
;
686 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
688 *ptb
= tb1
->page_next
[n1
];
691 ptb
= &tb1
->page_next
[n1
];
695 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
697 TranslationBlock
*tb1
, **ptb
;
700 ptb
= &tb
->jmp_next
[n
];
703 /* find tb(n) in circular list */
707 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
708 if (n1
== n
&& tb1
== tb
)
711 ptb
= &tb1
->jmp_first
;
713 ptb
= &tb1
->jmp_next
[n1
];
716 /* now we can suppress tb(n) from the list */
717 *ptb
= tb
->jmp_next
[n
];
719 tb
->jmp_next
[n
] = NULL
;
723 /* reset the jump entry 'n' of a TB so that it is not chained to
725 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
727 tb_set_jmp_target(tb
, n
, (unsigned long)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
730 void tb_phys_invalidate(TranslationBlock
*tb
, target_ulong page_addr
)
735 target_phys_addr_t phys_pc
;
736 TranslationBlock
*tb1
, *tb2
;
738 /* remove the TB from the hash list */
739 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
740 h
= tb_phys_hash_func(phys_pc
);
741 tb_remove(&tb_phys_hash
[h
], tb
,
742 offsetof(TranslationBlock
, phys_hash_next
));
744 /* remove the TB from the page list */
745 if (tb
->page_addr
[0] != page_addr
) {
746 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
747 tb_page_remove(&p
->first_tb
, tb
);
748 invalidate_page_bitmap(p
);
750 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
751 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
752 tb_page_remove(&p
->first_tb
, tb
);
753 invalidate_page_bitmap(p
);
756 tb_invalidated_flag
= 1;
758 /* remove the TB from the hash list */
759 h
= tb_jmp_cache_hash_func(tb
->pc
);
760 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
761 if (env
->tb_jmp_cache
[h
] == tb
)
762 env
->tb_jmp_cache
[h
] = NULL
;
765 /* suppress this TB from the two jump lists */
766 tb_jmp_remove(tb
, 0);
767 tb_jmp_remove(tb
, 1);
769 /* suppress any remaining jumps to this TB */
775 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
776 tb2
= tb1
->jmp_next
[n1
];
777 tb_reset_jump(tb1
, n1
);
778 tb1
->jmp_next
[n1
] = NULL
;
781 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2); /* fail safe */
783 tb_phys_invalidate_count
++;
786 static inline void set_bits(uint8_t *tab
, int start
, int len
)
792 mask
= 0xff << (start
& 7);
793 if ((start
& ~7) == (end
& ~7)) {
795 mask
&= ~(0xff << (end
& 7));
800 start
= (start
+ 8) & ~7;
802 while (start
< end1
) {
807 mask
= ~(0xff << (end
& 7));
813 static void build_page_bitmap(PageDesc
*p
)
815 int n
, tb_start
, tb_end
;
816 TranslationBlock
*tb
;
818 p
->code_bitmap
= qemu_mallocz(TARGET_PAGE_SIZE
/ 8);
825 tb
= (TranslationBlock
*)((long)tb
& ~3);
826 /* NOTE: this is subtle as a TB may span two physical pages */
828 /* NOTE: tb_end may be after the end of the page, but
829 it is not a problem */
830 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
831 tb_end
= tb_start
+ tb
->size
;
832 if (tb_end
> TARGET_PAGE_SIZE
)
833 tb_end
= TARGET_PAGE_SIZE
;
836 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
838 set_bits(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
839 tb
= tb
->page_next
[n
];
843 TranslationBlock
*tb_gen_code(CPUState
*env
,
844 target_ulong pc
, target_ulong cs_base
,
845 int flags
, int cflags
)
847 TranslationBlock
*tb
;
849 target_ulong phys_pc
, phys_page2
, virt_page2
;
852 phys_pc
= get_phys_addr_code(env
, pc
);
855 /* flush must be done */
857 /* cannot fail at this point */
859 /* Don't forget to invalidate previous TB info. */
860 tb_invalidated_flag
= 1;
862 tc_ptr
= code_gen_ptr
;
864 tb
->cs_base
= cs_base
;
867 cpu_gen_code(env
, tb
, &code_gen_size
);
868 code_gen_ptr
= (void *)(((unsigned long)code_gen_ptr
+ code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
870 /* check next page if needed */
871 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
873 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
874 phys_page2
= get_phys_addr_code(env
, virt_page2
);
876 tb_link_phys(tb
, phys_pc
, phys_page2
);
880 /* invalidate all TBs which intersect with the target physical page
881 starting in range [start;end[. NOTE: start and end must refer to
882 the same physical page. 'is_cpu_write_access' should be true if called
883 from a real cpu write access: the virtual CPU will exit the current
884 TB if code is modified inside this TB. */
885 void tb_invalidate_phys_page_range(target_phys_addr_t start
, target_phys_addr_t end
,
886 int is_cpu_write_access
)
888 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
889 CPUState
*env
= cpu_single_env
;
890 target_ulong tb_start
, tb_end
;
893 #ifdef TARGET_HAS_PRECISE_SMC
894 int current_tb_not_found
= is_cpu_write_access
;
895 TranslationBlock
*current_tb
= NULL
;
896 int current_tb_modified
= 0;
897 target_ulong current_pc
= 0;
898 target_ulong current_cs_base
= 0;
899 int current_flags
= 0;
900 #endif /* TARGET_HAS_PRECISE_SMC */
902 p
= page_find(start
>> TARGET_PAGE_BITS
);
905 if (!p
->code_bitmap
&&
906 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
&&
907 is_cpu_write_access
) {
908 /* build code bitmap */
909 build_page_bitmap(p
);
912 /* we remove all the TBs in the range [start, end[ */
913 /* XXX: see if in some cases it could be faster to invalidate all the code */
917 tb
= (TranslationBlock
*)((long)tb
& ~3);
918 tb_next
= tb
->page_next
[n
];
919 /* NOTE: this is subtle as a TB may span two physical pages */
921 /* NOTE: tb_end may be after the end of the page, but
922 it is not a problem */
923 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
924 tb_end
= tb_start
+ tb
->size
;
926 tb_start
= tb
->page_addr
[1];
927 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
929 if (!(tb_end
<= start
|| tb_start
>= end
)) {
930 #ifdef TARGET_HAS_PRECISE_SMC
931 if (current_tb_not_found
) {
932 current_tb_not_found
= 0;
934 if (env
->mem_io_pc
) {
935 /* now we have a real cpu fault */
936 current_tb
= tb_find_pc(env
->mem_io_pc
);
939 if (current_tb
== tb
&&
940 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
941 /* If we are modifying the current TB, we must stop
942 its execution. We could be more precise by checking
943 that the modification is after the current PC, but it
944 would require a specialized function to partially
945 restore the CPU state */
947 current_tb_modified
= 1;
948 cpu_restore_state(current_tb
, env
,
949 env
->mem_io_pc
, NULL
);
950 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
953 #endif /* TARGET_HAS_PRECISE_SMC */
954 /* we need to do that to handle the case where a signal
955 occurs while doing tb_phys_invalidate() */
958 saved_tb
= env
->current_tb
;
959 env
->current_tb
= NULL
;
961 tb_phys_invalidate(tb
, -1);
963 env
->current_tb
= saved_tb
;
964 if (env
->interrupt_request
&& env
->current_tb
)
965 cpu_interrupt(env
, env
->interrupt_request
);
970 #if !defined(CONFIG_USER_ONLY)
971 /* if no code remaining, no need to continue to use slow writes */
973 invalidate_page_bitmap(p
);
974 if (is_cpu_write_access
) {
975 tlb_unprotect_code_phys(env
, start
, env
->mem_io_vaddr
);
979 #ifdef TARGET_HAS_PRECISE_SMC
980 if (current_tb_modified
) {
981 /* we generate a block containing just the instruction
982 modifying the memory. It will ensure that it cannot modify
984 env
->current_tb
= NULL
;
985 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
986 cpu_resume_from_signal(env
, NULL
);
991 /* len must be <= 8 and start must be a multiple of len */
992 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start
, int len
)
999 fprintf(logfile
, "modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1000 cpu_single_env
->mem_io_vaddr
, len
,
1001 cpu_single_env
->eip
,
1002 cpu_single_env
->eip
+ (long)cpu_single_env
->segs
[R_CS
].base
);
1006 p
= page_find(start
>> TARGET_PAGE_BITS
);
1009 if (p
->code_bitmap
) {
1010 offset
= start
& ~TARGET_PAGE_MASK
;
1011 b
= p
->code_bitmap
[offset
>> 3] >> (offset
& 7);
1012 if (b
& ((1 << len
) - 1))
1016 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1020 #if !defined(CONFIG_SOFTMMU)
1021 static void tb_invalidate_phys_page(target_phys_addr_t addr
,
1022 unsigned long pc
, void *puc
)
1024 TranslationBlock
*tb
;
1027 #ifdef TARGET_HAS_PRECISE_SMC
1028 TranslationBlock
*current_tb
= NULL
;
1029 CPUState
*env
= cpu_single_env
;
1030 int current_tb_modified
= 0;
1031 target_ulong current_pc
= 0;
1032 target_ulong current_cs_base
= 0;
1033 int current_flags
= 0;
1036 addr
&= TARGET_PAGE_MASK
;
1037 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1041 #ifdef TARGET_HAS_PRECISE_SMC
1042 if (tb
&& pc
!= 0) {
1043 current_tb
= tb_find_pc(pc
);
1046 while (tb
!= NULL
) {
1048 tb
= (TranslationBlock
*)((long)tb
& ~3);
1049 #ifdef TARGET_HAS_PRECISE_SMC
1050 if (current_tb
== tb
&&
1051 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1052 /* If we are modifying the current TB, we must stop
1053 its execution. We could be more precise by checking
1054 that the modification is after the current PC, but it
1055 would require a specialized function to partially
1056 restore the CPU state */
1058 current_tb_modified
= 1;
1059 cpu_restore_state(current_tb
, env
, pc
, puc
);
1060 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1063 #endif /* TARGET_HAS_PRECISE_SMC */
1064 tb_phys_invalidate(tb
, addr
);
1065 tb
= tb
->page_next
[n
];
1068 #ifdef TARGET_HAS_PRECISE_SMC
1069 if (current_tb_modified
) {
1070 /* we generate a block containing just the instruction
1071 modifying the memory. It will ensure that it cannot modify
1073 env
->current_tb
= NULL
;
1074 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1075 cpu_resume_from_signal(env
, puc
);
1081 /* add the tb in the target page and protect it if necessary */
1082 static inline void tb_alloc_page(TranslationBlock
*tb
,
1083 unsigned int n
, target_ulong page_addr
)
1086 TranslationBlock
*last_first_tb
;
1088 tb
->page_addr
[n
] = page_addr
;
1089 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
);
1090 tb
->page_next
[n
] = p
->first_tb
;
1091 last_first_tb
= p
->first_tb
;
1092 p
->first_tb
= (TranslationBlock
*)((long)tb
| n
);
1093 invalidate_page_bitmap(p
);
1095 #if defined(TARGET_HAS_SMC) || 1
1097 #if defined(CONFIG_USER_ONLY)
1098 if (p
->flags
& PAGE_WRITE
) {
1103 /* force the host page as non writable (writes will have a
1104 page fault + mprotect overhead) */
1105 page_addr
&= qemu_host_page_mask
;
1107 for(addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1108 addr
+= TARGET_PAGE_SIZE
) {
1110 p2
= page_find (addr
>> TARGET_PAGE_BITS
);
1114 p2
->flags
&= ~PAGE_WRITE
;
1115 page_get_flags(addr
);
1117 mprotect(g2h(page_addr
), qemu_host_page_size
,
1118 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1119 #ifdef DEBUG_TB_INVALIDATE
1120 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1125 /* if some code is already present, then the pages are already
1126 protected. So we handle the case where only the first TB is
1127 allocated in a physical page */
1128 if (!last_first_tb
) {
1129 tlb_protect_code(page_addr
);
1133 #endif /* TARGET_HAS_SMC */
1136 /* Allocate a new translation block. Flush the translation buffer if
1137 too many translation blocks or too much generated code. */
1138 TranslationBlock
*tb_alloc(target_ulong pc
)
1140 TranslationBlock
*tb
;
1142 if (nb_tbs
>= code_gen_max_blocks
||
1143 (code_gen_ptr
- code_gen_buffer
) >= code_gen_buffer_max_size
)
1145 tb
= &tbs
[nb_tbs
++];
1151 void tb_free(TranslationBlock
*tb
)
1153 /* In practice this is mostly used for single use temporary TB
1154 Ignore the hard cases and just back up if this TB happens to
1155 be the last one generated. */
1156 if (nb_tbs
> 0 && tb
== &tbs
[nb_tbs
- 1]) {
1157 code_gen_ptr
= tb
->tc_ptr
;
1162 /* add a new TB and link it to the physical page tables. phys_page2 is
1163 (-1) to indicate that only one page contains the TB. */
1164 void tb_link_phys(TranslationBlock
*tb
,
1165 target_ulong phys_pc
, target_ulong phys_page2
)
1168 TranslationBlock
**ptb
;
1170 /* Grab the mmap lock to stop another thread invalidating this TB
1171 before we are done. */
1173 /* add in the physical hash table */
1174 h
= tb_phys_hash_func(phys_pc
);
1175 ptb
= &tb_phys_hash
[h
];
1176 tb
->phys_hash_next
= *ptb
;
1179 /* add in the page list */
1180 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1181 if (phys_page2
!= -1)
1182 tb_alloc_page(tb
, 1, phys_page2
);
1184 tb
->page_addr
[1] = -1;
1186 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2);
1187 tb
->jmp_next
[0] = NULL
;
1188 tb
->jmp_next
[1] = NULL
;
1190 /* init original jump addresses */
1191 if (tb
->tb_next_offset
[0] != 0xffff)
1192 tb_reset_jump(tb
, 0);
1193 if (tb
->tb_next_offset
[1] != 0xffff)
1194 tb_reset_jump(tb
, 1);
1196 #ifdef DEBUG_TB_CHECK
1202 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1203 tb[1].tc_ptr. Return NULL if not found */
1204 TranslationBlock
*tb_find_pc(unsigned long tc_ptr
)
1206 int m_min
, m_max
, m
;
1208 TranslationBlock
*tb
;
1212 if (tc_ptr
< (unsigned long)code_gen_buffer
||
1213 tc_ptr
>= (unsigned long)code_gen_ptr
)
1215 /* binary search (cf Knuth) */
1218 while (m_min
<= m_max
) {
1219 m
= (m_min
+ m_max
) >> 1;
1221 v
= (unsigned long)tb
->tc_ptr
;
1224 else if (tc_ptr
< v
) {
1233 static void tb_reset_jump_recursive(TranslationBlock
*tb
);
1235 static inline void tb_reset_jump_recursive2(TranslationBlock
*tb
, int n
)
1237 TranslationBlock
*tb1
, *tb_next
, **ptb
;
1240 tb1
= tb
->jmp_next
[n
];
1242 /* find head of list */
1245 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1248 tb1
= tb1
->jmp_next
[n1
];
1250 /* we are now sure now that tb jumps to tb1 */
1253 /* remove tb from the jmp_first list */
1254 ptb
= &tb_next
->jmp_first
;
1258 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1259 if (n1
== n
&& tb1
== tb
)
1261 ptb
= &tb1
->jmp_next
[n1
];
1263 *ptb
= tb
->jmp_next
[n
];
1264 tb
->jmp_next
[n
] = NULL
;
1266 /* suppress the jump to next tb in generated code */
1267 tb_reset_jump(tb
, n
);
1269 /* suppress jumps in the tb on which we could have jumped */
1270 tb_reset_jump_recursive(tb_next
);
1274 static void tb_reset_jump_recursive(TranslationBlock
*tb
)
1276 tb_reset_jump_recursive2(tb
, 0);
1277 tb_reset_jump_recursive2(tb
, 1);
1280 #if defined(TARGET_HAS_ICE)
1281 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1283 target_phys_addr_t addr
;
1285 ram_addr_t ram_addr
;
1288 addr
= cpu_get_phys_page_debug(env
, pc
);
1289 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
1291 pd
= IO_MEM_UNASSIGNED
;
1293 pd
= p
->phys_offset
;
1295 ram_addr
= (pd
& TARGET_PAGE_MASK
) | (pc
& ~TARGET_PAGE_MASK
);
1296 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1300 /* Add a watchpoint. */
1301 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1302 int flags
, CPUWatchpoint
**watchpoint
)
1304 target_ulong len_mask
= ~(len
- 1);
1307 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1308 if ((len
!= 1 && len
!= 2 && len
!= 4 && len
!= 8) || (addr
& ~len_mask
)) {
1309 fprintf(stderr
, "qemu: tried to set invalid watchpoint at "
1310 TARGET_FMT_lx
", len=" TARGET_FMT_lu
"\n", addr
, len
);
1313 wp
= qemu_malloc(sizeof(*wp
));
1318 wp
->len_mask
= len_mask
;
1321 wp
->next
= env
->watchpoints
;
1324 wp
->next
->prev
= wp
;
1325 env
->watchpoints
= wp
;
1327 tlb_flush_page(env
, addr
);
1328 /* FIXME: This flush is needed because of the hack to make memory ops
1329 terminate the TB. It can be removed once the proper IO trap and
1330 re-execute bits are in. */
1338 /* Remove a specific watchpoint. */
1339 int cpu_watchpoint_remove(CPUState
*env
, target_ulong addr
, target_ulong len
,
1342 target_ulong len_mask
= ~(len
- 1);
1345 for (wp
= env
->watchpoints
; wp
!= NULL
; wp
= wp
->next
) {
1346 if (addr
== wp
->vaddr
&& len_mask
== wp
->len_mask
1347 && flags
== wp
->flags
) {
1348 cpu_watchpoint_remove_by_ref(env
, wp
);
1355 /* Remove a specific watchpoint by reference. */
1356 void cpu_watchpoint_remove_by_ref(CPUState
*env
, CPUWatchpoint
*watchpoint
)
1358 if (watchpoint
->next
)
1359 watchpoint
->next
->prev
= watchpoint
->prev
;
1360 if (watchpoint
->prev
)
1361 watchpoint
->prev
->next
= watchpoint
->next
;
1363 env
->watchpoints
= watchpoint
->next
;
1365 tlb_flush_page(env
, watchpoint
->vaddr
);
1367 qemu_free(watchpoint
);
1370 /* Remove all matching watchpoints. */
1371 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1375 for (wp
= env
->watchpoints
; wp
!= NULL
; wp
= wp
->next
)
1376 if (wp
->flags
& mask
)
1377 cpu_watchpoint_remove_by_ref(env
, wp
);
1380 /* Add a breakpoint. */
1381 int cpu_breakpoint_insert(CPUState
*env
, target_ulong pc
, int flags
,
1382 CPUBreakpoint
**breakpoint
)
1384 #if defined(TARGET_HAS_ICE)
1387 bp
= qemu_malloc(sizeof(*bp
));
1394 bp
->next
= env
->breakpoints
;
1397 bp
->next
->prev
= bp
;
1398 env
->breakpoints
= bp
;
1400 breakpoint_invalidate(env
, pc
);
1410 /* Remove a specific breakpoint. */
1411 int cpu_breakpoint_remove(CPUState
*env
, target_ulong pc
, int flags
)
1413 #if defined(TARGET_HAS_ICE)
1416 for (bp
= env
->breakpoints
; bp
!= NULL
; bp
= bp
->next
) {
1417 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
1418 cpu_breakpoint_remove_by_ref(env
, bp
);
1428 /* Remove a specific breakpoint by reference. */
1429 void cpu_breakpoint_remove_by_ref(CPUState
*env
, CPUBreakpoint
*breakpoint
)
1431 #if defined(TARGET_HAS_ICE)
1432 if (breakpoint
->next
)
1433 breakpoint
->next
->prev
= breakpoint
->prev
;
1434 if (breakpoint
->prev
)
1435 breakpoint
->prev
->next
= breakpoint
->next
;
1437 env
->breakpoints
= breakpoint
->next
;
1439 breakpoint_invalidate(env
, breakpoint
->pc
);
1441 qemu_free(breakpoint
);
1445 /* Remove all matching breakpoints. */
1446 void cpu_breakpoint_remove_all(CPUState
*env
, int mask
)
1448 #if defined(TARGET_HAS_ICE)
1451 for (bp
= env
->breakpoints
; bp
!= NULL
; bp
= bp
->next
)
1452 if (bp
->flags
& mask
)
1453 cpu_breakpoint_remove_by_ref(env
, bp
);
1457 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1458 CPU loop after each instruction */
1459 void cpu_single_step(CPUState
*env
, int enabled
)
1461 #if defined(TARGET_HAS_ICE)
1462 if (env
->singlestep_enabled
!= enabled
) {
1463 env
->singlestep_enabled
= enabled
;
1464 /* must flush all the translated code to avoid inconsistancies */
1465 /* XXX: only flush what is necessary */
1471 /* enable or disable low levels log */
1472 void cpu_set_log(int log_flags
)
1474 loglevel
= log_flags
;
1475 if (loglevel
&& !logfile
) {
1476 logfile
= fopen(logfilename
, log_append
? "a" : "w");
1478 perror(logfilename
);
1481 #if !defined(CONFIG_SOFTMMU)
1482 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1484 static char logfile_buf
[4096];
1485 setvbuf(logfile
, logfile_buf
, _IOLBF
, sizeof(logfile_buf
));
1488 setvbuf(logfile
, NULL
, _IOLBF
, 0);
1492 if (!loglevel
&& logfile
) {
1498 void cpu_set_log_filename(const char *filename
)
1500 logfilename
= strdup(filename
);
1505 cpu_set_log(loglevel
);
1508 /* mask must never be zero, except for A20 change call */
1509 void cpu_interrupt(CPUState
*env
, int mask
)
1511 #if !defined(USE_NPTL)
1512 TranslationBlock
*tb
;
1513 static spinlock_t interrupt_lock
= SPIN_LOCK_UNLOCKED
;
1517 old_mask
= env
->interrupt_request
;
1518 /* FIXME: This is probably not threadsafe. A different thread could
1519 be in the middle of a read-modify-write operation. */
1520 env
->interrupt_request
|= mask
;
1521 #if defined(USE_NPTL)
1522 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1523 problem and hope the cpu will stop of its own accord. For userspace
1524 emulation this often isn't actually as bad as it sounds. Often
1525 signals are used primarily to interrupt blocking syscalls. */
1528 env
->icount_decr
.u16
.high
= 0xffff;
1529 #ifndef CONFIG_USER_ONLY
1530 /* CPU_INTERRUPT_EXIT isn't a real interrupt. It just means
1531 an async event happened and we need to process it. */
1533 && (mask
& ~(old_mask
| CPU_INTERRUPT_EXIT
)) != 0) {
1534 cpu_abort(env
, "Raised interrupt while not in I/O function");
1538 tb
= env
->current_tb
;
1539 /* if the cpu is currently executing code, we must unlink it and
1540 all the potentially executing TB */
1541 if (tb
&& !testandset(&interrupt_lock
)) {
1542 env
->current_tb
= NULL
;
1543 tb_reset_jump_recursive(tb
);
1544 resetlock(&interrupt_lock
);
1550 void cpu_reset_interrupt(CPUState
*env
, int mask
)
1552 env
->interrupt_request
&= ~mask
;
1555 const CPULogItem cpu_log_items
[] = {
1556 { CPU_LOG_TB_OUT_ASM
, "out_asm",
1557 "show generated host assembly code for each compiled TB" },
1558 { CPU_LOG_TB_IN_ASM
, "in_asm",
1559 "show target assembly code for each compiled TB" },
1560 { CPU_LOG_TB_OP
, "op",
1561 "show micro ops for each compiled TB" },
1562 { CPU_LOG_TB_OP_OPT
, "op_opt",
1565 "before eflags optimization and "
1567 "after liveness analysis" },
1568 { CPU_LOG_INT
, "int",
1569 "show interrupts/exceptions in short format" },
1570 { CPU_LOG_EXEC
, "exec",
1571 "show trace before each executed TB (lots of logs)" },
1572 { CPU_LOG_TB_CPU
, "cpu",
1573 "show CPU state before block translation" },
1575 { CPU_LOG_PCALL
, "pcall",
1576 "show protected mode far calls/returns/exceptions" },
1579 { CPU_LOG_IOPORT
, "ioport",
1580 "show all i/o ports accesses" },
1585 static int cmp1(const char *s1
, int n
, const char *s2
)
1587 if (strlen(s2
) != n
)
1589 return memcmp(s1
, s2
, n
) == 0;
1592 /* takes a comma separated list of log masks. Return 0 if error. */
1593 int cpu_str_to_log_mask(const char *str
)
1595 const CPULogItem
*item
;
1602 p1
= strchr(p
, ',');
1605 if(cmp1(p
,p1
-p
,"all")) {
1606 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1610 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1611 if (cmp1(p
, p1
- p
, item
->name
))
1625 void cpu_abort(CPUState
*env
, const char *fmt
, ...)
1632 fprintf(stderr
, "qemu: fatal: ");
1633 vfprintf(stderr
, fmt
, ap
);
1634 fprintf(stderr
, "\n");
1636 cpu_dump_state(env
, stderr
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1638 cpu_dump_state(env
, stderr
, fprintf
, 0);
1641 fprintf(logfile
, "qemu: fatal: ");
1642 vfprintf(logfile
, fmt
, ap2
);
1643 fprintf(logfile
, "\n");
1645 cpu_dump_state(env
, logfile
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1647 cpu_dump_state(env
, logfile
, fprintf
, 0);
1657 CPUState
*cpu_copy(CPUState
*env
)
1659 CPUState
*new_env
= cpu_init(env
->cpu_model_str
);
1660 /* preserve chaining and index */
1661 CPUState
*next_cpu
= new_env
->next_cpu
;
1662 int cpu_index
= new_env
->cpu_index
;
1663 memcpy(new_env
, env
, sizeof(CPUState
));
1664 new_env
->next_cpu
= next_cpu
;
1665 new_env
->cpu_index
= cpu_index
;
1669 #if !defined(CONFIG_USER_ONLY)
1671 static inline void tlb_flush_jmp_cache(CPUState
*env
, target_ulong addr
)
1675 /* Discard jump cache entries for any tb which might potentially
1676 overlap the flushed page. */
1677 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1678 memset (&env
->tb_jmp_cache
[i
], 0,
1679 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1681 i
= tb_jmp_cache_hash_page(addr
);
1682 memset (&env
->tb_jmp_cache
[i
], 0,
1683 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1686 /* NOTE: if flush_global is true, also flush global entries (not
1688 void tlb_flush(CPUState
*env
, int flush_global
)
1692 #if defined(DEBUG_TLB)
1693 printf("tlb_flush:\n");
1695 /* must reset current TB so that interrupts cannot modify the
1696 links while we are modifying them */
1697 env
->current_tb
= NULL
;
1699 for(i
= 0; i
< CPU_TLB_SIZE
; i
++) {
1700 env
->tlb_table
[0][i
].addr_read
= -1;
1701 env
->tlb_table
[0][i
].addr_write
= -1;
1702 env
->tlb_table
[0][i
].addr_code
= -1;
1703 env
->tlb_table
[1][i
].addr_read
= -1;
1704 env
->tlb_table
[1][i
].addr_write
= -1;
1705 env
->tlb_table
[1][i
].addr_code
= -1;
1706 #if (NB_MMU_MODES >= 3)
1707 env
->tlb_table
[2][i
].addr_read
= -1;
1708 env
->tlb_table
[2][i
].addr_write
= -1;
1709 env
->tlb_table
[2][i
].addr_code
= -1;
1710 #if (NB_MMU_MODES == 4)
1711 env
->tlb_table
[3][i
].addr_read
= -1;
1712 env
->tlb_table
[3][i
].addr_write
= -1;
1713 env
->tlb_table
[3][i
].addr_code
= -1;
1718 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
1721 if (env
->kqemu_enabled
) {
1722 kqemu_flush(env
, flush_global
);
1728 static inline void tlb_flush_entry(CPUTLBEntry
*tlb_entry
, target_ulong addr
)
1730 if (addr
== (tlb_entry
->addr_read
&
1731 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1732 addr
== (tlb_entry
->addr_write
&
1733 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1734 addr
== (tlb_entry
->addr_code
&
1735 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
))) {
1736 tlb_entry
->addr_read
= -1;
1737 tlb_entry
->addr_write
= -1;
1738 tlb_entry
->addr_code
= -1;
1742 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
1746 #if defined(DEBUG_TLB)
1747 printf("tlb_flush_page: " TARGET_FMT_lx
"\n", addr
);
1749 /* must reset current TB so that interrupts cannot modify the
1750 links while we are modifying them */
1751 env
->current_tb
= NULL
;
1753 addr
&= TARGET_PAGE_MASK
;
1754 i
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1755 tlb_flush_entry(&env
->tlb_table
[0][i
], addr
);
1756 tlb_flush_entry(&env
->tlb_table
[1][i
], addr
);
1757 #if (NB_MMU_MODES >= 3)
1758 tlb_flush_entry(&env
->tlb_table
[2][i
], addr
);
1759 #if (NB_MMU_MODES == 4)
1760 tlb_flush_entry(&env
->tlb_table
[3][i
], addr
);
1764 tlb_flush_jmp_cache(env
, addr
);
1767 if (env
->kqemu_enabled
) {
1768 kqemu_flush_page(env
, addr
);
1773 /* update the TLBs so that writes to code in the virtual page 'addr'
1775 static void tlb_protect_code(ram_addr_t ram_addr
)
1777 cpu_physical_memory_reset_dirty(ram_addr
,
1778 ram_addr
+ TARGET_PAGE_SIZE
,
1782 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1783 tested for self modifying code */
1784 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
1787 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] |= CODE_DIRTY_FLAG
;
1790 static inline void tlb_reset_dirty_range(CPUTLBEntry
*tlb_entry
,
1791 unsigned long start
, unsigned long length
)
1794 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
1795 addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) + tlb_entry
->addend
;
1796 if ((addr
- start
) < length
) {
1797 tlb_entry
->addr_write
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) | TLB_NOTDIRTY
;
1802 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t end
,
1806 unsigned long length
, start1
;
1810 start
&= TARGET_PAGE_MASK
;
1811 end
= TARGET_PAGE_ALIGN(end
);
1813 length
= end
- start
;
1816 len
= length
>> TARGET_PAGE_BITS
;
1818 /* XXX: should not depend on cpu context */
1820 if (env
->kqemu_enabled
) {
1823 for(i
= 0; i
< len
; i
++) {
1824 kqemu_set_notdirty(env
, addr
);
1825 addr
+= TARGET_PAGE_SIZE
;
1829 mask
= ~dirty_flags
;
1830 p
= phys_ram_dirty
+ (start
>> TARGET_PAGE_BITS
);
1831 for(i
= 0; i
< len
; i
++)
1834 /* we modify the TLB cache so that the dirty bit will be set again
1835 when accessing the range */
1836 start1
= start
+ (unsigned long)phys_ram_base
;
1837 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
1838 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1839 tlb_reset_dirty_range(&env
->tlb_table
[0][i
], start1
, length
);
1840 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1841 tlb_reset_dirty_range(&env
->tlb_table
[1][i
], start1
, length
);
1842 #if (NB_MMU_MODES >= 3)
1843 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1844 tlb_reset_dirty_range(&env
->tlb_table
[2][i
], start1
, length
);
1845 #if (NB_MMU_MODES == 4)
1846 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1847 tlb_reset_dirty_range(&env
->tlb_table
[3][i
], start1
, length
);
1853 int cpu_physical_memory_set_dirty_tracking(int enable
)
1855 in_migration
= enable
;
1859 int cpu_physical_memory_get_dirty_tracking(void)
1861 return in_migration
;
1864 static inline void tlb_update_dirty(CPUTLBEntry
*tlb_entry
)
1866 ram_addr_t ram_addr
;
1868 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
1869 ram_addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) +
1870 tlb_entry
->addend
- (unsigned long)phys_ram_base
;
1871 if (!cpu_physical_memory_is_dirty(ram_addr
)) {
1872 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
1877 /* update the TLB according to the current state of the dirty bits */
1878 void cpu_tlb_update_dirty(CPUState
*env
)
1881 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1882 tlb_update_dirty(&env
->tlb_table
[0][i
]);
1883 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1884 tlb_update_dirty(&env
->tlb_table
[1][i
]);
1885 #if (NB_MMU_MODES >= 3)
1886 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1887 tlb_update_dirty(&env
->tlb_table
[2][i
]);
1888 #if (NB_MMU_MODES == 4)
1889 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1890 tlb_update_dirty(&env
->tlb_table
[3][i
]);
1895 static inline void tlb_set_dirty1(CPUTLBEntry
*tlb_entry
, target_ulong vaddr
)
1897 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
))
1898 tlb_entry
->addr_write
= vaddr
;
1901 /* update the TLB corresponding to virtual page vaddr
1902 so that it is no longer dirty */
1903 static inline void tlb_set_dirty(CPUState
*env
, target_ulong vaddr
)
1907 vaddr
&= TARGET_PAGE_MASK
;
1908 i
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1909 tlb_set_dirty1(&env
->tlb_table
[0][i
], vaddr
);
1910 tlb_set_dirty1(&env
->tlb_table
[1][i
], vaddr
);
1911 #if (NB_MMU_MODES >= 3)
1912 tlb_set_dirty1(&env
->tlb_table
[2][i
], vaddr
);
1913 #if (NB_MMU_MODES == 4)
1914 tlb_set_dirty1(&env
->tlb_table
[3][i
], vaddr
);
1919 /* add a new TLB entry. At most one entry for a given virtual address
1920 is permitted. Return 0 if OK or 2 if the page could not be mapped
1921 (can only happen in non SOFTMMU mode for I/O pages or pages
1922 conflicting with the host address space). */
1923 int tlb_set_page_exec(CPUState
*env
, target_ulong vaddr
,
1924 target_phys_addr_t paddr
, int prot
,
1925 int mmu_idx
, int is_softmmu
)
1930 target_ulong address
;
1931 target_ulong code_address
;
1932 target_phys_addr_t addend
;
1936 target_phys_addr_t iotlb
;
1938 p
= phys_page_find(paddr
>> TARGET_PAGE_BITS
);
1940 pd
= IO_MEM_UNASSIGNED
;
1942 pd
= p
->phys_offset
;
1944 #if defined(DEBUG_TLB)
1945 printf("tlb_set_page: vaddr=" TARGET_FMT_lx
" paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1946 vaddr
, (int)paddr
, prot
, mmu_idx
, is_softmmu
, pd
);
1951 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&& !(pd
& IO_MEM_ROMD
)) {
1952 /* IO memory case (romd handled later) */
1953 address
|= TLB_MMIO
;
1955 addend
= (unsigned long)phys_ram_base
+ (pd
& TARGET_PAGE_MASK
);
1956 if ((pd
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
) {
1958 iotlb
= pd
& TARGET_PAGE_MASK
;
1959 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
)
1960 iotlb
|= IO_MEM_NOTDIRTY
;
1962 iotlb
|= IO_MEM_ROM
;
1964 /* IO handlers are currently passed a phsical address.
1965 It would be nice to pass an offset from the base address
1966 of that region. This would avoid having to special case RAM,
1967 and avoid full address decoding in every device.
1968 We can't use the high bits of pd for this because
1969 IO_MEM_ROMD uses these as a ram address. */
1970 iotlb
= (pd
& ~TARGET_PAGE_MASK
) + paddr
;
1973 code_address
= address
;
1974 /* Make accesses to pages with watchpoints go via the
1975 watchpoint trap routines. */
1976 for (wp
= env
->watchpoints
; wp
!= NULL
; wp
= wp
->next
) {
1977 if (vaddr
== (wp
->vaddr
& TARGET_PAGE_MASK
)) {
1978 iotlb
= io_mem_watch
+ paddr
;
1979 /* TODO: The memory case can be optimized by not trapping
1980 reads of pages with a write breakpoint. */
1981 address
|= TLB_MMIO
;
1985 index
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1986 env
->iotlb
[mmu_idx
][index
] = iotlb
- vaddr
;
1987 te
= &env
->tlb_table
[mmu_idx
][index
];
1988 te
->addend
= addend
- vaddr
;
1989 if (prot
& PAGE_READ
) {
1990 te
->addr_read
= address
;
1995 if (prot
& PAGE_EXEC
) {
1996 te
->addr_code
= code_address
;
2000 if (prot
& PAGE_WRITE
) {
2001 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_ROM
||
2002 (pd
& IO_MEM_ROMD
)) {
2003 /* Write access calls the I/O callback. */
2004 te
->addr_write
= address
| TLB_MMIO
;
2005 } else if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
&&
2006 !cpu_physical_memory_is_dirty(pd
)) {
2007 te
->addr_write
= address
| TLB_NOTDIRTY
;
2009 te
->addr_write
= address
;
2012 te
->addr_write
= -1;
2019 void tlb_flush(CPUState
*env
, int flush_global
)
2023 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
2027 int tlb_set_page_exec(CPUState
*env
, target_ulong vaddr
,
2028 target_phys_addr_t paddr
, int prot
,
2029 int mmu_idx
, int is_softmmu
)
2034 /* dump memory mappings */
2035 void page_dump(FILE *f
)
2037 unsigned long start
, end
;
2038 int i
, j
, prot
, prot1
;
2041 fprintf(f
, "%-8s %-8s %-8s %s\n",
2042 "start", "end", "size", "prot");
2046 for(i
= 0; i
<= L1_SIZE
; i
++) {
2051 for(j
= 0;j
< L2_SIZE
; j
++) {
2056 if (prot1
!= prot
) {
2057 end
= (i
<< (32 - L1_BITS
)) | (j
<< TARGET_PAGE_BITS
);
2059 fprintf(f
, "%08lx-%08lx %08lx %c%c%c\n",
2060 start
, end
, end
- start
,
2061 prot
& PAGE_READ
? 'r' : '-',
2062 prot
& PAGE_WRITE
? 'w' : '-',
2063 prot
& PAGE_EXEC
? 'x' : '-');
2077 int page_get_flags(target_ulong address
)
2081 p
= page_find(address
>> TARGET_PAGE_BITS
);
2087 /* modify the flags of a page and invalidate the code if
2088 necessary. The flag PAGE_WRITE_ORG is positionned automatically
2089 depending on PAGE_WRITE */
2090 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2095 /* mmap_lock should already be held. */
2096 start
= start
& TARGET_PAGE_MASK
;
2097 end
= TARGET_PAGE_ALIGN(end
);
2098 if (flags
& PAGE_WRITE
)
2099 flags
|= PAGE_WRITE_ORG
;
2100 for(addr
= start
; addr
< end
; addr
+= TARGET_PAGE_SIZE
) {
2101 p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
);
2102 /* We may be called for host regions that are outside guest
2106 /* if the write protection is set, then we invalidate the code
2108 if (!(p
->flags
& PAGE_WRITE
) &&
2109 (flags
& PAGE_WRITE
) &&
2111 tb_invalidate_phys_page(addr
, 0, NULL
);
2117 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2123 if (start
+ len
< start
)
2124 /* we've wrapped around */
2127 end
= TARGET_PAGE_ALIGN(start
+len
); /* must do before we loose bits in the next step */
2128 start
= start
& TARGET_PAGE_MASK
;
2130 for(addr
= start
; addr
< end
; addr
+= TARGET_PAGE_SIZE
) {
2131 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2134 if( !(p
->flags
& PAGE_VALID
) )
2137 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
))
2139 if (flags
& PAGE_WRITE
) {
2140 if (!(p
->flags
& PAGE_WRITE_ORG
))
2142 /* unprotect the page if it was put read-only because it
2143 contains translated code */
2144 if (!(p
->flags
& PAGE_WRITE
)) {
2145 if (!page_unprotect(addr
, 0, NULL
))
2154 /* called from signal handler: invalidate the code and unprotect the
2155 page. Return TRUE if the fault was succesfully handled. */
2156 int page_unprotect(target_ulong address
, unsigned long pc
, void *puc
)
2158 unsigned int page_index
, prot
, pindex
;
2160 target_ulong host_start
, host_end
, addr
;
2162 /* Technically this isn't safe inside a signal handler. However we
2163 know this only ever happens in a synchronous SEGV handler, so in
2164 practice it seems to be ok. */
2167 host_start
= address
& qemu_host_page_mask
;
2168 page_index
= host_start
>> TARGET_PAGE_BITS
;
2169 p1
= page_find(page_index
);
2174 host_end
= host_start
+ qemu_host_page_size
;
2177 for(addr
= host_start
;addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2181 /* if the page was really writable, then we change its
2182 protection back to writable */
2183 if (prot
& PAGE_WRITE_ORG
) {
2184 pindex
= (address
- host_start
) >> TARGET_PAGE_BITS
;
2185 if (!(p1
[pindex
].flags
& PAGE_WRITE
)) {
2186 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2187 (prot
& PAGE_BITS
) | PAGE_WRITE
);
2188 p1
[pindex
].flags
|= PAGE_WRITE
;
2189 /* and since the content will be modified, we must invalidate
2190 the corresponding translated code. */
2191 tb_invalidate_phys_page(address
, pc
, puc
);
2192 #ifdef DEBUG_TB_CHECK
2193 tb_invalidate_check(address
);
2203 static inline void tlb_set_dirty(CPUState
*env
,
2204 unsigned long addr
, target_ulong vaddr
)
2207 #endif /* defined(CONFIG_USER_ONLY) */
2209 #if !defined(CONFIG_USER_ONLY)
2210 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2212 static void *subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
2213 ram_addr_t orig_memory
);
2214 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2217 if (addr > start_addr) \
2220 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2221 if (start_addr2 > 0) \
2225 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2226 end_addr2 = TARGET_PAGE_SIZE - 1; \
2228 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2229 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2234 /* register physical memory. 'size' must be a multiple of the target
2235 page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2237 void cpu_register_physical_memory(target_phys_addr_t start_addr
,
2239 ram_addr_t phys_offset
)
2241 target_phys_addr_t addr
, end_addr
;
2244 ram_addr_t orig_size
= size
;
2248 /* XXX: should not depend on cpu context */
2250 if (env
->kqemu_enabled
) {
2251 kqemu_set_phys_mem(start_addr
, size
, phys_offset
);
2255 kvm_set_phys_mem(start_addr
, size
, phys_offset
);
2257 size
= (size
+ TARGET_PAGE_SIZE
- 1) & TARGET_PAGE_MASK
;
2258 end_addr
= start_addr
+ (target_phys_addr_t
)size
;
2259 for(addr
= start_addr
; addr
!= end_addr
; addr
+= TARGET_PAGE_SIZE
) {
2260 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2261 if (p
&& p
->phys_offset
!= IO_MEM_UNASSIGNED
) {
2262 ram_addr_t orig_memory
= p
->phys_offset
;
2263 target_phys_addr_t start_addr2
, end_addr2
;
2264 int need_subpage
= 0;
2266 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
, end_addr2
,
2268 if (need_subpage
|| phys_offset
& IO_MEM_SUBWIDTH
) {
2269 if (!(orig_memory
& IO_MEM_SUBPAGE
)) {
2270 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2271 &p
->phys_offset
, orig_memory
);
2273 subpage
= io_mem_opaque
[(orig_memory
& ~TARGET_PAGE_MASK
)
2276 subpage_register(subpage
, start_addr2
, end_addr2
, phys_offset
);
2278 p
->phys_offset
= phys_offset
;
2279 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2280 (phys_offset
& IO_MEM_ROMD
))
2281 phys_offset
+= TARGET_PAGE_SIZE
;
2284 p
= phys_page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2285 p
->phys_offset
= phys_offset
;
2286 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2287 (phys_offset
& IO_MEM_ROMD
))
2288 phys_offset
+= TARGET_PAGE_SIZE
;
2290 target_phys_addr_t start_addr2
, end_addr2
;
2291 int need_subpage
= 0;
2293 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
,
2294 end_addr2
, need_subpage
);
2296 if (need_subpage
|| phys_offset
& IO_MEM_SUBWIDTH
) {
2297 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2298 &p
->phys_offset
, IO_MEM_UNASSIGNED
);
2299 subpage_register(subpage
, start_addr2
, end_addr2
,
2306 /* since each CPU stores ram addresses in its TLB cache, we must
2307 reset the modified entries */
2309 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2314 /* XXX: temporary until new memory mapping API */
2315 ram_addr_t
cpu_get_physical_page_desc(target_phys_addr_t addr
)
2319 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2321 return IO_MEM_UNASSIGNED
;
2322 return p
->phys_offset
;
2325 /* XXX: better than nothing */
2326 ram_addr_t
qemu_ram_alloc(ram_addr_t size
)
2329 if ((phys_ram_alloc_offset
+ size
) > phys_ram_size
) {
2330 fprintf(stderr
, "Not enough memory (requested_size = %" PRIu64
", max memory = %" PRIu64
")\n",
2331 (uint64_t)size
, (uint64_t)phys_ram_size
);
2334 addr
= phys_ram_alloc_offset
;
2335 phys_ram_alloc_offset
= TARGET_PAGE_ALIGN(phys_ram_alloc_offset
+ size
);
2339 void qemu_ram_free(ram_addr_t addr
)
2343 static uint32_t unassigned_mem_readb(void *opaque
, target_phys_addr_t addr
)
2345 #ifdef DEBUG_UNASSIGNED
2346 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2348 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2349 do_unassigned_access(addr
, 0, 0, 0, 1);
2354 static uint32_t unassigned_mem_readw(void *opaque
, target_phys_addr_t addr
)
2356 #ifdef DEBUG_UNASSIGNED
2357 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2359 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2360 do_unassigned_access(addr
, 0, 0, 0, 2);
2365 static uint32_t unassigned_mem_readl(void *opaque
, target_phys_addr_t addr
)
2367 #ifdef DEBUG_UNASSIGNED
2368 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2370 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2371 do_unassigned_access(addr
, 0, 0, 0, 4);
2376 static void unassigned_mem_writeb(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2378 #ifdef DEBUG_UNASSIGNED
2379 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2381 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2382 do_unassigned_access(addr
, 1, 0, 0, 1);
2386 static void unassigned_mem_writew(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2388 #ifdef DEBUG_UNASSIGNED
2389 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2391 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2392 do_unassigned_access(addr
, 1, 0, 0, 2);
2396 static void unassigned_mem_writel(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2398 #ifdef DEBUG_UNASSIGNED
2399 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2401 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2402 do_unassigned_access(addr
, 1, 0, 0, 4);
2406 static CPUReadMemoryFunc
*unassigned_mem_read
[3] = {
2407 unassigned_mem_readb
,
2408 unassigned_mem_readw
,
2409 unassigned_mem_readl
,
2412 static CPUWriteMemoryFunc
*unassigned_mem_write
[3] = {
2413 unassigned_mem_writeb
,
2414 unassigned_mem_writew
,
2415 unassigned_mem_writel
,
2418 static void notdirty_mem_writeb(void *opaque
, target_phys_addr_t ram_addr
,
2422 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2423 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2424 #if !defined(CONFIG_USER_ONLY)
2425 tb_invalidate_phys_page_fast(ram_addr
, 1);
2426 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2429 stb_p(phys_ram_base
+ ram_addr
, val
);
2431 if (cpu_single_env
->kqemu_enabled
&&
2432 (dirty_flags
& KQEMU_MODIFY_PAGE_MASK
) != KQEMU_MODIFY_PAGE_MASK
)
2433 kqemu_modify_page(cpu_single_env
, ram_addr
);
2435 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2436 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2437 /* we remove the notdirty callback only if the code has been
2439 if (dirty_flags
== 0xff)
2440 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2443 static void notdirty_mem_writew(void *opaque
, target_phys_addr_t ram_addr
,
2447 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2448 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2449 #if !defined(CONFIG_USER_ONLY)
2450 tb_invalidate_phys_page_fast(ram_addr
, 2);
2451 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2454 stw_p(phys_ram_base
+ ram_addr
, val
);
2456 if (cpu_single_env
->kqemu_enabled
&&
2457 (dirty_flags
& KQEMU_MODIFY_PAGE_MASK
) != KQEMU_MODIFY_PAGE_MASK
)
2458 kqemu_modify_page(cpu_single_env
, ram_addr
);
2460 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2461 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2462 /* we remove the notdirty callback only if the code has been
2464 if (dirty_flags
== 0xff)
2465 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2468 static void notdirty_mem_writel(void *opaque
, target_phys_addr_t ram_addr
,
2472 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2473 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2474 #if !defined(CONFIG_USER_ONLY)
2475 tb_invalidate_phys_page_fast(ram_addr
, 4);
2476 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2479 stl_p(phys_ram_base
+ ram_addr
, val
);
2481 if (cpu_single_env
->kqemu_enabled
&&
2482 (dirty_flags
& KQEMU_MODIFY_PAGE_MASK
) != KQEMU_MODIFY_PAGE_MASK
)
2483 kqemu_modify_page(cpu_single_env
, ram_addr
);
2485 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2486 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2487 /* we remove the notdirty callback only if the code has been
2489 if (dirty_flags
== 0xff)
2490 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2493 static CPUReadMemoryFunc
*error_mem_read
[3] = {
2494 NULL
, /* never used */
2495 NULL
, /* never used */
2496 NULL
, /* never used */
2499 static CPUWriteMemoryFunc
*notdirty_mem_write
[3] = {
2500 notdirty_mem_writeb
,
2501 notdirty_mem_writew
,
2502 notdirty_mem_writel
,
2505 /* Generate a debug exception if a watchpoint has been hit. */
2506 static void check_watchpoint(int offset
, int len_mask
, int flags
)
2508 CPUState
*env
= cpu_single_env
;
2512 vaddr
= (env
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
2513 for (wp
= env
->watchpoints
; wp
!= NULL
; wp
= wp
->next
) {
2514 if ((vaddr
== (wp
->vaddr
& len_mask
) ||
2515 (vaddr
& wp
->len_mask
) == wp
->vaddr
) && (wp
->flags
& flags
)) {
2516 env
->watchpoint_hit
= wp
;
2517 cpu_interrupt(env
, CPU_INTERRUPT_DEBUG
);
2523 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2524 so these check for a hit then pass through to the normal out-of-line
2526 static uint32_t watch_mem_readb(void *opaque
, target_phys_addr_t addr
)
2528 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_READ
);
2529 return ldub_phys(addr
);
2532 static uint32_t watch_mem_readw(void *opaque
, target_phys_addr_t addr
)
2534 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_READ
);
2535 return lduw_phys(addr
);
2538 static uint32_t watch_mem_readl(void *opaque
, target_phys_addr_t addr
)
2540 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_READ
);
2541 return ldl_phys(addr
);
2544 static void watch_mem_writeb(void *opaque
, target_phys_addr_t addr
,
2547 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_WRITE
);
2548 stb_phys(addr
, val
);
2551 static void watch_mem_writew(void *opaque
, target_phys_addr_t addr
,
2554 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_WRITE
);
2555 stw_phys(addr
, val
);
2558 static void watch_mem_writel(void *opaque
, target_phys_addr_t addr
,
2561 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_WRITE
);
2562 stl_phys(addr
, val
);
2565 static CPUReadMemoryFunc
*watch_mem_read
[3] = {
2571 static CPUWriteMemoryFunc
*watch_mem_write
[3] = {
2577 static inline uint32_t subpage_readlen (subpage_t
*mmio
, target_phys_addr_t addr
,
2583 idx
= SUBPAGE_IDX(addr
- mmio
->base
);
2584 #if defined(DEBUG_SUBPAGE)
2585 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d\n", __func__
,
2586 mmio
, len
, addr
, idx
);
2588 ret
= (**mmio
->mem_read
[idx
][len
])(mmio
->opaque
[idx
][0][len
], addr
);
2593 static inline void subpage_writelen (subpage_t
*mmio
, target_phys_addr_t addr
,
2594 uint32_t value
, unsigned int len
)
2598 idx
= SUBPAGE_IDX(addr
- mmio
->base
);
2599 #if defined(DEBUG_SUBPAGE)
2600 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d value %08x\n", __func__
,
2601 mmio
, len
, addr
, idx
, value
);
2603 (**mmio
->mem_write
[idx
][len
])(mmio
->opaque
[idx
][1][len
], addr
, value
);
2606 static uint32_t subpage_readb (void *opaque
, target_phys_addr_t addr
)
2608 #if defined(DEBUG_SUBPAGE)
2609 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2612 return subpage_readlen(opaque
, addr
, 0);
2615 static void subpage_writeb (void *opaque
, target_phys_addr_t addr
,
2618 #if defined(DEBUG_SUBPAGE)
2619 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2621 subpage_writelen(opaque
, addr
, value
, 0);
2624 static uint32_t subpage_readw (void *opaque
, target_phys_addr_t addr
)
2626 #if defined(DEBUG_SUBPAGE)
2627 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2630 return subpage_readlen(opaque
, addr
, 1);
2633 static void subpage_writew (void *opaque
, target_phys_addr_t addr
,
2636 #if defined(DEBUG_SUBPAGE)
2637 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2639 subpage_writelen(opaque
, addr
, value
, 1);
2642 static uint32_t subpage_readl (void *opaque
, target_phys_addr_t addr
)
2644 #if defined(DEBUG_SUBPAGE)
2645 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2648 return subpage_readlen(opaque
, addr
, 2);
2651 static void subpage_writel (void *opaque
,
2652 target_phys_addr_t addr
, uint32_t value
)
2654 #if defined(DEBUG_SUBPAGE)
2655 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2657 subpage_writelen(opaque
, addr
, value
, 2);
2660 static CPUReadMemoryFunc
*subpage_read
[] = {
2666 static CPUWriteMemoryFunc
*subpage_write
[] = {
2672 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2678 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
2680 idx
= SUBPAGE_IDX(start
);
2681 eidx
= SUBPAGE_IDX(end
);
2682 #if defined(DEBUG_SUBPAGE)
2683 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %d\n", __func__
,
2684 mmio
, start
, end
, idx
, eidx
, memory
);
2686 memory
>>= IO_MEM_SHIFT
;
2687 for (; idx
<= eidx
; idx
++) {
2688 for (i
= 0; i
< 4; i
++) {
2689 if (io_mem_read
[memory
][i
]) {
2690 mmio
->mem_read
[idx
][i
] = &io_mem_read
[memory
][i
];
2691 mmio
->opaque
[idx
][0][i
] = io_mem_opaque
[memory
];
2693 if (io_mem_write
[memory
][i
]) {
2694 mmio
->mem_write
[idx
][i
] = &io_mem_write
[memory
][i
];
2695 mmio
->opaque
[idx
][1][i
] = io_mem_opaque
[memory
];
2703 static void *subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
2704 ram_addr_t orig_memory
)
2709 mmio
= qemu_mallocz(sizeof(subpage_t
));
2712 subpage_memory
= cpu_register_io_memory(0, subpage_read
, subpage_write
, mmio
);
2713 #if defined(DEBUG_SUBPAGE)
2714 printf("%s: %p base " TARGET_FMT_plx
" len %08x %d\n", __func__
,
2715 mmio
, base
, TARGET_PAGE_SIZE
, subpage_memory
);
2717 *phys
= subpage_memory
| IO_MEM_SUBPAGE
;
2718 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
- 1, orig_memory
);
2724 static void io_mem_init(void)
2726 cpu_register_io_memory(IO_MEM_ROM
>> IO_MEM_SHIFT
, error_mem_read
, unassigned_mem_write
, NULL
);
2727 cpu_register_io_memory(IO_MEM_UNASSIGNED
>> IO_MEM_SHIFT
, unassigned_mem_read
, unassigned_mem_write
, NULL
);
2728 cpu_register_io_memory(IO_MEM_NOTDIRTY
>> IO_MEM_SHIFT
, error_mem_read
, notdirty_mem_write
, NULL
);
2731 io_mem_watch
= cpu_register_io_memory(0, watch_mem_read
,
2732 watch_mem_write
, NULL
);
2733 /* alloc dirty bits array */
2734 phys_ram_dirty
= qemu_vmalloc(phys_ram_size
>> TARGET_PAGE_BITS
);
2735 memset(phys_ram_dirty
, 0xff, phys_ram_size
>> TARGET_PAGE_BITS
);
2738 /* mem_read and mem_write are arrays of functions containing the
2739 function to access byte (index 0), word (index 1) and dword (index
2740 2). Functions can be omitted with a NULL function pointer. The
2741 registered functions may be modified dynamically later.
2742 If io_index is non zero, the corresponding io zone is
2743 modified. If it is zero, a new io zone is allocated. The return
2744 value can be used with cpu_register_physical_memory(). (-1) is
2745 returned if error. */
2746 int cpu_register_io_memory(int io_index
,
2747 CPUReadMemoryFunc
**mem_read
,
2748 CPUWriteMemoryFunc
**mem_write
,
2751 int i
, subwidth
= 0;
2753 if (io_index
<= 0) {
2754 if (io_mem_nb
>= IO_MEM_NB_ENTRIES
)
2756 io_index
= io_mem_nb
++;
2758 if (io_index
>= IO_MEM_NB_ENTRIES
)
2762 for(i
= 0;i
< 3; i
++) {
2763 if (!mem_read
[i
] || !mem_write
[i
])
2764 subwidth
= IO_MEM_SUBWIDTH
;
2765 io_mem_read
[io_index
][i
] = mem_read
[i
];
2766 io_mem_write
[io_index
][i
] = mem_write
[i
];
2768 io_mem_opaque
[io_index
] = opaque
;
2769 return (io_index
<< IO_MEM_SHIFT
) | subwidth
;
2772 CPUWriteMemoryFunc
**cpu_get_io_memory_write(int io_index
)
2774 return io_mem_write
[io_index
>> IO_MEM_SHIFT
];
2777 CPUReadMemoryFunc
**cpu_get_io_memory_read(int io_index
)
2779 return io_mem_read
[io_index
>> IO_MEM_SHIFT
];
2782 #endif /* !defined(CONFIG_USER_ONLY) */
2784 /* physical memory access (slow version, mainly for debug) */
2785 #if defined(CONFIG_USER_ONLY)
2786 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
2787 int len
, int is_write
)
2794 page
= addr
& TARGET_PAGE_MASK
;
2795 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2798 flags
= page_get_flags(page
);
2799 if (!(flags
& PAGE_VALID
))
2802 if (!(flags
& PAGE_WRITE
))
2804 /* XXX: this code should not depend on lock_user */
2805 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
2806 /* FIXME - should this return an error rather than just fail? */
2809 unlock_user(p
, addr
, l
);
2811 if (!(flags
& PAGE_READ
))
2813 /* XXX: this code should not depend on lock_user */
2814 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
2815 /* FIXME - should this return an error rather than just fail? */
2818 unlock_user(p
, addr
, 0);
2827 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
2828 int len
, int is_write
)
2833 target_phys_addr_t page
;
2838 page
= addr
& TARGET_PAGE_MASK
;
2839 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2842 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
2844 pd
= IO_MEM_UNASSIGNED
;
2846 pd
= p
->phys_offset
;
2850 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
2851 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
2852 /* XXX: could force cpu_single_env to NULL to avoid
2854 if (l
>= 4 && ((addr
& 3) == 0)) {
2855 /* 32 bit write access */
2857 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
2859 } else if (l
>= 2 && ((addr
& 1) == 0)) {
2860 /* 16 bit write access */
2862 io_mem_write
[io_index
][1](io_mem_opaque
[io_index
], addr
, val
);
2865 /* 8 bit write access */
2867 io_mem_write
[io_index
][0](io_mem_opaque
[io_index
], addr
, val
);
2871 unsigned long addr1
;
2872 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
2874 ptr
= phys_ram_base
+ addr1
;
2875 memcpy(ptr
, buf
, l
);
2876 if (!cpu_physical_memory_is_dirty(addr1
)) {
2877 /* invalidate code */
2878 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
2880 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
2881 (0xff & ~CODE_DIRTY_FLAG
);
2885 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
2886 !(pd
& IO_MEM_ROMD
)) {
2888 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
2889 if (l
>= 4 && ((addr
& 3) == 0)) {
2890 /* 32 bit read access */
2891 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
2894 } else if (l
>= 2 && ((addr
& 1) == 0)) {
2895 /* 16 bit read access */
2896 val
= io_mem_read
[io_index
][1](io_mem_opaque
[io_index
], addr
);
2900 /* 8 bit read access */
2901 val
= io_mem_read
[io_index
][0](io_mem_opaque
[io_index
], addr
);
2907 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
2908 (addr
& ~TARGET_PAGE_MASK
);
2909 memcpy(buf
, ptr
, l
);
2918 /* used for ROM loading : can write in RAM and ROM */
2919 void cpu_physical_memory_write_rom(target_phys_addr_t addr
,
2920 const uint8_t *buf
, int len
)
2924 target_phys_addr_t page
;
2929 page
= addr
& TARGET_PAGE_MASK
;
2930 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2933 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
2935 pd
= IO_MEM_UNASSIGNED
;
2937 pd
= p
->phys_offset
;
2940 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
&&
2941 (pd
& ~TARGET_PAGE_MASK
) != IO_MEM_ROM
&&
2942 !(pd
& IO_MEM_ROMD
)) {
2945 unsigned long addr1
;
2946 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
2948 ptr
= phys_ram_base
+ addr1
;
2949 memcpy(ptr
, buf
, l
);
2958 /* warning: addr must be aligned */
2959 uint32_t ldl_phys(target_phys_addr_t addr
)
2967 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2969 pd
= IO_MEM_UNASSIGNED
;
2971 pd
= p
->phys_offset
;
2974 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
2975 !(pd
& IO_MEM_ROMD
)) {
2977 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
2978 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
2981 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
2982 (addr
& ~TARGET_PAGE_MASK
);
2988 /* warning: addr must be aligned */
2989 uint64_t ldq_phys(target_phys_addr_t addr
)
2997 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2999 pd
= IO_MEM_UNASSIGNED
;
3001 pd
= p
->phys_offset
;
3004 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3005 !(pd
& IO_MEM_ROMD
)) {
3007 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3008 #ifdef TARGET_WORDS_BIGENDIAN
3009 val
= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
) << 32;
3010 val
|= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4);
3012 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
3013 val
|= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4) << 32;
3017 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
3018 (addr
& ~TARGET_PAGE_MASK
);
3025 uint32_t ldub_phys(target_phys_addr_t addr
)
3028 cpu_physical_memory_read(addr
, &val
, 1);
3033 uint32_t lduw_phys(target_phys_addr_t addr
)
3036 cpu_physical_memory_read(addr
, (uint8_t *)&val
, 2);
3037 return tswap16(val
);
3040 /* warning: addr must be aligned. The ram page is not masked as dirty
3041 and the code inside is not invalidated. It is useful if the dirty
3042 bits are used to track modified PTEs */
3043 void stl_phys_notdirty(target_phys_addr_t addr
, uint32_t val
)
3050 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3052 pd
= IO_MEM_UNASSIGNED
;
3054 pd
= p
->phys_offset
;
3057 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3058 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3059 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3061 unsigned long addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3062 ptr
= phys_ram_base
+ addr1
;
3065 if (unlikely(in_migration
)) {
3066 if (!cpu_physical_memory_is_dirty(addr1
)) {
3067 /* invalidate code */
3068 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3070 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3071 (0xff & ~CODE_DIRTY_FLAG
);
3077 void stq_phys_notdirty(target_phys_addr_t addr
, uint64_t val
)
3084 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3086 pd
= IO_MEM_UNASSIGNED
;
3088 pd
= p
->phys_offset
;
3091 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3092 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3093 #ifdef TARGET_WORDS_BIGENDIAN
3094 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
>> 32);
3095 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
);
3097 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3098 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
>> 32);
3101 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
3102 (addr
& ~TARGET_PAGE_MASK
);
3107 /* warning: addr must be aligned */
3108 void stl_phys(target_phys_addr_t addr
, uint32_t val
)
3115 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3117 pd
= IO_MEM_UNASSIGNED
;
3119 pd
= p
->phys_offset
;
3122 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3123 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3124 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3126 unsigned long addr1
;
3127 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3129 ptr
= phys_ram_base
+ addr1
;
3131 if (!cpu_physical_memory_is_dirty(addr1
)) {
3132 /* invalidate code */
3133 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3135 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3136 (0xff & ~CODE_DIRTY_FLAG
);
3142 void stb_phys(target_phys_addr_t addr
, uint32_t val
)
3145 cpu_physical_memory_write(addr
, &v
, 1);
3149 void stw_phys(target_phys_addr_t addr
, uint32_t val
)
3151 uint16_t v
= tswap16(val
);
3152 cpu_physical_memory_write(addr
, (const uint8_t *)&v
, 2);
3156 void stq_phys(target_phys_addr_t addr
, uint64_t val
)
3159 cpu_physical_memory_write(addr
, (const uint8_t *)&val
, 8);
3164 /* virtual memory access for debug */
3165 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
3166 uint8_t *buf
, int len
, int is_write
)
3169 target_phys_addr_t phys_addr
;
3173 page
= addr
& TARGET_PAGE_MASK
;
3174 phys_addr
= cpu_get_phys_page_debug(env
, page
);
3175 /* if no physical page mapped, return an error */
3176 if (phys_addr
== -1)
3178 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3181 cpu_physical_memory_rw(phys_addr
+ (addr
& ~TARGET_PAGE_MASK
),
3190 /* in deterministic execution mode, instructions doing device I/Os
3191 must be at the end of the TB */
3192 void cpu_io_recompile(CPUState
*env
, void *retaddr
)
3194 TranslationBlock
*tb
;
3196 target_ulong pc
, cs_base
;
3199 tb
= tb_find_pc((unsigned long)retaddr
);
3201 cpu_abort(env
, "cpu_io_recompile: could not find TB for pc=%p",
3204 n
= env
->icount_decr
.u16
.low
+ tb
->icount
;
3205 cpu_restore_state(tb
, env
, (unsigned long)retaddr
, NULL
);
3206 /* Calculate how many instructions had been executed before the fault
3208 n
= n
- env
->icount_decr
.u16
.low
;
3209 /* Generate a new TB ending on the I/O insn. */
3211 /* On MIPS and SH, delay slot instructions can only be restarted if
3212 they were already the first instruction in the TB. If this is not
3213 the first instruction in a TB then re-execute the preceding
3215 #if defined(TARGET_MIPS)
3216 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
3217 env
->active_tc
.PC
-= 4;
3218 env
->icount_decr
.u16
.low
++;
3219 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
3221 #elif defined(TARGET_SH4)
3222 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
3225 env
->icount_decr
.u16
.low
++;
3226 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
3229 /* This should never happen. */
3230 if (n
> CF_COUNT_MASK
)
3231 cpu_abort(env
, "TB too big during recompile");
3233 cflags
= n
| CF_LAST_IO
;
3235 cs_base
= tb
->cs_base
;
3237 tb_phys_invalidate(tb
, -1);
3238 /* FIXME: In theory this could raise an exception. In practice
3239 we have already translated the block once so it's probably ok. */
3240 tb_gen_code(env
, pc
, cs_base
, flags
, cflags
);
3241 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3242 the first in the TB) then we end up generating a whole new TB and
3243 repeating the fault, which is horribly inefficient.
3244 Better would be to execute just this insn uncached, or generate a
3246 cpu_resume_from_signal(env
, NULL
);
3249 void dump_exec_info(FILE *f
,
3250 int (*cpu_fprintf
)(FILE *f
, const char *fmt
, ...))
3252 int i
, target_code_size
, max_target_code_size
;
3253 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
3254 TranslationBlock
*tb
;
3256 target_code_size
= 0;
3257 max_target_code_size
= 0;
3259 direct_jmp_count
= 0;
3260 direct_jmp2_count
= 0;
3261 for(i
= 0; i
< nb_tbs
; i
++) {
3263 target_code_size
+= tb
->size
;
3264 if (tb
->size
> max_target_code_size
)
3265 max_target_code_size
= tb
->size
;
3266 if (tb
->page_addr
[1] != -1)
3268 if (tb
->tb_next_offset
[0] != 0xffff) {
3270 if (tb
->tb_next_offset
[1] != 0xffff) {
3271 direct_jmp2_count
++;
3275 /* XXX: avoid using doubles ? */
3276 cpu_fprintf(f
, "Translation buffer state:\n");
3277 cpu_fprintf(f
, "gen code size %ld/%ld\n",
3278 code_gen_ptr
- code_gen_buffer
, code_gen_buffer_max_size
);
3279 cpu_fprintf(f
, "TB count %d/%d\n",
3280 nb_tbs
, code_gen_max_blocks
);
3281 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
3282 nb_tbs
? target_code_size
/ nb_tbs
: 0,
3283 max_target_code_size
);
3284 cpu_fprintf(f
, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3285 nb_tbs
? (code_gen_ptr
- code_gen_buffer
) / nb_tbs
: 0,
3286 target_code_size
? (double) (code_gen_ptr
- code_gen_buffer
) / target_code_size
: 0);
3287 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n",
3289 nb_tbs
? (cross_page
* 100) / nb_tbs
: 0);
3290 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3292 nb_tbs
? (direct_jmp_count
* 100) / nb_tbs
: 0,
3294 nb_tbs
? (direct_jmp2_count
* 100) / nb_tbs
: 0);
3295 cpu_fprintf(f
, "\nStatistics:\n");
3296 cpu_fprintf(f
, "TB flush count %d\n", tb_flush_count
);
3297 cpu_fprintf(f
, "TB invalidate count %d\n", tb_phys_invalidate_count
);
3298 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
3299 tcg_dump_info(f
, cpu_fprintf
);
3302 #if !defined(CONFIG_USER_ONLY)
3304 #define MMUSUFFIX _cmmu
3305 #define GETPC() NULL
3306 #define env cpu_single_env
3307 #define SOFTMMU_CODE_ACCESS
3310 #include "softmmu_template.h"
3313 #include "softmmu_template.h"
3316 #include "softmmu_template.h"
3319 #include "softmmu_template.h"