2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 #define WIN32_LEAN_AND_MEAN
25 #include <sys/types.h>
38 #include "qemu-common.h"
43 #if defined(CONFIG_USER_ONLY)
47 //#define DEBUG_TB_INVALIDATE
50 //#define DEBUG_UNASSIGNED
52 /* make various TB consistency checks */
53 //#define DEBUG_TB_CHECK
54 //#define DEBUG_TLB_CHECK
56 //#define DEBUG_IOPORT
57 //#define DEBUG_SUBPAGE
59 #if !defined(CONFIG_USER_ONLY)
60 /* TB consistency checks only implemented for usermode emulation. */
64 #define SMC_BITMAP_USE_THRESHOLD 10
66 #define MMAP_AREA_START 0x00000000
67 #define MMAP_AREA_END 0xa8000000
69 #if defined(TARGET_SPARC64)
70 #define TARGET_PHYS_ADDR_SPACE_BITS 41
71 #elif defined(TARGET_SPARC)
72 #define TARGET_PHYS_ADDR_SPACE_BITS 36
73 #elif defined(TARGET_ALPHA)
74 #define TARGET_PHYS_ADDR_SPACE_BITS 42
75 #define TARGET_VIRT_ADDR_SPACE_BITS 42
76 #elif defined(TARGET_PPC64)
77 #define TARGET_PHYS_ADDR_SPACE_BITS 42
78 #elif defined(TARGET_X86_64) && !defined(USE_KQEMU)
79 #define TARGET_PHYS_ADDR_SPACE_BITS 42
80 #elif defined(TARGET_I386) && !defined(USE_KQEMU)
81 #define TARGET_PHYS_ADDR_SPACE_BITS 36
83 /* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
84 #define TARGET_PHYS_ADDR_SPACE_BITS 32
87 static TranslationBlock
*tbs
;
88 int code_gen_max_blocks
;
89 TranslationBlock
*tb_phys_hash
[CODE_GEN_PHYS_HASH_SIZE
];
91 /* any access to the tbs or the page table must use this lock */
92 spinlock_t tb_lock
= SPIN_LOCK_UNLOCKED
;
94 #if defined(__arm__) || defined(__sparc_v9__)
95 /* The prologue must be reachable with a direct jump. ARM and Sparc64
96 have limited branch ranges (possibly also PPC) so place it in a
97 section close to code segment. */
98 #define code_gen_section \
99 __attribute__((__section__(".gen_code"))) \
100 __attribute__((aligned (32)))
102 #define code_gen_section \
103 __attribute__((aligned (32)))
106 uint8_t code_gen_prologue
[1024] code_gen_section
;
107 static uint8_t *code_gen_buffer
;
108 static unsigned long code_gen_buffer_size
;
109 /* threshold to flush the translated code buffer */
110 static unsigned long code_gen_buffer_max_size
;
111 uint8_t *code_gen_ptr
;
113 #if !defined(CONFIG_USER_ONLY)
114 ram_addr_t phys_ram_size
;
116 uint8_t *phys_ram_base
;
117 uint8_t *phys_ram_dirty
;
118 static int in_migration
;
119 static ram_addr_t phys_ram_alloc_offset
= 0;
123 /* current CPU in the current thread. It is only valid inside
125 CPUState
*cpu_single_env
;
126 /* 0 = Do not count executed instructions.
127 1 = Precise instruction counting.
128 2 = Adaptive rate instruction counting. */
130 /* Current instruction counter. While executing translated code this may
131 include some instructions that have not yet been executed. */
134 typedef struct PageDesc
{
135 /* list of TBs intersecting this ram page */
136 TranslationBlock
*first_tb
;
137 /* in order to optimize self modifying code, we count the number
138 of lookups we do to a given page to use a bitmap */
139 unsigned int code_write_count
;
140 uint8_t *code_bitmap
;
141 #if defined(CONFIG_USER_ONLY)
146 typedef struct PhysPageDesc
{
147 /* offset in host memory of the page + io_index in the low bits */
148 ram_addr_t phys_offset
;
152 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
153 /* XXX: this is a temporary hack for alpha target.
154 * In the future, this is to be replaced by a multi-level table
155 * to actually be able to handle the complete 64 bits address space.
157 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
159 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
162 #define L1_SIZE (1 << L1_BITS)
163 #define L2_SIZE (1 << L2_BITS)
165 unsigned long qemu_real_host_page_size
;
166 unsigned long qemu_host_page_bits
;
167 unsigned long qemu_host_page_size
;
168 unsigned long qemu_host_page_mask
;
170 /* XXX: for system emulation, it could just be an array */
171 static PageDesc
*l1_map
[L1_SIZE
];
172 static PhysPageDesc
**l1_phys_map
;
174 #if !defined(CONFIG_USER_ONLY)
175 static void io_mem_init(void);
177 /* io memory support */
178 CPUWriteMemoryFunc
*io_mem_write
[IO_MEM_NB_ENTRIES
][4];
179 CPUReadMemoryFunc
*io_mem_read
[IO_MEM_NB_ENTRIES
][4];
180 void *io_mem_opaque
[IO_MEM_NB_ENTRIES
];
181 static int io_mem_nb
;
182 static int io_mem_watch
;
186 static const char *logfilename
= "/tmp/qemu.log";
189 static int log_append
= 0;
192 static int tlb_flush_count
;
193 static int tb_flush_count
;
194 static int tb_phys_invalidate_count
;
196 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
197 typedef struct subpage_t
{
198 target_phys_addr_t base
;
199 CPUReadMemoryFunc
**mem_read
[TARGET_PAGE_SIZE
][4];
200 CPUWriteMemoryFunc
**mem_write
[TARGET_PAGE_SIZE
][4];
201 void *opaque
[TARGET_PAGE_SIZE
][2][4];
205 static void map_exec(void *addr
, long size
)
208 VirtualProtect(addr
, size
,
209 PAGE_EXECUTE_READWRITE
, &old_protect
);
213 static void map_exec(void *addr
, long size
)
215 unsigned long start
, end
, page_size
;
217 page_size
= getpagesize();
218 start
= (unsigned long)addr
;
219 start
&= ~(page_size
- 1);
221 end
= (unsigned long)addr
+ size
;
222 end
+= page_size
- 1;
223 end
&= ~(page_size
- 1);
225 mprotect((void *)start
, end
- start
,
226 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
230 static void page_init(void)
232 /* NOTE: we can always suppose that qemu_host_page_size >=
236 SYSTEM_INFO system_info
;
238 GetSystemInfo(&system_info
);
239 qemu_real_host_page_size
= system_info
.dwPageSize
;
242 qemu_real_host_page_size
= getpagesize();
244 if (qemu_host_page_size
== 0)
245 qemu_host_page_size
= qemu_real_host_page_size
;
246 if (qemu_host_page_size
< TARGET_PAGE_SIZE
)
247 qemu_host_page_size
= TARGET_PAGE_SIZE
;
248 qemu_host_page_bits
= 0;
249 while ((1 << qemu_host_page_bits
) < qemu_host_page_size
)
250 qemu_host_page_bits
++;
251 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
252 l1_phys_map
= qemu_vmalloc(L1_SIZE
* sizeof(void *));
253 memset(l1_phys_map
, 0, L1_SIZE
* sizeof(void *));
255 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
257 long long startaddr
, endaddr
;
262 last_brk
= (unsigned long)sbrk(0);
263 f
= fopen("/proc/self/maps", "r");
266 n
= fscanf (f
, "%llx-%llx %*[^\n]\n", &startaddr
, &endaddr
);
268 startaddr
= MIN(startaddr
,
269 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS
) - 1);
270 endaddr
= MIN(endaddr
,
271 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS
) - 1);
272 page_set_flags(startaddr
& TARGET_PAGE_MASK
,
273 TARGET_PAGE_ALIGN(endaddr
),
284 static inline PageDesc
**page_l1_map(target_ulong index
)
286 #if TARGET_LONG_BITS > 32
287 /* Host memory outside guest VM. For 32-bit targets we have already
288 excluded high addresses. */
289 if (index
> ((target_ulong
)L2_SIZE
* L1_SIZE
))
292 return &l1_map
[index
>> L2_BITS
];
295 static inline PageDesc
*page_find_alloc(target_ulong index
)
298 lp
= page_l1_map(index
);
304 /* allocate if not found */
305 #if defined(CONFIG_USER_ONLY)
307 size_t len
= sizeof(PageDesc
) * L2_SIZE
;
308 /* Don't use qemu_malloc because it may recurse. */
309 p
= mmap(0, len
, PROT_READ
| PROT_WRITE
,
310 MAP_PRIVATE
| MAP_ANONYMOUS
, -1, 0);
313 if (addr
== (target_ulong
)addr
) {
314 page_set_flags(addr
& TARGET_PAGE_MASK
,
315 TARGET_PAGE_ALIGN(addr
+ len
),
319 p
= qemu_mallocz(sizeof(PageDesc
) * L2_SIZE
);
323 return p
+ (index
& (L2_SIZE
- 1));
326 static inline PageDesc
*page_find(target_ulong index
)
329 lp
= page_l1_map(index
);
336 return p
+ (index
& (L2_SIZE
- 1));
339 static PhysPageDesc
*phys_page_find_alloc(target_phys_addr_t index
, int alloc
)
344 p
= (void **)l1_phys_map
;
345 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
347 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
348 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
350 lp
= p
+ ((index
>> (L1_BITS
+ L2_BITS
)) & (L1_SIZE
- 1));
353 /* allocate if not found */
356 p
= qemu_vmalloc(sizeof(void *) * L1_SIZE
);
357 memset(p
, 0, sizeof(void *) * L1_SIZE
);
361 lp
= p
+ ((index
>> L2_BITS
) & (L1_SIZE
- 1));
365 /* allocate if not found */
368 pd
= qemu_vmalloc(sizeof(PhysPageDesc
) * L2_SIZE
);
370 for (i
= 0; i
< L2_SIZE
; i
++)
371 pd
[i
].phys_offset
= IO_MEM_UNASSIGNED
;
373 return ((PhysPageDesc
*)pd
) + (index
& (L2_SIZE
- 1));
376 static inline PhysPageDesc
*phys_page_find(target_phys_addr_t index
)
378 return phys_page_find_alloc(index
, 0);
381 #if !defined(CONFIG_USER_ONLY)
382 static void tlb_protect_code(ram_addr_t ram_addr
);
383 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
385 #define mmap_lock() do { } while(0)
386 #define mmap_unlock() do { } while(0)
389 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
391 #if defined(CONFIG_USER_ONLY)
392 /* Currently it is not recommanded to allocate big chunks of data in
393 user mode. It will change when a dedicated libc will be used */
394 #define USE_STATIC_CODE_GEN_BUFFER
397 #ifdef USE_STATIC_CODE_GEN_BUFFER
398 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
];
401 static void code_gen_alloc(unsigned long tb_size
)
403 #ifdef USE_STATIC_CODE_GEN_BUFFER
404 code_gen_buffer
= static_code_gen_buffer
;
405 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
406 map_exec(code_gen_buffer
, code_gen_buffer_size
);
408 code_gen_buffer_size
= tb_size
;
409 if (code_gen_buffer_size
== 0) {
410 #if defined(CONFIG_USER_ONLY)
411 /* in user mode, phys_ram_size is not meaningful */
412 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
414 /* XXX: needs ajustments */
415 code_gen_buffer_size
= (unsigned long)(phys_ram_size
/ 4);
418 if (code_gen_buffer_size
< MIN_CODE_GEN_BUFFER_SIZE
)
419 code_gen_buffer_size
= MIN_CODE_GEN_BUFFER_SIZE
;
420 /* The code gen buffer location may have constraints depending on
421 the host cpu and OS */
422 #if defined(__linux__)
427 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
428 #if defined(__x86_64__)
430 /* Cannot map more than that */
431 if (code_gen_buffer_size
> (800 * 1024 * 1024))
432 code_gen_buffer_size
= (800 * 1024 * 1024);
433 #elif defined(__sparc_v9__)
434 // Map the buffer below 2G, so we can use direct calls and branches
436 start
= (void *) 0x60000000UL
;
437 if (code_gen_buffer_size
> (512 * 1024 * 1024))
438 code_gen_buffer_size
= (512 * 1024 * 1024);
440 code_gen_buffer
= mmap(start
, code_gen_buffer_size
,
441 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
443 if (code_gen_buffer
== MAP_FAILED
) {
444 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
448 #elif defined(__FreeBSD__)
452 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
453 #if defined(__x86_64__)
454 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
455 * 0x40000000 is free */
457 addr
= (void *)0x40000000;
458 /* Cannot map more than that */
459 if (code_gen_buffer_size
> (800 * 1024 * 1024))
460 code_gen_buffer_size
= (800 * 1024 * 1024);
462 code_gen_buffer
= mmap(addr
, code_gen_buffer_size
,
463 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
465 if (code_gen_buffer
== MAP_FAILED
) {
466 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
471 code_gen_buffer
= qemu_malloc(code_gen_buffer_size
);
472 if (!code_gen_buffer
) {
473 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
476 map_exec(code_gen_buffer
, code_gen_buffer_size
);
478 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
479 map_exec(code_gen_prologue
, sizeof(code_gen_prologue
));
480 code_gen_buffer_max_size
= code_gen_buffer_size
-
481 code_gen_max_block_size();
482 code_gen_max_blocks
= code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
483 tbs
= qemu_malloc(code_gen_max_blocks
* sizeof(TranslationBlock
));
486 /* Must be called before using the QEMU cpus. 'tb_size' is the size
487 (in bytes) allocated to the translation buffer. Zero means default
489 void cpu_exec_init_all(unsigned long tb_size
)
492 code_gen_alloc(tb_size
);
493 code_gen_ptr
= code_gen_buffer
;
495 #if !defined(CONFIG_USER_ONLY)
500 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
502 #define CPU_COMMON_SAVE_VERSION 1
504 static void cpu_common_save(QEMUFile
*f
, void *opaque
)
506 CPUState
*env
= opaque
;
508 qemu_put_be32s(f
, &env
->halted
);
509 qemu_put_be32s(f
, &env
->interrupt_request
);
512 static int cpu_common_load(QEMUFile
*f
, void *opaque
, int version_id
)
514 CPUState
*env
= opaque
;
516 if (version_id
!= CPU_COMMON_SAVE_VERSION
)
519 qemu_get_be32s(f
, &env
->halted
);
520 qemu_get_be32s(f
, &env
->interrupt_request
);
527 void cpu_exec_init(CPUState
*env
)
532 env
->next_cpu
= NULL
;
535 while (*penv
!= NULL
) {
536 penv
= (CPUState
**)&(*penv
)->next_cpu
;
539 env
->cpu_index
= cpu_index
;
541 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
542 register_savevm("cpu_common", cpu_index
, CPU_COMMON_SAVE_VERSION
,
543 cpu_common_save
, cpu_common_load
, env
);
544 register_savevm("cpu", cpu_index
, CPU_SAVE_VERSION
,
545 cpu_save
, cpu_load
, env
);
549 static inline void invalidate_page_bitmap(PageDesc
*p
)
551 if (p
->code_bitmap
) {
552 qemu_free(p
->code_bitmap
);
553 p
->code_bitmap
= NULL
;
555 p
->code_write_count
= 0;
558 /* set to NULL all the 'first_tb' fields in all PageDescs */
559 static void page_flush_tb(void)
564 for(i
= 0; i
< L1_SIZE
; i
++) {
567 for(j
= 0; j
< L2_SIZE
; j
++) {
569 invalidate_page_bitmap(p
);
576 /* flush all the translation blocks */
577 /* XXX: tb_flush is currently not thread safe */
578 void tb_flush(CPUState
*env1
)
581 #if defined(DEBUG_FLUSH)
582 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
583 (unsigned long)(code_gen_ptr
- code_gen_buffer
),
585 ((unsigned long)(code_gen_ptr
- code_gen_buffer
)) / nb_tbs
: 0);
587 if ((unsigned long)(code_gen_ptr
- code_gen_buffer
) > code_gen_buffer_size
)
588 cpu_abort(env1
, "Internal error: code buffer overflow\n");
592 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
593 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
596 memset (tb_phys_hash
, 0, CODE_GEN_PHYS_HASH_SIZE
* sizeof (void *));
599 code_gen_ptr
= code_gen_buffer
;
600 /* XXX: flush processor icache at this point if cache flush is
605 #ifdef DEBUG_TB_CHECK
607 static void tb_invalidate_check(target_ulong address
)
609 TranslationBlock
*tb
;
611 address
&= TARGET_PAGE_MASK
;
612 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
613 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
614 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
615 address
>= tb
->pc
+ tb
->size
)) {
616 printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
617 address
, (long)tb
->pc
, tb
->size
);
623 /* verify that all the pages have correct rights for code */
624 static void tb_page_check(void)
626 TranslationBlock
*tb
;
627 int i
, flags1
, flags2
;
629 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
630 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
631 flags1
= page_get_flags(tb
->pc
);
632 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
633 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
634 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
635 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
641 static void tb_jmp_check(TranslationBlock
*tb
)
643 TranslationBlock
*tb1
;
646 /* suppress any remaining jumps to this TB */
650 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
653 tb1
= tb1
->jmp_next
[n1
];
655 /* check end of list */
657 printf("ERROR: jmp_list from 0x%08lx\n", (long)tb
);
663 /* invalidate one TB */
664 static inline void tb_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
,
667 TranslationBlock
*tb1
;
671 *ptb
= *(TranslationBlock
**)((char *)tb1
+ next_offset
);
674 ptb
= (TranslationBlock
**)((char *)tb1
+ next_offset
);
678 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
680 TranslationBlock
*tb1
;
686 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
688 *ptb
= tb1
->page_next
[n1
];
691 ptb
= &tb1
->page_next
[n1
];
695 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
697 TranslationBlock
*tb1
, **ptb
;
700 ptb
= &tb
->jmp_next
[n
];
703 /* find tb(n) in circular list */
707 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
708 if (n1
== n
&& tb1
== tb
)
711 ptb
= &tb1
->jmp_first
;
713 ptb
= &tb1
->jmp_next
[n1
];
716 /* now we can suppress tb(n) from the list */
717 *ptb
= tb
->jmp_next
[n
];
719 tb
->jmp_next
[n
] = NULL
;
723 /* reset the jump entry 'n' of a TB so that it is not chained to
725 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
727 tb_set_jmp_target(tb
, n
, (unsigned long)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
730 void tb_phys_invalidate(TranslationBlock
*tb
, target_ulong page_addr
)
735 target_phys_addr_t phys_pc
;
736 TranslationBlock
*tb1
, *tb2
;
738 /* remove the TB from the hash list */
739 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
740 h
= tb_phys_hash_func(phys_pc
);
741 tb_remove(&tb_phys_hash
[h
], tb
,
742 offsetof(TranslationBlock
, phys_hash_next
));
744 /* remove the TB from the page list */
745 if (tb
->page_addr
[0] != page_addr
) {
746 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
747 tb_page_remove(&p
->first_tb
, tb
);
748 invalidate_page_bitmap(p
);
750 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
751 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
752 tb_page_remove(&p
->first_tb
, tb
);
753 invalidate_page_bitmap(p
);
756 tb_invalidated_flag
= 1;
758 /* remove the TB from the hash list */
759 h
= tb_jmp_cache_hash_func(tb
->pc
);
760 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
761 if (env
->tb_jmp_cache
[h
] == tb
)
762 env
->tb_jmp_cache
[h
] = NULL
;
765 /* suppress this TB from the two jump lists */
766 tb_jmp_remove(tb
, 0);
767 tb_jmp_remove(tb
, 1);
769 /* suppress any remaining jumps to this TB */
775 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
776 tb2
= tb1
->jmp_next
[n1
];
777 tb_reset_jump(tb1
, n1
);
778 tb1
->jmp_next
[n1
] = NULL
;
781 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2); /* fail safe */
783 tb_phys_invalidate_count
++;
786 static inline void set_bits(uint8_t *tab
, int start
, int len
)
792 mask
= 0xff << (start
& 7);
793 if ((start
& ~7) == (end
& ~7)) {
795 mask
&= ~(0xff << (end
& 7));
800 start
= (start
+ 8) & ~7;
802 while (start
< end1
) {
807 mask
= ~(0xff << (end
& 7));
813 static void build_page_bitmap(PageDesc
*p
)
815 int n
, tb_start
, tb_end
;
816 TranslationBlock
*tb
;
818 p
->code_bitmap
= qemu_mallocz(TARGET_PAGE_SIZE
/ 8);
825 tb
= (TranslationBlock
*)((long)tb
& ~3);
826 /* NOTE: this is subtle as a TB may span two physical pages */
828 /* NOTE: tb_end may be after the end of the page, but
829 it is not a problem */
830 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
831 tb_end
= tb_start
+ tb
->size
;
832 if (tb_end
> TARGET_PAGE_SIZE
)
833 tb_end
= TARGET_PAGE_SIZE
;
836 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
838 set_bits(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
839 tb
= tb
->page_next
[n
];
843 TranslationBlock
*tb_gen_code(CPUState
*env
,
844 target_ulong pc
, target_ulong cs_base
,
845 int flags
, int cflags
)
847 TranslationBlock
*tb
;
849 target_ulong phys_pc
, phys_page2
, virt_page2
;
852 phys_pc
= get_phys_addr_code(env
, pc
);
855 /* flush must be done */
857 /* cannot fail at this point */
859 /* Don't forget to invalidate previous TB info. */
860 tb_invalidated_flag
= 1;
862 tc_ptr
= code_gen_ptr
;
864 tb
->cs_base
= cs_base
;
867 cpu_gen_code(env
, tb
, &code_gen_size
);
868 code_gen_ptr
= (void *)(((unsigned long)code_gen_ptr
+ code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
870 /* check next page if needed */
871 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
873 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
874 phys_page2
= get_phys_addr_code(env
, virt_page2
);
876 tb_link_phys(tb
, phys_pc
, phys_page2
);
880 /* invalidate all TBs which intersect with the target physical page
881 starting in range [start;end[. NOTE: start and end must refer to
882 the same physical page. 'is_cpu_write_access' should be true if called
883 from a real cpu write access: the virtual CPU will exit the current
884 TB if code is modified inside this TB. */
885 void tb_invalidate_phys_page_range(target_phys_addr_t start
, target_phys_addr_t end
,
886 int is_cpu_write_access
)
888 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
889 CPUState
*env
= cpu_single_env
;
890 target_ulong tb_start
, tb_end
;
893 #ifdef TARGET_HAS_PRECISE_SMC
894 int current_tb_not_found
= is_cpu_write_access
;
895 TranslationBlock
*current_tb
= NULL
;
896 int current_tb_modified
= 0;
897 target_ulong current_pc
= 0;
898 target_ulong current_cs_base
= 0;
899 int current_flags
= 0;
900 #endif /* TARGET_HAS_PRECISE_SMC */
902 p
= page_find(start
>> TARGET_PAGE_BITS
);
905 if (!p
->code_bitmap
&&
906 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
&&
907 is_cpu_write_access
) {
908 /* build code bitmap */
909 build_page_bitmap(p
);
912 /* we remove all the TBs in the range [start, end[ */
913 /* XXX: see if in some cases it could be faster to invalidate all the code */
917 tb
= (TranslationBlock
*)((long)tb
& ~3);
918 tb_next
= tb
->page_next
[n
];
919 /* NOTE: this is subtle as a TB may span two physical pages */
921 /* NOTE: tb_end may be after the end of the page, but
922 it is not a problem */
923 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
924 tb_end
= tb_start
+ tb
->size
;
926 tb_start
= tb
->page_addr
[1];
927 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
929 if (!(tb_end
<= start
|| tb_start
>= end
)) {
930 #ifdef TARGET_HAS_PRECISE_SMC
931 if (current_tb_not_found
) {
932 current_tb_not_found
= 0;
934 if (env
->mem_io_pc
) {
935 /* now we have a real cpu fault */
936 current_tb
= tb_find_pc(env
->mem_io_pc
);
939 if (current_tb
== tb
&&
940 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
941 /* If we are modifying the current TB, we must stop
942 its execution. We could be more precise by checking
943 that the modification is after the current PC, but it
944 would require a specialized function to partially
945 restore the CPU state */
947 current_tb_modified
= 1;
948 cpu_restore_state(current_tb
, env
,
949 env
->mem_io_pc
, NULL
);
950 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
953 #endif /* TARGET_HAS_PRECISE_SMC */
954 /* we need to do that to handle the case where a signal
955 occurs while doing tb_phys_invalidate() */
958 saved_tb
= env
->current_tb
;
959 env
->current_tb
= NULL
;
961 tb_phys_invalidate(tb
, -1);
963 env
->current_tb
= saved_tb
;
964 if (env
->interrupt_request
&& env
->current_tb
)
965 cpu_interrupt(env
, env
->interrupt_request
);
970 #if !defined(CONFIG_USER_ONLY)
971 /* if no code remaining, no need to continue to use slow writes */
973 invalidate_page_bitmap(p
);
974 if (is_cpu_write_access
) {
975 tlb_unprotect_code_phys(env
, start
, env
->mem_io_vaddr
);
979 #ifdef TARGET_HAS_PRECISE_SMC
980 if (current_tb_modified
) {
981 /* we generate a block containing just the instruction
982 modifying the memory. It will ensure that it cannot modify
984 env
->current_tb
= NULL
;
985 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
986 cpu_resume_from_signal(env
, NULL
);
991 /* len must be <= 8 and start must be a multiple of len */
992 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start
, int len
)
999 fprintf(logfile
, "modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1000 cpu_single_env
->mem_io_vaddr
, len
,
1001 cpu_single_env
->eip
,
1002 cpu_single_env
->eip
+ (long)cpu_single_env
->segs
[R_CS
].base
);
1006 p
= page_find(start
>> TARGET_PAGE_BITS
);
1009 if (p
->code_bitmap
) {
1010 offset
= start
& ~TARGET_PAGE_MASK
;
1011 b
= p
->code_bitmap
[offset
>> 3] >> (offset
& 7);
1012 if (b
& ((1 << len
) - 1))
1016 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1020 #if !defined(CONFIG_SOFTMMU)
1021 static void tb_invalidate_phys_page(target_phys_addr_t addr
,
1022 unsigned long pc
, void *puc
)
1024 TranslationBlock
*tb
;
1027 #ifdef TARGET_HAS_PRECISE_SMC
1028 TranslationBlock
*current_tb
= NULL
;
1029 CPUState
*env
= cpu_single_env
;
1030 int current_tb_modified
= 0;
1031 target_ulong current_pc
= 0;
1032 target_ulong current_cs_base
= 0;
1033 int current_flags
= 0;
1036 addr
&= TARGET_PAGE_MASK
;
1037 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1041 #ifdef TARGET_HAS_PRECISE_SMC
1042 if (tb
&& pc
!= 0) {
1043 current_tb
= tb_find_pc(pc
);
1046 while (tb
!= NULL
) {
1048 tb
= (TranslationBlock
*)((long)tb
& ~3);
1049 #ifdef TARGET_HAS_PRECISE_SMC
1050 if (current_tb
== tb
&&
1051 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1052 /* If we are modifying the current TB, we must stop
1053 its execution. We could be more precise by checking
1054 that the modification is after the current PC, but it
1055 would require a specialized function to partially
1056 restore the CPU state */
1058 current_tb_modified
= 1;
1059 cpu_restore_state(current_tb
, env
, pc
, puc
);
1060 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1063 #endif /* TARGET_HAS_PRECISE_SMC */
1064 tb_phys_invalidate(tb
, addr
);
1065 tb
= tb
->page_next
[n
];
1068 #ifdef TARGET_HAS_PRECISE_SMC
1069 if (current_tb_modified
) {
1070 /* we generate a block containing just the instruction
1071 modifying the memory. It will ensure that it cannot modify
1073 env
->current_tb
= NULL
;
1074 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1075 cpu_resume_from_signal(env
, puc
);
1081 /* add the tb in the target page and protect it if necessary */
1082 static inline void tb_alloc_page(TranslationBlock
*tb
,
1083 unsigned int n
, target_ulong page_addr
)
1086 TranslationBlock
*last_first_tb
;
1088 tb
->page_addr
[n
] = page_addr
;
1089 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
);
1090 tb
->page_next
[n
] = p
->first_tb
;
1091 last_first_tb
= p
->first_tb
;
1092 p
->first_tb
= (TranslationBlock
*)((long)tb
| n
);
1093 invalidate_page_bitmap(p
);
1095 #if defined(TARGET_HAS_SMC) || 1
1097 #if defined(CONFIG_USER_ONLY)
1098 if (p
->flags
& PAGE_WRITE
) {
1103 /* force the host page as non writable (writes will have a
1104 page fault + mprotect overhead) */
1105 page_addr
&= qemu_host_page_mask
;
1107 for(addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1108 addr
+= TARGET_PAGE_SIZE
) {
1110 p2
= page_find (addr
>> TARGET_PAGE_BITS
);
1114 p2
->flags
&= ~PAGE_WRITE
;
1115 page_get_flags(addr
);
1117 mprotect(g2h(page_addr
), qemu_host_page_size
,
1118 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1119 #ifdef DEBUG_TB_INVALIDATE
1120 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1125 /* if some code is already present, then the pages are already
1126 protected. So we handle the case where only the first TB is
1127 allocated in a physical page */
1128 if (!last_first_tb
) {
1129 tlb_protect_code(page_addr
);
1133 #endif /* TARGET_HAS_SMC */
1136 /* Allocate a new translation block. Flush the translation buffer if
1137 too many translation blocks or too much generated code. */
1138 TranslationBlock
*tb_alloc(target_ulong pc
)
1140 TranslationBlock
*tb
;
1142 if (nb_tbs
>= code_gen_max_blocks
||
1143 (code_gen_ptr
- code_gen_buffer
) >= code_gen_buffer_max_size
)
1145 tb
= &tbs
[nb_tbs
++];
1151 void tb_free(TranslationBlock
*tb
)
1153 /* In practice this is mostly used for single use temporary TB
1154 Ignore the hard cases and just back up if this TB happens to
1155 be the last one generated. */
1156 if (nb_tbs
> 0 && tb
== &tbs
[nb_tbs
- 1]) {
1157 code_gen_ptr
= tb
->tc_ptr
;
1162 /* add a new TB and link it to the physical page tables. phys_page2 is
1163 (-1) to indicate that only one page contains the TB. */
1164 void tb_link_phys(TranslationBlock
*tb
,
1165 target_ulong phys_pc
, target_ulong phys_page2
)
1168 TranslationBlock
**ptb
;
1170 /* Grab the mmap lock to stop another thread invalidating this TB
1171 before we are done. */
1173 /* add in the physical hash table */
1174 h
= tb_phys_hash_func(phys_pc
);
1175 ptb
= &tb_phys_hash
[h
];
1176 tb
->phys_hash_next
= *ptb
;
1179 /* add in the page list */
1180 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1181 if (phys_page2
!= -1)
1182 tb_alloc_page(tb
, 1, phys_page2
);
1184 tb
->page_addr
[1] = -1;
1186 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2);
1187 tb
->jmp_next
[0] = NULL
;
1188 tb
->jmp_next
[1] = NULL
;
1190 /* init original jump addresses */
1191 if (tb
->tb_next_offset
[0] != 0xffff)
1192 tb_reset_jump(tb
, 0);
1193 if (tb
->tb_next_offset
[1] != 0xffff)
1194 tb_reset_jump(tb
, 1);
1196 #ifdef DEBUG_TB_CHECK
1202 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1203 tb[1].tc_ptr. Return NULL if not found */
1204 TranslationBlock
*tb_find_pc(unsigned long tc_ptr
)
1206 int m_min
, m_max
, m
;
1208 TranslationBlock
*tb
;
1212 if (tc_ptr
< (unsigned long)code_gen_buffer
||
1213 tc_ptr
>= (unsigned long)code_gen_ptr
)
1215 /* binary search (cf Knuth) */
1218 while (m_min
<= m_max
) {
1219 m
= (m_min
+ m_max
) >> 1;
1221 v
= (unsigned long)tb
->tc_ptr
;
1224 else if (tc_ptr
< v
) {
1233 static void tb_reset_jump_recursive(TranslationBlock
*tb
);
1235 static inline void tb_reset_jump_recursive2(TranslationBlock
*tb
, int n
)
1237 TranslationBlock
*tb1
, *tb_next
, **ptb
;
1240 tb1
= tb
->jmp_next
[n
];
1242 /* find head of list */
1245 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1248 tb1
= tb1
->jmp_next
[n1
];
1250 /* we are now sure now that tb jumps to tb1 */
1253 /* remove tb from the jmp_first list */
1254 ptb
= &tb_next
->jmp_first
;
1258 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1259 if (n1
== n
&& tb1
== tb
)
1261 ptb
= &tb1
->jmp_next
[n1
];
1263 *ptb
= tb
->jmp_next
[n
];
1264 tb
->jmp_next
[n
] = NULL
;
1266 /* suppress the jump to next tb in generated code */
1267 tb_reset_jump(tb
, n
);
1269 /* suppress jumps in the tb on which we could have jumped */
1270 tb_reset_jump_recursive(tb_next
);
1274 static void tb_reset_jump_recursive(TranslationBlock
*tb
)
1276 tb_reset_jump_recursive2(tb
, 0);
1277 tb_reset_jump_recursive2(tb
, 1);
1280 #if defined(TARGET_HAS_ICE)
1281 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1283 target_phys_addr_t addr
;
1285 ram_addr_t ram_addr
;
1288 addr
= cpu_get_phys_page_debug(env
, pc
);
1289 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
1291 pd
= IO_MEM_UNASSIGNED
;
1293 pd
= p
->phys_offset
;
1295 ram_addr
= (pd
& TARGET_PAGE_MASK
) | (pc
& ~TARGET_PAGE_MASK
);
1296 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1300 /* Add a watchpoint. */
1301 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1302 int flags
, CPUWatchpoint
**watchpoint
)
1304 target_ulong len_mask
= ~(len
- 1);
1307 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1308 if ((len
!= 1 && len
!= 2 && len
!= 4 && len
!= 8) || (addr
& ~len_mask
)) {
1309 fprintf(stderr
, "qemu: tried to set invalid watchpoint at "
1310 TARGET_FMT_lx
", len=" TARGET_FMT_lu
"\n", addr
, len
);
1313 wp
= qemu_malloc(sizeof(*wp
));
1318 wp
->len_mask
= len_mask
;
1321 wp
->next
= env
->watchpoints
;
1324 wp
->next
->prev
= wp
;
1325 env
->watchpoints
= wp
;
1327 tlb_flush_page(env
, addr
);
1328 /* FIXME: This flush is needed because of the hack to make memory ops
1329 terminate the TB. It can be removed once the proper IO trap and
1330 re-execute bits are in. */
1338 /* Remove a specific watchpoint. */
1339 int cpu_watchpoint_remove(CPUState
*env
, target_ulong addr
, target_ulong len
,
1342 target_ulong len_mask
= ~(len
- 1);
1345 for (wp
= env
->watchpoints
; wp
!= NULL
; wp
= wp
->next
) {
1346 if (addr
== wp
->vaddr
&& len_mask
== wp
->len_mask
1347 && flags
== wp
->flags
) {
1348 cpu_watchpoint_remove_by_ref(env
, wp
);
1355 /* Remove a specific watchpoint by reference. */
1356 void cpu_watchpoint_remove_by_ref(CPUState
*env
, CPUWatchpoint
*watchpoint
)
1358 if (watchpoint
->next
)
1359 watchpoint
->next
->prev
= watchpoint
->prev
;
1360 if (watchpoint
->prev
)
1361 watchpoint
->prev
->next
= watchpoint
->next
;
1363 env
->watchpoints
= watchpoint
->next
;
1365 tlb_flush_page(env
, watchpoint
->vaddr
);
1367 qemu_free(watchpoint
);
1370 /* Remove all matching watchpoints. */
1371 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1375 for (wp
= env
->watchpoints
; wp
!= NULL
; wp
= wp
->next
)
1376 if (wp
->flags
& mask
)
1377 cpu_watchpoint_remove_by_ref(env
, wp
);
1380 /* Add a breakpoint. */
1381 int cpu_breakpoint_insert(CPUState
*env
, target_ulong pc
, int flags
,
1382 CPUBreakpoint
**breakpoint
)
1384 #if defined(TARGET_HAS_ICE)
1387 bp
= qemu_malloc(sizeof(*bp
));
1394 bp
->next
= env
->breakpoints
;
1397 bp
->next
->prev
= bp
;
1398 env
->breakpoints
= bp
;
1400 breakpoint_invalidate(env
, pc
);
1410 /* Remove a specific breakpoint. */
1411 int cpu_breakpoint_remove(CPUState
*env
, target_ulong pc
, int flags
)
1413 #if defined(TARGET_HAS_ICE)
1416 for (bp
= env
->breakpoints
; bp
!= NULL
; bp
= bp
->next
) {
1417 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
1418 cpu_breakpoint_remove_by_ref(env
, bp
);
1428 /* Remove a specific breakpoint by reference. */
1429 void cpu_breakpoint_remove_by_ref(CPUState
*env
, CPUBreakpoint
*breakpoint
)
1431 #if defined(TARGET_HAS_ICE)
1432 if (breakpoint
->next
)
1433 breakpoint
->next
->prev
= breakpoint
->prev
;
1434 if (breakpoint
->prev
)
1435 breakpoint
->prev
->next
= breakpoint
->next
;
1437 env
->breakpoints
= breakpoint
->next
;
1439 breakpoint_invalidate(env
, breakpoint
->pc
);
1441 qemu_free(breakpoint
);
1445 /* Remove all matching breakpoints. */
1446 void cpu_breakpoint_remove_all(CPUState
*env
, int mask
)
1448 #if defined(TARGET_HAS_ICE)
1451 for (bp
= env
->breakpoints
; bp
!= NULL
; bp
= bp
->next
)
1452 if (bp
->flags
& mask
)
1453 cpu_breakpoint_remove_by_ref(env
, bp
);
1457 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1458 CPU loop after each instruction */
1459 void cpu_single_step(CPUState
*env
, int enabled
)
1461 #if defined(TARGET_HAS_ICE)
1462 if (env
->singlestep_enabled
!= enabled
) {
1463 env
->singlestep_enabled
= enabled
;
1464 /* must flush all the translated code to avoid inconsistancies */
1465 /* XXX: only flush what is necessary */
1471 /* enable or disable low levels log */
1472 void cpu_set_log(int log_flags
)
1474 loglevel
= log_flags
;
1475 if (loglevel
&& !logfile
) {
1476 logfile
= fopen(logfilename
, log_append
? "a" : "w");
1478 perror(logfilename
);
1481 #if !defined(CONFIG_SOFTMMU)
1482 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1484 static char logfile_buf
[4096];
1485 setvbuf(logfile
, logfile_buf
, _IOLBF
, sizeof(logfile_buf
));
1488 setvbuf(logfile
, NULL
, _IOLBF
, 0);
1492 if (!loglevel
&& logfile
) {
1498 void cpu_set_log_filename(const char *filename
)
1500 logfilename
= strdup(filename
);
1505 cpu_set_log(loglevel
);
1508 /* mask must never be zero, except for A20 change call */
1509 void cpu_interrupt(CPUState
*env
, int mask
)
1511 #if !defined(USE_NPTL)
1512 TranslationBlock
*tb
;
1513 static spinlock_t interrupt_lock
= SPIN_LOCK_UNLOCKED
;
1517 old_mask
= env
->interrupt_request
;
1518 /* FIXME: This is probably not threadsafe. A different thread could
1519 be in the middle of a read-modify-write operation. */
1520 env
->interrupt_request
|= mask
;
1521 #if defined(USE_NPTL)
1522 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1523 problem and hope the cpu will stop of its own accord. For userspace
1524 emulation this often isn't actually as bad as it sounds. Often
1525 signals are used primarily to interrupt blocking syscalls. */
1528 env
->icount_decr
.u16
.high
= 0xffff;
1529 #ifndef CONFIG_USER_ONLY
1530 /* CPU_INTERRUPT_EXIT isn't a real interrupt. It just means
1531 an async event happened and we need to process it. */
1533 && (mask
& ~(old_mask
| CPU_INTERRUPT_EXIT
)) != 0) {
1534 cpu_abort(env
, "Raised interrupt while not in I/O function");
1538 tb
= env
->current_tb
;
1539 /* if the cpu is currently executing code, we must unlink it and
1540 all the potentially executing TB */
1541 if (tb
&& !testandset(&interrupt_lock
)) {
1542 env
->current_tb
= NULL
;
1543 tb_reset_jump_recursive(tb
);
1544 resetlock(&interrupt_lock
);
1550 void cpu_reset_interrupt(CPUState
*env
, int mask
)
1552 env
->interrupt_request
&= ~mask
;
1555 const CPULogItem cpu_log_items
[] = {
1556 { CPU_LOG_TB_OUT_ASM
, "out_asm",
1557 "show generated host assembly code for each compiled TB" },
1558 { CPU_LOG_TB_IN_ASM
, "in_asm",
1559 "show target assembly code for each compiled TB" },
1560 { CPU_LOG_TB_OP
, "op",
1561 "show micro ops for each compiled TB" },
1562 { CPU_LOG_TB_OP_OPT
, "op_opt",
1565 "before eflags optimization and "
1567 "after liveness analysis" },
1568 { CPU_LOG_INT
, "int",
1569 "show interrupts/exceptions in short format" },
1570 { CPU_LOG_EXEC
, "exec",
1571 "show trace before each executed TB (lots of logs)" },
1572 { CPU_LOG_TB_CPU
, "cpu",
1573 "show CPU state before block translation" },
1575 { CPU_LOG_PCALL
, "pcall",
1576 "show protected mode far calls/returns/exceptions" },
1579 { CPU_LOG_IOPORT
, "ioport",
1580 "show all i/o ports accesses" },
1585 static int cmp1(const char *s1
, int n
, const char *s2
)
1587 if (strlen(s2
) != n
)
1589 return memcmp(s1
, s2
, n
) == 0;
1592 /* takes a comma separated list of log masks. Return 0 if error. */
1593 int cpu_str_to_log_mask(const char *str
)
1595 const CPULogItem
*item
;
1602 p1
= strchr(p
, ',');
1605 if(cmp1(p
,p1
-p
,"all")) {
1606 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1610 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1611 if (cmp1(p
, p1
- p
, item
->name
))
1625 void cpu_abort(CPUState
*env
, const char *fmt
, ...)
1632 fprintf(stderr
, "qemu: fatal: ");
1633 vfprintf(stderr
, fmt
, ap
);
1634 fprintf(stderr
, "\n");
1636 cpu_dump_state(env
, stderr
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1638 cpu_dump_state(env
, stderr
, fprintf
, 0);
1641 fprintf(logfile
, "qemu: fatal: ");
1642 vfprintf(logfile
, fmt
, ap2
);
1643 fprintf(logfile
, "\n");
1645 cpu_dump_state(env
, logfile
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1647 cpu_dump_state(env
, logfile
, fprintf
, 0);
1657 CPUState
*cpu_copy(CPUState
*env
)
1659 CPUState
*new_env
= cpu_init(env
->cpu_model_str
);
1660 /* preserve chaining and index */
1661 CPUState
*next_cpu
= new_env
->next_cpu
;
1662 int cpu_index
= new_env
->cpu_index
;
1663 memcpy(new_env
, env
, sizeof(CPUState
));
1664 new_env
->next_cpu
= next_cpu
;
1665 new_env
->cpu_index
= cpu_index
;
1669 #if !defined(CONFIG_USER_ONLY)
1671 static inline void tlb_flush_jmp_cache(CPUState
*env
, target_ulong addr
)
1675 /* Discard jump cache entries for any tb which might potentially
1676 overlap the flushed page. */
1677 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1678 memset (&env
->tb_jmp_cache
[i
], 0,
1679 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1681 i
= tb_jmp_cache_hash_page(addr
);
1682 memset (&env
->tb_jmp_cache
[i
], 0,
1683 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1686 /* NOTE: if flush_global is true, also flush global entries (not
1688 void tlb_flush(CPUState
*env
, int flush_global
)
1692 #if defined(DEBUG_TLB)
1693 printf("tlb_flush:\n");
1695 /* must reset current TB so that interrupts cannot modify the
1696 links while we are modifying them */
1697 env
->current_tb
= NULL
;
1699 for(i
= 0; i
< CPU_TLB_SIZE
; i
++) {
1700 env
->tlb_table
[0][i
].addr_read
= -1;
1701 env
->tlb_table
[0][i
].addr_write
= -1;
1702 env
->tlb_table
[0][i
].addr_code
= -1;
1703 env
->tlb_table
[1][i
].addr_read
= -1;
1704 env
->tlb_table
[1][i
].addr_write
= -1;
1705 env
->tlb_table
[1][i
].addr_code
= -1;
1706 #if (NB_MMU_MODES >= 3)
1707 env
->tlb_table
[2][i
].addr_read
= -1;
1708 env
->tlb_table
[2][i
].addr_write
= -1;
1709 env
->tlb_table
[2][i
].addr_code
= -1;
1710 #if (NB_MMU_MODES == 4)
1711 env
->tlb_table
[3][i
].addr_read
= -1;
1712 env
->tlb_table
[3][i
].addr_write
= -1;
1713 env
->tlb_table
[3][i
].addr_code
= -1;
1718 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
1721 if (env
->kqemu_enabled
) {
1722 kqemu_flush(env
, flush_global
);
1728 static inline void tlb_flush_entry(CPUTLBEntry
*tlb_entry
, target_ulong addr
)
1730 if (addr
== (tlb_entry
->addr_read
&
1731 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1732 addr
== (tlb_entry
->addr_write
&
1733 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1734 addr
== (tlb_entry
->addr_code
&
1735 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
))) {
1736 tlb_entry
->addr_read
= -1;
1737 tlb_entry
->addr_write
= -1;
1738 tlb_entry
->addr_code
= -1;
1742 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
1746 #if defined(DEBUG_TLB)
1747 printf("tlb_flush_page: " TARGET_FMT_lx
"\n", addr
);
1749 /* must reset current TB so that interrupts cannot modify the
1750 links while we are modifying them */
1751 env
->current_tb
= NULL
;
1753 addr
&= TARGET_PAGE_MASK
;
1754 i
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1755 tlb_flush_entry(&env
->tlb_table
[0][i
], addr
);
1756 tlb_flush_entry(&env
->tlb_table
[1][i
], addr
);
1757 #if (NB_MMU_MODES >= 3)
1758 tlb_flush_entry(&env
->tlb_table
[2][i
], addr
);
1759 #if (NB_MMU_MODES == 4)
1760 tlb_flush_entry(&env
->tlb_table
[3][i
], addr
);
1764 tlb_flush_jmp_cache(env
, addr
);
1767 if (env
->kqemu_enabled
) {
1768 kqemu_flush_page(env
, addr
);
1773 /* update the TLBs so that writes to code in the virtual page 'addr'
1775 static void tlb_protect_code(ram_addr_t ram_addr
)
1777 cpu_physical_memory_reset_dirty(ram_addr
,
1778 ram_addr
+ TARGET_PAGE_SIZE
,
1782 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1783 tested for self modifying code */
1784 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
1787 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] |= CODE_DIRTY_FLAG
;
1790 static inline void tlb_reset_dirty_range(CPUTLBEntry
*tlb_entry
,
1791 unsigned long start
, unsigned long length
)
1794 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
1795 addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) + tlb_entry
->addend
;
1796 if ((addr
- start
) < length
) {
1797 tlb_entry
->addr_write
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) | TLB_NOTDIRTY
;
1802 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t end
,
1806 unsigned long length
, start1
;
1810 start
&= TARGET_PAGE_MASK
;
1811 end
= TARGET_PAGE_ALIGN(end
);
1813 length
= end
- start
;
1816 len
= length
>> TARGET_PAGE_BITS
;
1818 /* XXX: should not depend on cpu context */
1820 if (env
->kqemu_enabled
) {
1823 for(i
= 0; i
< len
; i
++) {
1824 kqemu_set_notdirty(env
, addr
);
1825 addr
+= TARGET_PAGE_SIZE
;
1829 mask
= ~dirty_flags
;
1830 p
= phys_ram_dirty
+ (start
>> TARGET_PAGE_BITS
);
1831 for(i
= 0; i
< len
; i
++)
1834 /* we modify the TLB cache so that the dirty bit will be set again
1835 when accessing the range */
1836 start1
= start
+ (unsigned long)phys_ram_base
;
1837 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
1838 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1839 tlb_reset_dirty_range(&env
->tlb_table
[0][i
], start1
, length
);
1840 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1841 tlb_reset_dirty_range(&env
->tlb_table
[1][i
], start1
, length
);
1842 #if (NB_MMU_MODES >= 3)
1843 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1844 tlb_reset_dirty_range(&env
->tlb_table
[2][i
], start1
, length
);
1845 #if (NB_MMU_MODES == 4)
1846 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1847 tlb_reset_dirty_range(&env
->tlb_table
[3][i
], start1
, length
);
1853 int cpu_physical_memory_set_dirty_tracking(int enable
)
1855 in_migration
= enable
;
1859 int cpu_physical_memory_get_dirty_tracking(void)
1861 return in_migration
;
1864 static inline void tlb_update_dirty(CPUTLBEntry
*tlb_entry
)
1866 ram_addr_t ram_addr
;
1868 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
1869 ram_addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) +
1870 tlb_entry
->addend
- (unsigned long)phys_ram_base
;
1871 if (!cpu_physical_memory_is_dirty(ram_addr
)) {
1872 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
1877 /* update the TLB according to the current state of the dirty bits */
1878 void cpu_tlb_update_dirty(CPUState
*env
)
1881 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1882 tlb_update_dirty(&env
->tlb_table
[0][i
]);
1883 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1884 tlb_update_dirty(&env
->tlb_table
[1][i
]);
1885 #if (NB_MMU_MODES >= 3)
1886 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1887 tlb_update_dirty(&env
->tlb_table
[2][i
]);
1888 #if (NB_MMU_MODES == 4)
1889 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1890 tlb_update_dirty(&env
->tlb_table
[3][i
]);
1895 static inline void tlb_set_dirty1(CPUTLBEntry
*tlb_entry
, target_ulong vaddr
)
1897 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
))
1898 tlb_entry
->addr_write
= vaddr
;
1901 /* update the TLB corresponding to virtual page vaddr
1902 so that it is no longer dirty */
1903 static inline void tlb_set_dirty(CPUState
*env
, target_ulong vaddr
)
1907 vaddr
&= TARGET_PAGE_MASK
;
1908 i
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1909 tlb_set_dirty1(&env
->tlb_table
[0][i
], vaddr
);
1910 tlb_set_dirty1(&env
->tlb_table
[1][i
], vaddr
);
1911 #if (NB_MMU_MODES >= 3)
1912 tlb_set_dirty1(&env
->tlb_table
[2][i
], vaddr
);
1913 #if (NB_MMU_MODES == 4)
1914 tlb_set_dirty1(&env
->tlb_table
[3][i
], vaddr
);
1919 /* add a new TLB entry. At most one entry for a given virtual address
1920 is permitted. Return 0 if OK or 2 if the page could not be mapped
1921 (can only happen in non SOFTMMU mode for I/O pages or pages
1922 conflicting with the host address space). */
1923 int tlb_set_page_exec(CPUState
*env
, target_ulong vaddr
,
1924 target_phys_addr_t paddr
, int prot
,
1925 int mmu_idx
, int is_softmmu
)
1930 target_ulong address
;
1931 target_ulong code_address
;
1932 target_phys_addr_t addend
;
1936 target_phys_addr_t iotlb
;
1938 p
= phys_page_find(paddr
>> TARGET_PAGE_BITS
);
1940 pd
= IO_MEM_UNASSIGNED
;
1942 pd
= p
->phys_offset
;
1944 #if defined(DEBUG_TLB)
1945 printf("tlb_set_page: vaddr=" TARGET_FMT_lx
" paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1946 vaddr
, (int)paddr
, prot
, mmu_idx
, is_softmmu
, pd
);
1951 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&& !(pd
& IO_MEM_ROMD
)) {
1952 /* IO memory case (romd handled later) */
1953 address
|= TLB_MMIO
;
1955 addend
= (unsigned long)phys_ram_base
+ (pd
& TARGET_PAGE_MASK
);
1956 if ((pd
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
) {
1958 iotlb
= pd
& TARGET_PAGE_MASK
;
1959 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
)
1960 iotlb
|= IO_MEM_NOTDIRTY
;
1962 iotlb
|= IO_MEM_ROM
;
1964 /* IO handlers are currently passed a phsical address.
1965 It would be nice to pass an offset from the base address
1966 of that region. This would avoid having to special case RAM,
1967 and avoid full address decoding in every device.
1968 We can't use the high bits of pd for this because
1969 IO_MEM_ROMD uses these as a ram address. */
1970 iotlb
= (pd
& ~TARGET_PAGE_MASK
) + paddr
;
1973 code_address
= address
;
1974 /* Make accesses to pages with watchpoints go via the
1975 watchpoint trap routines. */
1976 for (wp
= env
->watchpoints
; wp
!= NULL
; wp
= wp
->next
) {
1977 if (vaddr
== (wp
->vaddr
& TARGET_PAGE_MASK
)) {
1978 iotlb
= io_mem_watch
+ paddr
;
1979 /* TODO: The memory case can be optimized by not trapping
1980 reads of pages with a write breakpoint. */
1981 address
|= TLB_MMIO
;
1985 index
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1986 env
->iotlb
[mmu_idx
][index
] = iotlb
- vaddr
;
1987 te
= &env
->tlb_table
[mmu_idx
][index
];
1988 te
->addend
= addend
- vaddr
;
1989 if (prot
& PAGE_READ
) {
1990 te
->addr_read
= address
;
1995 if (prot
& PAGE_EXEC
) {
1996 te
->addr_code
= code_address
;
2000 if (prot
& PAGE_WRITE
) {
2001 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_ROM
||
2002 (pd
& IO_MEM_ROMD
)) {
2003 /* Write access calls the I/O callback. */
2004 te
->addr_write
= address
| TLB_MMIO
;
2005 } else if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
&&
2006 !cpu_physical_memory_is_dirty(pd
)) {
2007 te
->addr_write
= address
| TLB_NOTDIRTY
;
2009 te
->addr_write
= address
;
2012 te
->addr_write
= -1;
2019 void tlb_flush(CPUState
*env
, int flush_global
)
2023 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
2027 int tlb_set_page_exec(CPUState
*env
, target_ulong vaddr
,
2028 target_phys_addr_t paddr
, int prot
,
2029 int mmu_idx
, int is_softmmu
)
2034 /* dump memory mappings */
2035 void page_dump(FILE *f
)
2037 unsigned long start
, end
;
2038 int i
, j
, prot
, prot1
;
2041 fprintf(f
, "%-8s %-8s %-8s %s\n",
2042 "start", "end", "size", "prot");
2046 for(i
= 0; i
<= L1_SIZE
; i
++) {
2051 for(j
= 0;j
< L2_SIZE
; j
++) {
2056 if (prot1
!= prot
) {
2057 end
= (i
<< (32 - L1_BITS
)) | (j
<< TARGET_PAGE_BITS
);
2059 fprintf(f
, "%08lx-%08lx %08lx %c%c%c\n",
2060 start
, end
, end
- start
,
2061 prot
& PAGE_READ
? 'r' : '-',
2062 prot
& PAGE_WRITE
? 'w' : '-',
2063 prot
& PAGE_EXEC
? 'x' : '-');
2077 int page_get_flags(target_ulong address
)
2081 p
= page_find(address
>> TARGET_PAGE_BITS
);
2087 /* modify the flags of a page and invalidate the code if
2088 necessary. The flag PAGE_WRITE_ORG is positionned automatically
2089 depending on PAGE_WRITE */
2090 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2095 /* mmap_lock should already be held. */
2096 start
= start
& TARGET_PAGE_MASK
;
2097 end
= TARGET_PAGE_ALIGN(end
);
2098 if (flags
& PAGE_WRITE
)
2099 flags
|= PAGE_WRITE_ORG
;
2100 for(addr
= start
; addr
< end
; addr
+= TARGET_PAGE_SIZE
) {
2101 p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
);
2102 /* We may be called for host regions that are outside guest
2106 /* if the write protection is set, then we invalidate the code
2108 if (!(p
->flags
& PAGE_WRITE
) &&
2109 (flags
& PAGE_WRITE
) &&
2111 tb_invalidate_phys_page(addr
, 0, NULL
);
2117 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2123 if (start
+ len
< start
)
2124 /* we've wrapped around */
2127 end
= TARGET_PAGE_ALIGN(start
+len
); /* must do before we loose bits in the next step */
2128 start
= start
& TARGET_PAGE_MASK
;
2130 for(addr
= start
; addr
< end
; addr
+= TARGET_PAGE_SIZE
) {
2131 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2134 if( !(p
->flags
& PAGE_VALID
) )
2137 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
))
2139 if (flags
& PAGE_WRITE
) {
2140 if (!(p
->flags
& PAGE_WRITE_ORG
))
2142 /* unprotect the page if it was put read-only because it
2143 contains translated code */
2144 if (!(p
->flags
& PAGE_WRITE
)) {
2145 if (!page_unprotect(addr
, 0, NULL
))
2154 /* called from signal handler: invalidate the code and unprotect the
2155 page. Return TRUE if the fault was succesfully handled. */
2156 int page_unprotect(target_ulong address
, unsigned long pc
, void *puc
)
2158 unsigned int page_index
, prot
, pindex
;
2160 target_ulong host_start
, host_end
, addr
;
2162 /* Technically this isn't safe inside a signal handler. However we
2163 know this only ever happens in a synchronous SEGV handler, so in
2164 practice it seems to be ok. */
2167 host_start
= address
& qemu_host_page_mask
;
2168 page_index
= host_start
>> TARGET_PAGE_BITS
;
2169 p1
= page_find(page_index
);
2174 host_end
= host_start
+ qemu_host_page_size
;
2177 for(addr
= host_start
;addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2181 /* if the page was really writable, then we change its
2182 protection back to writable */
2183 if (prot
& PAGE_WRITE_ORG
) {
2184 pindex
= (address
- host_start
) >> TARGET_PAGE_BITS
;
2185 if (!(p1
[pindex
].flags
& PAGE_WRITE
)) {
2186 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2187 (prot
& PAGE_BITS
) | PAGE_WRITE
);
2188 p1
[pindex
].flags
|= PAGE_WRITE
;
2189 /* and since the content will be modified, we must invalidate
2190 the corresponding translated code. */
2191 tb_invalidate_phys_page(address
, pc
, puc
);
2192 #ifdef DEBUG_TB_CHECK
2193 tb_invalidate_check(address
);
2203 static inline void tlb_set_dirty(CPUState
*env
,
2204 unsigned long addr
, target_ulong vaddr
)
2207 #endif /* defined(CONFIG_USER_ONLY) */
2209 #if !defined(CONFIG_USER_ONLY)
2210 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2212 static void *subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
2213 ram_addr_t orig_memory
);
2214 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2217 if (addr > start_addr) \
2220 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2221 if (start_addr2 > 0) \
2225 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2226 end_addr2 = TARGET_PAGE_SIZE - 1; \
2228 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2229 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2234 /* register physical memory. 'size' must be a multiple of the target
2235 page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2237 void cpu_register_physical_memory(target_phys_addr_t start_addr
,
2239 ram_addr_t phys_offset
)
2241 target_phys_addr_t addr
, end_addr
;
2244 ram_addr_t orig_size
= size
;
2248 /* XXX: should not depend on cpu context */
2250 if (env
->kqemu_enabled
) {
2251 kqemu_set_phys_mem(start_addr
, size
, phys_offset
);
2255 kvm_set_phys_mem(start_addr
, size
, phys_offset
);
2257 size
= (size
+ TARGET_PAGE_SIZE
- 1) & TARGET_PAGE_MASK
;
2258 end_addr
= start_addr
+ (target_phys_addr_t
)size
;
2259 for(addr
= start_addr
; addr
!= end_addr
; addr
+= TARGET_PAGE_SIZE
) {
2260 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2261 if (p
&& p
->phys_offset
!= IO_MEM_UNASSIGNED
) {
2262 ram_addr_t orig_memory
= p
->phys_offset
;
2263 target_phys_addr_t start_addr2
, end_addr2
;
2264 int need_subpage
= 0;
2266 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
, end_addr2
,
2268 if (need_subpage
|| phys_offset
& IO_MEM_SUBWIDTH
) {
2269 if (!(orig_memory
& IO_MEM_SUBPAGE
)) {
2270 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2271 &p
->phys_offset
, orig_memory
);
2273 subpage
= io_mem_opaque
[(orig_memory
& ~TARGET_PAGE_MASK
)
2276 subpage_register(subpage
, start_addr2
, end_addr2
, phys_offset
);
2278 p
->phys_offset
= phys_offset
;
2279 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2280 (phys_offset
& IO_MEM_ROMD
))
2281 phys_offset
+= TARGET_PAGE_SIZE
;
2284 p
= phys_page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2285 p
->phys_offset
= phys_offset
;
2286 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2287 (phys_offset
& IO_MEM_ROMD
))
2288 phys_offset
+= TARGET_PAGE_SIZE
;
2290 target_phys_addr_t start_addr2
, end_addr2
;
2291 int need_subpage
= 0;
2293 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
,
2294 end_addr2
, need_subpage
);
2296 if (need_subpage
|| phys_offset
& IO_MEM_SUBWIDTH
) {
2297 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2298 &p
->phys_offset
, IO_MEM_UNASSIGNED
);
2299 subpage_register(subpage
, start_addr2
, end_addr2
,
2306 /* since each CPU stores ram addresses in its TLB cache, we must
2307 reset the modified entries */
2309 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2314 /* XXX: temporary until new memory mapping API */
2315 ram_addr_t
cpu_get_physical_page_desc(target_phys_addr_t addr
)
2319 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2321 return IO_MEM_UNASSIGNED
;
2322 return p
->phys_offset
;
2325 /* XXX: better than nothing */
2326 ram_addr_t
qemu_ram_alloc(ram_addr_t size
)
2329 if ((phys_ram_alloc_offset
+ size
) > phys_ram_size
) {
2330 fprintf(stderr
, "Not enough memory (requested_size = %" PRIu64
", max memory = %" PRIu64
")\n",
2331 (uint64_t)size
, (uint64_t)phys_ram_size
);
2334 addr
= phys_ram_alloc_offset
;
2335 phys_ram_alloc_offset
= TARGET_PAGE_ALIGN(phys_ram_alloc_offset
+ size
);
2339 void qemu_ram_free(ram_addr_t addr
)
2343 static uint32_t unassigned_mem_readb(void *opaque
, target_phys_addr_t addr
)
2345 #ifdef DEBUG_UNASSIGNED
2346 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2348 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2349 do_unassigned_access(addr
, 0, 0, 0, 1);
2354 static uint32_t unassigned_mem_readw(void *opaque
, target_phys_addr_t addr
)
2356 #ifdef DEBUG_UNASSIGNED
2357 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2359 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2360 do_unassigned_access(addr
, 0, 0, 0, 2);
2365 static uint32_t unassigned_mem_readl(void *opaque
, target_phys_addr_t addr
)
2367 #ifdef DEBUG_UNASSIGNED
2368 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2370 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2371 do_unassigned_access(addr
, 0, 0, 0, 4);
2376 static void unassigned_mem_writeb(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2378 #ifdef DEBUG_UNASSIGNED
2379 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2381 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2382 do_unassigned_access(addr
, 1, 0, 0, 1);
2386 static void unassigned_mem_writew(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2388 #ifdef DEBUG_UNASSIGNED
2389 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2391 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2392 do_unassigned_access(addr
, 1, 0, 0, 2);
2396 static void unassigned_mem_writel(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2398 #ifdef DEBUG_UNASSIGNED
2399 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2401 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2402 do_unassigned_access(addr
, 1, 0, 0, 4);
2406 static CPUReadMemoryFunc
*unassigned_mem_read
[3] = {
2407 unassigned_mem_readb
,
2408 unassigned_mem_readw
,
2409 unassigned_mem_readl
,
2412 static CPUWriteMemoryFunc
*unassigned_mem_write
[3] = {
2413 unassigned_mem_writeb
,
2414 unassigned_mem_writew
,
2415 unassigned_mem_writel
,
2418 static void notdirty_mem_writeb(void *opaque
, target_phys_addr_t ram_addr
,
2422 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2423 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2424 #if !defined(CONFIG_USER_ONLY)
2425 tb_invalidate_phys_page_fast(ram_addr
, 1);
2426 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2429 stb_p(phys_ram_base
+ ram_addr
, val
);
2431 if (cpu_single_env
->kqemu_enabled
&&
2432 (dirty_flags
& KQEMU_MODIFY_PAGE_MASK
) != KQEMU_MODIFY_PAGE_MASK
)
2433 kqemu_modify_page(cpu_single_env
, ram_addr
);
2435 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2436 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2437 /* we remove the notdirty callback only if the code has been
2439 if (dirty_flags
== 0xff)
2440 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2443 static void notdirty_mem_writew(void *opaque
, target_phys_addr_t ram_addr
,
2447 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2448 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2449 #if !defined(CONFIG_USER_ONLY)
2450 tb_invalidate_phys_page_fast(ram_addr
, 2);
2451 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2454 stw_p(phys_ram_base
+ ram_addr
, val
);
2456 if (cpu_single_env
->kqemu_enabled
&&
2457 (dirty_flags
& KQEMU_MODIFY_PAGE_MASK
) != KQEMU_MODIFY_PAGE_MASK
)
2458 kqemu_modify_page(cpu_single_env
, ram_addr
);
2460 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2461 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2462 /* we remove the notdirty callback only if the code has been
2464 if (dirty_flags
== 0xff)
2465 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2468 static void notdirty_mem_writel(void *opaque
, target_phys_addr_t ram_addr
,
2472 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2473 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2474 #if !defined(CONFIG_USER_ONLY)
2475 tb_invalidate_phys_page_fast(ram_addr
, 4);
2476 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2479 stl_p(phys_ram_base
+ ram_addr
, val
);
2481 if (cpu_single_env
->kqemu_enabled
&&
2482 (dirty_flags
& KQEMU_MODIFY_PAGE_MASK
) != KQEMU_MODIFY_PAGE_MASK
)
2483 kqemu_modify_page(cpu_single_env
, ram_addr
);
2485 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2486 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2487 /* we remove the notdirty callback only if the code has been
2489 if (dirty_flags
== 0xff)
2490 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2493 static CPUReadMemoryFunc
*error_mem_read
[3] = {
2494 NULL
, /* never used */
2495 NULL
, /* never used */
2496 NULL
, /* never used */
2499 static CPUWriteMemoryFunc
*notdirty_mem_write
[3] = {
2500 notdirty_mem_writeb
,
2501 notdirty_mem_writew
,
2502 notdirty_mem_writel
,
2505 /* Generate a debug exception if a watchpoint has been hit. */
2506 static void check_watchpoint(int offset
, int len_mask
, int flags
)
2508 CPUState
*env
= cpu_single_env
;
2509 target_ulong pc
, cs_base
;
2510 TranslationBlock
*tb
;
2515 if (env
->watchpoint_hit
) {
2516 /* We re-entered the check after replacing the TB. Now raise
2517 * the debug interrupt so that is will trigger after the
2518 * current instruction. */
2519 cpu_interrupt(env
, CPU_INTERRUPT_DEBUG
);
2522 vaddr
= (env
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
2523 for (wp
= env
->watchpoints
; wp
!= NULL
; wp
= wp
->next
) {
2524 if ((vaddr
== (wp
->vaddr
& len_mask
) ||
2525 (vaddr
& wp
->len_mask
) == wp
->vaddr
) && (wp
->flags
& flags
)) {
2526 env
->watchpoint_hit
= wp
;
2527 tb
= tb_find_pc(env
->mem_io_pc
);
2529 cpu_abort(env
, "check_watchpoint: could not find TB for pc=%p",
2530 (void *)env
->mem_io_pc
);
2532 cpu_restore_state(tb
, env
, env
->mem_io_pc
, NULL
);
2533 tb_phys_invalidate(tb
, -1);
2534 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
2535 env
->exception_index
= EXCP_DEBUG
;
2537 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
2538 tb_gen_code(env
, pc
, cs_base
, cpu_flags
, 1);
2540 cpu_resume_from_signal(env
, NULL
);
2545 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2546 so these check for a hit then pass through to the normal out-of-line
2548 static uint32_t watch_mem_readb(void *opaque
, target_phys_addr_t addr
)
2550 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_READ
);
2551 return ldub_phys(addr
);
2554 static uint32_t watch_mem_readw(void *opaque
, target_phys_addr_t addr
)
2556 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_READ
);
2557 return lduw_phys(addr
);
2560 static uint32_t watch_mem_readl(void *opaque
, target_phys_addr_t addr
)
2562 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_READ
);
2563 return ldl_phys(addr
);
2566 static void watch_mem_writeb(void *opaque
, target_phys_addr_t addr
,
2569 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_WRITE
);
2570 stb_phys(addr
, val
);
2573 static void watch_mem_writew(void *opaque
, target_phys_addr_t addr
,
2576 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_WRITE
);
2577 stw_phys(addr
, val
);
2580 static void watch_mem_writel(void *opaque
, target_phys_addr_t addr
,
2583 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_WRITE
);
2584 stl_phys(addr
, val
);
2587 static CPUReadMemoryFunc
*watch_mem_read
[3] = {
2593 static CPUWriteMemoryFunc
*watch_mem_write
[3] = {
2599 static inline uint32_t subpage_readlen (subpage_t
*mmio
, target_phys_addr_t addr
,
2605 idx
= SUBPAGE_IDX(addr
- mmio
->base
);
2606 #if defined(DEBUG_SUBPAGE)
2607 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d\n", __func__
,
2608 mmio
, len
, addr
, idx
);
2610 ret
= (**mmio
->mem_read
[idx
][len
])(mmio
->opaque
[idx
][0][len
], addr
);
2615 static inline void subpage_writelen (subpage_t
*mmio
, target_phys_addr_t addr
,
2616 uint32_t value
, unsigned int len
)
2620 idx
= SUBPAGE_IDX(addr
- mmio
->base
);
2621 #if defined(DEBUG_SUBPAGE)
2622 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d value %08x\n", __func__
,
2623 mmio
, len
, addr
, idx
, value
);
2625 (**mmio
->mem_write
[idx
][len
])(mmio
->opaque
[idx
][1][len
], addr
, value
);
2628 static uint32_t subpage_readb (void *opaque
, target_phys_addr_t addr
)
2630 #if defined(DEBUG_SUBPAGE)
2631 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2634 return subpage_readlen(opaque
, addr
, 0);
2637 static void subpage_writeb (void *opaque
, target_phys_addr_t addr
,
2640 #if defined(DEBUG_SUBPAGE)
2641 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2643 subpage_writelen(opaque
, addr
, value
, 0);
2646 static uint32_t subpage_readw (void *opaque
, target_phys_addr_t addr
)
2648 #if defined(DEBUG_SUBPAGE)
2649 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2652 return subpage_readlen(opaque
, addr
, 1);
2655 static void subpage_writew (void *opaque
, target_phys_addr_t addr
,
2658 #if defined(DEBUG_SUBPAGE)
2659 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2661 subpage_writelen(opaque
, addr
, value
, 1);
2664 static uint32_t subpage_readl (void *opaque
, target_phys_addr_t addr
)
2666 #if defined(DEBUG_SUBPAGE)
2667 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2670 return subpage_readlen(opaque
, addr
, 2);
2673 static void subpage_writel (void *opaque
,
2674 target_phys_addr_t addr
, uint32_t value
)
2676 #if defined(DEBUG_SUBPAGE)
2677 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2679 subpage_writelen(opaque
, addr
, value
, 2);
2682 static CPUReadMemoryFunc
*subpage_read
[] = {
2688 static CPUWriteMemoryFunc
*subpage_write
[] = {
2694 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2700 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
2702 idx
= SUBPAGE_IDX(start
);
2703 eidx
= SUBPAGE_IDX(end
);
2704 #if defined(DEBUG_SUBPAGE)
2705 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %d\n", __func__
,
2706 mmio
, start
, end
, idx
, eidx
, memory
);
2708 memory
>>= IO_MEM_SHIFT
;
2709 for (; idx
<= eidx
; idx
++) {
2710 for (i
= 0; i
< 4; i
++) {
2711 if (io_mem_read
[memory
][i
]) {
2712 mmio
->mem_read
[idx
][i
] = &io_mem_read
[memory
][i
];
2713 mmio
->opaque
[idx
][0][i
] = io_mem_opaque
[memory
];
2715 if (io_mem_write
[memory
][i
]) {
2716 mmio
->mem_write
[idx
][i
] = &io_mem_write
[memory
][i
];
2717 mmio
->opaque
[idx
][1][i
] = io_mem_opaque
[memory
];
2725 static void *subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
2726 ram_addr_t orig_memory
)
2731 mmio
= qemu_mallocz(sizeof(subpage_t
));
2734 subpage_memory
= cpu_register_io_memory(0, subpage_read
, subpage_write
, mmio
);
2735 #if defined(DEBUG_SUBPAGE)
2736 printf("%s: %p base " TARGET_FMT_plx
" len %08x %d\n", __func__
,
2737 mmio
, base
, TARGET_PAGE_SIZE
, subpage_memory
);
2739 *phys
= subpage_memory
| IO_MEM_SUBPAGE
;
2740 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
- 1, orig_memory
);
2746 static void io_mem_init(void)
2748 cpu_register_io_memory(IO_MEM_ROM
>> IO_MEM_SHIFT
, error_mem_read
, unassigned_mem_write
, NULL
);
2749 cpu_register_io_memory(IO_MEM_UNASSIGNED
>> IO_MEM_SHIFT
, unassigned_mem_read
, unassigned_mem_write
, NULL
);
2750 cpu_register_io_memory(IO_MEM_NOTDIRTY
>> IO_MEM_SHIFT
, error_mem_read
, notdirty_mem_write
, NULL
);
2753 io_mem_watch
= cpu_register_io_memory(0, watch_mem_read
,
2754 watch_mem_write
, NULL
);
2755 /* alloc dirty bits array */
2756 phys_ram_dirty
= qemu_vmalloc(phys_ram_size
>> TARGET_PAGE_BITS
);
2757 memset(phys_ram_dirty
, 0xff, phys_ram_size
>> TARGET_PAGE_BITS
);
2760 /* mem_read and mem_write are arrays of functions containing the
2761 function to access byte (index 0), word (index 1) and dword (index
2762 2). Functions can be omitted with a NULL function pointer. The
2763 registered functions may be modified dynamically later.
2764 If io_index is non zero, the corresponding io zone is
2765 modified. If it is zero, a new io zone is allocated. The return
2766 value can be used with cpu_register_physical_memory(). (-1) is
2767 returned if error. */
2768 int cpu_register_io_memory(int io_index
,
2769 CPUReadMemoryFunc
**mem_read
,
2770 CPUWriteMemoryFunc
**mem_write
,
2773 int i
, subwidth
= 0;
2775 if (io_index
<= 0) {
2776 if (io_mem_nb
>= IO_MEM_NB_ENTRIES
)
2778 io_index
= io_mem_nb
++;
2780 if (io_index
>= IO_MEM_NB_ENTRIES
)
2784 for(i
= 0;i
< 3; i
++) {
2785 if (!mem_read
[i
] || !mem_write
[i
])
2786 subwidth
= IO_MEM_SUBWIDTH
;
2787 io_mem_read
[io_index
][i
] = mem_read
[i
];
2788 io_mem_write
[io_index
][i
] = mem_write
[i
];
2790 io_mem_opaque
[io_index
] = opaque
;
2791 return (io_index
<< IO_MEM_SHIFT
) | subwidth
;
2794 CPUWriteMemoryFunc
**cpu_get_io_memory_write(int io_index
)
2796 return io_mem_write
[io_index
>> IO_MEM_SHIFT
];
2799 CPUReadMemoryFunc
**cpu_get_io_memory_read(int io_index
)
2801 return io_mem_read
[io_index
>> IO_MEM_SHIFT
];
2804 #endif /* !defined(CONFIG_USER_ONLY) */
2806 /* physical memory access (slow version, mainly for debug) */
2807 #if defined(CONFIG_USER_ONLY)
2808 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
2809 int len
, int is_write
)
2816 page
= addr
& TARGET_PAGE_MASK
;
2817 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2820 flags
= page_get_flags(page
);
2821 if (!(flags
& PAGE_VALID
))
2824 if (!(flags
& PAGE_WRITE
))
2826 /* XXX: this code should not depend on lock_user */
2827 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
2828 /* FIXME - should this return an error rather than just fail? */
2831 unlock_user(p
, addr
, l
);
2833 if (!(flags
& PAGE_READ
))
2835 /* XXX: this code should not depend on lock_user */
2836 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
2837 /* FIXME - should this return an error rather than just fail? */
2840 unlock_user(p
, addr
, 0);
2849 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
2850 int len
, int is_write
)
2855 target_phys_addr_t page
;
2860 page
= addr
& TARGET_PAGE_MASK
;
2861 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2864 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
2866 pd
= IO_MEM_UNASSIGNED
;
2868 pd
= p
->phys_offset
;
2872 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
2873 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
2874 /* XXX: could force cpu_single_env to NULL to avoid
2876 if (l
>= 4 && ((addr
& 3) == 0)) {
2877 /* 32 bit write access */
2879 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
2881 } else if (l
>= 2 && ((addr
& 1) == 0)) {
2882 /* 16 bit write access */
2884 io_mem_write
[io_index
][1](io_mem_opaque
[io_index
], addr
, val
);
2887 /* 8 bit write access */
2889 io_mem_write
[io_index
][0](io_mem_opaque
[io_index
], addr
, val
);
2893 unsigned long addr1
;
2894 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
2896 ptr
= phys_ram_base
+ addr1
;
2897 memcpy(ptr
, buf
, l
);
2898 if (!cpu_physical_memory_is_dirty(addr1
)) {
2899 /* invalidate code */
2900 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
2902 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
2903 (0xff & ~CODE_DIRTY_FLAG
);
2907 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
2908 !(pd
& IO_MEM_ROMD
)) {
2910 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
2911 if (l
>= 4 && ((addr
& 3) == 0)) {
2912 /* 32 bit read access */
2913 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
2916 } else if (l
>= 2 && ((addr
& 1) == 0)) {
2917 /* 16 bit read access */
2918 val
= io_mem_read
[io_index
][1](io_mem_opaque
[io_index
], addr
);
2922 /* 8 bit read access */
2923 val
= io_mem_read
[io_index
][0](io_mem_opaque
[io_index
], addr
);
2929 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
2930 (addr
& ~TARGET_PAGE_MASK
);
2931 memcpy(buf
, ptr
, l
);
2940 /* used for ROM loading : can write in RAM and ROM */
2941 void cpu_physical_memory_write_rom(target_phys_addr_t addr
,
2942 const uint8_t *buf
, int len
)
2946 target_phys_addr_t page
;
2951 page
= addr
& TARGET_PAGE_MASK
;
2952 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2955 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
2957 pd
= IO_MEM_UNASSIGNED
;
2959 pd
= p
->phys_offset
;
2962 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
&&
2963 (pd
& ~TARGET_PAGE_MASK
) != IO_MEM_ROM
&&
2964 !(pd
& IO_MEM_ROMD
)) {
2967 unsigned long addr1
;
2968 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
2970 ptr
= phys_ram_base
+ addr1
;
2971 memcpy(ptr
, buf
, l
);
2980 /* warning: addr must be aligned */
2981 uint32_t ldl_phys(target_phys_addr_t addr
)
2989 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2991 pd
= IO_MEM_UNASSIGNED
;
2993 pd
= p
->phys_offset
;
2996 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
2997 !(pd
& IO_MEM_ROMD
)) {
2999 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3000 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
3003 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
3004 (addr
& ~TARGET_PAGE_MASK
);
3010 /* warning: addr must be aligned */
3011 uint64_t ldq_phys(target_phys_addr_t addr
)
3019 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3021 pd
= IO_MEM_UNASSIGNED
;
3023 pd
= p
->phys_offset
;
3026 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3027 !(pd
& IO_MEM_ROMD
)) {
3029 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3030 #ifdef TARGET_WORDS_BIGENDIAN
3031 val
= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
) << 32;
3032 val
|= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4);
3034 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
3035 val
|= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4) << 32;
3039 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
3040 (addr
& ~TARGET_PAGE_MASK
);
3047 uint32_t ldub_phys(target_phys_addr_t addr
)
3050 cpu_physical_memory_read(addr
, &val
, 1);
3055 uint32_t lduw_phys(target_phys_addr_t addr
)
3058 cpu_physical_memory_read(addr
, (uint8_t *)&val
, 2);
3059 return tswap16(val
);
3062 /* warning: addr must be aligned. The ram page is not masked as dirty
3063 and the code inside is not invalidated. It is useful if the dirty
3064 bits are used to track modified PTEs */
3065 void stl_phys_notdirty(target_phys_addr_t addr
, uint32_t val
)
3072 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3074 pd
= IO_MEM_UNASSIGNED
;
3076 pd
= p
->phys_offset
;
3079 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3080 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3081 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3083 unsigned long addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3084 ptr
= phys_ram_base
+ addr1
;
3087 if (unlikely(in_migration
)) {
3088 if (!cpu_physical_memory_is_dirty(addr1
)) {
3089 /* invalidate code */
3090 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3092 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3093 (0xff & ~CODE_DIRTY_FLAG
);
3099 void stq_phys_notdirty(target_phys_addr_t addr
, uint64_t val
)
3106 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3108 pd
= IO_MEM_UNASSIGNED
;
3110 pd
= p
->phys_offset
;
3113 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3114 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3115 #ifdef TARGET_WORDS_BIGENDIAN
3116 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
>> 32);
3117 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
);
3119 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3120 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
>> 32);
3123 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
3124 (addr
& ~TARGET_PAGE_MASK
);
3129 /* warning: addr must be aligned */
3130 void stl_phys(target_phys_addr_t addr
, uint32_t val
)
3137 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3139 pd
= IO_MEM_UNASSIGNED
;
3141 pd
= p
->phys_offset
;
3144 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3145 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3146 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3148 unsigned long addr1
;
3149 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3151 ptr
= phys_ram_base
+ addr1
;
3153 if (!cpu_physical_memory_is_dirty(addr1
)) {
3154 /* invalidate code */
3155 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3157 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3158 (0xff & ~CODE_DIRTY_FLAG
);
3164 void stb_phys(target_phys_addr_t addr
, uint32_t val
)
3167 cpu_physical_memory_write(addr
, &v
, 1);
3171 void stw_phys(target_phys_addr_t addr
, uint32_t val
)
3173 uint16_t v
= tswap16(val
);
3174 cpu_physical_memory_write(addr
, (const uint8_t *)&v
, 2);
3178 void stq_phys(target_phys_addr_t addr
, uint64_t val
)
3181 cpu_physical_memory_write(addr
, (const uint8_t *)&val
, 8);
3186 /* virtual memory access for debug */
3187 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
3188 uint8_t *buf
, int len
, int is_write
)
3191 target_phys_addr_t phys_addr
;
3195 page
= addr
& TARGET_PAGE_MASK
;
3196 phys_addr
= cpu_get_phys_page_debug(env
, page
);
3197 /* if no physical page mapped, return an error */
3198 if (phys_addr
== -1)
3200 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3203 cpu_physical_memory_rw(phys_addr
+ (addr
& ~TARGET_PAGE_MASK
),
3212 /* in deterministic execution mode, instructions doing device I/Os
3213 must be at the end of the TB */
3214 void cpu_io_recompile(CPUState
*env
, void *retaddr
)
3216 TranslationBlock
*tb
;
3218 target_ulong pc
, cs_base
;
3221 tb
= tb_find_pc((unsigned long)retaddr
);
3223 cpu_abort(env
, "cpu_io_recompile: could not find TB for pc=%p",
3226 n
= env
->icount_decr
.u16
.low
+ tb
->icount
;
3227 cpu_restore_state(tb
, env
, (unsigned long)retaddr
, NULL
);
3228 /* Calculate how many instructions had been executed before the fault
3230 n
= n
- env
->icount_decr
.u16
.low
;
3231 /* Generate a new TB ending on the I/O insn. */
3233 /* On MIPS and SH, delay slot instructions can only be restarted if
3234 they were already the first instruction in the TB. If this is not
3235 the first instruction in a TB then re-execute the preceding
3237 #if defined(TARGET_MIPS)
3238 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
3239 env
->active_tc
.PC
-= 4;
3240 env
->icount_decr
.u16
.low
++;
3241 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
3243 #elif defined(TARGET_SH4)
3244 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
3247 env
->icount_decr
.u16
.low
++;
3248 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
3251 /* This should never happen. */
3252 if (n
> CF_COUNT_MASK
)
3253 cpu_abort(env
, "TB too big during recompile");
3255 cflags
= n
| CF_LAST_IO
;
3257 cs_base
= tb
->cs_base
;
3259 tb_phys_invalidate(tb
, -1);
3260 /* FIXME: In theory this could raise an exception. In practice
3261 we have already translated the block once so it's probably ok. */
3262 tb_gen_code(env
, pc
, cs_base
, flags
, cflags
);
3263 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3264 the first in the TB) then we end up generating a whole new TB and
3265 repeating the fault, which is horribly inefficient.
3266 Better would be to execute just this insn uncached, or generate a
3268 cpu_resume_from_signal(env
, NULL
);
3271 void dump_exec_info(FILE *f
,
3272 int (*cpu_fprintf
)(FILE *f
, const char *fmt
, ...))
3274 int i
, target_code_size
, max_target_code_size
;
3275 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
3276 TranslationBlock
*tb
;
3278 target_code_size
= 0;
3279 max_target_code_size
= 0;
3281 direct_jmp_count
= 0;
3282 direct_jmp2_count
= 0;
3283 for(i
= 0; i
< nb_tbs
; i
++) {
3285 target_code_size
+= tb
->size
;
3286 if (tb
->size
> max_target_code_size
)
3287 max_target_code_size
= tb
->size
;
3288 if (tb
->page_addr
[1] != -1)
3290 if (tb
->tb_next_offset
[0] != 0xffff) {
3292 if (tb
->tb_next_offset
[1] != 0xffff) {
3293 direct_jmp2_count
++;
3297 /* XXX: avoid using doubles ? */
3298 cpu_fprintf(f
, "Translation buffer state:\n");
3299 cpu_fprintf(f
, "gen code size %ld/%ld\n",
3300 code_gen_ptr
- code_gen_buffer
, code_gen_buffer_max_size
);
3301 cpu_fprintf(f
, "TB count %d/%d\n",
3302 nb_tbs
, code_gen_max_blocks
);
3303 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
3304 nb_tbs
? target_code_size
/ nb_tbs
: 0,
3305 max_target_code_size
);
3306 cpu_fprintf(f
, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3307 nb_tbs
? (code_gen_ptr
- code_gen_buffer
) / nb_tbs
: 0,
3308 target_code_size
? (double) (code_gen_ptr
- code_gen_buffer
) / target_code_size
: 0);
3309 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n",
3311 nb_tbs
? (cross_page
* 100) / nb_tbs
: 0);
3312 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3314 nb_tbs
? (direct_jmp_count
* 100) / nb_tbs
: 0,
3316 nb_tbs
? (direct_jmp2_count
* 100) / nb_tbs
: 0);
3317 cpu_fprintf(f
, "\nStatistics:\n");
3318 cpu_fprintf(f
, "TB flush count %d\n", tb_flush_count
);
3319 cpu_fprintf(f
, "TB invalidate count %d\n", tb_phys_invalidate_count
);
3320 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
3321 tcg_dump_info(f
, cpu_fprintf
);
3324 #if !defined(CONFIG_USER_ONLY)
3326 #define MMUSUFFIX _cmmu
3327 #define GETPC() NULL
3328 #define env cpu_single_env
3329 #define SOFTMMU_CODE_ACCESS
3332 #include "softmmu_template.h"
3335 #include "softmmu_template.h"
3338 #include "softmmu_template.h"
3341 #include "softmmu_template.h"