4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
22 #include "qemu/osdep.h"
25 #include "qemu-common.h"
26 #define NO_CPU_IO_DEFS
29 #include "disas/disas.h"
30 #include "exec/exec-all.h"
32 #if defined(CONFIG_USER_ONLY)
34 #include "exec/exec-all.h"
35 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
36 #include <sys/param.h>
37 #if __FreeBSD_version >= 700104
38 #define HAVE_KINFO_GETVMMAP
39 #define sigqueue sigqueue_freebsd /* avoid redefinition */
41 #include <machine/profile.h>
50 #include "exec/address-spaces.h"
53 #include "exec/cputlb.h"
54 #include "exec/tb-hash.h"
55 #include "translate-all.h"
56 #include "qemu/bitmap.h"
57 #include "qemu/timer.h"
60 /* #define DEBUG_TB_INVALIDATE */
61 /* #define DEBUG_TB_FLUSH */
62 /* #define DEBUG_LOCKING */
63 /* make various TB consistency checks */
64 /* #define DEBUG_TB_CHECK */
66 #if !defined(CONFIG_USER_ONLY)
67 /* TB consistency checks only implemented for usermode emulation. */
71 /* Access to the various translations structures need to be serialised via locks
72 * for consistency. This is automatic for SoftMMU based system
73 * emulation due to its single threaded nature. In user-mode emulation
74 * access to the memory related structures are protected with the
78 #define DEBUG_MEM_LOCKS 1
80 #define DEBUG_MEM_LOCKS 0
84 #define assert_memory_lock() do { /* nothing */ } while (0)
86 #define assert_memory_lock() do { \
87 if (DEBUG_MEM_LOCKS) { \
88 g_assert(have_mmap_lock()); \
93 #define SMC_BITMAP_USE_THRESHOLD 10
95 typedef struct PageDesc
{
96 /* list of TBs intersecting this ram page */
97 TranslationBlock
*first_tb
;
99 /* in order to optimize self modifying code, we count the number
100 of lookups we do to a given page to use a bitmap */
101 unsigned int code_write_count
;
102 unsigned long *code_bitmap
;
108 /* In system mode we want L1_MAP to be based on ram offsets,
109 while in user mode we want it to be based on virtual addresses. */
110 #if !defined(CONFIG_USER_ONLY)
111 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
112 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
114 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
117 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
120 /* Size of the L2 (and L3, etc) page tables. */
122 #define V_L2_SIZE (1 << V_L2_BITS)
124 uintptr_t qemu_host_page_size
;
125 intptr_t qemu_host_page_mask
;
128 * L1 Mapping properties
130 static int v_l1_size
;
131 static int v_l1_shift
;
132 static int v_l2_levels
;
134 /* The bottom level has pointers to PageDesc, and is indexed by
135 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size.
137 #define V_L1_MIN_BITS 4
138 #define V_L1_MAX_BITS (V_L2_BITS + 3)
139 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS)
141 static void *l1_map
[V_L1_MAX_SIZE
];
143 /* code generation context */
147 /* translation block context */
148 #ifdef CONFIG_USER_ONLY
149 __thread
int have_tb_lock
;
152 static void page_table_config_init(void)
156 assert(TARGET_PAGE_BITS
);
157 /* The bits remaining after N lower levels of page tables. */
158 v_l1_bits
= (L1_MAP_ADDR_SPACE_BITS
- TARGET_PAGE_BITS
) % V_L2_BITS
;
159 if (v_l1_bits
< V_L1_MIN_BITS
) {
160 v_l1_bits
+= V_L2_BITS
;
163 v_l1_size
= 1 << v_l1_bits
;
164 v_l1_shift
= L1_MAP_ADDR_SPACE_BITS
- TARGET_PAGE_BITS
- v_l1_bits
;
165 v_l2_levels
= v_l1_shift
/ V_L2_BITS
- 1;
167 assert(v_l1_bits
<= V_L1_MAX_BITS
);
168 assert(v_l1_shift
% V_L2_BITS
== 0);
169 assert(v_l2_levels
>= 0);
174 #ifdef CONFIG_USER_ONLY
175 assert(!have_tb_lock
);
176 qemu_mutex_lock(&tcg_ctx
.tb_ctx
.tb_lock
);
183 #ifdef CONFIG_USER_ONLY
184 assert(have_tb_lock
);
186 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
190 void tb_lock_reset(void)
192 #ifdef CONFIG_USER_ONLY
194 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
201 #define DEBUG_TB_LOCKS 1
203 #define DEBUG_TB_LOCKS 0
206 #ifdef CONFIG_SOFTMMU
207 #define assert_tb_lock() do { /* nothing */ } while (0)
209 #define assert_tb_lock() do { \
210 if (DEBUG_TB_LOCKS) { \
211 g_assert(have_tb_lock); \
217 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
);
219 void cpu_gen_init(void)
221 tcg_context_init(&tcg_ctx
);
224 /* Encode VAL as a signed leb128 sequence at P.
225 Return P incremented past the encoded value. */
226 static uint8_t *encode_sleb128(uint8_t *p
, target_long val
)
233 more
= !((val
== 0 && (byte
& 0x40) == 0)
234 || (val
== -1 && (byte
& 0x40) != 0));
244 /* Decode a signed leb128 sequence at *PP; increment *PP past the
245 decoded value. Return the decoded value. */
246 static target_long
decode_sleb128(uint8_t **pp
)
254 val
|= (target_ulong
)(byte
& 0x7f) << shift
;
256 } while (byte
& 0x80);
257 if (shift
< TARGET_LONG_BITS
&& (byte
& 0x40)) {
258 val
|= -(target_ulong
)1 << shift
;
265 /* Encode the data collected about the instructions while compiling TB.
266 Place the data at BLOCK, and return the number of bytes consumed.
268 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
269 which come from the target's insn_start data, followed by a uintptr_t
270 which comes from the host pc of the end of the code implementing the insn.
272 Each line of the table is encoded as sleb128 deltas from the previous
273 line. The seed for the first line is { tb->pc, 0..., tb->tc_ptr }.
274 That is, the first column is seeded with the guest pc, the last column
275 with the host pc, and the middle columns with zeros. */
277 static int encode_search(TranslationBlock
*tb
, uint8_t *block
)
279 uint8_t *highwater
= tcg_ctx
.code_gen_highwater
;
283 tb
->tc_search
= block
;
285 for (i
= 0, n
= tb
->icount
; i
< n
; ++i
) {
288 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
290 prev
= (j
== 0 ? tb
->pc
: 0);
292 prev
= tcg_ctx
.gen_insn_data
[i
- 1][j
];
294 p
= encode_sleb128(p
, tcg_ctx
.gen_insn_data
[i
][j
] - prev
);
296 prev
= (i
== 0 ? 0 : tcg_ctx
.gen_insn_end_off
[i
- 1]);
297 p
= encode_sleb128(p
, tcg_ctx
.gen_insn_end_off
[i
] - prev
);
299 /* Test for (pending) buffer overflow. The assumption is that any
300 one row beginning below the high water mark cannot overrun
301 the buffer completely. Thus we can test for overflow after
302 encoding a row without having to check during encoding. */
303 if (unlikely(p
> highwater
)) {
311 /* The cpu state corresponding to 'searched_pc' is restored. */
312 static int cpu_restore_state_from_tb(CPUState
*cpu
, TranslationBlock
*tb
,
313 uintptr_t searched_pc
)
315 target_ulong data
[TARGET_INSN_START_WORDS
] = { tb
->pc
};
316 uintptr_t host_pc
= (uintptr_t)tb
->tc_ptr
;
317 CPUArchState
*env
= cpu
->env_ptr
;
318 uint8_t *p
= tb
->tc_search
;
319 int i
, j
, num_insns
= tb
->icount
;
320 #ifdef CONFIG_PROFILER
321 int64_t ti
= profile_getclock();
324 searched_pc
-= GETPC_ADJ
;
326 if (searched_pc
< host_pc
) {
330 /* Reconstruct the stored insn data while looking for the point at
331 which the end of the insn exceeds the searched_pc. */
332 for (i
= 0; i
< num_insns
; ++i
) {
333 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
334 data
[j
] += decode_sleb128(&p
);
336 host_pc
+= decode_sleb128(&p
);
337 if (host_pc
> searched_pc
) {
344 if (tb
->cflags
& CF_USE_ICOUNT
) {
346 /* Reset the cycle counter to the start of the block. */
347 cpu
->icount_decr
.u16
.low
+= num_insns
;
348 /* Clear the IO flag. */
351 cpu
->icount_decr
.u16
.low
-= i
;
352 restore_state_to_opc(env
, tb
, data
);
354 #ifdef CONFIG_PROFILER
355 tcg_ctx
.restore_time
+= profile_getclock() - ti
;
356 tcg_ctx
.restore_count
++;
361 bool cpu_restore_state(CPUState
*cpu
, uintptr_t retaddr
)
363 TranslationBlock
*tb
;
365 tb
= tb_find_pc(retaddr
);
367 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
368 if (tb
->cflags
& CF_NOCACHE
) {
369 /* one-shot translation, invalidate it immediately */
370 tb_phys_invalidate(tb
, -1);
378 void page_size_init(void)
380 /* NOTE: we can always suppose that qemu_host_page_size >=
382 qemu_real_host_page_size
= getpagesize();
383 qemu_real_host_page_mask
= -(intptr_t)qemu_real_host_page_size
;
384 if (qemu_host_page_size
== 0) {
385 qemu_host_page_size
= qemu_real_host_page_size
;
387 if (qemu_host_page_size
< TARGET_PAGE_SIZE
) {
388 qemu_host_page_size
= TARGET_PAGE_SIZE
;
390 qemu_host_page_mask
= -(intptr_t)qemu_host_page_size
;
393 static void page_init(void)
396 page_table_config_init();
398 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
400 #ifdef HAVE_KINFO_GETVMMAP
401 struct kinfo_vmentry
*freep
;
404 freep
= kinfo_getvmmap(getpid(), &cnt
);
407 for (i
= 0; i
< cnt
; i
++) {
408 unsigned long startaddr
, endaddr
;
410 startaddr
= freep
[i
].kve_start
;
411 endaddr
= freep
[i
].kve_end
;
412 if (h2g_valid(startaddr
)) {
413 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
415 if (h2g_valid(endaddr
)) {
416 endaddr
= h2g(endaddr
);
417 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
419 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
421 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
432 last_brk
= (unsigned long)sbrk(0);
434 f
= fopen("/compat/linux/proc/self/maps", "r");
439 unsigned long startaddr
, endaddr
;
442 n
= fscanf(f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
444 if (n
== 2 && h2g_valid(startaddr
)) {
445 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
447 if (h2g_valid(endaddr
)) {
448 endaddr
= h2g(endaddr
);
452 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
465 * Called with mmap_lock held for user-mode emulation.
467 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
473 /* Level 1. Always allocated. */
474 lp
= l1_map
+ ((index
>> v_l1_shift
) & (v_l1_size
- 1));
477 for (i
= v_l2_levels
; i
> 0; i
--) {
478 void **p
= atomic_rcu_read(lp
);
484 p
= g_new0(void *, V_L2_SIZE
);
485 atomic_rcu_set(lp
, p
);
488 lp
= p
+ ((index
>> (i
* V_L2_BITS
)) & (V_L2_SIZE
- 1));
491 pd
= atomic_rcu_read(lp
);
496 pd
= g_new0(PageDesc
, V_L2_SIZE
);
497 atomic_rcu_set(lp
, pd
);
500 return pd
+ (index
& (V_L2_SIZE
- 1));
503 static inline PageDesc
*page_find(tb_page_addr_t index
)
505 return page_find_alloc(index
, 0);
508 #if defined(CONFIG_USER_ONLY)
509 /* Currently it is not recommended to allocate big chunks of data in
510 user mode. It will change when a dedicated libc will be used. */
511 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
512 region in which the guest needs to run. Revisit this. */
513 #define USE_STATIC_CODE_GEN_BUFFER
516 /* Minimum size of the code gen buffer. This number is randomly chosen,
517 but not so small that we can't have a fair number of TB's live. */
518 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
520 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
521 indicated, this is constrained by the range of direct branches on the
522 host cpu, as used by the TCG implementation of goto_tb. */
523 #if defined(__x86_64__)
524 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
525 #elif defined(__sparc__)
526 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
527 #elif defined(__powerpc64__)
528 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
529 #elif defined(__powerpc__)
530 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
531 #elif defined(__aarch64__)
532 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
533 #elif defined(__arm__)
534 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
535 #elif defined(__s390x__)
536 /* We have a +- 4GB range on the branches; leave some slop. */
537 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
538 #elif defined(__mips__)
539 /* We have a 256MB branch region, but leave room to make sure the
540 main executable is also within that region. */
541 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
543 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
546 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
548 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
549 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
550 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
552 static inline size_t size_code_gen_buffer(size_t tb_size
)
554 /* Size the buffer. */
556 #ifdef USE_STATIC_CODE_GEN_BUFFER
557 tb_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
559 /* ??? Needs adjustments. */
560 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
561 static buffer, we could size this on RESERVED_VA, on the text
562 segment size of the executable, or continue to use the default. */
563 tb_size
= (unsigned long)(ram_size
/ 4);
566 if (tb_size
< MIN_CODE_GEN_BUFFER_SIZE
) {
567 tb_size
= MIN_CODE_GEN_BUFFER_SIZE
;
569 if (tb_size
> MAX_CODE_GEN_BUFFER_SIZE
) {
570 tb_size
= MAX_CODE_GEN_BUFFER_SIZE
;
576 /* In order to use J and JAL within the code_gen_buffer, we require
577 that the buffer not cross a 256MB boundary. */
578 static inline bool cross_256mb(void *addr
, size_t size
)
580 return ((uintptr_t)addr
^ ((uintptr_t)addr
+ size
)) & ~0x0ffffffful
;
583 /* We weren't able to allocate a buffer without crossing that boundary,
584 so make do with the larger portion of the buffer that doesn't cross.
585 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
586 static inline void *split_cross_256mb(void *buf1
, size_t size1
)
588 void *buf2
= (void *)(((uintptr_t)buf1
+ size1
) & ~0x0ffffffful
);
589 size_t size2
= buf1
+ size1
- buf2
;
597 tcg_ctx
.code_gen_buffer_size
= size1
;
602 #ifdef USE_STATIC_CODE_GEN_BUFFER
603 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
604 __attribute__((aligned(CODE_GEN_ALIGN
)));
607 static inline void do_protect(void *addr
, long size
, int prot
)
610 VirtualProtect(addr
, size
, prot
, &old_protect
);
613 static inline void map_exec(void *addr
, long size
)
615 do_protect(addr
, size
, PAGE_EXECUTE_READWRITE
);
618 static inline void map_none(void *addr
, long size
)
620 do_protect(addr
, size
, PAGE_NOACCESS
);
623 static inline void do_protect(void *addr
, long size
, int prot
)
625 uintptr_t start
, end
;
627 start
= (uintptr_t)addr
;
628 start
&= qemu_real_host_page_mask
;
630 end
= (uintptr_t)addr
+ size
;
631 end
= ROUND_UP(end
, qemu_real_host_page_size
);
633 mprotect((void *)start
, end
- start
, prot
);
636 static inline void map_exec(void *addr
, long size
)
638 do_protect(addr
, size
, PROT_READ
| PROT_WRITE
| PROT_EXEC
);
641 static inline void map_none(void *addr
, long size
)
643 do_protect(addr
, size
, PROT_NONE
);
647 static inline void *alloc_code_gen_buffer(void)
649 void *buf
= static_code_gen_buffer
;
650 size_t full_size
, size
;
652 /* The size of the buffer, rounded down to end on a page boundary. */
653 full_size
= (((uintptr_t)buf
+ sizeof(static_code_gen_buffer
))
654 & qemu_real_host_page_mask
) - (uintptr_t)buf
;
656 /* Reserve a guard page. */
657 size
= full_size
- qemu_real_host_page_size
;
659 /* Honor a command-line option limiting the size of the buffer. */
660 if (size
> tcg_ctx
.code_gen_buffer_size
) {
661 size
= (((uintptr_t)buf
+ tcg_ctx
.code_gen_buffer_size
)
662 & qemu_real_host_page_mask
) - (uintptr_t)buf
;
664 tcg_ctx
.code_gen_buffer_size
= size
;
667 if (cross_256mb(buf
, size
)) {
668 buf
= split_cross_256mb(buf
, size
);
669 size
= tcg_ctx
.code_gen_buffer_size
;
674 map_none(buf
+ size
, qemu_real_host_page_size
);
675 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
679 #elif defined(_WIN32)
680 static inline void *alloc_code_gen_buffer(void)
682 size_t size
= tcg_ctx
.code_gen_buffer_size
;
685 /* Perform the allocation in two steps, so that the guard page
686 is reserved but uncommitted. */
687 buf1
= VirtualAlloc(NULL
, size
+ qemu_real_host_page_size
,
688 MEM_RESERVE
, PAGE_NOACCESS
);
690 buf2
= VirtualAlloc(buf1
, size
, MEM_COMMIT
, PAGE_EXECUTE_READWRITE
);
691 assert(buf1
== buf2
);
697 static inline void *alloc_code_gen_buffer(void)
699 int flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
701 size_t size
= tcg_ctx
.code_gen_buffer_size
;
704 /* Constrain the position of the buffer based on the host cpu.
705 Note that these addresses are chosen in concert with the
706 addresses assigned in the relevant linker script file. */
707 # if defined(__PIE__) || defined(__PIC__)
708 /* Don't bother setting a preferred location if we're building
709 a position-independent executable. We're more likely to get
710 an address near the main executable if we let the kernel
711 choose the address. */
712 # elif defined(__x86_64__) && defined(MAP_32BIT)
713 /* Force the memory down into low memory with the executable.
714 Leave the choice of exact location with the kernel. */
716 /* Cannot expect to map more than 800MB in low memory. */
717 if (size
> 800u * 1024 * 1024) {
718 tcg_ctx
.code_gen_buffer_size
= size
= 800u * 1024 * 1024;
720 # elif defined(__sparc__)
721 start
= 0x40000000ul
;
722 # elif defined(__s390x__)
723 start
= 0x90000000ul
;
724 # elif defined(__mips__)
725 # if _MIPS_SIM == _ABI64
726 start
= 0x128000000ul
;
728 start
= 0x08000000ul
;
732 buf
= mmap((void *)start
, size
+ qemu_real_host_page_size
,
733 PROT_NONE
, flags
, -1, 0);
734 if (buf
== MAP_FAILED
) {
739 if (cross_256mb(buf
, size
)) {
740 /* Try again, with the original still mapped, to avoid re-acquiring
741 that 256mb crossing. This time don't specify an address. */
743 void *buf2
= mmap(NULL
, size
+ qemu_real_host_page_size
,
744 PROT_NONE
, flags
, -1, 0);
745 switch (buf2
!= MAP_FAILED
) {
747 if (!cross_256mb(buf2
, size
)) {
748 /* Success! Use the new buffer. */
749 munmap(buf
, size
+ qemu_real_host_page_size
);
752 /* Failure. Work with what we had. */
753 munmap(buf2
, size
+ qemu_real_host_page_size
);
756 /* Split the original buffer. Free the smaller half. */
757 buf2
= split_cross_256mb(buf
, size
);
758 size2
= tcg_ctx
.code_gen_buffer_size
;
760 munmap(buf
+ size2
+ qemu_real_host_page_size
, size
- size2
);
762 munmap(buf
, size
- size2
);
771 /* Make the final buffer accessible. The guard page at the end
772 will remain inaccessible with PROT_NONE. */
773 mprotect(buf
, size
, PROT_WRITE
| PROT_READ
| PROT_EXEC
);
775 /* Request large pages for the buffer. */
776 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
780 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
782 static inline void code_gen_alloc(size_t tb_size
)
784 tcg_ctx
.code_gen_buffer_size
= size_code_gen_buffer(tb_size
);
785 tcg_ctx
.code_gen_buffer
= alloc_code_gen_buffer();
786 if (tcg_ctx
.code_gen_buffer
== NULL
) {
787 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
791 /* Estimate a good size for the number of TBs we can support. We
792 still haven't deducted the prologue from the buffer size here,
793 but that's minimal and won't affect the estimate much. */
794 tcg_ctx
.code_gen_max_blocks
795 = tcg_ctx
.code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
796 tcg_ctx
.tb_ctx
.tbs
= g_new(TranslationBlock
, tcg_ctx
.code_gen_max_blocks
);
798 qemu_mutex_init(&tcg_ctx
.tb_ctx
.tb_lock
);
801 static void tb_htable_init(void)
803 unsigned int mode
= QHT_MODE_AUTO_RESIZE
;
805 qht_init(&tcg_ctx
.tb_ctx
.htable
, CODE_GEN_HTABLE_SIZE
, mode
);
808 /* Must be called before using the QEMU cpus. 'tb_size' is the size
809 (in bytes) allocated to the translation buffer. Zero means default
811 void tcg_exec_init(unsigned long tb_size
)
816 code_gen_alloc(tb_size
);
817 #if defined(CONFIG_SOFTMMU)
818 /* There's no guest base to take into account, so go ahead and
819 initialize the prologue now. */
820 tcg_prologue_init(&tcg_ctx
);
824 bool tcg_enabled(void)
826 return tcg_ctx
.code_gen_buffer
!= NULL
;
829 /* Allocate a new translation block. Flush the translation buffer if
830 too many translation blocks or too much generated code. */
831 static TranslationBlock
*tb_alloc(target_ulong pc
)
833 TranslationBlock
*tb
;
835 if (tcg_ctx
.tb_ctx
.nb_tbs
>= tcg_ctx
.code_gen_max_blocks
) {
838 tb
= &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
++];
845 void tb_free(TranslationBlock
*tb
)
847 /* In practice this is mostly used for single use temporary TB
848 Ignore the hard cases and just back up if this TB happens to
849 be the last one generated. */
850 if (tcg_ctx
.tb_ctx
.nb_tbs
> 0 &&
851 tb
== &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
- 1]) {
852 tcg_ctx
.code_gen_ptr
= tb
->tc_ptr
;
853 tcg_ctx
.tb_ctx
.nb_tbs
--;
857 static inline void invalidate_page_bitmap(PageDesc
*p
)
859 #ifdef CONFIG_SOFTMMU
860 g_free(p
->code_bitmap
);
861 p
->code_bitmap
= NULL
;
862 p
->code_write_count
= 0;
866 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
867 static void page_flush_tb_1(int level
, void **lp
)
877 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
878 pd
[i
].first_tb
= NULL
;
879 invalidate_page_bitmap(pd
+ i
);
884 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
885 page_flush_tb_1(level
- 1, pp
+ i
);
890 static void page_flush_tb(void)
892 int i
, l1_sz
= v_l1_size
;
894 for (i
= 0; i
< l1_sz
; i
++) {
895 page_flush_tb_1(v_l2_levels
, l1_map
+ i
);
899 /* flush all the translation blocks */
900 static void do_tb_flush(CPUState
*cpu
, void *data
)
902 unsigned tb_flush_req
= (unsigned) (uintptr_t) data
;
906 /* If it's already been done on request of another CPU,
909 if (tcg_ctx
.tb_ctx
.tb_flush_count
!= tb_flush_req
) {
913 #if defined(DEBUG_TB_FLUSH)
914 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
915 (unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
),
916 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.tb_ctx
.nb_tbs
> 0 ?
917 ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)) /
918 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
920 if ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)
921 > tcg_ctx
.code_gen_buffer_size
) {
922 cpu_abort(cpu
, "Internal error: code buffer overflow\n");
928 for (i
= 0; i
< TB_JMP_CACHE_SIZE
; ++i
) {
929 atomic_set(&cpu
->tb_jmp_cache
[i
], NULL
);
933 tcg_ctx
.tb_ctx
.nb_tbs
= 0;
934 qht_reset_size(&tcg_ctx
.tb_ctx
.htable
, CODE_GEN_HTABLE_SIZE
);
937 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
938 /* XXX: flush processor icache at this point if cache flush is
940 atomic_mb_set(&tcg_ctx
.tb_ctx
.tb_flush_count
,
941 tcg_ctx
.tb_ctx
.tb_flush_count
+ 1);
947 void tb_flush(CPUState
*cpu
)
950 uintptr_t tb_flush_req
= atomic_mb_read(&tcg_ctx
.tb_ctx
.tb_flush_count
);
951 async_safe_run_on_cpu(cpu
, do_tb_flush
, (void *) tb_flush_req
);
955 #ifdef DEBUG_TB_CHECK
958 do_tb_invalidate_check(struct qht
*ht
, void *p
, uint32_t hash
, void *userp
)
960 TranslationBlock
*tb
= p
;
961 target_ulong addr
= *(target_ulong
*)userp
;
963 if (!(addr
+ TARGET_PAGE_SIZE
<= tb
->pc
|| addr
>= tb
->pc
+ tb
->size
)) {
964 printf("ERROR invalidate: address=" TARGET_FMT_lx
965 " PC=%08lx size=%04x\n", addr
, (long)tb
->pc
, tb
->size
);
969 static void tb_invalidate_check(target_ulong address
)
971 address
&= TARGET_PAGE_MASK
;
972 qht_iter(&tcg_ctx
.tb_ctx
.htable
, do_tb_invalidate_check
, &address
);
976 do_tb_page_check(struct qht
*ht
, void *p
, uint32_t hash
, void *userp
)
978 TranslationBlock
*tb
= p
;
981 flags1
= page_get_flags(tb
->pc
);
982 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
983 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
984 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
985 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
989 /* verify that all the pages have correct rights for code */
990 static void tb_page_check(void)
992 qht_iter(&tcg_ctx
.tb_ctx
.htable
, do_tb_page_check
, NULL
);
997 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
999 TranslationBlock
*tb1
;
1004 n1
= (uintptr_t)tb1
& 3;
1005 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
1007 *ptb
= tb1
->page_next
[n1
];
1010 ptb
= &tb1
->page_next
[n1
];
1014 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */
1015 static inline void tb_remove_from_jmp_list(TranslationBlock
*tb
, int n
)
1017 TranslationBlock
*tb1
;
1018 uintptr_t *ptb
, ntb
;
1021 ptb
= &tb
->jmp_list_next
[n
];
1023 /* find tb(n) in circular list */
1027 tb1
= (TranslationBlock
*)(ntb
& ~3);
1028 if (n1
== n
&& tb1
== tb
) {
1032 ptb
= &tb1
->jmp_list_first
;
1034 ptb
= &tb1
->jmp_list_next
[n1
];
1037 /* now we can suppress tb(n) from the list */
1038 *ptb
= tb
->jmp_list_next
[n
];
1040 tb
->jmp_list_next
[n
] = (uintptr_t)NULL
;
1044 /* reset the jump entry 'n' of a TB so that it is not chained to
1046 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
1048 uintptr_t addr
= (uintptr_t)(tb
->tc_ptr
+ tb
->jmp_reset_offset
[n
]);
1049 tb_set_jmp_target(tb
, n
, addr
);
1052 /* remove any jumps to the TB */
1053 static inline void tb_jmp_unlink(TranslationBlock
*tb
)
1055 TranslationBlock
*tb1
;
1056 uintptr_t *ptb
, ntb
;
1059 ptb
= &tb
->jmp_list_first
;
1063 tb1
= (TranslationBlock
*)(ntb
& ~3);
1067 tb_reset_jump(tb1
, n1
);
1068 *ptb
= tb1
->jmp_list_next
[n1
];
1069 tb1
->jmp_list_next
[n1
] = (uintptr_t)NULL
;
1073 /* invalidate one TB */
1074 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
1079 tb_page_addr_t phys_pc
;
1081 atomic_set(&tb
->invalid
, true);
1083 /* remove the TB from the hash list */
1084 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1085 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
);
1086 qht_remove(&tcg_ctx
.tb_ctx
.htable
, tb
, h
);
1088 /* remove the TB from the page list */
1089 if (tb
->page_addr
[0] != page_addr
) {
1090 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
1091 tb_page_remove(&p
->first_tb
, tb
);
1092 invalidate_page_bitmap(p
);
1094 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
1095 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
1096 tb_page_remove(&p
->first_tb
, tb
);
1097 invalidate_page_bitmap(p
);
1100 /* remove the TB from the hash list */
1101 h
= tb_jmp_cache_hash_func(tb
->pc
);
1103 if (atomic_read(&cpu
->tb_jmp_cache
[h
]) == tb
) {
1104 atomic_set(&cpu
->tb_jmp_cache
[h
], NULL
);
1108 /* suppress this TB from the two jump lists */
1109 tb_remove_from_jmp_list(tb
, 0);
1110 tb_remove_from_jmp_list(tb
, 1);
1112 /* suppress any remaining jumps to this TB */
1115 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
++;
1118 #ifdef CONFIG_SOFTMMU
1119 static void build_page_bitmap(PageDesc
*p
)
1121 int n
, tb_start
, tb_end
;
1122 TranslationBlock
*tb
;
1124 p
->code_bitmap
= bitmap_new(TARGET_PAGE_SIZE
);
1127 while (tb
!= NULL
) {
1128 n
= (uintptr_t)tb
& 3;
1129 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1130 /* NOTE: this is subtle as a TB may span two physical pages */
1132 /* NOTE: tb_end may be after the end of the page, but
1133 it is not a problem */
1134 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
1135 tb_end
= tb_start
+ tb
->size
;
1136 if (tb_end
> TARGET_PAGE_SIZE
) {
1137 tb_end
= TARGET_PAGE_SIZE
;
1141 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1143 bitmap_set(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
1144 tb
= tb
->page_next
[n
];
1149 /* add the tb in the target page and protect it if necessary
1151 * Called with mmap_lock held for user-mode emulation.
1153 static inline void tb_alloc_page(TranslationBlock
*tb
,
1154 unsigned int n
, tb_page_addr_t page_addr
)
1157 #ifndef CONFIG_USER_ONLY
1158 bool page_already_protected
;
1161 tb
->page_addr
[n
] = page_addr
;
1162 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1163 tb
->page_next
[n
] = p
->first_tb
;
1164 #ifndef CONFIG_USER_ONLY
1165 page_already_protected
= p
->first_tb
!= NULL
;
1167 p
->first_tb
= (TranslationBlock
*)((uintptr_t)tb
| n
);
1168 invalidate_page_bitmap(p
);
1170 #if defined(CONFIG_USER_ONLY)
1171 if (p
->flags
& PAGE_WRITE
) {
1176 /* force the host page as non writable (writes will have a
1177 page fault + mprotect overhead) */
1178 page_addr
&= qemu_host_page_mask
;
1180 for (addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1181 addr
+= TARGET_PAGE_SIZE
) {
1183 p2
= page_find(addr
>> TARGET_PAGE_BITS
);
1188 p2
->flags
&= ~PAGE_WRITE
;
1190 mprotect(g2h(page_addr
), qemu_host_page_size
,
1191 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1192 #ifdef DEBUG_TB_INVALIDATE
1193 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1198 /* if some code is already present, then the pages are already
1199 protected. So we handle the case where only the first TB is
1200 allocated in a physical page */
1201 if (!page_already_protected
) {
1202 tlb_protect_code(page_addr
);
1207 /* add a new TB and link it to the physical page tables. phys_page2 is
1208 * (-1) to indicate that only one page contains the TB.
1210 * Called with mmap_lock held for user-mode emulation.
1212 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
1213 tb_page_addr_t phys_page2
)
1217 /* add in the page list */
1218 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1219 if (phys_page2
!= -1) {
1220 tb_alloc_page(tb
, 1, phys_page2
);
1222 tb
->page_addr
[1] = -1;
1225 /* add in the hash table */
1226 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
);
1227 qht_insert(&tcg_ctx
.tb_ctx
.htable
, tb
, h
);
1229 #ifdef DEBUG_TB_CHECK
1234 /* Called with mmap_lock held for user mode emulation. */
1235 TranslationBlock
*tb_gen_code(CPUState
*cpu
,
1236 target_ulong pc
, target_ulong cs_base
,
1237 uint32_t flags
, int cflags
)
1239 CPUArchState
*env
= cpu
->env_ptr
;
1240 TranslationBlock
*tb
;
1241 tb_page_addr_t phys_pc
, phys_page2
;
1242 target_ulong virt_page2
;
1243 tcg_insn_unit
*gen_code_buf
;
1244 int gen_code_size
, search_size
;
1245 #ifdef CONFIG_PROFILER
1249 phys_pc
= get_page_addr_code(env
, pc
);
1250 if (use_icount
&& !(cflags
& CF_IGNORE_ICOUNT
)) {
1251 cflags
|= CF_USE_ICOUNT
;
1255 if (unlikely(!tb
)) {
1257 /* flush must be done */
1263 gen_code_buf
= tcg_ctx
.code_gen_ptr
;
1264 tb
->tc_ptr
= gen_code_buf
;
1265 tb
->cs_base
= cs_base
;
1267 tb
->cflags
= cflags
;
1269 #ifdef CONFIG_PROFILER
1270 tcg_ctx
.tb_count1
++; /* includes aborted translations because of
1272 ti
= profile_getclock();
1275 tcg_func_start(&tcg_ctx
);
1277 tcg_ctx
.cpu
= ENV_GET_CPU(env
);
1278 gen_intermediate_code(env
, tb
);
1281 trace_translate_block(tb
, tb
->pc
, tb
->tc_ptr
);
1283 /* generate machine code */
1284 tb
->jmp_reset_offset
[0] = TB_JMP_RESET_OFFSET_INVALID
;
1285 tb
->jmp_reset_offset
[1] = TB_JMP_RESET_OFFSET_INVALID
;
1286 tcg_ctx
.tb_jmp_reset_offset
= tb
->jmp_reset_offset
;
1287 #ifdef USE_DIRECT_JUMP
1288 tcg_ctx
.tb_jmp_insn_offset
= tb
->jmp_insn_offset
;
1289 tcg_ctx
.tb_jmp_target_addr
= NULL
;
1291 tcg_ctx
.tb_jmp_insn_offset
= NULL
;
1292 tcg_ctx
.tb_jmp_target_addr
= tb
->jmp_target_addr
;
1295 #ifdef CONFIG_PROFILER
1297 tcg_ctx
.interm_time
+= profile_getclock() - ti
;
1298 tcg_ctx
.code_time
-= profile_getclock();
1301 /* ??? Overflow could be handled better here. In particular, we
1302 don't need to re-do gen_intermediate_code, nor should we re-do
1303 the tcg optimization currently hidden inside tcg_gen_code. All
1304 that should be required is to flush the TBs, allocate a new TB,
1305 re-initialize it per above, and re-do the actual code generation. */
1306 gen_code_size
= tcg_gen_code(&tcg_ctx
, tb
);
1307 if (unlikely(gen_code_size
< 0)) {
1308 goto buffer_overflow
;
1310 search_size
= encode_search(tb
, (void *)gen_code_buf
+ gen_code_size
);
1311 if (unlikely(search_size
< 0)) {
1312 goto buffer_overflow
;
1315 #ifdef CONFIG_PROFILER
1316 tcg_ctx
.code_time
+= profile_getclock();
1317 tcg_ctx
.code_in_len
+= tb
->size
;
1318 tcg_ctx
.code_out_len
+= gen_code_size
;
1319 tcg_ctx
.search_out_len
+= search_size
;
1323 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
) &&
1324 qemu_log_in_addr_range(tb
->pc
)) {
1325 qemu_log("OUT: [size=%d]\n", gen_code_size
);
1326 log_disas(tb
->tc_ptr
, gen_code_size
);
1332 tcg_ctx
.code_gen_ptr
= (void *)
1333 ROUND_UP((uintptr_t)gen_code_buf
+ gen_code_size
+ search_size
,
1336 /* init jump list */
1337 assert(((uintptr_t)tb
& 3) == 0);
1338 tb
->jmp_list_first
= (uintptr_t)tb
| 2;
1339 tb
->jmp_list_next
[0] = (uintptr_t)NULL
;
1340 tb
->jmp_list_next
[1] = (uintptr_t)NULL
;
1342 /* init original jump addresses wich has been set during tcg_gen_code() */
1343 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1344 tb_reset_jump(tb
, 0);
1346 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1347 tb_reset_jump(tb
, 1);
1350 /* check next page if needed */
1351 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1353 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1354 phys_page2
= get_page_addr_code(env
, virt_page2
);
1356 /* As long as consistency of the TB stuff is provided by tb_lock in user
1357 * mode and is implicit in single-threaded softmmu emulation, no explicit
1358 * memory barrier is required before tb_link_page() makes the TB visible
1359 * through the physical hash table and physical page list.
1361 tb_link_page(tb
, phys_pc
, phys_page2
);
1366 * Invalidate all TBs which intersect with the target physical address range
1367 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1368 * 'is_cpu_write_access' should be true if called from a real cpu write
1369 * access: the virtual CPU will exit the current TB if code is modified inside
1372 * Called with mmap_lock held for user-mode emulation
1374 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
)
1376 while (start
< end
) {
1377 tb_invalidate_phys_page_range(start
, end
, 0);
1378 start
&= TARGET_PAGE_MASK
;
1379 start
+= TARGET_PAGE_SIZE
;
1384 * Invalidate all TBs which intersect with the target physical address range
1385 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1386 * 'is_cpu_write_access' should be true if called from a real cpu write
1387 * access: the virtual CPU will exit the current TB if code is modified inside
1390 * Called with mmap_lock held for user-mode emulation
1392 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1393 int is_cpu_write_access
)
1395 TranslationBlock
*tb
, *tb_next
;
1396 #if defined(TARGET_HAS_PRECISE_SMC)
1397 CPUState
*cpu
= current_cpu
;
1398 CPUArchState
*env
= NULL
;
1400 tb_page_addr_t tb_start
, tb_end
;
1403 #ifdef TARGET_HAS_PRECISE_SMC
1404 int current_tb_not_found
= is_cpu_write_access
;
1405 TranslationBlock
*current_tb
= NULL
;
1406 int current_tb_modified
= 0;
1407 target_ulong current_pc
= 0;
1408 target_ulong current_cs_base
= 0;
1409 uint32_t current_flags
= 0;
1410 #endif /* TARGET_HAS_PRECISE_SMC */
1412 p
= page_find(start
>> TARGET_PAGE_BITS
);
1416 #if defined(TARGET_HAS_PRECISE_SMC)
1422 /* we remove all the TBs in the range [start, end[ */
1423 /* XXX: see if in some cases it could be faster to invalidate all
1426 while (tb
!= NULL
) {
1427 n
= (uintptr_t)tb
& 3;
1428 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1429 tb_next
= tb
->page_next
[n
];
1430 /* NOTE: this is subtle as a TB may span two physical pages */
1432 /* NOTE: tb_end may be after the end of the page, but
1433 it is not a problem */
1434 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1435 tb_end
= tb_start
+ tb
->size
;
1437 tb_start
= tb
->page_addr
[1];
1438 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1440 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1441 #ifdef TARGET_HAS_PRECISE_SMC
1442 if (current_tb_not_found
) {
1443 current_tb_not_found
= 0;
1445 if (cpu
->mem_io_pc
) {
1446 /* now we have a real cpu fault */
1447 current_tb
= tb_find_pc(cpu
->mem_io_pc
);
1450 if (current_tb
== tb
&&
1451 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1452 /* If we are modifying the current TB, we must stop
1453 its execution. We could be more precise by checking
1454 that the modification is after the current PC, but it
1455 would require a specialized function to partially
1456 restore the CPU state */
1458 current_tb_modified
= 1;
1459 cpu_restore_state_from_tb(cpu
, current_tb
, cpu
->mem_io_pc
);
1460 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1463 #endif /* TARGET_HAS_PRECISE_SMC */
1464 tb_phys_invalidate(tb
, -1);
1468 #if !defined(CONFIG_USER_ONLY)
1469 /* if no code remaining, no need to continue to use slow writes */
1471 invalidate_page_bitmap(p
);
1472 tlb_unprotect_code(start
);
1475 #ifdef TARGET_HAS_PRECISE_SMC
1476 if (current_tb_modified
) {
1477 /* we generate a block containing just the instruction
1478 modifying the memory. It will ensure that it cannot modify
1480 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1481 cpu_loop_exit_noexc(cpu
);
1486 #ifdef CONFIG_SOFTMMU
1487 /* len must be <= 8 and start must be a multiple of len */
1488 void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1494 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1495 cpu_single_env
->mem_io_vaddr
, len
,
1496 cpu_single_env
->eip
,
1497 cpu_single_env
->eip
+
1498 (intptr_t)cpu_single_env
->segs
[R_CS
].base
);
1501 p
= page_find(start
>> TARGET_PAGE_BITS
);
1505 if (!p
->code_bitmap
&&
1506 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
) {
1507 /* build code bitmap */
1508 build_page_bitmap(p
);
1510 if (p
->code_bitmap
) {
1514 nr
= start
& ~TARGET_PAGE_MASK
;
1515 b
= p
->code_bitmap
[BIT_WORD(nr
)] >> (nr
& (BITS_PER_LONG
- 1));
1516 if (b
& ((1 << len
) - 1)) {
1521 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1525 /* Called with mmap_lock held. If pc is not 0 then it indicates the
1526 * host PC of the faulting store instruction that caused this invalidate.
1527 * Returns true if the caller needs to abort execution of the current
1528 * TB (because it was modified by this store and the guest CPU has
1529 * precise-SMC semantics).
1531 static bool tb_invalidate_phys_page(tb_page_addr_t addr
, uintptr_t pc
)
1533 TranslationBlock
*tb
;
1536 #ifdef TARGET_HAS_PRECISE_SMC
1537 TranslationBlock
*current_tb
= NULL
;
1538 CPUState
*cpu
= current_cpu
;
1539 CPUArchState
*env
= NULL
;
1540 int current_tb_modified
= 0;
1541 target_ulong current_pc
= 0;
1542 target_ulong current_cs_base
= 0;
1543 uint32_t current_flags
= 0;
1546 addr
&= TARGET_PAGE_MASK
;
1547 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1552 #ifdef TARGET_HAS_PRECISE_SMC
1553 if (tb
&& pc
!= 0) {
1554 current_tb
= tb_find_pc(pc
);
1560 while (tb
!= NULL
) {
1561 n
= (uintptr_t)tb
& 3;
1562 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1563 #ifdef TARGET_HAS_PRECISE_SMC
1564 if (current_tb
== tb
&&
1565 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1566 /* If we are modifying the current TB, we must stop
1567 its execution. We could be more precise by checking
1568 that the modification is after the current PC, but it
1569 would require a specialized function to partially
1570 restore the CPU state */
1572 current_tb_modified
= 1;
1573 cpu_restore_state_from_tb(cpu
, current_tb
, pc
);
1574 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1577 #endif /* TARGET_HAS_PRECISE_SMC */
1578 tb_phys_invalidate(tb
, addr
);
1579 tb
= tb
->page_next
[n
];
1582 #ifdef TARGET_HAS_PRECISE_SMC
1583 if (current_tb_modified
) {
1584 /* we generate a block containing just the instruction
1585 modifying the memory. It will ensure that it cannot modify
1587 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1595 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1596 tb[1].tc_ptr. Return NULL if not found */
1597 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
)
1599 int m_min
, m_max
, m
;
1601 TranslationBlock
*tb
;
1603 if (tcg_ctx
.tb_ctx
.nb_tbs
<= 0) {
1606 if (tc_ptr
< (uintptr_t)tcg_ctx
.code_gen_buffer
||
1607 tc_ptr
>= (uintptr_t)tcg_ctx
.code_gen_ptr
) {
1610 /* binary search (cf Knuth) */
1612 m_max
= tcg_ctx
.tb_ctx
.nb_tbs
- 1;
1613 while (m_min
<= m_max
) {
1614 m
= (m_min
+ m_max
) >> 1;
1615 tb
= &tcg_ctx
.tb_ctx
.tbs
[m
];
1616 v
= (uintptr_t)tb
->tc_ptr
;
1619 } else if (tc_ptr
< v
) {
1625 return &tcg_ctx
.tb_ctx
.tbs
[m_max
];
1628 #if !defined(CONFIG_USER_ONLY)
1629 void tb_invalidate_phys_addr(AddressSpace
*as
, hwaddr addr
)
1631 ram_addr_t ram_addr
;
1636 mr
= address_space_translate(as
, addr
, &addr
, &l
, false);
1637 if (!(memory_region_is_ram(mr
)
1638 || memory_region_is_romd(mr
))) {
1642 ram_addr
= memory_region_get_ram_addr(mr
) + addr
;
1643 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1646 #endif /* !defined(CONFIG_USER_ONLY) */
1648 void tb_check_watchpoint(CPUState
*cpu
)
1650 TranslationBlock
*tb
;
1652 tb
= tb_find_pc(cpu
->mem_io_pc
);
1654 /* We can use retranslation to find the PC. */
1655 cpu_restore_state_from_tb(cpu
, tb
, cpu
->mem_io_pc
);
1656 tb_phys_invalidate(tb
, -1);
1658 /* The exception probably happened in a helper. The CPU state should
1659 have been saved before calling it. Fetch the PC from there. */
1660 CPUArchState
*env
= cpu
->env_ptr
;
1661 target_ulong pc
, cs_base
;
1662 tb_page_addr_t addr
;
1665 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &flags
);
1666 addr
= get_page_addr_code(env
, pc
);
1667 tb_invalidate_phys_range(addr
, addr
+ 1);
1671 #ifndef CONFIG_USER_ONLY
1672 /* in deterministic execution mode, instructions doing device I/Os
1673 must be at the end of the TB */
1674 void cpu_io_recompile(CPUState
*cpu
, uintptr_t retaddr
)
1676 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1677 CPUArchState
*env
= cpu
->env_ptr
;
1679 TranslationBlock
*tb
;
1681 target_ulong pc
, cs_base
;
1684 tb
= tb_find_pc(retaddr
);
1686 cpu_abort(cpu
, "cpu_io_recompile: could not find TB for pc=%p",
1689 n
= cpu
->icount_decr
.u16
.low
+ tb
->icount
;
1690 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
1691 /* Calculate how many instructions had been executed before the fault
1693 n
= n
- cpu
->icount_decr
.u16
.low
;
1694 /* Generate a new TB ending on the I/O insn. */
1696 /* On MIPS and SH, delay slot instructions can only be restarted if
1697 they were already the first instruction in the TB. If this is not
1698 the first instruction in a TB then re-execute the preceding
1700 #if defined(TARGET_MIPS)
1701 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
1702 env
->active_tc
.PC
-= (env
->hflags
& MIPS_HFLAG_B16
? 2 : 4);
1703 cpu
->icount_decr
.u16
.low
++;
1704 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
1706 #elif defined(TARGET_SH4)
1707 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
1710 cpu
->icount_decr
.u16
.low
++;
1711 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
1714 /* This should never happen. */
1715 if (n
> CF_COUNT_MASK
) {
1716 cpu_abort(cpu
, "TB too big during recompile");
1719 cflags
= n
| CF_LAST_IO
;
1721 cs_base
= tb
->cs_base
;
1723 tb_phys_invalidate(tb
, -1);
1724 if (tb
->cflags
& CF_NOCACHE
) {
1726 /* Invalidate original TB if this TB was generated in
1727 * cpu_exec_nocache() */
1728 tb_phys_invalidate(tb
->orig_tb
, -1);
1732 /* FIXME: In theory this could raise an exception. In practice
1733 we have already translated the block once so it's probably ok. */
1734 tb_gen_code(cpu
, pc
, cs_base
, flags
, cflags
);
1735 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1736 the first in the TB) then we end up generating a whole new TB and
1737 repeating the fault, which is horribly inefficient.
1738 Better would be to execute just this insn uncached, or generate a
1740 cpu_loop_exit_noexc(cpu
);
1743 void tb_flush_jmp_cache(CPUState
*cpu
, target_ulong addr
)
1747 /* Discard jump cache entries for any tb which might potentially
1748 overlap the flushed page. */
1749 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1750 memset(&cpu
->tb_jmp_cache
[i
], 0,
1751 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1753 i
= tb_jmp_cache_hash_page(addr
);
1754 memset(&cpu
->tb_jmp_cache
[i
], 0,
1755 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1758 static void print_qht_statistics(FILE *f
, fprintf_function cpu_fprintf
,
1759 struct qht_stats hst
)
1761 uint32_t hgram_opts
;
1765 if (!hst
.head_buckets
) {
1768 cpu_fprintf(f
, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n",
1769 hst
.used_head_buckets
, hst
.head_buckets
,
1770 (double)hst
.used_head_buckets
/ hst
.head_buckets
* 100);
1772 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
1773 hgram_opts
|= QDIST_PR_100X
| QDIST_PR_PERCENT
;
1774 if (qdist_xmax(&hst
.occupancy
) - qdist_xmin(&hst
.occupancy
) == 1) {
1775 hgram_opts
|= QDIST_PR_NODECIMAL
;
1777 hgram
= qdist_pr(&hst
.occupancy
, 10, hgram_opts
);
1778 cpu_fprintf(f
, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n",
1779 qdist_avg(&hst
.occupancy
) * 100, hgram
);
1782 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
1783 hgram_bins
= qdist_xmax(&hst
.chain
) - qdist_xmin(&hst
.chain
);
1784 if (hgram_bins
> 10) {
1788 hgram_opts
|= QDIST_PR_NODECIMAL
| QDIST_PR_NOBINRANGE
;
1790 hgram
= qdist_pr(&hst
.chain
, hgram_bins
, hgram_opts
);
1791 cpu_fprintf(f
, "TB hash avg chain %0.3f buckets. Histogram: %s\n",
1792 qdist_avg(&hst
.chain
), hgram
);
1796 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
1798 int i
, target_code_size
, max_target_code_size
;
1799 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
1800 TranslationBlock
*tb
;
1801 struct qht_stats hst
;
1803 target_code_size
= 0;
1804 max_target_code_size
= 0;
1806 direct_jmp_count
= 0;
1807 direct_jmp2_count
= 0;
1808 for (i
= 0; i
< tcg_ctx
.tb_ctx
.nb_tbs
; i
++) {
1809 tb
= &tcg_ctx
.tb_ctx
.tbs
[i
];
1810 target_code_size
+= tb
->size
;
1811 if (tb
->size
> max_target_code_size
) {
1812 max_target_code_size
= tb
->size
;
1814 if (tb
->page_addr
[1] != -1) {
1817 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1819 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1820 direct_jmp2_count
++;
1824 /* XXX: avoid using doubles ? */
1825 cpu_fprintf(f
, "Translation buffer state:\n");
1826 cpu_fprintf(f
, "gen code size %td/%zd\n",
1827 tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
,
1828 tcg_ctx
.code_gen_highwater
- tcg_ctx
.code_gen_buffer
);
1829 cpu_fprintf(f
, "TB count %d/%d\n",
1830 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.code_gen_max_blocks
);
1831 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
1832 tcg_ctx
.tb_ctx
.nb_tbs
? target_code_size
/
1833 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1834 max_target_code_size
);
1835 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1836 tcg_ctx
.tb_ctx
.nb_tbs
? (tcg_ctx
.code_gen_ptr
-
1837 tcg_ctx
.code_gen_buffer
) /
1838 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1839 target_code_size
? (double) (tcg_ctx
.code_gen_ptr
-
1840 tcg_ctx
.code_gen_buffer
) /
1841 target_code_size
: 0);
1842 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n", cross_page
,
1843 tcg_ctx
.tb_ctx
.nb_tbs
? (cross_page
* 100) /
1844 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1845 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1847 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp_count
* 100) /
1848 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1850 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp2_count
* 100) /
1851 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1853 qht_statistics_init(&tcg_ctx
.tb_ctx
.htable
, &hst
);
1854 print_qht_statistics(f
, cpu_fprintf
, hst
);
1855 qht_statistics_destroy(&hst
);
1857 cpu_fprintf(f
, "\nStatistics:\n");
1858 cpu_fprintf(f
, "TB flush count %u\n",
1859 atomic_read(&tcg_ctx
.tb_ctx
.tb_flush_count
));
1860 cpu_fprintf(f
, "TB invalidate count %d\n",
1861 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
);
1862 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
1863 tcg_dump_info(f
, cpu_fprintf
);
1866 void dump_opcount_info(FILE *f
, fprintf_function cpu_fprintf
)
1868 tcg_dump_op_count(f
, cpu_fprintf
);
1871 #else /* CONFIG_USER_ONLY */
1873 void cpu_interrupt(CPUState
*cpu
, int mask
)
1875 cpu
->interrupt_request
|= mask
;
1876 cpu
->tcg_exit_req
= 1;
1880 * Walks guest process memory "regions" one by one
1881 * and calls callback function 'fn' for each region.
1883 struct walk_memory_regions_data
{
1884 walk_memory_regions_fn fn
;
1890 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
1891 target_ulong end
, int new_prot
)
1893 if (data
->start
!= -1u) {
1894 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
1900 data
->start
= (new_prot
? end
: -1u);
1901 data
->prot
= new_prot
;
1906 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
1907 target_ulong base
, int level
, void **lp
)
1913 return walk_memory_regions_end(data
, base
, 0);
1919 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1920 int prot
= pd
[i
].flags
;
1922 pa
= base
| (i
<< TARGET_PAGE_BITS
);
1923 if (prot
!= data
->prot
) {
1924 rc
= walk_memory_regions_end(data
, pa
, prot
);
1933 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1934 pa
= base
| ((target_ulong
)i
<<
1935 (TARGET_PAGE_BITS
+ V_L2_BITS
* level
));
1936 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
1946 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
1948 struct walk_memory_regions_data data
;
1949 uintptr_t i
, l1_sz
= v_l1_size
;
1956 for (i
= 0; i
< l1_sz
; i
++) {
1957 target_ulong base
= i
<< (v_l1_shift
+ TARGET_PAGE_BITS
);
1958 int rc
= walk_memory_regions_1(&data
, base
, v_l2_levels
, l1_map
+ i
);
1964 return walk_memory_regions_end(&data
, 0, 0);
1967 static int dump_region(void *priv
, target_ulong start
,
1968 target_ulong end
, unsigned long prot
)
1970 FILE *f
= (FILE *)priv
;
1972 (void) fprintf(f
, TARGET_FMT_lx
"-"TARGET_FMT_lx
1973 " "TARGET_FMT_lx
" %c%c%c\n",
1974 start
, end
, end
- start
,
1975 ((prot
& PAGE_READ
) ? 'r' : '-'),
1976 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
1977 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
1982 /* dump memory mappings */
1983 void page_dump(FILE *f
)
1985 const int length
= sizeof(target_ulong
) * 2;
1986 (void) fprintf(f
, "%-*s %-*s %-*s %s\n",
1987 length
, "start", length
, "end", length
, "size", "prot");
1988 walk_memory_regions(f
, dump_region
);
1991 int page_get_flags(target_ulong address
)
1995 p
= page_find(address
>> TARGET_PAGE_BITS
);
2002 /* Modify the flags of a page and invalidate the code if necessary.
2003 The flag PAGE_WRITE_ORG is positioned automatically depending
2004 on PAGE_WRITE. The mmap_lock should already be held. */
2005 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2007 target_ulong addr
, len
;
2009 /* This function should never be called with addresses outside the
2010 guest address space. If this assert fires, it probably indicates
2011 a missing call to h2g_valid. */
2012 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2013 assert(end
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2015 assert(start
< end
);
2017 start
= start
& TARGET_PAGE_MASK
;
2018 end
= TARGET_PAGE_ALIGN(end
);
2020 if (flags
& PAGE_WRITE
) {
2021 flags
|= PAGE_WRITE_ORG
;
2024 for (addr
= start
, len
= end
- start
;
2026 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2027 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2029 /* If the write protection bit is set, then we invalidate
2031 if (!(p
->flags
& PAGE_WRITE
) &&
2032 (flags
& PAGE_WRITE
) &&
2034 tb_invalidate_phys_page(addr
, 0);
2040 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2046 /* This function should never be called with addresses outside the
2047 guest address space. If this assert fires, it probably indicates
2048 a missing call to h2g_valid. */
2049 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2050 assert(start
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2056 if (start
+ len
- 1 < start
) {
2057 /* We've wrapped around. */
2061 /* must do before we loose bits in the next step */
2062 end
= TARGET_PAGE_ALIGN(start
+ len
);
2063 start
= start
& TARGET_PAGE_MASK
;
2065 for (addr
= start
, len
= end
- start
;
2067 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2068 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2072 if (!(p
->flags
& PAGE_VALID
)) {
2076 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
)) {
2079 if (flags
& PAGE_WRITE
) {
2080 if (!(p
->flags
& PAGE_WRITE_ORG
)) {
2083 /* unprotect the page if it was put read-only because it
2084 contains translated code */
2085 if (!(p
->flags
& PAGE_WRITE
)) {
2086 if (!page_unprotect(addr
, 0)) {
2095 /* called from signal handler: invalidate the code and unprotect the
2096 * page. Return 0 if the fault was not handled, 1 if it was handled,
2097 * and 2 if it was handled but the caller must cause the TB to be
2098 * immediately exited. (We can only return 2 if the 'pc' argument is
2101 int page_unprotect(target_ulong address
, uintptr_t pc
)
2104 bool current_tb_invalidated
;
2106 target_ulong host_start
, host_end
, addr
;
2108 /* Technically this isn't safe inside a signal handler. However we
2109 know this only ever happens in a synchronous SEGV handler, so in
2110 practice it seems to be ok. */
2113 p
= page_find(address
>> TARGET_PAGE_BITS
);
2119 /* if the page was really writable, then we change its
2120 protection back to writable */
2121 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
2122 host_start
= address
& qemu_host_page_mask
;
2123 host_end
= host_start
+ qemu_host_page_size
;
2126 current_tb_invalidated
= false;
2127 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2128 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2129 p
->flags
|= PAGE_WRITE
;
2132 /* and since the content will be modified, we must invalidate
2133 the corresponding translated code. */
2134 current_tb_invalidated
|= tb_invalidate_phys_page(addr
, pc
);
2135 #ifdef DEBUG_TB_CHECK
2136 tb_invalidate_check(addr
);
2139 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2143 /* If current TB was invalidated return to main loop */
2144 return current_tb_invalidated
? 2 : 1;
2149 #endif /* CONFIG_USER_ONLY */