4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
22 #include "qemu/osdep.h"
25 #include "qemu-common.h"
26 #define NO_CPU_IO_DEFS
29 #include "disas/disas.h"
30 #include "exec/exec-all.h"
32 #if defined(CONFIG_USER_ONLY)
34 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
35 #include <sys/param.h>
36 #if __FreeBSD_version >= 700104
37 #define HAVE_KINFO_GETVMMAP
38 #define sigqueue sigqueue_freebsd /* avoid redefinition */
40 #include <machine/profile.h>
49 #include "exec/address-spaces.h"
52 #include "exec/cputlb.h"
53 #include "exec/tb-hash.h"
54 #include "translate-all.h"
55 #include "qemu/bitmap.h"
56 #include "qemu/timer.h"
59 //#define DEBUG_TB_INVALIDATE
61 /* make various TB consistency checks */
62 //#define DEBUG_TB_CHECK
64 #if !defined(CONFIG_USER_ONLY)
65 /* TB consistency checks only implemented for usermode emulation. */
69 #define SMC_BITMAP_USE_THRESHOLD 10
71 typedef struct PageDesc
{
72 /* list of TBs intersecting this ram page */
73 TranslationBlock
*first_tb
;
75 /* in order to optimize self modifying code, we count the number
76 of lookups we do to a given page to use a bitmap */
77 unsigned int code_write_count
;
78 unsigned long *code_bitmap
;
84 /* In system mode we want L1_MAP to be based on ram offsets,
85 while in user mode we want it to be based on virtual addresses. */
86 #if !defined(CONFIG_USER_ONLY)
87 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
88 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
90 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
93 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
96 /* Size of the L2 (and L3, etc) page tables. */
98 #define V_L2_SIZE (1 << V_L2_BITS)
100 /* The bits remaining after N lower levels of page tables. */
101 #define V_L1_BITS_REM \
102 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
104 #if V_L1_BITS_REM < 4
105 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
107 #define V_L1_BITS V_L1_BITS_REM
110 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
112 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
114 uintptr_t qemu_host_page_size
;
115 intptr_t qemu_host_page_mask
;
117 /* The bottom level has pointers to PageDesc */
118 static void *l1_map
[V_L1_SIZE
];
120 /* code generation context */
123 /* translation block context */
124 #ifdef CONFIG_USER_ONLY
125 __thread
int have_tb_lock
;
130 #ifdef CONFIG_USER_ONLY
131 assert(!have_tb_lock
);
132 qemu_mutex_lock(&tcg_ctx
.tb_ctx
.tb_lock
);
139 #ifdef CONFIG_USER_ONLY
140 assert(have_tb_lock
);
142 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
146 void tb_lock_reset(void)
148 #ifdef CONFIG_USER_ONLY
150 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
156 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
);
158 void cpu_gen_init(void)
160 tcg_context_init(&tcg_ctx
);
163 /* Encode VAL as a signed leb128 sequence at P.
164 Return P incremented past the encoded value. */
165 static uint8_t *encode_sleb128(uint8_t *p
, target_long val
)
172 more
= !((val
== 0 && (byte
& 0x40) == 0)
173 || (val
== -1 && (byte
& 0x40) != 0));
183 /* Decode a signed leb128 sequence at *PP; increment *PP past the
184 decoded value. Return the decoded value. */
185 static target_long
decode_sleb128(uint8_t **pp
)
193 val
|= (target_ulong
)(byte
& 0x7f) << shift
;
195 } while (byte
& 0x80);
196 if (shift
< TARGET_LONG_BITS
&& (byte
& 0x40)) {
197 val
|= -(target_ulong
)1 << shift
;
204 /* Encode the data collected about the instructions while compiling TB.
205 Place the data at BLOCK, and return the number of bytes consumed.
207 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
208 which come from the target's insn_start data, followed by a uintptr_t
209 which comes from the host pc of the end of the code implementing the insn.
211 Each line of the table is encoded as sleb128 deltas from the previous
212 line. The seed for the first line is { tb->pc, 0..., tb->tc_ptr }.
213 That is, the first column is seeded with the guest pc, the last column
214 with the host pc, and the middle columns with zeros. */
216 static int encode_search(TranslationBlock
*tb
, uint8_t *block
)
218 uint8_t *highwater
= tcg_ctx
.code_gen_highwater
;
222 tb
->tc_search
= block
;
224 for (i
= 0, n
= tb
->icount
; i
< n
; ++i
) {
227 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
229 prev
= (j
== 0 ? tb
->pc
: 0);
231 prev
= tcg_ctx
.gen_insn_data
[i
- 1][j
];
233 p
= encode_sleb128(p
, tcg_ctx
.gen_insn_data
[i
][j
] - prev
);
235 prev
= (i
== 0 ? 0 : tcg_ctx
.gen_insn_end_off
[i
- 1]);
236 p
= encode_sleb128(p
, tcg_ctx
.gen_insn_end_off
[i
] - prev
);
238 /* Test for (pending) buffer overflow. The assumption is that any
239 one row beginning below the high water mark cannot overrun
240 the buffer completely. Thus we can test for overflow after
241 encoding a row without having to check during encoding. */
242 if (unlikely(p
> highwater
)) {
250 /* The cpu state corresponding to 'searched_pc' is restored. */
251 static int cpu_restore_state_from_tb(CPUState
*cpu
, TranslationBlock
*tb
,
252 uintptr_t searched_pc
)
254 target_ulong data
[TARGET_INSN_START_WORDS
] = { tb
->pc
};
255 uintptr_t host_pc
= (uintptr_t)tb
->tc_ptr
;
256 CPUArchState
*env
= cpu
->env_ptr
;
257 uint8_t *p
= tb
->tc_search
;
258 int i
, j
, num_insns
= tb
->icount
;
259 #ifdef CONFIG_PROFILER
260 int64_t ti
= profile_getclock();
263 if (searched_pc
< host_pc
) {
267 /* Reconstruct the stored insn data while looking for the point at
268 which the end of the insn exceeds the searched_pc. */
269 for (i
= 0; i
< num_insns
; ++i
) {
270 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
271 data
[j
] += decode_sleb128(&p
);
273 host_pc
+= decode_sleb128(&p
);
274 if (host_pc
> searched_pc
) {
281 if (tb
->cflags
& CF_USE_ICOUNT
) {
283 /* Reset the cycle counter to the start of the block. */
284 cpu
->icount_decr
.u16
.low
+= num_insns
;
285 /* Clear the IO flag. */
288 cpu
->icount_decr
.u16
.low
-= i
;
289 restore_state_to_opc(env
, tb
, data
);
291 #ifdef CONFIG_PROFILER
292 tcg_ctx
.restore_time
+= profile_getclock() - ti
;
293 tcg_ctx
.restore_count
++;
298 bool cpu_restore_state(CPUState
*cpu
, uintptr_t retaddr
)
300 TranslationBlock
*tb
;
302 tb
= tb_find_pc(retaddr
);
304 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
305 if (tb
->cflags
& CF_NOCACHE
) {
306 /* one-shot translation, invalidate it immediately */
307 tb_phys_invalidate(tb
, -1);
315 void page_size_init(void)
317 /* NOTE: we can always suppose that qemu_host_page_size >=
319 qemu_real_host_page_size
= getpagesize();
320 qemu_real_host_page_mask
= -(intptr_t)qemu_real_host_page_size
;
321 if (qemu_host_page_size
== 0) {
322 qemu_host_page_size
= qemu_real_host_page_size
;
324 if (qemu_host_page_size
< TARGET_PAGE_SIZE
) {
325 qemu_host_page_size
= TARGET_PAGE_SIZE
;
327 qemu_host_page_mask
= -(intptr_t)qemu_host_page_size
;
330 static void page_init(void)
333 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
335 #ifdef HAVE_KINFO_GETVMMAP
336 struct kinfo_vmentry
*freep
;
339 freep
= kinfo_getvmmap(getpid(), &cnt
);
342 for (i
= 0; i
< cnt
; i
++) {
343 unsigned long startaddr
, endaddr
;
345 startaddr
= freep
[i
].kve_start
;
346 endaddr
= freep
[i
].kve_end
;
347 if (h2g_valid(startaddr
)) {
348 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
350 if (h2g_valid(endaddr
)) {
351 endaddr
= h2g(endaddr
);
352 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
354 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
356 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
367 last_brk
= (unsigned long)sbrk(0);
369 f
= fopen("/compat/linux/proc/self/maps", "r");
374 unsigned long startaddr
, endaddr
;
377 n
= fscanf(f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
379 if (n
== 2 && h2g_valid(startaddr
)) {
380 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
382 if (h2g_valid(endaddr
)) {
383 endaddr
= h2g(endaddr
);
387 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
400 * Called with mmap_lock held for user-mode emulation.
402 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
408 /* Level 1. Always allocated. */
409 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
412 for (i
= V_L1_SHIFT
/ V_L2_BITS
- 1; i
> 0; i
--) {
413 void **p
= atomic_rcu_read(lp
);
419 p
= g_new0(void *, V_L2_SIZE
);
420 atomic_rcu_set(lp
, p
);
423 lp
= p
+ ((index
>> (i
* V_L2_BITS
)) & (V_L2_SIZE
- 1));
426 pd
= atomic_rcu_read(lp
);
431 pd
= g_new0(PageDesc
, V_L2_SIZE
);
432 atomic_rcu_set(lp
, pd
);
435 return pd
+ (index
& (V_L2_SIZE
- 1));
438 static inline PageDesc
*page_find(tb_page_addr_t index
)
440 return page_find_alloc(index
, 0);
443 #if defined(CONFIG_USER_ONLY)
444 /* Currently it is not recommended to allocate big chunks of data in
445 user mode. It will change when a dedicated libc will be used. */
446 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
447 region in which the guest needs to run. Revisit this. */
448 #define USE_STATIC_CODE_GEN_BUFFER
451 /* Minimum size of the code gen buffer. This number is randomly chosen,
452 but not so small that we can't have a fair number of TB's live. */
453 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
455 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
456 indicated, this is constrained by the range of direct branches on the
457 host cpu, as used by the TCG implementation of goto_tb. */
458 #if defined(__x86_64__)
459 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
460 #elif defined(__sparc__)
461 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
462 #elif defined(__powerpc64__)
463 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
464 #elif defined(__powerpc__)
465 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
466 #elif defined(__aarch64__)
467 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
468 #elif defined(__arm__)
469 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
470 #elif defined(__s390x__)
471 /* We have a +- 4GB range on the branches; leave some slop. */
472 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
473 #elif defined(__mips__)
474 /* We have a 256MB branch region, but leave room to make sure the
475 main executable is also within that region. */
476 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
478 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
481 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
483 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
484 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
485 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
487 static inline size_t size_code_gen_buffer(size_t tb_size
)
489 /* Size the buffer. */
491 #ifdef USE_STATIC_CODE_GEN_BUFFER
492 tb_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
494 /* ??? Needs adjustments. */
495 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
496 static buffer, we could size this on RESERVED_VA, on the text
497 segment size of the executable, or continue to use the default. */
498 tb_size
= (unsigned long)(ram_size
/ 4);
501 if (tb_size
< MIN_CODE_GEN_BUFFER_SIZE
) {
502 tb_size
= MIN_CODE_GEN_BUFFER_SIZE
;
504 if (tb_size
> MAX_CODE_GEN_BUFFER_SIZE
) {
505 tb_size
= MAX_CODE_GEN_BUFFER_SIZE
;
511 /* In order to use J and JAL within the code_gen_buffer, we require
512 that the buffer not cross a 256MB boundary. */
513 static inline bool cross_256mb(void *addr
, size_t size
)
515 return ((uintptr_t)addr
^ ((uintptr_t)addr
+ size
)) & ~0x0ffffffful
;
518 /* We weren't able to allocate a buffer without crossing that boundary,
519 so make do with the larger portion of the buffer that doesn't cross.
520 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
521 static inline void *split_cross_256mb(void *buf1
, size_t size1
)
523 void *buf2
= (void *)(((uintptr_t)buf1
+ size1
) & ~0x0ffffffful
);
524 size_t size2
= buf1
+ size1
- buf2
;
532 tcg_ctx
.code_gen_buffer_size
= size1
;
537 #ifdef USE_STATIC_CODE_GEN_BUFFER
538 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
539 __attribute__((aligned(CODE_GEN_ALIGN
)));
542 static inline void do_protect(void *addr
, long size
, int prot
)
545 VirtualProtect(addr
, size
, prot
, &old_protect
);
548 static inline void map_exec(void *addr
, long size
)
550 do_protect(addr
, size
, PAGE_EXECUTE_READWRITE
);
553 static inline void map_none(void *addr
, long size
)
555 do_protect(addr
, size
, PAGE_NOACCESS
);
558 static inline void do_protect(void *addr
, long size
, int prot
)
560 uintptr_t start
, end
;
562 start
= (uintptr_t)addr
;
563 start
&= qemu_real_host_page_mask
;
565 end
= (uintptr_t)addr
+ size
;
566 end
= ROUND_UP(end
, qemu_real_host_page_size
);
568 mprotect((void *)start
, end
- start
, prot
);
571 static inline void map_exec(void *addr
, long size
)
573 do_protect(addr
, size
, PROT_READ
| PROT_WRITE
| PROT_EXEC
);
576 static inline void map_none(void *addr
, long size
)
578 do_protect(addr
, size
, PROT_NONE
);
582 static inline void *alloc_code_gen_buffer(void)
584 void *buf
= static_code_gen_buffer
;
585 size_t full_size
, size
;
587 /* The size of the buffer, rounded down to end on a page boundary. */
588 full_size
= (((uintptr_t)buf
+ sizeof(static_code_gen_buffer
))
589 & qemu_real_host_page_mask
) - (uintptr_t)buf
;
591 /* Reserve a guard page. */
592 size
= full_size
- qemu_real_host_page_size
;
594 /* Honor a command-line option limiting the size of the buffer. */
595 if (size
> tcg_ctx
.code_gen_buffer_size
) {
596 size
= (((uintptr_t)buf
+ tcg_ctx
.code_gen_buffer_size
)
597 & qemu_real_host_page_mask
) - (uintptr_t)buf
;
599 tcg_ctx
.code_gen_buffer_size
= size
;
602 if (cross_256mb(buf
, size
)) {
603 buf
= split_cross_256mb(buf
, size
);
604 size
= tcg_ctx
.code_gen_buffer_size
;
609 map_none(buf
+ size
, qemu_real_host_page_size
);
610 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
614 #elif defined(_WIN32)
615 static inline void *alloc_code_gen_buffer(void)
617 size_t size
= tcg_ctx
.code_gen_buffer_size
;
620 /* Perform the allocation in two steps, so that the guard page
621 is reserved but uncommitted. */
622 buf1
= VirtualAlloc(NULL
, size
+ qemu_real_host_page_size
,
623 MEM_RESERVE
, PAGE_NOACCESS
);
625 buf2
= VirtualAlloc(buf1
, size
, MEM_COMMIT
, PAGE_EXECUTE_READWRITE
);
626 assert(buf1
== buf2
);
632 static inline void *alloc_code_gen_buffer(void)
634 int flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
636 size_t size
= tcg_ctx
.code_gen_buffer_size
;
639 /* Constrain the position of the buffer based on the host cpu.
640 Note that these addresses are chosen in concert with the
641 addresses assigned in the relevant linker script file. */
642 # if defined(__PIE__) || defined(__PIC__)
643 /* Don't bother setting a preferred location if we're building
644 a position-independent executable. We're more likely to get
645 an address near the main executable if we let the kernel
646 choose the address. */
647 # elif defined(__x86_64__) && defined(MAP_32BIT)
648 /* Force the memory down into low memory with the executable.
649 Leave the choice of exact location with the kernel. */
651 /* Cannot expect to map more than 800MB in low memory. */
652 if (size
> 800u * 1024 * 1024) {
653 tcg_ctx
.code_gen_buffer_size
= size
= 800u * 1024 * 1024;
655 # elif defined(__sparc__)
656 start
= 0x40000000ul
;
657 # elif defined(__s390x__)
658 start
= 0x90000000ul
;
659 # elif defined(__mips__)
660 # if _MIPS_SIM == _ABI64
661 start
= 0x128000000ul
;
663 start
= 0x08000000ul
;
667 buf
= mmap((void *)start
, size
+ qemu_real_host_page_size
,
668 PROT_NONE
, flags
, -1, 0);
669 if (buf
== MAP_FAILED
) {
674 if (cross_256mb(buf
, size
)) {
675 /* Try again, with the original still mapped, to avoid re-acquiring
676 that 256mb crossing. This time don't specify an address. */
678 void *buf2
= mmap(NULL
, size
+ qemu_real_host_page_size
,
679 PROT_NONE
, flags
, -1, 0);
680 switch (buf2
!= MAP_FAILED
) {
682 if (!cross_256mb(buf2
, size
)) {
683 /* Success! Use the new buffer. */
684 munmap(buf
, size
+ qemu_real_host_page_size
);
687 /* Failure. Work with what we had. */
688 munmap(buf2
, size
+ qemu_real_host_page_size
);
691 /* Split the original buffer. Free the smaller half. */
692 buf2
= split_cross_256mb(buf
, size
);
693 size2
= tcg_ctx
.code_gen_buffer_size
;
695 munmap(buf
+ size2
+ qemu_real_host_page_size
, size
- size2
);
697 munmap(buf
, size
- size2
);
706 /* Make the final buffer accessible. The guard page at the end
707 will remain inaccessible with PROT_NONE. */
708 mprotect(buf
, size
, PROT_WRITE
| PROT_READ
| PROT_EXEC
);
710 /* Request large pages for the buffer. */
711 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
715 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
717 static inline void code_gen_alloc(size_t tb_size
)
719 tcg_ctx
.code_gen_buffer_size
= size_code_gen_buffer(tb_size
);
720 tcg_ctx
.code_gen_buffer
= alloc_code_gen_buffer();
721 if (tcg_ctx
.code_gen_buffer
== NULL
) {
722 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
726 /* Estimate a good size for the number of TBs we can support. We
727 still haven't deducted the prologue from the buffer size here,
728 but that's minimal and won't affect the estimate much. */
729 tcg_ctx
.code_gen_max_blocks
730 = tcg_ctx
.code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
731 tcg_ctx
.tb_ctx
.tbs
= g_new(TranslationBlock
, tcg_ctx
.code_gen_max_blocks
);
733 qemu_mutex_init(&tcg_ctx
.tb_ctx
.tb_lock
);
736 static void tb_htable_init(void)
738 unsigned int mode
= QHT_MODE_AUTO_RESIZE
;
740 qht_init(&tcg_ctx
.tb_ctx
.htable
, CODE_GEN_HTABLE_SIZE
, mode
);
743 /* Must be called before using the QEMU cpus. 'tb_size' is the size
744 (in bytes) allocated to the translation buffer. Zero means default
746 void tcg_exec_init(unsigned long tb_size
)
751 code_gen_alloc(tb_size
);
752 #if defined(CONFIG_SOFTMMU)
753 /* There's no guest base to take into account, so go ahead and
754 initialize the prologue now. */
755 tcg_prologue_init(&tcg_ctx
);
759 bool tcg_enabled(void)
761 return tcg_ctx
.code_gen_buffer
!= NULL
;
764 /* Allocate a new translation block. Flush the translation buffer if
765 too many translation blocks or too much generated code. */
766 static TranslationBlock
*tb_alloc(target_ulong pc
)
768 TranslationBlock
*tb
;
770 if (tcg_ctx
.tb_ctx
.nb_tbs
>= tcg_ctx
.code_gen_max_blocks
) {
773 tb
= &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
++];
779 void tb_free(TranslationBlock
*tb
)
781 /* In practice this is mostly used for single use temporary TB
782 Ignore the hard cases and just back up if this TB happens to
783 be the last one generated. */
784 if (tcg_ctx
.tb_ctx
.nb_tbs
> 0 &&
785 tb
== &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
- 1]) {
786 tcg_ctx
.code_gen_ptr
= tb
->tc_ptr
;
787 tcg_ctx
.tb_ctx
.nb_tbs
--;
791 static inline void invalidate_page_bitmap(PageDesc
*p
)
793 #ifdef CONFIG_SOFTMMU
794 g_free(p
->code_bitmap
);
795 p
->code_bitmap
= NULL
;
796 p
->code_write_count
= 0;
800 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
801 static void page_flush_tb_1(int level
, void **lp
)
811 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
812 pd
[i
].first_tb
= NULL
;
813 invalidate_page_bitmap(pd
+ i
);
818 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
819 page_flush_tb_1(level
- 1, pp
+ i
);
824 static void page_flush_tb(void)
828 for (i
= 0; i
< V_L1_SIZE
; i
++) {
829 page_flush_tb_1(V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
833 /* flush all the translation blocks */
834 /* XXX: tb_flush is currently not thread safe */
835 void tb_flush(CPUState
*cpu
)
837 if (!tcg_enabled()) {
840 #if defined(DEBUG_FLUSH)
841 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
842 (unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
),
843 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.tb_ctx
.nb_tbs
> 0 ?
844 ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)) /
845 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
847 if ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)
848 > tcg_ctx
.code_gen_buffer_size
) {
849 cpu_abort(cpu
, "Internal error: code buffer overflow\n");
851 tcg_ctx
.tb_ctx
.nb_tbs
= 0;
856 for (i
= 0; i
< TB_JMP_CACHE_SIZE
; ++i
) {
857 atomic_set(&cpu
->tb_jmp_cache
[i
], NULL
);
859 cpu
->tb_flushed
= true;
862 qht_reset_size(&tcg_ctx
.tb_ctx
.htable
, CODE_GEN_HTABLE_SIZE
);
865 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
866 /* XXX: flush processor icache at this point if cache flush is
868 tcg_ctx
.tb_ctx
.tb_flush_count
++;
871 #ifdef DEBUG_TB_CHECK
874 do_tb_invalidate_check(struct qht
*ht
, void *p
, uint32_t hash
, void *userp
)
876 TranslationBlock
*tb
= p
;
877 target_ulong addr
= *(target_ulong
*)userp
;
879 if (!(addr
+ TARGET_PAGE_SIZE
<= tb
->pc
|| addr
>= tb
->pc
+ tb
->size
)) {
880 printf("ERROR invalidate: address=" TARGET_FMT_lx
881 " PC=%08lx size=%04x\n", addr
, (long)tb
->pc
, tb
->size
);
885 static void tb_invalidate_check(target_ulong address
)
887 address
&= TARGET_PAGE_MASK
;
888 qht_iter(&tcg_ctx
.tb_ctx
.htable
, do_tb_invalidate_check
, &address
);
892 do_tb_page_check(struct qht
*ht
, void *p
, uint32_t hash
, void *userp
)
894 TranslationBlock
*tb
= p
;
897 flags1
= page_get_flags(tb
->pc
);
898 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
899 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
900 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
901 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
905 /* verify that all the pages have correct rights for code */
906 static void tb_page_check(void)
908 qht_iter(&tcg_ctx
.tb_ctx
.htable
, do_tb_page_check
, NULL
);
913 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
915 TranslationBlock
*tb1
;
920 n1
= (uintptr_t)tb1
& 3;
921 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
923 *ptb
= tb1
->page_next
[n1
];
926 ptb
= &tb1
->page_next
[n1
];
930 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */
931 static inline void tb_remove_from_jmp_list(TranslationBlock
*tb
, int n
)
933 TranslationBlock
*tb1
;
937 ptb
= &tb
->jmp_list_next
[n
];
939 /* find tb(n) in circular list */
943 tb1
= (TranslationBlock
*)(ntb
& ~3);
944 if (n1
== n
&& tb1
== tb
) {
948 ptb
= &tb1
->jmp_list_first
;
950 ptb
= &tb1
->jmp_list_next
[n1
];
953 /* now we can suppress tb(n) from the list */
954 *ptb
= tb
->jmp_list_next
[n
];
956 tb
->jmp_list_next
[n
] = (uintptr_t)NULL
;
960 /* reset the jump entry 'n' of a TB so that it is not chained to
962 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
964 uintptr_t addr
= (uintptr_t)(tb
->tc_ptr
+ tb
->jmp_reset_offset
[n
]);
965 tb_set_jmp_target(tb
, n
, addr
);
968 /* remove any jumps to the TB */
969 static inline void tb_jmp_unlink(TranslationBlock
*tb
)
971 TranslationBlock
*tb1
;
975 ptb
= &tb
->jmp_list_first
;
979 tb1
= (TranslationBlock
*)(ntb
& ~3);
983 tb_reset_jump(tb1
, n1
);
984 *ptb
= tb1
->jmp_list_next
[n1
];
985 tb1
->jmp_list_next
[n1
] = (uintptr_t)NULL
;
989 /* invalidate one TB */
990 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
995 tb_page_addr_t phys_pc
;
997 /* remove the TB from the hash list */
998 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
999 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
);
1000 qht_remove(&tcg_ctx
.tb_ctx
.htable
, tb
, h
);
1002 /* remove the TB from the page list */
1003 if (tb
->page_addr
[0] != page_addr
) {
1004 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
1005 tb_page_remove(&p
->first_tb
, tb
);
1006 invalidate_page_bitmap(p
);
1008 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
1009 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
1010 tb_page_remove(&p
->first_tb
, tb
);
1011 invalidate_page_bitmap(p
);
1014 /* remove the TB from the hash list */
1015 h
= tb_jmp_cache_hash_func(tb
->pc
);
1017 if (atomic_read(&cpu
->tb_jmp_cache
[h
]) == tb
) {
1018 atomic_set(&cpu
->tb_jmp_cache
[h
], NULL
);
1022 /* suppress this TB from the two jump lists */
1023 tb_remove_from_jmp_list(tb
, 0);
1024 tb_remove_from_jmp_list(tb
, 1);
1026 /* suppress any remaining jumps to this TB */
1029 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
++;
1032 #ifdef CONFIG_SOFTMMU
1033 static void build_page_bitmap(PageDesc
*p
)
1035 int n
, tb_start
, tb_end
;
1036 TranslationBlock
*tb
;
1038 p
->code_bitmap
= bitmap_new(TARGET_PAGE_SIZE
);
1041 while (tb
!= NULL
) {
1042 n
= (uintptr_t)tb
& 3;
1043 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1044 /* NOTE: this is subtle as a TB may span two physical pages */
1046 /* NOTE: tb_end may be after the end of the page, but
1047 it is not a problem */
1048 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
1049 tb_end
= tb_start
+ tb
->size
;
1050 if (tb_end
> TARGET_PAGE_SIZE
) {
1051 tb_end
= TARGET_PAGE_SIZE
;
1055 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1057 bitmap_set(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
1058 tb
= tb
->page_next
[n
];
1063 /* add the tb in the target page and protect it if necessary
1065 * Called with mmap_lock held for user-mode emulation.
1067 static inline void tb_alloc_page(TranslationBlock
*tb
,
1068 unsigned int n
, tb_page_addr_t page_addr
)
1071 #ifndef CONFIG_USER_ONLY
1072 bool page_already_protected
;
1075 tb
->page_addr
[n
] = page_addr
;
1076 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1077 tb
->page_next
[n
] = p
->first_tb
;
1078 #ifndef CONFIG_USER_ONLY
1079 page_already_protected
= p
->first_tb
!= NULL
;
1081 p
->first_tb
= (TranslationBlock
*)((uintptr_t)tb
| n
);
1082 invalidate_page_bitmap(p
);
1084 #if defined(CONFIG_USER_ONLY)
1085 if (p
->flags
& PAGE_WRITE
) {
1090 /* force the host page as non writable (writes will have a
1091 page fault + mprotect overhead) */
1092 page_addr
&= qemu_host_page_mask
;
1094 for (addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1095 addr
+= TARGET_PAGE_SIZE
) {
1097 p2
= page_find(addr
>> TARGET_PAGE_BITS
);
1102 p2
->flags
&= ~PAGE_WRITE
;
1104 mprotect(g2h(page_addr
), qemu_host_page_size
,
1105 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1106 #ifdef DEBUG_TB_INVALIDATE
1107 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1112 /* if some code is already present, then the pages are already
1113 protected. So we handle the case where only the first TB is
1114 allocated in a physical page */
1115 if (!page_already_protected
) {
1116 tlb_protect_code(page_addr
);
1121 /* add a new TB and link it to the physical page tables. phys_page2 is
1122 * (-1) to indicate that only one page contains the TB.
1124 * Called with mmap_lock held for user-mode emulation.
1126 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
1127 tb_page_addr_t phys_page2
)
1131 /* add in the hash table */
1132 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
);
1133 qht_insert(&tcg_ctx
.tb_ctx
.htable
, tb
, h
);
1135 /* add in the page list */
1136 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1137 if (phys_page2
!= -1) {
1138 tb_alloc_page(tb
, 1, phys_page2
);
1140 tb
->page_addr
[1] = -1;
1143 #ifdef DEBUG_TB_CHECK
1148 /* Called with mmap_lock held for user mode emulation. */
1149 TranslationBlock
*tb_gen_code(CPUState
*cpu
,
1150 target_ulong pc
, target_ulong cs_base
,
1151 uint32_t flags
, int cflags
)
1153 CPUArchState
*env
= cpu
->env_ptr
;
1154 TranslationBlock
*tb
;
1155 tb_page_addr_t phys_pc
, phys_page2
;
1156 target_ulong virt_page2
;
1157 tcg_insn_unit
*gen_code_buf
;
1158 int gen_code_size
, search_size
;
1159 #ifdef CONFIG_PROFILER
1163 phys_pc
= get_page_addr_code(env
, pc
);
1164 if (use_icount
&& !(cflags
& CF_IGNORE_ICOUNT
)) {
1165 cflags
|= CF_USE_ICOUNT
;
1169 if (unlikely(!tb
)) {
1171 /* flush must be done */
1173 /* cannot fail at this point */
1178 gen_code_buf
= tcg_ctx
.code_gen_ptr
;
1179 tb
->tc_ptr
= gen_code_buf
;
1180 tb
->cs_base
= cs_base
;
1182 tb
->cflags
= cflags
;
1184 #ifdef CONFIG_PROFILER
1185 tcg_ctx
.tb_count1
++; /* includes aborted translations because of
1187 ti
= profile_getclock();
1190 tcg_func_start(&tcg_ctx
);
1192 tcg_ctx
.cpu
= ENV_GET_CPU(env
);
1193 gen_intermediate_code(env
, tb
);
1196 trace_translate_block(tb
, tb
->pc
, tb
->tc_ptr
);
1198 /* generate machine code */
1199 tb
->jmp_reset_offset
[0] = TB_JMP_RESET_OFFSET_INVALID
;
1200 tb
->jmp_reset_offset
[1] = TB_JMP_RESET_OFFSET_INVALID
;
1201 tcg_ctx
.tb_jmp_reset_offset
= tb
->jmp_reset_offset
;
1202 #ifdef USE_DIRECT_JUMP
1203 tcg_ctx
.tb_jmp_insn_offset
= tb
->jmp_insn_offset
;
1204 tcg_ctx
.tb_jmp_target_addr
= NULL
;
1206 tcg_ctx
.tb_jmp_insn_offset
= NULL
;
1207 tcg_ctx
.tb_jmp_target_addr
= tb
->jmp_target_addr
;
1210 #ifdef CONFIG_PROFILER
1212 tcg_ctx
.interm_time
+= profile_getclock() - ti
;
1213 tcg_ctx
.code_time
-= profile_getclock();
1216 /* ??? Overflow could be handled better here. In particular, we
1217 don't need to re-do gen_intermediate_code, nor should we re-do
1218 the tcg optimization currently hidden inside tcg_gen_code. All
1219 that should be required is to flush the TBs, allocate a new TB,
1220 re-initialize it per above, and re-do the actual code generation. */
1221 gen_code_size
= tcg_gen_code(&tcg_ctx
, tb
);
1222 if (unlikely(gen_code_size
< 0)) {
1223 goto buffer_overflow
;
1225 search_size
= encode_search(tb
, (void *)gen_code_buf
+ gen_code_size
);
1226 if (unlikely(search_size
< 0)) {
1227 goto buffer_overflow
;
1230 #ifdef CONFIG_PROFILER
1231 tcg_ctx
.code_time
+= profile_getclock();
1232 tcg_ctx
.code_in_len
+= tb
->size
;
1233 tcg_ctx
.code_out_len
+= gen_code_size
;
1234 tcg_ctx
.search_out_len
+= search_size
;
1238 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
) &&
1239 qemu_log_in_addr_range(tb
->pc
)) {
1240 qemu_log("OUT: [size=%d]\n", gen_code_size
);
1241 log_disas(tb
->tc_ptr
, gen_code_size
);
1247 tcg_ctx
.code_gen_ptr
= (void *)
1248 ROUND_UP((uintptr_t)gen_code_buf
+ gen_code_size
+ search_size
,
1251 /* init jump list */
1252 assert(((uintptr_t)tb
& 3) == 0);
1253 tb
->jmp_list_first
= (uintptr_t)tb
| 2;
1254 tb
->jmp_list_next
[0] = (uintptr_t)NULL
;
1255 tb
->jmp_list_next
[1] = (uintptr_t)NULL
;
1257 /* init original jump addresses wich has been set during tcg_gen_code() */
1258 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1259 tb_reset_jump(tb
, 0);
1261 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1262 tb_reset_jump(tb
, 1);
1265 /* check next page if needed */
1266 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1268 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1269 phys_page2
= get_page_addr_code(env
, virt_page2
);
1271 /* As long as consistency of the TB stuff is provided by tb_lock in user
1272 * mode and is implicit in single-threaded softmmu emulation, no explicit
1273 * memory barrier is required before tb_link_page() makes the TB visible
1274 * through the physical hash table and physical page list.
1276 tb_link_page(tb
, phys_pc
, phys_page2
);
1281 * Invalidate all TBs which intersect with the target physical address range
1282 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1283 * 'is_cpu_write_access' should be true if called from a real cpu write
1284 * access: the virtual CPU will exit the current TB if code is modified inside
1287 * Called with mmap_lock held for user-mode emulation
1289 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
)
1291 while (start
< end
) {
1292 tb_invalidate_phys_page_range(start
, end
, 0);
1293 start
&= TARGET_PAGE_MASK
;
1294 start
+= TARGET_PAGE_SIZE
;
1299 * Invalidate all TBs which intersect with the target physical address range
1300 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1301 * 'is_cpu_write_access' should be true if called from a real cpu write
1302 * access: the virtual CPU will exit the current TB if code is modified inside
1305 * Called with mmap_lock held for user-mode emulation
1307 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1308 int is_cpu_write_access
)
1310 TranslationBlock
*tb
, *tb_next
;
1311 #if defined(TARGET_HAS_PRECISE_SMC)
1312 CPUState
*cpu
= current_cpu
;
1313 CPUArchState
*env
= NULL
;
1315 tb_page_addr_t tb_start
, tb_end
;
1318 #ifdef TARGET_HAS_PRECISE_SMC
1319 int current_tb_not_found
= is_cpu_write_access
;
1320 TranslationBlock
*current_tb
= NULL
;
1321 int current_tb_modified
= 0;
1322 target_ulong current_pc
= 0;
1323 target_ulong current_cs_base
= 0;
1324 uint32_t current_flags
= 0;
1325 #endif /* TARGET_HAS_PRECISE_SMC */
1327 p
= page_find(start
>> TARGET_PAGE_BITS
);
1331 #if defined(TARGET_HAS_PRECISE_SMC)
1337 /* we remove all the TBs in the range [start, end[ */
1338 /* XXX: see if in some cases it could be faster to invalidate all
1341 while (tb
!= NULL
) {
1342 n
= (uintptr_t)tb
& 3;
1343 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1344 tb_next
= tb
->page_next
[n
];
1345 /* NOTE: this is subtle as a TB may span two physical pages */
1347 /* NOTE: tb_end may be after the end of the page, but
1348 it is not a problem */
1349 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1350 tb_end
= tb_start
+ tb
->size
;
1352 tb_start
= tb
->page_addr
[1];
1353 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1355 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1356 #ifdef TARGET_HAS_PRECISE_SMC
1357 if (current_tb_not_found
) {
1358 current_tb_not_found
= 0;
1360 if (cpu
->mem_io_pc
) {
1361 /* now we have a real cpu fault */
1362 current_tb
= tb_find_pc(cpu
->mem_io_pc
);
1365 if (current_tb
== tb
&&
1366 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1367 /* If we are modifying the current TB, we must stop
1368 its execution. We could be more precise by checking
1369 that the modification is after the current PC, but it
1370 would require a specialized function to partially
1371 restore the CPU state */
1373 current_tb_modified
= 1;
1374 cpu_restore_state_from_tb(cpu
, current_tb
, cpu
->mem_io_pc
);
1375 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1378 #endif /* TARGET_HAS_PRECISE_SMC */
1379 tb_phys_invalidate(tb
, -1);
1383 #if !defined(CONFIG_USER_ONLY)
1384 /* if no code remaining, no need to continue to use slow writes */
1386 invalidate_page_bitmap(p
);
1387 tlb_unprotect_code(start
);
1390 #ifdef TARGET_HAS_PRECISE_SMC
1391 if (current_tb_modified
) {
1392 /* we generate a block containing just the instruction
1393 modifying the memory. It will ensure that it cannot modify
1395 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1396 cpu_loop_exit_noexc(cpu
);
1401 #ifdef CONFIG_SOFTMMU
1402 /* len must be <= 8 and start must be a multiple of len */
1403 void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1409 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1410 cpu_single_env
->mem_io_vaddr
, len
,
1411 cpu_single_env
->eip
,
1412 cpu_single_env
->eip
+
1413 (intptr_t)cpu_single_env
->segs
[R_CS
].base
);
1416 p
= page_find(start
>> TARGET_PAGE_BITS
);
1420 if (!p
->code_bitmap
&&
1421 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
) {
1422 /* build code bitmap */
1423 build_page_bitmap(p
);
1425 if (p
->code_bitmap
) {
1429 nr
= start
& ~TARGET_PAGE_MASK
;
1430 b
= p
->code_bitmap
[BIT_WORD(nr
)] >> (nr
& (BITS_PER_LONG
- 1));
1431 if (b
& ((1 << len
) - 1)) {
1436 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1440 /* Called with mmap_lock held. If pc is not 0 then it indicates the
1441 * host PC of the faulting store instruction that caused this invalidate.
1442 * Returns true if the caller needs to abort execution of the current
1443 * TB (because it was modified by this store and the guest CPU has
1444 * precise-SMC semantics).
1446 static bool tb_invalidate_phys_page(tb_page_addr_t addr
, uintptr_t pc
)
1448 TranslationBlock
*tb
;
1451 #ifdef TARGET_HAS_PRECISE_SMC
1452 TranslationBlock
*current_tb
= NULL
;
1453 CPUState
*cpu
= current_cpu
;
1454 CPUArchState
*env
= NULL
;
1455 int current_tb_modified
= 0;
1456 target_ulong current_pc
= 0;
1457 target_ulong current_cs_base
= 0;
1458 uint32_t current_flags
= 0;
1461 addr
&= TARGET_PAGE_MASK
;
1462 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1467 #ifdef TARGET_HAS_PRECISE_SMC
1468 if (tb
&& pc
!= 0) {
1469 current_tb
= tb_find_pc(pc
);
1475 while (tb
!= NULL
) {
1476 n
= (uintptr_t)tb
& 3;
1477 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1478 #ifdef TARGET_HAS_PRECISE_SMC
1479 if (current_tb
== tb
&&
1480 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1481 /* If we are modifying the current TB, we must stop
1482 its execution. We could be more precise by checking
1483 that the modification is after the current PC, but it
1484 would require a specialized function to partially
1485 restore the CPU state */
1487 current_tb_modified
= 1;
1488 cpu_restore_state_from_tb(cpu
, current_tb
, pc
);
1489 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1492 #endif /* TARGET_HAS_PRECISE_SMC */
1493 tb_phys_invalidate(tb
, addr
);
1494 tb
= tb
->page_next
[n
];
1497 #ifdef TARGET_HAS_PRECISE_SMC
1498 if (current_tb_modified
) {
1499 /* we generate a block containing just the instruction
1500 modifying the memory. It will ensure that it cannot modify
1502 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1510 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1511 tb[1].tc_ptr. Return NULL if not found */
1512 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
)
1514 int m_min
, m_max
, m
;
1516 TranslationBlock
*tb
;
1518 if (tcg_ctx
.tb_ctx
.nb_tbs
<= 0) {
1521 if (tc_ptr
< (uintptr_t)tcg_ctx
.code_gen_buffer
||
1522 tc_ptr
>= (uintptr_t)tcg_ctx
.code_gen_ptr
) {
1525 /* binary search (cf Knuth) */
1527 m_max
= tcg_ctx
.tb_ctx
.nb_tbs
- 1;
1528 while (m_min
<= m_max
) {
1529 m
= (m_min
+ m_max
) >> 1;
1530 tb
= &tcg_ctx
.tb_ctx
.tbs
[m
];
1531 v
= (uintptr_t)tb
->tc_ptr
;
1534 } else if (tc_ptr
< v
) {
1540 return &tcg_ctx
.tb_ctx
.tbs
[m_max
];
1543 #if !defined(CONFIG_USER_ONLY)
1544 void tb_invalidate_phys_addr(AddressSpace
*as
, hwaddr addr
)
1546 ram_addr_t ram_addr
;
1551 mr
= address_space_translate(as
, addr
, &addr
, &l
, false);
1552 if (!(memory_region_is_ram(mr
)
1553 || memory_region_is_romd(mr
))) {
1557 ram_addr
= memory_region_get_ram_addr(mr
) + addr
;
1558 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1561 #endif /* !defined(CONFIG_USER_ONLY) */
1563 void tb_check_watchpoint(CPUState
*cpu
)
1565 TranslationBlock
*tb
;
1567 tb
= tb_find_pc(cpu
->mem_io_pc
);
1569 /* We can use retranslation to find the PC. */
1570 cpu_restore_state_from_tb(cpu
, tb
, cpu
->mem_io_pc
);
1571 tb_phys_invalidate(tb
, -1);
1573 /* The exception probably happened in a helper. The CPU state should
1574 have been saved before calling it. Fetch the PC from there. */
1575 CPUArchState
*env
= cpu
->env_ptr
;
1576 target_ulong pc
, cs_base
;
1577 tb_page_addr_t addr
;
1580 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &flags
);
1581 addr
= get_page_addr_code(env
, pc
);
1582 tb_invalidate_phys_range(addr
, addr
+ 1);
1586 #ifndef CONFIG_USER_ONLY
1587 /* in deterministic execution mode, instructions doing device I/Os
1588 must be at the end of the TB */
1589 void cpu_io_recompile(CPUState
*cpu
, uintptr_t retaddr
)
1591 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1592 CPUArchState
*env
= cpu
->env_ptr
;
1594 TranslationBlock
*tb
;
1596 target_ulong pc
, cs_base
;
1599 tb
= tb_find_pc(retaddr
);
1601 cpu_abort(cpu
, "cpu_io_recompile: could not find TB for pc=%p",
1604 n
= cpu
->icount_decr
.u16
.low
+ tb
->icount
;
1605 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
1606 /* Calculate how many instructions had been executed before the fault
1608 n
= n
- cpu
->icount_decr
.u16
.low
;
1609 /* Generate a new TB ending on the I/O insn. */
1611 /* On MIPS and SH, delay slot instructions can only be restarted if
1612 they were already the first instruction in the TB. If this is not
1613 the first instruction in a TB then re-execute the preceding
1615 #if defined(TARGET_MIPS)
1616 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
1617 env
->active_tc
.PC
-= (env
->hflags
& MIPS_HFLAG_B16
? 2 : 4);
1618 cpu
->icount_decr
.u16
.low
++;
1619 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
1621 #elif defined(TARGET_SH4)
1622 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
1625 cpu
->icount_decr
.u16
.low
++;
1626 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
1629 /* This should never happen. */
1630 if (n
> CF_COUNT_MASK
) {
1631 cpu_abort(cpu
, "TB too big during recompile");
1634 cflags
= n
| CF_LAST_IO
;
1636 cs_base
= tb
->cs_base
;
1638 tb_phys_invalidate(tb
, -1);
1639 if (tb
->cflags
& CF_NOCACHE
) {
1641 /* Invalidate original TB if this TB was generated in
1642 * cpu_exec_nocache() */
1643 tb_phys_invalidate(tb
->orig_tb
, -1);
1647 /* FIXME: In theory this could raise an exception. In practice
1648 we have already translated the block once so it's probably ok. */
1649 tb_gen_code(cpu
, pc
, cs_base
, flags
, cflags
);
1650 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1651 the first in the TB) then we end up generating a whole new TB and
1652 repeating the fault, which is horribly inefficient.
1653 Better would be to execute just this insn uncached, or generate a
1655 cpu_loop_exit_noexc(cpu
);
1658 void tb_flush_jmp_cache(CPUState
*cpu
, target_ulong addr
)
1662 /* Discard jump cache entries for any tb which might potentially
1663 overlap the flushed page. */
1664 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1665 memset(&cpu
->tb_jmp_cache
[i
], 0,
1666 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1668 i
= tb_jmp_cache_hash_page(addr
);
1669 memset(&cpu
->tb_jmp_cache
[i
], 0,
1670 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1673 static void print_qht_statistics(FILE *f
, fprintf_function cpu_fprintf
,
1674 struct qht_stats hst
)
1676 uint32_t hgram_opts
;
1680 if (!hst
.head_buckets
) {
1683 cpu_fprintf(f
, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n",
1684 hst
.used_head_buckets
, hst
.head_buckets
,
1685 (double)hst
.used_head_buckets
/ hst
.head_buckets
* 100);
1687 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
1688 hgram_opts
|= QDIST_PR_100X
| QDIST_PR_PERCENT
;
1689 if (qdist_xmax(&hst
.occupancy
) - qdist_xmin(&hst
.occupancy
) == 1) {
1690 hgram_opts
|= QDIST_PR_NODECIMAL
;
1692 hgram
= qdist_pr(&hst
.occupancy
, 10, hgram_opts
);
1693 cpu_fprintf(f
, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n",
1694 qdist_avg(&hst
.occupancy
) * 100, hgram
);
1697 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
1698 hgram_bins
= qdist_xmax(&hst
.chain
) - qdist_xmin(&hst
.chain
);
1699 if (hgram_bins
> 10) {
1703 hgram_opts
|= QDIST_PR_NODECIMAL
| QDIST_PR_NOBINRANGE
;
1705 hgram
= qdist_pr(&hst
.chain
, hgram_bins
, hgram_opts
);
1706 cpu_fprintf(f
, "TB hash avg chain %0.3f buckets. Histogram: %s\n",
1707 qdist_avg(&hst
.chain
), hgram
);
1711 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
1713 int i
, target_code_size
, max_target_code_size
;
1714 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
1715 TranslationBlock
*tb
;
1716 struct qht_stats hst
;
1718 target_code_size
= 0;
1719 max_target_code_size
= 0;
1721 direct_jmp_count
= 0;
1722 direct_jmp2_count
= 0;
1723 for (i
= 0; i
< tcg_ctx
.tb_ctx
.nb_tbs
; i
++) {
1724 tb
= &tcg_ctx
.tb_ctx
.tbs
[i
];
1725 target_code_size
+= tb
->size
;
1726 if (tb
->size
> max_target_code_size
) {
1727 max_target_code_size
= tb
->size
;
1729 if (tb
->page_addr
[1] != -1) {
1732 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1734 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1735 direct_jmp2_count
++;
1739 /* XXX: avoid using doubles ? */
1740 cpu_fprintf(f
, "Translation buffer state:\n");
1741 cpu_fprintf(f
, "gen code size %td/%zd\n",
1742 tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
,
1743 tcg_ctx
.code_gen_highwater
- tcg_ctx
.code_gen_buffer
);
1744 cpu_fprintf(f
, "TB count %d/%d\n",
1745 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.code_gen_max_blocks
);
1746 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
1747 tcg_ctx
.tb_ctx
.nb_tbs
? target_code_size
/
1748 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1749 max_target_code_size
);
1750 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1751 tcg_ctx
.tb_ctx
.nb_tbs
? (tcg_ctx
.code_gen_ptr
-
1752 tcg_ctx
.code_gen_buffer
) /
1753 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1754 target_code_size
? (double) (tcg_ctx
.code_gen_ptr
-
1755 tcg_ctx
.code_gen_buffer
) /
1756 target_code_size
: 0);
1757 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n", cross_page
,
1758 tcg_ctx
.tb_ctx
.nb_tbs
? (cross_page
* 100) /
1759 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1760 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1762 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp_count
* 100) /
1763 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1765 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp2_count
* 100) /
1766 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1768 qht_statistics_init(&tcg_ctx
.tb_ctx
.htable
, &hst
);
1769 print_qht_statistics(f
, cpu_fprintf
, hst
);
1770 qht_statistics_destroy(&hst
);
1772 cpu_fprintf(f
, "\nStatistics:\n");
1773 cpu_fprintf(f
, "TB flush count %d\n", tcg_ctx
.tb_ctx
.tb_flush_count
);
1774 cpu_fprintf(f
, "TB invalidate count %d\n",
1775 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
);
1776 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
1777 tcg_dump_info(f
, cpu_fprintf
);
1780 void dump_opcount_info(FILE *f
, fprintf_function cpu_fprintf
)
1782 tcg_dump_op_count(f
, cpu_fprintf
);
1785 #else /* CONFIG_USER_ONLY */
1787 void cpu_interrupt(CPUState
*cpu
, int mask
)
1789 cpu
->interrupt_request
|= mask
;
1790 cpu
->tcg_exit_req
= 1;
1794 * Walks guest process memory "regions" one by one
1795 * and calls callback function 'fn' for each region.
1797 struct walk_memory_regions_data
{
1798 walk_memory_regions_fn fn
;
1804 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
1805 target_ulong end
, int new_prot
)
1807 if (data
->start
!= -1u) {
1808 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
1814 data
->start
= (new_prot
? end
: -1u);
1815 data
->prot
= new_prot
;
1820 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
1821 target_ulong base
, int level
, void **lp
)
1827 return walk_memory_regions_end(data
, base
, 0);
1833 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1834 int prot
= pd
[i
].flags
;
1836 pa
= base
| (i
<< TARGET_PAGE_BITS
);
1837 if (prot
!= data
->prot
) {
1838 rc
= walk_memory_regions_end(data
, pa
, prot
);
1847 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1848 pa
= base
| ((target_ulong
)i
<<
1849 (TARGET_PAGE_BITS
+ V_L2_BITS
* level
));
1850 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
1860 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
1862 struct walk_memory_regions_data data
;
1870 for (i
= 0; i
< V_L1_SIZE
; i
++) {
1871 int rc
= walk_memory_regions_1(&data
, (target_ulong
)i
<< (V_L1_SHIFT
+ TARGET_PAGE_BITS
),
1872 V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
1878 return walk_memory_regions_end(&data
, 0, 0);
1881 static int dump_region(void *priv
, target_ulong start
,
1882 target_ulong end
, unsigned long prot
)
1884 FILE *f
= (FILE *)priv
;
1886 (void) fprintf(f
, TARGET_FMT_lx
"-"TARGET_FMT_lx
1887 " "TARGET_FMT_lx
" %c%c%c\n",
1888 start
, end
, end
- start
,
1889 ((prot
& PAGE_READ
) ? 'r' : '-'),
1890 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
1891 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
1896 /* dump memory mappings */
1897 void page_dump(FILE *f
)
1899 const int length
= sizeof(target_ulong
) * 2;
1900 (void) fprintf(f
, "%-*s %-*s %-*s %s\n",
1901 length
, "start", length
, "end", length
, "size", "prot");
1902 walk_memory_regions(f
, dump_region
);
1905 int page_get_flags(target_ulong address
)
1909 p
= page_find(address
>> TARGET_PAGE_BITS
);
1916 /* Modify the flags of a page and invalidate the code if necessary.
1917 The flag PAGE_WRITE_ORG is positioned automatically depending
1918 on PAGE_WRITE. The mmap_lock should already be held. */
1919 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
1921 target_ulong addr
, len
;
1923 /* This function should never be called with addresses outside the
1924 guest address space. If this assert fires, it probably indicates
1925 a missing call to h2g_valid. */
1926 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1927 assert(end
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1929 assert(start
< end
);
1931 start
= start
& TARGET_PAGE_MASK
;
1932 end
= TARGET_PAGE_ALIGN(end
);
1934 if (flags
& PAGE_WRITE
) {
1935 flags
|= PAGE_WRITE_ORG
;
1938 for (addr
= start
, len
= end
- start
;
1940 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1941 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
1943 /* If the write protection bit is set, then we invalidate
1945 if (!(p
->flags
& PAGE_WRITE
) &&
1946 (flags
& PAGE_WRITE
) &&
1948 tb_invalidate_phys_page(addr
, 0);
1954 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
1960 /* This function should never be called with addresses outside the
1961 guest address space. If this assert fires, it probably indicates
1962 a missing call to h2g_valid. */
1963 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1964 assert(start
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1970 if (start
+ len
- 1 < start
) {
1971 /* We've wrapped around. */
1975 /* must do before we loose bits in the next step */
1976 end
= TARGET_PAGE_ALIGN(start
+ len
);
1977 start
= start
& TARGET_PAGE_MASK
;
1979 for (addr
= start
, len
= end
- start
;
1981 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1982 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1986 if (!(p
->flags
& PAGE_VALID
)) {
1990 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
)) {
1993 if (flags
& PAGE_WRITE
) {
1994 if (!(p
->flags
& PAGE_WRITE_ORG
)) {
1997 /* unprotect the page if it was put read-only because it
1998 contains translated code */
1999 if (!(p
->flags
& PAGE_WRITE
)) {
2000 if (!page_unprotect(addr
, 0)) {
2009 /* called from signal handler: invalidate the code and unprotect the
2010 * page. Return 0 if the fault was not handled, 1 if it was handled,
2011 * and 2 if it was handled but the caller must cause the TB to be
2012 * immediately exited. (We can only return 2 if the 'pc' argument is
2015 int page_unprotect(target_ulong address
, uintptr_t pc
)
2018 bool current_tb_invalidated
;
2020 target_ulong host_start
, host_end
, addr
;
2022 /* Technically this isn't safe inside a signal handler. However we
2023 know this only ever happens in a synchronous SEGV handler, so in
2024 practice it seems to be ok. */
2027 p
= page_find(address
>> TARGET_PAGE_BITS
);
2033 /* if the page was really writable, then we change its
2034 protection back to writable */
2035 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
2036 host_start
= address
& qemu_host_page_mask
;
2037 host_end
= host_start
+ qemu_host_page_size
;
2040 current_tb_invalidated
= false;
2041 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2042 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2043 p
->flags
|= PAGE_WRITE
;
2046 /* and since the content will be modified, we must invalidate
2047 the corresponding translated code. */
2048 current_tb_invalidated
|= tb_invalidate_phys_page(addr
, pc
);
2049 #ifdef DEBUG_TB_CHECK
2050 tb_invalidate_check(addr
);
2053 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2057 /* If current TB was invalidated return to main loop */
2058 return current_tb_invalidated
? 2 : 1;
2063 #endif /* CONFIG_USER_ONLY */