]> git.proxmox.com Git - mirror_qemu.git/blob - translate-all.c
target-ppc: VSX Stage 4: Refactor lxsdx
[mirror_qemu.git] / translate-all.c
1 /*
2 * Host code generation
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #ifdef _WIN32
20 #include <windows.h>
21 #else
22 #include <sys/types.h>
23 #include <sys/mman.h>
24 #endif
25 #include <stdarg.h>
26 #include <stdlib.h>
27 #include <stdio.h>
28 #include <string.h>
29 #include <inttypes.h>
30
31 #include "config.h"
32
33 #include "qemu-common.h"
34 #define NO_CPU_IO_DEFS
35 #include "cpu.h"
36 #include "disas/disas.h"
37 #include "tcg.h"
38 #if defined(CONFIG_USER_ONLY)
39 #include "qemu.h"
40 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
41 #include <sys/param.h>
42 #if __FreeBSD_version >= 700104
43 #define HAVE_KINFO_GETVMMAP
44 #define sigqueue sigqueue_freebsd /* avoid redefinition */
45 #include <sys/time.h>
46 #include <sys/proc.h>
47 #include <machine/profile.h>
48 #define _KERNEL
49 #include <sys/user.h>
50 #undef _KERNEL
51 #undef sigqueue
52 #include <libutil.h>
53 #endif
54 #endif
55 #else
56 #include "exec/address-spaces.h"
57 #endif
58
59 #include "exec/cputlb.h"
60 #include "translate-all.h"
61 #include "qemu/timer.h"
62
63 //#define DEBUG_TB_INVALIDATE
64 //#define DEBUG_FLUSH
65 /* make various TB consistency checks */
66 //#define DEBUG_TB_CHECK
67
68 #if !defined(CONFIG_USER_ONLY)
69 /* TB consistency checks only implemented for usermode emulation. */
70 #undef DEBUG_TB_CHECK
71 #endif
72
73 #define SMC_BITMAP_USE_THRESHOLD 10
74
75 typedef struct PageDesc {
76 /* list of TBs intersecting this ram page */
77 TranslationBlock *first_tb;
78 /* in order to optimize self modifying code, we count the number
79 of lookups we do to a given page to use a bitmap */
80 unsigned int code_write_count;
81 uint8_t *code_bitmap;
82 #if defined(CONFIG_USER_ONLY)
83 unsigned long flags;
84 #endif
85 } PageDesc;
86
87 /* In system mode we want L1_MAP to be based on ram offsets,
88 while in user mode we want it to be based on virtual addresses. */
89 #if !defined(CONFIG_USER_ONLY)
90 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
91 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
92 #else
93 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
94 #endif
95 #else
96 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
97 #endif
98
99 /* Size of the L2 (and L3, etc) page tables. */
100 #define V_L2_BITS 10
101 #define V_L2_SIZE (1 << V_L2_BITS)
102
103 /* The bits remaining after N lower levels of page tables. */
104 #define V_L1_BITS_REM \
105 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
106
107 #if V_L1_BITS_REM < 4
108 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
109 #else
110 #define V_L1_BITS V_L1_BITS_REM
111 #endif
112
113 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
114
115 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
116
117 uintptr_t qemu_real_host_page_size;
118 uintptr_t qemu_host_page_size;
119 uintptr_t qemu_host_page_mask;
120
121 /* This is a multi-level map on the virtual address space.
122 The bottom level has pointers to PageDesc. */
123 static void *l1_map[V_L1_SIZE];
124
125 /* code generation context */
126 TCGContext tcg_ctx;
127
128 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
129 tb_page_addr_t phys_page2);
130 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
131
132 void cpu_gen_init(void)
133 {
134 tcg_context_init(&tcg_ctx);
135 }
136
137 /* return non zero if the very first instruction is invalid so that
138 the virtual CPU can trigger an exception.
139
140 '*gen_code_size_ptr' contains the size of the generated code (host
141 code).
142 */
143 int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr)
144 {
145 TCGContext *s = &tcg_ctx;
146 uint8_t *gen_code_buf;
147 int gen_code_size;
148 #ifdef CONFIG_PROFILER
149 int64_t ti;
150 #endif
151
152 #ifdef CONFIG_PROFILER
153 s->tb_count1++; /* includes aborted translations because of
154 exceptions */
155 ti = profile_getclock();
156 #endif
157 tcg_func_start(s);
158
159 gen_intermediate_code(env, tb);
160
161 /* generate machine code */
162 gen_code_buf = tb->tc_ptr;
163 tb->tb_next_offset[0] = 0xffff;
164 tb->tb_next_offset[1] = 0xffff;
165 s->tb_next_offset = tb->tb_next_offset;
166 #ifdef USE_DIRECT_JUMP
167 s->tb_jmp_offset = tb->tb_jmp_offset;
168 s->tb_next = NULL;
169 #else
170 s->tb_jmp_offset = NULL;
171 s->tb_next = tb->tb_next;
172 #endif
173
174 #ifdef CONFIG_PROFILER
175 s->tb_count++;
176 s->interm_time += profile_getclock() - ti;
177 s->code_time -= profile_getclock();
178 #endif
179 gen_code_size = tcg_gen_code(s, gen_code_buf);
180 *gen_code_size_ptr = gen_code_size;
181 #ifdef CONFIG_PROFILER
182 s->code_time += profile_getclock();
183 s->code_in_len += tb->size;
184 s->code_out_len += gen_code_size;
185 #endif
186
187 #ifdef DEBUG_DISAS
188 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
189 qemu_log("OUT: [size=%d]\n", *gen_code_size_ptr);
190 log_disas(tb->tc_ptr, *gen_code_size_ptr);
191 qemu_log("\n");
192 qemu_log_flush();
193 }
194 #endif
195 return 0;
196 }
197
198 /* The cpu state corresponding to 'searched_pc' is restored.
199 */
200 static int cpu_restore_state_from_tb(TranslationBlock *tb, CPUArchState *env,
201 uintptr_t searched_pc)
202 {
203 TCGContext *s = &tcg_ctx;
204 int j;
205 uintptr_t tc_ptr;
206 #ifdef CONFIG_PROFILER
207 int64_t ti;
208 #endif
209
210 #ifdef CONFIG_PROFILER
211 ti = profile_getclock();
212 #endif
213 tcg_func_start(s);
214
215 gen_intermediate_code_pc(env, tb);
216
217 if (use_icount) {
218 /* Reset the cycle counter to the start of the block. */
219 env->icount_decr.u16.low += tb->icount;
220 /* Clear the IO flag. */
221 env->can_do_io = 0;
222 }
223
224 /* find opc index corresponding to search_pc */
225 tc_ptr = (uintptr_t)tb->tc_ptr;
226 if (searched_pc < tc_ptr)
227 return -1;
228
229 s->tb_next_offset = tb->tb_next_offset;
230 #ifdef USE_DIRECT_JUMP
231 s->tb_jmp_offset = tb->tb_jmp_offset;
232 s->tb_next = NULL;
233 #else
234 s->tb_jmp_offset = NULL;
235 s->tb_next = tb->tb_next;
236 #endif
237 j = tcg_gen_code_search_pc(s, (uint8_t *)tc_ptr, searched_pc - tc_ptr);
238 if (j < 0)
239 return -1;
240 /* now find start of instruction before */
241 while (s->gen_opc_instr_start[j] == 0) {
242 j--;
243 }
244 env->icount_decr.u16.low -= s->gen_opc_icount[j];
245
246 restore_state_to_opc(env, tb, j);
247
248 #ifdef CONFIG_PROFILER
249 s->restore_time += profile_getclock() - ti;
250 s->restore_count++;
251 #endif
252 return 0;
253 }
254
255 bool cpu_restore_state(CPUArchState *env, uintptr_t retaddr)
256 {
257 TranslationBlock *tb;
258
259 tb = tb_find_pc(retaddr);
260 if (tb) {
261 cpu_restore_state_from_tb(tb, env, retaddr);
262 return true;
263 }
264 return false;
265 }
266
267 #ifdef _WIN32
268 static inline void map_exec(void *addr, long size)
269 {
270 DWORD old_protect;
271 VirtualProtect(addr, size,
272 PAGE_EXECUTE_READWRITE, &old_protect);
273 }
274 #else
275 static inline void map_exec(void *addr, long size)
276 {
277 unsigned long start, end, page_size;
278
279 page_size = getpagesize();
280 start = (unsigned long)addr;
281 start &= ~(page_size - 1);
282
283 end = (unsigned long)addr + size;
284 end += page_size - 1;
285 end &= ~(page_size - 1);
286
287 mprotect((void *)start, end - start,
288 PROT_READ | PROT_WRITE | PROT_EXEC);
289 }
290 #endif
291
292 void page_size_init(void)
293 {
294 /* NOTE: we can always suppose that qemu_host_page_size >=
295 TARGET_PAGE_SIZE */
296 #ifdef _WIN32
297 SYSTEM_INFO system_info;
298
299 GetSystemInfo(&system_info);
300 qemu_real_host_page_size = system_info.dwPageSize;
301 #else
302 qemu_real_host_page_size = getpagesize();
303 #endif
304 if (qemu_host_page_size == 0) {
305 qemu_host_page_size = qemu_real_host_page_size;
306 }
307 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
308 qemu_host_page_size = TARGET_PAGE_SIZE;
309 }
310 qemu_host_page_mask = ~(qemu_host_page_size - 1);
311 }
312
313 static void page_init(void)
314 {
315 page_size_init();
316 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
317 {
318 #ifdef HAVE_KINFO_GETVMMAP
319 struct kinfo_vmentry *freep;
320 int i, cnt;
321
322 freep = kinfo_getvmmap(getpid(), &cnt);
323 if (freep) {
324 mmap_lock();
325 for (i = 0; i < cnt; i++) {
326 unsigned long startaddr, endaddr;
327
328 startaddr = freep[i].kve_start;
329 endaddr = freep[i].kve_end;
330 if (h2g_valid(startaddr)) {
331 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
332
333 if (h2g_valid(endaddr)) {
334 endaddr = h2g(endaddr);
335 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
336 } else {
337 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
338 endaddr = ~0ul;
339 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
340 #endif
341 }
342 }
343 }
344 free(freep);
345 mmap_unlock();
346 }
347 #else
348 FILE *f;
349
350 last_brk = (unsigned long)sbrk(0);
351
352 f = fopen("/compat/linux/proc/self/maps", "r");
353 if (f) {
354 mmap_lock();
355
356 do {
357 unsigned long startaddr, endaddr;
358 int n;
359
360 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
361
362 if (n == 2 && h2g_valid(startaddr)) {
363 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
364
365 if (h2g_valid(endaddr)) {
366 endaddr = h2g(endaddr);
367 } else {
368 endaddr = ~0ul;
369 }
370 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
371 }
372 } while (!feof(f));
373
374 fclose(f);
375 mmap_unlock();
376 }
377 #endif
378 }
379 #endif
380 }
381
382 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
383 {
384 PageDesc *pd;
385 void **lp;
386 int i;
387
388 #if defined(CONFIG_USER_ONLY)
389 /* We can't use g_malloc because it may recurse into a locked mutex. */
390 # define ALLOC(P, SIZE) \
391 do { \
392 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
393 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
394 } while (0)
395 #else
396 # define ALLOC(P, SIZE) \
397 do { P = g_malloc0(SIZE); } while (0)
398 #endif
399
400 /* Level 1. Always allocated. */
401 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
402
403 /* Level 2..N-1. */
404 for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
405 void **p = *lp;
406
407 if (p == NULL) {
408 if (!alloc) {
409 return NULL;
410 }
411 ALLOC(p, sizeof(void *) * V_L2_SIZE);
412 *lp = p;
413 }
414
415 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
416 }
417
418 pd = *lp;
419 if (pd == NULL) {
420 if (!alloc) {
421 return NULL;
422 }
423 ALLOC(pd, sizeof(PageDesc) * V_L2_SIZE);
424 *lp = pd;
425 }
426
427 #undef ALLOC
428
429 return pd + (index & (V_L2_SIZE - 1));
430 }
431
432 static inline PageDesc *page_find(tb_page_addr_t index)
433 {
434 return page_find_alloc(index, 0);
435 }
436
437 #if !defined(CONFIG_USER_ONLY)
438 #define mmap_lock() do { } while (0)
439 #define mmap_unlock() do { } while (0)
440 #endif
441
442 #if defined(CONFIG_USER_ONLY)
443 /* Currently it is not recommended to allocate big chunks of data in
444 user mode. It will change when a dedicated libc will be used. */
445 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
446 region in which the guest needs to run. Revisit this. */
447 #define USE_STATIC_CODE_GEN_BUFFER
448 #endif
449
450 /* ??? Should configure for this, not list operating systems here. */
451 #if (defined(__linux__) \
452 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
453 || defined(__DragonFly__) || defined(__OpenBSD__) \
454 || defined(__NetBSD__))
455 # define USE_MMAP
456 #endif
457
458 /* Minimum size of the code gen buffer. This number is randomly chosen,
459 but not so small that we can't have a fair number of TB's live. */
460 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
461
462 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
463 indicated, this is constrained by the range of direct branches on the
464 host cpu, as used by the TCG implementation of goto_tb. */
465 #if defined(__x86_64__)
466 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
467 #elif defined(__sparc__)
468 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
469 #elif defined(__aarch64__)
470 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
471 #elif defined(__arm__)
472 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
473 #elif defined(__s390x__)
474 /* We have a +- 4GB range on the branches; leave some slop. */
475 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
476 #else
477 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
478 #endif
479
480 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
481
482 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
483 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
484 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
485
486 static inline size_t size_code_gen_buffer(size_t tb_size)
487 {
488 /* Size the buffer. */
489 if (tb_size == 0) {
490 #ifdef USE_STATIC_CODE_GEN_BUFFER
491 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
492 #else
493 /* ??? Needs adjustments. */
494 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
495 static buffer, we could size this on RESERVED_VA, on the text
496 segment size of the executable, or continue to use the default. */
497 tb_size = (unsigned long)(ram_size / 4);
498 #endif
499 }
500 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
501 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
502 }
503 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
504 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
505 }
506 tcg_ctx.code_gen_buffer_size = tb_size;
507 return tb_size;
508 }
509
510 #ifdef USE_STATIC_CODE_GEN_BUFFER
511 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
512 __attribute__((aligned(CODE_GEN_ALIGN)));
513
514 static inline void *alloc_code_gen_buffer(void)
515 {
516 map_exec(static_code_gen_buffer, tcg_ctx.code_gen_buffer_size);
517 return static_code_gen_buffer;
518 }
519 #elif defined(USE_MMAP)
520 static inline void *alloc_code_gen_buffer(void)
521 {
522 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
523 uintptr_t start = 0;
524 void *buf;
525
526 /* Constrain the position of the buffer based on the host cpu.
527 Note that these addresses are chosen in concert with the
528 addresses assigned in the relevant linker script file. */
529 # if defined(__PIE__) || defined(__PIC__)
530 /* Don't bother setting a preferred location if we're building
531 a position-independent executable. We're more likely to get
532 an address near the main executable if we let the kernel
533 choose the address. */
534 # elif defined(__x86_64__) && defined(MAP_32BIT)
535 /* Force the memory down into low memory with the executable.
536 Leave the choice of exact location with the kernel. */
537 flags |= MAP_32BIT;
538 /* Cannot expect to map more than 800MB in low memory. */
539 if (tcg_ctx.code_gen_buffer_size > 800u * 1024 * 1024) {
540 tcg_ctx.code_gen_buffer_size = 800u * 1024 * 1024;
541 }
542 # elif defined(__sparc__)
543 start = 0x40000000ul;
544 # elif defined(__s390x__)
545 start = 0x90000000ul;
546 # endif
547
548 buf = mmap((void *)start, tcg_ctx.code_gen_buffer_size,
549 PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
550 return buf == MAP_FAILED ? NULL : buf;
551 }
552 #else
553 static inline void *alloc_code_gen_buffer(void)
554 {
555 void *buf = g_malloc(tcg_ctx.code_gen_buffer_size);
556
557 if (buf) {
558 map_exec(buf, tcg_ctx.code_gen_buffer_size);
559 }
560 return buf;
561 }
562 #endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
563
564 static inline void code_gen_alloc(size_t tb_size)
565 {
566 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size);
567 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer();
568 if (tcg_ctx.code_gen_buffer == NULL) {
569 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
570 exit(1);
571 }
572
573 qemu_madvise(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size,
574 QEMU_MADV_HUGEPAGE);
575
576 /* Steal room for the prologue at the end of the buffer. This ensures
577 (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
578 from TB's to the prologue are going to be in range. It also means
579 that we don't need to mark (additional) portions of the data segment
580 as executable. */
581 tcg_ctx.code_gen_prologue = tcg_ctx.code_gen_buffer +
582 tcg_ctx.code_gen_buffer_size - 1024;
583 tcg_ctx.code_gen_buffer_size -= 1024;
584
585 tcg_ctx.code_gen_buffer_max_size = tcg_ctx.code_gen_buffer_size -
586 (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
587 tcg_ctx.code_gen_max_blocks = tcg_ctx.code_gen_buffer_size /
588 CODE_GEN_AVG_BLOCK_SIZE;
589 tcg_ctx.tb_ctx.tbs =
590 g_malloc(tcg_ctx.code_gen_max_blocks * sizeof(TranslationBlock));
591 }
592
593 /* Must be called before using the QEMU cpus. 'tb_size' is the size
594 (in bytes) allocated to the translation buffer. Zero means default
595 size. */
596 void tcg_exec_init(unsigned long tb_size)
597 {
598 cpu_gen_init();
599 code_gen_alloc(tb_size);
600 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
601 tcg_register_jit(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size);
602 page_init();
603 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
604 /* There's no guest base to take into account, so go ahead and
605 initialize the prologue now. */
606 tcg_prologue_init(&tcg_ctx);
607 #endif
608 }
609
610 bool tcg_enabled(void)
611 {
612 return tcg_ctx.code_gen_buffer != NULL;
613 }
614
615 /* Allocate a new translation block. Flush the translation buffer if
616 too many translation blocks or too much generated code. */
617 static TranslationBlock *tb_alloc(target_ulong pc)
618 {
619 TranslationBlock *tb;
620
621 if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks ||
622 (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) >=
623 tcg_ctx.code_gen_buffer_max_size) {
624 return NULL;
625 }
626 tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
627 tb->pc = pc;
628 tb->cflags = 0;
629 return tb;
630 }
631
632 void tb_free(TranslationBlock *tb)
633 {
634 /* In practice this is mostly used for single use temporary TB
635 Ignore the hard cases and just back up if this TB happens to
636 be the last one generated. */
637 if (tcg_ctx.tb_ctx.nb_tbs > 0 &&
638 tb == &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) {
639 tcg_ctx.code_gen_ptr = tb->tc_ptr;
640 tcg_ctx.tb_ctx.nb_tbs--;
641 }
642 }
643
644 static inline void invalidate_page_bitmap(PageDesc *p)
645 {
646 if (p->code_bitmap) {
647 g_free(p->code_bitmap);
648 p->code_bitmap = NULL;
649 }
650 p->code_write_count = 0;
651 }
652
653 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
654 static void page_flush_tb_1(int level, void **lp)
655 {
656 int i;
657
658 if (*lp == NULL) {
659 return;
660 }
661 if (level == 0) {
662 PageDesc *pd = *lp;
663
664 for (i = 0; i < V_L2_SIZE; ++i) {
665 pd[i].first_tb = NULL;
666 invalidate_page_bitmap(pd + i);
667 }
668 } else {
669 void **pp = *lp;
670
671 for (i = 0; i < V_L2_SIZE; ++i) {
672 page_flush_tb_1(level - 1, pp + i);
673 }
674 }
675 }
676
677 static void page_flush_tb(void)
678 {
679 int i;
680
681 for (i = 0; i < V_L1_SIZE; i++) {
682 page_flush_tb_1(V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
683 }
684 }
685
686 /* flush all the translation blocks */
687 /* XXX: tb_flush is currently not thread safe */
688 void tb_flush(CPUArchState *env1)
689 {
690 CPUState *cpu;
691
692 #if defined(DEBUG_FLUSH)
693 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
694 (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
695 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
696 ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) /
697 tcg_ctx.tb_ctx.nb_tbs : 0);
698 #endif
699 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)
700 > tcg_ctx.code_gen_buffer_size) {
701 cpu_abort(env1, "Internal error: code buffer overflow\n");
702 }
703 tcg_ctx.tb_ctx.nb_tbs = 0;
704
705 CPU_FOREACH(cpu) {
706 CPUArchState *env = cpu->env_ptr;
707
708 memset(env->tb_jmp_cache, 0, sizeof(env->tb_jmp_cache));
709 }
710
711 memset(tcg_ctx.tb_ctx.tb_phys_hash, 0, sizeof(tcg_ctx.tb_ctx.tb_phys_hash));
712 page_flush_tb();
713
714 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
715 /* XXX: flush processor icache at this point if cache flush is
716 expensive */
717 tcg_ctx.tb_ctx.tb_flush_count++;
718 }
719
720 #ifdef DEBUG_TB_CHECK
721
722 static void tb_invalidate_check(target_ulong address)
723 {
724 TranslationBlock *tb;
725 int i;
726
727 address &= TARGET_PAGE_MASK;
728 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
729 for (tb = tb_ctx.tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
730 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
731 address >= tb->pc + tb->size)) {
732 printf("ERROR invalidate: address=" TARGET_FMT_lx
733 " PC=%08lx size=%04x\n",
734 address, (long)tb->pc, tb->size);
735 }
736 }
737 }
738 }
739
740 /* verify that all the pages have correct rights for code */
741 static void tb_page_check(void)
742 {
743 TranslationBlock *tb;
744 int i, flags1, flags2;
745
746 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
747 for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
748 tb = tb->phys_hash_next) {
749 flags1 = page_get_flags(tb->pc);
750 flags2 = page_get_flags(tb->pc + tb->size - 1);
751 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
752 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
753 (long)tb->pc, tb->size, flags1, flags2);
754 }
755 }
756 }
757 }
758
759 #endif
760
761 static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
762 {
763 TranslationBlock *tb1;
764
765 for (;;) {
766 tb1 = *ptb;
767 if (tb1 == tb) {
768 *ptb = tb1->phys_hash_next;
769 break;
770 }
771 ptb = &tb1->phys_hash_next;
772 }
773 }
774
775 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
776 {
777 TranslationBlock *tb1;
778 unsigned int n1;
779
780 for (;;) {
781 tb1 = *ptb;
782 n1 = (uintptr_t)tb1 & 3;
783 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
784 if (tb1 == tb) {
785 *ptb = tb1->page_next[n1];
786 break;
787 }
788 ptb = &tb1->page_next[n1];
789 }
790 }
791
792 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
793 {
794 TranslationBlock *tb1, **ptb;
795 unsigned int n1;
796
797 ptb = &tb->jmp_next[n];
798 tb1 = *ptb;
799 if (tb1) {
800 /* find tb(n) in circular list */
801 for (;;) {
802 tb1 = *ptb;
803 n1 = (uintptr_t)tb1 & 3;
804 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
805 if (n1 == n && tb1 == tb) {
806 break;
807 }
808 if (n1 == 2) {
809 ptb = &tb1->jmp_first;
810 } else {
811 ptb = &tb1->jmp_next[n1];
812 }
813 }
814 /* now we can suppress tb(n) from the list */
815 *ptb = tb->jmp_next[n];
816
817 tb->jmp_next[n] = NULL;
818 }
819 }
820
821 /* reset the jump entry 'n' of a TB so that it is not chained to
822 another TB */
823 static inline void tb_reset_jump(TranslationBlock *tb, int n)
824 {
825 tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
826 }
827
828 /* invalidate one TB */
829 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
830 {
831 CPUState *cpu;
832 PageDesc *p;
833 unsigned int h, n1;
834 tb_page_addr_t phys_pc;
835 TranslationBlock *tb1, *tb2;
836
837 /* remove the TB from the hash list */
838 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
839 h = tb_phys_hash_func(phys_pc);
840 tb_hash_remove(&tcg_ctx.tb_ctx.tb_phys_hash[h], tb);
841
842 /* remove the TB from the page list */
843 if (tb->page_addr[0] != page_addr) {
844 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
845 tb_page_remove(&p->first_tb, tb);
846 invalidate_page_bitmap(p);
847 }
848 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
849 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
850 tb_page_remove(&p->first_tb, tb);
851 invalidate_page_bitmap(p);
852 }
853
854 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
855
856 /* remove the TB from the hash list */
857 h = tb_jmp_cache_hash_func(tb->pc);
858 CPU_FOREACH(cpu) {
859 CPUArchState *env = cpu->env_ptr;
860
861 if (env->tb_jmp_cache[h] == tb) {
862 env->tb_jmp_cache[h] = NULL;
863 }
864 }
865
866 /* suppress this TB from the two jump lists */
867 tb_jmp_remove(tb, 0);
868 tb_jmp_remove(tb, 1);
869
870 /* suppress any remaining jumps to this TB */
871 tb1 = tb->jmp_first;
872 for (;;) {
873 n1 = (uintptr_t)tb1 & 3;
874 if (n1 == 2) {
875 break;
876 }
877 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
878 tb2 = tb1->jmp_next[n1];
879 tb_reset_jump(tb1, n1);
880 tb1->jmp_next[n1] = NULL;
881 tb1 = tb2;
882 }
883 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
884
885 tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
886 }
887
888 static inline void set_bits(uint8_t *tab, int start, int len)
889 {
890 int end, mask, end1;
891
892 end = start + len;
893 tab += start >> 3;
894 mask = 0xff << (start & 7);
895 if ((start & ~7) == (end & ~7)) {
896 if (start < end) {
897 mask &= ~(0xff << (end & 7));
898 *tab |= mask;
899 }
900 } else {
901 *tab++ |= mask;
902 start = (start + 8) & ~7;
903 end1 = end & ~7;
904 while (start < end1) {
905 *tab++ = 0xff;
906 start += 8;
907 }
908 if (start < end) {
909 mask = ~(0xff << (end & 7));
910 *tab |= mask;
911 }
912 }
913 }
914
915 static void build_page_bitmap(PageDesc *p)
916 {
917 int n, tb_start, tb_end;
918 TranslationBlock *tb;
919
920 p->code_bitmap = g_malloc0(TARGET_PAGE_SIZE / 8);
921
922 tb = p->first_tb;
923 while (tb != NULL) {
924 n = (uintptr_t)tb & 3;
925 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
926 /* NOTE: this is subtle as a TB may span two physical pages */
927 if (n == 0) {
928 /* NOTE: tb_end may be after the end of the page, but
929 it is not a problem */
930 tb_start = tb->pc & ~TARGET_PAGE_MASK;
931 tb_end = tb_start + tb->size;
932 if (tb_end > TARGET_PAGE_SIZE) {
933 tb_end = TARGET_PAGE_SIZE;
934 }
935 } else {
936 tb_start = 0;
937 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
938 }
939 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
940 tb = tb->page_next[n];
941 }
942 }
943
944 TranslationBlock *tb_gen_code(CPUArchState *env,
945 target_ulong pc, target_ulong cs_base,
946 int flags, int cflags)
947 {
948 TranslationBlock *tb;
949 uint8_t *tc_ptr;
950 tb_page_addr_t phys_pc, phys_page2;
951 target_ulong virt_page2;
952 int code_gen_size;
953
954 phys_pc = get_page_addr_code(env, pc);
955 tb = tb_alloc(pc);
956 if (!tb) {
957 /* flush must be done */
958 tb_flush(env);
959 /* cannot fail at this point */
960 tb = tb_alloc(pc);
961 /* Don't forget to invalidate previous TB info. */
962 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
963 }
964 tc_ptr = tcg_ctx.code_gen_ptr;
965 tb->tc_ptr = tc_ptr;
966 tb->cs_base = cs_base;
967 tb->flags = flags;
968 tb->cflags = cflags;
969 cpu_gen_code(env, tb, &code_gen_size);
970 tcg_ctx.code_gen_ptr = (void *)(((uintptr_t)tcg_ctx.code_gen_ptr +
971 code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
972
973 /* check next page if needed */
974 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
975 phys_page2 = -1;
976 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
977 phys_page2 = get_page_addr_code(env, virt_page2);
978 }
979 tb_link_page(tb, phys_pc, phys_page2);
980 return tb;
981 }
982
983 /*
984 * Invalidate all TBs which intersect with the target physical address range
985 * [start;end[. NOTE: start and end may refer to *different* physical pages.
986 * 'is_cpu_write_access' should be true if called from a real cpu write
987 * access: the virtual CPU will exit the current TB if code is modified inside
988 * this TB.
989 */
990 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end,
991 int is_cpu_write_access)
992 {
993 while (start < end) {
994 tb_invalidate_phys_page_range(start, end, is_cpu_write_access);
995 start &= TARGET_PAGE_MASK;
996 start += TARGET_PAGE_SIZE;
997 }
998 }
999
1000 /*
1001 * Invalidate all TBs which intersect with the target physical address range
1002 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1003 * 'is_cpu_write_access' should be true if called from a real cpu write
1004 * access: the virtual CPU will exit the current TB if code is modified inside
1005 * this TB.
1006 */
1007 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1008 int is_cpu_write_access)
1009 {
1010 TranslationBlock *tb, *tb_next, *saved_tb;
1011 CPUState *cpu = current_cpu;
1012 #if defined(TARGET_HAS_PRECISE_SMC) || !defined(CONFIG_USER_ONLY)
1013 CPUArchState *env = NULL;
1014 #endif
1015 tb_page_addr_t tb_start, tb_end;
1016 PageDesc *p;
1017 int n;
1018 #ifdef TARGET_HAS_PRECISE_SMC
1019 int current_tb_not_found = is_cpu_write_access;
1020 TranslationBlock *current_tb = NULL;
1021 int current_tb_modified = 0;
1022 target_ulong current_pc = 0;
1023 target_ulong current_cs_base = 0;
1024 int current_flags = 0;
1025 #endif /* TARGET_HAS_PRECISE_SMC */
1026
1027 p = page_find(start >> TARGET_PAGE_BITS);
1028 if (!p) {
1029 return;
1030 }
1031 if (!p->code_bitmap &&
1032 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1033 is_cpu_write_access) {
1034 /* build code bitmap */
1035 build_page_bitmap(p);
1036 }
1037 #if defined(TARGET_HAS_PRECISE_SMC) || !defined(CONFIG_USER_ONLY)
1038 if (cpu != NULL) {
1039 env = cpu->env_ptr;
1040 }
1041 #endif
1042
1043 /* we remove all the TBs in the range [start, end[ */
1044 /* XXX: see if in some cases it could be faster to invalidate all
1045 the code */
1046 tb = p->first_tb;
1047 while (tb != NULL) {
1048 n = (uintptr_t)tb & 3;
1049 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1050 tb_next = tb->page_next[n];
1051 /* NOTE: this is subtle as a TB may span two physical pages */
1052 if (n == 0) {
1053 /* NOTE: tb_end may be after the end of the page, but
1054 it is not a problem */
1055 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1056 tb_end = tb_start + tb->size;
1057 } else {
1058 tb_start = tb->page_addr[1];
1059 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1060 }
1061 if (!(tb_end <= start || tb_start >= end)) {
1062 #ifdef TARGET_HAS_PRECISE_SMC
1063 if (current_tb_not_found) {
1064 current_tb_not_found = 0;
1065 current_tb = NULL;
1066 if (env->mem_io_pc) {
1067 /* now we have a real cpu fault */
1068 current_tb = tb_find_pc(env->mem_io_pc);
1069 }
1070 }
1071 if (current_tb == tb &&
1072 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1073 /* If we are modifying the current TB, we must stop
1074 its execution. We could be more precise by checking
1075 that the modification is after the current PC, but it
1076 would require a specialized function to partially
1077 restore the CPU state */
1078
1079 current_tb_modified = 1;
1080 cpu_restore_state_from_tb(current_tb, env, env->mem_io_pc);
1081 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1082 &current_flags);
1083 }
1084 #endif /* TARGET_HAS_PRECISE_SMC */
1085 /* we need to do that to handle the case where a signal
1086 occurs while doing tb_phys_invalidate() */
1087 saved_tb = NULL;
1088 if (cpu != NULL) {
1089 saved_tb = cpu->current_tb;
1090 cpu->current_tb = NULL;
1091 }
1092 tb_phys_invalidate(tb, -1);
1093 if (cpu != NULL) {
1094 cpu->current_tb = saved_tb;
1095 if (cpu->interrupt_request && cpu->current_tb) {
1096 cpu_interrupt(cpu, cpu->interrupt_request);
1097 }
1098 }
1099 }
1100 tb = tb_next;
1101 }
1102 #if !defined(CONFIG_USER_ONLY)
1103 /* if no code remaining, no need to continue to use slow writes */
1104 if (!p->first_tb) {
1105 invalidate_page_bitmap(p);
1106 if (is_cpu_write_access) {
1107 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1108 }
1109 }
1110 #endif
1111 #ifdef TARGET_HAS_PRECISE_SMC
1112 if (current_tb_modified) {
1113 /* we generate a block containing just the instruction
1114 modifying the memory. It will ensure that it cannot modify
1115 itself */
1116 cpu->current_tb = NULL;
1117 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1118 cpu_resume_from_signal(env, NULL);
1119 }
1120 #endif
1121 }
1122
1123 /* len must be <= 8 and start must be a multiple of len */
1124 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1125 {
1126 PageDesc *p;
1127 int offset, b;
1128
1129 #if 0
1130 if (1) {
1131 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1132 cpu_single_env->mem_io_vaddr, len,
1133 cpu_single_env->eip,
1134 cpu_single_env->eip +
1135 (intptr_t)cpu_single_env->segs[R_CS].base);
1136 }
1137 #endif
1138 p = page_find(start >> TARGET_PAGE_BITS);
1139 if (!p) {
1140 return;
1141 }
1142 if (p->code_bitmap) {
1143 offset = start & ~TARGET_PAGE_MASK;
1144 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1145 if (b & ((1 << len) - 1)) {
1146 goto do_invalidate;
1147 }
1148 } else {
1149 do_invalidate:
1150 tb_invalidate_phys_page_range(start, start + len, 1);
1151 }
1152 }
1153
1154 #if !defined(CONFIG_SOFTMMU)
1155 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1156 uintptr_t pc, void *puc,
1157 bool locked)
1158 {
1159 TranslationBlock *tb;
1160 PageDesc *p;
1161 int n;
1162 #ifdef TARGET_HAS_PRECISE_SMC
1163 TranslationBlock *current_tb = NULL;
1164 CPUState *cpu = current_cpu;
1165 CPUArchState *env = NULL;
1166 int current_tb_modified = 0;
1167 target_ulong current_pc = 0;
1168 target_ulong current_cs_base = 0;
1169 int current_flags = 0;
1170 #endif
1171
1172 addr &= TARGET_PAGE_MASK;
1173 p = page_find(addr >> TARGET_PAGE_BITS);
1174 if (!p) {
1175 return;
1176 }
1177 tb = p->first_tb;
1178 #ifdef TARGET_HAS_PRECISE_SMC
1179 if (tb && pc != 0) {
1180 current_tb = tb_find_pc(pc);
1181 }
1182 if (cpu != NULL) {
1183 env = cpu->env_ptr;
1184 }
1185 #endif
1186 while (tb != NULL) {
1187 n = (uintptr_t)tb & 3;
1188 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1189 #ifdef TARGET_HAS_PRECISE_SMC
1190 if (current_tb == tb &&
1191 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1192 /* If we are modifying the current TB, we must stop
1193 its execution. We could be more precise by checking
1194 that the modification is after the current PC, but it
1195 would require a specialized function to partially
1196 restore the CPU state */
1197
1198 current_tb_modified = 1;
1199 cpu_restore_state_from_tb(current_tb, env, pc);
1200 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1201 &current_flags);
1202 }
1203 #endif /* TARGET_HAS_PRECISE_SMC */
1204 tb_phys_invalidate(tb, addr);
1205 tb = tb->page_next[n];
1206 }
1207 p->first_tb = NULL;
1208 #ifdef TARGET_HAS_PRECISE_SMC
1209 if (current_tb_modified) {
1210 /* we generate a block containing just the instruction
1211 modifying the memory. It will ensure that it cannot modify
1212 itself */
1213 cpu->current_tb = NULL;
1214 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1215 if (locked) {
1216 mmap_unlock();
1217 }
1218 cpu_resume_from_signal(env, puc);
1219 }
1220 #endif
1221 }
1222 #endif
1223
1224 /* add the tb in the target page and protect it if necessary */
1225 static inline void tb_alloc_page(TranslationBlock *tb,
1226 unsigned int n, tb_page_addr_t page_addr)
1227 {
1228 PageDesc *p;
1229 #ifndef CONFIG_USER_ONLY
1230 bool page_already_protected;
1231 #endif
1232
1233 tb->page_addr[n] = page_addr;
1234 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1235 tb->page_next[n] = p->first_tb;
1236 #ifndef CONFIG_USER_ONLY
1237 page_already_protected = p->first_tb != NULL;
1238 #endif
1239 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
1240 invalidate_page_bitmap(p);
1241
1242 #if defined(TARGET_HAS_SMC) || 1
1243
1244 #if defined(CONFIG_USER_ONLY)
1245 if (p->flags & PAGE_WRITE) {
1246 target_ulong addr;
1247 PageDesc *p2;
1248 int prot;
1249
1250 /* force the host page as non writable (writes will have a
1251 page fault + mprotect overhead) */
1252 page_addr &= qemu_host_page_mask;
1253 prot = 0;
1254 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1255 addr += TARGET_PAGE_SIZE) {
1256
1257 p2 = page_find(addr >> TARGET_PAGE_BITS);
1258 if (!p2) {
1259 continue;
1260 }
1261 prot |= p2->flags;
1262 p2->flags &= ~PAGE_WRITE;
1263 }
1264 mprotect(g2h(page_addr), qemu_host_page_size,
1265 (prot & PAGE_BITS) & ~PAGE_WRITE);
1266 #ifdef DEBUG_TB_INVALIDATE
1267 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1268 page_addr);
1269 #endif
1270 }
1271 #else
1272 /* if some code is already present, then the pages are already
1273 protected. So we handle the case where only the first TB is
1274 allocated in a physical page */
1275 if (!page_already_protected) {
1276 tlb_protect_code(page_addr);
1277 }
1278 #endif
1279
1280 #endif /* TARGET_HAS_SMC */
1281 }
1282
1283 /* add a new TB and link it to the physical page tables. phys_page2 is
1284 (-1) to indicate that only one page contains the TB. */
1285 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1286 tb_page_addr_t phys_page2)
1287 {
1288 unsigned int h;
1289 TranslationBlock **ptb;
1290
1291 /* Grab the mmap lock to stop another thread invalidating this TB
1292 before we are done. */
1293 mmap_lock();
1294 /* add in the physical hash table */
1295 h = tb_phys_hash_func(phys_pc);
1296 ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
1297 tb->phys_hash_next = *ptb;
1298 *ptb = tb;
1299
1300 /* add in the page list */
1301 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1302 if (phys_page2 != -1) {
1303 tb_alloc_page(tb, 1, phys_page2);
1304 } else {
1305 tb->page_addr[1] = -1;
1306 }
1307
1308 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
1309 tb->jmp_next[0] = NULL;
1310 tb->jmp_next[1] = NULL;
1311
1312 /* init original jump addresses */
1313 if (tb->tb_next_offset[0] != 0xffff) {
1314 tb_reset_jump(tb, 0);
1315 }
1316 if (tb->tb_next_offset[1] != 0xffff) {
1317 tb_reset_jump(tb, 1);
1318 }
1319
1320 #ifdef DEBUG_TB_CHECK
1321 tb_page_check();
1322 #endif
1323 mmap_unlock();
1324 }
1325
1326 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1327 tb[1].tc_ptr. Return NULL if not found */
1328 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
1329 {
1330 int m_min, m_max, m;
1331 uintptr_t v;
1332 TranslationBlock *tb;
1333
1334 if (tcg_ctx.tb_ctx.nb_tbs <= 0) {
1335 return NULL;
1336 }
1337 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer ||
1338 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) {
1339 return NULL;
1340 }
1341 /* binary search (cf Knuth) */
1342 m_min = 0;
1343 m_max = tcg_ctx.tb_ctx.nb_tbs - 1;
1344 while (m_min <= m_max) {
1345 m = (m_min + m_max) >> 1;
1346 tb = &tcg_ctx.tb_ctx.tbs[m];
1347 v = (uintptr_t)tb->tc_ptr;
1348 if (v == tc_ptr) {
1349 return tb;
1350 } else if (tc_ptr < v) {
1351 m_max = m - 1;
1352 } else {
1353 m_min = m + 1;
1354 }
1355 }
1356 return &tcg_ctx.tb_ctx.tbs[m_max];
1357 }
1358
1359 #if defined(TARGET_HAS_ICE) && !defined(CONFIG_USER_ONLY)
1360 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr)
1361 {
1362 ram_addr_t ram_addr;
1363 MemoryRegion *mr;
1364 hwaddr l = 1;
1365
1366 mr = address_space_translate(as, addr, &addr, &l, false);
1367 if (!(memory_region_is_ram(mr)
1368 || memory_region_is_romd(mr))) {
1369 return;
1370 }
1371 ram_addr = (memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK)
1372 + addr;
1373 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1374 }
1375 #endif /* TARGET_HAS_ICE && !defined(CONFIG_USER_ONLY) */
1376
1377 void tb_check_watchpoint(CPUArchState *env)
1378 {
1379 TranslationBlock *tb;
1380
1381 tb = tb_find_pc(env->mem_io_pc);
1382 if (!tb) {
1383 cpu_abort(env, "check_watchpoint: could not find TB for pc=%p",
1384 (void *)env->mem_io_pc);
1385 }
1386 cpu_restore_state_from_tb(tb, env, env->mem_io_pc);
1387 tb_phys_invalidate(tb, -1);
1388 }
1389
1390 #ifndef CONFIG_USER_ONLY
1391 /* mask must never be zero, except for A20 change call */
1392 static void tcg_handle_interrupt(CPUState *cpu, int mask)
1393 {
1394 CPUArchState *env = cpu->env_ptr;
1395 int old_mask;
1396
1397 old_mask = cpu->interrupt_request;
1398 cpu->interrupt_request |= mask;
1399
1400 /*
1401 * If called from iothread context, wake the target cpu in
1402 * case its halted.
1403 */
1404 if (!qemu_cpu_is_self(cpu)) {
1405 qemu_cpu_kick(cpu);
1406 return;
1407 }
1408
1409 if (use_icount) {
1410 env->icount_decr.u16.high = 0xffff;
1411 if (!can_do_io(env)
1412 && (mask & ~old_mask) != 0) {
1413 cpu_abort(env, "Raised interrupt while not in I/O function");
1414 }
1415 } else {
1416 cpu->tcg_exit_req = 1;
1417 }
1418 }
1419
1420 CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
1421
1422 /* in deterministic execution mode, instructions doing device I/Os
1423 must be at the end of the TB */
1424 void cpu_io_recompile(CPUArchState *env, uintptr_t retaddr)
1425 {
1426 TranslationBlock *tb;
1427 uint32_t n, cflags;
1428 target_ulong pc, cs_base;
1429 uint64_t flags;
1430
1431 tb = tb_find_pc(retaddr);
1432 if (!tb) {
1433 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
1434 (void *)retaddr);
1435 }
1436 n = env->icount_decr.u16.low + tb->icount;
1437 cpu_restore_state_from_tb(tb, env, retaddr);
1438 /* Calculate how many instructions had been executed before the fault
1439 occurred. */
1440 n = n - env->icount_decr.u16.low;
1441 /* Generate a new TB ending on the I/O insn. */
1442 n++;
1443 /* On MIPS and SH, delay slot instructions can only be restarted if
1444 they were already the first instruction in the TB. If this is not
1445 the first instruction in a TB then re-execute the preceding
1446 branch. */
1447 #if defined(TARGET_MIPS)
1448 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
1449 env->active_tc.PC -= 4;
1450 env->icount_decr.u16.low++;
1451 env->hflags &= ~MIPS_HFLAG_BMASK;
1452 }
1453 #elif defined(TARGET_SH4)
1454 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
1455 && n > 1) {
1456 env->pc -= 2;
1457 env->icount_decr.u16.low++;
1458 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
1459 }
1460 #endif
1461 /* This should never happen. */
1462 if (n > CF_COUNT_MASK) {
1463 cpu_abort(env, "TB too big during recompile");
1464 }
1465
1466 cflags = n | CF_LAST_IO;
1467 pc = tb->pc;
1468 cs_base = tb->cs_base;
1469 flags = tb->flags;
1470 tb_phys_invalidate(tb, -1);
1471 /* FIXME: In theory this could raise an exception. In practice
1472 we have already translated the block once so it's probably ok. */
1473 tb_gen_code(env, pc, cs_base, flags, cflags);
1474 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1475 the first in the TB) then we end up generating a whole new TB and
1476 repeating the fault, which is horribly inefficient.
1477 Better would be to execute just this insn uncached, or generate a
1478 second new TB. */
1479 cpu_resume_from_signal(env, NULL);
1480 }
1481
1482 void tb_flush_jmp_cache(CPUArchState *env, target_ulong addr)
1483 {
1484 unsigned int i;
1485
1486 /* Discard jump cache entries for any tb which might potentially
1487 overlap the flushed page. */
1488 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1489 memset(&env->tb_jmp_cache[i], 0,
1490 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1491
1492 i = tb_jmp_cache_hash_page(addr);
1493 memset(&env->tb_jmp_cache[i], 0,
1494 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1495 }
1496
1497 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1498 {
1499 int i, target_code_size, max_target_code_size;
1500 int direct_jmp_count, direct_jmp2_count, cross_page;
1501 TranslationBlock *tb;
1502
1503 target_code_size = 0;
1504 max_target_code_size = 0;
1505 cross_page = 0;
1506 direct_jmp_count = 0;
1507 direct_jmp2_count = 0;
1508 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) {
1509 tb = &tcg_ctx.tb_ctx.tbs[i];
1510 target_code_size += tb->size;
1511 if (tb->size > max_target_code_size) {
1512 max_target_code_size = tb->size;
1513 }
1514 if (tb->page_addr[1] != -1) {
1515 cross_page++;
1516 }
1517 if (tb->tb_next_offset[0] != 0xffff) {
1518 direct_jmp_count++;
1519 if (tb->tb_next_offset[1] != 0xffff) {
1520 direct_jmp2_count++;
1521 }
1522 }
1523 }
1524 /* XXX: avoid using doubles ? */
1525 cpu_fprintf(f, "Translation buffer state:\n");
1526 cpu_fprintf(f, "gen code size %td/%zd\n",
1527 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
1528 tcg_ctx.code_gen_buffer_max_size);
1529 cpu_fprintf(f, "TB count %d/%d\n",
1530 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
1531 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
1532 tcg_ctx.tb_ctx.nb_tbs ? target_code_size /
1533 tcg_ctx.tb_ctx.nb_tbs : 0,
1534 max_target_code_size);
1535 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1536 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr -
1537 tcg_ctx.code_gen_buffer) /
1538 tcg_ctx.tb_ctx.nb_tbs : 0,
1539 target_code_size ? (double) (tcg_ctx.code_gen_ptr -
1540 tcg_ctx.code_gen_buffer) /
1541 target_code_size : 0);
1542 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page,
1543 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) /
1544 tcg_ctx.tb_ctx.nb_tbs : 0);
1545 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1546 direct_jmp_count,
1547 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) /
1548 tcg_ctx.tb_ctx.nb_tbs : 0,
1549 direct_jmp2_count,
1550 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
1551 tcg_ctx.tb_ctx.nb_tbs : 0);
1552 cpu_fprintf(f, "\nStatistics:\n");
1553 cpu_fprintf(f, "TB flush count %d\n", tcg_ctx.tb_ctx.tb_flush_count);
1554 cpu_fprintf(f, "TB invalidate count %d\n",
1555 tcg_ctx.tb_ctx.tb_phys_invalidate_count);
1556 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
1557 tcg_dump_info(f, cpu_fprintf);
1558 }
1559
1560 #else /* CONFIG_USER_ONLY */
1561
1562 void cpu_interrupt(CPUState *cpu, int mask)
1563 {
1564 cpu->interrupt_request |= mask;
1565 cpu->tcg_exit_req = 1;
1566 }
1567
1568 /*
1569 * Walks guest process memory "regions" one by one
1570 * and calls callback function 'fn' for each region.
1571 */
1572 struct walk_memory_regions_data {
1573 walk_memory_regions_fn fn;
1574 void *priv;
1575 uintptr_t start;
1576 int prot;
1577 };
1578
1579 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
1580 abi_ulong end, int new_prot)
1581 {
1582 if (data->start != -1ul) {
1583 int rc = data->fn(data->priv, data->start, end, data->prot);
1584 if (rc != 0) {
1585 return rc;
1586 }
1587 }
1588
1589 data->start = (new_prot ? end : -1ul);
1590 data->prot = new_prot;
1591
1592 return 0;
1593 }
1594
1595 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
1596 abi_ulong base, int level, void **lp)
1597 {
1598 abi_ulong pa;
1599 int i, rc;
1600
1601 if (*lp == NULL) {
1602 return walk_memory_regions_end(data, base, 0);
1603 }
1604
1605 if (level == 0) {
1606 PageDesc *pd = *lp;
1607
1608 for (i = 0; i < V_L2_SIZE; ++i) {
1609 int prot = pd[i].flags;
1610
1611 pa = base | (i << TARGET_PAGE_BITS);
1612 if (prot != data->prot) {
1613 rc = walk_memory_regions_end(data, pa, prot);
1614 if (rc != 0) {
1615 return rc;
1616 }
1617 }
1618 }
1619 } else {
1620 void **pp = *lp;
1621
1622 for (i = 0; i < V_L2_SIZE; ++i) {
1623 pa = base | ((abi_ulong)i <<
1624 (TARGET_PAGE_BITS + V_L2_BITS * level));
1625 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
1626 if (rc != 0) {
1627 return rc;
1628 }
1629 }
1630 }
1631
1632 return 0;
1633 }
1634
1635 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
1636 {
1637 struct walk_memory_regions_data data;
1638 uintptr_t i;
1639
1640 data.fn = fn;
1641 data.priv = priv;
1642 data.start = -1ul;
1643 data.prot = 0;
1644
1645 for (i = 0; i < V_L1_SIZE; i++) {
1646 int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
1647 V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
1648
1649 if (rc != 0) {
1650 return rc;
1651 }
1652 }
1653
1654 return walk_memory_regions_end(&data, 0, 0);
1655 }
1656
1657 static int dump_region(void *priv, abi_ulong start,
1658 abi_ulong end, unsigned long prot)
1659 {
1660 FILE *f = (FILE *)priv;
1661
1662 (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
1663 " "TARGET_ABI_FMT_lx" %c%c%c\n",
1664 start, end, end - start,
1665 ((prot & PAGE_READ) ? 'r' : '-'),
1666 ((prot & PAGE_WRITE) ? 'w' : '-'),
1667 ((prot & PAGE_EXEC) ? 'x' : '-'));
1668
1669 return 0;
1670 }
1671
1672 /* dump memory mappings */
1673 void page_dump(FILE *f)
1674 {
1675 const int length = sizeof(abi_ulong) * 2;
1676 (void) fprintf(f, "%-*s %-*s %-*s %s\n",
1677 length, "start", length, "end", length, "size", "prot");
1678 walk_memory_regions(f, dump_region);
1679 }
1680
1681 int page_get_flags(target_ulong address)
1682 {
1683 PageDesc *p;
1684
1685 p = page_find(address >> TARGET_PAGE_BITS);
1686 if (!p) {
1687 return 0;
1688 }
1689 return p->flags;
1690 }
1691
1692 /* Modify the flags of a page and invalidate the code if necessary.
1693 The flag PAGE_WRITE_ORG is positioned automatically depending
1694 on PAGE_WRITE. The mmap_lock should already be held. */
1695 void page_set_flags(target_ulong start, target_ulong end, int flags)
1696 {
1697 target_ulong addr, len;
1698
1699 /* This function should never be called with addresses outside the
1700 guest address space. If this assert fires, it probably indicates
1701 a missing call to h2g_valid. */
1702 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1703 assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1704 #endif
1705 assert(start < end);
1706
1707 start = start & TARGET_PAGE_MASK;
1708 end = TARGET_PAGE_ALIGN(end);
1709
1710 if (flags & PAGE_WRITE) {
1711 flags |= PAGE_WRITE_ORG;
1712 }
1713
1714 for (addr = start, len = end - start;
1715 len != 0;
1716 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1717 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
1718
1719 /* If the write protection bit is set, then we invalidate
1720 the code inside. */
1721 if (!(p->flags & PAGE_WRITE) &&
1722 (flags & PAGE_WRITE) &&
1723 p->first_tb) {
1724 tb_invalidate_phys_page(addr, 0, NULL, false);
1725 }
1726 p->flags = flags;
1727 }
1728 }
1729
1730 int page_check_range(target_ulong start, target_ulong len, int flags)
1731 {
1732 PageDesc *p;
1733 target_ulong end;
1734 target_ulong addr;
1735
1736 /* This function should never be called with addresses outside the
1737 guest address space. If this assert fires, it probably indicates
1738 a missing call to h2g_valid. */
1739 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1740 assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1741 #endif
1742
1743 if (len == 0) {
1744 return 0;
1745 }
1746 if (start + len - 1 < start) {
1747 /* We've wrapped around. */
1748 return -1;
1749 }
1750
1751 /* must do before we loose bits in the next step */
1752 end = TARGET_PAGE_ALIGN(start + len);
1753 start = start & TARGET_PAGE_MASK;
1754
1755 for (addr = start, len = end - start;
1756 len != 0;
1757 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1758 p = page_find(addr >> TARGET_PAGE_BITS);
1759 if (!p) {
1760 return -1;
1761 }
1762 if (!(p->flags & PAGE_VALID)) {
1763 return -1;
1764 }
1765
1766 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
1767 return -1;
1768 }
1769 if (flags & PAGE_WRITE) {
1770 if (!(p->flags & PAGE_WRITE_ORG)) {
1771 return -1;
1772 }
1773 /* unprotect the page if it was put read-only because it
1774 contains translated code */
1775 if (!(p->flags & PAGE_WRITE)) {
1776 if (!page_unprotect(addr, 0, NULL)) {
1777 return -1;
1778 }
1779 }
1780 return 0;
1781 }
1782 }
1783 return 0;
1784 }
1785
1786 /* called from signal handler: invalidate the code and unprotect the
1787 page. Return TRUE if the fault was successfully handled. */
1788 int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
1789 {
1790 unsigned int prot;
1791 PageDesc *p;
1792 target_ulong host_start, host_end, addr;
1793
1794 /* Technically this isn't safe inside a signal handler. However we
1795 know this only ever happens in a synchronous SEGV handler, so in
1796 practice it seems to be ok. */
1797 mmap_lock();
1798
1799 p = page_find(address >> TARGET_PAGE_BITS);
1800 if (!p) {
1801 mmap_unlock();
1802 return 0;
1803 }
1804
1805 /* if the page was really writable, then we change its
1806 protection back to writable */
1807 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
1808 host_start = address & qemu_host_page_mask;
1809 host_end = host_start + qemu_host_page_size;
1810
1811 prot = 0;
1812 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
1813 p = page_find(addr >> TARGET_PAGE_BITS);
1814 p->flags |= PAGE_WRITE;
1815 prot |= p->flags;
1816
1817 /* and since the content will be modified, we must invalidate
1818 the corresponding translated code. */
1819 tb_invalidate_phys_page(addr, pc, puc, true);
1820 #ifdef DEBUG_TB_CHECK
1821 tb_invalidate_check(addr);
1822 #endif
1823 }
1824 mprotect((void *)g2h(host_start), qemu_host_page_size,
1825 prot & PAGE_BITS);
1826
1827 mmap_unlock();
1828 return 1;
1829 }
1830 mmap_unlock();
1831 return 0;
1832 }
1833 #endif /* CONFIG_USER_ONLY */