]> git.proxmox.com Git - mirror_qemu.git/blame - translate-all.c
mmu-hash32: Split direct store segment handling into a helper
[mirror_qemu.git] / translate-all.c
CommitLineData
d19893da
FB
1/*
2 * Host code generation
5fafdf24 3 *
d19893da
FB
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
8167ee88 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
d19893da 18 */
5b6dd868
BS
19#ifdef _WIN32
20#include <windows.h>
21#else
22#include <sys/types.h>
23#include <sys/mman.h>
24#endif
d19893da
FB
25#include <stdarg.h>
26#include <stdlib.h>
27#include <stdio.h>
28#include <string.h>
29#include <inttypes.h>
30
31#include "config.h"
2054396a 32
5b6dd868 33#include "qemu-common.h"
af5ad107 34#define NO_CPU_IO_DEFS
d3eead2e 35#include "cpu.h"
76cad711 36#include "disas/disas.h"
57fec1fe 37#include "tcg.h"
1de7afc9 38#include "qemu/timer.h"
022c62cb
PB
39#include "exec/memory.h"
40#include "exec/address-spaces.h"
5b6dd868
BS
41#if defined(CONFIG_USER_ONLY)
42#include "qemu.h"
43#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
44#include <sys/param.h>
45#if __FreeBSD_version >= 700104
46#define HAVE_KINFO_GETVMMAP
47#define sigqueue sigqueue_freebsd /* avoid redefinition */
48#include <sys/time.h>
49#include <sys/proc.h>
50#include <machine/profile.h>
51#define _KERNEL
52#include <sys/user.h>
53#undef _KERNEL
54#undef sigqueue
55#include <libutil.h>
56#endif
57#endif
58#endif
59
022c62cb 60#include "exec/cputlb.h"
5b6dd868
BS
61#include "translate-all.h"
62
63//#define DEBUG_TB_INVALIDATE
64//#define DEBUG_FLUSH
65/* make various TB consistency checks */
66//#define DEBUG_TB_CHECK
67
68#if !defined(CONFIG_USER_ONLY)
69/* TB consistency checks only implemented for usermode emulation. */
70#undef DEBUG_TB_CHECK
71#endif
72
73#define SMC_BITMAP_USE_THRESHOLD 10
74
5b6dd868
BS
75typedef struct PageDesc {
76 /* list of TBs intersecting this ram page */
77 TranslationBlock *first_tb;
78 /* in order to optimize self modifying code, we count the number
79 of lookups we do to a given page to use a bitmap */
80 unsigned int code_write_count;
81 uint8_t *code_bitmap;
82#if defined(CONFIG_USER_ONLY)
83 unsigned long flags;
84#endif
85} PageDesc;
86
87/* In system mode we want L1_MAP to be based on ram offsets,
88 while in user mode we want it to be based on virtual addresses. */
89#if !defined(CONFIG_USER_ONLY)
90#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
91# define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
92#else
93# define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
94#endif
95#else
96# define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
97#endif
98
99/* The bits remaining after N lower levels of page tables. */
100#define V_L1_BITS_REM \
101 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
102
103#if V_L1_BITS_REM < 4
104#define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
105#else
106#define V_L1_BITS V_L1_BITS_REM
107#endif
108
109#define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
110
111#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
112
113uintptr_t qemu_real_host_page_size;
114uintptr_t qemu_host_page_size;
115uintptr_t qemu_host_page_mask;
116
117/* This is a multi-level map on the virtual address space.
118 The bottom level has pointers to PageDesc. */
119static void *l1_map[V_L1_SIZE];
120
57fec1fe
FB
121/* code generation context */
122TCGContext tcg_ctx;
d19893da 123
5b6dd868
BS
124static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
125 tb_page_addr_t phys_page2);
a8a826a3 126static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
5b6dd868 127
57fec1fe
FB
128void cpu_gen_init(void)
129{
130 tcg_context_init(&tcg_ctx);
57fec1fe
FB
131}
132
d19893da 133/* return non zero if the very first instruction is invalid so that
5fafdf24 134 the virtual CPU can trigger an exception.
d19893da
FB
135
136 '*gen_code_size_ptr' contains the size of the generated code (host
137 code).
138*/
9349b4f9 139int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr)
d19893da 140{
57fec1fe 141 TCGContext *s = &tcg_ctx;
d19893da
FB
142 uint8_t *gen_code_buf;
143 int gen_code_size;
57fec1fe
FB
144#ifdef CONFIG_PROFILER
145 int64_t ti;
146#endif
147
148#ifdef CONFIG_PROFILER
b67d9a52
FB
149 s->tb_count1++; /* includes aborted translations because of
150 exceptions */
57fec1fe
FB
151 ti = profile_getclock();
152#endif
153 tcg_func_start(s);
d19893da 154
2cfc5f17
TS
155 gen_intermediate_code(env, tb);
156
ec6338ba 157 /* generate machine code */
57fec1fe 158 gen_code_buf = tb->tc_ptr;
ec6338ba
FB
159 tb->tb_next_offset[0] = 0xffff;
160 tb->tb_next_offset[1] = 0xffff;
57fec1fe 161 s->tb_next_offset = tb->tb_next_offset;
4cbb86e1 162#ifdef USE_DIRECT_JUMP
57fec1fe
FB
163 s->tb_jmp_offset = tb->tb_jmp_offset;
164 s->tb_next = NULL;
d19893da 165#else
57fec1fe
FB
166 s->tb_jmp_offset = NULL;
167 s->tb_next = tb->tb_next;
d19893da 168#endif
57fec1fe
FB
169
170#ifdef CONFIG_PROFILER
b67d9a52
FB
171 s->tb_count++;
172 s->interm_time += profile_getclock() - ti;
173 s->code_time -= profile_getclock();
57fec1fe 174#endif
54604f74 175 gen_code_size = tcg_gen_code(s, gen_code_buf);
d19893da 176 *gen_code_size_ptr = gen_code_size;
57fec1fe 177#ifdef CONFIG_PROFILER
b67d9a52
FB
178 s->code_time += profile_getclock();
179 s->code_in_len += tb->size;
180 s->code_out_len += gen_code_size;
57fec1fe
FB
181#endif
182
d19893da 183#ifdef DEBUG_DISAS
8fec2b8c 184 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
93fcfe39
AL
185 qemu_log("OUT: [size=%d]\n", *gen_code_size_ptr);
186 log_disas(tb->tc_ptr, *gen_code_size_ptr);
187 qemu_log("\n");
31b1a7b4 188 qemu_log_flush();
d19893da
FB
189 }
190#endif
191 return 0;
192}
193
5fafdf24 194/* The cpu state corresponding to 'searched_pc' is restored.
d19893da 195 */
a8a826a3
BS
196static int cpu_restore_state_from_tb(TranslationBlock *tb, CPUArchState *env,
197 uintptr_t searched_pc)
d19893da 198{
57fec1fe
FB
199 TCGContext *s = &tcg_ctx;
200 int j;
6375e09e 201 uintptr_t tc_ptr;
57fec1fe
FB
202#ifdef CONFIG_PROFILER
203 int64_t ti;
204#endif
205
206#ifdef CONFIG_PROFILER
207 ti = profile_getclock();
208#endif
209 tcg_func_start(s);
d19893da 210
2cfc5f17 211 gen_intermediate_code_pc(env, tb);
3b46e624 212
2e70f6ef
PB
213 if (use_icount) {
214 /* Reset the cycle counter to the start of the block. */
215 env->icount_decr.u16.low += tb->icount;
216 /* Clear the IO flag. */
217 env->can_do_io = 0;
218 }
219
d19893da 220 /* find opc index corresponding to search_pc */
6375e09e 221 tc_ptr = (uintptr_t)tb->tc_ptr;
d19893da
FB
222 if (searched_pc < tc_ptr)
223 return -1;
57fec1fe
FB
224
225 s->tb_next_offset = tb->tb_next_offset;
226#ifdef USE_DIRECT_JUMP
227 s->tb_jmp_offset = tb->tb_jmp_offset;
228 s->tb_next = NULL;
229#else
230 s->tb_jmp_offset = NULL;
231 s->tb_next = tb->tb_next;
232#endif
54604f74 233 j = tcg_gen_code_search_pc(s, (uint8_t *)tc_ptr, searched_pc - tc_ptr);
57fec1fe
FB
234 if (j < 0)
235 return -1;
d19893da 236 /* now find start of instruction before */
ab1103de 237 while (s->gen_opc_instr_start[j] == 0) {
d19893da 238 j--;
ab1103de 239 }
c9c99c22 240 env->icount_decr.u16.low -= s->gen_opc_icount[j];
3b46e624 241
e87b7cb0 242 restore_state_to_opc(env, tb, j);
57fec1fe
FB
243
244#ifdef CONFIG_PROFILER
b67d9a52
FB
245 s->restore_time += profile_getclock() - ti;
246 s->restore_count++;
57fec1fe 247#endif
d19893da
FB
248 return 0;
249}
5b6dd868 250
a8a826a3
BS
251bool cpu_restore_state(CPUArchState *env, uintptr_t retaddr)
252{
253 TranslationBlock *tb;
254
255 tb = tb_find_pc(retaddr);
256 if (tb) {
257 cpu_restore_state_from_tb(tb, env, retaddr);
258 return true;
259 }
260 return false;
261}
262
5b6dd868
BS
263#ifdef _WIN32
264static inline void map_exec(void *addr, long size)
265{
266 DWORD old_protect;
267 VirtualProtect(addr, size,
268 PAGE_EXECUTE_READWRITE, &old_protect);
269}
270#else
271static inline void map_exec(void *addr, long size)
272{
273 unsigned long start, end, page_size;
274
275 page_size = getpagesize();
276 start = (unsigned long)addr;
277 start &= ~(page_size - 1);
278
279 end = (unsigned long)addr + size;
280 end += page_size - 1;
281 end &= ~(page_size - 1);
282
283 mprotect((void *)start, end - start,
284 PROT_READ | PROT_WRITE | PROT_EXEC);
285}
286#endif
287
288static void page_init(void)
289{
290 /* NOTE: we can always suppose that qemu_host_page_size >=
291 TARGET_PAGE_SIZE */
292#ifdef _WIN32
293 {
294 SYSTEM_INFO system_info;
295
296 GetSystemInfo(&system_info);
297 qemu_real_host_page_size = system_info.dwPageSize;
298 }
299#else
300 qemu_real_host_page_size = getpagesize();
301#endif
302 if (qemu_host_page_size == 0) {
303 qemu_host_page_size = qemu_real_host_page_size;
304 }
305 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
306 qemu_host_page_size = TARGET_PAGE_SIZE;
307 }
308 qemu_host_page_mask = ~(qemu_host_page_size - 1);
309
310#if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
311 {
312#ifdef HAVE_KINFO_GETVMMAP
313 struct kinfo_vmentry *freep;
314 int i, cnt;
315
316 freep = kinfo_getvmmap(getpid(), &cnt);
317 if (freep) {
318 mmap_lock();
319 for (i = 0; i < cnt; i++) {
320 unsigned long startaddr, endaddr;
321
322 startaddr = freep[i].kve_start;
323 endaddr = freep[i].kve_end;
324 if (h2g_valid(startaddr)) {
325 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
326
327 if (h2g_valid(endaddr)) {
328 endaddr = h2g(endaddr);
329 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
330 } else {
331#if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
332 endaddr = ~0ul;
333 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
334#endif
335 }
336 }
337 }
338 free(freep);
339 mmap_unlock();
340 }
341#else
342 FILE *f;
343
344 last_brk = (unsigned long)sbrk(0);
345
346 f = fopen("/compat/linux/proc/self/maps", "r");
347 if (f) {
348 mmap_lock();
349
350 do {
351 unsigned long startaddr, endaddr;
352 int n;
353
354 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
355
356 if (n == 2 && h2g_valid(startaddr)) {
357 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
358
359 if (h2g_valid(endaddr)) {
360 endaddr = h2g(endaddr);
361 } else {
362 endaddr = ~0ul;
363 }
364 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
365 }
366 } while (!feof(f));
367
368 fclose(f);
369 mmap_unlock();
370 }
371#endif
372 }
373#endif
374}
375
376static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
377{
378 PageDesc *pd;
379 void **lp;
380 int i;
381
382#if defined(CONFIG_USER_ONLY)
383 /* We can't use g_malloc because it may recurse into a locked mutex. */
384# define ALLOC(P, SIZE) \
385 do { \
386 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
387 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
388 } while (0)
389#else
390# define ALLOC(P, SIZE) \
391 do { P = g_malloc0(SIZE); } while (0)
392#endif
393
394 /* Level 1. Always allocated. */
395 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
396
397 /* Level 2..N-1. */
398 for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
399 void **p = *lp;
400
401 if (p == NULL) {
402 if (!alloc) {
403 return NULL;
404 }
405 ALLOC(p, sizeof(void *) * L2_SIZE);
406 *lp = p;
407 }
408
409 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
410 }
411
412 pd = *lp;
413 if (pd == NULL) {
414 if (!alloc) {
415 return NULL;
416 }
417 ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
418 *lp = pd;
419 }
420
421#undef ALLOC
422
423 return pd + (index & (L2_SIZE - 1));
424}
425
426static inline PageDesc *page_find(tb_page_addr_t index)
427{
428 return page_find_alloc(index, 0);
429}
430
431#if !defined(CONFIG_USER_ONLY)
432#define mmap_lock() do { } while (0)
433#define mmap_unlock() do { } while (0)
434#endif
435
436#if defined(CONFIG_USER_ONLY)
437/* Currently it is not recommended to allocate big chunks of data in
438 user mode. It will change when a dedicated libc will be used. */
439/* ??? 64-bit hosts ought to have no problem mmaping data outside the
440 region in which the guest needs to run. Revisit this. */
441#define USE_STATIC_CODE_GEN_BUFFER
442#endif
443
444/* ??? Should configure for this, not list operating systems here. */
445#if (defined(__linux__) \
446 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
447 || defined(__DragonFly__) || defined(__OpenBSD__) \
448 || defined(__NetBSD__))
449# define USE_MMAP
450#endif
451
452/* Minimum size of the code gen buffer. This number is randomly chosen,
453 but not so small that we can't have a fair number of TB's live. */
454#define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
455
456/* Maximum size of the code gen buffer we'd like to use. Unless otherwise
457 indicated, this is constrained by the range of direct branches on the
458 host cpu, as used by the TCG implementation of goto_tb. */
459#if defined(__x86_64__)
460# define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
461#elif defined(__sparc__)
462# define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
463#elif defined(__arm__)
464# define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
465#elif defined(__s390x__)
466 /* We have a +- 4GB range on the branches; leave some slop. */
467# define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
468#else
469# define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
470#endif
471
472#define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
473
474#define DEFAULT_CODE_GEN_BUFFER_SIZE \
475 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
476 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
477
478static inline size_t size_code_gen_buffer(size_t tb_size)
479{
480 /* Size the buffer. */
481 if (tb_size == 0) {
482#ifdef USE_STATIC_CODE_GEN_BUFFER
483 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
484#else
485 /* ??? Needs adjustments. */
486 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
487 static buffer, we could size this on RESERVED_VA, on the text
488 segment size of the executable, or continue to use the default. */
489 tb_size = (unsigned long)(ram_size / 4);
490#endif
491 }
492 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
493 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
494 }
495 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
496 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
497 }
0b0d3320 498 tcg_ctx.code_gen_buffer_size = tb_size;
5b6dd868
BS
499 return tb_size;
500}
501
502#ifdef USE_STATIC_CODE_GEN_BUFFER
503static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
504 __attribute__((aligned(CODE_GEN_ALIGN)));
505
506static inline void *alloc_code_gen_buffer(void)
507{
0b0d3320 508 map_exec(static_code_gen_buffer, tcg_ctx.code_gen_buffer_size);
5b6dd868
BS
509 return static_code_gen_buffer;
510}
511#elif defined(USE_MMAP)
512static inline void *alloc_code_gen_buffer(void)
513{
514 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
515 uintptr_t start = 0;
516 void *buf;
517
518 /* Constrain the position of the buffer based on the host cpu.
519 Note that these addresses are chosen in concert with the
520 addresses assigned in the relevant linker script file. */
521# if defined(__PIE__) || defined(__PIC__)
522 /* Don't bother setting a preferred location if we're building
523 a position-independent executable. We're more likely to get
524 an address near the main executable if we let the kernel
525 choose the address. */
526# elif defined(__x86_64__) && defined(MAP_32BIT)
527 /* Force the memory down into low memory with the executable.
528 Leave the choice of exact location with the kernel. */
529 flags |= MAP_32BIT;
530 /* Cannot expect to map more than 800MB in low memory. */
0b0d3320
EV
531 if (tcg_ctx.code_gen_buffer_size > 800u * 1024 * 1024) {
532 tcg_ctx.code_gen_buffer_size = 800u * 1024 * 1024;
5b6dd868
BS
533 }
534# elif defined(__sparc__)
535 start = 0x40000000ul;
536# elif defined(__s390x__)
537 start = 0x90000000ul;
538# endif
539
0b0d3320 540 buf = mmap((void *)start, tcg_ctx.code_gen_buffer_size,
5b6dd868
BS
541 PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
542 return buf == MAP_FAILED ? NULL : buf;
543}
544#else
545static inline void *alloc_code_gen_buffer(void)
546{
0b0d3320 547 void *buf = g_malloc(tcg_ctx.code_gen_buffer_size);
5b6dd868
BS
548
549 if (buf) {
0b0d3320 550 map_exec(buf, tcg_ctx.code_gen_buffer_size);
5b6dd868
BS
551 }
552 return buf;
553}
554#endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
555
556static inline void code_gen_alloc(size_t tb_size)
557{
0b0d3320
EV
558 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size);
559 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer();
560 if (tcg_ctx.code_gen_buffer == NULL) {
5b6dd868
BS
561 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
562 exit(1);
563 }
564
0b0d3320
EV
565 qemu_madvise(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size,
566 QEMU_MADV_HUGEPAGE);
5b6dd868
BS
567
568 /* Steal room for the prologue at the end of the buffer. This ensures
569 (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
570 from TB's to the prologue are going to be in range. It also means
571 that we don't need to mark (additional) portions of the data segment
572 as executable. */
0b0d3320
EV
573 tcg_ctx.code_gen_prologue = tcg_ctx.code_gen_buffer +
574 tcg_ctx.code_gen_buffer_size - 1024;
575 tcg_ctx.code_gen_buffer_size -= 1024;
5b6dd868 576
0b0d3320 577 tcg_ctx.code_gen_buffer_max_size = tcg_ctx.code_gen_buffer_size -
5b6dd868 578 (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
0b0d3320
EV
579 tcg_ctx.code_gen_max_blocks = tcg_ctx.code_gen_buffer_size /
580 CODE_GEN_AVG_BLOCK_SIZE;
5e5f07e0
EV
581 tcg_ctx.tb_ctx.tbs =
582 g_malloc(tcg_ctx.code_gen_max_blocks * sizeof(TranslationBlock));
5b6dd868
BS
583}
584
585/* Must be called before using the QEMU cpus. 'tb_size' is the size
586 (in bytes) allocated to the translation buffer. Zero means default
587 size. */
588void tcg_exec_init(unsigned long tb_size)
589{
590 cpu_gen_init();
591 code_gen_alloc(tb_size);
0b0d3320
EV
592 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
593 tcg_register_jit(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size);
5b6dd868
BS
594 page_init();
595#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
596 /* There's no guest base to take into account, so go ahead and
597 initialize the prologue now. */
598 tcg_prologue_init(&tcg_ctx);
599#endif
600}
601
602bool tcg_enabled(void)
603{
0b0d3320 604 return tcg_ctx.code_gen_buffer != NULL;
5b6dd868
BS
605}
606
607/* Allocate a new translation block. Flush the translation buffer if
608 too many translation blocks or too much generated code. */
609static TranslationBlock *tb_alloc(target_ulong pc)
610{
611 TranslationBlock *tb;
612
5e5f07e0 613 if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks ||
0b0d3320
EV
614 (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) >=
615 tcg_ctx.code_gen_buffer_max_size) {
5b6dd868
BS
616 return NULL;
617 }
5e5f07e0 618 tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
5b6dd868
BS
619 tb->pc = pc;
620 tb->cflags = 0;
621 return tb;
622}
623
624void tb_free(TranslationBlock *tb)
625{
626 /* In practice this is mostly used for single use temporary TB
627 Ignore the hard cases and just back up if this TB happens to
628 be the last one generated. */
5e5f07e0
EV
629 if (tcg_ctx.tb_ctx.nb_tbs > 0 &&
630 tb == &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) {
0b0d3320 631 tcg_ctx.code_gen_ptr = tb->tc_ptr;
5e5f07e0 632 tcg_ctx.tb_ctx.nb_tbs--;
5b6dd868
BS
633 }
634}
635
636static inline void invalidate_page_bitmap(PageDesc *p)
637{
638 if (p->code_bitmap) {
639 g_free(p->code_bitmap);
640 p->code_bitmap = NULL;
641 }
642 p->code_write_count = 0;
643}
644
645/* Set to NULL all the 'first_tb' fields in all PageDescs. */
646static void page_flush_tb_1(int level, void **lp)
647{
648 int i;
649
650 if (*lp == NULL) {
651 return;
652 }
653 if (level == 0) {
654 PageDesc *pd = *lp;
655
656 for (i = 0; i < L2_SIZE; ++i) {
657 pd[i].first_tb = NULL;
658 invalidate_page_bitmap(pd + i);
659 }
660 } else {
661 void **pp = *lp;
662
663 for (i = 0; i < L2_SIZE; ++i) {
664 page_flush_tb_1(level - 1, pp + i);
665 }
666 }
667}
668
669static void page_flush_tb(void)
670{
671 int i;
672
673 for (i = 0; i < V_L1_SIZE; i++) {
674 page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
675 }
676}
677
678/* flush all the translation blocks */
679/* XXX: tb_flush is currently not thread safe */
680void tb_flush(CPUArchState *env1)
681{
682 CPUArchState *env;
683
684#if defined(DEBUG_FLUSH)
685 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
0b0d3320 686 (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
5e5f07e0 687 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
0b0d3320 688 ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) /
5e5f07e0 689 tcg_ctx.tb_ctx.nb_tbs : 0);
5b6dd868 690#endif
0b0d3320
EV
691 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)
692 > tcg_ctx.code_gen_buffer_size) {
5b6dd868
BS
693 cpu_abort(env1, "Internal error: code buffer overflow\n");
694 }
5e5f07e0 695 tcg_ctx.tb_ctx.nb_tbs = 0;
5b6dd868
BS
696
697 for (env = first_cpu; env != NULL; env = env->next_cpu) {
698 memset(env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof(void *));
699 }
700
5e5f07e0
EV
701 memset(tcg_ctx.tb_ctx.tb_phys_hash, 0,
702 CODE_GEN_PHYS_HASH_SIZE * sizeof(void *));
5b6dd868
BS
703 page_flush_tb();
704
0b0d3320 705 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
5b6dd868
BS
706 /* XXX: flush processor icache at this point if cache flush is
707 expensive */
5e5f07e0 708 tcg_ctx.tb_ctx.tb_flush_count++;
5b6dd868
BS
709}
710
711#ifdef DEBUG_TB_CHECK
712
713static void tb_invalidate_check(target_ulong address)
714{
715 TranslationBlock *tb;
716 int i;
717
718 address &= TARGET_PAGE_MASK;
719 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
5e5f07e0 720 for (tb = tb_ctx.tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
5b6dd868
BS
721 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
722 address >= tb->pc + tb->size)) {
723 printf("ERROR invalidate: address=" TARGET_FMT_lx
724 " PC=%08lx size=%04x\n",
725 address, (long)tb->pc, tb->size);
726 }
727 }
728 }
729}
730
731/* verify that all the pages have correct rights for code */
732static void tb_page_check(void)
733{
734 TranslationBlock *tb;
735 int i, flags1, flags2;
736
737 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
5e5f07e0
EV
738 for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
739 tb = tb->phys_hash_next) {
5b6dd868
BS
740 flags1 = page_get_flags(tb->pc);
741 flags2 = page_get_flags(tb->pc + tb->size - 1);
742 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
743 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
744 (long)tb->pc, tb->size, flags1, flags2);
745 }
746 }
747 }
748}
749
750#endif
751
0c884d16 752static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
5b6dd868
BS
753{
754 TranslationBlock *tb1;
755
756 for (;;) {
757 tb1 = *ptb;
758 if (tb1 == tb) {
0c884d16 759 *ptb = tb1->phys_hash_next;
5b6dd868
BS
760 break;
761 }
0c884d16 762 ptb = &tb1->phys_hash_next;
5b6dd868
BS
763 }
764}
765
766static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
767{
768 TranslationBlock *tb1;
769 unsigned int n1;
770
771 for (;;) {
772 tb1 = *ptb;
773 n1 = (uintptr_t)tb1 & 3;
774 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
775 if (tb1 == tb) {
776 *ptb = tb1->page_next[n1];
777 break;
778 }
779 ptb = &tb1->page_next[n1];
780 }
781}
782
783static inline void tb_jmp_remove(TranslationBlock *tb, int n)
784{
785 TranslationBlock *tb1, **ptb;
786 unsigned int n1;
787
788 ptb = &tb->jmp_next[n];
789 tb1 = *ptb;
790 if (tb1) {
791 /* find tb(n) in circular list */
792 for (;;) {
793 tb1 = *ptb;
794 n1 = (uintptr_t)tb1 & 3;
795 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
796 if (n1 == n && tb1 == tb) {
797 break;
798 }
799 if (n1 == 2) {
800 ptb = &tb1->jmp_first;
801 } else {
802 ptb = &tb1->jmp_next[n1];
803 }
804 }
805 /* now we can suppress tb(n) from the list */
806 *ptb = tb->jmp_next[n];
807
808 tb->jmp_next[n] = NULL;
809 }
810}
811
812/* reset the jump entry 'n' of a TB so that it is not chained to
813 another TB */
814static inline void tb_reset_jump(TranslationBlock *tb, int n)
815{
816 tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
817}
818
0c884d16 819/* invalidate one TB */
5b6dd868
BS
820void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
821{
822 CPUArchState *env;
823 PageDesc *p;
824 unsigned int h, n1;
825 tb_page_addr_t phys_pc;
826 TranslationBlock *tb1, *tb2;
827
828 /* remove the TB from the hash list */
829 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
830 h = tb_phys_hash_func(phys_pc);
5e5f07e0 831 tb_hash_remove(&tcg_ctx.tb_ctx.tb_phys_hash[h], tb);
5b6dd868
BS
832
833 /* remove the TB from the page list */
834 if (tb->page_addr[0] != page_addr) {
835 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
836 tb_page_remove(&p->first_tb, tb);
837 invalidate_page_bitmap(p);
838 }
839 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
840 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
841 tb_page_remove(&p->first_tb, tb);
842 invalidate_page_bitmap(p);
843 }
844
5e5f07e0 845 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
5b6dd868
BS
846
847 /* remove the TB from the hash list */
848 h = tb_jmp_cache_hash_func(tb->pc);
849 for (env = first_cpu; env != NULL; env = env->next_cpu) {
850 if (env->tb_jmp_cache[h] == tb) {
851 env->tb_jmp_cache[h] = NULL;
852 }
853 }
854
855 /* suppress this TB from the two jump lists */
856 tb_jmp_remove(tb, 0);
857 tb_jmp_remove(tb, 1);
858
859 /* suppress any remaining jumps to this TB */
860 tb1 = tb->jmp_first;
861 for (;;) {
862 n1 = (uintptr_t)tb1 & 3;
863 if (n1 == 2) {
864 break;
865 }
866 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
867 tb2 = tb1->jmp_next[n1];
868 tb_reset_jump(tb1, n1);
869 tb1->jmp_next[n1] = NULL;
870 tb1 = tb2;
871 }
872 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
873
5e5f07e0 874 tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
5b6dd868
BS
875}
876
877static inline void set_bits(uint8_t *tab, int start, int len)
878{
879 int end, mask, end1;
880
881 end = start + len;
882 tab += start >> 3;
883 mask = 0xff << (start & 7);
884 if ((start & ~7) == (end & ~7)) {
885 if (start < end) {
886 mask &= ~(0xff << (end & 7));
887 *tab |= mask;
888 }
889 } else {
890 *tab++ |= mask;
891 start = (start + 8) & ~7;
892 end1 = end & ~7;
893 while (start < end1) {
894 *tab++ = 0xff;
895 start += 8;
896 }
897 if (start < end) {
898 mask = ~(0xff << (end & 7));
899 *tab |= mask;
900 }
901 }
902}
903
904static void build_page_bitmap(PageDesc *p)
905{
906 int n, tb_start, tb_end;
907 TranslationBlock *tb;
908
909 p->code_bitmap = g_malloc0(TARGET_PAGE_SIZE / 8);
910
911 tb = p->first_tb;
912 while (tb != NULL) {
913 n = (uintptr_t)tb & 3;
914 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
915 /* NOTE: this is subtle as a TB may span two physical pages */
916 if (n == 0) {
917 /* NOTE: tb_end may be after the end of the page, but
918 it is not a problem */
919 tb_start = tb->pc & ~TARGET_PAGE_MASK;
920 tb_end = tb_start + tb->size;
921 if (tb_end > TARGET_PAGE_SIZE) {
922 tb_end = TARGET_PAGE_SIZE;
923 }
924 } else {
925 tb_start = 0;
926 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
927 }
928 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
929 tb = tb->page_next[n];
930 }
931}
932
933TranslationBlock *tb_gen_code(CPUArchState *env,
934 target_ulong pc, target_ulong cs_base,
935 int flags, int cflags)
936{
937 TranslationBlock *tb;
938 uint8_t *tc_ptr;
939 tb_page_addr_t phys_pc, phys_page2;
940 target_ulong virt_page2;
941 int code_gen_size;
942
943 phys_pc = get_page_addr_code(env, pc);
944 tb = tb_alloc(pc);
945 if (!tb) {
946 /* flush must be done */
947 tb_flush(env);
948 /* cannot fail at this point */
949 tb = tb_alloc(pc);
950 /* Don't forget to invalidate previous TB info. */
5e5f07e0 951 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
5b6dd868 952 }
0b0d3320 953 tc_ptr = tcg_ctx.code_gen_ptr;
5b6dd868
BS
954 tb->tc_ptr = tc_ptr;
955 tb->cs_base = cs_base;
956 tb->flags = flags;
957 tb->cflags = cflags;
958 cpu_gen_code(env, tb, &code_gen_size);
0b0d3320
EV
959 tcg_ctx.code_gen_ptr = (void *)(((uintptr_t)tcg_ctx.code_gen_ptr +
960 code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
5b6dd868
BS
961
962 /* check next page if needed */
963 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
964 phys_page2 = -1;
965 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
966 phys_page2 = get_page_addr_code(env, virt_page2);
967 }
968 tb_link_page(tb, phys_pc, phys_page2);
969 return tb;
970}
971
972/*
973 * Invalidate all TBs which intersect with the target physical address range
974 * [start;end[. NOTE: start and end may refer to *different* physical pages.
975 * 'is_cpu_write_access' should be true if called from a real cpu write
976 * access: the virtual CPU will exit the current TB if code is modified inside
977 * this TB.
978 */
979void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end,
980 int is_cpu_write_access)
981{
982 while (start < end) {
983 tb_invalidate_phys_page_range(start, end, is_cpu_write_access);
984 start &= TARGET_PAGE_MASK;
985 start += TARGET_PAGE_SIZE;
986 }
987}
988
989/*
990 * Invalidate all TBs which intersect with the target physical address range
991 * [start;end[. NOTE: start and end must refer to the *same* physical page.
992 * 'is_cpu_write_access' should be true if called from a real cpu write
993 * access: the virtual CPU will exit the current TB if code is modified inside
994 * this TB.
995 */
996void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
997 int is_cpu_write_access)
998{
999 TranslationBlock *tb, *tb_next, *saved_tb;
1000 CPUArchState *env = cpu_single_env;
d77953b9 1001 CPUState *cpu = NULL;
5b6dd868
BS
1002 tb_page_addr_t tb_start, tb_end;
1003 PageDesc *p;
1004 int n;
1005#ifdef TARGET_HAS_PRECISE_SMC
1006 int current_tb_not_found = is_cpu_write_access;
1007 TranslationBlock *current_tb = NULL;
1008 int current_tb_modified = 0;
1009 target_ulong current_pc = 0;
1010 target_ulong current_cs_base = 0;
1011 int current_flags = 0;
1012#endif /* TARGET_HAS_PRECISE_SMC */
1013
1014 p = page_find(start >> TARGET_PAGE_BITS);
1015 if (!p) {
1016 return;
1017 }
1018 if (!p->code_bitmap &&
1019 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1020 is_cpu_write_access) {
1021 /* build code bitmap */
1022 build_page_bitmap(p);
1023 }
d77953b9
AF
1024 if (env != NULL) {
1025 cpu = ENV_GET_CPU(env);
1026 }
5b6dd868
BS
1027
1028 /* we remove all the TBs in the range [start, end[ */
1029 /* XXX: see if in some cases it could be faster to invalidate all
1030 the code */
1031 tb = p->first_tb;
1032 while (tb != NULL) {
1033 n = (uintptr_t)tb & 3;
1034 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1035 tb_next = tb->page_next[n];
1036 /* NOTE: this is subtle as a TB may span two physical pages */
1037 if (n == 0) {
1038 /* NOTE: tb_end may be after the end of the page, but
1039 it is not a problem */
1040 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1041 tb_end = tb_start + tb->size;
1042 } else {
1043 tb_start = tb->page_addr[1];
1044 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1045 }
1046 if (!(tb_end <= start || tb_start >= end)) {
1047#ifdef TARGET_HAS_PRECISE_SMC
1048 if (current_tb_not_found) {
1049 current_tb_not_found = 0;
1050 current_tb = NULL;
1051 if (env->mem_io_pc) {
1052 /* now we have a real cpu fault */
1053 current_tb = tb_find_pc(env->mem_io_pc);
1054 }
1055 }
1056 if (current_tb == tb &&
1057 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1058 /* If we are modifying the current TB, we must stop
1059 its execution. We could be more precise by checking
1060 that the modification is after the current PC, but it
1061 would require a specialized function to partially
1062 restore the CPU state */
1063
1064 current_tb_modified = 1;
a8a826a3 1065 cpu_restore_state_from_tb(current_tb, env, env->mem_io_pc);
5b6dd868
BS
1066 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1067 &current_flags);
1068 }
1069#endif /* TARGET_HAS_PRECISE_SMC */
1070 /* we need to do that to handle the case where a signal
1071 occurs while doing tb_phys_invalidate() */
1072 saved_tb = NULL;
d77953b9
AF
1073 if (cpu != NULL) {
1074 saved_tb = cpu->current_tb;
1075 cpu->current_tb = NULL;
5b6dd868
BS
1076 }
1077 tb_phys_invalidate(tb, -1);
d77953b9
AF
1078 if (cpu != NULL) {
1079 cpu->current_tb = saved_tb;
c3affe56
AF
1080 if (cpu->interrupt_request && cpu->current_tb) {
1081 cpu_interrupt(cpu, cpu->interrupt_request);
5b6dd868
BS
1082 }
1083 }
1084 }
1085 tb = tb_next;
1086 }
1087#if !defined(CONFIG_USER_ONLY)
1088 /* if no code remaining, no need to continue to use slow writes */
1089 if (!p->first_tb) {
1090 invalidate_page_bitmap(p);
1091 if (is_cpu_write_access) {
1092 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1093 }
1094 }
1095#endif
1096#ifdef TARGET_HAS_PRECISE_SMC
1097 if (current_tb_modified) {
1098 /* we generate a block containing just the instruction
1099 modifying the memory. It will ensure that it cannot modify
1100 itself */
d77953b9 1101 cpu->current_tb = NULL;
5b6dd868
BS
1102 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1103 cpu_resume_from_signal(env, NULL);
1104 }
1105#endif
1106}
1107
1108/* len must be <= 8 and start must be a multiple of len */
1109void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1110{
1111 PageDesc *p;
1112 int offset, b;
1113
1114#if 0
1115 if (1) {
1116 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1117 cpu_single_env->mem_io_vaddr, len,
1118 cpu_single_env->eip,
1119 cpu_single_env->eip +
1120 (intptr_t)cpu_single_env->segs[R_CS].base);
1121 }
1122#endif
1123 p = page_find(start >> TARGET_PAGE_BITS);
1124 if (!p) {
1125 return;
1126 }
1127 if (p->code_bitmap) {
1128 offset = start & ~TARGET_PAGE_MASK;
1129 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1130 if (b & ((1 << len) - 1)) {
1131 goto do_invalidate;
1132 }
1133 } else {
1134 do_invalidate:
1135 tb_invalidate_phys_page_range(start, start + len, 1);
1136 }
1137}
1138
1139#if !defined(CONFIG_SOFTMMU)
1140static void tb_invalidate_phys_page(tb_page_addr_t addr,
1141 uintptr_t pc, void *puc)
1142{
1143 TranslationBlock *tb;
1144 PageDesc *p;
1145 int n;
1146#ifdef TARGET_HAS_PRECISE_SMC
1147 TranslationBlock *current_tb = NULL;
1148 CPUArchState *env = cpu_single_env;
d77953b9 1149 CPUState *cpu = NULL;
5b6dd868
BS
1150 int current_tb_modified = 0;
1151 target_ulong current_pc = 0;
1152 target_ulong current_cs_base = 0;
1153 int current_flags = 0;
1154#endif
1155
1156 addr &= TARGET_PAGE_MASK;
1157 p = page_find(addr >> TARGET_PAGE_BITS);
1158 if (!p) {
1159 return;
1160 }
1161 tb = p->first_tb;
1162#ifdef TARGET_HAS_PRECISE_SMC
1163 if (tb && pc != 0) {
1164 current_tb = tb_find_pc(pc);
1165 }
d77953b9
AF
1166 if (env != NULL) {
1167 cpu = ENV_GET_CPU(env);
1168 }
5b6dd868
BS
1169#endif
1170 while (tb != NULL) {
1171 n = (uintptr_t)tb & 3;
1172 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1173#ifdef TARGET_HAS_PRECISE_SMC
1174 if (current_tb == tb &&
1175 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1176 /* If we are modifying the current TB, we must stop
1177 its execution. We could be more precise by checking
1178 that the modification is after the current PC, but it
1179 would require a specialized function to partially
1180 restore the CPU state */
1181
1182 current_tb_modified = 1;
a8a826a3 1183 cpu_restore_state_from_tb(current_tb, env, pc);
5b6dd868
BS
1184 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1185 &current_flags);
1186 }
1187#endif /* TARGET_HAS_PRECISE_SMC */
1188 tb_phys_invalidate(tb, addr);
1189 tb = tb->page_next[n];
1190 }
1191 p->first_tb = NULL;
1192#ifdef TARGET_HAS_PRECISE_SMC
1193 if (current_tb_modified) {
1194 /* we generate a block containing just the instruction
1195 modifying the memory. It will ensure that it cannot modify
1196 itself */
d77953b9 1197 cpu->current_tb = NULL;
5b6dd868
BS
1198 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1199 cpu_resume_from_signal(env, puc);
1200 }
1201#endif
1202}
1203#endif
1204
1205/* add the tb in the target page and protect it if necessary */
1206static inline void tb_alloc_page(TranslationBlock *tb,
1207 unsigned int n, tb_page_addr_t page_addr)
1208{
1209 PageDesc *p;
1210#ifndef CONFIG_USER_ONLY
1211 bool page_already_protected;
1212#endif
1213
1214 tb->page_addr[n] = page_addr;
1215 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1216 tb->page_next[n] = p->first_tb;
1217#ifndef CONFIG_USER_ONLY
1218 page_already_protected = p->first_tb != NULL;
1219#endif
1220 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
1221 invalidate_page_bitmap(p);
1222
1223#if defined(TARGET_HAS_SMC) || 1
1224
1225#if defined(CONFIG_USER_ONLY)
1226 if (p->flags & PAGE_WRITE) {
1227 target_ulong addr;
1228 PageDesc *p2;
1229 int prot;
1230
1231 /* force the host page as non writable (writes will have a
1232 page fault + mprotect overhead) */
1233 page_addr &= qemu_host_page_mask;
1234 prot = 0;
1235 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1236 addr += TARGET_PAGE_SIZE) {
1237
1238 p2 = page_find(addr >> TARGET_PAGE_BITS);
1239 if (!p2) {
1240 continue;
1241 }
1242 prot |= p2->flags;
1243 p2->flags &= ~PAGE_WRITE;
1244 }
1245 mprotect(g2h(page_addr), qemu_host_page_size,
1246 (prot & PAGE_BITS) & ~PAGE_WRITE);
1247#ifdef DEBUG_TB_INVALIDATE
1248 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1249 page_addr);
1250#endif
1251 }
1252#else
1253 /* if some code is already present, then the pages are already
1254 protected. So we handle the case where only the first TB is
1255 allocated in a physical page */
1256 if (!page_already_protected) {
1257 tlb_protect_code(page_addr);
1258 }
1259#endif
1260
1261#endif /* TARGET_HAS_SMC */
1262}
1263
1264/* add a new TB and link it to the physical page tables. phys_page2 is
1265 (-1) to indicate that only one page contains the TB. */
1266static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1267 tb_page_addr_t phys_page2)
1268{
1269 unsigned int h;
1270 TranslationBlock **ptb;
1271
1272 /* Grab the mmap lock to stop another thread invalidating this TB
1273 before we are done. */
1274 mmap_lock();
1275 /* add in the physical hash table */
1276 h = tb_phys_hash_func(phys_pc);
5e5f07e0 1277 ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
5b6dd868
BS
1278 tb->phys_hash_next = *ptb;
1279 *ptb = tb;
1280
1281 /* add in the page list */
1282 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1283 if (phys_page2 != -1) {
1284 tb_alloc_page(tb, 1, phys_page2);
1285 } else {
1286 tb->page_addr[1] = -1;
1287 }
1288
1289 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
1290 tb->jmp_next[0] = NULL;
1291 tb->jmp_next[1] = NULL;
1292
1293 /* init original jump addresses */
1294 if (tb->tb_next_offset[0] != 0xffff) {
1295 tb_reset_jump(tb, 0);
1296 }
1297 if (tb->tb_next_offset[1] != 0xffff) {
1298 tb_reset_jump(tb, 1);
1299 }
1300
1301#ifdef DEBUG_TB_CHECK
1302 tb_page_check();
1303#endif
1304 mmap_unlock();
1305}
1306
1307#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
1308/* check whether the given addr is in TCG generated code buffer or not */
1309bool is_tcg_gen_code(uintptr_t tc_ptr)
1310{
1311 /* This can be called during code generation, code_gen_buffer_max_size
1312 is used instead of code_gen_ptr for upper boundary checking */
0b0d3320
EV
1313 return (tc_ptr >= (uintptr_t)tcg_ctx.code_gen_buffer &&
1314 tc_ptr < (uintptr_t)(tcg_ctx.code_gen_buffer +
1315 tcg_ctx.code_gen_buffer_max_size));
5b6dd868
BS
1316}
1317#endif
1318
1319/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1320 tb[1].tc_ptr. Return NULL if not found */
a8a826a3 1321static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
5b6dd868
BS
1322{
1323 int m_min, m_max, m;
1324 uintptr_t v;
1325 TranslationBlock *tb;
1326
5e5f07e0 1327 if (tcg_ctx.tb_ctx.nb_tbs <= 0) {
5b6dd868
BS
1328 return NULL;
1329 }
0b0d3320
EV
1330 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer ||
1331 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) {
5b6dd868
BS
1332 return NULL;
1333 }
1334 /* binary search (cf Knuth) */
1335 m_min = 0;
5e5f07e0 1336 m_max = tcg_ctx.tb_ctx.nb_tbs - 1;
5b6dd868
BS
1337 while (m_min <= m_max) {
1338 m = (m_min + m_max) >> 1;
5e5f07e0 1339 tb = &tcg_ctx.tb_ctx.tbs[m];
5b6dd868
BS
1340 v = (uintptr_t)tb->tc_ptr;
1341 if (v == tc_ptr) {
1342 return tb;
1343 } else if (tc_ptr < v) {
1344 m_max = m - 1;
1345 } else {
1346 m_min = m + 1;
1347 }
1348 }
5e5f07e0 1349 return &tcg_ctx.tb_ctx.tbs[m_max];
5b6dd868
BS
1350}
1351
5b6dd868
BS
1352#if defined(TARGET_HAS_ICE) && !defined(CONFIG_USER_ONLY)
1353void tb_invalidate_phys_addr(hwaddr addr)
1354{
1355 ram_addr_t ram_addr;
1356 MemoryRegionSection *section;
1357
1358 section = phys_page_find(address_space_memory.dispatch,
1359 addr >> TARGET_PAGE_BITS);
1360 if (!(memory_region_is_ram(section->mr)
1361 || (section->mr->rom_device && section->mr->readable))) {
1362 return;
1363 }
1364 ram_addr = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
1365 + memory_region_section_addr(section, addr);
1366 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1367}
1368#endif /* TARGET_HAS_ICE && !defined(CONFIG_USER_ONLY) */
1369
5b6dd868
BS
1370void tb_check_watchpoint(CPUArchState *env)
1371{
1372 TranslationBlock *tb;
1373
1374 tb = tb_find_pc(env->mem_io_pc);
1375 if (!tb) {
1376 cpu_abort(env, "check_watchpoint: could not find TB for pc=%p",
1377 (void *)env->mem_io_pc);
1378 }
a8a826a3 1379 cpu_restore_state_from_tb(tb, env, env->mem_io_pc);
5b6dd868
BS
1380 tb_phys_invalidate(tb, -1);
1381}
1382
1383#ifndef CONFIG_USER_ONLY
1384/* mask must never be zero, except for A20 change call */
c3affe56 1385static void tcg_handle_interrupt(CPUState *cpu, int mask)
5b6dd868 1386{
c3affe56 1387 CPUArchState *env = cpu->env_ptr;
5b6dd868
BS
1388 int old_mask;
1389
259186a7
AF
1390 old_mask = cpu->interrupt_request;
1391 cpu->interrupt_request |= mask;
5b6dd868
BS
1392
1393 /*
1394 * If called from iothread context, wake the target cpu in
1395 * case its halted.
1396 */
1397 if (!qemu_cpu_is_self(cpu)) {
1398 qemu_cpu_kick(cpu);
1399 return;
1400 }
1401
1402 if (use_icount) {
1403 env->icount_decr.u16.high = 0xffff;
1404 if (!can_do_io(env)
1405 && (mask & ~old_mask) != 0) {
1406 cpu_abort(env, "Raised interrupt while not in I/O function");
1407 }
1408 } else {
378df4b2 1409 cpu->tcg_exit_req = 1;
5b6dd868
BS
1410 }
1411}
1412
1413CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
1414
1415/* in deterministic execution mode, instructions doing device I/Os
1416 must be at the end of the TB */
1417void cpu_io_recompile(CPUArchState *env, uintptr_t retaddr)
1418{
1419 TranslationBlock *tb;
1420 uint32_t n, cflags;
1421 target_ulong pc, cs_base;
1422 uint64_t flags;
1423
1424 tb = tb_find_pc(retaddr);
1425 if (!tb) {
1426 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
1427 (void *)retaddr);
1428 }
1429 n = env->icount_decr.u16.low + tb->icount;
a8a826a3 1430 cpu_restore_state_from_tb(tb, env, retaddr);
5b6dd868
BS
1431 /* Calculate how many instructions had been executed before the fault
1432 occurred. */
1433 n = n - env->icount_decr.u16.low;
1434 /* Generate a new TB ending on the I/O insn. */
1435 n++;
1436 /* On MIPS and SH, delay slot instructions can only be restarted if
1437 they were already the first instruction in the TB. If this is not
1438 the first instruction in a TB then re-execute the preceding
1439 branch. */
1440#if defined(TARGET_MIPS)
1441 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
1442 env->active_tc.PC -= 4;
1443 env->icount_decr.u16.low++;
1444 env->hflags &= ~MIPS_HFLAG_BMASK;
1445 }
1446#elif defined(TARGET_SH4)
1447 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
1448 && n > 1) {
1449 env->pc -= 2;
1450 env->icount_decr.u16.low++;
1451 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
1452 }
1453#endif
1454 /* This should never happen. */
1455 if (n > CF_COUNT_MASK) {
1456 cpu_abort(env, "TB too big during recompile");
1457 }
1458
1459 cflags = n | CF_LAST_IO;
1460 pc = tb->pc;
1461 cs_base = tb->cs_base;
1462 flags = tb->flags;
1463 tb_phys_invalidate(tb, -1);
1464 /* FIXME: In theory this could raise an exception. In practice
1465 we have already translated the block once so it's probably ok. */
1466 tb_gen_code(env, pc, cs_base, flags, cflags);
1467 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1468 the first in the TB) then we end up generating a whole new TB and
1469 repeating the fault, which is horribly inefficient.
1470 Better would be to execute just this insn uncached, or generate a
1471 second new TB. */
1472 cpu_resume_from_signal(env, NULL);
1473}
1474
1475void tb_flush_jmp_cache(CPUArchState *env, target_ulong addr)
1476{
1477 unsigned int i;
1478
1479 /* Discard jump cache entries for any tb which might potentially
1480 overlap the flushed page. */
1481 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1482 memset(&env->tb_jmp_cache[i], 0,
1483 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1484
1485 i = tb_jmp_cache_hash_page(addr);
1486 memset(&env->tb_jmp_cache[i], 0,
1487 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1488}
1489
1490void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1491{
1492 int i, target_code_size, max_target_code_size;
1493 int direct_jmp_count, direct_jmp2_count, cross_page;
1494 TranslationBlock *tb;
1495
1496 target_code_size = 0;
1497 max_target_code_size = 0;
1498 cross_page = 0;
1499 direct_jmp_count = 0;
1500 direct_jmp2_count = 0;
5e5f07e0
EV
1501 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) {
1502 tb = &tcg_ctx.tb_ctx.tbs[i];
5b6dd868
BS
1503 target_code_size += tb->size;
1504 if (tb->size > max_target_code_size) {
1505 max_target_code_size = tb->size;
1506 }
1507 if (tb->page_addr[1] != -1) {
1508 cross_page++;
1509 }
1510 if (tb->tb_next_offset[0] != 0xffff) {
1511 direct_jmp_count++;
1512 if (tb->tb_next_offset[1] != 0xffff) {
1513 direct_jmp2_count++;
1514 }
1515 }
1516 }
1517 /* XXX: avoid using doubles ? */
1518 cpu_fprintf(f, "Translation buffer state:\n");
1519 cpu_fprintf(f, "gen code size %td/%zd\n",
0b0d3320
EV
1520 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
1521 tcg_ctx.code_gen_buffer_max_size);
5b6dd868 1522 cpu_fprintf(f, "TB count %d/%d\n",
5e5f07e0 1523 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
5b6dd868 1524 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
5e5f07e0
EV
1525 tcg_ctx.tb_ctx.nb_tbs ? target_code_size /
1526 tcg_ctx.tb_ctx.nb_tbs : 0,
1527 max_target_code_size);
5b6dd868 1528 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
5e5f07e0
EV
1529 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr -
1530 tcg_ctx.code_gen_buffer) /
1531 tcg_ctx.tb_ctx.nb_tbs : 0,
1532 target_code_size ? (double) (tcg_ctx.code_gen_ptr -
1533 tcg_ctx.code_gen_buffer) /
1534 target_code_size : 0);
1535 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page,
1536 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) /
1537 tcg_ctx.tb_ctx.nb_tbs : 0);
5b6dd868
BS
1538 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1539 direct_jmp_count,
5e5f07e0
EV
1540 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) /
1541 tcg_ctx.tb_ctx.nb_tbs : 0,
5b6dd868 1542 direct_jmp2_count,
5e5f07e0
EV
1543 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
1544 tcg_ctx.tb_ctx.nb_tbs : 0);
5b6dd868 1545 cpu_fprintf(f, "\nStatistics:\n");
5e5f07e0
EV
1546 cpu_fprintf(f, "TB flush count %d\n", tcg_ctx.tb_ctx.tb_flush_count);
1547 cpu_fprintf(f, "TB invalidate count %d\n",
1548 tcg_ctx.tb_ctx.tb_phys_invalidate_count);
5b6dd868
BS
1549 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
1550 tcg_dump_info(f, cpu_fprintf);
1551}
1552
1553#else /* CONFIG_USER_ONLY */
1554
c3affe56 1555void cpu_interrupt(CPUState *cpu, int mask)
5b6dd868 1556{
259186a7 1557 cpu->interrupt_request |= mask;
378df4b2 1558 cpu->tcg_exit_req = 1;
5b6dd868
BS
1559}
1560
1561/*
1562 * Walks guest process memory "regions" one by one
1563 * and calls callback function 'fn' for each region.
1564 */
1565struct walk_memory_regions_data {
1566 walk_memory_regions_fn fn;
1567 void *priv;
1568 uintptr_t start;
1569 int prot;
1570};
1571
1572static int walk_memory_regions_end(struct walk_memory_regions_data *data,
1573 abi_ulong end, int new_prot)
1574{
1575 if (data->start != -1ul) {
1576 int rc = data->fn(data->priv, data->start, end, data->prot);
1577 if (rc != 0) {
1578 return rc;
1579 }
1580 }
1581
1582 data->start = (new_prot ? end : -1ul);
1583 data->prot = new_prot;
1584
1585 return 0;
1586}
1587
1588static int walk_memory_regions_1(struct walk_memory_regions_data *data,
1589 abi_ulong base, int level, void **lp)
1590{
1591 abi_ulong pa;
1592 int i, rc;
1593
1594 if (*lp == NULL) {
1595 return walk_memory_regions_end(data, base, 0);
1596 }
1597
1598 if (level == 0) {
1599 PageDesc *pd = *lp;
1600
1601 for (i = 0; i < L2_SIZE; ++i) {
1602 int prot = pd[i].flags;
1603
1604 pa = base | (i << TARGET_PAGE_BITS);
1605 if (prot != data->prot) {
1606 rc = walk_memory_regions_end(data, pa, prot);
1607 if (rc != 0) {
1608 return rc;
1609 }
1610 }
1611 }
1612 } else {
1613 void **pp = *lp;
1614
1615 for (i = 0; i < L2_SIZE; ++i) {
1616 pa = base | ((abi_ulong)i <<
1617 (TARGET_PAGE_BITS + L2_BITS * level));
1618 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
1619 if (rc != 0) {
1620 return rc;
1621 }
1622 }
1623 }
1624
1625 return 0;
1626}
1627
1628int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
1629{
1630 struct walk_memory_regions_data data;
1631 uintptr_t i;
1632
1633 data.fn = fn;
1634 data.priv = priv;
1635 data.start = -1ul;
1636 data.prot = 0;
1637
1638 for (i = 0; i < V_L1_SIZE; i++) {
1639 int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
1640 V_L1_SHIFT / L2_BITS - 1, l1_map + i);
1641
1642 if (rc != 0) {
1643 return rc;
1644 }
1645 }
1646
1647 return walk_memory_regions_end(&data, 0, 0);
1648}
1649
1650static int dump_region(void *priv, abi_ulong start,
1651 abi_ulong end, unsigned long prot)
1652{
1653 FILE *f = (FILE *)priv;
1654
1655 (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
1656 " "TARGET_ABI_FMT_lx" %c%c%c\n",
1657 start, end, end - start,
1658 ((prot & PAGE_READ) ? 'r' : '-'),
1659 ((prot & PAGE_WRITE) ? 'w' : '-'),
1660 ((prot & PAGE_EXEC) ? 'x' : '-'));
1661
1662 return 0;
1663}
1664
1665/* dump memory mappings */
1666void page_dump(FILE *f)
1667{
1668 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
1669 "start", "end", "size", "prot");
1670 walk_memory_regions(f, dump_region);
1671}
1672
1673int page_get_flags(target_ulong address)
1674{
1675 PageDesc *p;
1676
1677 p = page_find(address >> TARGET_PAGE_BITS);
1678 if (!p) {
1679 return 0;
1680 }
1681 return p->flags;
1682}
1683
1684/* Modify the flags of a page and invalidate the code if necessary.
1685 The flag PAGE_WRITE_ORG is positioned automatically depending
1686 on PAGE_WRITE. The mmap_lock should already be held. */
1687void page_set_flags(target_ulong start, target_ulong end, int flags)
1688{
1689 target_ulong addr, len;
1690
1691 /* This function should never be called with addresses outside the
1692 guest address space. If this assert fires, it probably indicates
1693 a missing call to h2g_valid. */
1694#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1695 assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1696#endif
1697 assert(start < end);
1698
1699 start = start & TARGET_PAGE_MASK;
1700 end = TARGET_PAGE_ALIGN(end);
1701
1702 if (flags & PAGE_WRITE) {
1703 flags |= PAGE_WRITE_ORG;
1704 }
1705
1706 for (addr = start, len = end - start;
1707 len != 0;
1708 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1709 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
1710
1711 /* If the write protection bit is set, then we invalidate
1712 the code inside. */
1713 if (!(p->flags & PAGE_WRITE) &&
1714 (flags & PAGE_WRITE) &&
1715 p->first_tb) {
1716 tb_invalidate_phys_page(addr, 0, NULL);
1717 }
1718 p->flags = flags;
1719 }
1720}
1721
1722int page_check_range(target_ulong start, target_ulong len, int flags)
1723{
1724 PageDesc *p;
1725 target_ulong end;
1726 target_ulong addr;
1727
1728 /* This function should never be called with addresses outside the
1729 guest address space. If this assert fires, it probably indicates
1730 a missing call to h2g_valid. */
1731#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1732 assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1733#endif
1734
1735 if (len == 0) {
1736 return 0;
1737 }
1738 if (start + len - 1 < start) {
1739 /* We've wrapped around. */
1740 return -1;
1741 }
1742
1743 /* must do before we loose bits in the next step */
1744 end = TARGET_PAGE_ALIGN(start + len);
1745 start = start & TARGET_PAGE_MASK;
1746
1747 for (addr = start, len = end - start;
1748 len != 0;
1749 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1750 p = page_find(addr >> TARGET_PAGE_BITS);
1751 if (!p) {
1752 return -1;
1753 }
1754 if (!(p->flags & PAGE_VALID)) {
1755 return -1;
1756 }
1757
1758 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
1759 return -1;
1760 }
1761 if (flags & PAGE_WRITE) {
1762 if (!(p->flags & PAGE_WRITE_ORG)) {
1763 return -1;
1764 }
1765 /* unprotect the page if it was put read-only because it
1766 contains translated code */
1767 if (!(p->flags & PAGE_WRITE)) {
1768 if (!page_unprotect(addr, 0, NULL)) {
1769 return -1;
1770 }
1771 }
1772 return 0;
1773 }
1774 }
1775 return 0;
1776}
1777
1778/* called from signal handler: invalidate the code and unprotect the
1779 page. Return TRUE if the fault was successfully handled. */
1780int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
1781{
1782 unsigned int prot;
1783 PageDesc *p;
1784 target_ulong host_start, host_end, addr;
1785
1786 /* Technically this isn't safe inside a signal handler. However we
1787 know this only ever happens in a synchronous SEGV handler, so in
1788 practice it seems to be ok. */
1789 mmap_lock();
1790
1791 p = page_find(address >> TARGET_PAGE_BITS);
1792 if (!p) {
1793 mmap_unlock();
1794 return 0;
1795 }
1796
1797 /* if the page was really writable, then we change its
1798 protection back to writable */
1799 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
1800 host_start = address & qemu_host_page_mask;
1801 host_end = host_start + qemu_host_page_size;
1802
1803 prot = 0;
1804 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
1805 p = page_find(addr >> TARGET_PAGE_BITS);
1806 p->flags |= PAGE_WRITE;
1807 prot |= p->flags;
1808
1809 /* and since the content will be modified, we must invalidate
1810 the corresponding translated code. */
1811 tb_invalidate_phys_page(addr, pc, puc);
1812#ifdef DEBUG_TB_CHECK
1813 tb_invalidate_check(addr);
1814#endif
1815 }
1816 mprotect((void *)g2h(host_start), qemu_host_page_size,
1817 prot & PAGE_BITS);
1818
1819 mmap_unlock();
1820 return 1;
1821 }
1822 mmap_unlock();
1823 return 0;
1824}
1825#endif /* CONFIG_USER_ONLY */