]> git.proxmox.com Git - mirror_qemu.git/blame_incremental - exec.c
projects / mirror_qemu.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
linux-user: Remove ELFLOAD32.
[mirror_qemu.git] / exec.c
... / ...
CommitLineData
1/*
2 * virtual page mapping and translated block handling
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19#include "config.h"
20#ifdef _WIN32
21#include <windows.h>
22#else
23#include <sys/types.h>
24#include <sys/mman.h>
25#endif
26#include <stdlib.h>
27#include <stdio.h>
28#include <stdarg.h>
29#include <string.h>
30#include <errno.h>
31#include <unistd.h>
32#include <inttypes.h>
33
34#include "cpu.h"
35#include "exec-all.h"
36#include "qemu-common.h"
37#include "tcg.h"
38#include "hw/hw.h"
39#include "osdep.h"
40#include "kvm.h"
41#include "qemu-timer.h"
42#if defined(CONFIG_USER_ONLY)
43#include <qemu.h>
44#include <signal.h>
45#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
46#include <sys/param.h>
47#if __FreeBSD_version >= 700104
48#define HAVE_KINFO_GETVMMAP
49#define sigqueue sigqueue_freebsd /* avoid redefinition */
50#include <sys/time.h>
51#include <sys/proc.h>
52#include <machine/profile.h>
53#define _KERNEL
54#include <sys/user.h>
55#undef _KERNEL
56#undef sigqueue
57#include <libutil.h>
58#endif
59#endif
60#endif
61
62//#define DEBUG_TB_INVALIDATE
63//#define DEBUG_FLUSH
64//#define DEBUG_TLB
65//#define DEBUG_UNASSIGNED
66
67/* make various TB consistency checks */
68//#define DEBUG_TB_CHECK
69//#define DEBUG_TLB_CHECK
70
71//#define DEBUG_IOPORT
72//#define DEBUG_SUBPAGE
73
74#if !defined(CONFIG_USER_ONLY)
75/* TB consistency checks only implemented for usermode emulation. */
76#undef DEBUG_TB_CHECK
77#endif
78
79#define SMC_BITMAP_USE_THRESHOLD 10
80
81static TranslationBlock *tbs;
82int code_gen_max_blocks;
83TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
84static int nb_tbs;
85/* any access to the tbs or the page table must use this lock */
86spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
87
88#if defined(__arm__) || defined(__sparc_v9__)
89/* The prologue must be reachable with a direct jump. ARM and Sparc64
90 have limited branch ranges (possibly also PPC) so place it in a
91 section close to code segment. */
92#define code_gen_section \
93 __attribute__((__section__(".gen_code"))) \
94 __attribute__((aligned (32)))
95#elif defined(_WIN32)
96/* Maximum alignment for Win32 is 16. */
97#define code_gen_section \
98 __attribute__((aligned (16)))
99#else
100#define code_gen_section \
101 __attribute__((aligned (32)))
102#endif
103
104uint8_t code_gen_prologue[1024] code_gen_section;
105static uint8_t *code_gen_buffer;
106static unsigned long code_gen_buffer_size;
107/* threshold to flush the translated code buffer */
108static unsigned long code_gen_buffer_max_size;
109uint8_t *code_gen_ptr;
110
111#if !defined(CONFIG_USER_ONLY)
112int phys_ram_fd;
113uint8_t *phys_ram_dirty;
114static int in_migration;
115
116typedef struct RAMBlock {
117 uint8_t *host;
118 ram_addr_t offset;
119 ram_addr_t length;
120 struct RAMBlock *next;
121} RAMBlock;
122
123static RAMBlock *ram_blocks;
124/* TODO: When we implement (and use) ram deallocation (e.g. for hotplug)
125 then we can no longer assume contiguous ram offsets, and external uses
126 of this variable will break. */
127ram_addr_t last_ram_offset;
128#endif
129
130CPUState *first_cpu;
131/* current CPU in the current thread. It is only valid inside
132 cpu_exec() */
133CPUState *cpu_single_env;
134/* 0 = Do not count executed instructions.
135 1 = Precise instruction counting.
136 2 = Adaptive rate instruction counting. */
137int use_icount = 0;
138/* Current instruction counter. While executing translated code this may
139 include some instructions that have not yet been executed. */
140int64_t qemu_icount;
141
142typedef struct PageDesc {
143 /* list of TBs intersecting this ram page */
144 TranslationBlock *first_tb;
145 /* in order to optimize self modifying code, we count the number
146 of lookups we do to a given page to use a bitmap */
147 unsigned int code_write_count;
148 uint8_t *code_bitmap;
149#if defined(CONFIG_USER_ONLY)
150 unsigned long flags;
151#endif
152} PageDesc;
153
154/* In system mode we want L1_MAP to be based on ram offsets,
155 while in user mode we want it to be based on virtual addresses. */
156#if !defined(CONFIG_USER_ONLY)
157#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
158# define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
159#else
160# define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
161#endif
162#else
163# define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
164#endif
165
166/* Size of the L2 (and L3, etc) page tables. */
167#define L2_BITS 10
168#define L2_SIZE (1 << L2_BITS)
169
170/* The bits remaining after N lower levels of page tables. */
171#define P_L1_BITS_REM \
172 ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
173#define V_L1_BITS_REM \
174 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
175
176/* Size of the L1 page table. Avoid silly small sizes. */
177#if P_L1_BITS_REM < 4
178#define P_L1_BITS (P_L1_BITS_REM + L2_BITS)
179#else
180#define P_L1_BITS P_L1_BITS_REM
181#endif
182
183#if V_L1_BITS_REM < 4
184#define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
185#else
186#define V_L1_BITS V_L1_BITS_REM
187#endif
188
189#define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS)
190#define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
191
192#define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
193#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
194
195unsigned long qemu_real_host_page_size;
196unsigned long qemu_host_page_bits;
197unsigned long qemu_host_page_size;
198unsigned long qemu_host_page_mask;
199
200/* This is a multi-level map on the virtual address space.
201 The bottom level has pointers to PageDesc. */
202static void *l1_map[V_L1_SIZE];
203
204#if !defined(CONFIG_USER_ONLY)
205typedef struct PhysPageDesc {
206 /* offset in host memory of the page + io_index in the low bits */
207 ram_addr_t phys_offset;
208 ram_addr_t region_offset;
209} PhysPageDesc;
210
211/* This is a multi-level map on the physical address space.
212 The bottom level has pointers to PhysPageDesc. */
213static void *l1_phys_map[P_L1_SIZE];
214
215static void io_mem_init(void);
216
217/* io memory support */
218CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
219CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
220void *io_mem_opaque[IO_MEM_NB_ENTRIES];
221static char io_mem_used[IO_MEM_NB_ENTRIES];
222static int io_mem_watch;
223#endif
224
225/* log support */
226#ifdef WIN32
227static const char *logfilename = "qemu.log";
228#else
229static const char *logfilename = "/tmp/qemu.log";
230#endif
231FILE *logfile;
232int loglevel;
233static int log_append = 0;
234
235/* statistics */
236#if !defined(CONFIG_USER_ONLY)
237static int tlb_flush_count;
238#endif
239static int tb_flush_count;
240static int tb_phys_invalidate_count;
241
242#ifdef _WIN32
243static void map_exec(void *addr, long size)
244{
245 DWORD old_protect;
246 VirtualProtect(addr, size,
247 PAGE_EXECUTE_READWRITE, &old_protect);
248
249}
250#else
251static void map_exec(void *addr, long size)
252{
253 unsigned long start, end, page_size;
254
255 page_size = getpagesize();
256 start = (unsigned long)addr;
257 start &= ~(page_size - 1);
258
259 end = (unsigned long)addr + size;
260 end += page_size - 1;
261 end &= ~(page_size - 1);
262
263 mprotect((void *)start, end - start,
264 PROT_READ | PROT_WRITE | PROT_EXEC);
265}
266#endif
267
268static void page_init(void)
269{
270 /* NOTE: we can always suppose that qemu_host_page_size >=
271 TARGET_PAGE_SIZE */
272#ifdef _WIN32
273 {
274 SYSTEM_INFO system_info;
275
276 GetSystemInfo(&system_info);
277 qemu_real_host_page_size = system_info.dwPageSize;
278 }
279#else
280 qemu_real_host_page_size = getpagesize();
281#endif
282 if (qemu_host_page_size == 0)
283 qemu_host_page_size = qemu_real_host_page_size;
284 if (qemu_host_page_size < TARGET_PAGE_SIZE)
285 qemu_host_page_size = TARGET_PAGE_SIZE;
286 qemu_host_page_bits = 0;
287 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
288 qemu_host_page_bits++;
289 qemu_host_page_mask = ~(qemu_host_page_size - 1);
290
291#if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
292 {
293#ifdef HAVE_KINFO_GETVMMAP
294 struct kinfo_vmentry *freep;
295 int i, cnt;
296
297 freep = kinfo_getvmmap(getpid(), &cnt);
298 if (freep) {
299 mmap_lock();
300 for (i = 0; i < cnt; i++) {
301 unsigned long startaddr, endaddr;
302
303 startaddr = freep[i].kve_start;
304 endaddr = freep[i].kve_end;
305 if (h2g_valid(startaddr)) {
306 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
307
308 if (h2g_valid(endaddr)) {
309 endaddr = h2g(endaddr);
310 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
311 } else {
312#if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
313 endaddr = ~0ul;
314 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
315#endif
316 }
317 }
318 }
319 free(freep);
320 mmap_unlock();
321 }
322#else
323 FILE *f;
324
325 last_brk = (unsigned long)sbrk(0);
326
327#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
328 f = fopen("/compat/linux/proc/self/maps", "r");
329#else
330 f = fopen("/proc/self/maps", "r");
331#endif
332 if (f) {
333 mmap_lock();
334
335 do {
336 unsigned long startaddr, endaddr;
337 int n;
338
339 n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
340
341 if (n == 2 && h2g_valid(startaddr)) {
342 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
343
344 if (h2g_valid(endaddr)) {
345 endaddr = h2g(endaddr);
346 } else {
347 endaddr = ~0ul;
348 }
349 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
350 }
351 } while (!feof(f));
352
353 fclose(f);
354 mmap_unlock();
355 }
356#endif
357 }
358#endif
359}
360
361static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
362{
363 PageDesc *pd;
364 void **lp;
365 int i;
366
367#if defined(CONFIG_USER_ONLY)
368 /* We can't use qemu_malloc because it may recurse into a locked mutex.
369 Neither can we record the new pages we reserve while allocating a
370 given page because that may recurse into an unallocated page table
371 entry. Stuff the allocations we do make into a queue and process
372 them after having completed one entire page table allocation. */
373
374 unsigned long reserve[2 * (V_L1_SHIFT / L2_BITS)];
375 int reserve_idx = 0;
376
377# define ALLOC(P, SIZE) \
378 do { \
379 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
380 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
381 if (h2g_valid(P)) { \
382 reserve[reserve_idx] = h2g(P); \
383 reserve[reserve_idx + 1] = SIZE; \
384 reserve_idx += 2; \
385 } \
386 } while (0)
387#else
388# define ALLOC(P, SIZE) \
389 do { P = qemu_mallocz(SIZE); } while (0)
390#endif
391
392 /* Level 1. Always allocated. */
393 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
394
395 /* Level 2..N-1. */
396 for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
397 void **p = *lp;
398
399 if (p == NULL) {
400 if (!alloc) {
401 return NULL;
402 }
403 ALLOC(p, sizeof(void *) * L2_SIZE);
404 *lp = p;
405 }
406
407 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
408 }
409
410 pd = *lp;
411 if (pd == NULL) {
412 if (!alloc) {
413 return NULL;
414 }
415 ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
416 *lp = pd;
417 }
418
419#undef ALLOC
420#if defined(CONFIG_USER_ONLY)
421 for (i = 0; i < reserve_idx; i += 2) {
422 unsigned long addr = reserve[i];
423 unsigned long len = reserve[i + 1];
424
425 page_set_flags(addr & TARGET_PAGE_MASK,
426 TARGET_PAGE_ALIGN(addr + len),
427 PAGE_RESERVED);
428 }
429#endif
430
431 return pd + (index & (L2_SIZE - 1));
432}
433
434static inline PageDesc *page_find(tb_page_addr_t index)
435{
436 return page_find_alloc(index, 0);
437}
438
439#if !defined(CONFIG_USER_ONLY)
440static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
441{
442 PhysPageDesc *pd;
443 void **lp;
444 int i;
445
446 /* Level 1. Always allocated. */
447 lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
448
449 /* Level 2..N-1. */
450 for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
451 void **p = *lp;
452 if (p == NULL) {
453 if (!alloc) {
454 return NULL;
455 }
456 *lp = p = qemu_mallocz(sizeof(void *) * L2_SIZE);
457 }
458 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
459 }
460
461 pd = *lp;
462 if (pd == NULL) {
463 int i;
464
465 if (!alloc) {
466 return NULL;
467 }
468
469 *lp = pd = qemu_malloc(sizeof(PhysPageDesc) * L2_SIZE);
470
471 for (i = 0; i < L2_SIZE; i++) {
472 pd[i].phys_offset = IO_MEM_UNASSIGNED;
473 pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
474 }
475 }
476
477 return pd + (index & (L2_SIZE - 1));
478}
479
480static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
481{
482 return phys_page_find_alloc(index, 0);
483}
484
485static void tlb_protect_code(ram_addr_t ram_addr);
486static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
487 target_ulong vaddr);
488#define mmap_lock() do { } while(0)
489#define mmap_unlock() do { } while(0)
490#endif
491
492#define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
493
494#if defined(CONFIG_USER_ONLY)
495/* Currently it is not recommended to allocate big chunks of data in
496 user mode. It will change when a dedicated libc will be used */
497#define USE_STATIC_CODE_GEN_BUFFER
498#endif
499
500#ifdef USE_STATIC_CODE_GEN_BUFFER
501static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
502 __attribute__((aligned (CODE_GEN_ALIGN)));
503#endif
504
505static void code_gen_alloc(unsigned long tb_size)
506{
507#ifdef USE_STATIC_CODE_GEN_BUFFER
508 code_gen_buffer = static_code_gen_buffer;
509 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
510 map_exec(code_gen_buffer, code_gen_buffer_size);
511#else
512 code_gen_buffer_size = tb_size;
513 if (code_gen_buffer_size == 0) {
514#if defined(CONFIG_USER_ONLY)
515 /* in user mode, phys_ram_size is not meaningful */
516 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
517#else
518 /* XXX: needs adjustments */
519 code_gen_buffer_size = (unsigned long)(ram_size / 4);
520#endif
521 }
522 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
523 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
524 /* The code gen buffer location may have constraints depending on
525 the host cpu and OS */
526#if defined(__linux__)
527 {
528 int flags;
529 void *start = NULL;
530
531 flags = MAP_PRIVATE | MAP_ANONYMOUS;
532#if defined(__x86_64__)
533 flags |= MAP_32BIT;
534 /* Cannot map more than that */
535 if (code_gen_buffer_size > (800 * 1024 * 1024))
536 code_gen_buffer_size = (800 * 1024 * 1024);
537#elif defined(__sparc_v9__)
538 // Map the buffer below 2G, so we can use direct calls and branches
539 flags |= MAP_FIXED;
540 start = (void *) 0x60000000UL;
541 if (code_gen_buffer_size > (512 * 1024 * 1024))
542 code_gen_buffer_size = (512 * 1024 * 1024);
543#elif defined(__arm__)
544 /* Map the buffer below 32M, so we can use direct calls and branches */
545 flags |= MAP_FIXED;
546 start = (void *) 0x01000000UL;
547 if (code_gen_buffer_size > 16 * 1024 * 1024)
548 code_gen_buffer_size = 16 * 1024 * 1024;
549#endif
550 code_gen_buffer = mmap(start, code_gen_buffer_size,
551 PROT_WRITE | PROT_READ | PROT_EXEC,
552 flags, -1, 0);
553 if (code_gen_buffer == MAP_FAILED) {
554 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
555 exit(1);
556 }
557 }
558#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
559 {
560 int flags;
561 void *addr = NULL;
562 flags = MAP_PRIVATE | MAP_ANONYMOUS;
563#if defined(__x86_64__)
564 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
565 * 0x40000000 is free */
566 flags |= MAP_FIXED;
567 addr = (void *)0x40000000;
568 /* Cannot map more than that */
569 if (code_gen_buffer_size > (800 * 1024 * 1024))
570 code_gen_buffer_size = (800 * 1024 * 1024);
571#endif
572 code_gen_buffer = mmap(addr, code_gen_buffer_size,
573 PROT_WRITE | PROT_READ | PROT_EXEC,
574 flags, -1, 0);
575 if (code_gen_buffer == MAP_FAILED) {
576 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
577 exit(1);
578 }
579 }
580#else
581 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
582 map_exec(code_gen_buffer, code_gen_buffer_size);
583#endif
584#endif /* !USE_STATIC_CODE_GEN_BUFFER */
585 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
586 code_gen_buffer_max_size = code_gen_buffer_size -
587 code_gen_max_block_size();
588 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
589 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
590}
591
592/* Must be called before using the QEMU cpus. 'tb_size' is the size
593 (in bytes) allocated to the translation buffer. Zero means default
594 size. */
595void cpu_exec_init_all(unsigned long tb_size)
596{
597 cpu_gen_init();
598 code_gen_alloc(tb_size);
599 code_gen_ptr = code_gen_buffer;
600 page_init();
601#if !defined(CONFIG_USER_ONLY)
602 io_mem_init();
603#endif
604}
605
606#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
607
608static int cpu_common_post_load(void *opaque, int version_id)
609{
610 CPUState *env = opaque;
611
612 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
613 version_id is increased. */
614 env->interrupt_request &= ~0x01;
615 tlb_flush(env, 1);
616
617 return 0;
618}
619
620static const VMStateDescription vmstate_cpu_common = {
621 .name = "cpu_common",
622 .version_id = 1,
623 .minimum_version_id = 1,
624 .minimum_version_id_old = 1,
625 .post_load = cpu_common_post_load,
626 .fields = (VMStateField []) {
627 VMSTATE_UINT32(halted, CPUState),
628 VMSTATE_UINT32(interrupt_request, CPUState),
629 VMSTATE_END_OF_LIST()
630 }
631};
632#endif
633
634CPUState *qemu_get_cpu(int cpu)
635{
636 CPUState *env = first_cpu;
637
638 while (env) {
639 if (env->cpu_index == cpu)
640 break;
641 env = env->next_cpu;
642 }
643
644 return env;
645}
646
647void cpu_exec_init(CPUState *env)
648{
649 CPUState **penv;
650 int cpu_index;
651
652#if defined(CONFIG_USER_ONLY)
653 cpu_list_lock();
654#endif
655 env->next_cpu = NULL;
656 penv = &first_cpu;
657 cpu_index = 0;
658 while (*penv != NULL) {
659 penv = &(*penv)->next_cpu;
660 cpu_index++;
661 }
662 env->cpu_index = cpu_index;
663 env->numa_node = 0;
664 QTAILQ_INIT(&env->breakpoints);
665 QTAILQ_INIT(&env->watchpoints);
666 *penv = env;
667#if defined(CONFIG_USER_ONLY)
668 cpu_list_unlock();
669#endif
670#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
671 vmstate_register(cpu_index, &vmstate_cpu_common, env);
672 register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
673 cpu_save, cpu_load, env);
674#endif
675}
676
677static inline void invalidate_page_bitmap(PageDesc *p)
678{
679 if (p->code_bitmap) {
680 qemu_free(p->code_bitmap);
681 p->code_bitmap = NULL;
682 }
683 p->code_write_count = 0;
684}
685
686/* Set to NULL all the 'first_tb' fields in all PageDescs. */
687
688static void page_flush_tb_1 (int level, void **lp)
689{
690 int i;
691
692 if (*lp == NULL) {
693 return;
694 }
695 if (level == 0) {
696 PageDesc *pd = *lp;
697 for (i = 0; i < L2_SIZE; ++i) {
698 pd[i].first_tb = NULL;
699 invalidate_page_bitmap(pd + i);
700 }
701 } else {
702 void **pp = *lp;
703 for (i = 0; i < L2_SIZE; ++i) {
704 page_flush_tb_1 (level - 1, pp + i);
705 }
706 }
707}
708
709static void page_flush_tb(void)
710{
711 int i;
712 for (i = 0; i < V_L1_SIZE; i++) {
713 page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
714 }
715}
716
717/* flush all the translation blocks */
718/* XXX: tb_flush is currently not thread safe */
719void tb_flush(CPUState *env1)
720{
721 CPUState *env;
722#if defined(DEBUG_FLUSH)
723 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
724 (unsigned long)(code_gen_ptr - code_gen_buffer),
725 nb_tbs, nb_tbs > 0 ?
726 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
727#endif
728 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
729 cpu_abort(env1, "Internal error: code buffer overflow\n");
730
731 nb_tbs = 0;
732
733 for(env = first_cpu; env != NULL; env = env->next_cpu) {
734 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
735 }
736
737 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
738 page_flush_tb();
739
740 code_gen_ptr = code_gen_buffer;
741 /* XXX: flush processor icache at this point if cache flush is
742 expensive */
743 tb_flush_count++;
744}
745
746#ifdef DEBUG_TB_CHECK
747
748static void tb_invalidate_check(target_ulong address)
749{
750 TranslationBlock *tb;
751 int i;
752 address &= TARGET_PAGE_MASK;
753 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
754 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
755 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
756 address >= tb->pc + tb->size)) {
757 printf("ERROR invalidate: address=" TARGET_FMT_lx
758 " PC=%08lx size=%04x\n",
759 address, (long)tb->pc, tb->size);
760 }
761 }
762 }
763}
764
765/* verify that all the pages have correct rights for code */
766static void tb_page_check(void)
767{
768 TranslationBlock *tb;
769 int i, flags1, flags2;
770
771 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
772 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
773 flags1 = page_get_flags(tb->pc);
774 flags2 = page_get_flags(tb->pc + tb->size - 1);
775 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
776 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
777 (long)tb->pc, tb->size, flags1, flags2);
778 }
779 }
780 }
781}
782
783#endif
784
785/* invalidate one TB */
786static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
787 int next_offset)
788{
789 TranslationBlock *tb1;
790 for(;;) {
791 tb1 = *ptb;
792 if (tb1 == tb) {
793 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
794 break;
795 }
796 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
797 }
798}
799
800static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
801{
802 TranslationBlock *tb1;
803 unsigned int n1;
804
805 for(;;) {
806 tb1 = *ptb;
807 n1 = (long)tb1 & 3;
808 tb1 = (TranslationBlock *)((long)tb1 & ~3);
809 if (tb1 == tb) {
810 *ptb = tb1->page_next[n1];
811 break;
812 }
813 ptb = &tb1->page_next[n1];
814 }
815}
816
817static inline void tb_jmp_remove(TranslationBlock *tb, int n)
818{
819 TranslationBlock *tb1, **ptb;
820 unsigned int n1;
821
822 ptb = &tb->jmp_next[n];
823 tb1 = *ptb;
824 if (tb1) {
825 /* find tb(n) in circular list */
826 for(;;) {
827 tb1 = *ptb;
828 n1 = (long)tb1 & 3;
829 tb1 = (TranslationBlock *)((long)tb1 & ~3);
830 if (n1 == n && tb1 == tb)
831 break;
832 if (n1 == 2) {
833 ptb = &tb1->jmp_first;
834 } else {
835 ptb = &tb1->jmp_next[n1];
836 }
837 }
838 /* now we can suppress tb(n) from the list */
839 *ptb = tb->jmp_next[n];
840
841 tb->jmp_next[n] = NULL;
842 }
843}
844
845/* reset the jump entry 'n' of a TB so that it is not chained to
846 another TB */
847static inline void tb_reset_jump(TranslationBlock *tb, int n)
848{
849 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
850}
851
852void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
853{
854 CPUState *env;
855 PageDesc *p;
856 unsigned int h, n1;
857 tb_page_addr_t phys_pc;
858 TranslationBlock *tb1, *tb2;
859
860 /* remove the TB from the hash list */
861 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
862 h = tb_phys_hash_func(phys_pc);
863 tb_remove(&tb_phys_hash[h], tb,
864 offsetof(TranslationBlock, phys_hash_next));
865
866 /* remove the TB from the page list */
867 if (tb->page_addr[0] != page_addr) {
868 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
869 tb_page_remove(&p->first_tb, tb);
870 invalidate_page_bitmap(p);
871 }
872 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
873 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
874 tb_page_remove(&p->first_tb, tb);
875 invalidate_page_bitmap(p);
876 }
877
878 tb_invalidated_flag = 1;
879
880 /* remove the TB from the hash list */
881 h = tb_jmp_cache_hash_func(tb->pc);
882 for(env = first_cpu; env != NULL; env = env->next_cpu) {
883 if (env->tb_jmp_cache[h] == tb)
884 env->tb_jmp_cache[h] = NULL;
885 }
886
887 /* suppress this TB from the two jump lists */
888 tb_jmp_remove(tb, 0);
889 tb_jmp_remove(tb, 1);
890
891 /* suppress any remaining jumps to this TB */
892 tb1 = tb->jmp_first;
893 for(;;) {
894 n1 = (long)tb1 & 3;
895 if (n1 == 2)
896 break;
897 tb1 = (TranslationBlock *)((long)tb1 & ~3);
898 tb2 = tb1->jmp_next[n1];
899 tb_reset_jump(tb1, n1);
900 tb1->jmp_next[n1] = NULL;
901 tb1 = tb2;
902 }
903 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
904
905 tb_phys_invalidate_count++;
906}
907
908static inline void set_bits(uint8_t *tab, int start, int len)
909{
910 int end, mask, end1;
911
912 end = start + len;
913 tab += start >> 3;
914 mask = 0xff << (start & 7);
915 if ((start & ~7) == (end & ~7)) {
916 if (start < end) {
917 mask &= ~(0xff << (end & 7));
918 *tab |= mask;
919 }
920 } else {
921 *tab++ |= mask;
922 start = (start + 8) & ~7;
923 end1 = end & ~7;
924 while (start < end1) {
925 *tab++ = 0xff;
926 start += 8;
927 }
928 if (start < end) {
929 mask = ~(0xff << (end & 7));
930 *tab |= mask;
931 }
932 }
933}
934
935static void build_page_bitmap(PageDesc *p)
936{
937 int n, tb_start, tb_end;
938 TranslationBlock *tb;
939
940 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
941
942 tb = p->first_tb;
943 while (tb != NULL) {
944 n = (long)tb & 3;
945 tb = (TranslationBlock *)((long)tb & ~3);
946 /* NOTE: this is subtle as a TB may span two physical pages */
947 if (n == 0) {
948 /* NOTE: tb_end may be after the end of the page, but
949 it is not a problem */
950 tb_start = tb->pc & ~TARGET_PAGE_MASK;
951 tb_end = tb_start + tb->size;
952 if (tb_end > TARGET_PAGE_SIZE)
953 tb_end = TARGET_PAGE_SIZE;
954 } else {
955 tb_start = 0;
956 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
957 }
958 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
959 tb = tb->page_next[n];
960 }
961}
962
963TranslationBlock *tb_gen_code(CPUState *env,
964 target_ulong pc, target_ulong cs_base,
965 int flags, int cflags)
966{
967 TranslationBlock *tb;
968 uint8_t *tc_ptr;
969 tb_page_addr_t phys_pc, phys_page2;
970 target_ulong virt_page2;
971 int code_gen_size;
972
973 phys_pc = get_page_addr_code(env, pc);
974 tb = tb_alloc(pc);
975 if (!tb) {
976 /* flush must be done */
977 tb_flush(env);
978 /* cannot fail at this point */
979 tb = tb_alloc(pc);
980 /* Don't forget to invalidate previous TB info. */
981 tb_invalidated_flag = 1;
982 }
983 tc_ptr = code_gen_ptr;
984 tb->tc_ptr = tc_ptr;
985 tb->cs_base = cs_base;
986 tb->flags = flags;
987 tb->cflags = cflags;
988 cpu_gen_code(env, tb, &code_gen_size);
989 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
990
991 /* check next page if needed */
992 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
993 phys_page2 = -1;
994 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
995 phys_page2 = get_page_addr_code(env, virt_page2);
996 }
997 tb_link_page(tb, phys_pc, phys_page2);
998 return tb;
999}
1000
1001/* invalidate all TBs which intersect with the target physical page
1002 starting in range [start;end[. NOTE: start and end must refer to
1003 the same physical page. 'is_cpu_write_access' should be true if called
1004 from a real cpu write access: the virtual CPU will exit the current
1005 TB if code is modified inside this TB. */
1006void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1007 int is_cpu_write_access)
1008{
1009 TranslationBlock *tb, *tb_next, *saved_tb;
1010 CPUState *env = cpu_single_env;
1011 tb_page_addr_t tb_start, tb_end;
1012 PageDesc *p;
1013 int n;
1014#ifdef TARGET_HAS_PRECISE_SMC
1015 int current_tb_not_found = is_cpu_write_access;
1016 TranslationBlock *current_tb = NULL;
1017 int current_tb_modified = 0;
1018 target_ulong current_pc = 0;
1019 target_ulong current_cs_base = 0;
1020 int current_flags = 0;
1021#endif /* TARGET_HAS_PRECISE_SMC */
1022
1023 p = page_find(start >> TARGET_PAGE_BITS);
1024 if (!p)
1025 return;
1026 if (!p->code_bitmap &&
1027 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1028 is_cpu_write_access) {
1029 /* build code bitmap */
1030 build_page_bitmap(p);
1031 }
1032
1033 /* we remove all the TBs in the range [start, end[ */
1034 /* XXX: see if in some cases it could be faster to invalidate all the code */
1035 tb = p->first_tb;
1036 while (tb != NULL) {
1037 n = (long)tb & 3;
1038 tb = (TranslationBlock *)((long)tb & ~3);
1039 tb_next = tb->page_next[n];
1040 /* NOTE: this is subtle as a TB may span two physical pages */
1041 if (n == 0) {
1042 /* NOTE: tb_end may be after the end of the page, but
1043 it is not a problem */
1044 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1045 tb_end = tb_start + tb->size;
1046 } else {
1047 tb_start = tb->page_addr[1];
1048 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1049 }
1050 if (!(tb_end <= start || tb_start >= end)) {
1051#ifdef TARGET_HAS_PRECISE_SMC
1052 if (current_tb_not_found) {
1053 current_tb_not_found = 0;
1054 current_tb = NULL;
1055 if (env->mem_io_pc) {
1056 /* now we have a real cpu fault */
1057 current_tb = tb_find_pc(env->mem_io_pc);
1058 }
1059 }
1060 if (current_tb == tb &&
1061 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1062 /* If we are modifying the current TB, we must stop
1063 its execution. We could be more precise by checking
1064 that the modification is after the current PC, but it
1065 would require a specialized function to partially
1066 restore the CPU state */
1067
1068 current_tb_modified = 1;
1069 cpu_restore_state(current_tb, env,
1070 env->mem_io_pc, NULL);
1071 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1072 &current_flags);
1073 }
1074#endif /* TARGET_HAS_PRECISE_SMC */
1075 /* we need to do that to handle the case where a signal
1076 occurs while doing tb_phys_invalidate() */
1077 saved_tb = NULL;
1078 if (env) {
1079 saved_tb = env->current_tb;
1080 env->current_tb = NULL;
1081 }
1082 tb_phys_invalidate(tb, -1);
1083 if (env) {
1084 env->current_tb = saved_tb;
1085 if (env->interrupt_request && env->current_tb)
1086 cpu_interrupt(env, env->interrupt_request);
1087 }
1088 }
1089 tb = tb_next;
1090 }
1091#if !defined(CONFIG_USER_ONLY)
1092 /* if no code remaining, no need to continue to use slow writes */
1093 if (!p->first_tb) {
1094 invalidate_page_bitmap(p);
1095 if (is_cpu_write_access) {
1096 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1097 }
1098 }
1099#endif
1100#ifdef TARGET_HAS_PRECISE_SMC
1101 if (current_tb_modified) {
1102 /* we generate a block containing just the instruction
1103 modifying the memory. It will ensure that it cannot modify
1104 itself */
1105 env->current_tb = NULL;
1106 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1107 cpu_resume_from_signal(env, NULL);
1108 }
1109#endif
1110}
1111
1112/* len must be <= 8 and start must be a multiple of len */
1113static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1114{
1115 PageDesc *p;
1116 int offset, b;
1117#if 0
1118 if (1) {
1119 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1120 cpu_single_env->mem_io_vaddr, len,
1121 cpu_single_env->eip,
1122 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1123 }
1124#endif
1125 p = page_find(start >> TARGET_PAGE_BITS);
1126 if (!p)
1127 return;
1128 if (p->code_bitmap) {
1129 offset = start & ~TARGET_PAGE_MASK;
1130 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1131 if (b & ((1 << len) - 1))
1132 goto do_invalidate;
1133 } else {
1134 do_invalidate:
1135 tb_invalidate_phys_page_range(start, start + len, 1);
1136 }
1137}
1138
1139#if !defined(CONFIG_SOFTMMU)
1140static void tb_invalidate_phys_page(tb_page_addr_t addr,
1141 unsigned long pc, void *puc)
1142{
1143 TranslationBlock *tb;
1144 PageDesc *p;
1145 int n;
1146#ifdef TARGET_HAS_PRECISE_SMC
1147 TranslationBlock *current_tb = NULL;
1148 CPUState *env = cpu_single_env;
1149 int current_tb_modified = 0;
1150 target_ulong current_pc = 0;
1151 target_ulong current_cs_base = 0;
1152 int current_flags = 0;
1153#endif
1154
1155 addr &= TARGET_PAGE_MASK;
1156 p = page_find(addr >> TARGET_PAGE_BITS);
1157 if (!p)
1158 return;
1159 tb = p->first_tb;
1160#ifdef TARGET_HAS_PRECISE_SMC
1161 if (tb && pc != 0) {
1162 current_tb = tb_find_pc(pc);
1163 }
1164#endif
1165 while (tb != NULL) {
1166 n = (long)tb & 3;
1167 tb = (TranslationBlock *)((long)tb & ~3);
1168#ifdef TARGET_HAS_PRECISE_SMC
1169 if (current_tb == tb &&
1170 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1171 /* If we are modifying the current TB, we must stop
1172 its execution. We could be more precise by checking
1173 that the modification is after the current PC, but it
1174 would require a specialized function to partially
1175 restore the CPU state */
1176
1177 current_tb_modified = 1;
1178 cpu_restore_state(current_tb, env, pc, puc);
1179 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1180 &current_flags);
1181 }
1182#endif /* TARGET_HAS_PRECISE_SMC */
1183 tb_phys_invalidate(tb, addr);
1184 tb = tb->page_next[n];
1185 }
1186 p->first_tb = NULL;
1187#ifdef TARGET_HAS_PRECISE_SMC
1188 if (current_tb_modified) {
1189 /* we generate a block containing just the instruction
1190 modifying the memory. It will ensure that it cannot modify
1191 itself */
1192 env->current_tb = NULL;
1193 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1194 cpu_resume_from_signal(env, puc);
1195 }
1196#endif
1197}
1198#endif
1199
1200/* add the tb in the target page and protect it if necessary */
1201static inline void tb_alloc_page(TranslationBlock *tb,
1202 unsigned int n, tb_page_addr_t page_addr)
1203{
1204 PageDesc *p;
1205 TranslationBlock *last_first_tb;
1206
1207 tb->page_addr[n] = page_addr;
1208 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1209 tb->page_next[n] = p->first_tb;
1210 last_first_tb = p->first_tb;
1211 p->first_tb = (TranslationBlock *)((long)tb | n);
1212 invalidate_page_bitmap(p);
1213
1214#if defined(TARGET_HAS_SMC) || 1
1215
1216#if defined(CONFIG_USER_ONLY)
1217 if (p->flags & PAGE_WRITE) {
1218 target_ulong addr;
1219 PageDesc *p2;
1220 int prot;
1221
1222 /* force the host page as non writable (writes will have a
1223 page fault + mprotect overhead) */
1224 page_addr &= qemu_host_page_mask;
1225 prot = 0;
1226 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1227 addr += TARGET_PAGE_SIZE) {
1228
1229 p2 = page_find (addr >> TARGET_PAGE_BITS);
1230 if (!p2)
1231 continue;
1232 prot |= p2->flags;
1233 p2->flags &= ~PAGE_WRITE;
1234 }
1235 mprotect(g2h(page_addr), qemu_host_page_size,
1236 (prot & PAGE_BITS) & ~PAGE_WRITE);
1237#ifdef DEBUG_TB_INVALIDATE
1238 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1239 page_addr);
1240#endif
1241 }
1242#else
1243 /* if some code is already present, then the pages are already
1244 protected. So we handle the case where only the first TB is
1245 allocated in a physical page */
1246 if (!last_first_tb) {
1247 tlb_protect_code(page_addr);
1248 }
1249#endif
1250
1251#endif /* TARGET_HAS_SMC */
1252}
1253
1254/* Allocate a new translation block. Flush the translation buffer if
1255 too many translation blocks or too much generated code. */
1256TranslationBlock *tb_alloc(target_ulong pc)
1257{
1258 TranslationBlock *tb;
1259
1260 if (nb_tbs >= code_gen_max_blocks ||
1261 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1262 return NULL;
1263 tb = &tbs[nb_tbs++];
1264 tb->pc = pc;
1265 tb->cflags = 0;
1266 return tb;
1267}
1268
1269void tb_free(TranslationBlock *tb)
1270{
1271 /* In practice this is mostly used for single use temporary TB
1272 Ignore the hard cases and just back up if this TB happens to
1273 be the last one generated. */
1274 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1275 code_gen_ptr = tb->tc_ptr;
1276 nb_tbs--;
1277 }
1278}
1279
1280/* add a new TB and link it to the physical page tables. phys_page2 is
1281 (-1) to indicate that only one page contains the TB. */
1282void tb_link_page(TranslationBlock *tb,
1283 tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
1284{
1285 unsigned int h;
1286 TranslationBlock **ptb;
1287
1288 /* Grab the mmap lock to stop another thread invalidating this TB
1289 before we are done. */
1290 mmap_lock();
1291 /* add in the physical hash table */
1292 h = tb_phys_hash_func(phys_pc);
1293 ptb = &tb_phys_hash[h];
1294 tb->phys_hash_next = *ptb;
1295 *ptb = tb;
1296
1297 /* add in the page list */
1298 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1299 if (phys_page2 != -1)
1300 tb_alloc_page(tb, 1, phys_page2);
1301 else
1302 tb->page_addr[1] = -1;
1303
1304 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1305 tb->jmp_next[0] = NULL;
1306 tb->jmp_next[1] = NULL;
1307
1308 /* init original jump addresses */
1309 if (tb->tb_next_offset[0] != 0xffff)
1310 tb_reset_jump(tb, 0);
1311 if (tb->tb_next_offset[1] != 0xffff)
1312 tb_reset_jump(tb, 1);
1313
1314#ifdef DEBUG_TB_CHECK
1315 tb_page_check();
1316#endif
1317 mmap_unlock();
1318}
1319
1320/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1321 tb[1].tc_ptr. Return NULL if not found */
1322TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1323{
1324 int m_min, m_max, m;
1325 unsigned long v;
1326 TranslationBlock *tb;
1327
1328 if (nb_tbs <= 0)
1329 return NULL;
1330 if (tc_ptr < (unsigned long)code_gen_buffer ||
1331 tc_ptr >= (unsigned long)code_gen_ptr)
1332 return NULL;
1333 /* binary search (cf Knuth) */
1334 m_min = 0;
1335 m_max = nb_tbs - 1;
1336 while (m_min <= m_max) {
1337 m = (m_min + m_max) >> 1;
1338 tb = &tbs[m];
1339 v = (unsigned long)tb->tc_ptr;
1340 if (v == tc_ptr)
1341 return tb;
1342 else if (tc_ptr < v) {
1343 m_max = m - 1;
1344 } else {
1345 m_min = m + 1;
1346 }
1347 }
1348 return &tbs[m_max];
1349}
1350
1351static void tb_reset_jump_recursive(TranslationBlock *tb);
1352
1353static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1354{
1355 TranslationBlock *tb1, *tb_next, **ptb;
1356 unsigned int n1;
1357
1358 tb1 = tb->jmp_next[n];
1359 if (tb1 != NULL) {
1360 /* find head of list */
1361 for(;;) {
1362 n1 = (long)tb1 & 3;
1363 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1364 if (n1 == 2)
1365 break;
1366 tb1 = tb1->jmp_next[n1];
1367 }
1368 /* we are now sure now that tb jumps to tb1 */
1369 tb_next = tb1;
1370
1371 /* remove tb from the jmp_first list */
1372 ptb = &tb_next->jmp_first;
1373 for(;;) {
1374 tb1 = *ptb;
1375 n1 = (long)tb1 & 3;
1376 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1377 if (n1 == n && tb1 == tb)
1378 break;
1379 ptb = &tb1->jmp_next[n1];
1380 }
1381 *ptb = tb->jmp_next[n];
1382 tb->jmp_next[n] = NULL;
1383
1384 /* suppress the jump to next tb in generated code */
1385 tb_reset_jump(tb, n);
1386
1387 /* suppress jumps in the tb on which we could have jumped */
1388 tb_reset_jump_recursive(tb_next);
1389 }
1390}
1391
1392static void tb_reset_jump_recursive(TranslationBlock *tb)
1393{
1394 tb_reset_jump_recursive2(tb, 0);
1395 tb_reset_jump_recursive2(tb, 1);
1396}
1397
1398#if defined(TARGET_HAS_ICE)
1399#if defined(CONFIG_USER_ONLY)
1400static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1401{
1402 tb_invalidate_phys_page_range(pc, pc + 1, 0);
1403}
1404#else
1405static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1406{
1407 target_phys_addr_t addr;
1408 target_ulong pd;
1409 ram_addr_t ram_addr;
1410 PhysPageDesc *p;
1411
1412 addr = cpu_get_phys_page_debug(env, pc);
1413 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1414 if (!p) {
1415 pd = IO_MEM_UNASSIGNED;
1416 } else {
1417 pd = p->phys_offset;
1418 }
1419 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1420 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1421}
1422#endif
1423#endif /* TARGET_HAS_ICE */
1424
1425#if defined(CONFIG_USER_ONLY)
1426void cpu_watchpoint_remove_all(CPUState *env, int mask)
1427
1428{
1429}
1430
1431int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1432 int flags, CPUWatchpoint **watchpoint)
1433{
1434 return -ENOSYS;
1435}
1436#else
1437/* Add a watchpoint. */
1438int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1439 int flags, CPUWatchpoint **watchpoint)
1440{
1441 target_ulong len_mask = ~(len - 1);
1442 CPUWatchpoint *wp;
1443
1444 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1445 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1446 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1447 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1448 return -EINVAL;
1449 }
1450 wp = qemu_malloc(sizeof(*wp));
1451
1452 wp->vaddr = addr;
1453 wp->len_mask = len_mask;
1454 wp->flags = flags;
1455
1456 /* keep all GDB-injected watchpoints in front */
1457 if (flags & BP_GDB)
1458 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1459 else
1460 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1461
1462 tlb_flush_page(env, addr);
1463
1464 if (watchpoint)
1465 *watchpoint = wp;
1466 return 0;
1467}
1468
1469/* Remove a specific watchpoint. */
1470int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1471 int flags)
1472{
1473 target_ulong len_mask = ~(len - 1);
1474 CPUWatchpoint *wp;
1475
1476 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1477 if (addr == wp->vaddr && len_mask == wp->len_mask
1478 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1479 cpu_watchpoint_remove_by_ref(env, wp);
1480 return 0;
1481 }
1482 }
1483 return -ENOENT;
1484}
1485
1486/* Remove a specific watchpoint by reference. */
1487void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1488{
1489 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1490
1491 tlb_flush_page(env, watchpoint->vaddr);
1492
1493 qemu_free(watchpoint);
1494}
1495
1496/* Remove all matching watchpoints. */
1497void cpu_watchpoint_remove_all(CPUState *env, int mask)
1498{
1499 CPUWatchpoint *wp, *next;
1500
1501 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1502 if (wp->flags & mask)
1503 cpu_watchpoint_remove_by_ref(env, wp);
1504 }
1505}
1506#endif
1507
1508/* Add a breakpoint. */
1509int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1510 CPUBreakpoint **breakpoint)
1511{
1512#if defined(TARGET_HAS_ICE)
1513 CPUBreakpoint *bp;
1514
1515 bp = qemu_malloc(sizeof(*bp));
1516
1517 bp->pc = pc;
1518 bp->flags = flags;
1519
1520 /* keep all GDB-injected breakpoints in front */
1521 if (flags & BP_GDB)
1522 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1523 else
1524 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1525
1526 breakpoint_invalidate(env, pc);
1527
1528 if (breakpoint)
1529 *breakpoint = bp;
1530 return 0;
1531#else
1532 return -ENOSYS;
1533#endif
1534}
1535
1536/* Remove a specific breakpoint. */
1537int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1538{
1539#if defined(TARGET_HAS_ICE)
1540 CPUBreakpoint *bp;
1541
1542 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1543 if (bp->pc == pc && bp->flags == flags) {
1544 cpu_breakpoint_remove_by_ref(env, bp);
1545 return 0;
1546 }
1547 }
1548 return -ENOENT;
1549#else
1550 return -ENOSYS;
1551#endif
1552}
1553
1554/* Remove a specific breakpoint by reference. */
1555void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1556{
1557#if defined(TARGET_HAS_ICE)
1558 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1559
1560 breakpoint_invalidate(env, breakpoint->pc);
1561
1562 qemu_free(breakpoint);
1563#endif
1564}
1565
1566/* Remove all matching breakpoints. */
1567void cpu_breakpoint_remove_all(CPUState *env, int mask)
1568{
1569#if defined(TARGET_HAS_ICE)
1570 CPUBreakpoint *bp, *next;
1571
1572 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1573 if (bp->flags & mask)
1574 cpu_breakpoint_remove_by_ref(env, bp);
1575 }
1576#endif
1577}
1578
1579/* enable or disable single step mode. EXCP_DEBUG is returned by the
1580 CPU loop after each instruction */
1581void cpu_single_step(CPUState *env, int enabled)
1582{
1583#if defined(TARGET_HAS_ICE)
1584 if (env->singlestep_enabled != enabled) {
1585 env->singlestep_enabled = enabled;
1586 if (kvm_enabled())
1587 kvm_update_guest_debug(env, 0);
1588 else {
1589 /* must flush all the translated code to avoid inconsistencies */
1590 /* XXX: only flush what is necessary */
1591 tb_flush(env);
1592 }
1593 }
1594#endif
1595}
1596
1597/* enable or disable low levels log */
1598void cpu_set_log(int log_flags)
1599{
1600 loglevel = log_flags;
1601 if (loglevel && !logfile) {
1602 logfile = fopen(logfilename, log_append ? "a" : "w");
1603 if (!logfile) {
1604 perror(logfilename);
1605 _exit(1);
1606 }
1607#if !defined(CONFIG_SOFTMMU)
1608 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1609 {
1610 static char logfile_buf[4096];
1611 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1612 }
1613#elif !defined(_WIN32)
1614 /* Win32 doesn't support line-buffering and requires size >= 2 */
1615 setvbuf(logfile, NULL, _IOLBF, 0);
1616#endif
1617 log_append = 1;
1618 }
1619 if (!loglevel && logfile) {
1620 fclose(logfile);
1621 logfile = NULL;
1622 }
1623}
1624
1625void cpu_set_log_filename(const char *filename)
1626{
1627 logfilename = strdup(filename);
1628 if (logfile) {
1629 fclose(logfile);
1630 logfile = NULL;
1631 }
1632 cpu_set_log(loglevel);
1633}
1634
1635static void cpu_unlink_tb(CPUState *env)
1636{
1637 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1638 problem and hope the cpu will stop of its own accord. For userspace
1639 emulation this often isn't actually as bad as it sounds. Often
1640 signals are used primarily to interrupt blocking syscalls. */
1641 TranslationBlock *tb;
1642 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1643
1644 spin_lock(&interrupt_lock);
1645 tb = env->current_tb;
1646 /* if the cpu is currently executing code, we must unlink it and
1647 all the potentially executing TB */
1648 if (tb) {
1649 env->current_tb = NULL;
1650 tb_reset_jump_recursive(tb);
1651 }
1652 spin_unlock(&interrupt_lock);
1653}
1654
1655/* mask must never be zero, except for A20 change call */
1656void cpu_interrupt(CPUState *env, int mask)
1657{
1658 int old_mask;
1659
1660 old_mask = env->interrupt_request;
1661 env->interrupt_request |= mask;
1662
1663#ifndef CONFIG_USER_ONLY
1664 /*
1665 * If called from iothread context, wake the target cpu in
1666 * case its halted.
1667 */
1668 if (!qemu_cpu_self(env)) {
1669 qemu_cpu_kick(env);
1670 return;
1671 }
1672#endif
1673
1674 if (use_icount) {
1675 env->icount_decr.u16.high = 0xffff;
1676#ifndef CONFIG_USER_ONLY
1677 if (!can_do_io(env)
1678 && (mask & ~old_mask) != 0) {
1679 cpu_abort(env, "Raised interrupt while not in I/O function");
1680 }
1681#endif
1682 } else {
1683 cpu_unlink_tb(env);
1684 }
1685}
1686
1687void cpu_reset_interrupt(CPUState *env, int mask)
1688{
1689 env->interrupt_request &= ~mask;
1690}
1691
1692void cpu_exit(CPUState *env)
1693{
1694 env->exit_request = 1;
1695 cpu_unlink_tb(env);
1696}
1697
1698const CPULogItem cpu_log_items[] = {
1699 { CPU_LOG_TB_OUT_ASM, "out_asm",
1700 "show generated host assembly code for each compiled TB" },
1701 { CPU_LOG_TB_IN_ASM, "in_asm",
1702 "show target assembly code for each compiled TB" },
1703 { CPU_LOG_TB_OP, "op",
1704 "show micro ops for each compiled TB" },
1705 { CPU_LOG_TB_OP_OPT, "op_opt",
1706 "show micro ops "
1707#ifdef TARGET_I386
1708 "before eflags optimization and "
1709#endif
1710 "after liveness analysis" },
1711 { CPU_LOG_INT, "int",
1712 "show interrupts/exceptions in short format" },
1713 { CPU_LOG_EXEC, "exec",
1714 "show trace before each executed TB (lots of logs)" },
1715 { CPU_LOG_TB_CPU, "cpu",
1716 "show CPU state before block translation" },
1717#ifdef TARGET_I386
1718 { CPU_LOG_PCALL, "pcall",
1719 "show protected mode far calls/returns/exceptions" },
1720 { CPU_LOG_RESET, "cpu_reset",
1721 "show CPU state before CPU resets" },
1722#endif
1723#ifdef DEBUG_IOPORT
1724 { CPU_LOG_IOPORT, "ioport",
1725 "show all i/o ports accesses" },
1726#endif
1727 { 0, NULL, NULL },
1728};
1729
1730#ifndef CONFIG_USER_ONLY
1731static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
1732 = QLIST_HEAD_INITIALIZER(memory_client_list);
1733
1734static void cpu_notify_set_memory(target_phys_addr_t start_addr,
1735 ram_addr_t size,
1736 ram_addr_t phys_offset)
1737{
1738 CPUPhysMemoryClient *client;
1739 QLIST_FOREACH(client, &memory_client_list, list) {
1740 client->set_memory(client, start_addr, size, phys_offset);
1741 }
1742}
1743
1744static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
1745 target_phys_addr_t end)
1746{
1747 CPUPhysMemoryClient *client;
1748 QLIST_FOREACH(client, &memory_client_list, list) {
1749 int r = client->sync_dirty_bitmap(client, start, end);
1750 if (r < 0)
1751 return r;
1752 }
1753 return 0;
1754}
1755
1756static int cpu_notify_migration_log(int enable)
1757{
1758 CPUPhysMemoryClient *client;
1759 QLIST_FOREACH(client, &memory_client_list, list) {
1760 int r = client->migration_log(client, enable);
1761 if (r < 0)
1762 return r;
1763 }
1764 return 0;
1765}
1766
1767static void phys_page_for_each_1(CPUPhysMemoryClient *client,
1768 int level, void **lp)
1769{
1770 int i;
1771
1772 if (*lp == NULL) {
1773 return;
1774 }
1775 if (level == 0) {
1776 PhysPageDesc *pd = *lp;
1777 for (i = 0; i < L2_SIZE; ++i) {
1778 if (pd[i].phys_offset != IO_MEM_UNASSIGNED) {
1779 client->set_memory(client, pd[i].region_offset,
1780 TARGET_PAGE_SIZE, pd[i].phys_offset);
1781 }
1782 }
1783 } else {
1784 void **pp = *lp;
1785 for (i = 0; i < L2_SIZE; ++i) {
1786 phys_page_for_each_1(client, level - 1, pp + i);
1787 }
1788 }
1789}
1790
1791static void phys_page_for_each(CPUPhysMemoryClient *client)
1792{
1793 int i;
1794 for (i = 0; i < P_L1_SIZE; ++i) {
1795 phys_page_for_each_1(client, P_L1_SHIFT / L2_BITS - 1,
1796 l1_phys_map + 1);
1797 }
1798}
1799
1800void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
1801{
1802 QLIST_INSERT_HEAD(&memory_client_list, client, list);
1803 phys_page_for_each(client);
1804}
1805
1806void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
1807{
1808 QLIST_REMOVE(client, list);
1809}
1810#endif
1811
1812static int cmp1(const char *s1, int n, const char *s2)
1813{
1814 if (strlen(s2) != n)
1815 return 0;
1816 return memcmp(s1, s2, n) == 0;
1817}
1818
1819/* takes a comma separated list of log masks. Return 0 if error. */
1820int cpu_str_to_log_mask(const char *str)
1821{
1822 const CPULogItem *item;
1823 int mask;
1824 const char *p, *p1;
1825
1826 p = str;
1827 mask = 0;
1828 for(;;) {
1829 p1 = strchr(p, ',');
1830 if (!p1)
1831 p1 = p + strlen(p);
1832 if(cmp1(p,p1-p,"all")) {
1833 for(item = cpu_log_items; item->mask != 0; item++) {
1834 mask |= item->mask;
1835 }
1836 } else {
1837 for(item = cpu_log_items; item->mask != 0; item++) {
1838 if (cmp1(p, p1 - p, item->name))
1839 goto found;
1840 }
1841 return 0;
1842 }
1843 found:
1844 mask |= item->mask;
1845 if (*p1 != ',')
1846 break;
1847 p = p1 + 1;
1848 }
1849 return mask;
1850}
1851
1852void cpu_abort(CPUState *env, const char *fmt, ...)
1853{
1854 va_list ap;
1855 va_list ap2;
1856
1857 va_start(ap, fmt);
1858 va_copy(ap2, ap);
1859 fprintf(stderr, "qemu: fatal: ");
1860 vfprintf(stderr, fmt, ap);
1861 fprintf(stderr, "\n");
1862#ifdef TARGET_I386
1863 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1864#else
1865 cpu_dump_state(env, stderr, fprintf, 0);
1866#endif
1867 if (qemu_log_enabled()) {
1868 qemu_log("qemu: fatal: ");
1869 qemu_log_vprintf(fmt, ap2);
1870 qemu_log("\n");
1871#ifdef TARGET_I386
1872 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1873#else
1874 log_cpu_state(env, 0);
1875#endif
1876 qemu_log_flush();
1877 qemu_log_close();
1878 }
1879 va_end(ap2);
1880 va_end(ap);
1881#if defined(CONFIG_USER_ONLY)
1882 {
1883 struct sigaction act;
1884 sigfillset(&act.sa_mask);
1885 act.sa_handler = SIG_DFL;
1886 sigaction(SIGABRT, &act, NULL);
1887 }
1888#endif
1889 abort();
1890}
1891
1892CPUState *cpu_copy(CPUState *env)
1893{
1894 CPUState *new_env = cpu_init(env->cpu_model_str);
1895 CPUState *next_cpu = new_env->next_cpu;
1896 int cpu_index = new_env->cpu_index;
1897#if defined(TARGET_HAS_ICE)
1898 CPUBreakpoint *bp;
1899 CPUWatchpoint *wp;
1900#endif
1901
1902 memcpy(new_env, env, sizeof(CPUState));
1903
1904 /* Preserve chaining and index. */
1905 new_env->next_cpu = next_cpu;
1906 new_env->cpu_index = cpu_index;
1907
1908 /* Clone all break/watchpoints.
1909 Note: Once we support ptrace with hw-debug register access, make sure
1910 BP_CPU break/watchpoints are handled correctly on clone. */
1911 QTAILQ_INIT(&env->breakpoints);
1912 QTAILQ_INIT(&env->watchpoints);
1913#if defined(TARGET_HAS_ICE)
1914 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1915 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1916 }
1917 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1918 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1919 wp->flags, NULL);
1920 }
1921#endif
1922
1923 return new_env;
1924}
1925
1926#if !defined(CONFIG_USER_ONLY)
1927
1928static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1929{
1930 unsigned int i;
1931
1932 /* Discard jump cache entries for any tb which might potentially
1933 overlap the flushed page. */
1934 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1935 memset (&env->tb_jmp_cache[i], 0,
1936 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1937
1938 i = tb_jmp_cache_hash_page(addr);
1939 memset (&env->tb_jmp_cache[i], 0,
1940 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1941}
1942
1943static CPUTLBEntry s_cputlb_empty_entry = {
1944 .addr_read = -1,
1945 .addr_write = -1,
1946 .addr_code = -1,
1947 .addend = -1,
1948};
1949
1950/* NOTE: if flush_global is true, also flush global entries (not
1951 implemented yet) */
1952void tlb_flush(CPUState *env, int flush_global)
1953{
1954 int i;
1955
1956#if defined(DEBUG_TLB)
1957 printf("tlb_flush:\n");
1958#endif
1959 /* must reset current TB so that interrupts cannot modify the
1960 links while we are modifying them */
1961 env->current_tb = NULL;
1962
1963 for(i = 0; i < CPU_TLB_SIZE; i++) {
1964 int mmu_idx;
1965 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1966 env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
1967 }
1968 }
1969
1970 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1971
1972 env->tlb_flush_addr = -1;
1973 env->tlb_flush_mask = 0;
1974 tlb_flush_count++;
1975}
1976
1977static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1978{
1979 if (addr == (tlb_entry->addr_read &
1980 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1981 addr == (tlb_entry->addr_write &
1982 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1983 addr == (tlb_entry->addr_code &
1984 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1985 *tlb_entry = s_cputlb_empty_entry;
1986 }
1987}
1988
1989void tlb_flush_page(CPUState *env, target_ulong addr)
1990{
1991 int i;
1992 int mmu_idx;
1993
1994#if defined(DEBUG_TLB)
1995 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1996#endif
1997 /* Check if we need to flush due to large pages. */
1998 if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
1999#if defined(DEBUG_TLB)
2000 printf("tlb_flush_page: forced full flush ("
2001 TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
2002 env->tlb_flush_addr, env->tlb_flush_mask);
2003#endif
2004 tlb_flush(env, 1);
2005 return;
2006 }
2007 /* must reset current TB so that interrupts cannot modify the
2008 links while we are modifying them */
2009 env->current_tb = NULL;
2010
2011 addr &= TARGET_PAGE_MASK;
2012 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2013 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2014 tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
2015
2016 tlb_flush_jmp_cache(env, addr);
2017}
2018
2019/* update the TLBs so that writes to code in the virtual page 'addr'
2020 can be detected */
2021static void tlb_protect_code(ram_addr_t ram_addr)
2022{
2023 cpu_physical_memory_reset_dirty(ram_addr,
2024 ram_addr + TARGET_PAGE_SIZE,
2025 CODE_DIRTY_FLAG);
2026}
2027
2028/* update the TLB so that writes in physical page 'phys_addr' are no longer
2029 tested for self modifying code */
2030static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
2031 target_ulong vaddr)
2032{
2033 cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
2034}
2035
2036static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
2037 unsigned long start, unsigned long length)
2038{
2039 unsigned long addr;
2040 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2041 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
2042 if ((addr - start) < length) {
2043 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
2044 }
2045 }
2046}
2047
2048/* Note: start and end must be within the same ram block. */
2049void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
2050 int dirty_flags)
2051{
2052 CPUState *env;
2053 unsigned long length, start1;
2054 int i;
2055
2056 start &= TARGET_PAGE_MASK;
2057 end = TARGET_PAGE_ALIGN(end);
2058
2059 length = end - start;
2060 if (length == 0)
2061 return;
2062 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
2063
2064 /* we modify the TLB cache so that the dirty bit will be set again
2065 when accessing the range */
2066 start1 = (unsigned long)qemu_get_ram_ptr(start);
2067 /* Chek that we don't span multiple blocks - this breaks the
2068 address comparisons below. */
2069 if ((unsigned long)qemu_get_ram_ptr(end - 1) - start1
2070 != (end - 1) - start) {
2071 abort();
2072 }
2073
2074 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2075 int mmu_idx;
2076 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2077 for(i = 0; i < CPU_TLB_SIZE; i++)
2078 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
2079 start1, length);
2080 }
2081 }
2082}
2083
2084int cpu_physical_memory_set_dirty_tracking(int enable)
2085{
2086 int ret = 0;
2087 in_migration = enable;
2088 ret = cpu_notify_migration_log(!!enable);
2089 return ret;
2090}
2091
2092int cpu_physical_memory_get_dirty_tracking(void)
2093{
2094 return in_migration;
2095}
2096
2097int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
2098 target_phys_addr_t end_addr)
2099{
2100 int ret;
2101
2102 ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
2103 return ret;
2104}
2105
2106static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
2107{
2108 ram_addr_t ram_addr;
2109 void *p;
2110
2111 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2112 p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
2113 + tlb_entry->addend);
2114 ram_addr = qemu_ram_addr_from_host(p);
2115 if (!cpu_physical_memory_is_dirty(ram_addr)) {
2116 tlb_entry->addr_write |= TLB_NOTDIRTY;
2117 }
2118 }
2119}
2120
2121/* update the TLB according to the current state of the dirty bits */
2122void cpu_tlb_update_dirty(CPUState *env)
2123{
2124 int i;
2125 int mmu_idx;
2126 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2127 for(i = 0; i < CPU_TLB_SIZE; i++)
2128 tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
2129 }
2130}
2131
2132static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
2133{
2134 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
2135 tlb_entry->addr_write = vaddr;
2136}
2137
2138/* update the TLB corresponding to virtual page vaddr
2139 so that it is no longer dirty */
2140static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
2141{
2142 int i;
2143 int mmu_idx;
2144
2145 vaddr &= TARGET_PAGE_MASK;
2146 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2147 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2148 tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
2149}
2150
2151/* Our TLB does not support large pages, so remember the area covered by
2152 large pages and trigger a full TLB flush if these are invalidated. */
2153static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
2154 target_ulong size)
2155{
2156 target_ulong mask = ~(size - 1);
2157
2158 if (env->tlb_flush_addr == (target_ulong)-1) {
2159 env->tlb_flush_addr = vaddr & mask;
2160 env->tlb_flush_mask = mask;
2161 return;
2162 }
2163 /* Extend the existing region to include the new page.
2164 This is a compromise between unnecessary flushes and the cost
2165 of maintaining a full variable size TLB. */
2166 mask &= env->tlb_flush_mask;
2167 while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
2168 mask <<= 1;
2169 }
2170 env->tlb_flush_addr &= mask;
2171 env->tlb_flush_mask = mask;
2172}
2173
2174/* Add a new TLB entry. At most one entry for a given virtual address
2175 is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2176 supplied size is only used by tlb_flush_page. */
2177void tlb_set_page(CPUState *env, target_ulong vaddr,
2178 target_phys_addr_t paddr, int prot,
2179 int mmu_idx, target_ulong size)
2180{
2181 PhysPageDesc *p;
2182 unsigned long pd;
2183 unsigned int index;
2184 target_ulong address;
2185 target_ulong code_address;
2186 unsigned long addend;
2187 CPUTLBEntry *te;
2188 CPUWatchpoint *wp;
2189 target_phys_addr_t iotlb;
2190
2191 assert(size >= TARGET_PAGE_SIZE);
2192 if (size != TARGET_PAGE_SIZE) {
2193 tlb_add_large_page(env, vaddr, size);
2194 }
2195 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
2196 if (!p) {
2197 pd = IO_MEM_UNASSIGNED;
2198 } else {
2199 pd = p->phys_offset;
2200 }
2201#if defined(DEBUG_TLB)
2202 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
2203 vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
2204#endif
2205
2206 address = vaddr;
2207 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2208 /* IO memory case (romd handled later) */
2209 address |= TLB_MMIO;
2210 }
2211 addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
2212 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2213 /* Normal RAM. */
2214 iotlb = pd & TARGET_PAGE_MASK;
2215 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2216 iotlb |= IO_MEM_NOTDIRTY;
2217 else
2218 iotlb |= IO_MEM_ROM;
2219 } else {
2220 /* IO handlers are currently passed a physical address.
2221 It would be nice to pass an offset from the base address
2222 of that region. This would avoid having to special case RAM,
2223 and avoid full address decoding in every device.
2224 We can't use the high bits of pd for this because
2225 IO_MEM_ROMD uses these as a ram address. */
2226 iotlb = (pd & ~TARGET_PAGE_MASK);
2227 if (p) {
2228 iotlb += p->region_offset;
2229 } else {
2230 iotlb += paddr;
2231 }
2232 }
2233
2234 code_address = address;
2235 /* Make accesses to pages with watchpoints go via the
2236 watchpoint trap routines. */
2237 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2238 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2239 iotlb = io_mem_watch + paddr;
2240 /* TODO: The memory case can be optimized by not trapping
2241 reads of pages with a write breakpoint. */
2242 address |= TLB_MMIO;
2243 }
2244 }
2245
2246 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2247 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2248 te = &env->tlb_table[mmu_idx][index];
2249 te->addend = addend - vaddr;
2250 if (prot & PAGE_READ) {
2251 te->addr_read = address;
2252 } else {
2253 te->addr_read = -1;
2254 }
2255
2256 if (prot & PAGE_EXEC) {
2257 te->addr_code = code_address;
2258 } else {
2259 te->addr_code = -1;
2260 }
2261 if (prot & PAGE_WRITE) {
2262 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2263 (pd & IO_MEM_ROMD)) {
2264 /* Write access calls the I/O callback. */
2265 te->addr_write = address | TLB_MMIO;
2266 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2267 !cpu_physical_memory_is_dirty(pd)) {
2268 te->addr_write = address | TLB_NOTDIRTY;
2269 } else {
2270 te->addr_write = address;
2271 }
2272 } else {
2273 te->addr_write = -1;
2274 }
2275}
2276
2277#else
2278
2279void tlb_flush(CPUState *env, int flush_global)
2280{
2281}
2282
2283void tlb_flush_page(CPUState *env, target_ulong addr)
2284{
2285}
2286
2287/*
2288 * Walks guest process memory "regions" one by one
2289 * and calls callback function 'fn' for each region.
2290 */
2291
2292struct walk_memory_regions_data
2293{
2294 walk_memory_regions_fn fn;
2295 void *priv;
2296 unsigned long start;
2297 int prot;
2298};
2299
2300static int walk_memory_regions_end(struct walk_memory_regions_data *data,
2301 abi_ulong end, int new_prot)
2302{
2303 if (data->start != -1ul) {
2304 int rc = data->fn(data->priv, data->start, end, data->prot);
2305 if (rc != 0) {
2306 return rc;
2307 }
2308 }
2309
2310 data->start = (new_prot ? end : -1ul);
2311 data->prot = new_prot;
2312
2313 return 0;
2314}
2315
2316static int walk_memory_regions_1(struct walk_memory_regions_data *data,
2317 abi_ulong base, int level, void **lp)
2318{
2319 abi_ulong pa;
2320 int i, rc;
2321
2322 if (*lp == NULL) {
2323 return walk_memory_regions_end(data, base, 0);
2324 }
2325
2326 if (level == 0) {
2327 PageDesc *pd = *lp;
2328 for (i = 0; i < L2_SIZE; ++i) {
2329 int prot = pd[i].flags;
2330
2331 pa = base | (i << TARGET_PAGE_BITS);
2332 if (prot != data->prot) {
2333 rc = walk_memory_regions_end(data, pa, prot);
2334 if (rc != 0) {
2335 return rc;
2336 }
2337 }
2338 }
2339 } else {
2340 void **pp = *lp;
2341 for (i = 0; i < L2_SIZE; ++i) {
2342 pa = base | ((abi_ulong)i <<
2343 (TARGET_PAGE_BITS + L2_BITS * level));
2344 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
2345 if (rc != 0) {
2346 return rc;
2347 }
2348 }
2349 }
2350
2351 return 0;
2352}
2353
2354int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
2355{
2356 struct walk_memory_regions_data data;
2357 unsigned long i;
2358
2359 data.fn = fn;
2360 data.priv = priv;
2361 data.start = -1ul;
2362 data.prot = 0;
2363
2364 for (i = 0; i < V_L1_SIZE; i++) {
2365 int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
2366 V_L1_SHIFT / L2_BITS - 1, l1_map + i);
2367 if (rc != 0) {
2368 return rc;
2369 }
2370 }
2371
2372 return walk_memory_regions_end(&data, 0, 0);
2373}
2374
2375static int dump_region(void *priv, abi_ulong start,
2376 abi_ulong end, unsigned long prot)
2377{
2378 FILE *f = (FILE *)priv;
2379
2380 (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
2381 " "TARGET_ABI_FMT_lx" %c%c%c\n",
2382 start, end, end - start,
2383 ((prot & PAGE_READ) ? 'r' : '-'),
2384 ((prot & PAGE_WRITE) ? 'w' : '-'),
2385 ((prot & PAGE_EXEC) ? 'x' : '-'));
2386
2387 return (0);
2388}
2389
2390/* dump memory mappings */
2391void page_dump(FILE *f)
2392{
2393 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2394 "start", "end", "size", "prot");
2395 walk_memory_regions(f, dump_region);
2396}
2397
2398int page_get_flags(target_ulong address)
2399{
2400 PageDesc *p;
2401
2402 p = page_find(address >> TARGET_PAGE_BITS);
2403 if (!p)
2404 return 0;
2405 return p->flags;
2406}
2407
2408/* Modify the flags of a page and invalidate the code if necessary.
2409 The flag PAGE_WRITE_ORG is positioned automatically depending
2410 on PAGE_WRITE. The mmap_lock should already be held. */
2411void page_set_flags(target_ulong start, target_ulong end, int flags)
2412{
2413 target_ulong addr, len;
2414
2415 /* This function should never be called with addresses outside the
2416 guest address space. If this assert fires, it probably indicates
2417 a missing call to h2g_valid. */
2418#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2419 assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2420#endif
2421 assert(start < end);
2422
2423 start = start & TARGET_PAGE_MASK;
2424 end = TARGET_PAGE_ALIGN(end);
2425
2426 if (flags & PAGE_WRITE) {
2427 flags |= PAGE_WRITE_ORG;
2428 }
2429
2430 for (addr = start, len = end - start;
2431 len != 0;
2432 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2433 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2434
2435 /* If the write protection bit is set, then we invalidate
2436 the code inside. */
2437 if (!(p->flags & PAGE_WRITE) &&
2438 (flags & PAGE_WRITE) &&
2439 p->first_tb) {
2440 tb_invalidate_phys_page(addr, 0, NULL);
2441 }
2442 p->flags = flags;
2443 }
2444}
2445
2446int page_check_range(target_ulong start, target_ulong len, int flags)
2447{
2448 PageDesc *p;
2449 target_ulong end;
2450 target_ulong addr;
2451
2452 /* This function should never be called with addresses outside the
2453 guest address space. If this assert fires, it probably indicates
2454 a missing call to h2g_valid. */
2455#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2456 assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2457#endif
2458
2459 if (start + len - 1 < start) {
2460 /* We've wrapped around. */
2461 return -1;
2462 }
2463
2464 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2465 start = start & TARGET_PAGE_MASK;
2466
2467 for (addr = start, len = end - start;
2468 len != 0;
2469 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2470 p = page_find(addr >> TARGET_PAGE_BITS);
2471 if( !p )
2472 return -1;
2473 if( !(p->flags & PAGE_VALID) )
2474 return -1;
2475
2476 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2477 return -1;
2478 if (flags & PAGE_WRITE) {
2479 if (!(p->flags & PAGE_WRITE_ORG))
2480 return -1;
2481 /* unprotect the page if it was put read-only because it
2482 contains translated code */
2483 if (!(p->flags & PAGE_WRITE)) {
2484 if (!page_unprotect(addr, 0, NULL))
2485 return -1;
2486 }
2487 return 0;
2488 }
2489 }
2490 return 0;
2491}
2492
2493/* called from signal handler: invalidate the code and unprotect the
2494 page. Return TRUE if the fault was successfully handled. */
2495int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2496{
2497 unsigned int prot;
2498 PageDesc *p;
2499 target_ulong host_start, host_end, addr;
2500
2501 /* Technically this isn't safe inside a signal handler. However we
2502 know this only ever happens in a synchronous SEGV handler, so in
2503 practice it seems to be ok. */
2504 mmap_lock();
2505
2506 p = page_find(address >> TARGET_PAGE_BITS);
2507 if (!p) {
2508 mmap_unlock();
2509 return 0;
2510 }
2511
2512 /* if the page was really writable, then we change its
2513 protection back to writable */
2514 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
2515 host_start = address & qemu_host_page_mask;
2516 host_end = host_start + qemu_host_page_size;
2517
2518 prot = 0;
2519 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
2520 p = page_find(addr >> TARGET_PAGE_BITS);
2521 p->flags |= PAGE_WRITE;
2522 prot |= p->flags;
2523
2524 /* and since the content will be modified, we must invalidate
2525 the corresponding translated code. */
2526 tb_invalidate_phys_page(addr, pc, puc);
2527#ifdef DEBUG_TB_CHECK
2528 tb_invalidate_check(addr);
2529#endif
2530 }
2531 mprotect((void *)g2h(host_start), qemu_host_page_size,
2532 prot & PAGE_BITS);
2533
2534 mmap_unlock();
2535 return 1;
2536 }
2537 mmap_unlock();
2538 return 0;
2539}
2540
2541static inline void tlb_set_dirty(CPUState *env,
2542 unsigned long addr, target_ulong vaddr)
2543{
2544}
2545#endif /* defined(CONFIG_USER_ONLY) */
2546
2547#if !defined(CONFIG_USER_ONLY)
2548
2549#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2550typedef struct subpage_t {
2551 target_phys_addr_t base;
2552 CPUReadMemoryFunc * const *mem_read[TARGET_PAGE_SIZE][4];
2553 CPUWriteMemoryFunc * const *mem_write[TARGET_PAGE_SIZE][4];
2554 void *opaque[TARGET_PAGE_SIZE][2][4];
2555 ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
2556} subpage_t;
2557
2558static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2559 ram_addr_t memory, ram_addr_t region_offset);
2560static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2561 ram_addr_t orig_memory, ram_addr_t region_offset);
2562#define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2563 need_subpage) \
2564 do { \
2565 if (addr > start_addr) \
2566 start_addr2 = 0; \
2567 else { \
2568 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2569 if (start_addr2 > 0) \
2570 need_subpage = 1; \
2571 } \
2572 \
2573 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2574 end_addr2 = TARGET_PAGE_SIZE - 1; \
2575 else { \
2576 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2577 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2578 need_subpage = 1; \
2579 } \
2580 } while (0)
2581
2582/* register physical memory.
2583 For RAM, 'size' must be a multiple of the target page size.
2584 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2585 io memory page. The address used when calling the IO function is
2586 the offset from the start of the region, plus region_offset. Both
2587 start_addr and region_offset are rounded down to a page boundary
2588 before calculating this offset. This should not be a problem unless
2589 the low bits of start_addr and region_offset differ. */
2590void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2591 ram_addr_t size,
2592 ram_addr_t phys_offset,
2593 ram_addr_t region_offset)
2594{
2595 target_phys_addr_t addr, end_addr;
2596 PhysPageDesc *p;
2597 CPUState *env;
2598 ram_addr_t orig_size = size;
2599 void *subpage;
2600
2601 cpu_notify_set_memory(start_addr, size, phys_offset);
2602
2603 if (phys_offset == IO_MEM_UNASSIGNED) {
2604 region_offset = start_addr;
2605 }
2606 region_offset &= TARGET_PAGE_MASK;
2607 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2608 end_addr = start_addr + (target_phys_addr_t)size;
2609 for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2610 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2611 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2612 ram_addr_t orig_memory = p->phys_offset;
2613 target_phys_addr_t start_addr2, end_addr2;
2614 int need_subpage = 0;
2615
2616 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2617 need_subpage);
2618 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2619 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2620 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2621 &p->phys_offset, orig_memory,
2622 p->region_offset);
2623 } else {
2624 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2625 >> IO_MEM_SHIFT];
2626 }
2627 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2628 region_offset);
2629 p->region_offset = 0;
2630 } else {
2631 p->phys_offset = phys_offset;
2632 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2633 (phys_offset & IO_MEM_ROMD))
2634 phys_offset += TARGET_PAGE_SIZE;
2635 }
2636 } else {
2637 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2638 p->phys_offset = phys_offset;
2639 p->region_offset = region_offset;
2640 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2641 (phys_offset & IO_MEM_ROMD)) {
2642 phys_offset += TARGET_PAGE_SIZE;
2643 } else {
2644 target_phys_addr_t start_addr2, end_addr2;
2645 int need_subpage = 0;
2646
2647 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2648 end_addr2, need_subpage);
2649
2650 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2651 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2652 &p->phys_offset, IO_MEM_UNASSIGNED,
2653 addr & TARGET_PAGE_MASK);
2654 subpage_register(subpage, start_addr2, end_addr2,
2655 phys_offset, region_offset);
2656 p->region_offset = 0;
2657 }
2658 }
2659 }
2660 region_offset += TARGET_PAGE_SIZE;
2661 }
2662
2663 /* since each CPU stores ram addresses in its TLB cache, we must
2664 reset the modified entries */
2665 /* XXX: slow ! */
2666 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2667 tlb_flush(env, 1);
2668 }
2669}
2670
2671/* XXX: temporary until new memory mapping API */
2672ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2673{
2674 PhysPageDesc *p;
2675
2676 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2677 if (!p)
2678 return IO_MEM_UNASSIGNED;
2679 return p->phys_offset;
2680}
2681
2682void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2683{
2684 if (kvm_enabled())
2685 kvm_coalesce_mmio_region(addr, size);
2686}
2687
2688void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2689{
2690 if (kvm_enabled())
2691 kvm_uncoalesce_mmio_region(addr, size);
2692}
2693
2694void qemu_flush_coalesced_mmio_buffer(void)
2695{
2696 if (kvm_enabled())
2697 kvm_flush_coalesced_mmio_buffer();
2698}
2699
2700#if defined(__linux__) && !defined(TARGET_S390X)
2701
2702#include <sys/vfs.h>
2703
2704#define HUGETLBFS_MAGIC 0x958458f6
2705
2706static long gethugepagesize(const char *path)
2707{
2708 struct statfs fs;
2709 int ret;
2710
2711 do {
2712 ret = statfs(path, &fs);
2713 } while (ret != 0 && errno == EINTR);
2714
2715 if (ret != 0) {
2716 perror(path);
2717 return 0;
2718 }
2719
2720 if (fs.f_type != HUGETLBFS_MAGIC)
2721 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
2722
2723 return fs.f_bsize;
2724}
2725
2726static void *file_ram_alloc(ram_addr_t memory, const char *path)
2727{
2728 char *filename;
2729 void *area;
2730 int fd;
2731#ifdef MAP_POPULATE
2732 int flags;
2733#endif
2734 unsigned long hpagesize;
2735
2736 hpagesize = gethugepagesize(path);
2737 if (!hpagesize) {
2738 return NULL;
2739 }
2740
2741 if (memory < hpagesize) {
2742 return NULL;
2743 }
2744
2745 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2746 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2747 return NULL;
2748 }
2749
2750 if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
2751 return NULL;
2752 }
2753
2754 fd = mkstemp(filename);
2755 if (fd < 0) {
2756 perror("unable to create backing store for hugepages");
2757 free(filename);
2758 return NULL;
2759 }
2760 unlink(filename);
2761 free(filename);
2762
2763 memory = (memory+hpagesize-1) & ~(hpagesize-1);
2764
2765 /*
2766 * ftruncate is not supported by hugetlbfs in older
2767 * hosts, so don't bother bailing out on errors.
2768 * If anything goes wrong with it under other filesystems,
2769 * mmap will fail.
2770 */
2771 if (ftruncate(fd, memory))
2772 perror("ftruncate");
2773
2774#ifdef MAP_POPULATE
2775 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2776 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2777 * to sidestep this quirk.
2778 */
2779 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
2780 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
2781#else
2782 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
2783#endif
2784 if (area == MAP_FAILED) {
2785 perror("file_ram_alloc: can't mmap RAM pages");
2786 close(fd);
2787 return (NULL);
2788 }
2789 return area;
2790}
2791#endif
2792
2793ram_addr_t qemu_ram_alloc(ram_addr_t size)
2794{
2795 RAMBlock *new_block;
2796
2797 size = TARGET_PAGE_ALIGN(size);
2798 new_block = qemu_malloc(sizeof(*new_block));
2799
2800 if (mem_path) {
2801#if defined (__linux__) && !defined(TARGET_S390X)
2802 new_block->host = file_ram_alloc(size, mem_path);
2803 if (!new_block->host)
2804 exit(1);
2805#else
2806 fprintf(stderr, "-mem-path option unsupported\n");
2807 exit(1);
2808#endif
2809 } else {
2810#if defined(TARGET_S390X) && defined(CONFIG_KVM)
2811 /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
2812 new_block->host = mmap((void*)0x1000000, size,
2813 PROT_EXEC|PROT_READ|PROT_WRITE,
2814 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
2815#else
2816 new_block->host = qemu_vmalloc(size);
2817#endif
2818#ifdef MADV_MERGEABLE
2819 madvise(new_block->host, size, MADV_MERGEABLE);
2820#endif
2821 }
2822 new_block->offset = last_ram_offset;
2823 new_block->length = size;
2824
2825 new_block->next = ram_blocks;
2826 ram_blocks = new_block;
2827
2828 phys_ram_dirty = qemu_realloc(phys_ram_dirty,
2829 (last_ram_offset + size) >> TARGET_PAGE_BITS);
2830 memset(phys_ram_dirty + (last_ram_offset >> TARGET_PAGE_BITS),
2831 0xff, size >> TARGET_PAGE_BITS);
2832
2833 last_ram_offset += size;
2834
2835 if (kvm_enabled())
2836 kvm_setup_guest_memory(new_block->host, size);
2837
2838 return new_block->offset;
2839}
2840
2841void qemu_ram_free(ram_addr_t addr)
2842{
2843 /* TODO: implement this. */
2844}
2845
2846/* Return a host pointer to ram allocated with qemu_ram_alloc.
2847 With the exception of the softmmu code in this file, this should
2848 only be used for local memory (e.g. video ram) that the device owns,
2849 and knows it isn't going to access beyond the end of the block.
2850
2851 It should not be used for general purpose DMA.
2852 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2853 */
2854void *qemu_get_ram_ptr(ram_addr_t addr)
2855{
2856 RAMBlock *prev;
2857 RAMBlock **prevp;
2858 RAMBlock *block;
2859
2860 prev = NULL;
2861 prevp = &ram_blocks;
2862 block = ram_blocks;
2863 while (block && (block->offset > addr
2864 || block->offset + block->length <= addr)) {
2865 if (prev)
2866 prevp = &prev->next;
2867 prev = block;
2868 block = block->next;
2869 }
2870 if (!block) {
2871 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
2872 abort();
2873 }
2874 /* Move this entry to to start of the list. */
2875 if (prev) {
2876 prev->next = block->next;
2877 block->next = *prevp;
2878 *prevp = block;
2879 }
2880 return block->host + (addr - block->offset);
2881}
2882
2883/* Some of the softmmu routines need to translate from a host pointer
2884 (typically a TLB entry) back to a ram offset. */
2885ram_addr_t qemu_ram_addr_from_host(void *ptr)
2886{
2887 RAMBlock *block;
2888 uint8_t *host = ptr;
2889
2890 block = ram_blocks;
2891 while (block && (block->host > host
2892 || block->host + block->length <= host)) {
2893 block = block->next;
2894 }
2895 if (!block) {
2896 fprintf(stderr, "Bad ram pointer %p\n", ptr);
2897 abort();
2898 }
2899 return block->offset + (host - block->host);
2900}
2901
2902static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2903{
2904#ifdef DEBUG_UNASSIGNED
2905 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2906#endif
2907#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2908 do_unassigned_access(addr, 0, 0, 0, 1);
2909#endif
2910 return 0;
2911}
2912
2913static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2914{
2915#ifdef DEBUG_UNASSIGNED
2916 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2917#endif
2918#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2919 do_unassigned_access(addr, 0, 0, 0, 2);
2920#endif
2921 return 0;
2922}
2923
2924static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2925{
2926#ifdef DEBUG_UNASSIGNED
2927 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2928#endif
2929#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2930 do_unassigned_access(addr, 0, 0, 0, 4);
2931#endif
2932 return 0;
2933}
2934
2935static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2936{
2937#ifdef DEBUG_UNASSIGNED
2938 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2939#endif
2940#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2941 do_unassigned_access(addr, 1, 0, 0, 1);
2942#endif
2943}
2944
2945static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2946{
2947#ifdef DEBUG_UNASSIGNED
2948 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2949#endif
2950#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2951 do_unassigned_access(addr, 1, 0, 0, 2);
2952#endif
2953}
2954
2955static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2956{
2957#ifdef DEBUG_UNASSIGNED
2958 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2959#endif
2960#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2961 do_unassigned_access(addr, 1, 0, 0, 4);
2962#endif
2963}
2964
2965static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
2966 unassigned_mem_readb,
2967 unassigned_mem_readw,
2968 unassigned_mem_readl,
2969};
2970
2971static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
2972 unassigned_mem_writeb,
2973 unassigned_mem_writew,
2974 unassigned_mem_writel,
2975};
2976
2977static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2978 uint32_t val)
2979{
2980 int dirty_flags;
2981 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
2982 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2983#if !defined(CONFIG_USER_ONLY)
2984 tb_invalidate_phys_page_fast(ram_addr, 1);
2985 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
2986#endif
2987 }
2988 stb_p(qemu_get_ram_ptr(ram_addr), val);
2989 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2990 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
2991 /* we remove the notdirty callback only if the code has been
2992 flushed */
2993 if (dirty_flags == 0xff)
2994 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2995}
2996
2997static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2998 uint32_t val)
2999{
3000 int dirty_flags;
3001 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3002 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3003#if !defined(CONFIG_USER_ONLY)
3004 tb_invalidate_phys_page_fast(ram_addr, 2);
3005 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3006#endif
3007 }
3008 stw_p(qemu_get_ram_ptr(ram_addr), val);
3009 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3010 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3011 /* we remove the notdirty callback only if the code has been
3012 flushed */
3013 if (dirty_flags == 0xff)
3014 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3015}
3016
3017static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
3018 uint32_t val)
3019{
3020 int dirty_flags;
3021 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3022 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3023#if !defined(CONFIG_USER_ONLY)
3024 tb_invalidate_phys_page_fast(ram_addr, 4);
3025 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3026#endif
3027 }
3028 stl_p(qemu_get_ram_ptr(ram_addr), val);
3029 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3030 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3031 /* we remove the notdirty callback only if the code has been
3032 flushed */
3033 if (dirty_flags == 0xff)
3034 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3035}
3036
3037static CPUReadMemoryFunc * const error_mem_read[3] = {
3038 NULL, /* never used */
3039 NULL, /* never used */
3040 NULL, /* never used */
3041};
3042
3043static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
3044 notdirty_mem_writeb,
3045 notdirty_mem_writew,
3046 notdirty_mem_writel,
3047};
3048
3049/* Generate a debug exception if a watchpoint has been hit. */
3050static void check_watchpoint(int offset, int len_mask, int flags)
3051{
3052 CPUState *env = cpu_single_env;
3053 target_ulong pc, cs_base;
3054 TranslationBlock *tb;
3055 target_ulong vaddr;
3056 CPUWatchpoint *wp;
3057 int cpu_flags;
3058
3059 if (env->watchpoint_hit) {
3060 /* We re-entered the check after replacing the TB. Now raise
3061 * the debug interrupt so that is will trigger after the
3062 * current instruction. */
3063 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
3064 return;
3065 }
3066 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
3067 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
3068 if ((vaddr == (wp->vaddr & len_mask) ||
3069 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
3070 wp->flags |= BP_WATCHPOINT_HIT;
3071 if (!env->watchpoint_hit) {
3072 env->watchpoint_hit = wp;
3073 tb = tb_find_pc(env->mem_io_pc);
3074 if (!tb) {
3075 cpu_abort(env, "check_watchpoint: could not find TB for "
3076 "pc=%p", (void *)env->mem_io_pc);
3077 }
3078 cpu_restore_state(tb, env, env->mem_io_pc, NULL);
3079 tb_phys_invalidate(tb, -1);
3080 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
3081 env->exception_index = EXCP_DEBUG;
3082 } else {
3083 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
3084 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
3085 }
3086 cpu_resume_from_signal(env, NULL);
3087 }
3088 } else {
3089 wp->flags &= ~BP_WATCHPOINT_HIT;
3090 }
3091 }
3092}
3093
3094/* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3095 so these check for a hit then pass through to the normal out-of-line
3096 phys routines. */
3097static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
3098{
3099 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
3100 return ldub_phys(addr);
3101}
3102
3103static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
3104{
3105 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
3106 return lduw_phys(addr);
3107}
3108
3109static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
3110{
3111 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
3112 return ldl_phys(addr);
3113}
3114
3115static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
3116 uint32_t val)
3117{
3118 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
3119 stb_phys(addr, val);
3120}
3121
3122static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
3123 uint32_t val)
3124{
3125 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
3126 stw_phys(addr, val);
3127}
3128
3129static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
3130 uint32_t val)
3131{
3132 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
3133 stl_phys(addr, val);
3134}
3135
3136static CPUReadMemoryFunc * const watch_mem_read[3] = {
3137 watch_mem_readb,
3138 watch_mem_readw,
3139 watch_mem_readl,
3140};
3141
3142static CPUWriteMemoryFunc * const watch_mem_write[3] = {
3143 watch_mem_writeb,
3144 watch_mem_writew,
3145 watch_mem_writel,
3146};
3147
3148static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
3149 unsigned int len)
3150{
3151 uint32_t ret;
3152 unsigned int idx;
3153
3154 idx = SUBPAGE_IDX(addr);
3155#if defined(DEBUG_SUBPAGE)
3156 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
3157 mmio, len, addr, idx);
3158#endif
3159 ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
3160 addr + mmio->region_offset[idx][0][len]);
3161
3162 return ret;
3163}
3164
3165static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
3166 uint32_t value, unsigned int len)
3167{
3168 unsigned int idx;
3169
3170 idx = SUBPAGE_IDX(addr);
3171#if defined(DEBUG_SUBPAGE)
3172 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
3173 mmio, len, addr, idx, value);
3174#endif
3175 (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
3176 addr + mmio->region_offset[idx][1][len],
3177 value);
3178}
3179
3180static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
3181{
3182#if defined(DEBUG_SUBPAGE)
3183 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
3184#endif
3185
3186 return subpage_readlen(opaque, addr, 0);
3187}
3188
3189static void subpage_writeb (void *opaque, target_phys_addr_t addr,
3190 uint32_t value)
3191{
3192#if defined(DEBUG_SUBPAGE)
3193 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
3194#endif
3195 subpage_writelen(opaque, addr, value, 0);
3196}
3197
3198static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
3199{
3200#if defined(DEBUG_SUBPAGE)
3201 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
3202#endif
3203
3204 return subpage_readlen(opaque, addr, 1);
3205}
3206
3207static void subpage_writew (void *opaque, target_phys_addr_t addr,
3208 uint32_t value)
3209{
3210#if defined(DEBUG_SUBPAGE)
3211 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
3212#endif
3213 subpage_writelen(opaque, addr, value, 1);
3214}
3215
3216static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
3217{
3218#if defined(DEBUG_SUBPAGE)
3219 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
3220#endif
3221
3222 return subpage_readlen(opaque, addr, 2);
3223}
3224
3225static void subpage_writel (void *opaque,
3226 target_phys_addr_t addr, uint32_t value)
3227{
3228#if defined(DEBUG_SUBPAGE)
3229 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
3230#endif
3231 subpage_writelen(opaque, addr, value, 2);
3232}
3233
3234static CPUReadMemoryFunc * const subpage_read[] = {
3235 &subpage_readb,
3236 &subpage_readw,
3237 &subpage_readl,
3238};
3239
3240static CPUWriteMemoryFunc * const subpage_write[] = {
3241 &subpage_writeb,
3242 &subpage_writew,
3243 &subpage_writel,
3244};
3245
3246static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
3247 ram_addr_t memory, ram_addr_t region_offset)
3248{
3249 int idx, eidx;
3250 unsigned int i;
3251
3252 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
3253 return -1;
3254 idx = SUBPAGE_IDX(start);
3255 eidx = SUBPAGE_IDX(end);
3256#if defined(DEBUG_SUBPAGE)
3257 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
3258 mmio, start, end, idx, eidx, memory);
3259#endif
3260 memory >>= IO_MEM_SHIFT;
3261 for (; idx <= eidx; idx++) {
3262 for (i = 0; i < 4; i++) {
3263 if (io_mem_read[memory][i]) {
3264 mmio->mem_read[idx][i] = &io_mem_read[memory][i];
3265 mmio->opaque[idx][0][i] = io_mem_opaque[memory];
3266 mmio->region_offset[idx][0][i] = region_offset;
3267 }
3268 if (io_mem_write[memory][i]) {
3269 mmio->mem_write[idx][i] = &io_mem_write[memory][i];
3270 mmio->opaque[idx][1][i] = io_mem_opaque[memory];
3271 mmio->region_offset[idx][1][i] = region_offset;
3272 }
3273 }
3274 }
3275
3276 return 0;
3277}
3278
3279static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
3280 ram_addr_t orig_memory, ram_addr_t region_offset)
3281{
3282 subpage_t *mmio;
3283 int subpage_memory;
3284
3285 mmio = qemu_mallocz(sizeof(subpage_t));
3286
3287 mmio->base = base;
3288 subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio);
3289#if defined(DEBUG_SUBPAGE)
3290 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
3291 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
3292#endif
3293 *phys = subpage_memory | IO_MEM_SUBPAGE;
3294 subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
3295 region_offset);
3296
3297 return mmio;
3298}
3299
3300static int get_free_io_mem_idx(void)
3301{
3302 int i;
3303
3304 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
3305 if (!io_mem_used[i]) {
3306 io_mem_used[i] = 1;
3307 return i;
3308 }
3309 fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
3310 return -1;
3311}
3312
3313/* mem_read and mem_write are arrays of functions containing the
3314 function to access byte (index 0), word (index 1) and dword (index
3315 2). Functions can be omitted with a NULL function pointer.
3316 If io_index is non zero, the corresponding io zone is
3317 modified. If it is zero, a new io zone is allocated. The return
3318 value can be used with cpu_register_physical_memory(). (-1) is
3319 returned if error. */
3320static int cpu_register_io_memory_fixed(int io_index,
3321 CPUReadMemoryFunc * const *mem_read,
3322 CPUWriteMemoryFunc * const *mem_write,
3323 void *opaque)
3324{
3325 int i, subwidth = 0;
3326
3327 if (io_index <= 0) {
3328 io_index = get_free_io_mem_idx();
3329 if (io_index == -1)
3330 return io_index;
3331 } else {
3332 io_index >>= IO_MEM_SHIFT;
3333 if (io_index >= IO_MEM_NB_ENTRIES)
3334 return -1;
3335 }
3336
3337 for(i = 0;i < 3; i++) {
3338 if (!mem_read[i] || !mem_write[i])
3339 subwidth = IO_MEM_SUBWIDTH;
3340 io_mem_read[io_index][i] = mem_read[i];
3341 io_mem_write[io_index][i] = mem_write[i];
3342 }
3343 io_mem_opaque[io_index] = opaque;
3344 return (io_index << IO_MEM_SHIFT) | subwidth;
3345}
3346
3347int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
3348 CPUWriteMemoryFunc * const *mem_write,
3349 void *opaque)
3350{
3351 return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque);
3352}
3353
3354void cpu_unregister_io_memory(int io_table_address)
3355{
3356 int i;
3357 int io_index = io_table_address >> IO_MEM_SHIFT;
3358
3359 for (i=0;i < 3; i++) {
3360 io_mem_read[io_index][i] = unassigned_mem_read[i];
3361 io_mem_write[io_index][i] = unassigned_mem_write[i];
3362 }
3363 io_mem_opaque[io_index] = NULL;
3364 io_mem_used[io_index] = 0;
3365}
3366
3367static void io_mem_init(void)
3368{
3369 int i;
3370
3371 cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read, unassigned_mem_write, NULL);
3372 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read, unassigned_mem_write, NULL);
3373 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read, notdirty_mem_write, NULL);
3374 for (i=0; i<5; i++)
3375 io_mem_used[i] = 1;
3376
3377 io_mem_watch = cpu_register_io_memory(watch_mem_read,
3378 watch_mem_write, NULL);
3379}
3380
3381#endif /* !defined(CONFIG_USER_ONLY) */
3382
3383/* physical memory access (slow version, mainly for debug) */
3384#if defined(CONFIG_USER_ONLY)
3385int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3386 uint8_t *buf, int len, int is_write)
3387{
3388 int l, flags;
3389 target_ulong page;
3390 void * p;
3391
3392 while (len > 0) {
3393 page = addr & TARGET_PAGE_MASK;
3394 l = (page + TARGET_PAGE_SIZE) - addr;
3395 if (l > len)
3396 l = len;
3397 flags = page_get_flags(page);
3398 if (!(flags & PAGE_VALID))
3399 return -1;
3400 if (is_write) {
3401 if (!(flags & PAGE_WRITE))
3402 return -1;
3403 /* XXX: this code should not depend on lock_user */
3404 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
3405 return -1;
3406 memcpy(p, buf, l);
3407 unlock_user(p, addr, l);
3408 } else {
3409 if (!(flags & PAGE_READ))
3410 return -1;
3411 /* XXX: this code should not depend on lock_user */
3412 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
3413 return -1;
3414 memcpy(buf, p, l);
3415 unlock_user(p, addr, 0);
3416 }
3417 len -= l;
3418 buf += l;
3419 addr += l;
3420 }
3421 return 0;
3422}
3423
3424#else
3425void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3426 int len, int is_write)
3427{
3428 int l, io_index;
3429 uint8_t *ptr;
3430 uint32_t val;
3431 target_phys_addr_t page;
3432 unsigned long pd;
3433 PhysPageDesc *p;
3434
3435 while (len > 0) {
3436 page = addr & TARGET_PAGE_MASK;
3437 l = (page + TARGET_PAGE_SIZE) - addr;
3438 if (l > len)
3439 l = len;
3440 p = phys_page_find(page >> TARGET_PAGE_BITS);
3441 if (!p) {
3442 pd = IO_MEM_UNASSIGNED;
3443 } else {
3444 pd = p->phys_offset;
3445 }
3446
3447 if (is_write) {
3448 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3449 target_phys_addr_t addr1 = addr;
3450 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3451 if (p)
3452 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3453 /* XXX: could force cpu_single_env to NULL to avoid
3454 potential bugs */
3455 if (l >= 4 && ((addr1 & 3) == 0)) {
3456 /* 32 bit write access */
3457 val = ldl_p(buf);
3458 io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3459 l = 4;
3460 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3461 /* 16 bit write access */
3462 val = lduw_p(buf);
3463 io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3464 l = 2;
3465 } else {
3466 /* 8 bit write access */
3467 val = ldub_p(buf);
3468 io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3469 l = 1;
3470 }
3471 } else {
3472 unsigned long addr1;
3473 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3474 /* RAM case */
3475 ptr = qemu_get_ram_ptr(addr1);
3476 memcpy(ptr, buf, l);
3477 if (!cpu_physical_memory_is_dirty(addr1)) {
3478 /* invalidate code */
3479 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3480 /* set dirty bit */
3481 cpu_physical_memory_set_dirty_flags(
3482 addr1, (0xff & ~CODE_DIRTY_FLAG));
3483 }
3484 }
3485 } else {
3486 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3487 !(pd & IO_MEM_ROMD)) {
3488 target_phys_addr_t addr1 = addr;
3489 /* I/O case */
3490 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3491 if (p)
3492 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3493 if (l >= 4 && ((addr1 & 3) == 0)) {
3494 /* 32 bit read access */
3495 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3496 stl_p(buf, val);
3497 l = 4;
3498 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3499 /* 16 bit read access */
3500 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3501 stw_p(buf, val);
3502 l = 2;
3503 } else {
3504 /* 8 bit read access */
3505 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3506 stb_p(buf, val);
3507 l = 1;
3508 }
3509 } else {
3510 /* RAM case */
3511 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3512 (addr & ~TARGET_PAGE_MASK);
3513 memcpy(buf, ptr, l);
3514 }
3515 }
3516 len -= l;
3517 buf += l;
3518 addr += l;
3519 }
3520}
3521
3522/* used for ROM loading : can write in RAM and ROM */
3523void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3524 const uint8_t *buf, int len)
3525{
3526 int l;
3527 uint8_t *ptr;
3528 target_phys_addr_t page;
3529 unsigned long pd;
3530 PhysPageDesc *p;
3531
3532 while (len > 0) {
3533 page = addr & TARGET_PAGE_MASK;
3534 l = (page + TARGET_PAGE_SIZE) - addr;
3535 if (l > len)
3536 l = len;
3537 p = phys_page_find(page >> TARGET_PAGE_BITS);
3538 if (!p) {
3539 pd = IO_MEM_UNASSIGNED;
3540 } else {
3541 pd = p->phys_offset;
3542 }
3543
3544 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3545 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3546 !(pd & IO_MEM_ROMD)) {
3547 /* do nothing */
3548 } else {
3549 unsigned long addr1;
3550 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3551 /* ROM/RAM case */
3552 ptr = qemu_get_ram_ptr(addr1);
3553 memcpy(ptr, buf, l);
3554 }
3555 len -= l;
3556 buf += l;
3557 addr += l;
3558 }
3559}
3560
3561typedef struct {
3562 void *buffer;
3563 target_phys_addr_t addr;
3564 target_phys_addr_t len;
3565} BounceBuffer;
3566
3567static BounceBuffer bounce;
3568
3569typedef struct MapClient {
3570 void *opaque;
3571 void (*callback)(void *opaque);
3572 QLIST_ENTRY(MapClient) link;
3573} MapClient;
3574
3575static QLIST_HEAD(map_client_list, MapClient) map_client_list
3576 = QLIST_HEAD_INITIALIZER(map_client_list);
3577
3578void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3579{
3580 MapClient *client = qemu_malloc(sizeof(*client));
3581
3582 client->opaque = opaque;
3583 client->callback = callback;
3584 QLIST_INSERT_HEAD(&map_client_list, client, link);
3585 return client;
3586}
3587
3588void cpu_unregister_map_client(void *_client)
3589{
3590 MapClient *client = (MapClient *)_client;
3591
3592 QLIST_REMOVE(client, link);
3593 qemu_free(client);
3594}
3595
3596static void cpu_notify_map_clients(void)
3597{
3598 MapClient *client;
3599
3600 while (!QLIST_EMPTY(&map_client_list)) {
3601 client = QLIST_FIRST(&map_client_list);
3602 client->callback(client->opaque);
3603 cpu_unregister_map_client(client);
3604 }
3605}
3606
3607/* Map a physical memory region into a host virtual address.
3608 * May map a subset of the requested range, given by and returned in *plen.
3609 * May return NULL if resources needed to perform the mapping are exhausted.
3610 * Use only for reads OR writes - not for read-modify-write operations.
3611 * Use cpu_register_map_client() to know when retrying the map operation is
3612 * likely to succeed.
3613 */
3614void *cpu_physical_memory_map(target_phys_addr_t addr,
3615 target_phys_addr_t *plen,
3616 int is_write)
3617{
3618 target_phys_addr_t len = *plen;
3619 target_phys_addr_t done = 0;
3620 int l;
3621 uint8_t *ret = NULL;
3622 uint8_t *ptr;
3623 target_phys_addr_t page;
3624 unsigned long pd;
3625 PhysPageDesc *p;
3626 unsigned long addr1;
3627
3628 while (len > 0) {
3629 page = addr & TARGET_PAGE_MASK;
3630 l = (page + TARGET_PAGE_SIZE) - addr;
3631 if (l > len)
3632 l = len;
3633 p = phys_page_find(page >> TARGET_PAGE_BITS);
3634 if (!p) {
3635 pd = IO_MEM_UNASSIGNED;
3636 } else {
3637 pd = p->phys_offset;
3638 }
3639
3640 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3641 if (done || bounce.buffer) {
3642 break;
3643 }
3644 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3645 bounce.addr = addr;
3646 bounce.len = l;
3647 if (!is_write) {
3648 cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
3649 }
3650 ptr = bounce.buffer;
3651 } else {
3652 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3653 ptr = qemu_get_ram_ptr(addr1);
3654 }
3655 if (!done) {
3656 ret = ptr;
3657 } else if (ret + done != ptr) {
3658 break;
3659 }
3660
3661 len -= l;
3662 addr += l;
3663 done += l;
3664 }
3665 *plen = done;
3666 return ret;
3667}
3668
3669/* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3670 * Will also mark the memory as dirty if is_write == 1. access_len gives
3671 * the amount of memory that was actually read or written by the caller.
3672 */
3673void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3674 int is_write, target_phys_addr_t access_len)
3675{
3676 if (buffer != bounce.buffer) {
3677 if (is_write) {
3678 ram_addr_t addr1 = qemu_ram_addr_from_host(buffer);
3679 while (access_len) {
3680 unsigned l;
3681 l = TARGET_PAGE_SIZE;
3682 if (l > access_len)
3683 l = access_len;
3684 if (!cpu_physical_memory_is_dirty(addr1)) {
3685 /* invalidate code */
3686 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3687 /* set dirty bit */
3688 cpu_physical_memory_set_dirty_flags(
3689 addr1, (0xff & ~CODE_DIRTY_FLAG));
3690 }
3691 addr1 += l;
3692 access_len -= l;
3693 }
3694 }
3695 return;
3696 }
3697 if (is_write) {
3698 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3699 }
3700 qemu_vfree(bounce.buffer);
3701 bounce.buffer = NULL;
3702 cpu_notify_map_clients();
3703}
3704
3705/* warning: addr must be aligned */
3706uint32_t ldl_phys(target_phys_addr_t addr)
3707{
3708 int io_index;
3709 uint8_t *ptr;
3710 uint32_t val;
3711 unsigned long pd;
3712 PhysPageDesc *p;
3713
3714 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3715 if (!p) {
3716 pd = IO_MEM_UNASSIGNED;
3717 } else {
3718 pd = p->phys_offset;
3719 }
3720
3721 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3722 !(pd & IO_MEM_ROMD)) {
3723 /* I/O case */
3724 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3725 if (p)
3726 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3727 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3728 } else {
3729 /* RAM case */
3730 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3731 (addr & ~TARGET_PAGE_MASK);
3732 val = ldl_p(ptr);
3733 }
3734 return val;
3735}
3736
3737/* warning: addr must be aligned */
3738uint64_t ldq_phys(target_phys_addr_t addr)
3739{
3740 int io_index;
3741 uint8_t *ptr;
3742 uint64_t val;
3743 unsigned long pd;
3744 PhysPageDesc *p;
3745
3746 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3747 if (!p) {
3748 pd = IO_MEM_UNASSIGNED;
3749 } else {
3750 pd = p->phys_offset;
3751 }
3752
3753 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3754 !(pd & IO_MEM_ROMD)) {
3755 /* I/O case */
3756 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3757 if (p)
3758 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3759#ifdef TARGET_WORDS_BIGENDIAN
3760 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3761 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3762#else
3763 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3764 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3765#endif
3766 } else {
3767 /* RAM case */
3768 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3769 (addr & ~TARGET_PAGE_MASK);
3770 val = ldq_p(ptr);
3771 }
3772 return val;
3773}
3774
3775/* XXX: optimize */
3776uint32_t ldub_phys(target_phys_addr_t addr)
3777{
3778 uint8_t val;
3779 cpu_physical_memory_read(addr, &val, 1);
3780 return val;
3781}
3782
3783/* XXX: optimize */
3784uint32_t lduw_phys(target_phys_addr_t addr)
3785{
3786 uint16_t val;
3787 cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3788 return tswap16(val);
3789}
3790
3791/* warning: addr must be aligned. The ram page is not masked as dirty
3792 and the code inside is not invalidated. It is useful if the dirty
3793 bits are used to track modified PTEs */
3794void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3795{
3796 int io_index;
3797 uint8_t *ptr;
3798 unsigned long pd;
3799 PhysPageDesc *p;
3800
3801 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3802 if (!p) {
3803 pd = IO_MEM_UNASSIGNED;
3804 } else {
3805 pd = p->phys_offset;
3806 }
3807
3808 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3809 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3810 if (p)
3811 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3812 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3813 } else {
3814 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3815 ptr = qemu_get_ram_ptr(addr1);
3816 stl_p(ptr, val);
3817
3818 if (unlikely(in_migration)) {
3819 if (!cpu_physical_memory_is_dirty(addr1)) {
3820 /* invalidate code */
3821 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3822 /* set dirty bit */
3823 cpu_physical_memory_set_dirty_flags(
3824 addr1, (0xff & ~CODE_DIRTY_FLAG));
3825 }
3826 }
3827 }
3828}
3829
3830void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3831{
3832 int io_index;
3833 uint8_t *ptr;
3834 unsigned long pd;
3835 PhysPageDesc *p;
3836
3837 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3838 if (!p) {
3839 pd = IO_MEM_UNASSIGNED;
3840 } else {
3841 pd = p->phys_offset;
3842 }
3843
3844 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3845 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3846 if (p)
3847 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3848#ifdef TARGET_WORDS_BIGENDIAN
3849 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3850 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3851#else
3852 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3853 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3854#endif
3855 } else {
3856 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3857 (addr & ~TARGET_PAGE_MASK);
3858 stq_p(ptr, val);
3859 }
3860}
3861
3862/* warning: addr must be aligned */
3863void stl_phys(target_phys_addr_t addr, uint32_t val)
3864{
3865 int io_index;
3866 uint8_t *ptr;
3867 unsigned long pd;
3868 PhysPageDesc *p;
3869
3870 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3871 if (!p) {
3872 pd = IO_MEM_UNASSIGNED;
3873 } else {
3874 pd = p->phys_offset;
3875 }
3876
3877 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3878 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3879 if (p)
3880 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3881 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3882 } else {
3883 unsigned long addr1;
3884 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3885 /* RAM case */
3886 ptr = qemu_get_ram_ptr(addr1);
3887 stl_p(ptr, val);
3888 if (!cpu_physical_memory_is_dirty(addr1)) {
3889 /* invalidate code */
3890 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3891 /* set dirty bit */
3892 cpu_physical_memory_set_dirty_flags(addr1,
3893 (0xff & ~CODE_DIRTY_FLAG));
3894 }
3895 }
3896}
3897
3898/* XXX: optimize */
3899void stb_phys(target_phys_addr_t addr, uint32_t val)
3900{
3901 uint8_t v = val;
3902 cpu_physical_memory_write(addr, &v, 1);
3903}
3904
3905/* XXX: optimize */
3906void stw_phys(target_phys_addr_t addr, uint32_t val)
3907{
3908 uint16_t v = tswap16(val);
3909 cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
3910}
3911
3912/* XXX: optimize */
3913void stq_phys(target_phys_addr_t addr, uint64_t val)
3914{
3915 val = tswap64(val);
3916 cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3917}
3918
3919/* virtual memory access for debug (includes writing to ROM) */
3920int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3921 uint8_t *buf, int len, int is_write)
3922{
3923 int l;
3924 target_phys_addr_t phys_addr;
3925 target_ulong page;
3926
3927 while (len > 0) {
3928 page = addr & TARGET_PAGE_MASK;
3929 phys_addr = cpu_get_phys_page_debug(env, page);
3930 /* if no physical page mapped, return an error */
3931 if (phys_addr == -1)
3932 return -1;
3933 l = (page + TARGET_PAGE_SIZE) - addr;
3934 if (l > len)
3935 l = len;
3936 phys_addr += (addr & ~TARGET_PAGE_MASK);
3937 if (is_write)
3938 cpu_physical_memory_write_rom(phys_addr, buf, l);
3939 else
3940 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
3941 len -= l;
3942 buf += l;
3943 addr += l;
3944 }
3945 return 0;
3946}
3947#endif
3948
3949/* in deterministic execution mode, instructions doing device I/Os
3950 must be at the end of the TB */
3951void cpu_io_recompile(CPUState *env, void *retaddr)
3952{
3953 TranslationBlock *tb;
3954 uint32_t n, cflags;
3955 target_ulong pc, cs_base;
3956 uint64_t flags;
3957
3958 tb = tb_find_pc((unsigned long)retaddr);
3959 if (!tb) {
3960 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
3961 retaddr);
3962 }
3963 n = env->icount_decr.u16.low + tb->icount;
3964 cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3965 /* Calculate how many instructions had been executed before the fault
3966 occurred. */
3967 n = n - env->icount_decr.u16.low;
3968 /* Generate a new TB ending on the I/O insn. */
3969 n++;
3970 /* On MIPS and SH, delay slot instructions can only be restarted if
3971 they were already the first instruction in the TB. If this is not
3972 the first instruction in a TB then re-execute the preceding
3973 branch. */
3974#if defined(TARGET_MIPS)
3975 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3976 env->active_tc.PC -= 4;
3977 env->icount_decr.u16.low++;
3978 env->hflags &= ~MIPS_HFLAG_BMASK;
3979 }
3980#elif defined(TARGET_SH4)
3981 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3982 && n > 1) {
3983 env->pc -= 2;
3984 env->icount_decr.u16.low++;
3985 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3986 }
3987#endif
3988 /* This should never happen. */
3989 if (n > CF_COUNT_MASK)
3990 cpu_abort(env, "TB too big during recompile");
3991
3992 cflags = n | CF_LAST_IO;
3993 pc = tb->pc;
3994 cs_base = tb->cs_base;
3995 flags = tb->flags;
3996 tb_phys_invalidate(tb, -1);
3997 /* FIXME: In theory this could raise an exception. In practice
3998 we have already translated the block once so it's probably ok. */
3999 tb_gen_code(env, pc, cs_base, flags, cflags);
4000 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
4001 the first in the TB) then we end up generating a whole new TB and
4002 repeating the fault, which is horribly inefficient.
4003 Better would be to execute just this insn uncached, or generate a
4004 second new TB. */
4005 cpu_resume_from_signal(env, NULL);
4006}
4007
4008#if !defined(CONFIG_USER_ONLY)
4009
4010void dump_exec_info(FILE *f,
4011 int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
4012{
4013 int i, target_code_size, max_target_code_size;
4014 int direct_jmp_count, direct_jmp2_count, cross_page;
4015 TranslationBlock *tb;
4016
4017 target_code_size = 0;
4018 max_target_code_size = 0;
4019 cross_page = 0;
4020 direct_jmp_count = 0;
4021 direct_jmp2_count = 0;
4022 for(i = 0; i < nb_tbs; i++) {
4023 tb = &tbs[i];
4024 target_code_size += tb->size;
4025 if (tb->size > max_target_code_size)
4026 max_target_code_size = tb->size;
4027 if (tb->page_addr[1] != -1)
4028 cross_page++;
4029 if (tb->tb_next_offset[0] != 0xffff) {
4030 direct_jmp_count++;
4031 if (tb->tb_next_offset[1] != 0xffff) {
4032 direct_jmp2_count++;
4033 }
4034 }
4035 }
4036 /* XXX: avoid using doubles ? */
4037 cpu_fprintf(f, "Translation buffer state:\n");
4038 cpu_fprintf(f, "gen code size %ld/%ld\n",
4039 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
4040 cpu_fprintf(f, "TB count %d/%d\n",
4041 nb_tbs, code_gen_max_blocks);
4042 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
4043 nb_tbs ? target_code_size / nb_tbs : 0,
4044 max_target_code_size);
4045 cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
4046 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
4047 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
4048 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
4049 cross_page,
4050 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
4051 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
4052 direct_jmp_count,
4053 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
4054 direct_jmp2_count,
4055 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
4056 cpu_fprintf(f, "\nStatistics:\n");
4057 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
4058 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
4059 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
4060 tcg_dump_info(f, cpu_fprintf);
4061}
4062
4063#define MMUSUFFIX _cmmu
4064#define GETPC() NULL
4065#define env cpu_single_env
4066#define SOFTMMU_CODE_ACCESS
4067
4068#define SHIFT 0
4069#include "softmmu_template.h"
4070
4071#define SHIFT 1
4072#include "softmmu_template.h"
4073
4074#define SHIFT 2
4075#include "softmmu_template.h"
4076
4077#define SHIFT 3
4078#include "softmmu_template.h"
4079
4080#undef env
4081
4082#endif
QEMU mirror