]> git.proxmox.com Git - mirror_qemu.git/blame_incremental - exec.c
projects / mirror_qemu.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
linux-user: adapt uname machine to emulated CPU
[mirror_qemu.git] / exec.c
... / ...
CommitLineData
1/*
2 * virtual page mapping and translated block handling
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19#include "config.h"
20#ifdef _WIN32
21#include <windows.h>
22#else
23#include <sys/types.h>
24#include <sys/mman.h>
25#endif
26#include <stdlib.h>
27#include <stdio.h>
28#include <stdarg.h>
29#include <string.h>
30#include <errno.h>
31#include <unistd.h>
32#include <inttypes.h>
33
34#include "cpu.h"
35#include "exec-all.h"
36#include "qemu-common.h"
37#include "tcg.h"
38#include "hw/hw.h"
39#include "osdep.h"
40#include "kvm.h"
41#if defined(CONFIG_USER_ONLY)
42#include <qemu.h>
43#endif
44
45//#define DEBUG_TB_INVALIDATE
46//#define DEBUG_FLUSH
47//#define DEBUG_TLB
48//#define DEBUG_UNASSIGNED
49
50/* make various TB consistency checks */
51//#define DEBUG_TB_CHECK
52//#define DEBUG_TLB_CHECK
53
54//#define DEBUG_IOPORT
55//#define DEBUG_SUBPAGE
56
57#if !defined(CONFIG_USER_ONLY)
58/* TB consistency checks only implemented for usermode emulation. */
59#undef DEBUG_TB_CHECK
60#endif
61
62#define SMC_BITMAP_USE_THRESHOLD 10
63
64#if defined(TARGET_SPARC64)
65#define TARGET_PHYS_ADDR_SPACE_BITS 41
66#elif defined(TARGET_SPARC)
67#define TARGET_PHYS_ADDR_SPACE_BITS 36
68#elif defined(TARGET_ALPHA)
69#define TARGET_PHYS_ADDR_SPACE_BITS 42
70#define TARGET_VIRT_ADDR_SPACE_BITS 42
71#elif defined(TARGET_PPC64)
72#define TARGET_PHYS_ADDR_SPACE_BITS 42
73#elif defined(TARGET_X86_64)
74#define TARGET_PHYS_ADDR_SPACE_BITS 42
75#elif defined(TARGET_I386)
76#define TARGET_PHYS_ADDR_SPACE_BITS 36
77#else
78#define TARGET_PHYS_ADDR_SPACE_BITS 32
79#endif
80
81static TranslationBlock *tbs;
82int code_gen_max_blocks;
83TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
84static int nb_tbs;
85/* any access to the tbs or the page table must use this lock */
86spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
87
88#if defined(__arm__) || defined(__sparc_v9__)
89/* The prologue must be reachable with a direct jump. ARM and Sparc64
90 have limited branch ranges (possibly also PPC) so place it in a
91 section close to code segment. */
92#define code_gen_section \
93 __attribute__((__section__(".gen_code"))) \
94 __attribute__((aligned (32)))
95#elif defined(_WIN32)
96/* Maximum alignment for Win32 is 16. */
97#define code_gen_section \
98 __attribute__((aligned (16)))
99#else
100#define code_gen_section \
101 __attribute__((aligned (32)))
102#endif
103
104uint8_t code_gen_prologue[1024] code_gen_section;
105static uint8_t *code_gen_buffer;
106static unsigned long code_gen_buffer_size;
107/* threshold to flush the translated code buffer */
108static unsigned long code_gen_buffer_max_size;
109uint8_t *code_gen_ptr;
110
111#if !defined(CONFIG_USER_ONLY)
112int phys_ram_fd;
113uint8_t *phys_ram_dirty;
114static int in_migration;
115
116typedef struct RAMBlock {
117 uint8_t *host;
118 ram_addr_t offset;
119 ram_addr_t length;
120 struct RAMBlock *next;
121} RAMBlock;
122
123static RAMBlock *ram_blocks;
124/* TODO: When we implement (and use) ram deallocation (e.g. for hotplug)
125 then we can no longer assume contiguous ram offsets, and external uses
126 of this variable will break. */
127ram_addr_t last_ram_offset;
128#endif
129
130CPUState *first_cpu;
131/* current CPU in the current thread. It is only valid inside
132 cpu_exec() */
133CPUState *cpu_single_env;
134/* 0 = Do not count executed instructions.
135 1 = Precise instruction counting.
136 2 = Adaptive rate instruction counting. */
137int use_icount = 0;
138/* Current instruction counter. While executing translated code this may
139 include some instructions that have not yet been executed. */
140int64_t qemu_icount;
141
142typedef struct PageDesc {
143 /* list of TBs intersecting this ram page */
144 TranslationBlock *first_tb;
145 /* in order to optimize self modifying code, we count the number
146 of lookups we do to a given page to use a bitmap */
147 unsigned int code_write_count;
148 uint8_t *code_bitmap;
149#if defined(CONFIG_USER_ONLY)
150 unsigned long flags;
151#endif
152} PageDesc;
153
154typedef struct PhysPageDesc {
155 /* offset in host memory of the page + io_index in the low bits */
156 ram_addr_t phys_offset;
157 ram_addr_t region_offset;
158} PhysPageDesc;
159
160#define L2_BITS 10
161#if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
162/* XXX: this is a temporary hack for alpha target.
163 * In the future, this is to be replaced by a multi-level table
164 * to actually be able to handle the complete 64 bits address space.
165 */
166#define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
167#else
168#define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
169#endif
170
171#define L1_SIZE (1 << L1_BITS)
172#define L2_SIZE (1 << L2_BITS)
173
174unsigned long qemu_real_host_page_size;
175unsigned long qemu_host_page_bits;
176unsigned long qemu_host_page_size;
177unsigned long qemu_host_page_mask;
178
179/* XXX: for system emulation, it could just be an array */
180static PageDesc *l1_map[L1_SIZE];
181static PhysPageDesc **l1_phys_map;
182
183#if !defined(CONFIG_USER_ONLY)
184static void io_mem_init(void);
185
186/* io memory support */
187CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
188CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
189void *io_mem_opaque[IO_MEM_NB_ENTRIES];
190static char io_mem_used[IO_MEM_NB_ENTRIES];
191static int io_mem_watch;
192#endif
193
194/* log support */
195#ifdef WIN32
196static const char *logfilename = "qemu.log";
197#else
198static const char *logfilename = "/tmp/qemu.log";
199#endif
200FILE *logfile;
201int loglevel;
202static int log_append = 0;
203
204/* statistics */
205static int tlb_flush_count;
206static int tb_flush_count;
207static int tb_phys_invalidate_count;
208
209#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
210typedef struct subpage_t {
211 target_phys_addr_t base;
212 CPUReadMemoryFunc * const *mem_read[TARGET_PAGE_SIZE][4];
213 CPUWriteMemoryFunc * const *mem_write[TARGET_PAGE_SIZE][4];
214 void *opaque[TARGET_PAGE_SIZE][2][4];
215 ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
216} subpage_t;
217
218#ifdef _WIN32
219static void map_exec(void *addr, long size)
220{
221 DWORD old_protect;
222 VirtualProtect(addr, size,
223 PAGE_EXECUTE_READWRITE, &old_protect);
224
225}
226#else
227static void map_exec(void *addr, long size)
228{
229 unsigned long start, end, page_size;
230
231 page_size = getpagesize();
232 start = (unsigned long)addr;
233 start &= ~(page_size - 1);
234
235 end = (unsigned long)addr + size;
236 end += page_size - 1;
237 end &= ~(page_size - 1);
238
239 mprotect((void *)start, end - start,
240 PROT_READ | PROT_WRITE | PROT_EXEC);
241}
242#endif
243
244static void page_init(void)
245{
246 /* NOTE: we can always suppose that qemu_host_page_size >=
247 TARGET_PAGE_SIZE */
248#ifdef _WIN32
249 {
250 SYSTEM_INFO system_info;
251
252 GetSystemInfo(&system_info);
253 qemu_real_host_page_size = system_info.dwPageSize;
254 }
255#else
256 qemu_real_host_page_size = getpagesize();
257#endif
258 if (qemu_host_page_size == 0)
259 qemu_host_page_size = qemu_real_host_page_size;
260 if (qemu_host_page_size < TARGET_PAGE_SIZE)
261 qemu_host_page_size = TARGET_PAGE_SIZE;
262 qemu_host_page_bits = 0;
263 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
264 qemu_host_page_bits++;
265 qemu_host_page_mask = ~(qemu_host_page_size - 1);
266 l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
267 memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
268
269#if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
270 {
271 long long startaddr, endaddr;
272 FILE *f;
273 int n;
274
275 mmap_lock();
276 last_brk = (unsigned long)sbrk(0);
277 f = fopen("/proc/self/maps", "r");
278 if (f) {
279 do {
280 n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
281 if (n == 2) {
282 startaddr = MIN(startaddr,
283 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
284 endaddr = MIN(endaddr,
285 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
286 page_set_flags(startaddr & TARGET_PAGE_MASK,
287 TARGET_PAGE_ALIGN(endaddr),
288 PAGE_RESERVED);
289 }
290 } while (!feof(f));
291 fclose(f);
292 }
293 mmap_unlock();
294 }
295#endif
296}
297
298static inline PageDesc **page_l1_map(target_ulong index)
299{
300#if TARGET_LONG_BITS > 32
301 /* Host memory outside guest VM. For 32-bit targets we have already
302 excluded high addresses. */
303 if (index > ((target_ulong)L2_SIZE * L1_SIZE))
304 return NULL;
305#endif
306 return &l1_map[index >> L2_BITS];
307}
308
309static inline PageDesc *page_find_alloc(target_ulong index)
310{
311 PageDesc **lp, *p;
312 lp = page_l1_map(index);
313 if (!lp)
314 return NULL;
315
316 p = *lp;
317 if (!p) {
318 /* allocate if not found */
319#if defined(CONFIG_USER_ONLY)
320 size_t len = sizeof(PageDesc) * L2_SIZE;
321 /* Don't use qemu_malloc because it may recurse. */
322 p = mmap(NULL, len, PROT_READ | PROT_WRITE,
323 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
324 *lp = p;
325 if (h2g_valid(p)) {
326 unsigned long addr = h2g(p);
327 page_set_flags(addr & TARGET_PAGE_MASK,
328 TARGET_PAGE_ALIGN(addr + len),
329 PAGE_RESERVED);
330 }
331#else
332 p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
333 *lp = p;
334#endif
335 }
336 return p + (index & (L2_SIZE - 1));
337}
338
339static inline PageDesc *page_find(target_ulong index)
340{
341 PageDesc **lp, *p;
342 lp = page_l1_map(index);
343 if (!lp)
344 return NULL;
345
346 p = *lp;
347 if (!p) {
348 return NULL;
349 }
350 return p + (index & (L2_SIZE - 1));
351}
352
353static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
354{
355 void **lp, **p;
356 PhysPageDesc *pd;
357
358 p = (void **)l1_phys_map;
359#if TARGET_PHYS_ADDR_SPACE_BITS > 32
360
361#if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
362#error unsupported TARGET_PHYS_ADDR_SPACE_BITS
363#endif
364 lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
365 p = *lp;
366 if (!p) {
367 /* allocate if not found */
368 if (!alloc)
369 return NULL;
370 p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
371 memset(p, 0, sizeof(void *) * L1_SIZE);
372 *lp = p;
373 }
374#endif
375 lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
376 pd = *lp;
377 if (!pd) {
378 int i;
379 /* allocate if not found */
380 if (!alloc)
381 return NULL;
382 pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
383 *lp = pd;
384 for (i = 0; i < L2_SIZE; i++) {
385 pd[i].phys_offset = IO_MEM_UNASSIGNED;
386 pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
387 }
388 }
389 return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
390}
391
392static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
393{
394 return phys_page_find_alloc(index, 0);
395}
396
397#if !defined(CONFIG_USER_ONLY)
398static void tlb_protect_code(ram_addr_t ram_addr);
399static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
400 target_ulong vaddr);
401#define mmap_lock() do { } while(0)
402#define mmap_unlock() do { } while(0)
403#endif
404
405#define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
406
407#if defined(CONFIG_USER_ONLY)
408/* Currently it is not recommended to allocate big chunks of data in
409 user mode. It will change when a dedicated libc will be used */
410#define USE_STATIC_CODE_GEN_BUFFER
411#endif
412
413#ifdef USE_STATIC_CODE_GEN_BUFFER
414static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
415#endif
416
417static void code_gen_alloc(unsigned long tb_size)
418{
419#ifdef USE_STATIC_CODE_GEN_BUFFER
420 code_gen_buffer = static_code_gen_buffer;
421 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
422 map_exec(code_gen_buffer, code_gen_buffer_size);
423#else
424 code_gen_buffer_size = tb_size;
425 if (code_gen_buffer_size == 0) {
426#if defined(CONFIG_USER_ONLY)
427 /* in user mode, phys_ram_size is not meaningful */
428 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
429#else
430 /* XXX: needs adjustments */
431 code_gen_buffer_size = (unsigned long)(ram_size / 4);
432#endif
433 }
434 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
435 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
436 /* The code gen buffer location may have constraints depending on
437 the host cpu and OS */
438#if defined(__linux__)
439 {
440 int flags;
441 void *start = NULL;
442
443 flags = MAP_PRIVATE | MAP_ANONYMOUS;
444#if defined(__x86_64__)
445 flags |= MAP_32BIT;
446 /* Cannot map more than that */
447 if (code_gen_buffer_size > (800 * 1024 * 1024))
448 code_gen_buffer_size = (800 * 1024 * 1024);
449#elif defined(__sparc_v9__)
450 // Map the buffer below 2G, so we can use direct calls and branches
451 flags |= MAP_FIXED;
452 start = (void *) 0x60000000UL;
453 if (code_gen_buffer_size > (512 * 1024 * 1024))
454 code_gen_buffer_size = (512 * 1024 * 1024);
455#elif defined(__arm__)
456 /* Map the buffer below 32M, so we can use direct calls and branches */
457 flags |= MAP_FIXED;
458 start = (void *) 0x01000000UL;
459 if (code_gen_buffer_size > 16 * 1024 * 1024)
460 code_gen_buffer_size = 16 * 1024 * 1024;
461#endif
462 code_gen_buffer = mmap(start, code_gen_buffer_size,
463 PROT_WRITE | PROT_READ | PROT_EXEC,
464 flags, -1, 0);
465 if (code_gen_buffer == MAP_FAILED) {
466 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
467 exit(1);
468 }
469 }
470#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
471 {
472 int flags;
473 void *addr = NULL;
474 flags = MAP_PRIVATE | MAP_ANONYMOUS;
475#if defined(__x86_64__)
476 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
477 * 0x40000000 is free */
478 flags |= MAP_FIXED;
479 addr = (void *)0x40000000;
480 /* Cannot map more than that */
481 if (code_gen_buffer_size > (800 * 1024 * 1024))
482 code_gen_buffer_size = (800 * 1024 * 1024);
483#endif
484 code_gen_buffer = mmap(addr, code_gen_buffer_size,
485 PROT_WRITE | PROT_READ | PROT_EXEC,
486 flags, -1, 0);
487 if (code_gen_buffer == MAP_FAILED) {
488 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
489 exit(1);
490 }
491 }
492#else
493 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
494 map_exec(code_gen_buffer, code_gen_buffer_size);
495#endif
496#endif /* !USE_STATIC_CODE_GEN_BUFFER */
497 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
498 code_gen_buffer_max_size = code_gen_buffer_size -
499 code_gen_max_block_size();
500 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
501 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
502}
503
504/* Must be called before using the QEMU cpus. 'tb_size' is the size
505 (in bytes) allocated to the translation buffer. Zero means default
506 size. */
507void cpu_exec_init_all(unsigned long tb_size)
508{
509 cpu_gen_init();
510 code_gen_alloc(tb_size);
511 code_gen_ptr = code_gen_buffer;
512 page_init();
513#if !defined(CONFIG_USER_ONLY)
514 io_mem_init();
515#endif
516}
517
518#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
519
520static void cpu_common_pre_save(void *opaque)
521{
522 CPUState *env = opaque;
523
524 cpu_synchronize_state(env);
525}
526
527static int cpu_common_pre_load(void *opaque)
528{
529 CPUState *env = opaque;
530
531 cpu_synchronize_state(env);
532 return 0;
533}
534
535static int cpu_common_post_load(void *opaque, int version_id)
536{
537 CPUState *env = opaque;
538
539 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
540 version_id is increased. */
541 env->interrupt_request &= ~0x01;
542 tlb_flush(env, 1);
543
544 return 0;
545}
546
547static const VMStateDescription vmstate_cpu_common = {
548 .name = "cpu_common",
549 .version_id = 1,
550 .minimum_version_id = 1,
551 .minimum_version_id_old = 1,
552 .pre_save = cpu_common_pre_save,
553 .pre_load = cpu_common_pre_load,
554 .post_load = cpu_common_post_load,
555 .fields = (VMStateField []) {
556 VMSTATE_UINT32(halted, CPUState),
557 VMSTATE_UINT32(interrupt_request, CPUState),
558 VMSTATE_END_OF_LIST()
559 }
560};
561#endif
562
563CPUState *qemu_get_cpu(int cpu)
564{
565 CPUState *env = first_cpu;
566
567 while (env) {
568 if (env->cpu_index == cpu)
569 break;
570 env = env->next_cpu;
571 }
572
573 return env;
574}
575
576void cpu_exec_init(CPUState *env)
577{
578 CPUState **penv;
579 int cpu_index;
580
581#if defined(CONFIG_USER_ONLY)
582 cpu_list_lock();
583#endif
584 env->next_cpu = NULL;
585 penv = &first_cpu;
586 cpu_index = 0;
587 while (*penv != NULL) {
588 penv = &(*penv)->next_cpu;
589 cpu_index++;
590 }
591 env->cpu_index = cpu_index;
592 env->numa_node = 0;
593 QTAILQ_INIT(&env->breakpoints);
594 QTAILQ_INIT(&env->watchpoints);
595 *penv = env;
596#if defined(CONFIG_USER_ONLY)
597 cpu_list_unlock();
598#endif
599#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
600 vmstate_register(cpu_index, &vmstate_cpu_common, env);
601 register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
602 cpu_save, cpu_load, env);
603#endif
604}
605
606static inline void invalidate_page_bitmap(PageDesc *p)
607{
608 if (p->code_bitmap) {
609 qemu_free(p->code_bitmap);
610 p->code_bitmap = NULL;
611 }
612 p->code_write_count = 0;
613}
614
615/* set to NULL all the 'first_tb' fields in all PageDescs */
616static void page_flush_tb(void)
617{
618 int i, j;
619 PageDesc *p;
620
621 for(i = 0; i < L1_SIZE; i++) {
622 p = l1_map[i];
623 if (p) {
624 for(j = 0; j < L2_SIZE; j++) {
625 p->first_tb = NULL;
626 invalidate_page_bitmap(p);
627 p++;
628 }
629 }
630 }
631}
632
633/* flush all the translation blocks */
634/* XXX: tb_flush is currently not thread safe */
635void tb_flush(CPUState *env1)
636{
637 CPUState *env;
638#if defined(DEBUG_FLUSH)
639 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
640 (unsigned long)(code_gen_ptr - code_gen_buffer),
641 nb_tbs, nb_tbs > 0 ?
642 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
643#endif
644 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
645 cpu_abort(env1, "Internal error: code buffer overflow\n");
646
647 nb_tbs = 0;
648
649 for(env = first_cpu; env != NULL; env = env->next_cpu) {
650 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
651 }
652
653 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
654 page_flush_tb();
655
656 code_gen_ptr = code_gen_buffer;
657 /* XXX: flush processor icache at this point if cache flush is
658 expensive */
659 tb_flush_count++;
660}
661
662#ifdef DEBUG_TB_CHECK
663
664static void tb_invalidate_check(target_ulong address)
665{
666 TranslationBlock *tb;
667 int i;
668 address &= TARGET_PAGE_MASK;
669 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
670 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
671 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
672 address >= tb->pc + tb->size)) {
673 printf("ERROR invalidate: address=" TARGET_FMT_lx
674 " PC=%08lx size=%04x\n",
675 address, (long)tb->pc, tb->size);
676 }
677 }
678 }
679}
680
681/* verify that all the pages have correct rights for code */
682static void tb_page_check(void)
683{
684 TranslationBlock *tb;
685 int i, flags1, flags2;
686
687 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
688 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
689 flags1 = page_get_flags(tb->pc);
690 flags2 = page_get_flags(tb->pc + tb->size - 1);
691 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
692 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
693 (long)tb->pc, tb->size, flags1, flags2);
694 }
695 }
696 }
697}
698
699#endif
700
701/* invalidate one TB */
702static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
703 int next_offset)
704{
705 TranslationBlock *tb1;
706 for(;;) {
707 tb1 = *ptb;
708 if (tb1 == tb) {
709 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
710 break;
711 }
712 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
713 }
714}
715
716static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
717{
718 TranslationBlock *tb1;
719 unsigned int n1;
720
721 for(;;) {
722 tb1 = *ptb;
723 n1 = (long)tb1 & 3;
724 tb1 = (TranslationBlock *)((long)tb1 & ~3);
725 if (tb1 == tb) {
726 *ptb = tb1->page_next[n1];
727 break;
728 }
729 ptb = &tb1->page_next[n1];
730 }
731}
732
733static inline void tb_jmp_remove(TranslationBlock *tb, int n)
734{
735 TranslationBlock *tb1, **ptb;
736 unsigned int n1;
737
738 ptb = &tb->jmp_next[n];
739 tb1 = *ptb;
740 if (tb1) {
741 /* find tb(n) in circular list */
742 for(;;) {
743 tb1 = *ptb;
744 n1 = (long)tb1 & 3;
745 tb1 = (TranslationBlock *)((long)tb1 & ~3);
746 if (n1 == n && tb1 == tb)
747 break;
748 if (n1 == 2) {
749 ptb = &tb1->jmp_first;
750 } else {
751 ptb = &tb1->jmp_next[n1];
752 }
753 }
754 /* now we can suppress tb(n) from the list */
755 *ptb = tb->jmp_next[n];
756
757 tb->jmp_next[n] = NULL;
758 }
759}
760
761/* reset the jump entry 'n' of a TB so that it is not chained to
762 another TB */
763static inline void tb_reset_jump(TranslationBlock *tb, int n)
764{
765 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
766}
767
768void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
769{
770 CPUState *env;
771 PageDesc *p;
772 unsigned int h, n1;
773 target_phys_addr_t phys_pc;
774 TranslationBlock *tb1, *tb2;
775
776 /* remove the TB from the hash list */
777 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
778 h = tb_phys_hash_func(phys_pc);
779 tb_remove(&tb_phys_hash[h], tb,
780 offsetof(TranslationBlock, phys_hash_next));
781
782 /* remove the TB from the page list */
783 if (tb->page_addr[0] != page_addr) {
784 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
785 tb_page_remove(&p->first_tb, tb);
786 invalidate_page_bitmap(p);
787 }
788 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
789 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
790 tb_page_remove(&p->first_tb, tb);
791 invalidate_page_bitmap(p);
792 }
793
794 tb_invalidated_flag = 1;
795
796 /* remove the TB from the hash list */
797 h = tb_jmp_cache_hash_func(tb->pc);
798 for(env = first_cpu; env != NULL; env = env->next_cpu) {
799 if (env->tb_jmp_cache[h] == tb)
800 env->tb_jmp_cache[h] = NULL;
801 }
802
803 /* suppress this TB from the two jump lists */
804 tb_jmp_remove(tb, 0);
805 tb_jmp_remove(tb, 1);
806
807 /* suppress any remaining jumps to this TB */
808 tb1 = tb->jmp_first;
809 for(;;) {
810 n1 = (long)tb1 & 3;
811 if (n1 == 2)
812 break;
813 tb1 = (TranslationBlock *)((long)tb1 & ~3);
814 tb2 = tb1->jmp_next[n1];
815 tb_reset_jump(tb1, n1);
816 tb1->jmp_next[n1] = NULL;
817 tb1 = tb2;
818 }
819 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
820
821 tb_phys_invalidate_count++;
822}
823
824static inline void set_bits(uint8_t *tab, int start, int len)
825{
826 int end, mask, end1;
827
828 end = start + len;
829 tab += start >> 3;
830 mask = 0xff << (start & 7);
831 if ((start & ~7) == (end & ~7)) {
832 if (start < end) {
833 mask &= ~(0xff << (end & 7));
834 *tab |= mask;
835 }
836 } else {
837 *tab++ |= mask;
838 start = (start + 8) & ~7;
839 end1 = end & ~7;
840 while (start < end1) {
841 *tab++ = 0xff;
842 start += 8;
843 }
844 if (start < end) {
845 mask = ~(0xff << (end & 7));
846 *tab |= mask;
847 }
848 }
849}
850
851static void build_page_bitmap(PageDesc *p)
852{
853 int n, tb_start, tb_end;
854 TranslationBlock *tb;
855
856 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
857
858 tb = p->first_tb;
859 while (tb != NULL) {
860 n = (long)tb & 3;
861 tb = (TranslationBlock *)((long)tb & ~3);
862 /* NOTE: this is subtle as a TB may span two physical pages */
863 if (n == 0) {
864 /* NOTE: tb_end may be after the end of the page, but
865 it is not a problem */
866 tb_start = tb->pc & ~TARGET_PAGE_MASK;
867 tb_end = tb_start + tb->size;
868 if (tb_end > TARGET_PAGE_SIZE)
869 tb_end = TARGET_PAGE_SIZE;
870 } else {
871 tb_start = 0;
872 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
873 }
874 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
875 tb = tb->page_next[n];
876 }
877}
878
879TranslationBlock *tb_gen_code(CPUState *env,
880 target_ulong pc, target_ulong cs_base,
881 int flags, int cflags)
882{
883 TranslationBlock *tb;
884 uint8_t *tc_ptr;
885 target_ulong phys_pc, phys_page2, virt_page2;
886 int code_gen_size;
887
888 phys_pc = get_phys_addr_code(env, pc);
889 tb = tb_alloc(pc);
890 if (!tb) {
891 /* flush must be done */
892 tb_flush(env);
893 /* cannot fail at this point */
894 tb = tb_alloc(pc);
895 /* Don't forget to invalidate previous TB info. */
896 tb_invalidated_flag = 1;
897 }
898 tc_ptr = code_gen_ptr;
899 tb->tc_ptr = tc_ptr;
900 tb->cs_base = cs_base;
901 tb->flags = flags;
902 tb->cflags = cflags;
903 cpu_gen_code(env, tb, &code_gen_size);
904 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
905
906 /* check next page if needed */
907 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
908 phys_page2 = -1;
909 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
910 phys_page2 = get_phys_addr_code(env, virt_page2);
911 }
912 tb_link_phys(tb, phys_pc, phys_page2);
913 return tb;
914}
915
916/* invalidate all TBs which intersect with the target physical page
917 starting in range [start;end[. NOTE: start and end must refer to
918 the same physical page. 'is_cpu_write_access' should be true if called
919 from a real cpu write access: the virtual CPU will exit the current
920 TB if code is modified inside this TB. */
921void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
922 int is_cpu_write_access)
923{
924 TranslationBlock *tb, *tb_next, *saved_tb;
925 CPUState *env = cpu_single_env;
926 target_ulong tb_start, tb_end;
927 PageDesc *p;
928 int n;
929#ifdef TARGET_HAS_PRECISE_SMC
930 int current_tb_not_found = is_cpu_write_access;
931 TranslationBlock *current_tb = NULL;
932 int current_tb_modified = 0;
933 target_ulong current_pc = 0;
934 target_ulong current_cs_base = 0;
935 int current_flags = 0;
936#endif /* TARGET_HAS_PRECISE_SMC */
937
938 p = page_find(start >> TARGET_PAGE_BITS);
939 if (!p)
940 return;
941 if (!p->code_bitmap &&
942 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
943 is_cpu_write_access) {
944 /* build code bitmap */
945 build_page_bitmap(p);
946 }
947
948 /* we remove all the TBs in the range [start, end[ */
949 /* XXX: see if in some cases it could be faster to invalidate all the code */
950 tb = p->first_tb;
951 while (tb != NULL) {
952 n = (long)tb & 3;
953 tb = (TranslationBlock *)((long)tb & ~3);
954 tb_next = tb->page_next[n];
955 /* NOTE: this is subtle as a TB may span two physical pages */
956 if (n == 0) {
957 /* NOTE: tb_end may be after the end of the page, but
958 it is not a problem */
959 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
960 tb_end = tb_start + tb->size;
961 } else {
962 tb_start = tb->page_addr[1];
963 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
964 }
965 if (!(tb_end <= start || tb_start >= end)) {
966#ifdef TARGET_HAS_PRECISE_SMC
967 if (current_tb_not_found) {
968 current_tb_not_found = 0;
969 current_tb = NULL;
970 if (env->mem_io_pc) {
971 /* now we have a real cpu fault */
972 current_tb = tb_find_pc(env->mem_io_pc);
973 }
974 }
975 if (current_tb == tb &&
976 (current_tb->cflags & CF_COUNT_MASK) != 1) {
977 /* If we are modifying the current TB, we must stop
978 its execution. We could be more precise by checking
979 that the modification is after the current PC, but it
980 would require a specialized function to partially
981 restore the CPU state */
982
983 current_tb_modified = 1;
984 cpu_restore_state(current_tb, env,
985 env->mem_io_pc, NULL);
986 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
987 &current_flags);
988 }
989#endif /* TARGET_HAS_PRECISE_SMC */
990 /* we need to do that to handle the case where a signal
991 occurs while doing tb_phys_invalidate() */
992 saved_tb = NULL;
993 if (env) {
994 saved_tb = env->current_tb;
995 env->current_tb = NULL;
996 }
997 tb_phys_invalidate(tb, -1);
998 if (env) {
999 env->current_tb = saved_tb;
1000 if (env->interrupt_request && env->current_tb)
1001 cpu_interrupt(env, env->interrupt_request);
1002 }
1003 }
1004 tb = tb_next;
1005 }
1006#if !defined(CONFIG_USER_ONLY)
1007 /* if no code remaining, no need to continue to use slow writes */
1008 if (!p->first_tb) {
1009 invalidate_page_bitmap(p);
1010 if (is_cpu_write_access) {
1011 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1012 }
1013 }
1014#endif
1015#ifdef TARGET_HAS_PRECISE_SMC
1016 if (current_tb_modified) {
1017 /* we generate a block containing just the instruction
1018 modifying the memory. It will ensure that it cannot modify
1019 itself */
1020 env->current_tb = NULL;
1021 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1022 cpu_resume_from_signal(env, NULL);
1023 }
1024#endif
1025}
1026
1027/* len must be <= 8 and start must be a multiple of len */
1028static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
1029{
1030 PageDesc *p;
1031 int offset, b;
1032#if 0
1033 if (1) {
1034 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1035 cpu_single_env->mem_io_vaddr, len,
1036 cpu_single_env->eip,
1037 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1038 }
1039#endif
1040 p = page_find(start >> TARGET_PAGE_BITS);
1041 if (!p)
1042 return;
1043 if (p->code_bitmap) {
1044 offset = start & ~TARGET_PAGE_MASK;
1045 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1046 if (b & ((1 << len) - 1))
1047 goto do_invalidate;
1048 } else {
1049 do_invalidate:
1050 tb_invalidate_phys_page_range(start, start + len, 1);
1051 }
1052}
1053
1054#if !defined(CONFIG_SOFTMMU)
1055static void tb_invalidate_phys_page(target_phys_addr_t addr,
1056 unsigned long pc, void *puc)
1057{
1058 TranslationBlock *tb;
1059 PageDesc *p;
1060 int n;
1061#ifdef TARGET_HAS_PRECISE_SMC
1062 TranslationBlock *current_tb = NULL;
1063 CPUState *env = cpu_single_env;
1064 int current_tb_modified = 0;
1065 target_ulong current_pc = 0;
1066 target_ulong current_cs_base = 0;
1067 int current_flags = 0;
1068#endif
1069
1070 addr &= TARGET_PAGE_MASK;
1071 p = page_find(addr >> TARGET_PAGE_BITS);
1072 if (!p)
1073 return;
1074 tb = p->first_tb;
1075#ifdef TARGET_HAS_PRECISE_SMC
1076 if (tb && pc != 0) {
1077 current_tb = tb_find_pc(pc);
1078 }
1079#endif
1080 while (tb != NULL) {
1081 n = (long)tb & 3;
1082 tb = (TranslationBlock *)((long)tb & ~3);
1083#ifdef TARGET_HAS_PRECISE_SMC
1084 if (current_tb == tb &&
1085 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1086 /* If we are modifying the current TB, we must stop
1087 its execution. We could be more precise by checking
1088 that the modification is after the current PC, but it
1089 would require a specialized function to partially
1090 restore the CPU state */
1091
1092 current_tb_modified = 1;
1093 cpu_restore_state(current_tb, env, pc, puc);
1094 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1095 &current_flags);
1096 }
1097#endif /* TARGET_HAS_PRECISE_SMC */
1098 tb_phys_invalidate(tb, addr);
1099 tb = tb->page_next[n];
1100 }
1101 p->first_tb = NULL;
1102#ifdef TARGET_HAS_PRECISE_SMC
1103 if (current_tb_modified) {
1104 /* we generate a block containing just the instruction
1105 modifying the memory. It will ensure that it cannot modify
1106 itself */
1107 env->current_tb = NULL;
1108 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1109 cpu_resume_from_signal(env, puc);
1110 }
1111#endif
1112}
1113#endif
1114
1115/* add the tb in the target page and protect it if necessary */
1116static inline void tb_alloc_page(TranslationBlock *tb,
1117 unsigned int n, target_ulong page_addr)
1118{
1119 PageDesc *p;
1120 TranslationBlock *last_first_tb;
1121
1122 tb->page_addr[n] = page_addr;
1123 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
1124 tb->page_next[n] = p->first_tb;
1125 last_first_tb = p->first_tb;
1126 p->first_tb = (TranslationBlock *)((long)tb | n);
1127 invalidate_page_bitmap(p);
1128
1129#if defined(TARGET_HAS_SMC) || 1
1130
1131#if defined(CONFIG_USER_ONLY)
1132 if (p->flags & PAGE_WRITE) {
1133 target_ulong addr;
1134 PageDesc *p2;
1135 int prot;
1136
1137 /* force the host page as non writable (writes will have a
1138 page fault + mprotect overhead) */
1139 page_addr &= qemu_host_page_mask;
1140 prot = 0;
1141 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1142 addr += TARGET_PAGE_SIZE) {
1143
1144 p2 = page_find (addr >> TARGET_PAGE_BITS);
1145 if (!p2)
1146 continue;
1147 prot |= p2->flags;
1148 p2->flags &= ~PAGE_WRITE;
1149 page_get_flags(addr);
1150 }
1151 mprotect(g2h(page_addr), qemu_host_page_size,
1152 (prot & PAGE_BITS) & ~PAGE_WRITE);
1153#ifdef DEBUG_TB_INVALIDATE
1154 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1155 page_addr);
1156#endif
1157 }
1158#else
1159 /* if some code is already present, then the pages are already
1160 protected. So we handle the case where only the first TB is
1161 allocated in a physical page */
1162 if (!last_first_tb) {
1163 tlb_protect_code(page_addr);
1164 }
1165#endif
1166
1167#endif /* TARGET_HAS_SMC */
1168}
1169
1170/* Allocate a new translation block. Flush the translation buffer if
1171 too many translation blocks or too much generated code. */
1172TranslationBlock *tb_alloc(target_ulong pc)
1173{
1174 TranslationBlock *tb;
1175
1176 if (nb_tbs >= code_gen_max_blocks ||
1177 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1178 return NULL;
1179 tb = &tbs[nb_tbs++];
1180 tb->pc = pc;
1181 tb->cflags = 0;
1182 return tb;
1183}
1184
1185void tb_free(TranslationBlock *tb)
1186{
1187 /* In practice this is mostly used for single use temporary TB
1188 Ignore the hard cases and just back up if this TB happens to
1189 be the last one generated. */
1190 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1191 code_gen_ptr = tb->tc_ptr;
1192 nb_tbs--;
1193 }
1194}
1195
1196/* add a new TB and link it to the physical page tables. phys_page2 is
1197 (-1) to indicate that only one page contains the TB. */
1198void tb_link_phys(TranslationBlock *tb,
1199 target_ulong phys_pc, target_ulong phys_page2)
1200{
1201 unsigned int h;
1202 TranslationBlock **ptb;
1203
1204 /* Grab the mmap lock to stop another thread invalidating this TB
1205 before we are done. */
1206 mmap_lock();
1207 /* add in the physical hash table */
1208 h = tb_phys_hash_func(phys_pc);
1209 ptb = &tb_phys_hash[h];
1210 tb->phys_hash_next = *ptb;
1211 *ptb = tb;
1212
1213 /* add in the page list */
1214 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1215 if (phys_page2 != -1)
1216 tb_alloc_page(tb, 1, phys_page2);
1217 else
1218 tb->page_addr[1] = -1;
1219
1220 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1221 tb->jmp_next[0] = NULL;
1222 tb->jmp_next[1] = NULL;
1223
1224 /* init original jump addresses */
1225 if (tb->tb_next_offset[0] != 0xffff)
1226 tb_reset_jump(tb, 0);
1227 if (tb->tb_next_offset[1] != 0xffff)
1228 tb_reset_jump(tb, 1);
1229
1230#ifdef DEBUG_TB_CHECK
1231 tb_page_check();
1232#endif
1233 mmap_unlock();
1234}
1235
1236/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1237 tb[1].tc_ptr. Return NULL if not found */
1238TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1239{
1240 int m_min, m_max, m;
1241 unsigned long v;
1242 TranslationBlock *tb;
1243
1244 if (nb_tbs <= 0)
1245 return NULL;
1246 if (tc_ptr < (unsigned long)code_gen_buffer ||
1247 tc_ptr >= (unsigned long)code_gen_ptr)
1248 return NULL;
1249 /* binary search (cf Knuth) */
1250 m_min = 0;
1251 m_max = nb_tbs - 1;
1252 while (m_min <= m_max) {
1253 m = (m_min + m_max) >> 1;
1254 tb = &tbs[m];
1255 v = (unsigned long)tb->tc_ptr;
1256 if (v == tc_ptr)
1257 return tb;
1258 else if (tc_ptr < v) {
1259 m_max = m - 1;
1260 } else {
1261 m_min = m + 1;
1262 }
1263 }
1264 return &tbs[m_max];
1265}
1266
1267static void tb_reset_jump_recursive(TranslationBlock *tb);
1268
1269static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1270{
1271 TranslationBlock *tb1, *tb_next, **ptb;
1272 unsigned int n1;
1273
1274 tb1 = tb->jmp_next[n];
1275 if (tb1 != NULL) {
1276 /* find head of list */
1277 for(;;) {
1278 n1 = (long)tb1 & 3;
1279 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1280 if (n1 == 2)
1281 break;
1282 tb1 = tb1->jmp_next[n1];
1283 }
1284 /* we are now sure now that tb jumps to tb1 */
1285 tb_next = tb1;
1286
1287 /* remove tb from the jmp_first list */
1288 ptb = &tb_next->jmp_first;
1289 for(;;) {
1290 tb1 = *ptb;
1291 n1 = (long)tb1 & 3;
1292 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1293 if (n1 == n && tb1 == tb)
1294 break;
1295 ptb = &tb1->jmp_next[n1];
1296 }
1297 *ptb = tb->jmp_next[n];
1298 tb->jmp_next[n] = NULL;
1299
1300 /* suppress the jump to next tb in generated code */
1301 tb_reset_jump(tb, n);
1302
1303 /* suppress jumps in the tb on which we could have jumped */
1304 tb_reset_jump_recursive(tb_next);
1305 }
1306}
1307
1308static void tb_reset_jump_recursive(TranslationBlock *tb)
1309{
1310 tb_reset_jump_recursive2(tb, 0);
1311 tb_reset_jump_recursive2(tb, 1);
1312}
1313
1314#if defined(TARGET_HAS_ICE)
1315static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1316{
1317 target_phys_addr_t addr;
1318 target_ulong pd;
1319 ram_addr_t ram_addr;
1320 PhysPageDesc *p;
1321
1322 addr = cpu_get_phys_page_debug(env, pc);
1323 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1324 if (!p) {
1325 pd = IO_MEM_UNASSIGNED;
1326 } else {
1327 pd = p->phys_offset;
1328 }
1329 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1330 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1331}
1332#endif
1333
1334/* Add a watchpoint. */
1335int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1336 int flags, CPUWatchpoint **watchpoint)
1337{
1338 target_ulong len_mask = ~(len - 1);
1339 CPUWatchpoint *wp;
1340
1341 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1342 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1343 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1344 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1345 return -EINVAL;
1346 }
1347 wp = qemu_malloc(sizeof(*wp));
1348
1349 wp->vaddr = addr;
1350 wp->len_mask = len_mask;
1351 wp->flags = flags;
1352
1353 /* keep all GDB-injected watchpoints in front */
1354 if (flags & BP_GDB)
1355 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1356 else
1357 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1358
1359 tlb_flush_page(env, addr);
1360
1361 if (watchpoint)
1362 *watchpoint = wp;
1363 return 0;
1364}
1365
1366/* Remove a specific watchpoint. */
1367int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1368 int flags)
1369{
1370 target_ulong len_mask = ~(len - 1);
1371 CPUWatchpoint *wp;
1372
1373 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1374 if (addr == wp->vaddr && len_mask == wp->len_mask
1375 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1376 cpu_watchpoint_remove_by_ref(env, wp);
1377 return 0;
1378 }
1379 }
1380 return -ENOENT;
1381}
1382
1383/* Remove a specific watchpoint by reference. */
1384void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1385{
1386 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1387
1388 tlb_flush_page(env, watchpoint->vaddr);
1389
1390 qemu_free(watchpoint);
1391}
1392
1393/* Remove all matching watchpoints. */
1394void cpu_watchpoint_remove_all(CPUState *env, int mask)
1395{
1396 CPUWatchpoint *wp, *next;
1397
1398 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1399 if (wp->flags & mask)
1400 cpu_watchpoint_remove_by_ref(env, wp);
1401 }
1402}
1403
1404/* Add a breakpoint. */
1405int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1406 CPUBreakpoint **breakpoint)
1407{
1408#if defined(TARGET_HAS_ICE)
1409 CPUBreakpoint *bp;
1410
1411 bp = qemu_malloc(sizeof(*bp));
1412
1413 bp->pc = pc;
1414 bp->flags = flags;
1415
1416 /* keep all GDB-injected breakpoints in front */
1417 if (flags & BP_GDB)
1418 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1419 else
1420 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1421
1422 breakpoint_invalidate(env, pc);
1423
1424 if (breakpoint)
1425 *breakpoint = bp;
1426 return 0;
1427#else
1428 return -ENOSYS;
1429#endif
1430}
1431
1432/* Remove a specific breakpoint. */
1433int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1434{
1435#if defined(TARGET_HAS_ICE)
1436 CPUBreakpoint *bp;
1437
1438 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1439 if (bp->pc == pc && bp->flags == flags) {
1440 cpu_breakpoint_remove_by_ref(env, bp);
1441 return 0;
1442 }
1443 }
1444 return -ENOENT;
1445#else
1446 return -ENOSYS;
1447#endif
1448}
1449
1450/* Remove a specific breakpoint by reference. */
1451void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1452{
1453#if defined(TARGET_HAS_ICE)
1454 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1455
1456 breakpoint_invalidate(env, breakpoint->pc);
1457
1458 qemu_free(breakpoint);
1459#endif
1460}
1461
1462/* Remove all matching breakpoints. */
1463void cpu_breakpoint_remove_all(CPUState *env, int mask)
1464{
1465#if defined(TARGET_HAS_ICE)
1466 CPUBreakpoint *bp, *next;
1467
1468 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1469 if (bp->flags & mask)
1470 cpu_breakpoint_remove_by_ref(env, bp);
1471 }
1472#endif
1473}
1474
1475/* enable or disable single step mode. EXCP_DEBUG is returned by the
1476 CPU loop after each instruction */
1477void cpu_single_step(CPUState *env, int enabled)
1478{
1479#if defined(TARGET_HAS_ICE)
1480 if (env->singlestep_enabled != enabled) {
1481 env->singlestep_enabled = enabled;
1482 if (kvm_enabled())
1483 kvm_update_guest_debug(env, 0);
1484 else {
1485 /* must flush all the translated code to avoid inconsistencies */
1486 /* XXX: only flush what is necessary */
1487 tb_flush(env);
1488 }
1489 }
1490#endif
1491}
1492
1493/* enable or disable low levels log */
1494void cpu_set_log(int log_flags)
1495{
1496 loglevel = log_flags;
1497 if (loglevel && !logfile) {
1498 logfile = fopen(logfilename, log_append ? "a" : "w");
1499 if (!logfile) {
1500 perror(logfilename);
1501 _exit(1);
1502 }
1503#if !defined(CONFIG_SOFTMMU)
1504 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1505 {
1506 static char logfile_buf[4096];
1507 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1508 }
1509#elif !defined(_WIN32)
1510 /* Win32 doesn't support line-buffering and requires size >= 2 */
1511 setvbuf(logfile, NULL, _IOLBF, 0);
1512#endif
1513 log_append = 1;
1514 }
1515 if (!loglevel && logfile) {
1516 fclose(logfile);
1517 logfile = NULL;
1518 }
1519}
1520
1521void cpu_set_log_filename(const char *filename)
1522{
1523 logfilename = strdup(filename);
1524 if (logfile) {
1525 fclose(logfile);
1526 logfile = NULL;
1527 }
1528 cpu_set_log(loglevel);
1529}
1530
1531static void cpu_unlink_tb(CPUState *env)
1532{
1533 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1534 problem and hope the cpu will stop of its own accord. For userspace
1535 emulation this often isn't actually as bad as it sounds. Often
1536 signals are used primarily to interrupt blocking syscalls. */
1537 TranslationBlock *tb;
1538 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1539
1540 tb = env->current_tb;
1541 /* if the cpu is currently executing code, we must unlink it and
1542 all the potentially executing TB */
1543 if (tb) {
1544 spin_lock(&interrupt_lock);
1545 env->current_tb = NULL;
1546 tb_reset_jump_recursive(tb);
1547 spin_unlock(&interrupt_lock);
1548 }
1549}
1550
1551/* mask must never be zero, except for A20 change call */
1552void cpu_interrupt(CPUState *env, int mask)
1553{
1554 int old_mask;
1555
1556 old_mask = env->interrupt_request;
1557 env->interrupt_request |= mask;
1558
1559#ifndef CONFIG_USER_ONLY
1560 /*
1561 * If called from iothread context, wake the target cpu in
1562 * case its halted.
1563 */
1564 if (!qemu_cpu_self(env)) {
1565 qemu_cpu_kick(env);
1566 return;
1567 }
1568#endif
1569
1570 if (use_icount) {
1571 env->icount_decr.u16.high = 0xffff;
1572#ifndef CONFIG_USER_ONLY
1573 if (!can_do_io(env)
1574 && (mask & ~old_mask) != 0) {
1575 cpu_abort(env, "Raised interrupt while not in I/O function");
1576 }
1577#endif
1578 } else {
1579 cpu_unlink_tb(env);
1580 }
1581}
1582
1583void cpu_reset_interrupt(CPUState *env, int mask)
1584{
1585 env->interrupt_request &= ~mask;
1586}
1587
1588void cpu_exit(CPUState *env)
1589{
1590 env->exit_request = 1;
1591 cpu_unlink_tb(env);
1592}
1593
1594const CPULogItem cpu_log_items[] = {
1595 { CPU_LOG_TB_OUT_ASM, "out_asm",
1596 "show generated host assembly code for each compiled TB" },
1597 { CPU_LOG_TB_IN_ASM, "in_asm",
1598 "show target assembly code for each compiled TB" },
1599 { CPU_LOG_TB_OP, "op",
1600 "show micro ops for each compiled TB" },
1601 { CPU_LOG_TB_OP_OPT, "op_opt",
1602 "show micro ops "
1603#ifdef TARGET_I386
1604 "before eflags optimization and "
1605#endif
1606 "after liveness analysis" },
1607 { CPU_LOG_INT, "int",
1608 "show interrupts/exceptions in short format" },
1609 { CPU_LOG_EXEC, "exec",
1610 "show trace before each executed TB (lots of logs)" },
1611 { CPU_LOG_TB_CPU, "cpu",
1612 "show CPU state before block translation" },
1613#ifdef TARGET_I386
1614 { CPU_LOG_PCALL, "pcall",
1615 "show protected mode far calls/returns/exceptions" },
1616 { CPU_LOG_RESET, "cpu_reset",
1617 "show CPU state before CPU resets" },
1618#endif
1619#ifdef DEBUG_IOPORT
1620 { CPU_LOG_IOPORT, "ioport",
1621 "show all i/o ports accesses" },
1622#endif
1623 { 0, NULL, NULL },
1624};
1625
1626static int cmp1(const char *s1, int n, const char *s2)
1627{
1628 if (strlen(s2) != n)
1629 return 0;
1630 return memcmp(s1, s2, n) == 0;
1631}
1632
1633/* takes a comma separated list of log masks. Return 0 if error. */
1634int cpu_str_to_log_mask(const char *str)
1635{
1636 const CPULogItem *item;
1637 int mask;
1638 const char *p, *p1;
1639
1640 p = str;
1641 mask = 0;
1642 for(;;) {
1643 p1 = strchr(p, ',');
1644 if (!p1)
1645 p1 = p + strlen(p);
1646 if(cmp1(p,p1-p,"all")) {
1647 for(item = cpu_log_items; item->mask != 0; item++) {
1648 mask |= item->mask;
1649 }
1650 } else {
1651 for(item = cpu_log_items; item->mask != 0; item++) {
1652 if (cmp1(p, p1 - p, item->name))
1653 goto found;
1654 }
1655 return 0;
1656 }
1657 found:
1658 mask |= item->mask;
1659 if (*p1 != ',')
1660 break;
1661 p = p1 + 1;
1662 }
1663 return mask;
1664}
1665
1666void cpu_abort(CPUState *env, const char *fmt, ...)
1667{
1668 va_list ap;
1669 va_list ap2;
1670
1671 va_start(ap, fmt);
1672 va_copy(ap2, ap);
1673 fprintf(stderr, "qemu: fatal: ");
1674 vfprintf(stderr, fmt, ap);
1675 fprintf(stderr, "\n");
1676#ifdef TARGET_I386
1677 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1678#else
1679 cpu_dump_state(env, stderr, fprintf, 0);
1680#endif
1681 if (qemu_log_enabled()) {
1682 qemu_log("qemu: fatal: ");
1683 qemu_log_vprintf(fmt, ap2);
1684 qemu_log("\n");
1685#ifdef TARGET_I386
1686 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1687#else
1688 log_cpu_state(env, 0);
1689#endif
1690 qemu_log_flush();
1691 qemu_log_close();
1692 }
1693 va_end(ap2);
1694 va_end(ap);
1695 abort();
1696}
1697
1698CPUState *cpu_copy(CPUState *env)
1699{
1700 CPUState *new_env = cpu_init(env->cpu_model_str);
1701 CPUState *next_cpu = new_env->next_cpu;
1702 int cpu_index = new_env->cpu_index;
1703#if defined(TARGET_HAS_ICE)
1704 CPUBreakpoint *bp;
1705 CPUWatchpoint *wp;
1706#endif
1707
1708 memcpy(new_env, env, sizeof(CPUState));
1709
1710 /* Preserve chaining and index. */
1711 new_env->next_cpu = next_cpu;
1712 new_env->cpu_index = cpu_index;
1713
1714 /* Clone all break/watchpoints.
1715 Note: Once we support ptrace with hw-debug register access, make sure
1716 BP_CPU break/watchpoints are handled correctly on clone. */
1717 QTAILQ_INIT(&env->breakpoints);
1718 QTAILQ_INIT(&env->watchpoints);
1719#if defined(TARGET_HAS_ICE)
1720 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1721 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1722 }
1723 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1724 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1725 wp->flags, NULL);
1726 }
1727#endif
1728
1729 return new_env;
1730}
1731
1732#if !defined(CONFIG_USER_ONLY)
1733
1734static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1735{
1736 unsigned int i;
1737
1738 /* Discard jump cache entries for any tb which might potentially
1739 overlap the flushed page. */
1740 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1741 memset (&env->tb_jmp_cache[i], 0,
1742 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1743
1744 i = tb_jmp_cache_hash_page(addr);
1745 memset (&env->tb_jmp_cache[i], 0,
1746 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1747}
1748
1749static CPUTLBEntry s_cputlb_empty_entry = {
1750 .addr_read = -1,
1751 .addr_write = -1,
1752 .addr_code = -1,
1753 .addend = -1,
1754};
1755
1756/* NOTE: if flush_global is true, also flush global entries (not
1757 implemented yet) */
1758void tlb_flush(CPUState *env, int flush_global)
1759{
1760 int i;
1761
1762#if defined(DEBUG_TLB)
1763 printf("tlb_flush:\n");
1764#endif
1765 /* must reset current TB so that interrupts cannot modify the
1766 links while we are modifying them */
1767 env->current_tb = NULL;
1768
1769 for(i = 0; i < CPU_TLB_SIZE; i++) {
1770 int mmu_idx;
1771 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1772 env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
1773 }
1774 }
1775
1776 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1777
1778 tlb_flush_count++;
1779}
1780
1781static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1782{
1783 if (addr == (tlb_entry->addr_read &
1784 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1785 addr == (tlb_entry->addr_write &
1786 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1787 addr == (tlb_entry->addr_code &
1788 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1789 *tlb_entry = s_cputlb_empty_entry;
1790 }
1791}
1792
1793void tlb_flush_page(CPUState *env, target_ulong addr)
1794{
1795 int i;
1796 int mmu_idx;
1797
1798#if defined(DEBUG_TLB)
1799 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1800#endif
1801 /* must reset current TB so that interrupts cannot modify the
1802 links while we are modifying them */
1803 env->current_tb = NULL;
1804
1805 addr &= TARGET_PAGE_MASK;
1806 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1807 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
1808 tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
1809
1810 tlb_flush_jmp_cache(env, addr);
1811}
1812
1813/* update the TLBs so that writes to code in the virtual page 'addr'
1814 can be detected */
1815static void tlb_protect_code(ram_addr_t ram_addr)
1816{
1817 cpu_physical_memory_reset_dirty(ram_addr,
1818 ram_addr + TARGET_PAGE_SIZE,
1819 CODE_DIRTY_FLAG);
1820}
1821
1822/* update the TLB so that writes in physical page 'phys_addr' are no longer
1823 tested for self modifying code */
1824static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1825 target_ulong vaddr)
1826{
1827 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
1828}
1829
1830static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1831 unsigned long start, unsigned long length)
1832{
1833 unsigned long addr;
1834 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1835 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1836 if ((addr - start) < length) {
1837 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1838 }
1839 }
1840}
1841
1842/* Note: start and end must be within the same ram block. */
1843void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
1844 int dirty_flags)
1845{
1846 CPUState *env;
1847 unsigned long length, start1;
1848 int i, mask, len;
1849 uint8_t *p;
1850
1851 start &= TARGET_PAGE_MASK;
1852 end = TARGET_PAGE_ALIGN(end);
1853
1854 length = end - start;
1855 if (length == 0)
1856 return;
1857 len = length >> TARGET_PAGE_BITS;
1858 mask = ~dirty_flags;
1859 p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
1860 for(i = 0; i < len; i++)
1861 p[i] &= mask;
1862
1863 /* we modify the TLB cache so that the dirty bit will be set again
1864 when accessing the range */
1865 start1 = (unsigned long)qemu_get_ram_ptr(start);
1866 /* Chek that we don't span multiple blocks - this breaks the
1867 address comparisons below. */
1868 if ((unsigned long)qemu_get_ram_ptr(end - 1) - start1
1869 != (end - 1) - start) {
1870 abort();
1871 }
1872
1873 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1874 int mmu_idx;
1875 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1876 for(i = 0; i < CPU_TLB_SIZE; i++)
1877 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
1878 start1, length);
1879 }
1880 }
1881}
1882
1883int cpu_physical_memory_set_dirty_tracking(int enable)
1884{
1885 in_migration = enable;
1886 if (kvm_enabled()) {
1887 return kvm_set_migration_log(enable);
1888 }
1889 return 0;
1890}
1891
1892int cpu_physical_memory_get_dirty_tracking(void)
1893{
1894 return in_migration;
1895}
1896
1897int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
1898 target_phys_addr_t end_addr)
1899{
1900 int ret = 0;
1901
1902 if (kvm_enabled())
1903 ret = kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
1904 return ret;
1905}
1906
1907static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
1908{
1909 ram_addr_t ram_addr;
1910 void *p;
1911
1912 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1913 p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
1914 + tlb_entry->addend);
1915 ram_addr = qemu_ram_addr_from_host(p);
1916 if (!cpu_physical_memory_is_dirty(ram_addr)) {
1917 tlb_entry->addr_write |= TLB_NOTDIRTY;
1918 }
1919 }
1920}
1921
1922/* update the TLB according to the current state of the dirty bits */
1923void cpu_tlb_update_dirty(CPUState *env)
1924{
1925 int i;
1926 int mmu_idx;
1927 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1928 for(i = 0; i < CPU_TLB_SIZE; i++)
1929 tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
1930 }
1931}
1932
1933static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
1934{
1935 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
1936 tlb_entry->addr_write = vaddr;
1937}
1938
1939/* update the TLB corresponding to virtual page vaddr
1940 so that it is no longer dirty */
1941static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
1942{
1943 int i;
1944 int mmu_idx;
1945
1946 vaddr &= TARGET_PAGE_MASK;
1947 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1948 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
1949 tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
1950}
1951
1952/* add a new TLB entry. At most one entry for a given virtual address
1953 is permitted. Return 0 if OK or 2 if the page could not be mapped
1954 (can only happen in non SOFTMMU mode for I/O pages or pages
1955 conflicting with the host address space). */
1956int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
1957 target_phys_addr_t paddr, int prot,
1958 int mmu_idx, int is_softmmu)
1959{
1960 PhysPageDesc *p;
1961 unsigned long pd;
1962 unsigned int index;
1963 target_ulong address;
1964 target_ulong code_address;
1965 target_phys_addr_t addend;
1966 int ret;
1967 CPUTLBEntry *te;
1968 CPUWatchpoint *wp;
1969 target_phys_addr_t iotlb;
1970
1971 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
1972 if (!p) {
1973 pd = IO_MEM_UNASSIGNED;
1974 } else {
1975 pd = p->phys_offset;
1976 }
1977#if defined(DEBUG_TLB)
1978 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1979 vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
1980#endif
1981
1982 ret = 0;
1983 address = vaddr;
1984 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
1985 /* IO memory case (romd handled later) */
1986 address |= TLB_MMIO;
1987 }
1988 addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
1989 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
1990 /* Normal RAM. */
1991 iotlb = pd & TARGET_PAGE_MASK;
1992 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
1993 iotlb |= IO_MEM_NOTDIRTY;
1994 else
1995 iotlb |= IO_MEM_ROM;
1996 } else {
1997 /* IO handlers are currently passed a physical address.
1998 It would be nice to pass an offset from the base address
1999 of that region. This would avoid having to special case RAM,
2000 and avoid full address decoding in every device.
2001 We can't use the high bits of pd for this because
2002 IO_MEM_ROMD uses these as a ram address. */
2003 iotlb = (pd & ~TARGET_PAGE_MASK);
2004 if (p) {
2005 iotlb += p->region_offset;
2006 } else {
2007 iotlb += paddr;
2008 }
2009 }
2010
2011 code_address = address;
2012 /* Make accesses to pages with watchpoints go via the
2013 watchpoint trap routines. */
2014 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2015 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2016 iotlb = io_mem_watch + paddr;
2017 /* TODO: The memory case can be optimized by not trapping
2018 reads of pages with a write breakpoint. */
2019 address |= TLB_MMIO;
2020 }
2021 }
2022
2023 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2024 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2025 te = &env->tlb_table[mmu_idx][index];
2026 te->addend = addend - vaddr;
2027 if (prot & PAGE_READ) {
2028 te->addr_read = address;
2029 } else {
2030 te->addr_read = -1;
2031 }
2032
2033 if (prot & PAGE_EXEC) {
2034 te->addr_code = code_address;
2035 } else {
2036 te->addr_code = -1;
2037 }
2038 if (prot & PAGE_WRITE) {
2039 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2040 (pd & IO_MEM_ROMD)) {
2041 /* Write access calls the I/O callback. */
2042 te->addr_write = address | TLB_MMIO;
2043 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2044 !cpu_physical_memory_is_dirty(pd)) {
2045 te->addr_write = address | TLB_NOTDIRTY;
2046 } else {
2047 te->addr_write = address;
2048 }
2049 } else {
2050 te->addr_write = -1;
2051 }
2052 return ret;
2053}
2054
2055#else
2056
2057void tlb_flush(CPUState *env, int flush_global)
2058{
2059}
2060
2061void tlb_flush_page(CPUState *env, target_ulong addr)
2062{
2063}
2064
2065int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2066 target_phys_addr_t paddr, int prot,
2067 int mmu_idx, int is_softmmu)
2068{
2069 return 0;
2070}
2071
2072/*
2073 * Walks guest process memory "regions" one by one
2074 * and calls callback function 'fn' for each region.
2075 */
2076int walk_memory_regions(void *priv,
2077 int (*fn)(void *, unsigned long, unsigned long, unsigned long))
2078{
2079 unsigned long start, end;
2080 PageDesc *p = NULL;
2081 int i, j, prot, prot1;
2082 int rc = 0;
2083
2084 start = end = -1;
2085 prot = 0;
2086
2087 for (i = 0; i <= L1_SIZE; i++) {
2088 p = (i < L1_SIZE) ? l1_map[i] : NULL;
2089 for (j = 0; j < L2_SIZE; j++) {
2090 prot1 = (p == NULL) ? 0 : p[j].flags;
2091 /*
2092 * "region" is one continuous chunk of memory
2093 * that has same protection flags set.
2094 */
2095 if (prot1 != prot) {
2096 end = (i << (32 - L1_BITS)) | (j << TARGET_PAGE_BITS);
2097 if (start != -1) {
2098 rc = (*fn)(priv, start, end, prot);
2099 /* callback can stop iteration by returning != 0 */
2100 if (rc != 0)
2101 return (rc);
2102 }
2103 if (prot1 != 0)
2104 start = end;
2105 else
2106 start = -1;
2107 prot = prot1;
2108 }
2109 if (p == NULL)
2110 break;
2111 }
2112 }
2113 return (rc);
2114}
2115
2116static int dump_region(void *priv, unsigned long start,
2117 unsigned long end, unsigned long prot)
2118{
2119 FILE *f = (FILE *)priv;
2120
2121 (void) fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
2122 start, end, end - start,
2123 ((prot & PAGE_READ) ? 'r' : '-'),
2124 ((prot & PAGE_WRITE) ? 'w' : '-'),
2125 ((prot & PAGE_EXEC) ? 'x' : '-'));
2126
2127 return (0);
2128}
2129
2130/* dump memory mappings */
2131void page_dump(FILE *f)
2132{
2133 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2134 "start", "end", "size", "prot");
2135 walk_memory_regions(f, dump_region);
2136}
2137
2138int page_get_flags(target_ulong address)
2139{
2140 PageDesc *p;
2141
2142 p = page_find(address >> TARGET_PAGE_BITS);
2143 if (!p)
2144 return 0;
2145 return p->flags;
2146}
2147
2148/* modify the flags of a page and invalidate the code if
2149 necessary. The flag PAGE_WRITE_ORG is positioned automatically
2150 depending on PAGE_WRITE */
2151void page_set_flags(target_ulong start, target_ulong end, int flags)
2152{
2153 PageDesc *p;
2154 target_ulong addr;
2155
2156 /* mmap_lock should already be held. */
2157 start = start & TARGET_PAGE_MASK;
2158 end = TARGET_PAGE_ALIGN(end);
2159 if (flags & PAGE_WRITE)
2160 flags |= PAGE_WRITE_ORG;
2161 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2162 p = page_find_alloc(addr >> TARGET_PAGE_BITS);
2163 /* We may be called for host regions that are outside guest
2164 address space. */
2165 if (!p)
2166 return;
2167 /* if the write protection is set, then we invalidate the code
2168 inside */
2169 if (!(p->flags & PAGE_WRITE) &&
2170 (flags & PAGE_WRITE) &&
2171 p->first_tb) {
2172 tb_invalidate_phys_page(addr, 0, NULL);
2173 }
2174 p->flags = flags;
2175 }
2176}
2177
2178int page_check_range(target_ulong start, target_ulong len, int flags)
2179{
2180 PageDesc *p;
2181 target_ulong end;
2182 target_ulong addr;
2183
2184 if (start + len < start)
2185 /* we've wrapped around */
2186 return -1;
2187
2188 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2189 start = start & TARGET_PAGE_MASK;
2190
2191 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2192 p = page_find(addr >> TARGET_PAGE_BITS);
2193 if( !p )
2194 return -1;
2195 if( !(p->flags & PAGE_VALID) )
2196 return -1;
2197
2198 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2199 return -1;
2200 if (flags & PAGE_WRITE) {
2201 if (!(p->flags & PAGE_WRITE_ORG))
2202 return -1;
2203 /* unprotect the page if it was put read-only because it
2204 contains translated code */
2205 if (!(p->flags & PAGE_WRITE)) {
2206 if (!page_unprotect(addr, 0, NULL))
2207 return -1;
2208 }
2209 return 0;
2210 }
2211 }
2212 return 0;
2213}
2214
2215/* called from signal handler: invalidate the code and unprotect the
2216 page. Return TRUE if the fault was successfully handled. */
2217int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2218{
2219 unsigned int page_index, prot, pindex;
2220 PageDesc *p, *p1;
2221 target_ulong host_start, host_end, addr;
2222
2223 /* Technically this isn't safe inside a signal handler. However we
2224 know this only ever happens in a synchronous SEGV handler, so in
2225 practice it seems to be ok. */
2226 mmap_lock();
2227
2228 host_start = address & qemu_host_page_mask;
2229 page_index = host_start >> TARGET_PAGE_BITS;
2230 p1 = page_find(page_index);
2231 if (!p1) {
2232 mmap_unlock();
2233 return 0;
2234 }
2235 host_end = host_start + qemu_host_page_size;
2236 p = p1;
2237 prot = 0;
2238 for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2239 prot |= p->flags;
2240 p++;
2241 }
2242 /* if the page was really writable, then we change its
2243 protection back to writable */
2244 if (prot & PAGE_WRITE_ORG) {
2245 pindex = (address - host_start) >> TARGET_PAGE_BITS;
2246 if (!(p1[pindex].flags & PAGE_WRITE)) {
2247 mprotect((void *)g2h(host_start), qemu_host_page_size,
2248 (prot & PAGE_BITS) | PAGE_WRITE);
2249 p1[pindex].flags |= PAGE_WRITE;
2250 /* and since the content will be modified, we must invalidate
2251 the corresponding translated code. */
2252 tb_invalidate_phys_page(address, pc, puc);
2253#ifdef DEBUG_TB_CHECK
2254 tb_invalidate_check(address);
2255#endif
2256 mmap_unlock();
2257 return 1;
2258 }
2259 }
2260 mmap_unlock();
2261 return 0;
2262}
2263
2264static inline void tlb_set_dirty(CPUState *env,
2265 unsigned long addr, target_ulong vaddr)
2266{
2267}
2268#endif /* defined(CONFIG_USER_ONLY) */
2269
2270#if !defined(CONFIG_USER_ONLY)
2271
2272static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2273 ram_addr_t memory, ram_addr_t region_offset);
2274static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2275 ram_addr_t orig_memory, ram_addr_t region_offset);
2276#define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2277 need_subpage) \
2278 do { \
2279 if (addr > start_addr) \
2280 start_addr2 = 0; \
2281 else { \
2282 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2283 if (start_addr2 > 0) \
2284 need_subpage = 1; \
2285 } \
2286 \
2287 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2288 end_addr2 = TARGET_PAGE_SIZE - 1; \
2289 else { \
2290 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2291 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2292 need_subpage = 1; \
2293 } \
2294 } while (0)
2295
2296/* register physical memory.
2297 For RAM, 'size' must be a multiple of the target page size.
2298 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2299 io memory page. The address used when calling the IO function is
2300 the offset from the start of the region, plus region_offset. Both
2301 start_addr and region_offset are rounded down to a page boundary
2302 before calculating this offset. This should not be a problem unless
2303 the low bits of start_addr and region_offset differ. */
2304void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2305 ram_addr_t size,
2306 ram_addr_t phys_offset,
2307 ram_addr_t region_offset)
2308{
2309 target_phys_addr_t addr, end_addr;
2310 PhysPageDesc *p;
2311 CPUState *env;
2312 ram_addr_t orig_size = size;
2313 void *subpage;
2314
2315 if (kvm_enabled())
2316 kvm_set_phys_mem(start_addr, size, phys_offset);
2317
2318 if (phys_offset == IO_MEM_UNASSIGNED) {
2319 region_offset = start_addr;
2320 }
2321 region_offset &= TARGET_PAGE_MASK;
2322 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2323 end_addr = start_addr + (target_phys_addr_t)size;
2324 for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2325 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2326 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2327 ram_addr_t orig_memory = p->phys_offset;
2328 target_phys_addr_t start_addr2, end_addr2;
2329 int need_subpage = 0;
2330
2331 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2332 need_subpage);
2333 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2334 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2335 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2336 &p->phys_offset, orig_memory,
2337 p->region_offset);
2338 } else {
2339 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2340 >> IO_MEM_SHIFT];
2341 }
2342 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2343 region_offset);
2344 p->region_offset = 0;
2345 } else {
2346 p->phys_offset = phys_offset;
2347 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2348 (phys_offset & IO_MEM_ROMD))
2349 phys_offset += TARGET_PAGE_SIZE;
2350 }
2351 } else {
2352 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2353 p->phys_offset = phys_offset;
2354 p->region_offset = region_offset;
2355 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2356 (phys_offset & IO_MEM_ROMD)) {
2357 phys_offset += TARGET_PAGE_SIZE;
2358 } else {
2359 target_phys_addr_t start_addr2, end_addr2;
2360 int need_subpage = 0;
2361
2362 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2363 end_addr2, need_subpage);
2364
2365 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2366 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2367 &p->phys_offset, IO_MEM_UNASSIGNED,
2368 addr & TARGET_PAGE_MASK);
2369 subpage_register(subpage, start_addr2, end_addr2,
2370 phys_offset, region_offset);
2371 p->region_offset = 0;
2372 }
2373 }
2374 }
2375 region_offset += TARGET_PAGE_SIZE;
2376 }
2377
2378 /* since each CPU stores ram addresses in its TLB cache, we must
2379 reset the modified entries */
2380 /* XXX: slow ! */
2381 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2382 tlb_flush(env, 1);
2383 }
2384}
2385
2386/* XXX: temporary until new memory mapping API */
2387ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2388{
2389 PhysPageDesc *p;
2390
2391 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2392 if (!p)
2393 return IO_MEM_UNASSIGNED;
2394 return p->phys_offset;
2395}
2396
2397void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2398{
2399 if (kvm_enabled())
2400 kvm_coalesce_mmio_region(addr, size);
2401}
2402
2403void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2404{
2405 if (kvm_enabled())
2406 kvm_uncoalesce_mmio_region(addr, size);
2407}
2408
2409ram_addr_t qemu_ram_alloc(ram_addr_t size)
2410{
2411 RAMBlock *new_block;
2412
2413 size = TARGET_PAGE_ALIGN(size);
2414 new_block = qemu_malloc(sizeof(*new_block));
2415
2416#if defined(TARGET_S390X) && defined(CONFIG_KVM)
2417 /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
2418 new_block->host = mmap((void*)0x1000000, size, PROT_EXEC|PROT_READ|PROT_WRITE,
2419 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
2420#else
2421 new_block->host = qemu_vmalloc(size);
2422#endif
2423#ifdef MADV_MERGEABLE
2424 madvise(new_block->host, size, MADV_MERGEABLE);
2425#endif
2426 new_block->offset = last_ram_offset;
2427 new_block->length = size;
2428
2429 new_block->next = ram_blocks;
2430 ram_blocks = new_block;
2431
2432 phys_ram_dirty = qemu_realloc(phys_ram_dirty,
2433 (last_ram_offset + size) >> TARGET_PAGE_BITS);
2434 memset(phys_ram_dirty + (last_ram_offset >> TARGET_PAGE_BITS),
2435 0xff, size >> TARGET_PAGE_BITS);
2436
2437 last_ram_offset += size;
2438
2439 if (kvm_enabled())
2440 kvm_setup_guest_memory(new_block->host, size);
2441
2442 return new_block->offset;
2443}
2444
2445void qemu_ram_free(ram_addr_t addr)
2446{
2447 /* TODO: implement this. */
2448}
2449
2450/* Return a host pointer to ram allocated with qemu_ram_alloc.
2451 With the exception of the softmmu code in this file, this should
2452 only be used for local memory (e.g. video ram) that the device owns,
2453 and knows it isn't going to access beyond the end of the block.
2454
2455 It should not be used for general purpose DMA.
2456 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2457 */
2458void *qemu_get_ram_ptr(ram_addr_t addr)
2459{
2460 RAMBlock *prev;
2461 RAMBlock **prevp;
2462 RAMBlock *block;
2463
2464 prev = NULL;
2465 prevp = &ram_blocks;
2466 block = ram_blocks;
2467 while (block && (block->offset > addr
2468 || block->offset + block->length <= addr)) {
2469 if (prev)
2470 prevp = &prev->next;
2471 prev = block;
2472 block = block->next;
2473 }
2474 if (!block) {
2475 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
2476 abort();
2477 }
2478 /* Move this entry to to start of the list. */
2479 if (prev) {
2480 prev->next = block->next;
2481 block->next = *prevp;
2482 *prevp = block;
2483 }
2484 return block->host + (addr - block->offset);
2485}
2486
2487/* Some of the softmmu routines need to translate from a host pointer
2488 (typically a TLB entry) back to a ram offset. */
2489ram_addr_t qemu_ram_addr_from_host(void *ptr)
2490{
2491 RAMBlock *prev;
2492 RAMBlock *block;
2493 uint8_t *host = ptr;
2494
2495 prev = NULL;
2496 block = ram_blocks;
2497 while (block && (block->host > host
2498 || block->host + block->length <= host)) {
2499 prev = block;
2500 block = block->next;
2501 }
2502 if (!block) {
2503 fprintf(stderr, "Bad ram pointer %p\n", ptr);
2504 abort();
2505 }
2506 return block->offset + (host - block->host);
2507}
2508
2509static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2510{
2511#ifdef DEBUG_UNASSIGNED
2512 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2513#endif
2514#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2515 do_unassigned_access(addr, 0, 0, 0, 1);
2516#endif
2517 return 0;
2518}
2519
2520static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2521{
2522#ifdef DEBUG_UNASSIGNED
2523 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2524#endif
2525#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2526 do_unassigned_access(addr, 0, 0, 0, 2);
2527#endif
2528 return 0;
2529}
2530
2531static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2532{
2533#ifdef DEBUG_UNASSIGNED
2534 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2535#endif
2536#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2537 do_unassigned_access(addr, 0, 0, 0, 4);
2538#endif
2539 return 0;
2540}
2541
2542static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2543{
2544#ifdef DEBUG_UNASSIGNED
2545 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2546#endif
2547#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2548 do_unassigned_access(addr, 1, 0, 0, 1);
2549#endif
2550}
2551
2552static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2553{
2554#ifdef DEBUG_UNASSIGNED
2555 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2556#endif
2557#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2558 do_unassigned_access(addr, 1, 0, 0, 2);
2559#endif
2560}
2561
2562static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2563{
2564#ifdef DEBUG_UNASSIGNED
2565 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2566#endif
2567#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2568 do_unassigned_access(addr, 1, 0, 0, 4);
2569#endif
2570}
2571
2572static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
2573 unassigned_mem_readb,
2574 unassigned_mem_readw,
2575 unassigned_mem_readl,
2576};
2577
2578static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
2579 unassigned_mem_writeb,
2580 unassigned_mem_writew,
2581 unassigned_mem_writel,
2582};
2583
2584static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2585 uint32_t val)
2586{
2587 int dirty_flags;
2588 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2589 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2590#if !defined(CONFIG_USER_ONLY)
2591 tb_invalidate_phys_page_fast(ram_addr, 1);
2592 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2593#endif
2594 }
2595 stb_p(qemu_get_ram_ptr(ram_addr), val);
2596 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2597 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2598 /* we remove the notdirty callback only if the code has been
2599 flushed */
2600 if (dirty_flags == 0xff)
2601 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2602}
2603
2604static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2605 uint32_t val)
2606{
2607 int dirty_flags;
2608 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2609 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2610#if !defined(CONFIG_USER_ONLY)
2611 tb_invalidate_phys_page_fast(ram_addr, 2);
2612 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2613#endif
2614 }
2615 stw_p(qemu_get_ram_ptr(ram_addr), val);
2616 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2617 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2618 /* we remove the notdirty callback only if the code has been
2619 flushed */
2620 if (dirty_flags == 0xff)
2621 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2622}
2623
2624static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
2625 uint32_t val)
2626{
2627 int dirty_flags;
2628 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2629 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2630#if !defined(CONFIG_USER_ONLY)
2631 tb_invalidate_phys_page_fast(ram_addr, 4);
2632 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2633#endif
2634 }
2635 stl_p(qemu_get_ram_ptr(ram_addr), val);
2636 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2637 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2638 /* we remove the notdirty callback only if the code has been
2639 flushed */
2640 if (dirty_flags == 0xff)
2641 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2642}
2643
2644static CPUReadMemoryFunc * const error_mem_read[3] = {
2645 NULL, /* never used */
2646 NULL, /* never used */
2647 NULL, /* never used */
2648};
2649
2650static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
2651 notdirty_mem_writeb,
2652 notdirty_mem_writew,
2653 notdirty_mem_writel,
2654};
2655
2656/* Generate a debug exception if a watchpoint has been hit. */
2657static void check_watchpoint(int offset, int len_mask, int flags)
2658{
2659 CPUState *env = cpu_single_env;
2660 target_ulong pc, cs_base;
2661 TranslationBlock *tb;
2662 target_ulong vaddr;
2663 CPUWatchpoint *wp;
2664 int cpu_flags;
2665
2666 if (env->watchpoint_hit) {
2667 /* We re-entered the check after replacing the TB. Now raise
2668 * the debug interrupt so that is will trigger after the
2669 * current instruction. */
2670 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
2671 return;
2672 }
2673 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2674 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2675 if ((vaddr == (wp->vaddr & len_mask) ||
2676 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
2677 wp->flags |= BP_WATCHPOINT_HIT;
2678 if (!env->watchpoint_hit) {
2679 env->watchpoint_hit = wp;
2680 tb = tb_find_pc(env->mem_io_pc);
2681 if (!tb) {
2682 cpu_abort(env, "check_watchpoint: could not find TB for "
2683 "pc=%p", (void *)env->mem_io_pc);
2684 }
2685 cpu_restore_state(tb, env, env->mem_io_pc, NULL);
2686 tb_phys_invalidate(tb, -1);
2687 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2688 env->exception_index = EXCP_DEBUG;
2689 } else {
2690 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2691 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
2692 }
2693 cpu_resume_from_signal(env, NULL);
2694 }
2695 } else {
2696 wp->flags &= ~BP_WATCHPOINT_HIT;
2697 }
2698 }
2699}
2700
2701/* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2702 so these check for a hit then pass through to the normal out-of-line
2703 phys routines. */
2704static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
2705{
2706 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
2707 return ldub_phys(addr);
2708}
2709
2710static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
2711{
2712 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
2713 return lduw_phys(addr);
2714}
2715
2716static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
2717{
2718 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
2719 return ldl_phys(addr);
2720}
2721
2722static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
2723 uint32_t val)
2724{
2725 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
2726 stb_phys(addr, val);
2727}
2728
2729static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
2730 uint32_t val)
2731{
2732 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
2733 stw_phys(addr, val);
2734}
2735
2736static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
2737 uint32_t val)
2738{
2739 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
2740 stl_phys(addr, val);
2741}
2742
2743static CPUReadMemoryFunc * const watch_mem_read[3] = {
2744 watch_mem_readb,
2745 watch_mem_readw,
2746 watch_mem_readl,
2747};
2748
2749static CPUWriteMemoryFunc * const watch_mem_write[3] = {
2750 watch_mem_writeb,
2751 watch_mem_writew,
2752 watch_mem_writel,
2753};
2754
2755static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
2756 unsigned int len)
2757{
2758 uint32_t ret;
2759 unsigned int idx;
2760
2761 idx = SUBPAGE_IDX(addr);
2762#if defined(DEBUG_SUBPAGE)
2763 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
2764 mmio, len, addr, idx);
2765#endif
2766 ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
2767 addr + mmio->region_offset[idx][0][len]);
2768
2769 return ret;
2770}
2771
2772static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
2773 uint32_t value, unsigned int len)
2774{
2775 unsigned int idx;
2776
2777 idx = SUBPAGE_IDX(addr);
2778#if defined(DEBUG_SUBPAGE)
2779 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
2780 mmio, len, addr, idx, value);
2781#endif
2782 (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
2783 addr + mmio->region_offset[idx][1][len],
2784 value);
2785}
2786
2787static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
2788{
2789#if defined(DEBUG_SUBPAGE)
2790 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2791#endif
2792
2793 return subpage_readlen(opaque, addr, 0);
2794}
2795
2796static void subpage_writeb (void *opaque, target_phys_addr_t addr,
2797 uint32_t value)
2798{
2799#if defined(DEBUG_SUBPAGE)
2800 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2801#endif
2802 subpage_writelen(opaque, addr, value, 0);
2803}
2804
2805static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
2806{
2807#if defined(DEBUG_SUBPAGE)
2808 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2809#endif
2810
2811 return subpage_readlen(opaque, addr, 1);
2812}
2813
2814static void subpage_writew (void *opaque, target_phys_addr_t addr,
2815 uint32_t value)
2816{
2817#if defined(DEBUG_SUBPAGE)
2818 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2819#endif
2820 subpage_writelen(opaque, addr, value, 1);
2821}
2822
2823static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
2824{
2825#if defined(DEBUG_SUBPAGE)
2826 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2827#endif
2828
2829 return subpage_readlen(opaque, addr, 2);
2830}
2831
2832static void subpage_writel (void *opaque,
2833 target_phys_addr_t addr, uint32_t value)
2834{
2835#if defined(DEBUG_SUBPAGE)
2836 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2837#endif
2838 subpage_writelen(opaque, addr, value, 2);
2839}
2840
2841static CPUReadMemoryFunc * const subpage_read[] = {
2842 &subpage_readb,
2843 &subpage_readw,
2844 &subpage_readl,
2845};
2846
2847static CPUWriteMemoryFunc * const subpage_write[] = {
2848 &subpage_writeb,
2849 &subpage_writew,
2850 &subpage_writel,
2851};
2852
2853static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2854 ram_addr_t memory, ram_addr_t region_offset)
2855{
2856 int idx, eidx;
2857 unsigned int i;
2858
2859 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2860 return -1;
2861 idx = SUBPAGE_IDX(start);
2862 eidx = SUBPAGE_IDX(end);
2863#if defined(DEBUG_SUBPAGE)
2864 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
2865 mmio, start, end, idx, eidx, memory);
2866#endif
2867 memory >>= IO_MEM_SHIFT;
2868 for (; idx <= eidx; idx++) {
2869 for (i = 0; i < 4; i++) {
2870 if (io_mem_read[memory][i]) {
2871 mmio->mem_read[idx][i] = &io_mem_read[memory][i];
2872 mmio->opaque[idx][0][i] = io_mem_opaque[memory];
2873 mmio->region_offset[idx][0][i] = region_offset;
2874 }
2875 if (io_mem_write[memory][i]) {
2876 mmio->mem_write[idx][i] = &io_mem_write[memory][i];
2877 mmio->opaque[idx][1][i] = io_mem_opaque[memory];
2878 mmio->region_offset[idx][1][i] = region_offset;
2879 }
2880 }
2881 }
2882
2883 return 0;
2884}
2885
2886static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2887 ram_addr_t orig_memory, ram_addr_t region_offset)
2888{
2889 subpage_t *mmio;
2890 int subpage_memory;
2891
2892 mmio = qemu_mallocz(sizeof(subpage_t));
2893
2894 mmio->base = base;
2895 subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio);
2896#if defined(DEBUG_SUBPAGE)
2897 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
2898 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
2899#endif
2900 *phys = subpage_memory | IO_MEM_SUBPAGE;
2901 subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
2902 region_offset);
2903
2904 return mmio;
2905}
2906
2907static int get_free_io_mem_idx(void)
2908{
2909 int i;
2910
2911 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
2912 if (!io_mem_used[i]) {
2913 io_mem_used[i] = 1;
2914 return i;
2915 }
2916 fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
2917 return -1;
2918}
2919
2920/* mem_read and mem_write are arrays of functions containing the
2921 function to access byte (index 0), word (index 1) and dword (index
2922 2). Functions can be omitted with a NULL function pointer.
2923 If io_index is non zero, the corresponding io zone is
2924 modified. If it is zero, a new io zone is allocated. The return
2925 value can be used with cpu_register_physical_memory(). (-1) is
2926 returned if error. */
2927static int cpu_register_io_memory_fixed(int io_index,
2928 CPUReadMemoryFunc * const *mem_read,
2929 CPUWriteMemoryFunc * const *mem_write,
2930 void *opaque)
2931{
2932 int i, subwidth = 0;
2933
2934 if (io_index <= 0) {
2935 io_index = get_free_io_mem_idx();
2936 if (io_index == -1)
2937 return io_index;
2938 } else {
2939 io_index >>= IO_MEM_SHIFT;
2940 if (io_index >= IO_MEM_NB_ENTRIES)
2941 return -1;
2942 }
2943
2944 for(i = 0;i < 3; i++) {
2945 if (!mem_read[i] || !mem_write[i])
2946 subwidth = IO_MEM_SUBWIDTH;
2947 io_mem_read[io_index][i] = mem_read[i];
2948 io_mem_write[io_index][i] = mem_write[i];
2949 }
2950 io_mem_opaque[io_index] = opaque;
2951 return (io_index << IO_MEM_SHIFT) | subwidth;
2952}
2953
2954int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
2955 CPUWriteMemoryFunc * const *mem_write,
2956 void *opaque)
2957{
2958 return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque);
2959}
2960
2961void cpu_unregister_io_memory(int io_table_address)
2962{
2963 int i;
2964 int io_index = io_table_address >> IO_MEM_SHIFT;
2965
2966 for (i=0;i < 3; i++) {
2967 io_mem_read[io_index][i] = unassigned_mem_read[i];
2968 io_mem_write[io_index][i] = unassigned_mem_write[i];
2969 }
2970 io_mem_opaque[io_index] = NULL;
2971 io_mem_used[io_index] = 0;
2972}
2973
2974static void io_mem_init(void)
2975{
2976 int i;
2977
2978 cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read, unassigned_mem_write, NULL);
2979 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read, unassigned_mem_write, NULL);
2980 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read, notdirty_mem_write, NULL);
2981 for (i=0; i<5; i++)
2982 io_mem_used[i] = 1;
2983
2984 io_mem_watch = cpu_register_io_memory(watch_mem_read,
2985 watch_mem_write, NULL);
2986}
2987
2988#endif /* !defined(CONFIG_USER_ONLY) */
2989
2990/* physical memory access (slow version, mainly for debug) */
2991#if defined(CONFIG_USER_ONLY)
2992void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2993 int len, int is_write)
2994{
2995 int l, flags;
2996 target_ulong page;
2997 void * p;
2998
2999 while (len > 0) {
3000 page = addr & TARGET_PAGE_MASK;
3001 l = (page + TARGET_PAGE_SIZE) - addr;
3002 if (l > len)
3003 l = len;
3004 flags = page_get_flags(page);
3005 if (!(flags & PAGE_VALID))
3006 return;
3007 if (is_write) {
3008 if (!(flags & PAGE_WRITE))
3009 return;
3010 /* XXX: this code should not depend on lock_user */
3011 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
3012 /* FIXME - should this return an error rather than just fail? */
3013 return;
3014 memcpy(p, buf, l);
3015 unlock_user(p, addr, l);
3016 } else {
3017 if (!(flags & PAGE_READ))
3018 return;
3019 /* XXX: this code should not depend on lock_user */
3020 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
3021 /* FIXME - should this return an error rather than just fail? */
3022 return;
3023 memcpy(buf, p, l);
3024 unlock_user(p, addr, 0);
3025 }
3026 len -= l;
3027 buf += l;
3028 addr += l;
3029 }
3030}
3031
3032#else
3033void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3034 int len, int is_write)
3035{
3036 int l, io_index;
3037 uint8_t *ptr;
3038 uint32_t val;
3039 target_phys_addr_t page;
3040 unsigned long pd;
3041 PhysPageDesc *p;
3042
3043 while (len > 0) {
3044 page = addr & TARGET_PAGE_MASK;
3045 l = (page + TARGET_PAGE_SIZE) - addr;
3046 if (l > len)
3047 l = len;
3048 p = phys_page_find(page >> TARGET_PAGE_BITS);
3049 if (!p) {
3050 pd = IO_MEM_UNASSIGNED;
3051 } else {
3052 pd = p->phys_offset;
3053 }
3054
3055 if (is_write) {
3056 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3057 target_phys_addr_t addr1 = addr;
3058 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3059 if (p)
3060 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3061 /* XXX: could force cpu_single_env to NULL to avoid
3062 potential bugs */
3063 if (l >= 4 && ((addr1 & 3) == 0)) {
3064 /* 32 bit write access */
3065 val = ldl_p(buf);
3066 io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3067 l = 4;
3068 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3069 /* 16 bit write access */
3070 val = lduw_p(buf);
3071 io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3072 l = 2;
3073 } else {
3074 /* 8 bit write access */
3075 val = ldub_p(buf);
3076 io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3077 l = 1;
3078 }
3079 } else {
3080 unsigned long addr1;
3081 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3082 /* RAM case */
3083 ptr = qemu_get_ram_ptr(addr1);
3084 memcpy(ptr, buf, l);
3085 if (!cpu_physical_memory_is_dirty(addr1)) {
3086 /* invalidate code */
3087 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3088 /* set dirty bit */
3089 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3090 (0xff & ~CODE_DIRTY_FLAG);
3091 }
3092 }
3093 } else {
3094 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3095 !(pd & IO_MEM_ROMD)) {
3096 target_phys_addr_t addr1 = addr;
3097 /* I/O case */
3098 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3099 if (p)
3100 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3101 if (l >= 4 && ((addr1 & 3) == 0)) {
3102 /* 32 bit read access */
3103 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3104 stl_p(buf, val);
3105 l = 4;
3106 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3107 /* 16 bit read access */
3108 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3109 stw_p(buf, val);
3110 l = 2;
3111 } else {
3112 /* 8 bit read access */
3113 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3114 stb_p(buf, val);
3115 l = 1;
3116 }
3117 } else {
3118 /* RAM case */
3119 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3120 (addr & ~TARGET_PAGE_MASK);
3121 memcpy(buf, ptr, l);
3122 }
3123 }
3124 len -= l;
3125 buf += l;
3126 addr += l;
3127 }
3128}
3129
3130/* used for ROM loading : can write in RAM and ROM */
3131void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3132 const uint8_t *buf, int len)
3133{
3134 int l;
3135 uint8_t *ptr;
3136 target_phys_addr_t page;
3137 unsigned long pd;
3138 PhysPageDesc *p;
3139
3140 while (len > 0) {
3141 page = addr & TARGET_PAGE_MASK;
3142 l = (page + TARGET_PAGE_SIZE) - addr;
3143 if (l > len)
3144 l = len;
3145 p = phys_page_find(page >> TARGET_PAGE_BITS);
3146 if (!p) {
3147 pd = IO_MEM_UNASSIGNED;
3148 } else {
3149 pd = p->phys_offset;
3150 }
3151
3152 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3153 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3154 !(pd & IO_MEM_ROMD)) {
3155 /* do nothing */
3156 } else {
3157 unsigned long addr1;
3158 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3159 /* ROM/RAM case */
3160 ptr = qemu_get_ram_ptr(addr1);
3161 memcpy(ptr, buf, l);
3162 }
3163 len -= l;
3164 buf += l;
3165 addr += l;
3166 }
3167}
3168
3169typedef struct {
3170 void *buffer;
3171 target_phys_addr_t addr;
3172 target_phys_addr_t len;
3173} BounceBuffer;
3174
3175static BounceBuffer bounce;
3176
3177typedef struct MapClient {
3178 void *opaque;
3179 void (*callback)(void *opaque);
3180 QLIST_ENTRY(MapClient) link;
3181} MapClient;
3182
3183static QLIST_HEAD(map_client_list, MapClient) map_client_list
3184 = QLIST_HEAD_INITIALIZER(map_client_list);
3185
3186void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3187{
3188 MapClient *client = qemu_malloc(sizeof(*client));
3189
3190 client->opaque = opaque;
3191 client->callback = callback;
3192 QLIST_INSERT_HEAD(&map_client_list, client, link);
3193 return client;
3194}
3195
3196void cpu_unregister_map_client(void *_client)
3197{
3198 MapClient *client = (MapClient *)_client;
3199
3200 QLIST_REMOVE(client, link);
3201 qemu_free(client);
3202}
3203
3204static void cpu_notify_map_clients(void)
3205{
3206 MapClient *client;
3207
3208 while (!QLIST_EMPTY(&map_client_list)) {
3209 client = QLIST_FIRST(&map_client_list);
3210 client->callback(client->opaque);
3211 cpu_unregister_map_client(client);
3212 }
3213}
3214
3215/* Map a physical memory region into a host virtual address.
3216 * May map a subset of the requested range, given by and returned in *plen.
3217 * May return NULL if resources needed to perform the mapping are exhausted.
3218 * Use only for reads OR writes - not for read-modify-write operations.
3219 * Use cpu_register_map_client() to know when retrying the map operation is
3220 * likely to succeed.
3221 */
3222void *cpu_physical_memory_map(target_phys_addr_t addr,
3223 target_phys_addr_t *plen,
3224 int is_write)
3225{
3226 target_phys_addr_t len = *plen;
3227 target_phys_addr_t done = 0;
3228 int l;
3229 uint8_t *ret = NULL;
3230 uint8_t *ptr;
3231 target_phys_addr_t page;
3232 unsigned long pd;
3233 PhysPageDesc *p;
3234 unsigned long addr1;
3235
3236 while (len > 0) {
3237 page = addr & TARGET_PAGE_MASK;
3238 l = (page + TARGET_PAGE_SIZE) - addr;
3239 if (l > len)
3240 l = len;
3241 p = phys_page_find(page >> TARGET_PAGE_BITS);
3242 if (!p) {
3243 pd = IO_MEM_UNASSIGNED;
3244 } else {
3245 pd = p->phys_offset;
3246 }
3247
3248 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3249 if (done || bounce.buffer) {
3250 break;
3251 }
3252 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3253 bounce.addr = addr;
3254 bounce.len = l;
3255 if (!is_write) {
3256 cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
3257 }
3258 ptr = bounce.buffer;
3259 } else {
3260 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3261 ptr = qemu_get_ram_ptr(addr1);
3262 }
3263 if (!done) {
3264 ret = ptr;
3265 } else if (ret + done != ptr) {
3266 break;
3267 }
3268
3269 len -= l;
3270 addr += l;
3271 done += l;
3272 }
3273 *plen = done;
3274 return ret;
3275}
3276
3277/* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3278 * Will also mark the memory as dirty if is_write == 1. access_len gives
3279 * the amount of memory that was actually read or written by the caller.
3280 */
3281void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3282 int is_write, target_phys_addr_t access_len)
3283{
3284 if (buffer != bounce.buffer) {
3285 if (is_write) {
3286 ram_addr_t addr1 = qemu_ram_addr_from_host(buffer);
3287 while (access_len) {
3288 unsigned l;
3289 l = TARGET_PAGE_SIZE;
3290 if (l > access_len)
3291 l = access_len;
3292 if (!cpu_physical_memory_is_dirty(addr1)) {
3293 /* invalidate code */
3294 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3295 /* set dirty bit */
3296 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3297 (0xff & ~CODE_DIRTY_FLAG);
3298 }
3299 addr1 += l;
3300 access_len -= l;
3301 }
3302 }
3303 return;
3304 }
3305 if (is_write) {
3306 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3307 }
3308 qemu_vfree(bounce.buffer);
3309 bounce.buffer = NULL;
3310 cpu_notify_map_clients();
3311}
3312
3313/* warning: addr must be aligned */
3314uint32_t ldl_phys(target_phys_addr_t addr)
3315{
3316 int io_index;
3317 uint8_t *ptr;
3318 uint32_t val;
3319 unsigned long pd;
3320 PhysPageDesc *p;
3321
3322 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3323 if (!p) {
3324 pd = IO_MEM_UNASSIGNED;
3325 } else {
3326 pd = p->phys_offset;
3327 }
3328
3329 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3330 !(pd & IO_MEM_ROMD)) {
3331 /* I/O case */
3332 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3333 if (p)
3334 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3335 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3336 } else {
3337 /* RAM case */
3338 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3339 (addr & ~TARGET_PAGE_MASK);
3340 val = ldl_p(ptr);
3341 }
3342 return val;
3343}
3344
3345/* warning: addr must be aligned */
3346uint64_t ldq_phys(target_phys_addr_t addr)
3347{
3348 int io_index;
3349 uint8_t *ptr;
3350 uint64_t val;
3351 unsigned long pd;
3352 PhysPageDesc *p;
3353
3354 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3355 if (!p) {
3356 pd = IO_MEM_UNASSIGNED;
3357 } else {
3358 pd = p->phys_offset;
3359 }
3360
3361 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3362 !(pd & IO_MEM_ROMD)) {
3363 /* I/O case */
3364 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3365 if (p)
3366 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3367#ifdef TARGET_WORDS_BIGENDIAN
3368 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3369 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3370#else
3371 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3372 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3373#endif
3374 } else {
3375 /* RAM case */
3376 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3377 (addr & ~TARGET_PAGE_MASK);
3378 val = ldq_p(ptr);
3379 }
3380 return val;
3381}
3382
3383/* XXX: optimize */
3384uint32_t ldub_phys(target_phys_addr_t addr)
3385{
3386 uint8_t val;
3387 cpu_physical_memory_read(addr, &val, 1);
3388 return val;
3389}
3390
3391/* XXX: optimize */
3392uint32_t lduw_phys(target_phys_addr_t addr)
3393{
3394 uint16_t val;
3395 cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3396 return tswap16(val);
3397}
3398
3399/* warning: addr must be aligned. The ram page is not masked as dirty
3400 and the code inside is not invalidated. It is useful if the dirty
3401 bits are used to track modified PTEs */
3402void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3403{
3404 int io_index;
3405 uint8_t *ptr;
3406 unsigned long pd;
3407 PhysPageDesc *p;
3408
3409 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3410 if (!p) {
3411 pd = IO_MEM_UNASSIGNED;
3412 } else {
3413 pd = p->phys_offset;
3414 }
3415
3416 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3417 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3418 if (p)
3419 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3420 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3421 } else {
3422 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3423 ptr = qemu_get_ram_ptr(addr1);
3424 stl_p(ptr, val);
3425
3426 if (unlikely(in_migration)) {
3427 if (!cpu_physical_memory_is_dirty(addr1)) {
3428 /* invalidate code */
3429 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3430 /* set dirty bit */
3431 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3432 (0xff & ~CODE_DIRTY_FLAG);
3433 }
3434 }
3435 }
3436}
3437
3438void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3439{
3440 int io_index;
3441 uint8_t *ptr;
3442 unsigned long pd;
3443 PhysPageDesc *p;
3444
3445 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3446 if (!p) {
3447 pd = IO_MEM_UNASSIGNED;
3448 } else {
3449 pd = p->phys_offset;
3450 }
3451
3452 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3453 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3454 if (p)
3455 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3456#ifdef TARGET_WORDS_BIGENDIAN
3457 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3458 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3459#else
3460 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3461 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3462#endif
3463 } else {
3464 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3465 (addr & ~TARGET_PAGE_MASK);
3466 stq_p(ptr, val);
3467 }
3468}
3469
3470/* warning: addr must be aligned */
3471void stl_phys(target_phys_addr_t addr, uint32_t val)
3472{
3473 int io_index;
3474 uint8_t *ptr;
3475 unsigned long pd;
3476 PhysPageDesc *p;
3477
3478 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3479 if (!p) {
3480 pd = IO_MEM_UNASSIGNED;
3481 } else {
3482 pd = p->phys_offset;
3483 }
3484
3485 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3486 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3487 if (p)
3488 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3489 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3490 } else {
3491 unsigned long addr1;
3492 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3493 /* RAM case */
3494 ptr = qemu_get_ram_ptr(addr1);
3495 stl_p(ptr, val);
3496 if (!cpu_physical_memory_is_dirty(addr1)) {
3497 /* invalidate code */
3498 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3499 /* set dirty bit */
3500 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3501 (0xff & ~CODE_DIRTY_FLAG);
3502 }
3503 }
3504}
3505
3506/* XXX: optimize */
3507void stb_phys(target_phys_addr_t addr, uint32_t val)
3508{
3509 uint8_t v = val;
3510 cpu_physical_memory_write(addr, &v, 1);
3511}
3512
3513/* XXX: optimize */
3514void stw_phys(target_phys_addr_t addr, uint32_t val)
3515{
3516 uint16_t v = tswap16(val);
3517 cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
3518}
3519
3520/* XXX: optimize */
3521void stq_phys(target_phys_addr_t addr, uint64_t val)
3522{
3523 val = tswap64(val);
3524 cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3525}
3526
3527#endif
3528
3529/* virtual memory access for debug (includes writing to ROM) */
3530int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3531 uint8_t *buf, int len, int is_write)
3532{
3533 int l;
3534 target_phys_addr_t phys_addr;
3535 target_ulong page;
3536
3537 while (len > 0) {
3538 page = addr & TARGET_PAGE_MASK;
3539 phys_addr = cpu_get_phys_page_debug(env, page);
3540 /* if no physical page mapped, return an error */
3541 if (phys_addr == -1)
3542 return -1;
3543 l = (page + TARGET_PAGE_SIZE) - addr;
3544 if (l > len)
3545 l = len;
3546 phys_addr += (addr & ~TARGET_PAGE_MASK);
3547#if !defined(CONFIG_USER_ONLY)
3548 if (is_write)
3549 cpu_physical_memory_write_rom(phys_addr, buf, l);
3550 else
3551#endif
3552 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
3553 len -= l;
3554 buf += l;
3555 addr += l;
3556 }
3557 return 0;
3558}
3559
3560/* in deterministic execution mode, instructions doing device I/Os
3561 must be at the end of the TB */
3562void cpu_io_recompile(CPUState *env, void *retaddr)
3563{
3564 TranslationBlock *tb;
3565 uint32_t n, cflags;
3566 target_ulong pc, cs_base;
3567 uint64_t flags;
3568
3569 tb = tb_find_pc((unsigned long)retaddr);
3570 if (!tb) {
3571 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
3572 retaddr);
3573 }
3574 n = env->icount_decr.u16.low + tb->icount;
3575 cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3576 /* Calculate how many instructions had been executed before the fault
3577 occurred. */
3578 n = n - env->icount_decr.u16.low;
3579 /* Generate a new TB ending on the I/O insn. */
3580 n++;
3581 /* On MIPS and SH, delay slot instructions can only be restarted if
3582 they were already the first instruction in the TB. If this is not
3583 the first instruction in a TB then re-execute the preceding
3584 branch. */
3585#if defined(TARGET_MIPS)
3586 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3587 env->active_tc.PC -= 4;
3588 env->icount_decr.u16.low++;
3589 env->hflags &= ~MIPS_HFLAG_BMASK;
3590 }
3591#elif defined(TARGET_SH4)
3592 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3593 && n > 1) {
3594 env->pc -= 2;
3595 env->icount_decr.u16.low++;
3596 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3597 }
3598#endif
3599 /* This should never happen. */
3600 if (n > CF_COUNT_MASK)
3601 cpu_abort(env, "TB too big during recompile");
3602
3603 cflags = n | CF_LAST_IO;
3604 pc = tb->pc;
3605 cs_base = tb->cs_base;
3606 flags = tb->flags;
3607 tb_phys_invalidate(tb, -1);
3608 /* FIXME: In theory this could raise an exception. In practice
3609 we have already translated the block once so it's probably ok. */
3610 tb_gen_code(env, pc, cs_base, flags, cflags);
3611 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3612 the first in the TB) then we end up generating a whole new TB and
3613 repeating the fault, which is horribly inefficient.
3614 Better would be to execute just this insn uncached, or generate a
3615 second new TB. */
3616 cpu_resume_from_signal(env, NULL);
3617}
3618
3619void dump_exec_info(FILE *f,
3620 int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
3621{
3622 int i, target_code_size, max_target_code_size;
3623 int direct_jmp_count, direct_jmp2_count, cross_page;
3624 TranslationBlock *tb;
3625
3626 target_code_size = 0;
3627 max_target_code_size = 0;
3628 cross_page = 0;
3629 direct_jmp_count = 0;
3630 direct_jmp2_count = 0;
3631 for(i = 0; i < nb_tbs; i++) {
3632 tb = &tbs[i];
3633 target_code_size += tb->size;
3634 if (tb->size > max_target_code_size)
3635 max_target_code_size = tb->size;
3636 if (tb->page_addr[1] != -1)
3637 cross_page++;
3638 if (tb->tb_next_offset[0] != 0xffff) {
3639 direct_jmp_count++;
3640 if (tb->tb_next_offset[1] != 0xffff) {
3641 direct_jmp2_count++;
3642 }
3643 }
3644 }
3645 /* XXX: avoid using doubles ? */
3646 cpu_fprintf(f, "Translation buffer state:\n");
3647 cpu_fprintf(f, "gen code size %ld/%ld\n",
3648 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
3649 cpu_fprintf(f, "TB count %d/%d\n",
3650 nb_tbs, code_gen_max_blocks);
3651 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
3652 nb_tbs ? target_code_size / nb_tbs : 0,
3653 max_target_code_size);
3654 cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3655 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
3656 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
3657 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
3658 cross_page,
3659 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
3660 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3661 direct_jmp_count,
3662 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
3663 direct_jmp2_count,
3664 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
3665 cpu_fprintf(f, "\nStatistics:\n");
3666 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
3667 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
3668 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
3669 tcg_dump_info(f, cpu_fprintf);
3670}
3671
3672#if !defined(CONFIG_USER_ONLY)
3673
3674#define MMUSUFFIX _cmmu
3675#define GETPC() NULL
3676#define env cpu_single_env
3677#define SOFTMMU_CODE_ACCESS
3678
3679#define SHIFT 0
3680#include "softmmu_template.h"
3681
3682#define SHIFT 1
3683#include "softmmu_template.h"
3684
3685#define SHIFT 2
3686#include "softmmu_template.h"
3687
3688#define SHIFT 3
3689#include "softmmu_template.h"
3690
3691#undef env
3692
3693#endif
QEMU mirror