]>
Commit | Line | Data |
---|---|---|
7d13299d | 1 | /* |
e965fc38 | 2 | * emulator main execution loop |
5fafdf24 | 3 | * |
66321a11 | 4 | * Copyright (c) 2003-2005 Fabrice Bellard |
7d13299d | 5 | * |
3ef693a0 FB |
6 | * This library is free software; you can redistribute it and/or |
7 | * modify it under the terms of the GNU Lesser General Public | |
8 | * License as published by the Free Software Foundation; either | |
fb0343d5 | 9 | * version 2.1 of the License, or (at your option) any later version. |
7d13299d | 10 | * |
3ef693a0 FB |
11 | * This library is distributed in the hope that it will be useful, |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
14 | * Lesser General Public License for more details. | |
7d13299d | 15 | * |
3ef693a0 | 16 | * You should have received a copy of the GNU Lesser General Public |
8167ee88 | 17 | * License along with this library; if not, see <http://www.gnu.org/licenses/>. |
7d13299d | 18 | */ |
a8d25326 | 19 | |
7b31bbc2 | 20 | #include "qemu/osdep.h" |
a8d25326 | 21 | #include "qemu-common.h" |
740b1759 | 22 | #include "qemu/qemu-print.h" |
78271684 | 23 | #include "hw/core/tcg-cpu-ops.h" |
d9bb58e5 | 24 | #include "trace.h" |
76cad711 | 25 | #include "disas/disas.h" |
63c91552 | 26 | #include "exec/exec-all.h" |
dcb32f1d | 27 | #include "tcg/tcg.h" |
1de7afc9 | 28 | #include "qemu/atomic.h" |
c905a368 | 29 | #include "qemu/compiler.h" |
c2aa5f81 | 30 | #include "qemu/timer.h" |
79e2b9ae | 31 | #include "qemu/rcu.h" |
508127e2 | 32 | #include "exec/log.h" |
8d04fb55 | 33 | #include "qemu/main-loop.h" |
6220e900 PD |
34 | #if defined(TARGET_I386) && !defined(CONFIG_USER_ONLY) |
35 | #include "hw/i386/apic.h" | |
36 | #endif | |
d2528bdc | 37 | #include "sysemu/cpus.h" |
740b1759 CF |
38 | #include "exec/cpu-all.h" |
39 | #include "sysemu/cpu-timers.h" | |
6f060969 | 40 | #include "sysemu/replay.h" |
4288eb26 | 41 | #include "exec/helper-proto.h" |
e5ceadff | 42 | #include "tb-hash.h" |
e5ceadff | 43 | #include "tb-context.h" |
c03f041f | 44 | #include "internal.h" |
c2aa5f81 ST |
45 | |
46 | /* -icount align implementation. */ | |
47 | ||
48 | typedef struct SyncClocks { | |
49 | int64_t diff_clk; | |
50 | int64_t last_cpu_icount; | |
7f7bc144 | 51 | int64_t realtime_clock; |
c2aa5f81 ST |
52 | } SyncClocks; |
53 | ||
54 | #if !defined(CONFIG_USER_ONLY) | |
55 | /* Allow the guest to have a max 3ms advance. | |
56 | * The difference between the 2 clocks could therefore | |
57 | * oscillate around 0. | |
58 | */ | |
59 | #define VM_CLOCK_ADVANCE 3000000 | |
7f7bc144 ST |
60 | #define THRESHOLD_REDUCE 1.5 |
61 | #define MAX_DELAY_PRINT_RATE 2000000000LL | |
62 | #define MAX_NB_PRINTS 100 | |
c2aa5f81 | 63 | |
740b1759 CF |
64 | static int64_t max_delay; |
65 | static int64_t max_advance; | |
66 | ||
5e140196 | 67 | static void align_clocks(SyncClocks *sc, CPUState *cpu) |
c2aa5f81 ST |
68 | { |
69 | int64_t cpu_icount; | |
70 | ||
71 | if (!icount_align_option) { | |
72 | return; | |
73 | } | |
74 | ||
5e140196 | 75 | cpu_icount = cpu->icount_extra + cpu_neg(cpu)->icount_decr.u16.low; |
8191d368 | 76 | sc->diff_clk += icount_to_ns(sc->last_cpu_icount - cpu_icount); |
c2aa5f81 ST |
77 | sc->last_cpu_icount = cpu_icount; |
78 | ||
79 | if (sc->diff_clk > VM_CLOCK_ADVANCE) { | |
80 | #ifndef _WIN32 | |
81 | struct timespec sleep_delay, rem_delay; | |
82 | sleep_delay.tv_sec = sc->diff_clk / 1000000000LL; | |
83 | sleep_delay.tv_nsec = sc->diff_clk % 1000000000LL; | |
84 | if (nanosleep(&sleep_delay, &rem_delay) < 0) { | |
a498d0ef | 85 | sc->diff_clk = rem_delay.tv_sec * 1000000000LL + rem_delay.tv_nsec; |
c2aa5f81 ST |
86 | } else { |
87 | sc->diff_clk = 0; | |
88 | } | |
89 | #else | |
90 | Sleep(sc->diff_clk / SCALE_MS); | |
91 | sc->diff_clk = 0; | |
92 | #endif | |
93 | } | |
94 | } | |
95 | ||
7f7bc144 ST |
96 | static void print_delay(const SyncClocks *sc) |
97 | { | |
98 | static float threshold_delay; | |
99 | static int64_t last_realtime_clock; | |
100 | static int nb_prints; | |
101 | ||
102 | if (icount_align_option && | |
103 | sc->realtime_clock - last_realtime_clock >= MAX_DELAY_PRINT_RATE && | |
104 | nb_prints < MAX_NB_PRINTS) { | |
105 | if ((-sc->diff_clk / (float)1000000000LL > threshold_delay) || | |
106 | (-sc->diff_clk / (float)1000000000LL < | |
107 | (threshold_delay - THRESHOLD_REDUCE))) { | |
108 | threshold_delay = (-sc->diff_clk / 1000000000LL) + 1; | |
740b1759 CF |
109 | qemu_printf("Warning: The guest is now late by %.1f to %.1f seconds\n", |
110 | threshold_delay - 1, | |
111 | threshold_delay); | |
7f7bc144 ST |
112 | nb_prints++; |
113 | last_realtime_clock = sc->realtime_clock; | |
114 | } | |
115 | } | |
116 | } | |
117 | ||
5e140196 | 118 | static void init_delay_params(SyncClocks *sc, CPUState *cpu) |
c2aa5f81 ST |
119 | { |
120 | if (!icount_align_option) { | |
121 | return; | |
122 | } | |
2e91cc62 PB |
123 | sc->realtime_clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT); |
124 | sc->diff_clk = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - sc->realtime_clock; | |
5e140196 RH |
125 | sc->last_cpu_icount |
126 | = cpu->icount_extra + cpu_neg(cpu)->icount_decr.u16.low; | |
27498bef ST |
127 | if (sc->diff_clk < max_delay) { |
128 | max_delay = sc->diff_clk; | |
129 | } | |
130 | if (sc->diff_clk > max_advance) { | |
131 | max_advance = sc->diff_clk; | |
132 | } | |
7f7bc144 ST |
133 | |
134 | /* Print every 2s max if the guest is late. We limit the number | |
135 | of printed messages to NB_PRINT_MAX(currently 100) */ | |
136 | print_delay(sc); | |
c2aa5f81 ST |
137 | } |
138 | #else | |
139 | static void align_clocks(SyncClocks *sc, const CPUState *cpu) | |
140 | { | |
141 | } | |
142 | ||
143 | static void init_delay_params(SyncClocks *sc, const CPUState *cpu) | |
144 | { | |
145 | } | |
146 | #endif /* CONFIG USER ONLY */ | |
7d13299d | 147 | |
043e35d9 RH |
148 | uint32_t curr_cflags(CPUState *cpu) |
149 | { | |
150 | return cpu->tcg_cflags; | |
151 | } | |
152 | ||
632cb63d RH |
153 | /* Might cause an exception, so have a longjmp destination ready */ |
154 | static inline TranslationBlock *tb_lookup(CPUState *cpu, target_ulong pc, | |
155 | target_ulong cs_base, | |
156 | uint32_t flags, uint32_t cflags) | |
157 | { | |
158 | TranslationBlock *tb; | |
159 | uint32_t hash; | |
160 | ||
161 | /* we should never be trying to look up an INVALID tb */ | |
162 | tcg_debug_assert(!(cflags & CF_INVALID)); | |
163 | ||
164 | hash = tb_jmp_cache_hash_func(pc); | |
165 | tb = qatomic_rcu_read(&cpu->tb_jmp_cache[hash]); | |
166 | ||
167 | if (likely(tb && | |
168 | tb->pc == pc && | |
169 | tb->cs_base == cs_base && | |
170 | tb->flags == flags && | |
171 | tb->trace_vcpu_dstate == *cpu->trace_dstate && | |
172 | tb_cflags(tb) == cflags)) { | |
173 | return tb; | |
174 | } | |
175 | tb = tb_htable_lookup(cpu, pc, cs_base, flags, cflags); | |
176 | if (tb == NULL) { | |
177 | return NULL; | |
178 | } | |
179 | qatomic_set(&cpu->tb_jmp_cache[hash], tb); | |
180 | return tb; | |
181 | } | |
182 | ||
abb0cd93 RH |
183 | static inline void log_cpu_exec(target_ulong pc, CPUState *cpu, |
184 | const TranslationBlock *tb) | |
185 | { | |
186 | if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_CPU | CPU_LOG_EXEC)) | |
187 | && qemu_log_in_addr_range(pc)) { | |
188 | ||
189 | qemu_log_mask(CPU_LOG_EXEC, | |
190 | "Trace %d: %p [" TARGET_FMT_lx | |
7eabad36 RH |
191 | "/" TARGET_FMT_lx "/%08x/%08x] %s\n", |
192 | cpu->cpu_index, tb->tc.ptr, tb->cs_base, pc, | |
193 | tb->flags, tb->cflags, lookup_symbol(pc)); | |
abb0cd93 RH |
194 | |
195 | #if defined(DEBUG_DISAS) | |
196 | if (qemu_loglevel_mask(CPU_LOG_TB_CPU)) { | |
197 | FILE *logfile = qemu_log_lock(); | |
198 | int flags = 0; | |
199 | ||
200 | if (qemu_loglevel_mask(CPU_LOG_TB_FPU)) { | |
201 | flags |= CPU_DUMP_FPU; | |
202 | } | |
203 | #if defined(TARGET_I386) | |
204 | flags |= CPU_DUMP_CCOP; | |
205 | #endif | |
206 | log_cpu_state(cpu, flags); | |
207 | qemu_log_unlock(logfile); | |
208 | } | |
209 | #endif /* DEBUG_DISAS */ | |
210 | } | |
211 | } | |
212 | ||
4288eb26 RH |
213 | /** |
214 | * helper_lookup_tb_ptr: quick check for next tb | |
215 | * @env: current cpu state | |
216 | * | |
217 | * Look for an existing TB matching the current cpu state. | |
218 | * If found, return the code pointer. If not found, return | |
219 | * the tcg epilogue so that we return into cpu_tb_exec. | |
220 | */ | |
221 | const void *HELPER(lookup_tb_ptr)(CPUArchState *env) | |
222 | { | |
223 | CPUState *cpu = env_cpu(env); | |
224 | TranslationBlock *tb; | |
225 | target_ulong cs_base, pc; | |
226 | uint32_t flags; | |
227 | ||
228 | cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); | |
229 | ||
230 | tb = tb_lookup(cpu, pc, cs_base, flags, curr_cflags(cpu)); | |
231 | if (tb == NULL) { | |
232 | return tcg_code_gen_epilogue; | |
233 | } | |
abb0cd93 RH |
234 | |
235 | log_cpu_exec(pc, cpu, tb); | |
236 | ||
4288eb26 RH |
237 | return tb->tc.ptr; |
238 | } | |
239 | ||
77211379 | 240 | /* Execute a TB, and fix up the CPU state afterwards if necessary */ |
c905a368 DB |
241 | /* |
242 | * Disable CFI checks. | |
243 | * TCG creates binary blobs at runtime, with the transformed code. | |
244 | * A TB is a blob of binary code, created at runtime and called with an | |
245 | * indirect function call. Since such function did not exist at compile time, | |
246 | * the CFI runtime has no way to verify its signature and would fail. | |
247 | * TCG is not considered a security-sensitive part of QEMU so this does not | |
248 | * affect the impact of CFI in environment with high security requirements | |
249 | */ | |
eba40358 RH |
250 | static inline TranslationBlock * QEMU_DISABLE_CFI |
251 | cpu_tb_exec(CPUState *cpu, TranslationBlock *itb, int *tb_exit) | |
77211379 PM |
252 | { |
253 | CPUArchState *env = cpu->env_ptr; | |
819af24b SF |
254 | uintptr_t ret; |
255 | TranslationBlock *last_tb; | |
db0c51a3 | 256 | const void *tb_ptr = itb->tc.ptr; |
1a830635 | 257 | |
abb0cd93 | 258 | log_cpu_exec(itb->pc, cpu, itb); |
03afa5f8 | 259 | |
653b87eb | 260 | qemu_thread_jit_execute(); |
819af24b | 261 | ret = tcg_qemu_tb_exec(env, tb_ptr); |
626cf8f4 | 262 | cpu->can_do_io = 1; |
eba40358 RH |
263 | /* |
264 | * TODO: Delay swapping back to the read-write region of the TB | |
265 | * until we actually need to modify the TB. The read-only copy, | |
266 | * coming from the rx region, shares the same host TLB entry as | |
267 | * the code that executed the exit_tb opcode that arrived here. | |
268 | * If we insist on touching both the RX and the RW pages, we | |
269 | * double the host TLB pressure. | |
270 | */ | |
271 | last_tb = tcg_splitwx_to_rw((void *)(ret & ~TB_EXIT_MASK)); | |
272 | *tb_exit = ret & TB_EXIT_MASK; | |
273 | ||
274 | trace_exec_tb_exit(last_tb, *tb_exit); | |
6db8b538 | 275 | |
eba40358 | 276 | if (*tb_exit > TB_EXIT_IDX1) { |
77211379 PM |
277 | /* We didn't start executing this TB (eg because the instruction |
278 | * counter hit zero); we must restore the guest PC to the address | |
279 | * of the start of the TB. | |
280 | */ | |
bdf7ae5b | 281 | CPUClass *cc = CPU_GET_CLASS(cpu); |
819af24b | 282 | qemu_log_mask_and_addr(CPU_LOG_EXEC, last_tb->pc, |
d977e1c2 AB |
283 | "Stopped execution of TB chain before %p [" |
284 | TARGET_FMT_lx "] %s\n", | |
e7e168f4 | 285 | last_tb->tc.ptr, last_tb->pc, |
819af24b | 286 | lookup_symbol(last_tb->pc)); |
78271684 CF |
287 | if (cc->tcg_ops->synchronize_from_tb) { |
288 | cc->tcg_ops->synchronize_from_tb(cpu, last_tb); | |
bdf7ae5b AF |
289 | } else { |
290 | assert(cc->set_pc); | |
819af24b | 291 | cc->set_pc(cpu, last_tb->pc); |
bdf7ae5b | 292 | } |
77211379 | 293 | } |
eba40358 | 294 | return last_tb; |
77211379 PM |
295 | } |
296 | ||
2e70f6ef | 297 | |
035ba06c EH |
298 | static void cpu_exec_enter(CPUState *cpu) |
299 | { | |
300 | CPUClass *cc = CPU_GET_CLASS(cpu); | |
301 | ||
78271684 CF |
302 | if (cc->tcg_ops->cpu_exec_enter) { |
303 | cc->tcg_ops->cpu_exec_enter(cpu); | |
80c4750b | 304 | } |
035ba06c EH |
305 | } |
306 | ||
307 | static void cpu_exec_exit(CPUState *cpu) | |
fdbc2b57 | 308 | { |
08e73c48 | 309 | CPUClass *cc = CPU_GET_CLASS(cpu); |
035ba06c | 310 | |
78271684 CF |
311 | if (cc->tcg_ops->cpu_exec_exit) { |
312 | cc->tcg_ops->cpu_exec_exit(cpu); | |
80c4750b | 313 | } |
035ba06c EH |
314 | } |
315 | ||
316 | void cpu_exec_step_atomic(CPUState *cpu) | |
317 | { | |
6f04cb1c | 318 | CPUArchState *env = (CPUArchState *)cpu->env_ptr; |
fdbc2b57 RH |
319 | TranslationBlock *tb; |
320 | target_ulong cs_base, pc; | |
321 | uint32_t flags; | |
c0ae396a | 322 | uint32_t cflags = (curr_cflags(cpu) & ~CF_PARALLEL) | 1; |
eba40358 | 323 | int tb_exit; |
fdbc2b57 | 324 | |
08e73c48 | 325 | if (sigsetjmp(cpu->jmp_env, 0) == 0) { |
886cc689 | 326 | start_exclusive(); |
bfff072c DC |
327 | g_assert(cpu == current_cpu); |
328 | g_assert(!cpu->running); | |
329 | cpu->running = true; | |
886cc689 | 330 | |
6f04cb1c | 331 | cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); |
c0ae396a | 332 | tb = tb_lookup(cpu, pc, cs_base, flags, cflags); |
6f04cb1c | 333 | |
4e2ca83e EC |
334 | if (tb == NULL) { |
335 | mmap_lock(); | |
95590e24 | 336 | tb = tb_gen_code(cpu, pc, cs_base, flags, cflags); |
4e2ca83e EC |
337 | mmap_unlock(); |
338 | } | |
08e73c48 | 339 | |
035ba06c | 340 | cpu_exec_enter(cpu); |
08e73c48 | 341 | /* execute the generated code */ |
4e2ca83e | 342 | trace_exec_tb(tb, pc); |
eba40358 | 343 | cpu_tb_exec(cpu, tb, &tb_exit); |
035ba06c | 344 | cpu_exec_exit(cpu); |
08e73c48 | 345 | } else { |
0ac20318 | 346 | /* |
08e73c48 PK |
347 | * The mmap_lock is dropped by tb_gen_code if it runs out of |
348 | * memory. | |
349 | */ | |
350 | #ifndef CONFIG_SOFTMMU | |
351 | tcg_debug_assert(!have_mmap_lock()); | |
352 | #endif | |
6aaa24f9 EC |
353 | if (qemu_mutex_iothread_locked()) { |
354 | qemu_mutex_unlock_iothread(); | |
355 | } | |
faa9372c | 356 | assert_no_pages_locked(); |
e6d86bed | 357 | qemu_plugin_disable_mem_helpers(cpu); |
08e73c48 | 358 | } |
426eeecd | 359 | |
886cc689 AB |
360 | |
361 | /* | |
362 | * As we start the exclusive region before codegen we must still | |
363 | * be in the region if we longjump out of either the codegen or | |
364 | * the execution. | |
365 | */ | |
366 | g_assert(cpu_in_exclusive_context(cpu)); | |
bfff072c | 367 | cpu->running = false; |
886cc689 | 368 | end_exclusive(); |
fdbc2b57 RH |
369 | } |
370 | ||
909eaac9 EC |
371 | struct tb_desc { |
372 | target_ulong pc; | |
373 | target_ulong cs_base; | |
374 | CPUArchState *env; | |
375 | tb_page_addr_t phys_page1; | |
376 | uint32_t flags; | |
bf253ac6 | 377 | uint32_t cflags; |
61a67f71 | 378 | uint32_t trace_vcpu_dstate; |
909eaac9 EC |
379 | }; |
380 | ||
61b8cef1 | 381 | static bool tb_lookup_cmp(const void *p, const void *d) |
909eaac9 EC |
382 | { |
383 | const TranslationBlock *tb = p; | |
384 | const struct tb_desc *desc = d; | |
385 | ||
386 | if (tb->pc == desc->pc && | |
387 | tb->page_addr[0] == desc->phys_page1 && | |
388 | tb->cs_base == desc->cs_base && | |
6d21e420 | 389 | tb->flags == desc->flags && |
61a67f71 | 390 | tb->trace_vcpu_dstate == desc->trace_vcpu_dstate && |
bf253ac6 | 391 | tb_cflags(tb) == desc->cflags) { |
909eaac9 EC |
392 | /* check next page if needed */ |
393 | if (tb->page_addr[1] == -1) { | |
394 | return true; | |
395 | } else { | |
396 | tb_page_addr_t phys_page2; | |
397 | target_ulong virt_page2; | |
398 | ||
399 | virt_page2 = (desc->pc & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE; | |
400 | phys_page2 = get_page_addr_code(desc->env, virt_page2); | |
401 | if (tb->page_addr[1] == phys_page2) { | |
402 | return true; | |
403 | } | |
404 | } | |
405 | } | |
406 | return false; | |
407 | } | |
408 | ||
cedbcb01 | 409 | TranslationBlock *tb_htable_lookup(CPUState *cpu, target_ulong pc, |
4e2ca83e | 410 | target_ulong cs_base, uint32_t flags, |
bf253ac6 | 411 | uint32_t cflags) |
8a40a180 | 412 | { |
909eaac9 EC |
413 | tb_page_addr_t phys_pc; |
414 | struct tb_desc desc; | |
42bd3228 | 415 | uint32_t h; |
3b46e624 | 416 | |
909eaac9 EC |
417 | desc.env = (CPUArchState *)cpu->env_ptr; |
418 | desc.cs_base = cs_base; | |
419 | desc.flags = flags; | |
bf253ac6 | 420 | desc.cflags = cflags; |
61a67f71 | 421 | desc.trace_vcpu_dstate = *cpu->trace_dstate; |
909eaac9 EC |
422 | desc.pc = pc; |
423 | phys_pc = get_page_addr_code(desc.env, pc); | |
7252f2de PM |
424 | if (phys_pc == -1) { |
425 | return NULL; | |
426 | } | |
909eaac9 | 427 | desc.phys_page1 = phys_pc & TARGET_PAGE_MASK; |
bf253ac6 | 428 | h = tb_hash_func(phys_pc, pc, flags, cflags, *cpu->trace_dstate); |
61b8cef1 | 429 | return qht_lookup_custom(&tb_ctx.htable, &desc, h, tb_lookup_cmp); |
9fd1a948 PB |
430 | } |
431 | ||
a8583393 RH |
432 | void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr) |
433 | { | |
434 | if (TCG_TARGET_HAS_direct_jump) { | |
435 | uintptr_t offset = tb->jmp_target_arg[n]; | |
e7e168f4 | 436 | uintptr_t tc_ptr = (uintptr_t)tb->tc.ptr; |
1acbad0f RH |
437 | uintptr_t jmp_rx = tc_ptr + offset; |
438 | uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff; | |
439 | tb_target_set_jmp_target(tc_ptr, jmp_rx, jmp_rw, addr); | |
a8583393 RH |
440 | } else { |
441 | tb->jmp_target_arg[n] = addr; | |
442 | } | |
443 | } | |
444 | ||
a8583393 RH |
445 | static inline void tb_add_jump(TranslationBlock *tb, int n, |
446 | TranslationBlock *tb_next) | |
447 | { | |
194125e3 EC |
448 | uintptr_t old; |
449 | ||
653b87eb | 450 | qemu_thread_jit_write(); |
a8583393 | 451 | assert(n < ARRAY_SIZE(tb->jmp_list_next)); |
194125e3 EC |
452 | qemu_spin_lock(&tb_next->jmp_lock); |
453 | ||
454 | /* make sure the destination TB is valid */ | |
455 | if (tb_next->cflags & CF_INVALID) { | |
456 | goto out_unlock_next; | |
457 | } | |
458 | /* Atomically claim the jump destination slot only if it was NULL */ | |
d73415a3 SH |
459 | old = qatomic_cmpxchg(&tb->jmp_dest[n], (uintptr_t)NULL, |
460 | (uintptr_t)tb_next); | |
194125e3 EC |
461 | if (old) { |
462 | goto out_unlock_next; | |
a8583393 | 463 | } |
194125e3 EC |
464 | |
465 | /* patch the native jump address */ | |
466 | tb_set_jmp_target(tb, n, (uintptr_t)tb_next->tc.ptr); | |
467 | ||
468 | /* add in TB jmp list */ | |
469 | tb->jmp_list_next[n] = tb_next->jmp_list_head; | |
470 | tb_next->jmp_list_head = (uintptr_t)tb | n; | |
471 | ||
472 | qemu_spin_unlock(&tb_next->jmp_lock); | |
473 | ||
a8583393 RH |
474 | qemu_log_mask_and_addr(CPU_LOG_EXEC, tb->pc, |
475 | "Linking TBs %p [" TARGET_FMT_lx | |
476 | "] index %d -> %p [" TARGET_FMT_lx "]\n", | |
e7e168f4 EC |
477 | tb->tc.ptr, tb->pc, n, |
478 | tb_next->tc.ptr, tb_next->pc); | |
194125e3 | 479 | return; |
a8583393 | 480 | |
194125e3 EC |
481 | out_unlock_next: |
482 | qemu_spin_unlock(&tb_next->jmp_lock); | |
483 | return; | |
a8583393 RH |
484 | } |
485 | ||
bd2710d5 SF |
486 | static inline TranslationBlock *tb_find(CPUState *cpu, |
487 | TranslationBlock *last_tb, | |
bf253ac6 | 488 | int tb_exit, uint32_t cflags) |
8a40a180 | 489 | { |
6f04cb1c | 490 | CPUArchState *env = (CPUArchState *)cpu->env_ptr; |
8a40a180 FB |
491 | TranslationBlock *tb; |
492 | target_ulong cs_base, pc; | |
89fee74a | 493 | uint32_t flags; |
8a40a180 | 494 | |
6f04cb1c AB |
495 | cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); |
496 | ||
bf253ac6 | 497 | tb = tb_lookup(cpu, pc, cs_base, flags, cflags); |
f6bb84d5 | 498 | if (tb == NULL) { |
f6bb84d5 | 499 | mmap_lock(); |
bf253ac6 | 500 | tb = tb_gen_code(cpu, pc, cs_base, flags, cflags); |
f6bb84d5 | 501 | mmap_unlock(); |
bd2710d5 | 502 | /* We add the TB in the virtual pc hash table for the fast lookup */ |
d73415a3 | 503 | qatomic_set(&cpu->tb_jmp_cache[tb_jmp_cache_hash_func(pc)], tb); |
8a40a180 | 504 | } |
c88c67e5 SF |
505 | #ifndef CONFIG_USER_ONLY |
506 | /* We don't take care of direct jumps when address mapping changes in | |
507 | * system emulation. So it's not safe to make a direct jump to a TB | |
508 | * spanning two pages because the mapping for the second page can change. | |
509 | */ | |
510 | if (tb->page_addr[1] != -1) { | |
4b7e6950 | 511 | last_tb = NULL; |
c88c67e5 SF |
512 | } |
513 | #endif | |
a0522c7a | 514 | /* See if we can patch the calling TB. */ |
d7f425fd | 515 | if (last_tb) { |
194125e3 | 516 | tb_add_jump(last_tb, tb_exit, tb); |
74d356dd | 517 | } |
8a40a180 FB |
518 | return tb; |
519 | } | |
520 | ||
8b2d34e9 SF |
521 | static inline bool cpu_handle_halt(CPUState *cpu) |
522 | { | |
523 | if (cpu->halted) { | |
524 | #if defined(TARGET_I386) && !defined(CONFIG_USER_ONLY) | |
4084893d | 525 | if (cpu->interrupt_request & CPU_INTERRUPT_POLL) { |
8b2d34e9 | 526 | X86CPU *x86_cpu = X86_CPU(cpu); |
8d04fb55 | 527 | qemu_mutex_lock_iothread(); |
8b2d34e9 SF |
528 | apic_poll_irq(x86_cpu->apic_state); |
529 | cpu_reset_interrupt(cpu, CPU_INTERRUPT_POLL); | |
8d04fb55 | 530 | qemu_mutex_unlock_iothread(); |
8b2d34e9 SF |
531 | } |
532 | #endif | |
533 | if (!cpu_has_work(cpu)) { | |
8b2d34e9 SF |
534 | return true; |
535 | } | |
536 | ||
537 | cpu->halted = 0; | |
538 | } | |
539 | ||
540 | return false; | |
541 | } | |
542 | ||
ea284766 | 543 | static inline void cpu_handle_debug_exception(CPUState *cpu) |
1009d2ed | 544 | { |
86025ee4 | 545 | CPUClass *cc = CPU_GET_CLASS(cpu); |
1009d2ed JK |
546 | CPUWatchpoint *wp; |
547 | ||
ff4700b0 AF |
548 | if (!cpu->watchpoint_hit) { |
549 | QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) { | |
1009d2ed JK |
550 | wp->flags &= ~BP_WATCHPOINT_HIT; |
551 | } | |
552 | } | |
86025ee4 | 553 | |
78271684 CF |
554 | if (cc->tcg_ops->debug_excp_handler) { |
555 | cc->tcg_ops->debug_excp_handler(cpu); | |
710384d0 | 556 | } |
1009d2ed JK |
557 | } |
558 | ||
ea284766 SF |
559 | static inline bool cpu_handle_exception(CPUState *cpu, int *ret) |
560 | { | |
17b50b0c PD |
561 | if (cpu->exception_index < 0) { |
562 | #ifndef CONFIG_USER_ONLY | |
563 | if (replay_has_exception() | |
5e140196 | 564 | && cpu_neg(cpu)->icount_decr.u16.low + cpu->icount_extra == 0) { |
a11bbb6a | 565 | /* Execute just one insn to trigger exception pending in the log */ |
c0ae396a | 566 | cpu->cflags_next_tb = (curr_cflags(cpu) & ~CF_USE_ICOUNT) | 1; |
17b50b0c PD |
567 | } |
568 | #endif | |
a11bbb6a | 569 | return false; |
17b50b0c | 570 | } |
17b50b0c PD |
571 | if (cpu->exception_index >= EXCP_INTERRUPT) { |
572 | /* exit request from the cpu execution loop */ | |
573 | *ret = cpu->exception_index; | |
574 | if (*ret == EXCP_DEBUG) { | |
575 | cpu_handle_debug_exception(cpu); | |
576 | } | |
577 | cpu->exception_index = -1; | |
578 | return true; | |
579 | } else { | |
ea284766 | 580 | #if defined(CONFIG_USER_ONLY) |
17b50b0c PD |
581 | /* if user mode only, we simulate a fake exception |
582 | which will be handled outside the cpu execution | |
583 | loop */ | |
ea284766 | 584 | #if defined(TARGET_I386) |
17b50b0c | 585 | CPUClass *cc = CPU_GET_CLASS(cpu); |
78271684 | 586 | cc->tcg_ops->do_interrupt(cpu); |
17b50b0c PD |
587 | #endif |
588 | *ret = cpu->exception_index; | |
589 | cpu->exception_index = -1; | |
590 | return true; | |
591 | #else | |
592 | if (replay_exception()) { | |
ea284766 | 593 | CPUClass *cc = CPU_GET_CLASS(cpu); |
17b50b0c | 594 | qemu_mutex_lock_iothread(); |
78271684 | 595 | cc->tcg_ops->do_interrupt(cpu); |
17b50b0c | 596 | qemu_mutex_unlock_iothread(); |
ea284766 | 597 | cpu->exception_index = -1; |
a7ba744f LM |
598 | |
599 | if (unlikely(cpu->singlestep_enabled)) { | |
600 | /* | |
601 | * After processing the exception, ensure an EXCP_DEBUG is | |
602 | * raised when single-stepping so that GDB doesn't miss the | |
603 | * next instruction. | |
604 | */ | |
605 | *ret = EXCP_DEBUG; | |
606 | cpu_handle_debug_exception(cpu); | |
607 | return true; | |
608 | } | |
17b50b0c PD |
609 | } else if (!replay_has_interrupt()) { |
610 | /* give a chance to iothread in replay mode */ | |
611 | *ret = EXCP_INTERRUPT; | |
ea284766 | 612 | return true; |
ea284766 | 613 | } |
ea284766 SF |
614 | #endif |
615 | } | |
616 | ||
617 | return false; | |
618 | } | |
619 | ||
4084893d PD |
620 | /* |
621 | * CPU_INTERRUPT_POLL is a virtual event which gets converted into a | |
622 | * "real" interrupt event later. It does not need to be recorded for | |
623 | * replay purposes. | |
624 | */ | |
625 | static inline bool need_replay_interrupt(int interrupt_request) | |
626 | { | |
627 | #if defined(TARGET_I386) | |
628 | return !(interrupt_request & CPU_INTERRUPT_POLL); | |
629 | #else | |
630 | return true; | |
631 | #endif | |
632 | } | |
633 | ||
209b71b6 | 634 | static inline bool cpu_handle_interrupt(CPUState *cpu, |
c385e6e4 SF |
635 | TranslationBlock **last_tb) |
636 | { | |
637 | CPUClass *cc = CPU_GET_CLASS(cpu); | |
17b50b0c PD |
638 | |
639 | /* Clear the interrupt flag now since we're processing | |
640 | * cpu->interrupt_request and cpu->exit_request. | |
d84be02d DH |
641 | * Ensure zeroing happens before reading cpu->exit_request or |
642 | * cpu->interrupt_request (see also smp_wmb in cpu_exit()) | |
17b50b0c | 643 | */ |
d73415a3 | 644 | qatomic_mb_set(&cpu_neg(cpu)->icount_decr.u16.high, 0); |
c385e6e4 | 645 | |
d73415a3 | 646 | if (unlikely(qatomic_read(&cpu->interrupt_request))) { |
8d04fb55 JK |
647 | int interrupt_request; |
648 | qemu_mutex_lock_iothread(); | |
649 | interrupt_request = cpu->interrupt_request; | |
c385e6e4 SF |
650 | if (unlikely(cpu->singlestep_enabled & SSTEP_NOIRQ)) { |
651 | /* Mask out external interrupts for this step. */ | |
652 | interrupt_request &= ~CPU_INTERRUPT_SSTEP_MASK; | |
653 | } | |
654 | if (interrupt_request & CPU_INTERRUPT_DEBUG) { | |
655 | cpu->interrupt_request &= ~CPU_INTERRUPT_DEBUG; | |
656 | cpu->exception_index = EXCP_DEBUG; | |
8d04fb55 | 657 | qemu_mutex_unlock_iothread(); |
209b71b6 | 658 | return true; |
c385e6e4 SF |
659 | } |
660 | if (replay_mode == REPLAY_MODE_PLAY && !replay_has_interrupt()) { | |
661 | /* Do nothing */ | |
662 | } else if (interrupt_request & CPU_INTERRUPT_HALT) { | |
663 | replay_interrupt(); | |
664 | cpu->interrupt_request &= ~CPU_INTERRUPT_HALT; | |
665 | cpu->halted = 1; | |
666 | cpu->exception_index = EXCP_HLT; | |
8d04fb55 | 667 | qemu_mutex_unlock_iothread(); |
209b71b6 | 668 | return true; |
c385e6e4 SF |
669 | } |
670 | #if defined(TARGET_I386) | |
671 | else if (interrupt_request & CPU_INTERRUPT_INIT) { | |
672 | X86CPU *x86_cpu = X86_CPU(cpu); | |
673 | CPUArchState *env = &x86_cpu->env; | |
674 | replay_interrupt(); | |
65c9d60a | 675 | cpu_svm_check_intercept_param(env, SVM_EXIT_INIT, 0, 0); |
c385e6e4 SF |
676 | do_cpu_init(x86_cpu); |
677 | cpu->exception_index = EXCP_HALTED; | |
8d04fb55 | 678 | qemu_mutex_unlock_iothread(); |
209b71b6 | 679 | return true; |
c385e6e4 SF |
680 | } |
681 | #else | |
682 | else if (interrupt_request & CPU_INTERRUPT_RESET) { | |
683 | replay_interrupt(); | |
684 | cpu_reset(cpu); | |
8d04fb55 | 685 | qemu_mutex_unlock_iothread(); |
209b71b6 | 686 | return true; |
c385e6e4 SF |
687 | } |
688 | #endif | |
689 | /* The target hook has 3 exit conditions: | |
690 | False when the interrupt isn't processed, | |
691 | True when it is, and we should restart on a new TB, | |
692 | and via longjmp via cpu_loop_exit. */ | |
693 | else { | |
78271684 CF |
694 | if (cc->tcg_ops->cpu_exec_interrupt && |
695 | cc->tcg_ops->cpu_exec_interrupt(cpu, interrupt_request)) { | |
4084893d PD |
696 | if (need_replay_interrupt(interrupt_request)) { |
697 | replay_interrupt(); | |
698 | } | |
ba3c35d9 RH |
699 | /* |
700 | * After processing the interrupt, ensure an EXCP_DEBUG is | |
701 | * raised when single-stepping so that GDB doesn't miss the | |
702 | * next instruction. | |
703 | */ | |
704 | cpu->exception_index = | |
705 | (cpu->singlestep_enabled ? EXCP_DEBUG : -1); | |
c385e6e4 SF |
706 | *last_tb = NULL; |
707 | } | |
8b1fe3f4 SF |
708 | /* The target hook may have updated the 'cpu->interrupt_request'; |
709 | * reload the 'interrupt_request' value */ | |
710 | interrupt_request = cpu->interrupt_request; | |
c385e6e4 | 711 | } |
8b1fe3f4 | 712 | if (interrupt_request & CPU_INTERRUPT_EXITTB) { |
c385e6e4 SF |
713 | cpu->interrupt_request &= ~CPU_INTERRUPT_EXITTB; |
714 | /* ensure that no TB jump will be modified as | |
715 | the program flow was changed */ | |
716 | *last_tb = NULL; | |
717 | } | |
8d04fb55 JK |
718 | |
719 | /* If we exit via cpu_loop_exit/longjmp it is reset in cpu_exec */ | |
720 | qemu_mutex_unlock_iothread(); | |
c385e6e4 | 721 | } |
8d04fb55 | 722 | |
cfb2d02b | 723 | /* Finally, check if we need to exit to the main loop. */ |
d73415a3 | 724 | if (unlikely(qatomic_read(&cpu->exit_request)) |
740b1759 | 725 | || (icount_enabled() |
a11bbb6a | 726 | && (cpu->cflags_next_tb == -1 || cpu->cflags_next_tb & CF_USE_ICOUNT) |
5e140196 | 727 | && cpu_neg(cpu)->icount_decr.u16.low + cpu->icount_extra == 0)) { |
d73415a3 | 728 | qatomic_set(&cpu->exit_request, 0); |
5f3bdfd4 PD |
729 | if (cpu->exception_index == -1) { |
730 | cpu->exception_index = EXCP_INTERRUPT; | |
731 | } | |
209b71b6 | 732 | return true; |
c385e6e4 | 733 | } |
209b71b6 PB |
734 | |
735 | return false; | |
c385e6e4 SF |
736 | } |
737 | ||
928de9ee | 738 | static inline void cpu_loop_exec_tb(CPUState *cpu, TranslationBlock *tb, |
cfb2d02b | 739 | TranslationBlock **last_tb, int *tb_exit) |
928de9ee | 740 | { |
1aab16c2 | 741 | int32_t insns_left; |
928de9ee SF |
742 | |
743 | trace_exec_tb(tb, tb->pc); | |
eba40358 | 744 | tb = cpu_tb_exec(cpu, tb, tb_exit); |
1aab16c2 PB |
745 | if (*tb_exit != TB_EXIT_REQUESTED) { |
746 | *last_tb = tb; | |
747 | return; | |
748 | } | |
749 | ||
750 | *last_tb = NULL; | |
d73415a3 | 751 | insns_left = qatomic_read(&cpu_neg(cpu)->icount_decr.u32); |
1aab16c2 | 752 | if (insns_left < 0) { |
e5143e30 AB |
753 | /* Something asked us to stop executing chained TBs; just |
754 | * continue round the main loop. Whatever requested the exit | |
30f3dda2 | 755 | * will also have set something else (eg exit_request or |
17b50b0c PD |
756 | * interrupt_request) which will be handled by |
757 | * cpu_handle_interrupt. cpu_handle_interrupt will also | |
758 | * clear cpu->icount_decr.u16.high. | |
928de9ee | 759 | */ |
1aab16c2 | 760 | return; |
928de9ee | 761 | } |
1aab16c2 PB |
762 | |
763 | /* Instruction counter expired. */ | |
740b1759 | 764 | assert(icount_enabled()); |
1aab16c2 | 765 | #ifndef CONFIG_USER_ONLY |
eda5f7c6 | 766 | /* Ensure global icount has gone forward */ |
8191d368 | 767 | icount_update(cpu); |
eda5f7c6 | 768 | /* Refill decrementer and continue execution. */ |
bc662a33 | 769 | insns_left = MIN(CF_COUNT_MASK, cpu->icount_budget); |
5e140196 | 770 | cpu_neg(cpu)->icount_decr.u16.low = insns_left; |
eda5f7c6 | 771 | cpu->icount_extra = cpu->icount_budget - insns_left; |
bc662a33 AB |
772 | |
773 | /* | |
774 | * If the next tb has more instructions than we have left to | |
775 | * execute we need to ensure we find/generate a TB with exactly | |
776 | * insns_left instructions in it. | |
777 | */ | |
778 | if (!cpu->icount_extra && insns_left > 0 && insns_left < tb->icount) { | |
779 | cpu->cflags_next_tb = (tb->cflags & ~CF_COUNT_MASK) | insns_left; | |
928de9ee | 780 | } |
1aab16c2 | 781 | #endif |
928de9ee SF |
782 | } |
783 | ||
7d13299d FB |
784 | /* main execution loop */ |
785 | ||
ea3e9847 | 786 | int cpu_exec(CPUState *cpu) |
7d13299d | 787 | { |
97a8ea5a | 788 | CPUClass *cc = CPU_GET_CLASS(cpu); |
c385e6e4 | 789 | int ret; |
cfb2d02b | 790 | SyncClocks sc = { 0 }; |
c2aa5f81 | 791 | |
6f060969 PD |
792 | /* replay_interrupt may need current_cpu */ |
793 | current_cpu = cpu; | |
794 | ||
8b2d34e9 SF |
795 | if (cpu_handle_halt(cpu)) { |
796 | return EXCP_HALTED; | |
eda48c34 | 797 | } |
5a1e3cfc | 798 | |
79e2b9ae PB |
799 | rcu_read_lock(); |
800 | ||
035ba06c | 801 | cpu_exec_enter(cpu); |
9d27abd9 | 802 | |
c2aa5f81 ST |
803 | /* Calculate difference between guest clock and host clock. |
804 | * This delay includes the delay of the last cycle, so | |
805 | * what we have to do is sleep until it is 0. As for the | |
806 | * advance/delay we gain here, we try to fix it next time. | |
807 | */ | |
808 | init_delay_params(&sc, cpu); | |
809 | ||
4515e58d PB |
810 | /* prepare setjmp context for exception handling */ |
811 | if (sigsetjmp(cpu->jmp_env, 0) != 0) { | |
19a84318 | 812 | #if defined(__clang__) |
e6a41a04 PM |
813 | /* |
814 | * Some compilers wrongly smash all local variables after | |
815 | * siglongjmp (the spec requires that only non-volatile locals | |
816 | * which are changed between the sigsetjmp and siglongjmp are | |
817 | * permitted to be trashed). There were bug reports for gcc | |
818 | * 4.5.0 and clang. The bug is fixed in all versions of gcc | |
819 | * that we support, but is still unfixed in clang: | |
820 | * https://bugs.llvm.org/show_bug.cgi?id=21183 | |
821 | * | |
4515e58d | 822 | * Reload essential local variables here for those compilers. |
e6a41a04 PM |
823 | * Newer versions of gcc would complain about this code (-Wclobbered), |
824 | * so we only perform the workaround for clang. | |
825 | */ | |
4515e58d PB |
826 | cpu = current_cpu; |
827 | cc = CPU_GET_CLASS(cpu); | |
e6a41a04 PM |
828 | #else |
829 | /* | |
830 | * Non-buggy compilers preserve these locals; assert that | |
831 | * they have the correct value. | |
832 | */ | |
4515e58d PB |
833 | g_assert(cpu == current_cpu); |
834 | g_assert(cc == CPU_GET_CLASS(cpu)); | |
e6a41a04 PM |
835 | #endif |
836 | ||
0ac20318 EC |
837 | #ifndef CONFIG_SOFTMMU |
838 | tcg_debug_assert(!have_mmap_lock()); | |
839 | #endif | |
8d04fb55 JK |
840 | if (qemu_mutex_iothread_locked()) { |
841 | qemu_mutex_unlock_iothread(); | |
842 | } | |
e6d86bed EC |
843 | qemu_plugin_disable_mem_helpers(cpu); |
844 | ||
8fd3a9b8 | 845 | assert_no_pages_locked(); |
4515e58d PB |
846 | } |
847 | ||
848 | /* if an exception is pending, we execute it here */ | |
849 | while (!cpu_handle_exception(cpu, &ret)) { | |
850 | TranslationBlock *last_tb = NULL; | |
851 | int tb_exit = 0; | |
852 | ||
853 | while (!cpu_handle_interrupt(cpu, &last_tb)) { | |
9b990ee5 RH |
854 | uint32_t cflags = cpu->cflags_next_tb; |
855 | TranslationBlock *tb; | |
856 | ||
857 | /* When requested, use an exact setting for cflags for the next | |
858 | execution. This is used for icount, precise smc, and stop- | |
859 | after-access watchpoints. Since this request should never | |
860 | have CF_INVALID set, -1 is a convenient invalid value that | |
861 | does not require tcg headers for cpu_common_reset. */ | |
862 | if (cflags == -1) { | |
c0ae396a | 863 | cflags = curr_cflags(cpu); |
9b990ee5 RH |
864 | } else { |
865 | cpu->cflags_next_tb = -1; | |
866 | } | |
867 | ||
868 | tb = tb_find(cpu, last_tb, tb_exit, cflags); | |
cfb2d02b | 869 | cpu_loop_exec_tb(cpu, tb, &last_tb, &tb_exit); |
4515e58d PB |
870 | /* Try to align the host and virtual clocks |
871 | if the guest is in advance */ | |
872 | align_clocks(&sc, cpu); | |
7d13299d | 873 | } |
4515e58d | 874 | } |
3fb2ded1 | 875 | |
035ba06c | 876 | cpu_exec_exit(cpu); |
79e2b9ae | 877 | rcu_read_unlock(); |
1057eaa7 | 878 | |
7d13299d FB |
879 | return ret; |
880 | } | |
740b1759 | 881 | |
7df5e3d6 CF |
882 | void tcg_exec_realizefn(CPUState *cpu, Error **errp) |
883 | { | |
884 | static bool tcg_target_initialized; | |
885 | CPUClass *cc = CPU_GET_CLASS(cpu); | |
886 | ||
887 | if (!tcg_target_initialized) { | |
78271684 | 888 | cc->tcg_ops->initialize(); |
7df5e3d6 CF |
889 | tcg_target_initialized = true; |
890 | } | |
891 | tlb_init(cpu); | |
892 | qemu_plugin_vcpu_init_hook(cpu); | |
893 | ||
894 | #ifndef CONFIG_USER_ONLY | |
895 | tcg_iommu_init_notifier_list(cpu); | |
896 | #endif /* !CONFIG_USER_ONLY */ | |
897 | } | |
898 | ||
899 | /* undo the initializations in reverse order */ | |
900 | void tcg_exec_unrealizefn(CPUState *cpu) | |
901 | { | |
902 | #ifndef CONFIG_USER_ONLY | |
903 | tcg_iommu_free_notifier_list(cpu); | |
904 | #endif /* !CONFIG_USER_ONLY */ | |
905 | ||
906 | qemu_plugin_vcpu_exit_hook(cpu); | |
907 | tlb_destroy(cpu); | |
908 | } | |
909 | ||
740b1759 CF |
910 | #ifndef CONFIG_USER_ONLY |
911 | ||
912 | void dump_drift_info(void) | |
913 | { | |
914 | if (!icount_enabled()) { | |
915 | return; | |
916 | } | |
917 | ||
918 | qemu_printf("Host - Guest clock %"PRIi64" ms\n", | |
8191d368 | 919 | (cpu_get_clock() - icount_get()) / SCALE_MS); |
740b1759 CF |
920 | if (icount_align_option) { |
921 | qemu_printf("Max guest delay %"PRIi64" ms\n", | |
922 | -max_delay / SCALE_MS); | |
923 | qemu_printf("Max guest advance %"PRIi64" ms\n", | |
924 | max_advance / SCALE_MS); | |
925 | } else { | |
926 | qemu_printf("Max guest delay NA\n"); | |
927 | qemu_printf("Max guest advance NA\n"); | |
928 | } | |
929 | } | |
930 | ||
931 | #endif /* !CONFIG_USER_ONLY */ |