]>
Commit | Line | Data |
---|---|---|
7d13299d | 1 | /* |
e965fc38 | 2 | * emulator main execution loop |
5fafdf24 | 3 | * |
66321a11 | 4 | * Copyright (c) 2003-2005 Fabrice Bellard |
7d13299d | 5 | * |
3ef693a0 FB |
6 | * This library is free software; you can redistribute it and/or |
7 | * modify it under the terms of the GNU Lesser General Public | |
8 | * License as published by the Free Software Foundation; either | |
fb0343d5 | 9 | * version 2.1 of the License, or (at your option) any later version. |
7d13299d | 10 | * |
3ef693a0 FB |
11 | * This library is distributed in the hope that it will be useful, |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
14 | * Lesser General Public License for more details. | |
7d13299d | 15 | * |
3ef693a0 | 16 | * You should have received a copy of the GNU Lesser General Public |
8167ee88 | 17 | * License along with this library; if not, see <http://www.gnu.org/licenses/>. |
7d13299d | 18 | */ |
a8d25326 | 19 | |
7b31bbc2 | 20 | #include "qemu/osdep.h" |
740b1759 | 21 | #include "qemu/qemu-print.h" |
3a841ab5 | 22 | #include "qapi/error.h" |
3a841ab5 | 23 | #include "qapi/type-helpers.h" |
78271684 | 24 | #include "hw/core/tcg-cpu-ops.h" |
d9bb58e5 | 25 | #include "trace.h" |
76cad711 | 26 | #include "disas/disas.h" |
63c91552 | 27 | #include "exec/exec-all.h" |
dcb32f1d | 28 | #include "tcg/tcg.h" |
1de7afc9 | 29 | #include "qemu/atomic.h" |
79e2b9ae | 30 | #include "qemu/rcu.h" |
508127e2 | 31 | #include "exec/log.h" |
8d04fb55 | 32 | #include "qemu/main-loop.h" |
d2528bdc | 33 | #include "sysemu/cpus.h" |
740b1759 CF |
34 | #include "exec/cpu-all.h" |
35 | #include "sysemu/cpu-timers.h" | |
5b5968c4 | 36 | #include "exec/replay-core.h" |
3a841ab5 | 37 | #include "sysemu/tcg.h" |
a3e7f702 | 38 | #include "exec/helper-proto-common.h" |
a976a99a | 39 | #include "tb-jmp-cache.h" |
e5ceadff | 40 | #include "tb-hash.h" |
e5ceadff | 41 | #include "tb-context.h" |
5934660f | 42 | #include "internal-common.h" |
4c268d6d | 43 | #include "internal-target.h" |
3b28c270 PMD |
44 | #if defined(CONFIG_USER_ONLY) |
45 | #include "user-retaddr.h" | |
46 | #endif | |
c2aa5f81 ST |
47 | |
48 | /* -icount align implementation. */ | |
49 | ||
50 | typedef struct SyncClocks { | |
51 | int64_t diff_clk; | |
52 | int64_t last_cpu_icount; | |
7f7bc144 | 53 | int64_t realtime_clock; |
c2aa5f81 ST |
54 | } SyncClocks; |
55 | ||
56 | #if !defined(CONFIG_USER_ONLY) | |
57 | /* Allow the guest to have a max 3ms advance. | |
58 | * The difference between the 2 clocks could therefore | |
59 | * oscillate around 0. | |
60 | */ | |
61 | #define VM_CLOCK_ADVANCE 3000000 | |
7f7bc144 ST |
62 | #define THRESHOLD_REDUCE 1.5 |
63 | #define MAX_DELAY_PRINT_RATE 2000000000LL | |
64 | #define MAX_NB_PRINTS 100 | |
c2aa5f81 | 65 | |
00c9a5c2 PMD |
66 | int64_t max_delay; |
67 | int64_t max_advance; | |
740b1759 | 68 | |
5e140196 | 69 | static void align_clocks(SyncClocks *sc, CPUState *cpu) |
c2aa5f81 ST |
70 | { |
71 | int64_t cpu_icount; | |
72 | ||
73 | if (!icount_align_option) { | |
74 | return; | |
75 | } | |
76 | ||
a953b5fa | 77 | cpu_icount = cpu->icount_extra + cpu->neg.icount_decr.u16.low; |
8191d368 | 78 | sc->diff_clk += icount_to_ns(sc->last_cpu_icount - cpu_icount); |
c2aa5f81 ST |
79 | sc->last_cpu_icount = cpu_icount; |
80 | ||
81 | if (sc->diff_clk > VM_CLOCK_ADVANCE) { | |
82 | #ifndef _WIN32 | |
83 | struct timespec sleep_delay, rem_delay; | |
84 | sleep_delay.tv_sec = sc->diff_clk / 1000000000LL; | |
85 | sleep_delay.tv_nsec = sc->diff_clk % 1000000000LL; | |
86 | if (nanosleep(&sleep_delay, &rem_delay) < 0) { | |
a498d0ef | 87 | sc->diff_clk = rem_delay.tv_sec * 1000000000LL + rem_delay.tv_nsec; |
c2aa5f81 ST |
88 | } else { |
89 | sc->diff_clk = 0; | |
90 | } | |
91 | #else | |
92 | Sleep(sc->diff_clk / SCALE_MS); | |
93 | sc->diff_clk = 0; | |
94 | #endif | |
95 | } | |
96 | } | |
97 | ||
7f7bc144 ST |
98 | static void print_delay(const SyncClocks *sc) |
99 | { | |
100 | static float threshold_delay; | |
101 | static int64_t last_realtime_clock; | |
102 | static int nb_prints; | |
103 | ||
104 | if (icount_align_option && | |
105 | sc->realtime_clock - last_realtime_clock >= MAX_DELAY_PRINT_RATE && | |
106 | nb_prints < MAX_NB_PRINTS) { | |
107 | if ((-sc->diff_clk / (float)1000000000LL > threshold_delay) || | |
108 | (-sc->diff_clk / (float)1000000000LL < | |
109 | (threshold_delay - THRESHOLD_REDUCE))) { | |
110 | threshold_delay = (-sc->diff_clk / 1000000000LL) + 1; | |
740b1759 CF |
111 | qemu_printf("Warning: The guest is now late by %.1f to %.1f seconds\n", |
112 | threshold_delay - 1, | |
113 | threshold_delay); | |
7f7bc144 ST |
114 | nb_prints++; |
115 | last_realtime_clock = sc->realtime_clock; | |
116 | } | |
117 | } | |
118 | } | |
119 | ||
5e140196 | 120 | static void init_delay_params(SyncClocks *sc, CPUState *cpu) |
c2aa5f81 ST |
121 | { |
122 | if (!icount_align_option) { | |
123 | return; | |
124 | } | |
2e91cc62 PB |
125 | sc->realtime_clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT); |
126 | sc->diff_clk = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - sc->realtime_clock; | |
5e140196 | 127 | sc->last_cpu_icount |
a953b5fa | 128 | = cpu->icount_extra + cpu->neg.icount_decr.u16.low; |
27498bef ST |
129 | if (sc->diff_clk < max_delay) { |
130 | max_delay = sc->diff_clk; | |
131 | } | |
132 | if (sc->diff_clk > max_advance) { | |
133 | max_advance = sc->diff_clk; | |
134 | } | |
7f7bc144 ST |
135 | |
136 | /* Print every 2s max if the guest is late. We limit the number | |
137 | of printed messages to NB_PRINT_MAX(currently 100) */ | |
138 | print_delay(sc); | |
c2aa5f81 ST |
139 | } |
140 | #else | |
141 | static void align_clocks(SyncClocks *sc, const CPUState *cpu) | |
142 | { | |
143 | } | |
144 | ||
145 | static void init_delay_params(SyncClocks *sc, const CPUState *cpu) | |
146 | { | |
147 | } | |
148 | #endif /* CONFIG USER ONLY */ | |
7d13299d | 149 | |
b254c342 PMD |
150 | bool tcg_cflags_has(CPUState *cpu, uint32_t flags) |
151 | { | |
152 | return cpu->tcg_cflags & flags; | |
153 | } | |
154 | ||
155 | void tcg_cflags_set(CPUState *cpu, uint32_t flags) | |
156 | { | |
157 | cpu->tcg_cflags |= flags; | |
158 | } | |
159 | ||
043e35d9 RH |
160 | uint32_t curr_cflags(CPUState *cpu) |
161 | { | |
84f15616 RH |
162 | uint32_t cflags = cpu->tcg_cflags; |
163 | ||
04f5b647 | 164 | /* |
c2ffd754 RH |
165 | * Record gdb single-step. We should be exiting the TB by raising |
166 | * EXCP_DEBUG, but to simplify other tests, disable chaining too. | |
167 | * | |
04f5b647 RH |
168 | * For singlestep and -d nochain, suppress goto_tb so that |
169 | * we can log -d cpu,exec after every TB. | |
170 | */ | |
c2ffd754 RH |
171 | if (unlikely(cpu->singlestep_enabled)) { |
172 | cflags |= CF_NO_GOTO_TB | CF_NO_GOTO_PTR | CF_SINGLE_STEP | 1; | |
0e33928c | 173 | } else if (qatomic_read(&one_insn_per_tb)) { |
04f5b647 RH |
174 | cflags |= CF_NO_GOTO_TB | 1; |
175 | } else if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) { | |
fb957011 | 176 | cflags |= CF_NO_GOTO_TB; |
84f15616 RH |
177 | } |
178 | ||
179 | return cflags; | |
043e35d9 RH |
180 | } |
181 | ||
0c90ba16 | 182 | struct tb_desc { |
f0a08b09 AJ |
183 | vaddr pc; |
184 | uint64_t cs_base; | |
0c90ba16 | 185 | CPUArchState *env; |
93b99616 | 186 | tb_page_addr_t page_addr0; |
0c90ba16 RH |
187 | uint32_t flags; |
188 | uint32_t cflags; | |
0c90ba16 RH |
189 | }; |
190 | ||
191 | static bool tb_lookup_cmp(const void *p, const void *d) | |
192 | { | |
193 | const TranslationBlock *tb = p; | |
194 | const struct tb_desc *desc = d; | |
195 | ||
279513c7 | 196 | if ((tb_cflags(tb) & CF_PCREL || tb->pc == desc->pc) && |
28905cfb | 197 | tb_page_addr0(tb) == desc->page_addr0 && |
0c90ba16 RH |
198 | tb->cs_base == desc->cs_base && |
199 | tb->flags == desc->flags && | |
0c90ba16 RH |
200 | tb_cflags(tb) == desc->cflags) { |
201 | /* check next page if needed */ | |
28905cfb RH |
202 | tb_page_addr_t tb_phys_page1 = tb_page_addr1(tb); |
203 | if (tb_phys_page1 == -1) { | |
0c90ba16 RH |
204 | return true; |
205 | } else { | |
93b99616 | 206 | tb_page_addr_t phys_page1; |
f0a08b09 | 207 | vaddr virt_page1; |
0c90ba16 | 208 | |
9867b302 RH |
209 | /* |
210 | * We know that the first page matched, and an otherwise valid TB | |
211 | * encountered an incomplete instruction at the end of that page, | |
212 | * therefore we know that generating a new TB from the current PC | |
213 | * must also require reading from the next page -- even if the | |
214 | * second pages do not match, and therefore the resulting insn | |
215 | * is different for the new TB. Therefore any exception raised | |
216 | * here by the faulting lookup is not premature. | |
217 | */ | |
93b99616 RH |
218 | virt_page1 = TARGET_PAGE_ALIGN(desc->pc); |
219 | phys_page1 = get_page_addr_code(desc->env, virt_page1); | |
28905cfb | 220 | if (tb_phys_page1 == phys_page1) { |
0c90ba16 RH |
221 | return true; |
222 | } | |
223 | } | |
224 | } | |
225 | return false; | |
226 | } | |
227 | ||
f0a08b09 AJ |
228 | static TranslationBlock *tb_htable_lookup(CPUState *cpu, vaddr pc, |
229 | uint64_t cs_base, uint32_t flags, | |
0c90ba16 RH |
230 | uint32_t cflags) |
231 | { | |
232 | tb_page_addr_t phys_pc; | |
233 | struct tb_desc desc; | |
234 | uint32_t h; | |
235 | ||
b77af26e | 236 | desc.env = cpu_env(cpu); |
0c90ba16 RH |
237 | desc.cs_base = cs_base; |
238 | desc.flags = flags; | |
239 | desc.cflags = cflags; | |
0c90ba16 RH |
240 | desc.pc = pc; |
241 | phys_pc = get_page_addr_code(desc.env, pc); | |
242 | if (phys_pc == -1) { | |
243 | return NULL; | |
244 | } | |
93b99616 | 245 | desc.page_addr0 = phys_pc; |
4be79026 | 246 | h = tb_hash_func(phys_pc, (cflags & CF_PCREL ? 0 : pc), |
367189ef | 247 | flags, cs_base, cflags); |
0c90ba16 RH |
248 | return qht_lookup_custom(&tb_ctx.htable, &desc, h, tb_lookup_cmp); |
249 | } | |
250 | ||
632cb63d | 251 | /* Might cause an exception, so have a longjmp destination ready */ |
f0a08b09 AJ |
252 | static inline TranslationBlock *tb_lookup(CPUState *cpu, vaddr pc, |
253 | uint64_t cs_base, uint32_t flags, | |
254 | uint32_t cflags) | |
632cb63d RH |
255 | { |
256 | TranslationBlock *tb; | |
8ed558ec | 257 | CPUJumpCache *jc; |
632cb63d RH |
258 | uint32_t hash; |
259 | ||
260 | /* we should never be trying to look up an INVALID tb */ | |
261 | tcg_debug_assert(!(cflags & CF_INVALID)); | |
262 | ||
263 | hash = tb_jmp_cache_hash_func(pc); | |
8ed558ec | 264 | jc = cpu->tb_jmp_cache; |
632cb63d | 265 | |
d157e540 PB |
266 | tb = qatomic_read(&jc->array[hash].tb); |
267 | if (likely(tb && | |
268 | jc->array[hash].pc == pc && | |
269 | tb->cs_base == cs_base && | |
270 | tb->flags == flags && | |
271 | tb_cflags(tb) == cflags)) { | |
272 | goto hit; | |
632cb63d | 273 | } |
2dd5b7a1 | 274 | |
d157e540 PB |
275 | tb = tb_htable_lookup(cpu, pc, cs_base, flags, cflags); |
276 | if (tb == NULL) { | |
277 | return NULL; | |
278 | } | |
279 | ||
280 | jc->array[hash].pc = pc; | |
281 | qatomic_set(&jc->array[hash].tb, tb); | |
282 | ||
283 | hit: | |
284 | /* | |
285 | * As long as tb is not NULL, the contents are consistent. Therefore, | |
286 | * the virtual PC has to match for non-CF_PCREL translations. | |
287 | */ | |
288 | assert((tb_cflags(tb) & CF_PCREL) || tb->pc == pc); | |
632cb63d RH |
289 | return tb; |
290 | } | |
291 | ||
f0a08b09 | 292 | static void log_cpu_exec(vaddr pc, CPUState *cpu, |
fbf59aad | 293 | const TranslationBlock *tb) |
abb0cd93 | 294 | { |
fbf59aad | 295 | if (qemu_log_in_addr_range(pc)) { |
abb0cd93 | 296 | qemu_log_mask(CPU_LOG_EXEC, |
85314e13 | 297 | "Trace %d: %p [%08" PRIx64 |
e60a7d0d | 298 | "/%016" VADDR_PRIx "/%08x/%08x] %s\n", |
7eabad36 RH |
299 | cpu->cpu_index, tb->tc.ptr, tb->cs_base, pc, |
300 | tb->flags, tb->cflags, lookup_symbol(pc)); | |
abb0cd93 | 301 | |
abb0cd93 | 302 | if (qemu_loglevel_mask(CPU_LOG_TB_CPU)) { |
c60f599b | 303 | FILE *logfile = qemu_log_trylock(); |
78b54858 RH |
304 | if (logfile) { |
305 | int flags = 0; | |
abb0cd93 | 306 | |
78b54858 RH |
307 | if (qemu_loglevel_mask(CPU_LOG_TB_FPU)) { |
308 | flags |= CPU_DUMP_FPU; | |
309 | } | |
abb0cd93 | 310 | #if defined(TARGET_I386) |
78b54858 | 311 | flags |= CPU_DUMP_CCOP; |
abb0cd93 | 312 | #endif |
b84694de IK |
313 | if (qemu_loglevel_mask(CPU_LOG_TB_VPU)) { |
314 | flags |= CPU_DUMP_VPU; | |
315 | } | |
c769fbd7 | 316 | cpu_dump_state(cpu, logfile, flags); |
78b54858 RH |
317 | qemu_log_unlock(logfile); |
318 | } | |
abb0cd93 | 319 | } |
abb0cd93 RH |
320 | } |
321 | } | |
322 | ||
f0a08b09 | 323 | static bool check_for_breakpoints_slow(CPUState *cpu, vaddr pc, |
69993c4e | 324 | uint32_t *cflags) |
10c37828 RH |
325 | { |
326 | CPUBreakpoint *bp; | |
327 | bool match_page = false; | |
328 | ||
10c37828 RH |
329 | /* |
330 | * Singlestep overrides breakpoints. | |
331 | * This requirement is visible in the record-replay tests, where | |
332 | * we would fail to make forward progress in reverse-continue. | |
333 | * | |
334 | * TODO: gdb singlestep should only override gdb breakpoints, | |
335 | * so that one could (gdb) singlestep into the guest kernel's | |
336 | * architectural breakpoint handler. | |
337 | */ | |
338 | if (cpu->singlestep_enabled) { | |
339 | return false; | |
340 | } | |
341 | ||
342 | QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) { | |
343 | /* | |
344 | * If we have an exact pc match, trigger the breakpoint. | |
345 | * Otherwise, note matches within the page. | |
346 | */ | |
347 | if (pc == bp->pc) { | |
348 | bool match_bp = false; | |
349 | ||
350 | if (bp->flags & BP_GDB) { | |
351 | match_bp = true; | |
352 | } else if (bp->flags & BP_CPU) { | |
353 | #ifdef CONFIG_USER_ONLY | |
354 | g_assert_not_reached(); | |
355 | #else | |
991bd65d RH |
356 | const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops; |
357 | assert(tcg_ops->debug_check_breakpoint); | |
358 | match_bp = tcg_ops->debug_check_breakpoint(cpu); | |
10c37828 RH |
359 | #endif |
360 | } | |
361 | ||
362 | if (match_bp) { | |
363 | cpu->exception_index = EXCP_DEBUG; | |
364 | return true; | |
365 | } | |
366 | } else if (((pc ^ bp->pc) & TARGET_PAGE_MASK) == 0) { | |
367 | match_page = true; | |
368 | } | |
369 | } | |
370 | ||
371 | /* | |
372 | * Within the same page as a breakpoint, single-step, | |
373 | * returning to helper_lookup_tb_ptr after each insn looking | |
374 | * for the actual breakpoint. | |
375 | * | |
376 | * TODO: Perhaps better to record all of the TBs associated | |
377 | * with a given virtual page that contains a breakpoint, and | |
378 | * then invalidate them when a new overlapping breakpoint is | |
379 | * set on the page. Non-overlapping TBs would not be | |
380 | * invalidated, nor would any TB need to be invalidated as | |
381 | * breakpoints are removed. | |
382 | */ | |
383 | if (match_page) { | |
384 | *cflags = (*cflags & ~CF_COUNT_MASK) | CF_NO_GOTO_TB | 1; | |
385 | } | |
386 | return false; | |
387 | } | |
388 | ||
f0a08b09 | 389 | static inline bool check_for_breakpoints(CPUState *cpu, vaddr pc, |
69993c4e LL |
390 | uint32_t *cflags) |
391 | { | |
392 | return unlikely(!QTAILQ_EMPTY(&cpu->breakpoints)) && | |
393 | check_for_breakpoints_slow(cpu, pc, cflags); | |
394 | } | |
395 | ||
4288eb26 RH |
396 | /** |
397 | * helper_lookup_tb_ptr: quick check for next tb | |
398 | * @env: current cpu state | |
399 | * | |
400 | * Look for an existing TB matching the current cpu state. | |
401 | * If found, return the code pointer. If not found, return | |
402 | * the tcg epilogue so that we return into cpu_tb_exec. | |
403 | */ | |
404 | const void *HELPER(lookup_tb_ptr)(CPUArchState *env) | |
405 | { | |
406 | CPUState *cpu = env_cpu(env); | |
407 | TranslationBlock *tb; | |
bb5de525 AJ |
408 | vaddr pc; |
409 | uint64_t cs_base; | |
10c37828 | 410 | uint32_t flags, cflags; |
4288eb26 | 411 | |
62bcba83 PM |
412 | /* |
413 | * By definition we've just finished a TB, so I/O is OK. | |
414 | * Avoid the possibility of calling cpu_io_recompile() if | |
415 | * a page table walk triggered by tb_lookup() calling | |
416 | * probe_access_internal() happens to touch an MMIO device. | |
417 | * The next TB, if we chain to it, will clear the flag again. | |
418 | */ | |
419 | cpu->neg.can_do_io = true; | |
4288eb26 RH |
420 | cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); |
421 | ||
10c37828 RH |
422 | cflags = curr_cflags(cpu); |
423 | if (check_for_breakpoints(cpu, pc, &cflags)) { | |
424 | cpu_loop_exit(cpu); | |
425 | } | |
426 | ||
427 | tb = tb_lookup(cpu, pc, cs_base, flags, cflags); | |
4288eb26 RH |
428 | if (tb == NULL) { |
429 | return tcg_code_gen_epilogue; | |
430 | } | |
abb0cd93 | 431 | |
fbf59aad RH |
432 | if (qemu_loglevel_mask(CPU_LOG_TB_CPU | CPU_LOG_EXEC)) { |
433 | log_cpu_exec(pc, cpu, tb); | |
434 | } | |
abb0cd93 | 435 | |
4288eb26 RH |
436 | return tb->tc.ptr; |
437 | } | |
438 | ||
77211379 | 439 | /* Execute a TB, and fix up the CPU state afterwards if necessary */ |
c905a368 DB |
440 | /* |
441 | * Disable CFI checks. | |
442 | * TCG creates binary blobs at runtime, with the transformed code. | |
443 | * A TB is a blob of binary code, created at runtime and called with an | |
444 | * indirect function call. Since such function did not exist at compile time, | |
445 | * the CFI runtime has no way to verify its signature and would fail. | |
446 | * TCG is not considered a security-sensitive part of QEMU so this does not | |
447 | * affect the impact of CFI in environment with high security requirements | |
448 | */ | |
eba40358 RH |
449 | static inline TranslationBlock * QEMU_DISABLE_CFI |
450 | cpu_tb_exec(CPUState *cpu, TranslationBlock *itb, int *tb_exit) | |
77211379 | 451 | { |
819af24b SF |
452 | uintptr_t ret; |
453 | TranslationBlock *last_tb; | |
db0c51a3 | 454 | const void *tb_ptr = itb->tc.ptr; |
1a830635 | 455 | |
fbf59aad RH |
456 | if (qemu_loglevel_mask(CPU_LOG_TB_CPU | CPU_LOG_EXEC)) { |
457 | log_cpu_exec(log_pc(cpu, itb), cpu, itb); | |
458 | } | |
03afa5f8 | 459 | |
653b87eb | 460 | qemu_thread_jit_execute(); |
94956d7b | 461 | ret = tcg_qemu_tb_exec(cpu_env(cpu), tb_ptr); |
464dacf6 | 462 | cpu->neg.can_do_io = true; |
e04660af | 463 | qemu_plugin_disable_mem_helpers(cpu); |
eba40358 RH |
464 | /* |
465 | * TODO: Delay swapping back to the read-write region of the TB | |
466 | * until we actually need to modify the TB. The read-only copy, | |
467 | * coming from the rx region, shares the same host TLB entry as | |
468 | * the code that executed the exit_tb opcode that arrived here. | |
469 | * If we insist on touching both the RX and the RW pages, we | |
470 | * double the host TLB pressure. | |
471 | */ | |
472 | last_tb = tcg_splitwx_to_rw((void *)(ret & ~TB_EXIT_MASK)); | |
473 | *tb_exit = ret & TB_EXIT_MASK; | |
474 | ||
475 | trace_exec_tb_exit(last_tb, *tb_exit); | |
6db8b538 | 476 | |
eba40358 | 477 | if (*tb_exit > TB_EXIT_IDX1) { |
77211379 PM |
478 | /* We didn't start executing this TB (eg because the instruction |
479 | * counter hit zero); we must restore the guest PC to the address | |
480 | * of the start of the TB. | |
481 | */ | |
991bd65d RH |
482 | CPUClass *cc = cpu->cc; |
483 | const TCGCPUOps *tcg_ops = cc->tcg_ops; | |
fbf59aad | 484 | |
991bd65d RH |
485 | if (tcg_ops->synchronize_from_tb) { |
486 | tcg_ops->synchronize_from_tb(cpu, last_tb); | |
bdf7ae5b | 487 | } else { |
4be79026 | 488 | tcg_debug_assert(!(tb_cflags(last_tb) & CF_PCREL)); |
bdf7ae5b | 489 | assert(cc->set_pc); |
279513c7 | 490 | cc->set_pc(cpu, last_tb->pc); |
fbf59aad RH |
491 | } |
492 | if (qemu_loglevel_mask(CPU_LOG_EXEC)) { | |
f0a08b09 | 493 | vaddr pc = log_pc(cpu, last_tb); |
fbf59aad | 494 | if (qemu_log_in_addr_range(pc)) { |
e60a7d0d | 495 | qemu_log("Stopped execution of TB chain before %p [%016" |
f0a08b09 | 496 | VADDR_PRIx "] %s\n", |
fbf59aad RH |
497 | last_tb->tc.ptr, pc, lookup_symbol(pc)); |
498 | } | |
bdf7ae5b | 499 | } |
77211379 | 500 | } |
c9460d75 RH |
501 | |
502 | /* | |
503 | * If gdb single-step, and we haven't raised another exception, | |
504 | * raise a debug exception. Single-step with another exception | |
505 | * is handled in cpu_handle_exception. | |
506 | */ | |
507 | if (unlikely(cpu->singlestep_enabled) && cpu->exception_index == -1) { | |
508 | cpu->exception_index = EXCP_DEBUG; | |
509 | cpu_loop_exit(cpu); | |
510 | } | |
511 | ||
eba40358 | 512 | return last_tb; |
77211379 PM |
513 | } |
514 | ||
2e70f6ef | 515 | |
035ba06c EH |
516 | static void cpu_exec_enter(CPUState *cpu) |
517 | { | |
991bd65d | 518 | const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops; |
035ba06c | 519 | |
991bd65d RH |
520 | if (tcg_ops->cpu_exec_enter) { |
521 | tcg_ops->cpu_exec_enter(cpu); | |
80c4750b | 522 | } |
035ba06c EH |
523 | } |
524 | ||
525 | static void cpu_exec_exit(CPUState *cpu) | |
fdbc2b57 | 526 | { |
991bd65d | 527 | const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops; |
035ba06c | 528 | |
991bd65d RH |
529 | if (tcg_ops->cpu_exec_exit) { |
530 | tcg_ops->cpu_exec_exit(cpu); | |
80c4750b | 531 | } |
035ba06c EH |
532 | } |
533 | ||
cb62bd15 RH |
534 | static void cpu_exec_longjmp_cleanup(CPUState *cpu) |
535 | { | |
536 | /* Non-buggy compilers preserve this; assert the correct value. */ | |
537 | g_assert(cpu == current_cpu); | |
538 | ||
539 | #ifdef CONFIG_USER_ONLY | |
540 | clear_helper_retaddr(); | |
541 | if (have_mmap_lock()) { | |
542 | mmap_unlock(); | |
543 | } | |
deba7870 RH |
544 | #else |
545 | /* | |
546 | * For softmmu, a tlb_fill fault during translation will land here, | |
547 | * and we need to release any page locks held. In system mode we | |
548 | * have one tcg_ctx per thread, so we know it was this cpu doing | |
549 | * the translation. | |
550 | * | |
551 | * Alternative 1: Install a cleanup to be called via an exception | |
552 | * handling safe longjmp. It seems plausible that all our hosts | |
553 | * support such a thing. We'd have to properly register unwind info | |
554 | * for the JIT for EH, rather that just for GDB. | |
555 | * | |
556 | * Alternative 2: Set and restore cpu->jmp_env in tb_gen_code to | |
557 | * capture the cpu_loop_exit longjmp, perform the cleanup, and | |
558 | * jump again to arrive here. | |
559 | */ | |
560 | if (tcg_ctx->gen_tb) { | |
561 | tb_unlock_pages(tcg_ctx->gen_tb); | |
562 | tcg_ctx->gen_tb = NULL; | |
563 | } | |
cb62bd15 | 564 | #endif |
195801d7 SH |
565 | if (bql_locked()) { |
566 | bql_unlock(); | |
cb62bd15 RH |
567 | } |
568 | assert_no_pages_locked(); | |
569 | } | |
570 | ||
035ba06c EH |
571 | void cpu_exec_step_atomic(CPUState *cpu) |
572 | { | |
b77af26e | 573 | CPUArchState *env = cpu_env(cpu); |
fdbc2b57 | 574 | TranslationBlock *tb; |
bb5de525 AJ |
575 | vaddr pc; |
576 | uint64_t cs_base; | |
258afb48 | 577 | uint32_t flags, cflags; |
eba40358 | 578 | int tb_exit; |
fdbc2b57 | 579 | |
08e73c48 | 580 | if (sigsetjmp(cpu->jmp_env, 0) == 0) { |
886cc689 | 581 | start_exclusive(); |
bfff072c DC |
582 | g_assert(cpu == current_cpu); |
583 | g_assert(!cpu->running); | |
584 | cpu->running = true; | |
886cc689 | 585 | |
6f04cb1c | 586 | cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); |
6f04cb1c | 587 | |
258afb48 RH |
588 | cflags = curr_cflags(cpu); |
589 | /* Execute in a serial context. */ | |
590 | cflags &= ~CF_PARALLEL; | |
591 | /* After 1 insn, return and release the exclusive lock. */ | |
592 | cflags |= CF_NO_GOTO_TB | CF_NO_GOTO_PTR | 1; | |
10c37828 RH |
593 | /* |
594 | * No need to check_for_breakpoints here. | |
595 | * We only arrive in cpu_exec_step_atomic after beginning execution | |
596 | * of an insn that includes an atomic operation we can't handle. | |
597 | * Any breakpoint for this insn will have been recognized earlier. | |
598 | */ | |
258afb48 RH |
599 | |
600 | tb = tb_lookup(cpu, pc, cs_base, flags, cflags); | |
4e2ca83e EC |
601 | if (tb == NULL) { |
602 | mmap_lock(); | |
95590e24 | 603 | tb = tb_gen_code(cpu, pc, cs_base, flags, cflags); |
4e2ca83e EC |
604 | mmap_unlock(); |
605 | } | |
08e73c48 | 606 | |
035ba06c | 607 | cpu_exec_enter(cpu); |
08e73c48 | 608 | /* execute the generated code */ |
4e2ca83e | 609 | trace_exec_tb(tb, pc); |
eba40358 | 610 | cpu_tb_exec(cpu, tb, &tb_exit); |
035ba06c | 611 | cpu_exec_exit(cpu); |
08e73c48 | 612 | } else { |
cb62bd15 | 613 | cpu_exec_longjmp_cleanup(cpu); |
08e73c48 | 614 | } |
426eeecd | 615 | |
886cc689 AB |
616 | /* |
617 | * As we start the exclusive region before codegen we must still | |
618 | * be in the region if we longjump out of either the codegen or | |
619 | * the execution. | |
620 | */ | |
621 | g_assert(cpu_in_exclusive_context(cpu)); | |
bfff072c | 622 | cpu->running = false; |
886cc689 | 623 | end_exclusive(); |
fdbc2b57 RH |
624 | } |
625 | ||
a8583393 RH |
626 | void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr) |
627 | { | |
2fd2e78d RH |
628 | /* |
629 | * Get the rx view of the structure, from which we find the | |
630 | * executable code address, and tb_target_set_jmp_target can | |
631 | * produce a pc-relative displacement to jmp_target_addr[n]. | |
632 | */ | |
633 | const TranslationBlock *c_tb = tcg_splitwx_to_rx(tb); | |
634 | uintptr_t offset = tb->jmp_insn_offset[n]; | |
635 | uintptr_t jmp_rx = (uintptr_t)tb->tc.ptr + offset; | |
636 | uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff; | |
637 | ||
9da6079b | 638 | tb->jmp_target_addr[n] = addr; |
2fd2e78d | 639 | tb_target_set_jmp_target(c_tb, n, jmp_rx, jmp_rw); |
a8583393 RH |
640 | } |
641 | ||
a8583393 RH |
642 | static inline void tb_add_jump(TranslationBlock *tb, int n, |
643 | TranslationBlock *tb_next) | |
644 | { | |
194125e3 EC |
645 | uintptr_t old; |
646 | ||
653b87eb | 647 | qemu_thread_jit_write(); |
a8583393 | 648 | assert(n < ARRAY_SIZE(tb->jmp_list_next)); |
194125e3 EC |
649 | qemu_spin_lock(&tb_next->jmp_lock); |
650 | ||
651 | /* make sure the destination TB is valid */ | |
652 | if (tb_next->cflags & CF_INVALID) { | |
653 | goto out_unlock_next; | |
654 | } | |
655 | /* Atomically claim the jump destination slot only if it was NULL */ | |
d73415a3 SH |
656 | old = qatomic_cmpxchg(&tb->jmp_dest[n], (uintptr_t)NULL, |
657 | (uintptr_t)tb_next); | |
194125e3 EC |
658 | if (old) { |
659 | goto out_unlock_next; | |
a8583393 | 660 | } |
194125e3 EC |
661 | |
662 | /* patch the native jump address */ | |
663 | tb_set_jmp_target(tb, n, (uintptr_t)tb_next->tc.ptr); | |
664 | ||
665 | /* add in TB jmp list */ | |
666 | tb->jmp_list_next[n] = tb_next->jmp_list_head; | |
667 | tb_next->jmp_list_head = (uintptr_t)tb | n; | |
668 | ||
669 | qemu_spin_unlock(&tb_next->jmp_lock); | |
670 | ||
fbf59aad RH |
671 | qemu_log_mask(CPU_LOG_EXEC, "Linking TBs %p index %d -> %p\n", |
672 | tb->tc.ptr, n, tb_next->tc.ptr); | |
194125e3 | 673 | return; |
a8583393 | 674 | |
194125e3 EC |
675 | out_unlock_next: |
676 | qemu_spin_unlock(&tb_next->jmp_lock); | |
677 | return; | |
a8583393 RH |
678 | } |
679 | ||
8b2d34e9 SF |
680 | static inline bool cpu_handle_halt(CPUState *cpu) |
681 | { | |
0596fa11 | 682 | #ifndef CONFIG_USER_ONLY |
8b2d34e9 | 683 | if (cpu->halted) { |
aa6fb657 PMD |
684 | const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops; |
685 | ||
aa6fb657 PMD |
686 | if (tcg_ops->cpu_exec_halt) { |
687 | tcg_ops->cpu_exec_halt(cpu); | |
688 | } | |
8b2d34e9 | 689 | if (!cpu_has_work(cpu)) { |
8b2d34e9 SF |
690 | return true; |
691 | } | |
692 | ||
693 | cpu->halted = 0; | |
694 | } | |
0596fa11 | 695 | #endif /* !CONFIG_USER_ONLY */ |
8b2d34e9 SF |
696 | |
697 | return false; | |
698 | } | |
699 | ||
ea284766 | 700 | static inline void cpu_handle_debug_exception(CPUState *cpu) |
1009d2ed | 701 | { |
991bd65d | 702 | const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops; |
1009d2ed JK |
703 | CPUWatchpoint *wp; |
704 | ||
ff4700b0 AF |
705 | if (!cpu->watchpoint_hit) { |
706 | QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) { | |
1009d2ed JK |
707 | wp->flags &= ~BP_WATCHPOINT_HIT; |
708 | } | |
709 | } | |
86025ee4 | 710 | |
991bd65d RH |
711 | if (tcg_ops->debug_excp_handler) { |
712 | tcg_ops->debug_excp_handler(cpu); | |
710384d0 | 713 | } |
1009d2ed JK |
714 | } |
715 | ||
ea284766 SF |
716 | static inline bool cpu_handle_exception(CPUState *cpu, int *ret) |
717 | { | |
17b50b0c PD |
718 | if (cpu->exception_index < 0) { |
719 | #ifndef CONFIG_USER_ONLY | |
720 | if (replay_has_exception() | |
a953b5fa | 721 | && cpu->neg.icount_decr.u16.low + cpu->icount_extra == 0) { |
a11bbb6a | 722 | /* Execute just one insn to trigger exception pending in the log */ |
c3e97f64 | 723 | cpu->cflags_next_tb = (curr_cflags(cpu) & ~CF_USE_ICOUNT) |
cf9b5790 | 724 | | CF_NOIRQ | 1; |
17b50b0c PD |
725 | } |
726 | #endif | |
a11bbb6a | 727 | return false; |
17b50b0c | 728 | } |
991bd65d | 729 | |
17b50b0c PD |
730 | if (cpu->exception_index >= EXCP_INTERRUPT) { |
731 | /* exit request from the cpu execution loop */ | |
732 | *ret = cpu->exception_index; | |
733 | if (*ret == EXCP_DEBUG) { | |
734 | cpu_handle_debug_exception(cpu); | |
735 | } | |
736 | cpu->exception_index = -1; | |
737 | return true; | |
991bd65d RH |
738 | } |
739 | ||
ea284766 | 740 | #if defined(CONFIG_USER_ONLY) |
991bd65d RH |
741 | /* |
742 | * If user mode only, we simulate a fake exception which will be | |
743 | * handled outside the cpu execution loop. | |
744 | */ | |
ea284766 | 745 | #if defined(TARGET_I386) |
991bd65d RH |
746 | const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops; |
747 | tcg_ops->fake_user_interrupt(cpu); | |
12096421 | 748 | #endif /* TARGET_I386 */ |
991bd65d RH |
749 | *ret = cpu->exception_index; |
750 | cpu->exception_index = -1; | |
751 | return true; | |
17b50b0c | 752 | #else |
991bd65d RH |
753 | if (replay_exception()) { |
754 | const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops; | |
a7ba744f | 755 | |
991bd65d RH |
756 | bql_lock(); |
757 | tcg_ops->do_interrupt(cpu); | |
758 | bql_unlock(); | |
759 | cpu->exception_index = -1; | |
760 | ||
761 | if (unlikely(cpu->singlestep_enabled)) { | |
762 | /* | |
763 | * After processing the exception, ensure an EXCP_DEBUG is | |
764 | * raised when single-stepping so that GDB doesn't miss the | |
765 | * next instruction. | |
766 | */ | |
767 | *ret = EXCP_DEBUG; | |
768 | cpu_handle_debug_exception(cpu); | |
ea284766 | 769 | return true; |
ea284766 | 770 | } |
991bd65d RH |
771 | } else if (!replay_has_interrupt()) { |
772 | /* give a chance to iothread in replay mode */ | |
773 | *ret = EXCP_INTERRUPT; | |
774 | return true; | |
ea284766 | 775 | } |
991bd65d | 776 | #endif |
ea284766 SF |
777 | |
778 | return false; | |
779 | } | |
780 | ||
93c6091b PMD |
781 | static inline bool icount_exit_request(CPUState *cpu) |
782 | { | |
783 | if (!icount_enabled()) { | |
784 | return false; | |
785 | } | |
786 | if (cpu->cflags_next_tb != -1 && !(cpu->cflags_next_tb & CF_USE_ICOUNT)) { | |
787 | return false; | |
788 | } | |
789 | return cpu->neg.icount_decr.u16.low + cpu->icount_extra == 0; | |
790 | } | |
791 | ||
209b71b6 | 792 | static inline bool cpu_handle_interrupt(CPUState *cpu, |
c385e6e4 SF |
793 | TranslationBlock **last_tb) |
794 | { | |
aff0e204 AB |
795 | /* |
796 | * If we have requested custom cflags with CF_NOIRQ we should | |
797 | * skip checking here. Any pending interrupts will get picked up | |
798 | * by the next TB we execute under normal cflags. | |
799 | */ | |
800 | if (cpu->cflags_next_tb != -1 && cpu->cflags_next_tb & CF_NOIRQ) { | |
801 | return false; | |
802 | } | |
803 | ||
17b50b0c PD |
804 | /* Clear the interrupt flag now since we're processing |
805 | * cpu->interrupt_request and cpu->exit_request. | |
d84be02d DH |
806 | * Ensure zeroing happens before reading cpu->exit_request or |
807 | * cpu->interrupt_request (see also smp_wmb in cpu_exit()) | |
17b50b0c | 808 | */ |
a953b5fa | 809 | qatomic_set_mb(&cpu->neg.icount_decr.u16.high, 0); |
c385e6e4 | 810 | |
d73415a3 | 811 | if (unlikely(qatomic_read(&cpu->interrupt_request))) { |
8d04fb55 | 812 | int interrupt_request; |
195801d7 | 813 | bql_lock(); |
8d04fb55 | 814 | interrupt_request = cpu->interrupt_request; |
c385e6e4 SF |
815 | if (unlikely(cpu->singlestep_enabled & SSTEP_NOIRQ)) { |
816 | /* Mask out external interrupts for this step. */ | |
817 | interrupt_request &= ~CPU_INTERRUPT_SSTEP_MASK; | |
818 | } | |
819 | if (interrupt_request & CPU_INTERRUPT_DEBUG) { | |
820 | cpu->interrupt_request &= ~CPU_INTERRUPT_DEBUG; | |
821 | cpu->exception_index = EXCP_DEBUG; | |
195801d7 | 822 | bql_unlock(); |
209b71b6 | 823 | return true; |
c385e6e4 | 824 | } |
77c0fc4e | 825 | #if !defined(CONFIG_USER_ONLY) |
c385e6e4 SF |
826 | if (replay_mode == REPLAY_MODE_PLAY && !replay_has_interrupt()) { |
827 | /* Do nothing */ | |
828 | } else if (interrupt_request & CPU_INTERRUPT_HALT) { | |
829 | replay_interrupt(); | |
830 | cpu->interrupt_request &= ~CPU_INTERRUPT_HALT; | |
831 | cpu->halted = 1; | |
832 | cpu->exception_index = EXCP_HLT; | |
195801d7 | 833 | bql_unlock(); |
209b71b6 | 834 | return true; |
c385e6e4 SF |
835 | } |
836 | #if defined(TARGET_I386) | |
837 | else if (interrupt_request & CPU_INTERRUPT_INIT) { | |
838 | X86CPU *x86_cpu = X86_CPU(cpu); | |
839 | CPUArchState *env = &x86_cpu->env; | |
840 | replay_interrupt(); | |
65c9d60a | 841 | cpu_svm_check_intercept_param(env, SVM_EXIT_INIT, 0, 0); |
c385e6e4 SF |
842 | do_cpu_init(x86_cpu); |
843 | cpu->exception_index = EXCP_HALTED; | |
195801d7 | 844 | bql_unlock(); |
209b71b6 | 845 | return true; |
c385e6e4 SF |
846 | } |
847 | #else | |
848 | else if (interrupt_request & CPU_INTERRUPT_RESET) { | |
849 | replay_interrupt(); | |
850 | cpu_reset(cpu); | |
195801d7 | 851 | bql_unlock(); |
209b71b6 | 852 | return true; |
c385e6e4 | 853 | } |
77c0fc4e | 854 | #endif /* !TARGET_I386 */ |
c385e6e4 SF |
855 | /* The target hook has 3 exit conditions: |
856 | False when the interrupt isn't processed, | |
857 | True when it is, and we should restart on a new TB, | |
858 | and via longjmp via cpu_loop_exit. */ | |
859 | else { | |
991bd65d | 860 | const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops; |
77c0fc4e | 861 | |
991bd65d RH |
862 | if (tcg_ops->cpu_exec_interrupt && |
863 | tcg_ops->cpu_exec_interrupt(cpu, interrupt_request)) { | |
b7e9a4a9 RH |
864 | if (!tcg_ops->need_replay_interrupt || |
865 | tcg_ops->need_replay_interrupt(interrupt_request)) { | |
4084893d PD |
866 | replay_interrupt(); |
867 | } | |
ba3c35d9 RH |
868 | /* |
869 | * After processing the interrupt, ensure an EXCP_DEBUG is | |
870 | * raised when single-stepping so that GDB doesn't miss the | |
871 | * next instruction. | |
872 | */ | |
5b7b197c LM |
873 | if (unlikely(cpu->singlestep_enabled)) { |
874 | cpu->exception_index = EXCP_DEBUG; | |
195801d7 | 875 | bql_unlock(); |
5b7b197c LM |
876 | return true; |
877 | } | |
878 | cpu->exception_index = -1; | |
c385e6e4 SF |
879 | *last_tb = NULL; |
880 | } | |
8b1fe3f4 SF |
881 | /* The target hook may have updated the 'cpu->interrupt_request'; |
882 | * reload the 'interrupt_request' value */ | |
883 | interrupt_request = cpu->interrupt_request; | |
c385e6e4 | 884 | } |
77c0fc4e | 885 | #endif /* !CONFIG_USER_ONLY */ |
8b1fe3f4 | 886 | if (interrupt_request & CPU_INTERRUPT_EXITTB) { |
c385e6e4 SF |
887 | cpu->interrupt_request &= ~CPU_INTERRUPT_EXITTB; |
888 | /* ensure that no TB jump will be modified as | |
889 | the program flow was changed */ | |
890 | *last_tb = NULL; | |
891 | } | |
8d04fb55 JK |
892 | |
893 | /* If we exit via cpu_loop_exit/longjmp it is reset in cpu_exec */ | |
195801d7 | 894 | bql_unlock(); |
c385e6e4 | 895 | } |
8d04fb55 | 896 | |
cfb2d02b | 897 | /* Finally, check if we need to exit to the main loop. */ |
93c6091b | 898 | if (unlikely(qatomic_read(&cpu->exit_request)) || icount_exit_request(cpu)) { |
d73415a3 | 899 | qatomic_set(&cpu->exit_request, 0); |
5f3bdfd4 PD |
900 | if (cpu->exception_index == -1) { |
901 | cpu->exception_index = EXCP_INTERRUPT; | |
902 | } | |
209b71b6 | 903 | return true; |
c385e6e4 | 904 | } |
209b71b6 PB |
905 | |
906 | return false; | |
c385e6e4 SF |
907 | } |
908 | ||
928de9ee | 909 | static inline void cpu_loop_exec_tb(CPUState *cpu, TranslationBlock *tb, |
f0a08b09 AJ |
910 | vaddr pc, TranslationBlock **last_tb, |
911 | int *tb_exit) | |
928de9ee | 912 | { |
fbf59aad | 913 | trace_exec_tb(tb, pc); |
eba40358 | 914 | tb = cpu_tb_exec(cpu, tb, tb_exit); |
1aab16c2 PB |
915 | if (*tb_exit != TB_EXIT_REQUESTED) { |
916 | *last_tb = tb; | |
917 | return; | |
918 | } | |
919 | ||
920 | *last_tb = NULL; | |
0650fc1e | 921 | if (cpu_loop_exit_requested(cpu)) { |
e5143e30 AB |
922 | /* Something asked us to stop executing chained TBs; just |
923 | * continue round the main loop. Whatever requested the exit | |
30f3dda2 | 924 | * will also have set something else (eg exit_request or |
17b50b0c PD |
925 | * interrupt_request) which will be handled by |
926 | * cpu_handle_interrupt. cpu_handle_interrupt will also | |
927 | * clear cpu->icount_decr.u16.high. | |
928de9ee | 928 | */ |
1aab16c2 | 929 | return; |
928de9ee | 930 | } |
1aab16c2 PB |
931 | |
932 | /* Instruction counter expired. */ | |
740b1759 | 933 | assert(icount_enabled()); |
1aab16c2 | 934 | #ifndef CONFIG_USER_ONLY |
eda5f7c6 | 935 | /* Ensure global icount has gone forward */ |
8191d368 | 936 | icount_update(cpu); |
eda5f7c6 | 937 | /* Refill decrementer and continue execution. */ |
0650fc1e | 938 | int32_t insns_left = MIN(0xffff, cpu->icount_budget); |
a953b5fa | 939 | cpu->neg.icount_decr.u16.low = insns_left; |
eda5f7c6 | 940 | cpu->icount_extra = cpu->icount_budget - insns_left; |
bc662a33 AB |
941 | |
942 | /* | |
943 | * If the next tb has more instructions than we have left to | |
944 | * execute we need to ensure we find/generate a TB with exactly | |
945 | * insns_left instructions in it. | |
946 | */ | |
c8cf47a9 PM |
947 | if (insns_left > 0 && insns_left < tb->icount) { |
948 | assert(insns_left <= CF_COUNT_MASK); | |
949 | assert(cpu->icount_extra == 0); | |
bc662a33 | 950 | cpu->cflags_next_tb = (tb->cflags & ~CF_COUNT_MASK) | insns_left; |
928de9ee | 951 | } |
1aab16c2 | 952 | #endif |
928de9ee SF |
953 | } |
954 | ||
7d13299d FB |
955 | /* main execution loop */ |
956 | ||
61710a7e RH |
957 | static int __attribute__((noinline)) |
958 | cpu_exec_loop(CPUState *cpu, SyncClocks *sc) | |
7d13299d | 959 | { |
c385e6e4 | 960 | int ret; |
4515e58d PB |
961 | |
962 | /* if an exception is pending, we execute it here */ | |
963 | while (!cpu_handle_exception(cpu, &ret)) { | |
964 | TranslationBlock *last_tb = NULL; | |
965 | int tb_exit = 0; | |
966 | ||
967 | while (!cpu_handle_interrupt(cpu, &last_tb)) { | |
9b990ee5 | 968 | TranslationBlock *tb; |
bb5de525 AJ |
969 | vaddr pc; |
970 | uint64_t cs_base; | |
11c1d5f8 RH |
971 | uint32_t flags, cflags; |
972 | ||
b77af26e | 973 | cpu_get_tb_cpu_state(cpu_env(cpu), &pc, &cs_base, &flags); |
10c37828 | 974 | |
11c1d5f8 RH |
975 | /* |
976 | * When requested, use an exact setting for cflags for the next | |
977 | * execution. This is used for icount, precise smc, and stop- | |
978 | * after-access watchpoints. Since this request should never | |
979 | * have CF_INVALID set, -1 is a convenient invalid value that | |
980 | * does not require tcg headers for cpu_common_reset. | |
981 | */ | |
982 | cflags = cpu->cflags_next_tb; | |
9b990ee5 | 983 | if (cflags == -1) { |
c0ae396a | 984 | cflags = curr_cflags(cpu); |
9b990ee5 RH |
985 | } else { |
986 | cpu->cflags_next_tb = -1; | |
987 | } | |
988 | ||
10c37828 RH |
989 | if (check_for_breakpoints(cpu, pc, &cflags)) { |
990 | break; | |
991 | } | |
11c1d5f8 RH |
992 | |
993 | tb = tb_lookup(cpu, pc, cs_base, flags, cflags); | |
994 | if (tb == NULL) { | |
3371802f | 995 | CPUJumpCache *jc; |
a976a99a RH |
996 | uint32_t h; |
997 | ||
11c1d5f8 RH |
998 | mmap_lock(); |
999 | tb = tb_gen_code(cpu, pc, cs_base, flags, cflags); | |
1000 | mmap_unlock(); | |
3371802f | 1001 | |
11c1d5f8 RH |
1002 | /* |
1003 | * We add the TB in the virtual pc hash table | |
1004 | * for the fast lookup | |
1005 | */ | |
a976a99a | 1006 | h = tb_jmp_cache_hash_func(pc); |
3371802f | 1007 | jc = cpu->tb_jmp_cache; |
d157e540 PB |
1008 | jc->array[h].pc = pc; |
1009 | qatomic_set(&jc->array[h].tb, tb); | |
11c1d5f8 RH |
1010 | } |
1011 | ||
1012 | #ifndef CONFIG_USER_ONLY | |
1013 | /* | |
1014 | * We don't take care of direct jumps when address mapping | |
1015 | * changes in system emulation. So it's not safe to make a | |
1016 | * direct jump to a TB spanning two pages because the mapping | |
1017 | * for the second page can change. | |
1018 | */ | |
28905cfb | 1019 | if (tb_page_addr1(tb) != -1) { |
11c1d5f8 RH |
1020 | last_tb = NULL; |
1021 | } | |
1022 | #endif | |
1023 | /* See if we can patch the calling TB. */ | |
1024 | if (last_tb) { | |
1025 | tb_add_jump(last_tb, tb_exit, tb); | |
1026 | } | |
1027 | ||
fbf59aad | 1028 | cpu_loop_exec_tb(cpu, tb, pc, &last_tb, &tb_exit); |
11c1d5f8 | 1029 | |
4515e58d PB |
1030 | /* Try to align the host and virtual clocks |
1031 | if the guest is in advance */ | |
61710a7e RH |
1032 | align_clocks(sc, cpu); |
1033 | } | |
1034 | } | |
1035 | return ret; | |
1036 | } | |
1037 | ||
1038 | static int cpu_exec_setjmp(CPUState *cpu, SyncClocks *sc) | |
1039 | { | |
1040 | /* Prepare setjmp context for exception handling. */ | |
1041 | if (unlikely(sigsetjmp(cpu->jmp_env, 0) != 0)) { | |
cb62bd15 | 1042 | cpu_exec_longjmp_cleanup(cpu); |
61710a7e RH |
1043 | } |
1044 | ||
1045 | return cpu_exec_loop(cpu, sc); | |
1046 | } | |
1047 | ||
1048 | int cpu_exec(CPUState *cpu) | |
1049 | { | |
1050 | int ret; | |
1051 | SyncClocks sc = { 0 }; | |
1052 | ||
1053 | /* replay_interrupt may need current_cpu */ | |
1054 | current_cpu = cpu; | |
1055 | ||
1056 | if (cpu_handle_halt(cpu)) { | |
1057 | return EXCP_HALTED; | |
4515e58d | 1058 | } |
3fb2ded1 | 1059 | |
f5e9362a | 1060 | RCU_READ_LOCK_GUARD(); |
61710a7e RH |
1061 | cpu_exec_enter(cpu); |
1062 | ||
1063 | /* | |
1064 | * Calculate difference between guest clock and host clock. | |
1065 | * This delay includes the delay of the last cycle, so | |
1066 | * what we have to do is sleep until it is 0. As for the | |
1067 | * advance/delay we gain here, we try to fix it next time. | |
1068 | */ | |
1069 | init_delay_params(&sc, cpu); | |
1070 | ||
1071 | ret = cpu_exec_setjmp(cpu, &sc); | |
1072 | ||
035ba06c | 1073 | cpu_exec_exit(cpu); |
7d13299d FB |
1074 | return ret; |
1075 | } | |
740b1759 | 1076 | |
fa312f2e | 1077 | bool tcg_exec_realizefn(CPUState *cpu, Error **errp) |
7df5e3d6 CF |
1078 | { |
1079 | static bool tcg_target_initialized; | |
7df5e3d6 CF |
1080 | |
1081 | if (!tcg_target_initialized) { | |
991bd65d | 1082 | cpu->cc->tcg_ops->initialize(); |
7df5e3d6 CF |
1083 | tcg_target_initialized = true; |
1084 | } | |
7df5e3d6 | 1085 | |
4e4fa6c1 RH |
1086 | cpu->tb_jmp_cache = g_new0(CPUJumpCache, 1); |
1087 | tlb_init(cpu); | |
7df5e3d6 CF |
1088 | #ifndef CONFIG_USER_ONLY |
1089 | tcg_iommu_init_notifier_list(cpu); | |
1090 | #endif /* !CONFIG_USER_ONLY */ | |
4e4fa6c1 | 1091 | /* qemu_plugin_vcpu_init_hook delayed until cpu_index assigned. */ |
fa312f2e PMD |
1092 | |
1093 | return true; | |
7df5e3d6 CF |
1094 | } |
1095 | ||
1096 | /* undo the initializations in reverse order */ | |
1097 | void tcg_exec_unrealizefn(CPUState *cpu) | |
1098 | { | |
1099 | #ifndef CONFIG_USER_ONLY | |
1100 | tcg_iommu_free_notifier_list(cpu); | |
1101 | #endif /* !CONFIG_USER_ONLY */ | |
1102 | ||
7df5e3d6 | 1103 | tlb_destroy(cpu); |
4731f89b | 1104 | g_free_rcu(cpu->tb_jmp_cache, rcu); |
7df5e3d6 | 1105 | } |