cpus.c

   1 /*
   2  * QEMU System Emulator
   3  *
   4  * Copyright (c) 2003-2008 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24
  25 /* Needed early for CONFIG_BSD etc. */
  26 #include "config-host.h"
  27
  28 #include "monitor/monitor.h"
  29 #include "sysemu/sysemu.h"
  30 #include "exec/gdbstub.h"
  31 #include "sysemu/dma.h"
  32 #include "sysemu/kvm.h"
  33 #include "qmp-commands.h"
  34
  35 #include "qemu/thread.h"
  36 #include "sysemu/cpus.h"
  37 #include "sysemu/qtest.h"
  38 #include "qemu/main-loop.h"
  39 #include "qemu/bitmap.h"
  40
  41 #ifndef _WIN32
  42 #include "qemu/compatfd.h"
  43 #endif
  44
  45 #ifdef CONFIG_LINUX
  46
  47 #include <sys/prctl.h>
  48
  49 #ifndef PR_MCE_KILL
  50 #define PR_MCE_KILL 33
  51 #endif
  52
  53 #ifndef PR_MCE_KILL_SET
  54 #define PR_MCE_KILL_SET 1
  55 #endif
  56
  57 #ifndef PR_MCE_KILL_EARLY
  58 #define PR_MCE_KILL_EARLY 1
  59 #endif
  60
  61 #endif /* CONFIG_LINUX */
  62
  63 static CPUState *next_cpu;
  64
  65 static bool cpu_thread_is_idle(CPUState *cpu)
  66 {
  67     if (cpu->stop || cpu->queued_work_first) {
  68         return false;
  69     }
  70     if (cpu->stopped || !runstate_is_running()) {
  71         return true;
  72     }
  73     if (!cpu->halted || qemu_cpu_has_work(cpu) ||
  74         kvm_halt_in_kernel()) {
  75         return false;
  76     }
  77     return true;
  78 }
  79
  80 static bool all_cpu_threads_idle(void)
  81 {
  82     CPUState *cpu;
  83
  84     for (cpu = first_cpu; cpu != NULL; cpu = cpu->next_cpu) {
  85         if (!cpu_thread_is_idle(cpu)) {
  86             return false;
  87         }
  88     }
  89     return true;
  90 }
  91
  92 /***********************************************************/
  93 /* guest cycle counter */
  94
  95 /* Conversion factor from emulated instructions to virtual clock ticks.  */
  96 static int icount_time_shift;
  97 /* Arbitrarily pick 1MIPS as the minimum allowable speed.  */
  98 #define MAX_ICOUNT_SHIFT 10
  99 /* Compensate for varying guest execution speed.  */
 100 static int64_t qemu_icount_bias;
 101 static QEMUTimer *icount_rt_timer;
 102 static QEMUTimer *icount_vm_timer;
 103 static QEMUTimer *icount_warp_timer;
 104 static int64_t vm_clock_warp_start;
 105 static int64_t qemu_icount;
 106
 107 typedef struct TimersState {
 108     int64_t cpu_ticks_prev;
 109     int64_t cpu_ticks_offset;
 110     int64_t cpu_clock_offset;
 111     int32_t cpu_ticks_enabled;
 112     int64_t dummy;
 113 } TimersState;
 114
 115 TimersState timers_state;
 116
 117 /* Return the virtual CPU time, based on the instruction counter.  */
 118 int64_t cpu_get_icount(void)
 119 {
 120     int64_t icount;
 121     CPUState *cpu = current_cpu;
 122
 123     icount = qemu_icount;
 124     if (cpu) {
 125         CPUArchState *env = cpu->env_ptr;
 126         if (!can_do_io(env)) {
 127             fprintf(stderr, "Bad clock read\n");
 128         }
 129         icount -= (env->icount_decr.u16.low + env->icount_extra);
 130     }
 131     return qemu_icount_bias + (icount << icount_time_shift);
 132 }
 133
 134 /* return the host CPU cycle counter and handle stop/restart */
 135 int64_t cpu_get_ticks(void)
 136 {
 137     if (use_icount) {
 138         return cpu_get_icount();
 139     }
 140     if (!timers_state.cpu_ticks_enabled) {
 141         return timers_state.cpu_ticks_offset;
 142     } else {
 143         int64_t ticks;
 144         ticks = cpu_get_real_ticks();
 145         if (timers_state.cpu_ticks_prev > ticks) {
 146             /* Note: non increasing ticks may happen if the host uses
 147                software suspend */
 148             timers_state.cpu_ticks_offset += timers_state.cpu_ticks_prev - ticks;
 149         }
 150         timers_state.cpu_ticks_prev = ticks;
 151         return ticks + timers_state.cpu_ticks_offset;
 152     }
 153 }
 154
 155 /* return the host CPU monotonic timer and handle stop/restart */
 156 int64_t cpu_get_clock(void)
 157 {
 158     int64_t ti;
 159     if (!timers_state.cpu_ticks_enabled) {
 160         return timers_state.cpu_clock_offset;
 161     } else {
 162         ti = get_clock();
 163         return ti + timers_state.cpu_clock_offset;
 164     }
 165 }
 166
 167 /* enable cpu_get_ticks() */
 168 void cpu_enable_ticks(void)
 169 {
 170     if (!timers_state.cpu_ticks_enabled) {
 171         timers_state.cpu_ticks_offset -= cpu_get_real_ticks();
 172         timers_state.cpu_clock_offset -= get_clock();
 173         timers_state.cpu_ticks_enabled = 1;
 174     }
 175 }
 176
 177 /* disable cpu_get_ticks() : the clock is stopped. You must not call
 178    cpu_get_ticks() after that.  */
 179 void cpu_disable_ticks(void)
 180 {
 181     if (timers_state.cpu_ticks_enabled) {
 182         timers_state.cpu_ticks_offset = cpu_get_ticks();
 183         timers_state.cpu_clock_offset = cpu_get_clock();
 184         timers_state.cpu_ticks_enabled = 0;
 185     }
 186 }
 187
 188 /* Correlation between real and virtual time is always going to be
 189    fairly approximate, so ignore small variation.
 190    When the guest is idle real and virtual time will be aligned in
 191    the IO wait loop.  */
 192 #define ICOUNT_WOBBLE (get_ticks_per_sec() / 10)
 193
 194 static void icount_adjust(void)
 195 {
 196     int64_t cur_time;
 197     int64_t cur_icount;
 198     int64_t delta;
 199     static int64_t last_delta;
 200     /* If the VM is not running, then do nothing.  */
 201     if (!runstate_is_running()) {
 202         return;
 203     }
 204     cur_time = cpu_get_clock();
 205     cur_icount = qemu_get_clock_ns(vm_clock);
 206     delta = cur_icount - cur_time;
 207     /* FIXME: This is a very crude algorithm, somewhat prone to oscillation.  */
 208     if (delta > 0
 209         && last_delta + ICOUNT_WOBBLE < delta * 2
 210         && icount_time_shift > 0) {
 211         /* The guest is getting too far ahead.  Slow time down.  */
 212         icount_time_shift--;
 213     }
 214     if (delta < 0
 215         && last_delta - ICOUNT_WOBBLE > delta * 2
 216         && icount_time_shift < MAX_ICOUNT_SHIFT) {
 217         /* The guest is getting too far behind.  Speed time up.  */
 218         icount_time_shift++;
 219     }
 220     last_delta = delta;
 221     qemu_icount_bias = cur_icount - (qemu_icount << icount_time_shift);
 222 }
 223
 224 static void icount_adjust_rt(void *opaque)
 225 {
 226     qemu_mod_timer(icount_rt_timer,
 227                    qemu_get_clock_ms(rt_clock) + 1000);
 228     icount_adjust();
 229 }
 230
 231 static void icount_adjust_vm(void *opaque)
 232 {
 233     qemu_mod_timer(icount_vm_timer,
 234                    qemu_get_clock_ns(vm_clock) + get_ticks_per_sec() / 10);
 235     icount_adjust();
 236 }
 237
 238 static int64_t qemu_icount_round(int64_t count)
 239 {
 240     return (count + (1 << icount_time_shift) - 1) >> icount_time_shift;
 241 }
 242
 243 static void icount_warp_rt(void *opaque)
 244 {
 245     if (vm_clock_warp_start == -1) {
 246         return;
 247     }
 248
 249     if (runstate_is_running()) {
 250         int64_t clock = qemu_get_clock_ns(rt_clock);
 251         int64_t warp_delta = clock - vm_clock_warp_start;
 252         if (use_icount == 1) {
 253             qemu_icount_bias += warp_delta;
 254         } else {
 255             /*
 256              * In adaptive mode, do not let the vm_clock run too
 257              * far ahead of real time.
 258              */
 259             int64_t cur_time = cpu_get_clock();
 260             int64_t cur_icount = qemu_get_clock_ns(vm_clock);
 261             int64_t delta = cur_time - cur_icount;
 262             qemu_icount_bias += MIN(warp_delta, delta);
 263         }
 264         if (qemu_clock_expired(vm_clock)) {
 265             qemu_notify_event();
 266         }
 267     }
 268     vm_clock_warp_start = -1;
 269 }
 270
 271 void qtest_clock_warp(int64_t dest)
 272 {
 273     int64_t clock = qemu_get_clock_ns(vm_clock);
 274     assert(qtest_enabled());
 275     while (clock < dest) {
 276         int64_t deadline = qemu_clock_deadline(vm_clock);
 277         int64_t warp = MIN(dest - clock, deadline);
 278         qemu_icount_bias += warp;
 279         qemu_run_timers(vm_clock);
 280         clock = qemu_get_clock_ns(vm_clock);
 281     }
 282     qemu_notify_event();
 283 }
 284
 285 void qemu_clock_warp(QEMUClock *clock)
 286 {
 287     int64_t deadline;
 288
 289     /*
 290      * There are too many global variables to make the "warp" behavior
 291      * applicable to other clocks.  But a clock argument removes the
 292      * need for if statements all over the place.
 293      */
 294     if (clock != vm_clock || !use_icount) {
 295         return;
 296     }
 297
 298     /*
 299      * If the CPUs have been sleeping, advance the vm_clock timer now.  This
 300      * ensures that the deadline for the timer is computed correctly below.
 301      * This also makes sure that the insn counter is synchronized before the
 302      * CPU starts running, in case the CPU is woken by an event other than
 303      * the earliest vm_clock timer.
 304      */
 305     icount_warp_rt(NULL);
 306     if (!all_cpu_threads_idle() || !qemu_clock_has_timers(vm_clock)) {
 307         qemu_del_timer(icount_warp_timer);
 308         return;
 309     }
 310
 311     if (qtest_enabled()) {
 312         /* When testing, qtest commands advance icount.  */
 313         return;
 314     }
 315
 316     vm_clock_warp_start = qemu_get_clock_ns(rt_clock);
 317     deadline = qemu_clock_deadline(vm_clock);
 318     if (deadline > 0) {
 319         /*
 320          * Ensure the vm_clock proceeds even when the virtual CPU goes to
 321          * sleep.  Otherwise, the CPU might be waiting for a future timer
 322          * interrupt to wake it up, but the interrupt never comes because
 323          * the vCPU isn't running any insns and thus doesn't advance the
 324          * vm_clock.
 325          *
 326          * An extreme solution for this problem would be to never let VCPUs
 327          * sleep in icount mode if there is a pending vm_clock timer; rather
 328          * time could just advance to the next vm_clock event.  Instead, we
 329          * do stop VCPUs and only advance vm_clock after some "real" time,
 330          * (related to the time left until the next event) has passed.  This
 331          * rt_clock timer will do this.  This avoids that the warps are too
 332          * visible externally---for example, you will not be sending network
 333          * packets continuously instead of every 100ms.
 334          */
 335         qemu_mod_timer(icount_warp_timer, vm_clock_warp_start + deadline);
 336     } else {
 337         qemu_notify_event();
 338     }
 339 }
 340
 341 static const VMStateDescription vmstate_timers = {
 342     .name = "timer",
 343     .version_id = 2,
 344     .minimum_version_id = 1,
 345     .minimum_version_id_old = 1,
 346     .fields      = (VMStateField[]) {
 347         VMSTATE_INT64(cpu_ticks_offset, TimersState),
 348         VMSTATE_INT64(dummy, TimersState),
 349         VMSTATE_INT64_V(cpu_clock_offset, TimersState, 2),
 350         VMSTATE_END_OF_LIST()
 351     }
 352 };
 353
 354 void configure_icount(const char *option)
 355 {
 356     vmstate_register(NULL, 0, &vmstate_timers, &timers_state);
 357     if (!option) {
 358         return;
 359     }
 360
 361     icount_warp_timer = qemu_new_timer_ns(rt_clock, icount_warp_rt, NULL);
 362     if (strcmp(option, "auto") != 0) {
 363         icount_time_shift = strtol(option, NULL, 0);
 364         use_icount = 1;
 365         return;
 366     }
 367
 368     use_icount = 2;
 369
 370     /* 125MIPS seems a reasonable initial guess at the guest speed.
 371        It will be corrected fairly quickly anyway.  */
 372     icount_time_shift = 3;
 373
 374     /* Have both realtime and virtual time triggers for speed adjustment.
 375        The realtime trigger catches emulated time passing too slowly,
 376        the virtual time trigger catches emulated time passing too fast.
 377        Realtime triggers occur even when idle, so use them less frequently
 378        than VM triggers.  */
 379     icount_rt_timer = qemu_new_timer_ms(rt_clock, icount_adjust_rt, NULL);
 380     qemu_mod_timer(icount_rt_timer,
 381                    qemu_get_clock_ms(rt_clock) + 1000);
 382     icount_vm_timer = qemu_new_timer_ns(vm_clock, icount_adjust_vm, NULL);
 383     qemu_mod_timer(icount_vm_timer,
 384                    qemu_get_clock_ns(vm_clock) + get_ticks_per_sec() / 10);
 385 }
 386
 387 /***********************************************************/
 388 void hw_error(const char *fmt, ...)
 389 {
 390     va_list ap;
 391     CPUState *cpu;
 392
 393     va_start(ap, fmt);
 394     fprintf(stderr, "qemu: hardware error: ");
 395     vfprintf(stderr, fmt, ap);
 396     fprintf(stderr, "\n");
 397     for (cpu = first_cpu; cpu != NULL; cpu = cpu->next_cpu) {
 398         fprintf(stderr, "CPU #%d:\n", cpu->cpu_index);
 399         cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU);
 400     }
 401     va_end(ap);
 402     abort();
 403 }
 404
 405 void cpu_synchronize_all_states(void)
 406 {
 407     CPUState *cpu;
 408
 409     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
 410         cpu_synchronize_state(cpu);
 411     }
 412 }
 413
 414 void cpu_synchronize_all_post_reset(void)
 415 {
 416     CPUState *cpu;
 417
 418     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
 419         cpu_synchronize_post_reset(cpu);
 420     }
 421 }
 422
 423 void cpu_synchronize_all_post_init(void)
 424 {
 425     CPUState *cpu;
 426
 427     for (cpu = first_cpu; cpu; cpu = cpu->next_cpu) {
 428         cpu_synchronize_post_init(cpu);
 429     }
 430 }
 431
 432 bool cpu_is_stopped(CPUState *cpu)
 433 {
 434     return !runstate_is_running() || cpu->stopped;
 435 }
 436
 437 static int do_vm_stop(RunState state)
 438 {
 439     int ret = 0;
 440
 441     if (runstate_is_running()) {
 442         cpu_disable_ticks();
 443         pause_all_vcpus();
 444         runstate_set(state);
 445         vm_state_notify(0, state);
 446         bdrv_drain_all();
 447         ret = bdrv_flush_all();
 448         monitor_protocol_event(QEVENT_STOP, NULL);
 449     }
 450
 451     return ret;
 452 }
 453
 454 static bool cpu_can_run(CPUState *cpu)
 455 {
 456     if (cpu->stop) {
 457         return false;
 458     }
 459     if (cpu->stopped || !runstate_is_running()) {
 460         return false;
 461     }
 462     return true;
 463 }
 464
 465 static void cpu_handle_guest_debug(CPUState *cpu)
 466 {
 467     gdb_set_stop_cpu(cpu);
 468     qemu_system_debug_request();
 469     cpu->stopped = true;
 470 }
 471
 472 static void cpu_signal(int sig)
 473 {
 474     if (current_cpu) {
 475         cpu_exit(current_cpu);
 476     }
 477     exit_request = 1;
 478 }
 479
 480 #ifdef CONFIG_LINUX
 481 static void sigbus_reraise(void)
 482 {
 483     sigset_t set;
 484     struct sigaction action;
 485
 486     memset(&action, 0, sizeof(action));
 487     action.sa_handler = SIG_DFL;
 488     if (!sigaction(SIGBUS, &action, NULL)) {
 489         raise(SIGBUS);
 490         sigemptyset(&set);
 491         sigaddset(&set, SIGBUS);
 492         sigprocmask(SIG_UNBLOCK, &set, NULL);
 493     }
 494     perror("Failed to re-raise SIGBUS!\n");
 495     abort();
 496 }
 497
 498 static void sigbus_handler(int n, struct qemu_signalfd_siginfo *siginfo,
 499                            void *ctx)
 500 {
 501     if (kvm_on_sigbus(siginfo->ssi_code,
 502                       (void *)(intptr_t)siginfo->ssi_addr)) {
 503         sigbus_reraise();
 504     }
 505 }
 506
 507 static void qemu_init_sigbus(void)
 508 {
 509     struct sigaction action;
 510
 511     memset(&action, 0, sizeof(action));
 512     action.sa_flags = SA_SIGINFO;
 513     action.sa_sigaction = (void (*)(int, siginfo_t*, void*))sigbus_handler;
 514     sigaction(SIGBUS, &action, NULL);
 515
 516     prctl(PR_MCE_KILL, PR_MCE_KILL_SET, PR_MCE_KILL_EARLY, 0, 0);
 517 }
 518
 519 static void qemu_kvm_eat_signals(CPUState *cpu)
 520 {
 521     struct timespec ts = { 0, 0 };
 522     siginfo_t siginfo;
 523     sigset_t waitset;
 524     sigset_t chkset;
 525     int r;
 526
 527     sigemptyset(&waitset);
 528     sigaddset(&waitset, SIG_IPI);
 529     sigaddset(&waitset, SIGBUS);
 530
 531     do {
 532         r = sigtimedwait(&waitset, &siginfo, &ts);
 533         if (r == -1 && !(errno == EAGAIN || errno == EINTR)) {
 534             perror("sigtimedwait");
 535             exit(1);
 536         }
 537
 538         switch (r) {
 539         case SIGBUS:
 540             if (kvm_on_sigbus_vcpu(cpu, siginfo.si_code, siginfo.si_addr)) {
 541                 sigbus_reraise();
 542             }
 543             break;
 544         default:
 545             break;
 546         }
 547
 548         r = sigpending(&chkset);
 549         if (r == -1) {
 550             perror("sigpending");
 551             exit(1);
 552         }
 553     } while (sigismember(&chkset, SIG_IPI) || sigismember(&chkset, SIGBUS));
 554 }
 555
 556 #else /* !CONFIG_LINUX */
 557
 558 static void qemu_init_sigbus(void)
 559 {
 560 }
 561
 562 static void qemu_kvm_eat_signals(CPUState *cpu)
 563 {
 564 }
 565 #endif /* !CONFIG_LINUX */
 566
 567 #ifndef _WIN32
 568 static void dummy_signal(int sig)
 569 {
 570 }
 571
 572 static void qemu_kvm_init_cpu_signals(CPUState *cpu)
 573 {
 574     int r;
 575     sigset_t set;
 576     struct sigaction sigact;
 577
 578     memset(&sigact, 0, sizeof(sigact));
 579     sigact.sa_handler = dummy_signal;
 580     sigaction(SIG_IPI, &sigact, NULL);
 581
 582     pthread_sigmask(SIG_BLOCK, NULL, &set);
 583     sigdelset(&set, SIG_IPI);
 584     sigdelset(&set, SIGBUS);
 585     r = kvm_set_signal_mask(cpu, &set);
 586     if (r) {
 587         fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r));
 588         exit(1);
 589     }
 590 }
 591
 592 static void qemu_tcg_init_cpu_signals(void)
 593 {
 594     sigset_t set;
 595     struct sigaction sigact;
 596
 597     memset(&sigact, 0, sizeof(sigact));
 598     sigact.sa_handler = cpu_signal;
 599     sigaction(SIG_IPI, &sigact, NULL);
 600
 601     sigemptyset(&set);
 602     sigaddset(&set, SIG_IPI);
 603     pthread_sigmask(SIG_UNBLOCK, &set, NULL);
 604 }
 605
 606 #else /* _WIN32 */
 607 static void qemu_kvm_init_cpu_signals(CPUState *cpu)
 608 {
 609     abort();
 610 }
 611
 612 static void qemu_tcg_init_cpu_signals(void)
 613 {
 614 }
 615 #endif /* _WIN32 */
 616
 617 static QemuMutex qemu_global_mutex;
 618 static QemuCond qemu_io_proceeded_cond;
 619 static bool iothread_requesting_mutex;
 620
 621 static QemuThread io_thread;
 622
 623 static QemuThread *tcg_cpu_thread;
 624 static QemuCond *tcg_halt_cond;
 625
 626 /* cpu creation */
 627 static QemuCond qemu_cpu_cond;
 628 /* system init */
 629 static QemuCond qemu_pause_cond;
 630 static QemuCond qemu_work_cond;
 631
 632 void qemu_init_cpu_loop(void)
 633 {
 634     qemu_init_sigbus();
 635     qemu_cond_init(&qemu_cpu_cond);
 636     qemu_cond_init(&qemu_pause_cond);
 637     qemu_cond_init(&qemu_work_cond);
 638     qemu_cond_init(&qemu_io_proceeded_cond);
 639     qemu_mutex_init(&qemu_global_mutex);
 640
 641     qemu_thread_get_self(&io_thread);
 642 }
 643
 644 void run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
 645 {
 646     struct qemu_work_item wi;
 647
 648     if (qemu_cpu_is_self(cpu)) {
 649         func(data);
 650         return;
 651     }
 652
 653     wi.func = func;
 654     wi.data = data;
 655     if (cpu->queued_work_first == NULL) {
 656         cpu->queued_work_first = &wi;
 657     } else {
 658         cpu->queued_work_last->next = &wi;
 659     }
 660     cpu->queued_work_last = &wi;
 661     wi.next = NULL;
 662     wi.done = false;
 663
 664     qemu_cpu_kick(cpu);
 665     while (!wi.done) {
 666         CPUState *self_cpu = current_cpu;
 667
 668         qemu_cond_wait(&qemu_work_cond, &qemu_global_mutex);
 669         current_cpu = self_cpu;
 670     }
 671 }
 672
 673 static void flush_queued_work(CPUState *cpu)
 674 {
 675     struct qemu_work_item *wi;
 676
 677     if (cpu->queued_work_first == NULL) {
 678         return;
 679     }
 680
 681     while ((wi = cpu->queued_work_first)) {
 682         cpu->queued_work_first = wi->next;
 683         wi->func(wi->data);
 684         wi->done = true;
 685     }
 686     cpu->queued_work_last = NULL;
 687     qemu_cond_broadcast(&qemu_work_cond);
 688 }
 689
 690 static void qemu_wait_io_event_common(CPUState *cpu)
 691 {
 692     if (cpu->stop) {
 693         cpu->stop = false;
 694         cpu->stopped = true;
 695         qemu_cond_signal(&qemu_pause_cond);
 696     }
 697     flush_queued_work(cpu);
 698     cpu->thread_kicked = false;
 699 }
 700
 701 static void qemu_tcg_wait_io_event(void)
 702 {
 703     CPUState *cpu;
 704
 705     while (all_cpu_threads_idle()) {
 706        /* Start accounting real time to the virtual clock if the CPUs
 707           are idle.  */
 708         qemu_clock_warp(vm_clock);
 709         qemu_cond_wait(tcg_halt_cond, &qemu_global_mutex);
 710     }
 711
 712     while (iothread_requesting_mutex) {
 713         qemu_cond_wait(&qemu_io_proceeded_cond, &qemu_global_mutex);
 714     }
 715
 716     for (cpu = first_cpu; cpu != NULL; cpu = cpu->next_cpu) {
 717         qemu_wait_io_event_common(cpu);
 718     }
 719 }
 720
 721 static void qemu_kvm_wait_io_event(CPUState *cpu)
 722 {
 723     while (cpu_thread_is_idle(cpu)) {
 724         qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
 725     }
 726
 727     qemu_kvm_eat_signals(cpu);
 728     qemu_wait_io_event_common(cpu);
 729 }
 730
 731 static void *qemu_kvm_cpu_thread_fn(void *arg)
 732 {
 733     CPUState *cpu = arg;
 734     int r;
 735
 736     qemu_mutex_lock(&qemu_global_mutex);
 737     qemu_thread_get_self(cpu->thread);
 738     cpu->thread_id = qemu_get_thread_id();
 739     current_cpu = cpu;
 740
 741     r = kvm_init_vcpu(cpu);
 742     if (r < 0) {
 743         fprintf(stderr, "kvm_init_vcpu failed: %s\n", strerror(-r));
 744         exit(1);
 745     }
 746
 747     qemu_kvm_init_cpu_signals(cpu);
 748
 749     /* signal CPU creation */
 750     cpu->created = true;
 751     qemu_cond_signal(&qemu_cpu_cond);
 752
 753     while (1) {
 754         if (cpu_can_run(cpu)) {
 755             r = kvm_cpu_exec(cpu);
 756             if (r == EXCP_DEBUG) {
 757                 cpu_handle_guest_debug(cpu);
 758             }
 759         }
 760         qemu_kvm_wait_io_event(cpu);
 761     }
 762
 763     return NULL;
 764 }
 765
 766 static void *qemu_dummy_cpu_thread_fn(void *arg)
 767 {
 768 #ifdef _WIN32
 769     fprintf(stderr, "qtest is not supported under Windows\n");
 770     exit(1);
 771 #else
 772     CPUState *cpu = arg;
 773     sigset_t waitset;
 774     int r;
 775
 776     qemu_mutex_lock_iothread();
 777     qemu_thread_get_self(cpu->thread);
 778     cpu->thread_id = qemu_get_thread_id();
 779
 780     sigemptyset(&waitset);
 781     sigaddset(&waitset, SIG_IPI);
 782
 783     /* signal CPU creation */
 784     cpu->created = true;
 785     qemu_cond_signal(&qemu_cpu_cond);
 786
 787     current_cpu = cpu;
 788     while (1) {
 789         current_cpu = NULL;
 790         qemu_mutex_unlock_iothread();
 791         do {
 792             int sig;
 793             r = sigwait(&waitset, &sig);
 794         } while (r == -1 && (errno == EAGAIN || errno == EINTR));
 795         if (r == -1) {
 796             perror("sigwait");
 797             exit(1);
 798         }
 799         qemu_mutex_lock_iothread();
 800         current_cpu = cpu;
 801         qemu_wait_io_event_common(cpu);
 802     }
 803
 804     return NULL;
 805 #endif
 806 }
 807
 808 static void tcg_exec_all(void);
 809
 810 static void tcg_signal_cpu_creation(CPUState *cpu, void *data)
 811 {
 812     cpu->thread_id = qemu_get_thread_id();
 813     cpu->created = true;
 814 }
 815
 816 static void *qemu_tcg_cpu_thread_fn(void *arg)
 817 {
 818     CPUState *cpu = arg;
 819
 820     qemu_tcg_init_cpu_signals();
 821     qemu_thread_get_self(cpu->thread);
 822
 823     qemu_mutex_lock(&qemu_global_mutex);
 824     qemu_for_each_cpu(tcg_signal_cpu_creation, NULL);
 825     qemu_cond_signal(&qemu_cpu_cond);
 826
 827     /* wait for initial kick-off after machine start */
 828     while (first_cpu->stopped) {
 829         qemu_cond_wait(tcg_halt_cond, &qemu_global_mutex);
 830
 831         /* process any pending work */
 832         for (cpu = first_cpu; cpu != NULL; cpu = cpu->next_cpu) {
 833             qemu_wait_io_event_common(cpu);
 834         }
 835     }
 836
 837     while (1) {
 838         tcg_exec_all();
 839         if (use_icount && qemu_clock_deadline(vm_clock) <= 0) {
 840             qemu_notify_event();
 841         }
 842         qemu_tcg_wait_io_event();
 843     }
 844
 845     return NULL;
 846 }
 847
 848 static void qemu_cpu_kick_thread(CPUState *cpu)
 849 {
 850 #ifndef _WIN32
 851     int err;
 852
 853     err = pthread_kill(cpu->thread->thread, SIG_IPI);
 854     if (err) {
 855         fprintf(stderr, "qemu:%s: %s", __func__, strerror(err));
 856         exit(1);
 857     }
 858 #else /* _WIN32 */
 859     if (!qemu_cpu_is_self(cpu)) {
 860         CONTEXT tcgContext;
 861
 862         if (SuspendThread(cpu->hThread) == (DWORD)-1) {
 863             fprintf(stderr, "qemu:%s: GetLastError:%lu\n", __func__,
 864                     GetLastError());
 865             exit(1);
 866         }
 867
 868         /* On multi-core systems, we are not sure that the thread is actually
 869          * suspended until we can get the context.
 870          */
 871         tcgContext.ContextFlags = CONTEXT_CONTROL;
 872         while (GetThreadContext(cpu->hThread, &tcgContext) != 0) {
 873             continue;
 874         }
 875
 876         cpu_signal(0);
 877
 878         if (ResumeThread(cpu->hThread) == (DWORD)-1) {
 879             fprintf(stderr, "qemu:%s: GetLastError:%lu\n", __func__,
 880                     GetLastError());
 881             exit(1);
 882         }
 883     }
 884 #endif
 885 }
 886
 887 void qemu_cpu_kick(CPUState *cpu)
 888 {
 889     qemu_cond_broadcast(cpu->halt_cond);
 890     if (!tcg_enabled() && !cpu->thread_kicked) {
 891         qemu_cpu_kick_thread(cpu);
 892         cpu->thread_kicked = true;
 893     }
 894 }
 895
 896 void qemu_cpu_kick_self(void)
 897 {
 898 #ifndef _WIN32
 899     assert(current_cpu);
 900
 901     if (!current_cpu->thread_kicked) {
 902         qemu_cpu_kick_thread(current_cpu);
 903         current_cpu->thread_kicked = true;
 904     }
 905 #else
 906     abort();
 907 #endif
 908 }
 909
 910 bool qemu_cpu_is_self(CPUState *cpu)
 911 {
 912     return qemu_thread_is_self(cpu->thread);
 913 }
 914
 915 static bool qemu_in_vcpu_thread(void)
 916 {
 917     return current_cpu && qemu_cpu_is_self(current_cpu);
 918 }
 919
 920 void qemu_mutex_lock_iothread(void)
 921 {
 922     if (!tcg_enabled()) {
 923         qemu_mutex_lock(&qemu_global_mutex);
 924     } else {
 925         iothread_requesting_mutex = true;
 926         if (qemu_mutex_trylock(&qemu_global_mutex)) {
 927             qemu_cpu_kick_thread(first_cpu);
 928             qemu_mutex_lock(&qemu_global_mutex);
 929         }
 930         iothread_requesting_mutex = false;
 931         qemu_cond_broadcast(&qemu_io_proceeded_cond);
 932     }
 933 }
 934
 935 void qemu_mutex_unlock_iothread(void)
 936 {
 937     qemu_mutex_unlock(&qemu_global_mutex);
 938 }
 939
 940 static int all_vcpus_paused(void)
 941 {
 942     CPUState *cpu = first_cpu;
 943
 944     while (cpu) {
 945         if (!cpu->stopped) {
 946             return 0;
 947         }
 948         cpu = cpu->next_cpu;
 949     }
 950
 951     return 1;
 952 }
 953
 954 void pause_all_vcpus(void)
 955 {
 956     CPUState *cpu = first_cpu;
 957
 958     qemu_clock_enable(vm_clock, false);
 959     while (cpu) {
 960         cpu->stop = true;
 961         qemu_cpu_kick(cpu);
 962         cpu = cpu->next_cpu;
 963     }
 964
 965     if (qemu_in_vcpu_thread()) {
 966         cpu_stop_current();
 967         if (!kvm_enabled()) {
 968             cpu = first_cpu;
 969             while (cpu) {
 970                 cpu->stop = false;
 971                 cpu->stopped = true;
 972                 cpu = cpu->next_cpu;
 973             }
 974             return;
 975         }
 976     }
 977
 978     while (!all_vcpus_paused()) {
 979         qemu_cond_wait(&qemu_pause_cond, &qemu_global_mutex);
 980         cpu = first_cpu;
 981         while (cpu) {
 982             qemu_cpu_kick(cpu);
 983             cpu = cpu->next_cpu;
 984         }
 985     }
 986 }
 987
 988 void cpu_resume(CPUState *cpu)
 989 {
 990     cpu->stop = false;
 991     cpu->stopped = false;
 992     qemu_cpu_kick(cpu);
 993 }
 994
 995 void resume_all_vcpus(void)
 996 {
 997     CPUState *cpu = first_cpu;
 998
 999     qemu_clock_enable(vm_clock, true);
1000     while (cpu) {
1001         cpu_resume(cpu);
1002         cpu = cpu->next_cpu;
1003     }
1004 }
1005
1006 static void qemu_tcg_init_vcpu(CPUState *cpu)
1007 {
1008     /* share a single thread for all cpus with TCG */
1009     if (!tcg_cpu_thread) {
1010         cpu->thread = g_malloc0(sizeof(QemuThread));
1011         cpu->halt_cond = g_malloc0(sizeof(QemuCond));
1012         qemu_cond_init(cpu->halt_cond);
1013         tcg_halt_cond = cpu->halt_cond;
1014         qemu_thread_create(cpu->thread, qemu_tcg_cpu_thread_fn, cpu,
1015                            QEMU_THREAD_JOINABLE);
1016 #ifdef _WIN32
1017         cpu->hThread = qemu_thread_get_handle(cpu->thread);
1018 #endif
1019         while (!cpu->created) {
1020             qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
1021         }
1022         tcg_cpu_thread = cpu->thread;
1023     } else {
1024         cpu->thread = tcg_cpu_thread;
1025         cpu->halt_cond = tcg_halt_cond;
1026     }
1027 }
1028
1029 static void qemu_kvm_start_vcpu(CPUState *cpu)
1030 {
1031     cpu->thread = g_malloc0(sizeof(QemuThread));
1032     cpu->halt_cond = g_malloc0(sizeof(QemuCond));
1033     qemu_cond_init(cpu->halt_cond);
1034     qemu_thread_create(cpu->thread, qemu_kvm_cpu_thread_fn, cpu,
1035                        QEMU_THREAD_JOINABLE);
1036     while (!cpu->created) {
1037         qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
1038     }
1039 }
1040
1041 static void qemu_dummy_start_vcpu(CPUState *cpu)
1042 {
1043     cpu->thread = g_malloc0(sizeof(QemuThread));
1044     cpu->halt_cond = g_malloc0(sizeof(QemuCond));
1045     qemu_cond_init(cpu->halt_cond);
1046     qemu_thread_create(cpu->thread, qemu_dummy_cpu_thread_fn, cpu,
1047                        QEMU_THREAD_JOINABLE);
1048     while (!cpu->created) {
1049         qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
1050     }
1051 }
1052
1053 void qemu_init_vcpu(CPUState *cpu)
1054 {
1055     cpu->nr_cores = smp_cores;
1056     cpu->nr_threads = smp_threads;
1057     cpu->stopped = true;
1058     if (kvm_enabled()) {
1059         qemu_kvm_start_vcpu(cpu);
1060     } else if (tcg_enabled()) {
1061         qemu_tcg_init_vcpu(cpu);
1062     } else {
1063         qemu_dummy_start_vcpu(cpu);
1064     }
1065 }
1066
1067 void cpu_stop_current(void)
1068 {
1069     if (current_cpu) {
1070         current_cpu->stop = false;
1071         current_cpu->stopped = true;
1072         cpu_exit(current_cpu);
1073         qemu_cond_signal(&qemu_pause_cond);
1074     }
1075 }
1076
1077 int vm_stop(RunState state)
1078 {
1079     if (qemu_in_vcpu_thread()) {
1080         qemu_system_vmstop_request(state);
1081         /*
1082          * FIXME: should not return to device code in case
1083          * vm_stop() has been requested.
1084          */
1085         cpu_stop_current();
1086         return 0;
1087     }
1088
1089     return do_vm_stop(state);
1090 }
1091
1092 /* does a state transition even if the VM is already stopped,
1093    current state is forgotten forever */
1094 int vm_stop_force_state(RunState state)
1095 {
1096     if (runstate_is_running()) {
1097         return vm_stop(state);
1098     } else {
1099         runstate_set(state);
1100         return 0;
1101     }
1102 }
1103
1104 static int tcg_cpu_exec(CPUArchState *env)
1105 {
1106     int ret;
1107 #ifdef CONFIG_PROFILER
1108     int64_t ti;
1109 #endif
1110
1111 #ifdef CONFIG_PROFILER
1112     ti = profile_getclock();
1113 #endif
1114     if (use_icount) {
1115         int64_t count;
1116         int decr;
1117         qemu_icount -= (env->icount_decr.u16.low + env->icount_extra);
1118         env->icount_decr.u16.low = 0;
1119         env->icount_extra = 0;
1120         count = qemu_icount_round(qemu_clock_deadline(vm_clock));
1121         qemu_icount += count;
1122         decr = (count > 0xffff) ? 0xffff : count;
1123         count -= decr;
1124         env->icount_decr.u16.low = decr;
1125         env->icount_extra = count;
1126     }
1127     ret = cpu_exec(env);
1128 #ifdef CONFIG_PROFILER
1129     qemu_time += profile_getclock() - ti;
1130 #endif
1131     if (use_icount) {
1132         /* Fold pending instructions back into the
1133            instruction counter, and clear the interrupt flag.  */
1134         qemu_icount -= (env->icount_decr.u16.low
1135                         + env->icount_extra);
1136         env->icount_decr.u32 = 0;
1137         env->icount_extra = 0;
1138     }
1139     return ret;
1140 }
1141
1142 static void tcg_exec_all(void)
1143 {
1144     int r;
1145
1146     /* Account partial waits to the vm_clock.  */
1147     qemu_clock_warp(vm_clock);
1148
1149     if (next_cpu == NULL) {
1150         next_cpu = first_cpu;
1151     }
1152     for (; next_cpu != NULL && !exit_request; next_cpu = next_cpu->next_cpu) {
1153         CPUState *cpu = next_cpu;
1154         CPUArchState *env = cpu->env_ptr;
1155
1156         qemu_clock_enable(vm_clock,
1157                           (env->singlestep_enabled & SSTEP_NOTIMER) == 0);
1158
1159         if (cpu_can_run(cpu)) {
1160             r = tcg_cpu_exec(env);
1161             if (r == EXCP_DEBUG) {
1162                 cpu_handle_guest_debug(cpu);
1163                 break;
1164             }
1165         } else if (cpu->stop || cpu->stopped) {
1166             break;
1167         }
1168     }
1169     exit_request = 0;
1170 }
1171
1172 void set_numa_modes(void)
1173 {
1174     CPUState *cpu;
1175     int i;
1176
1177     for (cpu = first_cpu; cpu != NULL; cpu = cpu->next_cpu) {
1178         for (i = 0; i < nb_numa_nodes; i++) {
1179             if (test_bit(cpu->cpu_index, node_cpumask[i])) {
1180                 cpu->numa_node = i;
1181             }
1182         }
1183     }
1184 }
1185
1186 void list_cpus(FILE *f, fprintf_function cpu_fprintf, const char *optarg)
1187 {
1188     /* XXX: implement xxx_cpu_list for targets that still miss it */
1189 #if defined(cpu_list)
1190     cpu_list(f, cpu_fprintf);
1191 #endif
1192 }
1193
1194 CpuInfoList *qmp_query_cpus(Error **errp)
1195 {
1196     CpuInfoList *head = NULL, *cur_item = NULL;
1197     CPUState *cpu;
1198
1199     for (cpu = first_cpu; cpu != NULL; cpu = cpu->next_cpu) {
1200         CpuInfoList *info;
1201 #if defined(TARGET_I386)
1202         X86CPU *x86_cpu = X86_CPU(cpu);
1203         CPUX86State *env = &x86_cpu->env;
1204 #elif defined(TARGET_PPC)
1205         PowerPCCPU *ppc_cpu = POWERPC_CPU(cpu);
1206         CPUPPCState *env = &ppc_cpu->env;
1207 #elif defined(TARGET_SPARC)
1208         SPARCCPU *sparc_cpu = SPARC_CPU(cpu);
1209         CPUSPARCState *env = &sparc_cpu->env;
1210 #elif defined(TARGET_MIPS)
1211         MIPSCPU *mips_cpu = MIPS_CPU(cpu);
1212         CPUMIPSState *env = &mips_cpu->env;
1213 #endif
1214
1215         cpu_synchronize_state(cpu);
1216
1217         info = g_malloc0(sizeof(*info));
1218         info->value = g_malloc0(sizeof(*info->value));
1219         info->value->CPU = cpu->cpu_index;
1220         info->value->current = (cpu == first_cpu);
1221         info->value->halted = cpu->halted;
1222         info->value->thread_id = cpu->thread_id;
1223 #if defined(TARGET_I386)
1224         info->value->has_pc = true;
1225         info->value->pc = env->eip + env->segs[R_CS].base;
1226 #elif defined(TARGET_PPC)
1227         info->value->has_nip = true;
1228         info->value->nip = env->nip;
1229 #elif defined(TARGET_SPARC)
1230         info->value->has_pc = true;
1231         info->value->pc = env->pc;
1232         info->value->has_npc = true;
1233         info->value->npc = env->npc;
1234 #elif defined(TARGET_MIPS)
1235         info->value->has_PC = true;
1236         info->value->PC = env->active_tc.PC;
1237 #endif
1238
1239         /* XXX: waiting for the qapi to support GSList */
1240         if (!cur_item) {
1241             head = cur_item = info;
1242         } else {
1243             cur_item->next = info;
1244             cur_item = info;
1245         }
1246     }
1247
1248     return head;
1249 }
1250
1251 void qmp_memsave(int64_t addr, int64_t size, const char *filename,
1252                  bool has_cpu, int64_t cpu_index, Error **errp)
1253 {
1254     FILE *f;
1255     uint32_t l;
1256     CPUArchState *env;
1257     CPUState *cpu;
1258     uint8_t buf[1024];
1259
1260     if (!has_cpu) {
1261         cpu_index = 0;
1262     }
1263
1264     cpu = qemu_get_cpu(cpu_index);
1265     if (cpu == NULL) {
1266         error_set(errp, QERR_INVALID_PARAMETER_VALUE, "cpu-index",
1267                   "a CPU number");
1268         return;
1269     }
1270     env = cpu->env_ptr;
1271
1272     f = fopen(filename, "wb");
1273     if (!f) {
1274         error_setg_file_open(errp, errno, filename);
1275         return;
1276     }
1277
1278     while (size != 0) {
1279         l = sizeof(buf);
1280         if (l > size)
1281             l = size;
1282         cpu_memory_rw_debug(env, addr, buf, l, 0);
1283         if (fwrite(buf, 1, l, f) != l) {
1284             error_set(errp, QERR_IO_ERROR);
1285             goto exit;
1286         }
1287         addr += l;
1288         size -= l;
1289     }
1290
1291 exit:
1292     fclose(f);
1293 }
1294
1295 void qmp_pmemsave(int64_t addr, int64_t size, const char *filename,
1296                   Error **errp)
1297 {
1298     FILE *f;
1299     uint32_t l;
1300     uint8_t buf[1024];
1301
1302     f = fopen(filename, "wb");
1303     if (!f) {
1304         error_setg_file_open(errp, errno, filename);
1305         return;
1306     }
1307
1308     while (size != 0) {
1309         l = sizeof(buf);
1310         if (l > size)
1311             l = size;
1312         cpu_physical_memory_rw(addr, buf, l, 0);
1313         if (fwrite(buf, 1, l, f) != l) {
1314             error_set(errp, QERR_IO_ERROR);
1315             goto exit;
1316         }
1317         addr += l;
1318         size -= l;
1319     }
1320
1321 exit:
1322     fclose(f);
1323 }
1324
1325 void qmp_inject_nmi(Error **errp)
1326 {
1327 #if defined(TARGET_I386)
1328     CPUState *cs;
1329
1330     for (cs = first_cpu; cs != NULL; cs = cs->next_cpu) {
1331         X86CPU *cpu = X86_CPU(cs);
1332         CPUX86State *env = &cpu->env;
1333
1334         if (!env->apic_state) {
1335             cpu_interrupt(cs, CPU_INTERRUPT_NMI);
1336         } else {
1337             apic_deliver_nmi(env->apic_state);
1338         }
1339     }
1340 #else
1341     error_set(errp, QERR_UNSUPPORTED);
1342 #endif
1343 }