X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=cpus-common.c;h=59f751ecf917a90cf7f98fe1eb8f892dbc376e7a;hb=37c4a85cd256a51c5f58ee7e531f25f0e89b2c87;hp=d6cd4262353e8397305037fb1ae11fe34ce10880;hpb=0e55539c076a61b0b10a1aea1158fc20fb159d99;p=mirror_qemu.git diff --git a/cpus-common.c b/cpus-common.c index d6cd426235..59f751ecf9 100644 --- a/cpus-common.c +++ b/cpus-common.c @@ -18,16 +18,30 @@ */ #include "qemu/osdep.h" +#include "qemu/main-loop.h" #include "exec/cpu-common.h" #include "qom/cpu.h" #include "sysemu/cpus.h" static QemuMutex qemu_cpu_list_lock; +static QemuCond exclusive_cond; +static QemuCond exclusive_resume; static QemuCond qemu_work_cond; +/* >= 1 if a thread is inside start_exclusive/end_exclusive. Written + * under qemu_cpu_list_lock, read with atomic operations. + */ +static int pending_cpus; + void qemu_init_cpu_list(void) { + /* This is needed because qemu_init_cpu_list is also called by the + * child process in a fork. */ + pending_cpus = 0; + qemu_mutex_init(&qemu_cpu_list_lock); + qemu_cond_init(&exclusive_cond); + qemu_cond_init(&exclusive_resume); qemu_cond_init(&qemu_work_cond); } @@ -55,6 +69,12 @@ static int cpu_get_free_index(void) return cpu_index; } +static void finish_safe_work(CPUState *cpu) +{ + cpu_exec_start(cpu); + cpu_exec_end(cpu); +} + void cpu_list_add(CPUState *cpu) { qemu_mutex_lock(&qemu_cpu_list_lock); @@ -66,6 +86,8 @@ void cpu_list_add(CPUState *cpu) } QTAILQ_INSERT_TAIL(&cpus, cpu, node); qemu_mutex_unlock(&qemu_cpu_list_lock); + + finish_safe_work(cpu); } void cpu_list_remove(CPUState *cpu) @@ -87,8 +109,8 @@ void cpu_list_remove(CPUState *cpu) struct qemu_work_item { struct qemu_work_item *next; run_on_cpu_func func; - void *data; - bool free, done; + run_on_cpu_data data; + bool free, exclusive, done; }; static void queue_work_on_cpu(CPUState *cpu, struct qemu_work_item *wi) @@ -107,7 +129,7 @@ static void queue_work_on_cpu(CPUState *cpu, struct qemu_work_item *wi) qemu_cpu_kick(cpu); } -void do_run_on_cpu(CPUState *cpu, run_on_cpu_func func, void *data, +void do_run_on_cpu(CPUState *cpu, run_on_cpu_func func, run_on_cpu_data data, QemuMutex *mutex) { struct qemu_work_item wi; @@ -121,6 +143,7 @@ void do_run_on_cpu(CPUState *cpu, run_on_cpu_func func, void *data, wi.data = data; wi.done = false; wi.free = false; + wi.exclusive = false; queue_work_on_cpu(cpu, &wi); while (!atomic_mb_read(&wi.done)) { @@ -131,19 +154,158 @@ void do_run_on_cpu(CPUState *cpu, run_on_cpu_func func, void *data, } } -void async_run_on_cpu(CPUState *cpu, run_on_cpu_func func, void *data) +void async_run_on_cpu(CPUState *cpu, run_on_cpu_func func, run_on_cpu_data data) { struct qemu_work_item *wi; - if (qemu_cpu_is_self(cpu)) { - func(cpu, data); - return; + wi = g_malloc0(sizeof(struct qemu_work_item)); + wi->func = func; + wi->data = data; + wi->free = true; + + queue_work_on_cpu(cpu, wi); +} + +/* Wait for pending exclusive operations to complete. The CPU list lock + must be held. */ +static inline void exclusive_idle(void) +{ + while (pending_cpus) { + qemu_cond_wait(&exclusive_resume, &qemu_cpu_list_lock); + } +} + +/* Start an exclusive operation. + Must only be called from outside cpu_exec. */ +void start_exclusive(void) +{ + CPUState *other_cpu; + int running_cpus; + + qemu_mutex_lock(&qemu_cpu_list_lock); + exclusive_idle(); + + /* Make all other cpus stop executing. */ + atomic_set(&pending_cpus, 1); + + /* Write pending_cpus before reading other_cpu->running. */ + smp_mb(); + running_cpus = 0; + CPU_FOREACH(other_cpu) { + if (atomic_read(&other_cpu->running)) { + other_cpu->has_waiter = true; + running_cpus++; + qemu_cpu_kick(other_cpu); + } + } + + atomic_set(&pending_cpus, running_cpus + 1); + while (pending_cpus > 1) { + qemu_cond_wait(&exclusive_cond, &qemu_cpu_list_lock); } + /* Can release mutex, no one will enter another exclusive + * section until end_exclusive resets pending_cpus to 0. + */ + qemu_mutex_unlock(&qemu_cpu_list_lock); +} + +/* Finish an exclusive operation. */ +void end_exclusive(void) +{ + qemu_mutex_lock(&qemu_cpu_list_lock); + atomic_set(&pending_cpus, 0); + qemu_cond_broadcast(&exclusive_resume); + qemu_mutex_unlock(&qemu_cpu_list_lock); +} + +/* Wait for exclusive ops to finish, and begin cpu execution. */ +void cpu_exec_start(CPUState *cpu) +{ + atomic_set(&cpu->running, true); + + /* Write cpu->running before reading pending_cpus. */ + smp_mb(); + + /* 1. start_exclusive saw cpu->running == true and pending_cpus >= 1. + * After taking the lock we'll see cpu->has_waiter == true and run---not + * for long because start_exclusive kicked us. cpu_exec_end will + * decrement pending_cpus and signal the waiter. + * + * 2. start_exclusive saw cpu->running == false but pending_cpus >= 1. + * This includes the case when an exclusive item is running now. + * Then we'll see cpu->has_waiter == false and wait for the item to + * complete. + * + * 3. pending_cpus == 0. Then start_exclusive is definitely going to + * see cpu->running == true, and it will kick the CPU. + */ + if (unlikely(atomic_read(&pending_cpus))) { + qemu_mutex_lock(&qemu_cpu_list_lock); + if (!cpu->has_waiter) { + /* Not counted in pending_cpus, let the exclusive item + * run. Since we have the lock, just set cpu->running to true + * while holding it; no need to check pending_cpus again. + */ + atomic_set(&cpu->running, false); + exclusive_idle(); + /* Now pending_cpus is zero. */ + atomic_set(&cpu->running, true); + } else { + /* Counted in pending_cpus, go ahead and release the + * waiter at cpu_exec_end. + */ + } + qemu_mutex_unlock(&qemu_cpu_list_lock); + } +} + +/* Mark cpu as not executing, and release pending exclusive ops. */ +void cpu_exec_end(CPUState *cpu) +{ + atomic_set(&cpu->running, false); + + /* Write cpu->running before reading pending_cpus. */ + smp_mb(); + + /* 1. start_exclusive saw cpu->running == true. Then it will increment + * pending_cpus and wait for exclusive_cond. After taking the lock + * we'll see cpu->has_waiter == true. + * + * 2. start_exclusive saw cpu->running == false but here pending_cpus >= 1. + * This includes the case when an exclusive item started after setting + * cpu->running to false and before we read pending_cpus. Then we'll see + * cpu->has_waiter == false and not touch pending_cpus. The next call to + * cpu_exec_start will run exclusive_idle if still necessary, thus waiting + * for the item to complete. + * + * 3. pending_cpus == 0. Then start_exclusive is definitely going to + * see cpu->running == false, and it can ignore this CPU until the + * next cpu_exec_start. + */ + if (unlikely(atomic_read(&pending_cpus))) { + qemu_mutex_lock(&qemu_cpu_list_lock); + if (cpu->has_waiter) { + cpu->has_waiter = false; + atomic_set(&pending_cpus, pending_cpus - 1); + if (pending_cpus == 1) { + qemu_cond_signal(&exclusive_cond); + } + } + qemu_mutex_unlock(&qemu_cpu_list_lock); + } +} + +void async_safe_run_on_cpu(CPUState *cpu, run_on_cpu_func func, + run_on_cpu_data data) +{ + struct qemu_work_item *wi; + wi = g_malloc0(sizeof(struct qemu_work_item)); wi->func = func; wi->data = data; wi->free = true; + wi->exclusive = true; queue_work_on_cpu(cpu, wi); } @@ -164,7 +326,21 @@ void process_queued_cpu_work(CPUState *cpu) cpu->queued_work_last = NULL; } qemu_mutex_unlock(&cpu->work_mutex); - wi->func(cpu, wi->data); + if (wi->exclusive) { + /* Running work items outside the BQL avoids the following deadlock: + * 1) start_exclusive() is called with the BQL taken while another + * CPU is running; 2) cpu_exec in the other CPU tries to takes the + * BQL, so it goes to sleep; start_exclusive() is sleeping too, so + * neither CPU can proceed. + */ + qemu_mutex_unlock_iothread(); + start_exclusive(); + wi->func(cpu, wi->data); + end_exclusive(); + qemu_mutex_lock_iothread(); + } else { + wi->func(cpu, wi->data); + } qemu_mutex_lock(&cpu->work_mutex); if (wi->free) { g_free(wi);