*/
#include "qemu/osdep.h"
-#include "qemu/coroutine.h"
#include "qemu/coroutine_int.h"
#include "qemu/processor.h"
#include "qemu/queue.h"
QSIMPLEQ_INIT(&queue->entries);
}
-void coroutine_fn qemu_co_queue_wait_impl(CoQueue *queue, QemuLockable *lock)
+void coroutine_fn qemu_co_queue_wait_impl(CoQueue *queue, QemuLockable *lock,
+ CoQueueWaitFlags flags)
{
Coroutine *self = qemu_coroutine_self();
- QSIMPLEQ_INSERT_TAIL(&queue->entries, self, co_queue_next);
+ if (flags & CO_QUEUE_WAIT_FRONT) {
+ QSIMPLEQ_INSERT_HEAD(&queue->entries, self, co_queue_next);
+ } else {
+ QSIMPLEQ_INSERT_TAIL(&queue->entries, self, co_queue_next);
+ }
if (lock) {
qemu_lockable_unlock(lock);
}
}
-static bool qemu_co_queue_do_restart(CoQueue *queue, bool single)
-{
- Coroutine *next;
-
- if (QSIMPLEQ_EMPTY(&queue->entries)) {
- return false;
- }
-
- while ((next = QSIMPLEQ_FIRST(&queue->entries)) != NULL) {
- QSIMPLEQ_REMOVE_HEAD(&queue->entries, co_queue_next);
- aio_co_wake(next);
- if (single) {
- break;
- }
- }
- return true;
-}
-
-bool qemu_co_queue_next(CoQueue *queue)
-{
- return qemu_co_queue_do_restart(queue, true);
-}
-
-void qemu_co_queue_restart_all(CoQueue *queue)
-{
- qemu_co_queue_do_restart(queue, false);
-}
-
bool qemu_co_enter_next_impl(CoQueue *queue, QemuLockable *lock)
{
Coroutine *next;
return true;
}
+bool coroutine_fn qemu_co_queue_next(CoQueue *queue)
+{
+ /* No unlock/lock needed in coroutine context. */
+ return qemu_co_enter_next_impl(queue, NULL);
+}
+
+void qemu_co_enter_all_impl(CoQueue *queue, QemuLockable *lock)
+{
+ while (qemu_co_enter_next_impl(queue, lock)) {
+ /* just loop */
+ }
+}
+
+void coroutine_fn qemu_co_queue_restart_all(CoQueue *queue)
+{
+ /* No unlock/lock needed in coroutine context. */
+ qemu_co_enter_all_impl(queue, NULL);
+}
+
bool qemu_co_queue_empty(CoQueue *queue)
{
return QSIMPLEQ_FIRST(&queue->entries) == NULL;
QSLIST_ENTRY(CoWaitRecord) next;
} CoWaitRecord;
-static void push_waiter(CoMutex *mutex, CoWaitRecord *w)
+static void coroutine_fn push_waiter(CoMutex *mutex, CoWaitRecord *w)
{
w->co = qemu_coroutine_self();
QSLIST_INSERT_HEAD_ATOMIC(&mutex->from_push, w, next);
trace_qemu_co_mutex_lock_entry(mutex, self);
push_waiter(mutex, &w);
+ /*
+ * Add waiter before reading mutex->handoff. Pairs with qatomic_set_mb
+ * in qemu_co_mutex_unlock.
+ */
+ smp_mb__after_rmw();
+
/* This is the "Responsibility Hand-Off" protocol; a lock() picks from
* a concurrent unlock() the responsibility of waking somebody up.
*/
- old_handoff = qatomic_mb_read(&mutex->handoff);
+ old_handoff = qatomic_read(&mutex->handoff);
if (old_handoff &&
has_waiters(mutex) &&
qatomic_cmpxchg(&mutex->handoff, old_handoff, 0) == old_handoff) {
}
our_handoff = mutex->sequence;
- qatomic_mb_set(&mutex->handoff, our_handoff);
+ /* Set handoff before checking for waiters. */
+ qatomic_set_mb(&mutex->handoff, our_handoff);
if (!has_waiters(mutex)) {
/* The concurrent lock has not added itself yet, so it
* will be able to pick our handoff.
}
/* Releases the internal CoMutex. */
-static void qemu_co_rwlock_maybe_wake_one(CoRwlock *lock)
+static void coroutine_fn qemu_co_rwlock_maybe_wake_one(CoRwlock *lock)
{
CoRwTicket *tkt = QSIMPLEQ_FIRST(&lock->tickets);
Coroutine *co = NULL;
}
}
-void qemu_co_rwlock_rdlock(CoRwlock *lock)
+void coroutine_fn qemu_co_rwlock_rdlock(CoRwlock *lock)
{
Coroutine *self = qemu_coroutine_self();
self->locks_held++;
}
-void qemu_co_rwlock_unlock(CoRwlock *lock)
+void coroutine_fn qemu_co_rwlock_unlock(CoRwlock *lock)
{
Coroutine *self = qemu_coroutine_self();
qemu_co_rwlock_maybe_wake_one(lock);
}
-void qemu_co_rwlock_downgrade(CoRwlock *lock)
+void coroutine_fn qemu_co_rwlock_downgrade(CoRwlock *lock)
{
qemu_co_mutex_lock(&lock->mutex);
assert(lock->owners == -1);
qemu_co_rwlock_maybe_wake_one(lock);
}
-void qemu_co_rwlock_wrlock(CoRwlock *lock)
+void coroutine_fn qemu_co_rwlock_wrlock(CoRwlock *lock)
{
Coroutine *self = qemu_coroutine_self();
self->locks_held++;
}
-void qemu_co_rwlock_upgrade(CoRwlock *lock)
+void coroutine_fn qemu_co_rwlock_upgrade(CoRwlock *lock)
{
qemu_co_mutex_lock(&lock->mutex);
assert(lock->owners > 0);