#include "qemu/thread.h"
#include "qemu/queue.h"
#include "qemu/atomic.h"
+#include "qemu/notify.h"
+#include "qemu/sys_membarrier.h"
#ifdef __cplusplus
extern "C" {
/* Data used for registry, protected by rcu_registry_lock */
QLIST_ENTRY(rcu_reader_data) node;
+
+ /*
+ * NotifierList used to force an RCU grace period. Accessed under
+ * rcu_registry_lock. Note that the notifier is called _outside_
+ * the thread!
+ */
+ NotifierList force_rcu;
};
extern __thread struct rcu_reader_data rcu_reader;
return;
}
- ctr = atomic_read(&rcu_gp_ctr);
- atomic_xchg(&p_rcu_reader->ctr, ctr);
+ ctr = qatomic_read(&rcu_gp_ctr);
+ qatomic_set(&p_rcu_reader->ctr, ctr);
+
+ /* Write p_rcu_reader->ctr before reading RCU-protected pointers. */
+ smp_mb_placeholder();
}
static inline void rcu_read_unlock(void)
return;
}
- atomic_xchg(&p_rcu_reader->ctr, 0);
- if (unlikely(atomic_read(&p_rcu_reader->waiting))) {
- atomic_set(&p_rcu_reader->waiting, false);
+ /* Ensure that the critical section is seen to precede the
+ * store to p_rcu_reader->ctr. Together with the following
+ * smp_mb_placeholder(), this ensures writes to p_rcu_reader->ctr
+ * are sequentially consistent.
+ */
+ qatomic_store_release(&p_rcu_reader->ctr, 0);
+
+ /* Write p_rcu_reader->ctr before reading p_rcu_reader->waiting. */
+ smp_mb_placeholder();
+ if (unlikely(qatomic_read(&p_rcu_reader->waiting))) {
+ qatomic_set(&p_rcu_reader->waiting, false);
qemu_event_set(&rcu_gp_event);
}
}
*/
extern void rcu_register_thread(void);
extern void rcu_unregister_thread(void);
-extern void rcu_after_fork(void);
+
+/*
+ * Support for fork(). fork() support is enabled at startup.
+ */
+extern void rcu_enable_atfork(void);
+extern void rcu_disable_atfork(void);
struct rcu_head;
typedef void RCUCBFunc(struct rcu_head *head);
};
extern void call_rcu1(struct rcu_head *head, RCUCBFunc *func);
+extern void drain_call_rcu(void);
/* The operands of the minus operator must have the same type,
* which must be the one that we specify in the cast.
}), \
(RCUCBFunc *)g_free);
+typedef void RCUReadAuto;
+static inline RCUReadAuto *rcu_read_auto_lock(void)
+{
+ rcu_read_lock();
+ /* Anything non-NULL causes the cleanup function to be called */
+ return (void *)(uintptr_t)0x1;
+}
+
+static inline void rcu_read_auto_unlock(RCUReadAuto *r)
+{
+ rcu_read_unlock();
+}
+
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(RCUReadAuto, rcu_read_auto_unlock)
+
+#define WITH_RCU_READ_LOCK_GUARD() \
+ WITH_RCU_READ_LOCK_GUARD_(glue(_rcu_read_auto, __COUNTER__))
+
+#define WITH_RCU_READ_LOCK_GUARD_(var) \
+ for (g_autoptr(RCUReadAuto) var = rcu_read_auto_lock(); \
+ (var); rcu_read_auto_unlock(var), (var) = NULL)
+
+#define RCU_READ_LOCK_GUARD() \
+ g_autoptr(RCUReadAuto) _rcu_read_auto __attribute__((unused)) = rcu_read_auto_lock()
+
+/*
+ * Force-RCU notifiers tell readers that they should exit their
+ * read-side critical section.
+ */
+void rcu_add_force_rcu_notifier(Notifier *n);
+void rcu_remove_force_rcu_notifier(Notifier *n);
+
#ifdef __cplusplus
}
#endif