static void kvm_cpu_kick(CPUState *cpu)
{
- atomic_set(&cpu->kvm_run->immediate_exit, 1);
+ qatomic_set(&cpu->kvm_run->immediate_exit, 1);
}
static void kvm_cpu_kick_self(void)
int r;
if (kvm_immediate_exit) {
- atomic_set(&cpu->kvm_run->immediate_exit, 0);
+ qatomic_set(&cpu->kvm_run->immediate_exit, 0);
/* Write kvm_run->immediate_exit before the cpu->exit_request
* write in kvm_cpu_exec.
*/
DPRINTF("kvm_cpu_exec()\n");
if (kvm_arch_process_async_events(cpu)) {
- atomic_set(&cpu->exit_request, 0);
+ qatomic_set(&cpu->exit_request, 0);
return EXCP_HLT;
}
}
kvm_arch_pre_run(cpu, run);
- if (atomic_read(&cpu->exit_request)) {
+ if (qatomic_read(&cpu->exit_request)) {
DPRINTF("interrupt exit requested\n");
/*
* KVM requires us to reenter the kernel after IO exits to complete
vm_stop(RUN_STATE_INTERNAL_ERROR);
}
- atomic_set(&cpu->exit_request, 0);
+ qatomic_set(&cpu->exit_request, 0);
return ret;
}
have_sigbus_pending = true;
pending_sigbus_addr = addr;
pending_sigbus_code = code;
- atomic_set(&cpu->exit_request, 1);
+ qatomic_set(&cpu->exit_request, 1);
return 0;
#else
return 1;
#if DATA_SIZE == 16
ret = atomic16_cmpxchg(haddr, cmpv, newv);
#else
- ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
+ ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv);
#endif
ATOMIC_MMU_CLEANUP;
atomic_trace_rmw_post(env, addr, info);
ATOMIC_MMU_IDX);
atomic_trace_rmw_pre(env, addr, info);
- ret = atomic_xchg__nocheck(haddr, val);
+ ret = qatomic_xchg__nocheck(haddr, val);
ATOMIC_MMU_CLEANUP;
atomic_trace_rmw_post(env, addr, info);
return ret;
uint16_t info = trace_mem_build_info(SHIFT, false, 0, false, \
ATOMIC_MMU_IDX); \
atomic_trace_rmw_pre(env, addr, info); \
- ret = atomic_##X(haddr, val); \
+ ret = qatomic_##X(haddr, val); \
ATOMIC_MMU_CLEANUP; \
atomic_trace_rmw_post(env, addr, info); \
return ret; \
ATOMIC_MMU_IDX); \
atomic_trace_rmw_pre(env, addr, info); \
smp_mb(); \
- cmp = atomic_read__nocheck(haddr); \
+ cmp = qatomic_read__nocheck(haddr); \
do { \
old = cmp; new = FN(old, val); \
- cmp = atomic_cmpxchg__nocheck(haddr, old, new); \
+ cmp = qatomic_cmpxchg__nocheck(haddr, old, new); \
} while (cmp != old); \
ATOMIC_MMU_CLEANUP; \
atomic_trace_rmw_post(env, addr, info); \
#if DATA_SIZE == 16
ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
#else
- ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
+ ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
#endif
ATOMIC_MMU_CLEANUP;
atomic_trace_rmw_post(env, addr, info);
ATOMIC_MMU_IDX);
atomic_trace_rmw_pre(env, addr, info);
- ret = atomic_xchg__nocheck(haddr, BSWAP(val));
+ ret = qatomic_xchg__nocheck(haddr, BSWAP(val));
ATOMIC_MMU_CLEANUP;
atomic_trace_rmw_post(env, addr, info);
return BSWAP(ret);
uint16_t info = trace_mem_build_info(SHIFT, false, MO_BSWAP, \
false, ATOMIC_MMU_IDX); \
atomic_trace_rmw_pre(env, addr, info); \
- ret = atomic_##X(haddr, BSWAP(val)); \
+ ret = qatomic_##X(haddr, BSWAP(val)); \
ATOMIC_MMU_CLEANUP; \
atomic_trace_rmw_post(env, addr, info); \
return BSWAP(ret); \
false, ATOMIC_MMU_IDX); \
atomic_trace_rmw_pre(env, addr, info); \
smp_mb(); \
- ldn = atomic_read__nocheck(haddr); \
+ ldn = qatomic_read__nocheck(haddr); \
do { \
ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \
- ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \
+ ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \
} while (ldo != ldn); \
ATOMIC_MMU_CLEANUP; \
atomic_trace_rmw_post(env, addr, info); \
goto out_unlock_next;
}
/* Atomically claim the jump destination slot only if it was NULL */
- old = atomic_cmpxchg(&tb->jmp_dest[n], (uintptr_t)NULL, (uintptr_t)tb_next);
+ old = qatomic_cmpxchg(&tb->jmp_dest[n], (uintptr_t)NULL,
+ (uintptr_t)tb_next);
if (old) {
goto out_unlock_next;
}
tb = tb_gen_code(cpu, pc, cs_base, flags, cf_mask);
mmap_unlock();
/* We add the TB in the virtual pc hash table for the fast lookup */
- atomic_set(&cpu->tb_jmp_cache[tb_jmp_cache_hash_func(pc)], tb);
+ qatomic_set(&cpu->tb_jmp_cache[tb_jmp_cache_hash_func(pc)], tb);
}
#ifndef CONFIG_USER_ONLY
/* We don't take care of direct jumps when address mapping changes in
* Ensure zeroing happens before reading cpu->exit_request or
* cpu->interrupt_request (see also smp_wmb in cpu_exit())
*/
- atomic_mb_set(&cpu_neg(cpu)->icount_decr.u16.high, 0);
+ qatomic_mb_set(&cpu_neg(cpu)->icount_decr.u16.high, 0);
- if (unlikely(atomic_read(&cpu->interrupt_request))) {
+ if (unlikely(qatomic_read(&cpu->interrupt_request))) {
int interrupt_request;
qemu_mutex_lock_iothread();
interrupt_request = cpu->interrupt_request;
}
/* Finally, check if we need to exit to the main loop. */
- if (unlikely(atomic_read(&cpu->exit_request))
+ if (unlikely(qatomic_read(&cpu->exit_request))
|| (use_icount
&& cpu_neg(cpu)->icount_decr.u16.low + cpu->icount_extra == 0)) {
- atomic_set(&cpu->exit_request, 0);
+ qatomic_set(&cpu->exit_request, 0);
if (cpu->exception_index == -1) {
cpu->exception_index = EXCP_INTERRUPT;
}
}
*last_tb = NULL;
- insns_left = atomic_read(&cpu_neg(cpu)->icount_decr.u32);
+ insns_left = qatomic_read(&cpu_neg(cpu)->icount_decr.u32);
if (insns_left < 0) {
/* Something asked us to stop executing chained TBs; just
* continue round the main loop. Whatever requested the exit
CPU_FOREACH(cpu) {
CPUArchState *env = cpu->env_ptr;
- full += atomic_read(&env_tlb(env)->c.full_flush_count);
- part += atomic_read(&env_tlb(env)->c.part_flush_count);
- elide += atomic_read(&env_tlb(env)->c.elide_flush_count);
+ full += qatomic_read(&env_tlb(env)->c.full_flush_count);
+ part += qatomic_read(&env_tlb(env)->c.part_flush_count);
+ elide += qatomic_read(&env_tlb(env)->c.elide_flush_count);
}
*pfull = full;
*ppart = part;
cpu_tb_jmp_cache_clear(cpu);
if (to_clean == ALL_MMUIDX_BITS) {
- atomic_set(&env_tlb(env)->c.full_flush_count,
+ qatomic_set(&env_tlb(env)->c.full_flush_count,
env_tlb(env)->c.full_flush_count + 1);
} else {
- atomic_set(&env_tlb(env)->c.part_flush_count,
+ qatomic_set(&env_tlb(env)->c.part_flush_count,
env_tlb(env)->c.part_flush_count + ctpop16(to_clean));
if (to_clean != asked) {
- atomic_set(&env_tlb(env)->c.elide_flush_count,
+ qatomic_set(&env_tlb(env)->c.elide_flush_count,
env_tlb(env)->c.elide_flush_count +
ctpop16(asked & ~to_clean));
}
* generated code.
*
* Other vCPUs might be reading their TLBs during guest execution, so we update
- * te->addr_write with atomic_set. We don't need to worry about this for
+ * te->addr_write with qatomic_set. We don't need to worry about this for
* oversized guests as MTTCG is disabled for them.
*
* Called with tlb_c.lock held.
#if TCG_OVERSIZED_GUEST
tlb_entry->addr_write |= TLB_NOTDIRTY;
#else
- atomic_set(&tlb_entry->addr_write,
+ qatomic_set(&tlb_entry->addr_write,
tlb_entry->addr_write | TLB_NOTDIRTY);
#endif
}
#if TCG_OVERSIZED_GUEST
return *(target_ulong *)((uintptr_t)entry + ofs);
#else
- /* ofs might correspond to .addr_write, so use atomic_read */
- return atomic_read((target_ulong *)((uintptr_t)entry + ofs));
+ /* ofs might correspond to .addr_write, so use qatomic_read */
+ return qatomic_read((target_ulong *)((uintptr_t)entry + ofs));
#endif
}
CPUTLBEntry *vtlb = &env_tlb(env)->d[mmu_idx].vtable[vidx];
target_ulong cmp;
- /* elt_ofs might correspond to .addr_write, so use atomic_read */
+ /* elt_ofs might correspond to .addr_write, so use qatomic_read */
#if TCG_OVERSIZED_GUEST
cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs);
#else
- cmp = atomic_read((target_ulong *)((uintptr_t)vtlb + elt_ofs));
+ cmp = qatomic_read((target_ulong *)((uintptr_t)vtlb + elt_ofs));
#endif
if (cmp == page) {
if (!qemu_cpu_is_self(cpu)) {
qemu_cpu_kick(cpu);
} else {
- atomic_set(&cpu_neg(cpu)->icount_decr.u16.high, -1);
+ qatomic_set(&cpu_neg(cpu)->icount_decr.u16.high, -1);
if (use_icount &&
!cpu->can_do_io
&& (mask & ~old_mask) != 0) {
restore_state_to_opc(env, tb, data);
#ifdef CONFIG_PROFILER
- atomic_set(&prof->restore_time,
+ qatomic_set(&prof->restore_time,
prof->restore_time + profile_getclock() - ti);
- atomic_set(&prof->restore_count, prof->restore_count + 1);
+ qatomic_set(&prof->restore_count, prof->restore_count + 1);
#endif
return 0;
}
/* Level 2..N-1. */
for (i = v_l2_levels; i > 0; i--) {
- void **p = atomic_rcu_read(lp);
+ void **p = qatomic_rcu_read(lp);
if (p == NULL) {
void *existing;
return NULL;
}
p = g_new0(void *, V_L2_SIZE);
- existing = atomic_cmpxchg(lp, NULL, p);
+ existing = qatomic_cmpxchg(lp, NULL, p);
if (unlikely(existing)) {
g_free(p);
p = existing;
lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
}
- pd = atomic_rcu_read(lp);
+ pd = qatomic_rcu_read(lp);
if (pd == NULL) {
void *existing;
}
}
#endif
- existing = atomic_cmpxchg(lp, NULL, pd);
+ existing = qatomic_cmpxchg(lp, NULL, pd);
if (unlikely(existing)) {
#ifndef CONFIG_USER_ONLY
{
tcg_region_reset_all();
/* XXX: flush processor icache at this point if cache flush is
expensive */
- atomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1);
+ qatomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1);
done:
mmap_unlock();
void tb_flush(CPUState *cpu)
{
if (tcg_enabled()) {
- unsigned tb_flush_count = atomic_mb_read(&tb_ctx.tb_flush_count);
+ unsigned tb_flush_count = qatomic_mb_read(&tb_ctx.tb_flush_count);
if (cpu_in_exclusive_context(cpu)) {
do_tb_flush(cpu, RUN_ON_CPU_HOST_INT(tb_flush_count));
int n;
/* mark the LSB of jmp_dest[] so that no further jumps can be inserted */
- ptr = atomic_or_fetch(&orig->jmp_dest[n_orig], 1);
+ ptr = qatomic_or_fetch(&orig->jmp_dest[n_orig], 1);
dest = (TranslationBlock *)(ptr & ~1);
if (dest == NULL) {
return;
* While acquiring the lock, the jump might have been removed if the
* destination TB was invalidated; check again.
*/
- ptr_locked = atomic_read(&orig->jmp_dest[n_orig]);
+ ptr_locked = qatomic_read(&orig->jmp_dest[n_orig]);
if (ptr_locked != ptr) {
qemu_spin_unlock(&dest->jmp_lock);
/*
TB_FOR_EACH_JMP(dest, tb, n) {
tb_reset_jump(tb, n);
- atomic_and(&tb->jmp_dest[n], (uintptr_t)NULL | 1);
+ qatomic_and(&tb->jmp_dest[n], (uintptr_t)NULL | 1);
/* No need to clear the list entry; setting the dest ptr is enough */
}
dest->jmp_list_head = (uintptr_t)NULL;
/* make sure no further incoming jumps will be chained to this TB */
qemu_spin_lock(&tb->jmp_lock);
- atomic_set(&tb->cflags, tb->cflags | CF_INVALID);
+ qatomic_set(&tb->cflags, tb->cflags | CF_INVALID);
qemu_spin_unlock(&tb->jmp_lock);
/* remove the TB from the hash list */
/* remove the TB from the hash list */
h = tb_jmp_cache_hash_func(tb->pc);
CPU_FOREACH(cpu) {
- if (atomic_read(&cpu->tb_jmp_cache[h]) == tb) {
- atomic_set(&cpu->tb_jmp_cache[h], NULL);
+ if (qatomic_read(&cpu->tb_jmp_cache[h]) == tb) {
+ qatomic_set(&cpu->tb_jmp_cache[h], NULL);
}
}
/* suppress any remaining jumps to this TB */
tb_jmp_unlink(tb);
- atomic_set(&tcg_ctx->tb_phys_invalidate_count,
+ qatomic_set(&tcg_ctx->tb_phys_invalidate_count,
tcg_ctx->tb_phys_invalidate_count + 1);
}
#ifdef CONFIG_PROFILER
/* includes aborted translations because of exceptions */
- atomic_set(&prof->tb_count1, prof->tb_count1 + 1);
+ qatomic_set(&prof->tb_count1, prof->tb_count1 + 1);
ti = profile_getclock();
#endif
}
#ifdef CONFIG_PROFILER
- atomic_set(&prof->tb_count, prof->tb_count + 1);
- atomic_set(&prof->interm_time, prof->interm_time + profile_getclock() - ti);
+ qatomic_set(&prof->tb_count, prof->tb_count + 1);
+ qatomic_set(&prof->interm_time,
+ prof->interm_time + profile_getclock() - ti);
ti = profile_getclock();
#endif
tb->tc.size = gen_code_size;
#ifdef CONFIG_PROFILER
- atomic_set(&prof->code_time, prof->code_time + profile_getclock() - ti);
- atomic_set(&prof->code_in_len, prof->code_in_len + tb->size);
- atomic_set(&prof->code_out_len, prof->code_out_len + gen_code_size);
- atomic_set(&prof->search_out_len, prof->search_out_len + search_size);
+ qatomic_set(&prof->code_time, prof->code_time + profile_getclock() - ti);
+ qatomic_set(&prof->code_in_len, prof->code_in_len + tb->size);
+ qatomic_set(&prof->code_out_len, prof->code_out_len + gen_code_size);
+ qatomic_set(&prof->search_out_len, prof->search_out_len + search_size);
#endif
#ifdef DEBUG_DISAS
}
#endif
- atomic_set(&tcg_ctx->code_gen_ptr, (void *)
+ qatomic_set(&tcg_ctx->code_gen_ptr, (void *)
ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
CODE_GEN_ALIGN));
uintptr_t orig_aligned = (uintptr_t)gen_code_buf;
orig_aligned -= ROUND_UP(sizeof(*tb), qemu_icache_linesize);
- atomic_set(&tcg_ctx->code_gen_ptr, (void *)orig_aligned);
+ qatomic_set(&tcg_ctx->code_gen_ptr, (void *)orig_aligned);
tb_destroy(tb);
return existing_tb;
}
unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr);
for (i = 0; i < TB_JMP_PAGE_SIZE; i++) {
- atomic_set(&cpu->tb_jmp_cache[i0 + i], NULL);
+ qatomic_set(&cpu->tb_jmp_cache[i0 + i], NULL);
}
}
qemu_printf("\nStatistics:\n");
qemu_printf("TB flush count %u\n",
- atomic_read(&tb_ctx.tb_flush_count));
+ qatomic_read(&tb_ctx.tb_flush_count));
qemu_printf("TB invalidate count %zu\n",
tcg_tb_phys_invalidate_count());
{
g_assert(qemu_mutex_iothread_locked());
cpu->interrupt_request |= mask;
- atomic_set(&cpu_neg(cpu)->icount_decr.u16.high, -1);
+ qatomic_set(&cpu_neg(cpu)->icount_decr.u16.high, -1);
}
/*
static void qjack_buffer_clear(QJackBuffer *buffer)
{
assert(buffer->data);
- atomic_store_release(&buffer->used, 0);
+ qatomic_store_release(&buffer->used, 0);
buffer->rptr = 0;
buffer->wptr = 0;
}
assert(buffer->data);
const int samples = size / sizeof(float);
int frames = samples / buffer->channels;
- const int avail = buffer->frames - atomic_load_acquire(&buffer->used);
+ const int avail = buffer->frames - qatomic_load_acquire(&buffer->used);
if (frames > avail) {
frames = avail;
buffer->wptr = wptr;
- atomic_add(&buffer->used, frames);
+ qatomic_add(&buffer->used, frames);
return frames * buffer->channels * sizeof(float);
};
static int qjack_buffer_write_l(QJackBuffer *buffer, float **dest, int frames)
{
assert(buffer->data);
- const int avail = buffer->frames - atomic_load_acquire(&buffer->used);
+ const int avail = buffer->frames - qatomic_load_acquire(&buffer->used);
int wptr = buffer->wptr;
if (frames > avail) {
}
buffer->wptr = wptr;
- atomic_add(&buffer->used, frames);
+ qatomic_add(&buffer->used, frames);
return frames;
}
assert(buffer->data);
const int samples = size / sizeof(float);
int frames = samples / buffer->channels;
- const int avail = atomic_load_acquire(&buffer->used);
+ const int avail = qatomic_load_acquire(&buffer->used);
if (frames > avail) {
frames = avail;
buffer->rptr = rptr;
- atomic_sub(&buffer->used, frames);
+ qatomic_sub(&buffer->used, frames);
return frames * buffer->channels * sizeof(float);
}
{
assert(buffer->data);
int copy = frames;
- const int used = atomic_load_acquire(&buffer->used);
+ const int used = qatomic_load_acquire(&buffer->used);
int rptr = buffer->rptr;
if (copy > used) {
}
buffer->rptr = rptr;
- atomic_sub(&buffer->used, copy);
+ qatomic_sub(&buffer->used, copy);
return copy;
}
}
/* bdrv_new() and bdrv_close() make it so */
- assert(atomic_read(&bs->copy_on_read) == 0);
+ assert(qatomic_read(&bs->copy_on_read) == 0);
if (bs->open_flags & BDRV_O_COPY_ON_READ) {
if (!bs->read_only) {
bs->file = NULL;
g_free(bs->opaque);
bs->opaque = NULL;
- atomic_set(&bs->copy_on_read, 0);
+ qatomic_set(&bs->copy_on_read, 0);
bs->backing_file[0] = '\0';
bs->backing_format[0] = '\0';
bs->total_sectors = 0;
void blk_inc_in_flight(BlockBackend *blk)
{
- atomic_inc(&blk->in_flight);
+ qatomic_inc(&blk->in_flight);
}
void blk_dec_in_flight(BlockBackend *blk)
{
- atomic_dec(&blk->in_flight);
+ qatomic_dec(&blk->in_flight);
aio_wait_kick();
}
/* We may have -ENOMEDIUM completions in flight */
AIO_WAIT_WHILE(blk_get_aio_context(blk),
- atomic_mb_read(&blk->in_flight) > 0);
+ qatomic_mb_read(&blk->in_flight) > 0);
if (bs) {
bdrv_drained_end(bs);
aio_context_acquire(ctx);
/* We may have -ENOMEDIUM completions in flight */
- AIO_WAIT_WHILE(ctx, atomic_mb_read(&blk->in_flight) > 0);
+ AIO_WAIT_WHILE(ctx, qatomic_mb_read(&blk->in_flight) > 0);
aio_context_release(ctx);
}
static void blk_root_drained_begin(BdrvChild *child)
{
BlockBackend *blk = child->opaque;
+ ThrottleGroupMember *tgm = &blk->public.throttle_group_member;
if (++blk->quiesce_counter == 1) {
if (blk->dev_ops && blk->dev_ops->drained_begin) {
/* Note that blk->root may not be accessible here yet if we are just
* attaching to a BlockDriverState that is drained. Use child instead. */
- if (atomic_fetch_inc(&blk->public.throttle_group_member.io_limits_disabled) == 0) {
- throttle_group_restart_tgm(&blk->public.throttle_group_member);
+ if (qatomic_fetch_inc(&tgm->io_limits_disabled) == 0) {
+ throttle_group_restart_tgm(tgm);
}
}
assert(blk->quiesce_counter);
assert(blk->public.throttle_group_member.io_limits_disabled);
- atomic_dec(&blk->public.throttle_group_member.io_limits_disabled);
+ qatomic_dec(&blk->public.throttle_group_member.io_limits_disabled);
if (--blk->quiesce_counter == 0) {
if (blk->dev_ops && blk->dev_ops->drained_end) {
{
int drained_end_counter = 0;
bdrv_parent_drained_end_single_no_poll(c, &drained_end_counter);
- BDRV_POLL_WHILE(c->bs, atomic_read(&drained_end_counter) > 0);
+ BDRV_POLL_WHILE(c->bs, qatomic_read(&drained_end_counter) > 0);
}
static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
*/
void bdrv_enable_copy_on_read(BlockDriverState *bs)
{
- atomic_inc(&bs->copy_on_read);
+ qatomic_inc(&bs->copy_on_read);
}
void bdrv_disable_copy_on_read(BlockDriverState *bs)
{
- int old = atomic_fetch_dec(&bs->copy_on_read);
+ int old = qatomic_fetch_dec(&bs->copy_on_read);
assert(old >= 1);
}
}
/* Set data->done and decrement drained_end_counter before bdrv_wakeup() */
- atomic_mb_set(&data->done, true);
+ qatomic_mb_set(&data->done, true);
if (!data->begin) {
- atomic_dec(data->drained_end_counter);
+ qatomic_dec(data->drained_end_counter);
}
bdrv_dec_in_flight(bs);
};
if (!begin) {
- atomic_inc(drained_end_counter);
+ qatomic_inc(drained_end_counter);
}
/* Make sure the driver callback completes during the polling phase for
return true;
}
- if (atomic_read(&bs->in_flight)) {
+ if (qatomic_read(&bs->in_flight)) {
return true;
}
assert(!qemu_in_coroutine());
/* Stop things in parent-to-child order */
- if (atomic_fetch_inc(&bs->quiesce_counter) == 0) {
+ if (qatomic_fetch_inc(&bs->quiesce_counter) == 0) {
aio_disable_external(bdrv_get_aio_context(bs));
}
bdrv_parent_drained_end(bs, parent, ignore_bds_parents,
drained_end_counter);
- old_quiesce_counter = atomic_fetch_dec(&bs->quiesce_counter);
+ old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter);
if (old_quiesce_counter == 1) {
aio_enable_external(bdrv_get_aio_context(bs));
}
{
int drained_end_counter = 0;
bdrv_do_drained_end(bs, false, NULL, false, &drained_end_counter);
- BDRV_POLL_WHILE(bs, atomic_read(&drained_end_counter) > 0);
+ BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0);
}
void bdrv_drained_end_no_poll(BlockDriverState *bs, int *drained_end_counter)
{
int drained_end_counter = 0;
bdrv_do_drained_end(bs, true, NULL, false, &drained_end_counter);
- BDRV_POLL_WHILE(bs, atomic_read(&drained_end_counter) > 0);
+ BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0);
}
void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent)
&drained_end_counter);
}
- BDRV_POLL_WHILE(child->bs, atomic_read(&drained_end_counter) > 0);
+ BDRV_POLL_WHILE(child->bs, qatomic_read(&drained_end_counter) > 0);
}
/*
{
BdrvChild *child, *next;
- assert(atomic_read(&bs->in_flight) == 0);
+ assert(qatomic_read(&bs->in_flight) == 0);
QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
bdrv_drain_assert_idle(child->bs);
}
}
assert(qemu_get_current_aio_context() == qemu_get_aio_context());
- AIO_WAIT_WHILE(NULL, atomic_read(&drained_end_counter) > 0);
+ AIO_WAIT_WHILE(NULL, qatomic_read(&drained_end_counter) > 0);
assert(bdrv_drain_all_count > 0);
bdrv_drain_all_count--;
static void tracked_request_end(BdrvTrackedRequest *req)
{
if (req->serialising) {
- atomic_dec(&req->bs->serialising_in_flight);
+ qatomic_dec(&req->bs->serialising_in_flight);
}
qemu_co_mutex_lock(&req->bs->reqs_lock);
qemu_co_mutex_lock(&bs->reqs_lock);
if (!req->serialising) {
- atomic_inc(&req->bs->serialising_in_flight);
+ qatomic_inc(&req->bs->serialising_in_flight);
req->serialising = true;
}
void bdrv_inc_in_flight(BlockDriverState *bs)
{
- atomic_inc(&bs->in_flight);
+ qatomic_inc(&bs->in_flight);
}
void bdrv_wakeup(BlockDriverState *bs)
void bdrv_dec_in_flight(BlockDriverState *bs)
{
- atomic_dec(&bs->in_flight);
+ qatomic_dec(&bs->in_flight);
bdrv_wakeup(bs);
}
BlockDriverState *bs = self->bs;
bool waited = false;
- if (!atomic_read(&bs->serialising_in_flight)) {
+ if (!qatomic_read(&bs->serialising_in_flight)) {
return false;
}
bdrv_inc_in_flight(bs);
/* Don't do copy-on-read if we read data before write operation */
- if (atomic_read(&bs->copy_on_read)) {
+ if (qatomic_read(&bs->copy_on_read)) {
flags |= BDRV_REQ_COPY_ON_READ;
}
int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
BlockDriverState *bs = child->bs;
- atomic_inc(&bs->write_gen);
+ qatomic_inc(&bs->write_gen);
/*
* Discard cannot extend the image, but in error handling cases, such as
}
qemu_co_mutex_lock(&bs->reqs_lock);
- current_gen = atomic_read(&bs->write_gen);
+ current_gen = qatomic_read(&bs->write_gen);
/* Wait until any previous flushes are completed */
while (bs->active_flush_req) {
bdrv_io_plug(child->bs);
}
- if (atomic_fetch_inc(&bs->io_plugged) == 0) {
+ if (qatomic_fetch_inc(&bs->io_plugged) == 0) {
BlockDriver *drv = bs->drv;
if (drv && drv->bdrv_io_plug) {
drv->bdrv_io_plug(bs);
BdrvChild *child;
assert(bs->io_plugged);
- if (atomic_fetch_dec(&bs->io_plugged) == 1) {
+ if (qatomic_fetch_dec(&bs->io_plugged) == 1) {
BlockDriver *drv = bs->drv;
if (drv && drv->bdrv_io_unplug) {
drv->bdrv_io_unplug(bs);
}
/* Set task->complete before reading bs->wakeup. */
- atomic_mb_set(&task->complete, 1);
+ qatomic_mb_set(&task->complete, 1);
bdrv_wakeup(task->bs);
}
srco->co = NULL;
srco->ret = ret;
/* Set srco->finished before reading bs->wakeup. */
- atomic_mb_set(&srco->finished, true);
+ qatomic_mb_set(&srco->finished, true);
if (srco->bs) {
bdrv_wakeup(srco->bs);
}
* immediately if it has pending requests. Otherwise we could be
* forcing it to wait for other member's throttled requests. */
if (tgm_has_pending_reqs(tgm, is_write) &&
- atomic_read(&tgm->io_limits_disabled)) {
+ qatomic_read(&tgm->io_limits_disabled)) {
return tgm;
}
ThrottleTimers *tt = &tgm->throttle_timers;
bool must_wait;
- if (atomic_read(&tgm->io_limits_disabled)) {
+ if (qatomic_read(&tgm->io_limits_disabled)) {
return false;
}
g_free(data);
- atomic_dec(&tgm->restart_pending);
+ qatomic_dec(&tgm->restart_pending);
aio_wait_kick();
}
* be no timer pending on this tgm at this point */
assert(!timer_pending(tgm->throttle_timers.timers[is_write]));
- atomic_inc(&tgm->restart_pending);
+ qatomic_inc(&tgm->restart_pending);
co = qemu_coroutine_create(throttle_group_restart_queue_entry, rd);
aio_co_enter(tgm->aio_context, co);
tgm->throttle_state = ts;
tgm->aio_context = ctx;
- atomic_set(&tgm->restart_pending, 0);
+ qatomic_set(&tgm->restart_pending, 0);
qemu_mutex_lock(&tg->lock);
/* If the ThrottleGroup is new set this ThrottleGroupMember as the token */
}
/* Wait for throttle_group_restart_queue_entry() coroutines to finish */
- AIO_WAIT_WHILE(tgm->aio_context, atomic_read(&tgm->restart_pending) > 0);
+ AIO_WAIT_WHILE(tgm->aio_context, qatomic_read(&tgm->restart_pending) > 0);
qemu_mutex_lock(&tg->lock);
for (i = 0; i < 2; i++) {
static void coroutine_fn throttle_co_drain_begin(BlockDriverState *bs)
{
ThrottleGroupMember *tgm = bs->opaque;
- if (atomic_fetch_inc(&tgm->io_limits_disabled) == 0) {
+ if (qatomic_fetch_inc(&tgm->io_limits_disabled) == 0) {
throttle_group_restart_tgm(tgm);
}
}
{
ThrottleGroupMember *tgm = bs->opaque;
assert(tgm->io_limits_disabled);
- atomic_dec(&tgm->io_limits_disabled);
+ qatomic_dec(&tgm->io_limits_disabled);
}
static const char *const throttle_strong_runtime_opts[] = {
/* We don't need (or want) to use the transactional
* bdrv_reopen_multiple() across all the entries at once, because we
* don't want to abort all of them if one of them fails the reopen */
- if (!atomic_read(&state->old_bs->copy_on_read)) {
+ if (!qatomic_read(&state->old_bs->copy_on_read)) {
bdrv_reopen_set_read_only(state->old_bs, true, NULL);
}
info = g_new0(BlockJobInfo, 1);
info->type = g_strdup(job_type_str(&job->job));
info->device = g_strdup(job->job.id);
- info->busy = atomic_read(&job->job.busy);
+ info->busy = qatomic_read(&job->job.busy);
info->paused = job->job.pause_count > 0;
info->offset = job->job.progress.current;
info->len = job->job.progress.total;
vu_log_page(uint8_t *log_table, uint64_t page)
{
DPRINT("Logged dirty guest page: %"PRId64"\n", page);
- atomic_or(&log_table[page / 8], 1 << (page % 8));
+ qatomic_or(&log_table[page / 8], 1 << (page % 8));
}
static void
wi.exclusive = false;
queue_work_on_cpu(cpu, &wi);
- while (!atomic_mb_read(&wi.done)) {
+ while (!qatomic_mb_read(&wi.done)) {
CPUState *self_cpu = current_cpu;
qemu_cond_wait(&qemu_work_cond, mutex);
exclusive_idle();
/* Make all other cpus stop executing. */
- atomic_set(&pending_cpus, 1);
+ qatomic_set(&pending_cpus, 1);
/* Write pending_cpus before reading other_cpu->running. */
smp_mb();
running_cpus = 0;
CPU_FOREACH(other_cpu) {
- if (atomic_read(&other_cpu->running)) {
+ if (qatomic_read(&other_cpu->running)) {
other_cpu->has_waiter = true;
running_cpus++;
qemu_cpu_kick(other_cpu);
}
}
- atomic_set(&pending_cpus, running_cpus + 1);
+ qatomic_set(&pending_cpus, running_cpus + 1);
while (pending_cpus > 1) {
qemu_cond_wait(&exclusive_cond, &qemu_cpu_list_lock);
}
current_cpu->in_exclusive_context = false;
qemu_mutex_lock(&qemu_cpu_list_lock);
- atomic_set(&pending_cpus, 0);
+ qatomic_set(&pending_cpus, 0);
qemu_cond_broadcast(&exclusive_resume);
qemu_mutex_unlock(&qemu_cpu_list_lock);
}
/* Wait for exclusive ops to finish, and begin cpu execution. */
void cpu_exec_start(CPUState *cpu)
{
- atomic_set(&cpu->running, true);
+ qatomic_set(&cpu->running, true);
/* Write cpu->running before reading pending_cpus. */
smp_mb();
* 3. pending_cpus == 0. Then start_exclusive is definitely going to
* see cpu->running == true, and it will kick the CPU.
*/
- if (unlikely(atomic_read(&pending_cpus))) {
+ if (unlikely(qatomic_read(&pending_cpus))) {
QEMU_LOCK_GUARD(&qemu_cpu_list_lock);
if (!cpu->has_waiter) {
/* Not counted in pending_cpus, let the exclusive item
* run. Since we have the lock, just set cpu->running to true
* while holding it; no need to check pending_cpus again.
*/
- atomic_set(&cpu->running, false);
+ qatomic_set(&cpu->running, false);
exclusive_idle();
/* Now pending_cpus is zero. */
- atomic_set(&cpu->running, true);
+ qatomic_set(&cpu->running, true);
} else {
/* Counted in pending_cpus, go ahead and release the
* waiter at cpu_exec_end.
/* Mark cpu as not executing, and release pending exclusive ops. */
void cpu_exec_end(CPUState *cpu)
{
- atomic_set(&cpu->running, false);
+ qatomic_set(&cpu->running, false);
/* Write cpu->running before reading pending_cpus. */
smp_mb();
* see cpu->running == false, and it can ignore this CPU until the
* next cpu_exec_start.
*/
- if (unlikely(atomic_read(&pending_cpus))) {
+ if (unlikely(qatomic_read(&pending_cpus))) {
QEMU_LOCK_GUARD(&qemu_cpu_list_lock);
if (cpu->has_waiter) {
cpu->has_waiter = false;
- atomic_set(&pending_cpus, pending_cpus - 1);
+ qatomic_set(&pending_cpus, pending_cpus - 1);
if (pending_cpus == 1) {
qemu_cond_signal(&exclusive_cond);
}
if (wi->free) {
g_free(wi);
} else {
- atomic_mb_set(&wi->done, true);
+ qatomic_mb_set(&wi->done, true);
}
}
qemu_mutex_unlock(&cpu->work_mutex);
- compiler barriers: ``barrier()``;
-- weak atomic access and manual memory barriers: ``atomic_read()``,
- ``atomic_set()``, ``smp_rmb()``, ``smp_wmb()``, ``smp_mb()``, ``smp_mb_acquire()``,
- ``smp_mb_release()``, ``smp_read_barrier_depends()``;
+- weak atomic access and manual memory barriers: ``qatomic_read()``,
+ ``qatomic_set()``, ``smp_rmb()``, ``smp_wmb()``, ``smp_mb()``,
+ ``smp_mb_acquire()``, ``smp_mb_release()``, ``smp_read_barrier_depends()``;
- sequentially consistent atomic access: everything else.
``qemu/atomic.h`` provides the following set of atomic read-modify-write
operations::
- void atomic_inc(ptr)
- void atomic_dec(ptr)
- void atomic_add(ptr, val)
- void atomic_sub(ptr, val)
- void atomic_and(ptr, val)
- void atomic_or(ptr, val)
-
- typeof(*ptr) atomic_fetch_inc(ptr)
- typeof(*ptr) atomic_fetch_dec(ptr)
- typeof(*ptr) atomic_fetch_add(ptr, val)
- typeof(*ptr) atomic_fetch_sub(ptr, val)
- typeof(*ptr) atomic_fetch_and(ptr, val)
- typeof(*ptr) atomic_fetch_or(ptr, val)
- typeof(*ptr) atomic_fetch_xor(ptr, val)
- typeof(*ptr) atomic_fetch_inc_nonzero(ptr)
- typeof(*ptr) atomic_xchg(ptr, val)
- typeof(*ptr) atomic_cmpxchg(ptr, old, new)
+ void qatomic_inc(ptr)
+ void qatomic_dec(ptr)
+ void qatomic_add(ptr, val)
+ void qatomic_sub(ptr, val)
+ void qatomic_and(ptr, val)
+ void qatomic_or(ptr, val)
+
+ typeof(*ptr) qatomic_fetch_inc(ptr)
+ typeof(*ptr) qatomic_fetch_dec(ptr)
+ typeof(*ptr) qatomic_fetch_add(ptr, val)
+ typeof(*ptr) qatomic_fetch_sub(ptr, val)
+ typeof(*ptr) qatomic_fetch_and(ptr, val)
+ typeof(*ptr) qatomic_fetch_or(ptr, val)
+ typeof(*ptr) qatomic_fetch_xor(ptr, val)
+ typeof(*ptr) qatomic_fetch_inc_nonzero(ptr)
+ typeof(*ptr) qatomic_xchg(ptr, val)
+ typeof(*ptr) qatomic_cmpxchg(ptr, old, new)
all of which return the old value of ``*ptr``. These operations are
polymorphic; they operate on any type that is as wide as a pointer or
Similar operations return the new value of ``*ptr``::
- typeof(*ptr) atomic_inc_fetch(ptr)
- typeof(*ptr) atomic_dec_fetch(ptr)
- typeof(*ptr) atomic_add_fetch(ptr, val)
- typeof(*ptr) atomic_sub_fetch(ptr, val)
- typeof(*ptr) atomic_and_fetch(ptr, val)
- typeof(*ptr) atomic_or_fetch(ptr, val)
- typeof(*ptr) atomic_xor_fetch(ptr, val)
+ typeof(*ptr) qatomic_inc_fetch(ptr)
+ typeof(*ptr) qatomic_dec_fetch(ptr)
+ typeof(*ptr) qatomic_add_fetch(ptr, val)
+ typeof(*ptr) qatomic_sub_fetch(ptr, val)
+ typeof(*ptr) qatomic_and_fetch(ptr, val)
+ typeof(*ptr) qatomic_or_fetch(ptr, val)
+ typeof(*ptr) qatomic_xor_fetch(ptr, val)
``qemu/atomic.h`` also provides loads and stores that cannot be reordered
with each other::
- typeof(*ptr) atomic_mb_read(ptr)
- void atomic_mb_set(ptr, val)
+ typeof(*ptr) qatomic_mb_read(ptr)
+ void qatomic_mb_set(ptr, val)
However these do not provide sequential consistency and, in particular,
they do not participate in the total ordering enforced by
- lightweight synchronization primitives such as ``QemuEvent``
-- RCU operations (``atomic_rcu_read``, ``atomic_rcu_set``) when publishing
+- RCU operations (``qatomic_rcu_read``, ``qatomic_rcu_set``) when publishing
or accessing a new version of a data structure
-- other atomic accesses: ``atomic_read`` and ``atomic_load_acquire`` for
- loads, ``atomic_set`` and ``atomic_store_release`` for stores, ``smp_mb``
+- other atomic accesses: ``qatomic_read`` and ``qatomic_load_acquire`` for
+ loads, ``qatomic_set`` and ``qatomic_store_release`` for stores, ``smp_mb``
to forbid reordering subsequent loads before a store.
When using this model, variables are accessed with:
-- ``atomic_read()`` and ``atomic_set()``; these prevent the compiler from
+- ``qatomic_read()`` and ``qatomic_set()``; these prevent the compiler from
optimizing accesses out of existence and creating unsolicited
accesses, but do not otherwise impose any ordering on loads and
stores: both the compiler and the processor are free to reorder
them.
-- ``atomic_load_acquire()``, which guarantees the LOAD to appear to
+- ``qatomic_load_acquire()``, which guarantees the LOAD to appear to
happen, with respect to the other components of the system,
before all the LOAD or STORE operations specified afterwards.
- Operations coming before ``atomic_load_acquire()`` can still be
+ Operations coming before ``qatomic_load_acquire()`` can still be
reordered after it.
-- ``atomic_store_release()``, which guarantees the STORE to appear to
+- ``qatomic_store_release()``, which guarantees the STORE to appear to
happen, with respect to the other components of the system,
after all the LOAD or STORE operations specified before.
- Operations coming after ``atomic_store_release()`` can still be
+ Operations coming after ``qatomic_store_release()`` can still be
reordered before it.
Restrictions to the ordering of accesses can also be specified
dependency and a full read barrier or better is required.
-Memory barriers and ``atomic_load_acquire``/``atomic_store_release`` are
+Memory barriers and ``qatomic_load_acquire``/``qatomic_store_release`` are
mostly used when a data structure has one thread that is always a writer
and one thread that is always a reader:
+==================================+==================================+
| :: | :: |
| | |
- | atomic_store_release(&a, x); | y = atomic_load_acquire(&b); |
- | atomic_store_release(&b, y); | x = atomic_load_acquire(&a); |
+ | qatomic_store_release(&a, x); | y = qatomic_load_acquire(&b); |
+ | qatomic_store_release(&b, y); | x = qatomic_load_acquire(&a); |
+----------------------------------+----------------------------------+
In this case, correctness is easy to check for using the "pairing"
| | |
| n = 0; | n = 0; |
| for (i = 0; i < 10; i++) | for (i = 0; i < 10; i++) |
- | n += atomic_load_acquire(&a[i]); | n += atomic_read(&a[i]); |
+ | n += qatomic_load_acquire(&a[i]); | n += qatomic_read(&a[i]); |
| | smp_mb_acquire(); |
+------------------------------------------+----------------------------------+
| :: | :: |
| | |
| | smp_mb_release(); |
| for (i = 0; i < 10; i++) | for (i = 0; i < 10; i++) |
- | atomic_store_release(&a[i], false); | atomic_set(&a[i], false); |
+ | qatomic_store_release(&a[i], false); | qatomic_set(&a[i], false); |
+------------------------------------------+----------------------------------+
Splitting a loop can also be useful to reduce the number of barriers:
| | |
| n = 0; | smp_mb_release(); |
| for (i = 0; i < 10; i++) { | for (i = 0; i < 10; i++) |
- | atomic_store_release(&a[i], false); | atomic_set(&a[i], false); |
+ | qatomic_store_release(&a[i], false); | qatomic_set(&a[i], false); |
| smp_mb(); | smb_mb(); |
- | n += atomic_read(&b[i]); | n = 0; |
+ | n += qatomic_read(&b[i]); | n = 0; |
| } | for (i = 0; i < 10; i++) |
- | | n += atomic_read(&b[i]); |
+ | | n += qatomic_read(&b[i]); |
+------------------------------------------+----------------------------------+
In this case, a ``smp_mb_release()`` is also replaced with a (possibly cheaper, and clearer
| | |
| | smp_mb_release(); |
| for (i = 0; i < 10; i++) { | for (i = 0; i < 10; i++) |
- | atomic_store_release(&a[i], false); | atomic_set(&a[i], false); |
- | atomic_store_release(&b[i], false); | smb_wmb(); |
+ | qatomic_store_release(&a[i], false); | qatomic_set(&a[i], false); |
+ | qatomic_store_release(&b[i], false); | smb_wmb(); |
| } | for (i = 0; i < 10; i++) |
- | | atomic_set(&b[i], false); |
+ | | qatomic_set(&b[i], false); |
+------------------------------------------+----------------------------------+
Acquire/release pairing and the *synchronizes-with* relation
------------------------------------------------------------
-Atomic operations other than ``atomic_set()`` and ``atomic_read()`` have
+Atomic operations other than ``qatomic_set()`` and ``qatomic_read()`` have
either *acquire* or *release* semantics [#rmw]_. This has two effects:
.. [#rmw] Read-modify-write operations can have both---acquire applies to the
Synchronization between threads basically descends from this pairing of
a release operation and an acquire operation. Therefore, atomic operations
-other than ``atomic_set()`` and ``atomic_read()`` will almost always be
+other than ``qatomic_set()`` and ``qatomic_read()`` will almost always be
paired with another operation of the opposite kind: an acquire operation
will pair with a release operation and vice versa. This rule of thumb is
extremely useful; in the case of QEMU, however, note that the other
operation may actually be in a driver that runs in the guest!
``smp_read_barrier_depends()``, ``smp_rmb()``, ``smp_mb_acquire()``,
-``atomic_load_acquire()`` and ``atomic_rcu_read()`` all count
+``qatomic_load_acquire()`` and ``qatomic_rcu_read()`` all count
as acquire operations. ``smp_wmb()``, ``smp_mb_release()``,
-``atomic_store_release()`` and ``atomic_rcu_set()`` all count as release
+``qatomic_store_release()`` and ``qatomic_rcu_set()`` all count as release
operations. ``smp_mb()`` counts as both acquire and release, therefore
it can pair with any other atomic operation. Here is an example:
+======================+==============================+
| :: | :: |
| | |
- | atomic_set(&a, 1); | |
+ | qatomic_set(&a, 1);| |
| smp_wmb(); | |
- | atomic_set(&b, 2); | x = atomic_read(&b); |
+ | qatomic_set(&b, 2);| x = qatomic_read(&b); |
| | smp_rmb(); |
- | | y = atomic_read(&a); |
+ | | y = qatomic_read(&a); |
+----------------------+------------------------------+
Note that a load-store pair only counts if the two operations access the
+================================+================================+
| :: | :: |
| | |
- | atomic_set(&a, 1); | |
- | atomic_store_release(&b, 2); | x = atomic_load_acquire(&b); |
- | | y = atomic_read(&a); |
+ | qatomic_set(&a, 1); | |
+ | qatomic_store_release(&b, 2);| x = qatomic_load_acquire(&b);|
+ | | y = qatomic_read(&a); |
+--------------------------------+--------------------------------+
Acquire and release semantics of higher-level primitives can also be
| smp_wmb(); | |
| x->i = 2; | |
| smp_wmb(); | |
- | atomic_set(&a, x); | x = atomic_read(&a); |
+ | qatomic_set(&a, x);| x = qatomic_read(&a); |
| | smp_read_barrier_depends(); |
| | y = x->i; |
| | smp_read_barrier_depends(); |
at all. Linux 4.1 updated them to implement volatile
semantics via ``ACCESS_ONCE`` (or the more recent ``READ``/``WRITE_ONCE``).
- QEMU's ``atomic_read`` and ``atomic_set`` implement C11 atomic relaxed
+ QEMU's ``qatomic_read`` and ``qatomic_set`` implement C11 atomic relaxed
semantics if the compiler supports it, and volatile semantics otherwise.
Both semantics prevent the compiler from doing certain transformations;
the difference is that atomic accesses are guaranteed to be atomic,
since we assume the variables passed are machine-word sized and
properly aligned.
- No barriers are implied by ``atomic_read`` and ``atomic_set`` in either Linux
- or QEMU.
+ No barriers are implied by ``qatomic_read`` and ``qatomic_set`` in either
+ Linux or QEMU.
- atomic read-modify-write operations in Linux are of three kinds:
a different set of memory barriers; in QEMU, all of them enforce
sequential consistency.
-- in QEMU, ``atomic_read()`` and ``atomic_set()`` do not participate in
+- in QEMU, ``qatomic_read()`` and ``qatomic_set()`` do not participate in
the total ordering enforced by sequentially-consistent operations.
This is because QEMU uses the C11 memory model. The following example
is correct in Linux but not in QEMU:
+==================================+================================+
| :: | :: |
| | |
- | a = atomic_fetch_add(&x, 2); | a = atomic_fetch_add(&x, 2); |
- | b = READ_ONCE(&y); | b = atomic_read(&y); |
+ | a = atomic_fetch_add(&x, 2); | a = qatomic_fetch_add(&x, 2);|
+ | b = READ_ONCE(&y); | b = qatomic_read(&y); |
+----------------------------------+--------------------------------+
because the read of ``y`` can be moved (by either the processor or the
+================================+
| :: |
| |
- | a = atomic_read(&x); |
- | atomic_set(&x, a + 2); |
+ | a = qatomic_read(&x); |
+ | qatomic_set(&x, a + 2); |
| smp_mb(); |
- | b = atomic_read(&y); |
+ | b = qatomic_read(&y); |
+--------------------------------+
Sources
Reads, instead, can be done without taking the mutex, as long as the
readers and writers use the same macros that are used for RCU, for
-example atomic_rcu_read, atomic_rcu_set, QLIST_FOREACH_RCU, etc. This is
+example qatomic_rcu_read, qatomic_rcu_set, QLIST_FOREACH_RCU, etc. This is
because the reads are done outside a lock and a set or QLIST_INSERT_HEAD
can happen concurrently with the read. The RCU API ensures that the
processor and the compiler see all required memory barriers.
if (!xyz) {
new_xyz = g_new(XYZ, 1);
...
- atomic_rcu_set(&xyz, new_xyz);
+ qatomic_rcu_set(&xyz, new_xyz);
}
qemu_lockcnt_unlock(&xyz_lockcnt);
qemu_lockcnt_inc(&xyz_lockcnt);
if (xyz) {
- XYZ *p = atomic_rcu_read(&xyz);
+ XYZ *p = qatomic_rcu_read(&xyz);
...
/* Accesses can now be done through "p". */
}
qemu_lockcnt_inc(&xyz_lockcnt);
if (xyz) {
- XYZ *p = atomic_rcu_read(&xyz);
+ XYZ *p = qatomic_rcu_read(&xyz);
...
/* Accesses can now be done through "p". */
}
g_free_rcu(&foo, rcu);
- typeof(*p) atomic_rcu_read(p);
+ typeof(*p) qatomic_rcu_read(p);
- atomic_rcu_read() is similar to atomic_load_acquire(), but it makes
+ qatomic_rcu_read() is similar to qatomic_load_acquire(), but it makes
some assumptions on the code that calls it. This allows a more
optimized implementation.
- atomic_rcu_read assumes that whenever a single RCU critical
+ qatomic_rcu_read assumes that whenever a single RCU critical
section reads multiple shared data, these reads are either
data-dependent or need no ordering. This is almost always the
case when using RCU, because read-side critical sections typically
every update) until reaching a data structure of interest,
and then read from there.
- RCU read-side critical sections must use atomic_rcu_read() to
+ RCU read-side critical sections must use qatomic_rcu_read() to
read data, unless concurrent writes are prevented by another
synchronization mechanism.
data structure in a single direction, opposite to the direction
in which the updater initializes it.
- void atomic_rcu_set(p, typeof(*p) v);
+ void qatomic_rcu_set(p, typeof(*p) v);
- atomic_rcu_set() is similar to atomic_store_release(), though it also
+ qatomic_rcu_set() is similar to qatomic_store_release(), though it also
makes assumptions on the code that calls it in order to allow a more
optimized implementation.
- In particular, atomic_rcu_set() suffices for synchronization
+ In particular, qatomic_rcu_set() suffices for synchronization
with readers, if the updater never mutates a field within a
data item that is already accessible to readers. This is the
case when initializing a new copy of the RCU-protected data
structure; just ensure that initialization of *p is carried out
- before atomic_rcu_set() makes the data item visible to readers.
+ before qatomic_rcu_set() makes the data item visible to readers.
If this rule is observed, writes will happen in the opposite
order as reads in the RCU read-side critical sections (or if
there is just one update), and there will be no need for other
programming; not allowing this would prevent upgrading an RCU read-side
critical section to become an updater.
-- atomic_rcu_read and atomic_rcu_set replace rcu_dereference and
+- qatomic_rcu_read and qatomic_rcu_set replace rcu_dereference and
rcu_assign_pointer. They take a _pointer_ to the variable being accessed.
- call_rcu is a macro that has an extra argument (the name of the first
consider the following code fragment:
rcu_read_lock();
- p = atomic_rcu_read(&foo);
+ p = qatomic_rcu_read(&foo);
/* do something with p. */
rcu_read_unlock();
qemu_mutex_lock(&foo_mutex);
old = foo;
- atomic_rcu_set(&foo, new);
+ qatomic_rcu_set(&foo, new);
qemu_mutex_unlock(&foo_mutex);
synchronize_rcu();
free(old);
is possible to combine this idiom with a "real" reference count:
rcu_read_lock();
- p = atomic_rcu_read(&foo);
+ p = qatomic_rcu_read(&foo);
foo_ref(p);
rcu_read_unlock();
/* do something with p. */
qemu_mutex_lock(&foo_mutex);
old = foo;
- atomic_rcu_set(&foo, new);
+ qatomic_rcu_set(&foo, new);
qemu_mutex_unlock(&foo_mutex);
synchronize_rcu();
foo_unref(old);
qemu_mutex_lock(&foo_mutex);
old = foo;
- atomic_rcu_set(&foo, new);
+ qatomic_rcu_set(&foo, new);
qemu_mutex_unlock(&foo_mutex);
call_rcu(foo_unref, old, rcu);
use call_rcu() instead:
foo_unref(struct foo *p) {
- if (atomic_fetch_dec(&p->refcount) == 1) {
+ if (qatomic_fetch_dec(&p->refcount) == 1) {
call_rcu(foo_destroy, p, rcu);
}
}
read side:
rcu_read_lock();
- struct arr *array = atomic_rcu_read(&global_array);
+ struct arr *array = qatomic_rcu_read(&global_array);
x = i < array->size ? array->data[i] : -1;
rcu_read_unlock();
return x;
/* Removal phase. */
old_array = global_array;
- atomic_rcu_set(&new_array->data, new_array);
+ qatomic_rcu_set(&new_array->data, new_array);
synchronize_rcu();
/* Reclamation phase. */
bool dump_in_progress(void)
{
DumpState *state = &dump_state_global;
- return (atomic_read(&state->status) == DUMP_STATUS_ACTIVE);
+ return (qatomic_read(&state->status) == DUMP_STATUS_ACTIVE);
}
/* calculate total size of memory to be dumped (taking filter into
/* make sure status is written after written_size updates */
smp_wmb();
- atomic_set(&s->status,
+ qatomic_set(&s->status,
(local_err ? DUMP_STATUS_FAILED : DUMP_STATUS_COMPLETED));
/* send DUMP_COMPLETED message (unconditionally) */
{
DumpQueryResult *result = g_new(DumpQueryResult, 1);
DumpState *state = &dump_state_global;
- result->status = atomic_read(&state->status);
+ result->status = qatomic_read(&state->status);
/* make sure we are reading status and written_size in order */
smp_rmb();
result->completed = state->written_size;
begin, length, &local_err);
if (local_err) {
error_propagate(errp, local_err);
- atomic_set(&s->status, DUMP_STATUS_FAILED);
+ qatomic_set(&s->status, DUMP_STATUS_FAILED);
return;
}
hwaddr addr,
bool resolve_subpage)
{
- MemoryRegionSection *section = atomic_read(&d->mru_section);
+ MemoryRegionSection *section = qatomic_read(&d->mru_section);
subpage_t *subpage;
if (!section || section == &d->map.sections[PHYS_SECTION_UNASSIGNED] ||
!section_covers_addr(section, addr)) {
section = phys_page_find(d, addr);
- atomic_set(&d->mru_section, section);
+ qatomic_set(&d->mru_section, section);
}
if (resolve_subpage && section->mr->subpage) {
subpage = container_of(section->mr, subpage_t, iomem);
IOMMUMemoryRegionClass *imrc;
IOMMUTLBEntry iotlb;
int iommu_idx;
- AddressSpaceDispatch *d = atomic_rcu_read(&cpu->cpu_ases[asidx].memory_dispatch);
+ AddressSpaceDispatch *d =
+ qatomic_rcu_read(&cpu->cpu_ases[asidx].memory_dispatch);
for (;;) {
section = address_space_translate_internal(d, addr, &addr, plen, false);
{
RAMBlock *block;
- block = atomic_rcu_read(&ram_list.mru_block);
+ block = qatomic_rcu_read(&ram_list.mru_block);
if (block && addr - block->offset < block->max_length) {
return block;
}
* call_rcu(reclaim_ramblock, xxx);
* rcu_read_unlock()
*
- * atomic_rcu_set is not needed here. The block was already published
+ * qatomic_rcu_set is not needed here. The block was already published
* when it was placed into the list. Here we're just making an extra
* copy of the pointer.
*/
page = start_page;
WITH_RCU_READ_LOCK_GUARD() {
- blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
+ blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
ramblock = qemu_get_ram_block(start);
/* Range sanity check on the ramblock */
assert(start >= ramblock->offset &&
dest = 0;
WITH_RCU_READ_LOCK_GUARD() {
- blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
+ blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
while (page < end) {
unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
DirtyMemoryBlocks *new_blocks;
int j;
- old_blocks = atomic_rcu_read(&ram_list.dirty_memory[i]);
+ old_blocks = qatomic_rcu_read(&ram_list.dirty_memory[i]);
new_blocks = g_malloc(sizeof(*new_blocks) +
sizeof(new_blocks->blocks[0]) * new_num_blocks);
new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
}
- atomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
+ qatomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
if (old_blocks) {
g_free_rcu(old_blocks, rcu);
}
RCU_READ_LOCK_GUARD();
- block = atomic_rcu_read(&ram_list.mru_block);
+ block = qatomic_rcu_read(&ram_list.mru_block);
if (block && block->host && host - block->host < block->max_length) {
goto found;
}
{
int asidx = cpu_asidx_from_attrs(cpu, attrs);
CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
- AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
+ AddressSpaceDispatch *d = qatomic_rcu_read(&cpuas->memory_dispatch);
MemoryRegionSection *sections = d->map.sections;
return §ions[index & ~TARGET_PAGE_MASK];
* may have split the RCU critical section.
*/
d = address_space_to_dispatch(cpuas->as);
- atomic_rcu_set(&cpuas->memory_dispatch, d);
+ qatomic_rcu_set(&cpuas->memory_dispatch, d);
tlb_flush(cpuas->cpu);
}
qemu_mutex_lock(&map_client_list_lock);
client->bh = bh;
QLIST_INSERT_HEAD(&map_client_list, client, link);
- if (!atomic_read(&bounce.in_use)) {
+ if (!qatomic_read(&bounce.in_use)) {
cpu_notify_map_clients_locked();
}
qemu_mutex_unlock(&map_client_list_lock);
mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
if (!memory_access_is_direct(mr, is_write)) {
- if (atomic_xchg(&bounce.in_use, true)) {
+ if (qatomic_xchg(&bounce.in_use, true)) {
*plen = 0;
return NULL;
}
qemu_vfree(bounce.buffer);
bounce.buffer = NULL;
memory_region_unref(bounce.mr);
- atomic_mb_set(&bounce.in_use, false);
+ qatomic_mb_set(&bounce.in_use, false);
cpu_notify_map_clients();
}
int old;
if (!state) {
- atomic_dec(&ram_block_discard_disabled);
+ qatomic_dec(&ram_block_discard_disabled);
return 0;
}
do {
- old = atomic_read(&ram_block_discard_disabled);
+ old = qatomic_read(&ram_block_discard_disabled);
if (old < 0) {
return -EBUSY;
}
- } while (atomic_cmpxchg(&ram_block_discard_disabled, old, old + 1) != old);
+ } while (qatomic_cmpxchg(&ram_block_discard_disabled,
+ old, old + 1) != old);
return 0;
}
int old;
if (!state) {
- atomic_inc(&ram_block_discard_disabled);
+ qatomic_inc(&ram_block_discard_disabled);
return 0;
}
do {
- old = atomic_read(&ram_block_discard_disabled);
+ old = qatomic_read(&ram_block_discard_disabled);
if (old > 0) {
return -EBUSY;
}
- } while (atomic_cmpxchg(&ram_block_discard_disabled, old, old - 1) != old);
+ } while (qatomic_cmpxchg(&ram_block_discard_disabled,
+ old, old - 1) != old);
return 0;
}
bool ram_block_discard_is_disabled(void)
{
- return atomic_read(&ram_block_discard_disabled) > 0;
+ return qatomic_read(&ram_block_discard_disabled) > 0;
}
bool ram_block_discard_is_required(void)
{
- return atomic_read(&ram_block_discard_disabled) < 0;
+ return qatomic_read(&ram_block_discard_disabled) < 0;
}
#endif
void cpu_exit(CPUState *cpu)
{
- atomic_set(&cpu->exit_request, 1);
+ qatomic_set(&cpu->exit_request, 1);
/* Ensure cpu_exec will see the exit request after TCG has exited. */
smp_wmb();
- atomic_set(&cpu->icount_decr_ptr->u16.high, -1);
+ qatomic_set(&cpu->icount_decr_ptr->u16.high, -1);
}
int cpu_write_elf32_qemunote(WriteCoreDumpFunction f, CPUState *cpu,
cpu->halted = cpu->start_powered_off;
cpu->mem_io_pc = 0;
cpu->icount_extra = 0;
- atomic_set(&cpu->icount_decr_ptr->u32, 0);
+ qatomic_set(&cpu->icount_decr_ptr->u32, 0);
cpu->can_do_io = 1;
cpu->exception_index = -1;
cpu->crash_occurred = false;
/*
* Older versions of Spice forgot to define the QXLRam struct
* with the '__aligned__(4)' attribute. clang 7 and newer will
- * thus warn that atomic_fetch_or(&d->ram->int_pending, ...)
+ * thus warn that qatomic_fetch_or(&d->ram->int_pending, ...)
* might be a misaligned atomic access, and will generate an
* out-of-line call for it, which results in a link error since
* we don't currently link against libatomic.
#define ALIGNED_UINT32_PTR(P) ((uint32_t *)P)
#endif
- old_pending = atomic_fetch_or(ALIGNED_UINT32_PTR(&d->ram->int_pending),
+ old_pending = qatomic_fetch_or(ALIGNED_UINT32_PTR(&d->ram->int_pending),
le_events);
if ((old_pending & le_events) == le_events) {
return;
HvSintRoute *sint_route = opaque;
HvSintStagedMessage *staged_msg = sint_route->staged_msg;
- if (atomic_read(&staged_msg->state) != HV_STAGED_MSG_POSTED) {
+ if (qatomic_read(&staged_msg->state) != HV_STAGED_MSG_POSTED) {
/* status nor ready yet (spurious ack from guest?), ignore */
return;
}
staged_msg->status = 0;
/* staged message processing finished, ready to start over */
- atomic_set(&staged_msg->state, HV_STAGED_MSG_FREE);
+ qatomic_set(&staged_msg->state, HV_STAGED_MSG_FREE);
/* drop the reference taken in hyperv_post_msg */
hyperv_sint_route_unref(sint_route);
}
memory_region_set_dirty(&synic->msg_page_mr, 0, sizeof(*synic->msg_page));
posted:
- atomic_set(&staged_msg->state, HV_STAGED_MSG_POSTED);
+ qatomic_set(&staged_msg->state, HV_STAGED_MSG_POSTED);
/*
* Notify the msg originator of the progress made; if the slot was busy we
* set msg_pending flag in it so it will be the guest who will do EOM and
assert(staged_msg);
/* grab the staging area */
- if (atomic_cmpxchg(&staged_msg->state, HV_STAGED_MSG_FREE,
+ if (qatomic_cmpxchg(&staged_msg->state, HV_STAGED_MSG_FREE,
HV_STAGED_MSG_BUSY) != HV_STAGED_MSG_FREE) {
return -EAGAIN;
}
set_mask = BIT_MASK(eventno);
flags = synic->event_page->slot[sint_route->sint].flags;
- if ((atomic_fetch_or(&flags[set_idx], set_mask) & set_mask) != set_mask) {
+ if ((qatomic_fetch_or(&flags[set_idx], set_mask) & set_mask) != set_mask) {
memory_region_set_dirty(&synic->event_page_mr, 0,
sizeof(*synic->event_page));
ret = hyperv_sint_route_set_sint(sint_route);
idx = BIT_WORD(chan->id);
mask = BIT_MASK(chan->id);
- if ((atomic_fetch_or(&int_map[idx], mask) & mask) != mask) {
+ if ((qatomic_fetch_or(&int_map[idx], mask) & mask) != mask) {
res = hyperv_sint_route_set_sint(chan->notify_route);
dirty = len;
}
assert(req.dir == IOREQ_WRITE);
assert(!req.data_is_ptr);
- atomic_add(&buf_page->read_pointer, qw + 1);
+ qatomic_add(&buf_page->read_pointer, qw + 1);
}
return req.count;
int enable;
enable = icu->ier[n_IRQ / 8] & (1 << (n_IRQ & 7));
- if (n_IRQ > 0 && enable != 0 && atomic_read(&icu->req_irq) < 0) {
- atomic_set(&icu->req_irq, n_IRQ);
+ if (n_IRQ > 0 && enable != 0 && qatomic_read(&icu->req_irq) < 0) {
+ qatomic_set(&icu->req_irq, n_IRQ);
set_irq(icu, n_IRQ, rxicu_level(icu, n_IRQ));
}
}
}
if (issue == 0 && src->sense == TRG_LEVEL) {
icu->ir[n_IRQ] = 0;
- if (atomic_read(&icu->req_irq) == n_IRQ) {
+ if (qatomic_read(&icu->req_irq) == n_IRQ) {
/* clear request */
set_irq(icu, n_IRQ, 0);
- atomic_set(&icu->req_irq, -1);
+ qatomic_set(&icu->req_irq, -1);
}
return;
}
int n_IRQ;
int max_pri;
- n_IRQ = atomic_read(&icu->req_irq);
+ n_IRQ = qatomic_read(&icu->req_irq);
if (n_IRQ < 0) {
return;
}
- atomic_set(&icu->req_irq, -1);
+ qatomic_set(&icu->req_irq, -1);
if (icu->src[n_IRQ].sense != TRG_LEVEL) {
icu->ir[n_IRQ] = 0;
}
static uint32_t atomic_set_masked(uint32_t *a, uint32_t mask, uint32_t value)
{
- uint32_t old, new, cmp = atomic_read(a);
+ uint32_t old, new, cmp = qatomic_read(a);
do {
old = cmp;
new = (old & ~mask) | (value & mask);
- cmp = atomic_cmpxchg(a, old, new);
+ cmp = qatomic_cmpxchg(a, old, new);
} while (old != cmp);
return old;
qemu_mutex_unlock(&edu->thr_mutex);
break;
case 0x20:
- val = atomic_read(&edu->status);
+ val = qatomic_read(&edu->status);
break;
case 0x24:
val = edu->irq_status;
edu->addr4 = ~val;
break;
case 0x08:
- if (atomic_read(&edu->status) & EDU_STATUS_COMPUTING) {
+ if (qatomic_read(&edu->status) & EDU_STATUS_COMPUTING) {
break;
}
/* EDU_STATUS_COMPUTING cannot go 0->1 concurrently, because it is only
*/
qemu_mutex_lock(&edu->thr_mutex);
edu->fact = val;
- atomic_or(&edu->status, EDU_STATUS_COMPUTING);
+ qatomic_or(&edu->status, EDU_STATUS_COMPUTING);
qemu_cond_signal(&edu->thr_cond);
qemu_mutex_unlock(&edu->thr_mutex);
break;
case 0x20:
if (val & EDU_STATUS_IRQFACT) {
- atomic_or(&edu->status, EDU_STATUS_IRQFACT);
+ qatomic_or(&edu->status, EDU_STATUS_IRQFACT);
} else {
- atomic_and(&edu->status, ~EDU_STATUS_IRQFACT);
+ qatomic_and(&edu->status, ~EDU_STATUS_IRQFACT);
}
break;
case 0x60:
uint32_t val, ret = 1;
qemu_mutex_lock(&edu->thr_mutex);
- while ((atomic_read(&edu->status) & EDU_STATUS_COMPUTING) == 0 &&
+ while ((qatomic_read(&edu->status) & EDU_STATUS_COMPUTING) == 0 &&
!edu->stopping) {
qemu_cond_wait(&edu->thr_cond, &edu->thr_mutex);
}
qemu_mutex_lock(&edu->thr_mutex);
edu->fact = ret;
qemu_mutex_unlock(&edu->thr_mutex);
- atomic_and(&edu->status, ~EDU_STATUS_COMPUTING);
+ qatomic_and(&edu->status, ~EDU_STATUS_COMPUTING);
- if (atomic_read(&edu->status) & EDU_STATUS_IRQFACT) {
+ if (qatomic_read(&edu->status) & EDU_STATUS_IRQFACT) {
qemu_mutex_lock_iothread();
edu_raise_irq(edu, FACT_IRQ);
qemu_mutex_unlock_iothread();
if (virtio_has_feature(features, VIRTIO_NET_F_STANDBY)) {
qapi_event_send_failover_negotiated(n->netclient_name);
- atomic_set(&n->primary_should_be_hidden, false);
+ qatomic_set(&n->primary_should_be_hidden, false);
failover_add_primary(n, &err);
if (err) {
n->primary_dev = virtio_connect_failover_devices(n, n->qdev, &err);
bool should_be_hidden;
Error *err = NULL;
- should_be_hidden = atomic_read(&n->primary_should_be_hidden);
+ should_be_hidden = qatomic_read(&n->primary_should_be_hidden);
if (!n->primary_dev) {
n->primary_dev = virtio_connect_failover_devices(n, n->qdev, &err);
qdev_get_vmsd(n->primary_dev),
n->primary_dev);
qapi_event_send_unplug_primary(n->primary_device_id);
- atomic_set(&n->primary_should_be_hidden, true);
+ qatomic_set(&n->primary_should_be_hidden, true);
} else {
warn_report("couldn't unplug primary device");
}
n->primary_device_opts = device_opts;
/* primary_should_be_hidden is set during feature negotiation */
- hide = atomic_read(&n->primary_should_be_hidden);
+ hide = qatomic_read(&n->primary_should_be_hidden);
if (n->primary_device_dict) {
g_free(n->primary_device_id);
if (n->failover) {
n->primary_listener.should_be_hidden =
virtio_net_primary_should_be_hidden;
- atomic_set(&n->primary_should_be_hidden, true);
+ qatomic_set(&n->primary_should_be_hidden, true);
device_listener_register(&n->primary_listener);
n->migration_state.notify = virtio_net_migration_state_notifier;
add_migration_state_change_notifier(&n->migration_state);
bctx = rdma_rm_get_cqe_ctx(rdma_dev_res, cqe_ctx_id);
if (bctx) {
rdma_rm_dealloc_cqe_ctx(rdma_dev_res, cqe_ctx_id);
- atomic_dec(&rdma_dev_res->stats.missing_cqe);
+ qatomic_dec(&rdma_dev_res->stats.missing_cqe);
}
g_free(bctx);
}
cqe_ctx_id = rdma_protected_qlist_pop_int64(&backend_dev->
recv_mads_list);
if (cqe_ctx_id != -ENOENT) {
- atomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
+ qatomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
free_cqe_ctx(GINT_TO_POINTER(cqe_ctx_id),
backend_dev->rdma_dev_res);
}
}
total_ne += ne;
} while (ne > 0);
- atomic_sub(&rdma_dev_res->stats.missing_cqe, total_ne);
+ qatomic_sub(&rdma_dev_res->stats.missing_cqe, total_ne);
}
if (ne < 0) {
static inline void disable_rdmacm_mux_async(RdmaBackendDev *backend_dev)
{
- atomic_set(&backend_dev->rdmacm_mux.can_receive, 0);
+ qatomic_set(&backend_dev->rdmacm_mux.can_receive, 0);
}
static inline void enable_rdmacm_mux_async(RdmaBackendDev *backend_dev)
{
- atomic_set(&backend_dev->rdmacm_mux.can_receive, sizeof(RdmaCmMuxMsg));
+ qatomic_set(&backend_dev->rdmacm_mux.can_receive, sizeof(RdmaCmMuxMsg));
}
static inline int rdmacm_mux_can_process_async(RdmaBackendDev *backend_dev)
{
- return atomic_read(&backend_dev->rdmacm_mux.can_receive);
+ return qatomic_read(&backend_dev->rdmacm_mux.can_receive);
}
static int rdmacm_mux_check_op_status(CharBackend *mad_chr_be)
goto err_dealloc_cqe_ctx;
}
- atomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
+ qatomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
backend_dev->rdma_dev_res->stats.tx++;
return;
goto err_dealloc_cqe_ctx;
}
- atomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
+ qatomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
backend_dev->rdma_dev_res->stats.rx_bufs++;
return;
goto err_dealloc_cqe_ctx;
}
- atomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
+ qatomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
backend_dev->rdma_dev_res->stats.rx_bufs++;
backend_dev->rdma_dev_res->stats.rx_srq++;
qemu_mutex_init(&dev_res->lock);
memset(&dev_res->stats, 0, sizeof(dev_res->stats));
- atomic_set(&dev_res->stats.missing_cqe, 0);
+ qatomic_set(&dev_res->stats.missing_cqe, 0);
return 0;
}
ring->max_elems = max_elems;
ring->elem_sz = elem_sz;
/* TODO: Give a moment to think if we want to redo driver settings
- atomic_set(&ring->ring_state->prod_tail, 0);
- atomic_set(&ring->ring_state->cons_head, 0);
+ qatomic_set(&ring->ring_state->prod_tail, 0);
+ qatomic_set(&ring->ring_state->cons_head, 0);
*/
ring->npages = npages;
ring->pages = g_malloc(npages * sizeof(void *));
actual = *ind_addr;
do {
expected = actual;
- actual = atomic_cmpxchg(ind_addr, expected, expected | to_be_set);
+ actual = qatomic_cmpxchg(ind_addr, expected, expected | to_be_set);
} while (actual != expected);
cpu_physical_memory_unmap((void *)ind_addr, len, 1, len);
actual = *ind_addr;
do {
expected = actual;
- actual = atomic_cmpxchg(ind_addr, expected, expected | to_be_set);
+ actual = qatomic_cmpxchg(ind_addr, expected, expected | to_be_set);
} while (actual != expected);
trace_virtio_ccw_set_ind(ind_loc, actual, actual | to_be_set);
cpu_physical_memory_unmap((void *)ind_addr, len, 1, len);
}
/* Data must be read atomically. We don't really need barrier semantics
* but it's easier to use atomic_* than roll our own. */
- log = atomic_xchg(from, 0);
+ log = qatomic_xchg(from, 0);
while (log) {
int bit = ctzl(log);
hwaddr page_addr;
}
return proxy->vqs[vdev->queue_sel].enabled;
case VIRTIO_MMIO_INTERRUPT_STATUS:
- return atomic_read(&vdev->isr);
+ return qatomic_read(&vdev->isr);
case VIRTIO_MMIO_STATUS:
return vdev->status;
case VIRTIO_MMIO_CONFIG_GENERATION:
}
break;
case VIRTIO_MMIO_INTERRUPT_ACK:
- atomic_and(&vdev->isr, ~value);
+ qatomic_and(&vdev->isr, ~value);
virtio_update_irq(vdev);
break;
case VIRTIO_MMIO_STATUS:
if (!vdev) {
return;
}
- level = (atomic_read(&vdev->isr) != 0);
+ level = (qatomic_read(&vdev->isr) != 0);
trace_virtio_mmio_setting_irq(level);
qemu_set_irq(proxy->irq, level);
}
msix_notify(&proxy->pci_dev, vector);
else {
VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
- pci_set_irq(&proxy->pci_dev, atomic_read(&vdev->isr) & 1);
+ pci_set_irq(&proxy->pci_dev, qatomic_read(&vdev->isr) & 1);
}
}
break;
case VIRTIO_PCI_ISR:
/* reading from the ISR also clears it. */
- ret = atomic_xchg(&vdev->isr, 0);
+ ret = qatomic_xchg(&vdev->isr, 0);
pci_irq_deassert(&proxy->pci_dev);
break;
case VIRTIO_MSI_CONFIG_VECTOR:
{
VirtIOPCIProxy *proxy = opaque;
VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
- uint64_t val = atomic_xchg(&vdev->isr, 0);
+ uint64_t val = qatomic_xchg(&vdev->isr, 0);
pci_irq_deassert(&proxy->pci_dev);
return val;
{
VRingMemoryRegionCaches *caches;
- caches = atomic_read(&vq->vring.caches);
- atomic_rcu_set(&vq->vring.caches, NULL);
+ caches = qatomic_read(&vq->vring.caches);
+ qatomic_rcu_set(&vq->vring.caches, NULL);
if (caches) {
call_rcu(caches, virtio_free_region_cache, rcu);
}
goto err_avail;
}
- atomic_rcu_set(&vq->vring.caches, new);
+ qatomic_rcu_set(&vq->vring.caches, new);
if (old) {
call_rcu(old, virtio_free_region_cache, rcu);
}
/* Called within rcu_read_lock(). */
static VRingMemoryRegionCaches *vring_get_region_caches(struct VirtQueue *vq)
{
- return atomic_rcu_read(&vq->vring.caches);
+ return qatomic_rcu_read(&vq->vring.caches);
}
/* Called within rcu_read_lock(). */
vdev->queue_sel = 0;
vdev->status = 0;
vdev->disabled = false;
- atomic_set(&vdev->isr, 0);
+ qatomic_set(&vdev->isr, 0);
vdev->config_vector = VIRTIO_NO_VECTOR;
virtio_notify_vector(vdev, vdev->config_vector);
static void virtio_set_isr(VirtIODevice *vdev, int value)
{
- uint8_t old = atomic_read(&vdev->isr);
+ uint8_t old = qatomic_read(&vdev->isr);
/* Do not write ISR if it does not change, so that its cacheline remains
* shared in the common case where the guest does not read it.
*/
if ((old & value) != value) {
- atomic_or(&vdev->isr, value);
+ qatomic_or(&vdev->isr, value);
}
}
vdev->started = false;
vdev->device_id = device_id;
vdev->status = 0;
- atomic_set(&vdev->isr, 0);
+ qatomic_set(&vdev->isr, 0);
vdev->queue_sel = 0;
vdev->config_vector = VIRTIO_NO_VECTOR;
vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX);
uint32_t irq_bit = 1 << irq;
if (active) {
- atomic_or(&env->sregs[INTSET], irq_bit);
+ qatomic_or(&env->sregs[INTSET], irq_bit);
} else if (env->config->interrupt[irq].inttype == INTTYPE_LEVEL) {
- atomic_and(&env->sregs[INTSET], ~irq_bit);
+ qatomic_and(&env->sregs[INTSET], ~irq_bit);
}
check_interrupts(env);
AioWait *wait_ = &global_aio_wait; \
AioContext *ctx_ = (ctx); \
/* Increment wait_->num_waiters before evaluating cond. */ \
- atomic_inc(&wait_->num_waiters); \
+ qatomic_inc(&wait_->num_waiters); \
if (ctx_ && in_aio_context_home_thread(ctx_)) { \
while ((cond)) { \
aio_poll(ctx_, true); \
waited_ = true; \
} \
} \
- atomic_dec(&wait_->num_waiters); \
+ qatomic_dec(&wait_->num_waiters); \
waited_; })
/**
*/
static inline void aio_disable_external(AioContext *ctx)
{
- atomic_inc(&ctx->external_disable_cnt);
+ qatomic_inc(&ctx->external_disable_cnt);
}
/**
{
int old;
- old = atomic_fetch_dec(&ctx->external_disable_cnt);
+ old = qatomic_fetch_dec(&ctx->external_disable_cnt);
assert(old > 0);
if (old == 1) {
/* Kick event loop so it re-arms file descriptors */
*/
static inline bool aio_external_disabled(AioContext *ctx)
{
- return atomic_read(&ctx->external_disable_cnt);
+ return qatomic_read(&ctx->external_disable_cnt);
}
/**
*/
static inline bool aio_node_check(AioContext *ctx, bool is_external)
{
- return !is_external || !atomic_read(&ctx->external_disable_cnt);
+ return !is_external || !qatomic_read(&ctx->external_disable_cnt);
}
/**
#if TCG_OVERSIZED_GUEST
return entry->addr_write;
#else
- return atomic_read(&entry->addr_write);
+ return qatomic_read(&entry->addr_write);
#endif
}
*/
static inline bool cpu_loop_exit_requested(CPUState *cpu)
{
- return (int32_t)atomic_read(&cpu_neg(cpu)->icount_decr.u32) < 0;
+ return (int32_t)qatomic_read(&cpu_neg(cpu)->icount_decr.u32) < 0;
}
#if !defined(CONFIG_USER_ONLY)
extern bool parallel_cpus;
-/* Hide the atomic_read to make code a little easier on the eyes */
+/* Hide the qatomic_read to make code a little easier on the eyes */
static inline uint32_t tb_cflags(const TranslationBlock *tb)
{
- return atomic_read(&tb->cflags);
+ return qatomic_read(&tb->cflags);
}
/* current cflags for hashing/comparison */
if (qemu_log_enabled()) {
rcu_read_lock();
- logfile = atomic_rcu_read(&qemu_logfile);
+ logfile = qatomic_rcu_read(&qemu_logfile);
if (logfile) {
cpu_dump_state(cpu, logfile->fd, flags);
}
{
QemuLogFile *logfile;
rcu_read_lock();
- logfile = atomic_rcu_read(&qemu_logfile);
+ logfile = qatomic_rcu_read(&qemu_logfile);
if (logfile) {
target_disas(logfile->fd, cpu, start, len);
}
{
QemuLogFile *logfile;
rcu_read_lock();
- logfile = atomic_rcu_read(&qemu_logfile);
+ logfile = qatomic_rcu_read(&qemu_logfile);
if (logfile) {
disas(logfile->fd, code, size, note);
}
static inline FlatView *address_space_to_flatview(AddressSpace *as)
{
- return atomic_rcu_read(&as->current_map);
+ return qatomic_rcu_read(&as->current_map);
}
page = start >> TARGET_PAGE_BITS;
WITH_RCU_READ_LOCK_GUARD() {
- blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
+ blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
idx = page / DIRTY_MEMORY_BLOCK_SIZE;
offset = page % DIRTY_MEMORY_BLOCK_SIZE;
RCU_READ_LOCK_GUARD();
- blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
+ blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
idx = page / DIRTY_MEMORY_BLOCK_SIZE;
offset = page % DIRTY_MEMORY_BLOCK_SIZE;
RCU_READ_LOCK_GUARD();
- blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
+ blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
set_bit_atomic(offset, blocks->blocks[idx]);
}
WITH_RCU_READ_LOCK_GUARD() {
for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
- blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i]);
+ blocks[i] = qatomic_rcu_read(&ram_list.dirty_memory[i]);
}
idx = page / DIRTY_MEMORY_BLOCK_SIZE;
WITH_RCU_READ_LOCK_GUARD() {
for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
- blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
+ blocks[i] =
+ qatomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
}
for (k = 0; k < nr; k++) {
if (bitmap[k]) {
unsigned long temp = leul_to_cpu(bitmap[k]);
- atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
+ qatomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
if (global_dirty_log) {
- atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset],
- temp);
+ qatomic_or(
+ &blocks[DIRTY_MEMORY_MIGRATION][idx][offset],
+ temp);
}
if (tcg_enabled()) {
- atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset],
- temp);
+ qatomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset],
+ temp);
}
}
DIRTY_MEMORY_BLOCK_SIZE);
unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
- src = atomic_rcu_read(
+ src = qatomic_rcu_read(
&ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
for (k = page; k < page + nr; k++) {
if (src[idx][offset]) {
- unsigned long bits = atomic_xchg(&src[idx][offset], 0);
+ unsigned long bits = qatomic_xchg(&src[idx][offset], 0);
unsigned long new_dirty;
new_dirty = ~dest[k];
dest[k] |= bits;
* rcu_read_lock();
*
* DirtyMemoryBlocks *blocks =
- * atomic_rcu_read(&ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION]);
+ * qatomic_rcu_read(&ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION]);
*
* ram_addr_t idx = (addr >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
* unsigned long *block = blocks.blocks[idx];
cpu_get_tb_cpu_state(env, pc, cs_base, flags);
hash = tb_jmp_cache_hash_func(*pc);
- tb = atomic_rcu_read(&cpu->tb_jmp_cache[hash]);
+ tb = qatomic_rcu_read(&cpu->tb_jmp_cache[hash]);
cf_mask &= ~CF_CLUSTER_MASK;
cf_mask |= cpu->cluster_index << CF_CLUSTER_SHIFT;
if (tb == NULL) {
return NULL;
}
- atomic_set(&cpu->tb_jmp_cache[hash], tb);
+ qatomic_set(&cpu->tb_jmp_cache[hash], tb);
return tb;
}
unsigned int i;
for (i = 0; i < TB_JMP_CACHE_SIZE; i++) {
- atomic_set(&cpu->tb_jmp_cache[i], NULL);
+ qatomic_set(&cpu->tb_jmp_cache[i], NULL);
}
}
* no effect on the generated code but not using the atomic primitives
* will get flagged by sanitizers as a violation.
*/
-#define atomic_read__nocheck(ptr) \
+#define qatomic_read__nocheck(ptr) \
__atomic_load_n(ptr, __ATOMIC_RELAXED)
-#define atomic_read(ptr) \
- ({ \
+#define qatomic_read(ptr) \
+ ({ \
QEMU_BUILD_BUG_ON(sizeof(*ptr) > ATOMIC_REG_SIZE); \
- atomic_read__nocheck(ptr); \
+ qatomic_read__nocheck(ptr); \
})
-#define atomic_set__nocheck(ptr, i) \
+#define qatomic_set__nocheck(ptr, i) \
__atomic_store_n(ptr, i, __ATOMIC_RELAXED)
-#define atomic_set(ptr, i) do { \
+#define qatomic_set(ptr, i) do { \
QEMU_BUILD_BUG_ON(sizeof(*ptr) > ATOMIC_REG_SIZE); \
- atomic_set__nocheck(ptr, i); \
+ qatomic_set__nocheck(ptr, i); \
} while(0)
/* See above: most compilers currently treat consume and acquire the
- * same, but this slows down atomic_rcu_read unnecessarily.
+ * same, but this slows down qatomic_rcu_read unnecessarily.
*/
#ifdef __SANITIZE_THREAD__
-#define atomic_rcu_read__nocheck(ptr, valptr) \
+#define qatomic_rcu_read__nocheck(ptr, valptr) \
__atomic_load(ptr, valptr, __ATOMIC_CONSUME);
#else
-#define atomic_rcu_read__nocheck(ptr, valptr) \
- __atomic_load(ptr, valptr, __ATOMIC_RELAXED); \
+#define qatomic_rcu_read__nocheck(ptr, valptr) \
+ __atomic_load(ptr, valptr, __ATOMIC_RELAXED); \
smp_read_barrier_depends();
#endif
-#define atomic_rcu_read(ptr) \
- ({ \
+#define qatomic_rcu_read(ptr) \
+ ({ \
QEMU_BUILD_BUG_ON(sizeof(*ptr) > ATOMIC_REG_SIZE); \
- typeof_strip_qual(*ptr) _val; \
- atomic_rcu_read__nocheck(ptr, &_val); \
- _val; \
+ typeof_strip_qual(*ptr) _val; \
+ qatomic_rcu_read__nocheck(ptr, &_val); \
+ _val; \
})
-#define atomic_rcu_set(ptr, i) do { \
+#define qatomic_rcu_set(ptr, i) do { \
QEMU_BUILD_BUG_ON(sizeof(*ptr) > ATOMIC_REG_SIZE); \
- __atomic_store_n(ptr, i, __ATOMIC_RELEASE); \
+ __atomic_store_n(ptr, i, __ATOMIC_RELEASE); \
} while(0)
-#define atomic_load_acquire(ptr) \
+#define qatomic_load_acquire(ptr) \
({ \
QEMU_BUILD_BUG_ON(sizeof(*ptr) > ATOMIC_REG_SIZE); \
typeof_strip_qual(*ptr) _val; \
_val; \
})
-#define atomic_store_release(ptr, i) do { \
+#define qatomic_store_release(ptr, i) do { \
QEMU_BUILD_BUG_ON(sizeof(*ptr) > ATOMIC_REG_SIZE); \
__atomic_store_n(ptr, i, __ATOMIC_RELEASE); \
} while(0)
/* All the remaining operations are fully sequentially consistent */
-#define atomic_xchg__nocheck(ptr, i) ({ \
+#define qatomic_xchg__nocheck(ptr, i) ({ \
__atomic_exchange_n(ptr, (i), __ATOMIC_SEQ_CST); \
})
-#define atomic_xchg(ptr, i) ({ \
+#define qatomic_xchg(ptr, i) ({ \
QEMU_BUILD_BUG_ON(sizeof(*ptr) > ATOMIC_REG_SIZE); \
- atomic_xchg__nocheck(ptr, i); \
+ qatomic_xchg__nocheck(ptr, i); \
})
/* Returns the eventual value, failed or not */
-#define atomic_cmpxchg__nocheck(ptr, old, new) ({ \
+#define qatomic_cmpxchg__nocheck(ptr, old, new) ({ \
typeof_strip_qual(*ptr) _old = (old); \
(void)__atomic_compare_exchange_n(ptr, &_old, new, false, \
__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); \
_old; \
})
-#define atomic_cmpxchg(ptr, old, new) ({ \
+#define qatomic_cmpxchg(ptr, old, new) ({ \
QEMU_BUILD_BUG_ON(sizeof(*ptr) > ATOMIC_REG_SIZE); \
- atomic_cmpxchg__nocheck(ptr, old, new); \
+ qatomic_cmpxchg__nocheck(ptr, old, new); \
})
/* Provide shorter names for GCC atomic builtins, return old value */
-#define atomic_fetch_inc(ptr) __atomic_fetch_add(ptr, 1, __ATOMIC_SEQ_CST)
-#define atomic_fetch_dec(ptr) __atomic_fetch_sub(ptr, 1, __ATOMIC_SEQ_CST)
-
-#ifndef atomic_fetch_add
-#define atomic_fetch_add(ptr, n) __atomic_fetch_add(ptr, n, __ATOMIC_SEQ_CST)
-#define atomic_fetch_sub(ptr, n) __atomic_fetch_sub(ptr, n, __ATOMIC_SEQ_CST)
-#define atomic_fetch_and(ptr, n) __atomic_fetch_and(ptr, n, __ATOMIC_SEQ_CST)
-#define atomic_fetch_or(ptr, n) __atomic_fetch_or(ptr, n, __ATOMIC_SEQ_CST)
-#define atomic_fetch_xor(ptr, n) __atomic_fetch_xor(ptr, n, __ATOMIC_SEQ_CST)
-#endif
-
-#define atomic_inc_fetch(ptr) __atomic_add_fetch(ptr, 1, __ATOMIC_SEQ_CST)
-#define atomic_dec_fetch(ptr) __atomic_sub_fetch(ptr, 1, __ATOMIC_SEQ_CST)
-#define atomic_add_fetch(ptr, n) __atomic_add_fetch(ptr, n, __ATOMIC_SEQ_CST)
-#define atomic_sub_fetch(ptr, n) __atomic_sub_fetch(ptr, n, __ATOMIC_SEQ_CST)
-#define atomic_and_fetch(ptr, n) __atomic_and_fetch(ptr, n, __ATOMIC_SEQ_CST)
-#define atomic_or_fetch(ptr, n) __atomic_or_fetch(ptr, n, __ATOMIC_SEQ_CST)
-#define atomic_xor_fetch(ptr, n) __atomic_xor_fetch(ptr, n, __ATOMIC_SEQ_CST)
+#define qatomic_fetch_inc(ptr) __atomic_fetch_add(ptr, 1, __ATOMIC_SEQ_CST)
+#define qatomic_fetch_dec(ptr) __atomic_fetch_sub(ptr, 1, __ATOMIC_SEQ_CST)
+
+#define qatomic_fetch_add(ptr, n) __atomic_fetch_add(ptr, n, __ATOMIC_SEQ_CST)
+#define qatomic_fetch_sub(ptr, n) __atomic_fetch_sub(ptr, n, __ATOMIC_SEQ_CST)
+#define qatomic_fetch_and(ptr, n) __atomic_fetch_and(ptr, n, __ATOMIC_SEQ_CST)
+#define qatomic_fetch_or(ptr, n) __atomic_fetch_or(ptr, n, __ATOMIC_SEQ_CST)
+#define qatomic_fetch_xor(ptr, n) __atomic_fetch_xor(ptr, n, __ATOMIC_SEQ_CST)
+
+#define qatomic_inc_fetch(ptr) __atomic_add_fetch(ptr, 1, __ATOMIC_SEQ_CST)
+#define qatomic_dec_fetch(ptr) __atomic_sub_fetch(ptr, 1, __ATOMIC_SEQ_CST)
+#define qatomic_add_fetch(ptr, n) __atomic_add_fetch(ptr, n, __ATOMIC_SEQ_CST)
+#define qatomic_sub_fetch(ptr, n) __atomic_sub_fetch(ptr, n, __ATOMIC_SEQ_CST)
+#define qatomic_and_fetch(ptr, n) __atomic_and_fetch(ptr, n, __ATOMIC_SEQ_CST)
+#define qatomic_or_fetch(ptr, n) __atomic_or_fetch(ptr, n, __ATOMIC_SEQ_CST)
+#define qatomic_xor_fetch(ptr, n) __atomic_xor_fetch(ptr, n, __ATOMIC_SEQ_CST)
/* And even shorter names that return void. */
-#define atomic_inc(ptr) ((void) __atomic_fetch_add(ptr, 1, __ATOMIC_SEQ_CST))
-#define atomic_dec(ptr) ((void) __atomic_fetch_sub(ptr, 1, __ATOMIC_SEQ_CST))
-#define atomic_add(ptr, n) ((void) __atomic_fetch_add(ptr, n, __ATOMIC_SEQ_CST))
-#define atomic_sub(ptr, n) ((void) __atomic_fetch_sub(ptr, n, __ATOMIC_SEQ_CST))
-#define atomic_and(ptr, n) ((void) __atomic_fetch_and(ptr, n, __ATOMIC_SEQ_CST))
-#define atomic_or(ptr, n) ((void) __atomic_fetch_or(ptr, n, __ATOMIC_SEQ_CST))
-#define atomic_xor(ptr, n) ((void) __atomic_fetch_xor(ptr, n, __ATOMIC_SEQ_CST))
+#define qatomic_inc(ptr) \
+ ((void) __atomic_fetch_add(ptr, 1, __ATOMIC_SEQ_CST))
+#define qatomic_dec(ptr) \
+ ((void) __atomic_fetch_sub(ptr, 1, __ATOMIC_SEQ_CST))
+#define qatomic_add(ptr, n) \
+ ((void) __atomic_fetch_add(ptr, n, __ATOMIC_SEQ_CST))
+#define qatomic_sub(ptr, n) \
+ ((void) __atomic_fetch_sub(ptr, n, __ATOMIC_SEQ_CST))
+#define qatomic_and(ptr, n) \
+ ((void) __atomic_fetch_and(ptr, n, __ATOMIC_SEQ_CST))
+#define qatomic_or(ptr, n) \
+ ((void) __atomic_fetch_or(ptr, n, __ATOMIC_SEQ_CST))
+#define qatomic_xor(ptr, n) \
+ ((void) __atomic_fetch_xor(ptr, n, __ATOMIC_SEQ_CST))
#else /* __ATOMIC_RELAXED */
* but it is a full barrier at the hardware level. Add a compiler barrier
* to make it a full barrier also at the compiler level.
*/
-#define atomic_xchg(ptr, i) (barrier(), __sync_lock_test_and_set(ptr, i))
+#define qatomic_xchg(ptr, i) (barrier(), __sync_lock_test_and_set(ptr, i))
#elif defined(_ARCH_PPC)
/* These will only be atomic if the processor does the fetch or store
* in a single issue memory operation
*/
-#define atomic_read__nocheck(p) (*(__typeof__(*(p)) volatile*) (p))
-#define atomic_set__nocheck(p, i) ((*(__typeof__(*(p)) volatile*) (p)) = (i))
+#define qatomic_read__nocheck(p) (*(__typeof__(*(p)) volatile*) (p))
+#define qatomic_set__nocheck(p, i) ((*(__typeof__(*(p)) volatile*) (p)) = (i))
-#define atomic_read(ptr) atomic_read__nocheck(ptr)
-#define atomic_set(ptr, i) atomic_set__nocheck(ptr,i)
+#define qatomic_read(ptr) qatomic_read__nocheck(ptr)
+#define qatomic_set(ptr, i) qatomic_set__nocheck(ptr,i)
/**
- * atomic_rcu_read - reads a RCU-protected pointer to a local variable
+ * qatomic_rcu_read - reads a RCU-protected pointer to a local variable
* into a RCU read-side critical section. The pointer can later be safely
* dereferenced within the critical section.
*
* Inserts memory barriers on architectures that require them (currently only
* Alpha) and documents which pointers are protected by RCU.
*
- * atomic_rcu_read also includes a compiler barrier to ensure that
+ * qatomic_rcu_read also includes a compiler barrier to ensure that
* value-speculative optimizations (e.g. VSS: Value Speculation
* Scheduling) does not perform the data read before the pointer read
* by speculating the value of the pointer.
*
- * Should match atomic_rcu_set(), atomic_xchg(), atomic_cmpxchg().
+ * Should match qatomic_rcu_set(), qatomic_xchg(), qatomic_cmpxchg().
*/
-#define atomic_rcu_read(ptr) ({ \
- typeof(*ptr) _val = atomic_read(ptr); \
+#define qatomic_rcu_read(ptr) ({ \
+ typeof(*ptr) _val = qatomic_read(ptr); \
smp_read_barrier_depends(); \
_val; \
})
/**
- * atomic_rcu_set - assigns (publicizes) a pointer to a new data structure
+ * qatomic_rcu_set - assigns (publicizes) a pointer to a new data structure
* meant to be read by RCU read-side critical sections.
*
* Documents which pointers will be dereferenced by RCU read-side critical
* them. It also makes sure the compiler does not reorder code initializing the
* data structure before its publication.
*
- * Should match atomic_rcu_read().
+ * Should match qatomic_rcu_read().
*/
-#define atomic_rcu_set(ptr, i) do { \
+#define qatomic_rcu_set(ptr, i) do { \
smp_wmb(); \
- atomic_set(ptr, i); \
+ qatomic_set(ptr, i); \
} while (0)
-#define atomic_load_acquire(ptr) ({ \
- typeof(*ptr) _val = atomic_read(ptr); \
+#define qatomic_load_acquire(ptr) ({ \
+ typeof(*ptr) _val = qatomic_read(ptr); \
smp_mb_acquire(); \
_val; \
})
-#define atomic_store_release(ptr, i) do { \
+#define qatomic_store_release(ptr, i) do { \
smp_mb_release(); \
- atomic_set(ptr, i); \
+ qatomic_set(ptr, i); \
} while (0)
-#ifndef atomic_xchg
+#ifndef qatomic_xchg
#if defined(__clang__)
-#define atomic_xchg(ptr, i) __sync_swap(ptr, i)
+#define qatomic_xchg(ptr, i) __sync_swap(ptr, i)
#else
/* __sync_lock_test_and_set() is documented to be an acquire barrier only. */
-#define atomic_xchg(ptr, i) (smp_mb(), __sync_lock_test_and_set(ptr, i))
+#define qatomic_xchg(ptr, i) (smp_mb(), __sync_lock_test_and_set(ptr, i))
#endif
#endif
-#define atomic_xchg__nocheck atomic_xchg
+#define qatomic_xchg__nocheck qatomic_xchg
/* Provide shorter names for GCC atomic builtins. */
-#define atomic_fetch_inc(ptr) __sync_fetch_and_add(ptr, 1)
-#define atomic_fetch_dec(ptr) __sync_fetch_and_add(ptr, -1)
-
-#ifndef atomic_fetch_add
-#define atomic_fetch_add(ptr, n) __sync_fetch_and_add(ptr, n)
-#define atomic_fetch_sub(ptr, n) __sync_fetch_and_sub(ptr, n)
-#define atomic_fetch_and(ptr, n) __sync_fetch_and_and(ptr, n)
-#define atomic_fetch_or(ptr, n) __sync_fetch_and_or(ptr, n)
-#define atomic_fetch_xor(ptr, n) __sync_fetch_and_xor(ptr, n)
-#endif
-
-#define atomic_inc_fetch(ptr) __sync_add_and_fetch(ptr, 1)
-#define atomic_dec_fetch(ptr) __sync_add_and_fetch(ptr, -1)
-#define atomic_add_fetch(ptr, n) __sync_add_and_fetch(ptr, n)
-#define atomic_sub_fetch(ptr, n) __sync_sub_and_fetch(ptr, n)
-#define atomic_and_fetch(ptr, n) __sync_and_and_fetch(ptr, n)
-#define atomic_or_fetch(ptr, n) __sync_or_and_fetch(ptr, n)
-#define atomic_xor_fetch(ptr, n) __sync_xor_and_fetch(ptr, n)
-
-#define atomic_cmpxchg(ptr, old, new) __sync_val_compare_and_swap(ptr, old, new)
-#define atomic_cmpxchg__nocheck(ptr, old, new) atomic_cmpxchg(ptr, old, new)
+#define qatomic_fetch_inc(ptr) __sync_fetch_and_add(ptr, 1)
+#define qatomic_fetch_dec(ptr) __sync_fetch_and_add(ptr, -1)
+
+#define qatomic_fetch_add(ptr, n) __sync_fetch_and_add(ptr, n)
+#define qatomic_fetch_sub(ptr, n) __sync_fetch_and_sub(ptr, n)
+#define qatomic_fetch_and(ptr, n) __sync_fetch_and_and(ptr, n)
+#define qatomic_fetch_or(ptr, n) __sync_fetch_and_or(ptr, n)
+#define qatomic_fetch_xor(ptr, n) __sync_fetch_and_xor(ptr, n)
+
+#define qatomic_inc_fetch(ptr) __sync_add_and_fetch(ptr, 1)
+#define qatomic_dec_fetch(ptr) __sync_add_and_fetch(ptr, -1)
+#define qatomic_add_fetch(ptr, n) __sync_add_and_fetch(ptr, n)
+#define qatomic_sub_fetch(ptr, n) __sync_sub_and_fetch(ptr, n)
+#define qatomic_and_fetch(ptr, n) __sync_and_and_fetch(ptr, n)
+#define qatomic_or_fetch(ptr, n) __sync_or_and_fetch(ptr, n)
+#define qatomic_xor_fetch(ptr, n) __sync_xor_and_fetch(ptr, n)
+
+#define qatomic_cmpxchg(ptr, old, new) \
+ __sync_val_compare_and_swap(ptr, old, new)
+#define qatomic_cmpxchg__nocheck(ptr, old, new) qatomic_cmpxchg(ptr, old, new)
/* And even shorter names that return void. */
-#define atomic_inc(ptr) ((void) __sync_fetch_and_add(ptr, 1))
-#define atomic_dec(ptr) ((void) __sync_fetch_and_add(ptr, -1))
-#define atomic_add(ptr, n) ((void) __sync_fetch_and_add(ptr, n))
-#define atomic_sub(ptr, n) ((void) __sync_fetch_and_sub(ptr, n))
-#define atomic_and(ptr, n) ((void) __sync_fetch_and_and(ptr, n))
-#define atomic_or(ptr, n) ((void) __sync_fetch_and_or(ptr, n))
-#define atomic_xor(ptr, n) ((void) __sync_fetch_and_xor(ptr, n))
+#define qatomic_inc(ptr) ((void) __sync_fetch_and_add(ptr, 1))
+#define qatomic_dec(ptr) ((void) __sync_fetch_and_add(ptr, -1))
+#define qatomic_add(ptr, n) ((void) __sync_fetch_and_add(ptr, n))
+#define qatomic_sub(ptr, n) ((void) __sync_fetch_and_sub(ptr, n))
+#define qatomic_and(ptr, n) ((void) __sync_fetch_and_and(ptr, n))
+#define qatomic_or(ptr, n) ((void) __sync_fetch_and_or(ptr, n))
+#define qatomic_xor(ptr, n) ((void) __sync_fetch_and_xor(ptr, n))
#endif /* __ATOMIC_RELAXED */
/* This is more efficient than a store plus a fence. */
#if !defined(__SANITIZE_THREAD__)
#if defined(__i386__) || defined(__x86_64__) || defined(__s390x__)
-#define atomic_mb_set(ptr, i) ((void)atomic_xchg(ptr, i))
+#define qatomic_mb_set(ptr, i) ((void)qatomic_xchg(ptr, i))
#endif
#endif
-/* atomic_mb_read/set semantics map Java volatile variables. They are
+/* qatomic_mb_read/set semantics map Java volatile variables. They are
* less expensive on some platforms (notably POWER) than fully
* sequentially consistent operations.
*
* use. See docs/devel/atomics.txt for more discussion.
*/
-#ifndef atomic_mb_read
-#define atomic_mb_read(ptr) \
- atomic_load_acquire(ptr)
+#ifndef qatomic_mb_read
+#define qatomic_mb_read(ptr) \
+ qatomic_load_acquire(ptr)
#endif
-#ifndef atomic_mb_set
-#define atomic_mb_set(ptr, i) do { \
- atomic_store_release(ptr, i); \
+#ifndef qatomic_mb_set
+#define qatomic_mb_set(ptr, i) do { \
+ qatomic_store_release(ptr, i); \
smp_mb(); \
} while(0)
#endif
-#define atomic_fetch_inc_nonzero(ptr) ({ \
- typeof_strip_qual(*ptr) _oldn = atomic_read(ptr); \
- while (_oldn && atomic_cmpxchg(ptr, _oldn, _oldn + 1) != _oldn) { \
- _oldn = atomic_read(ptr); \
+#define qatomic_fetch_inc_nonzero(ptr) ({ \
+ typeof_strip_qual(*ptr) _oldn = qatomic_read(ptr); \
+ while (_oldn && qatomic_cmpxchg(ptr, _oldn, _oldn + 1) != _oldn) { \
+ _oldn = qatomic_read(ptr); \
} \
_oldn; \
})
/* Abstractions to access atomically (i.e. "once") i64/u64 variables */
#ifdef CONFIG_ATOMIC64
-static inline int64_t atomic_read_i64(const int64_t *ptr)
+static inline int64_t qatomic_read_i64(const int64_t *ptr)
{
/* use __nocheck because sizeof(void *) might be < sizeof(u64) */
- return atomic_read__nocheck(ptr);
+ return qatomic_read__nocheck(ptr);
}
-static inline uint64_t atomic_read_u64(const uint64_t *ptr)
+static inline uint64_t qatomic_read_u64(const uint64_t *ptr)
{
- return atomic_read__nocheck(ptr);
+ return qatomic_read__nocheck(ptr);
}
-static inline void atomic_set_i64(int64_t *ptr, int64_t val)
+static inline void qatomic_set_i64(int64_t *ptr, int64_t val)
{
- atomic_set__nocheck(ptr, val);
+ qatomic_set__nocheck(ptr, val);
}
-static inline void atomic_set_u64(uint64_t *ptr, uint64_t val)
+static inline void qatomic_set_u64(uint64_t *ptr, uint64_t val)
{
- atomic_set__nocheck(ptr, val);
+ qatomic_set__nocheck(ptr, val);
}
-static inline void atomic64_init(void)
+static inline void qatomic64_init(void)
{
}
#else /* !CONFIG_ATOMIC64 */
-int64_t atomic_read_i64(const int64_t *ptr);
-uint64_t atomic_read_u64(const uint64_t *ptr);
-void atomic_set_i64(int64_t *ptr, int64_t val);
-void atomic_set_u64(uint64_t *ptr, uint64_t val);
-void atomic64_init(void);
+int64_t qatomic_read_i64(const int64_t *ptr);
+uint64_t qatomic_read_u64(const uint64_t *ptr);
+void qatomic_set_i64(int64_t *ptr, int64_t val);
+void qatomic_set_u64(uint64_t *ptr, uint64_t val);
+void qatomic64_init(void);
#endif /* !CONFIG_ATOMIC64 */
#endif /* QEMU_ATOMIC_H */
#if defined(CONFIG_ATOMIC128)
static inline Int128 atomic16_cmpxchg(Int128 *ptr, Int128 cmp, Int128 new)
{
- return atomic_cmpxchg__nocheck(ptr, cmp, new);
+ return qatomic_cmpxchg__nocheck(ptr, cmp, new);
}
# define HAVE_CMPXCHG128 1
#elif defined(CONFIG_CMPXCHG128)
#if defined(CONFIG_ATOMIC128)
static inline Int128 atomic16_read(Int128 *ptr)
{
- return atomic_read__nocheck(ptr);
+ return qatomic_read__nocheck(ptr);
}
static inline void atomic16_set(Int128 *ptr, Int128 val)
{
- atomic_set__nocheck(ptr, val);
+ qatomic_set__nocheck(ptr, val);
}
# define HAVE_ATOMIC128 1
unsigned long mask = BIT_MASK(nr);
unsigned long *p = addr + BIT_WORD(nr);
- atomic_or(p, mask);
+ qatomic_or(p, mask);
}
/**
* because the condition will be false no matter whether we read NULL or
* the pointer for any other coroutine.
*/
- assert(atomic_read(&mutex->locked) &&
+ assert(qatomic_read(&mutex->locked) &&
mutex->holder == qemu_coroutine_self());
}
bool res = false;
rcu_read_lock();
- logfile = atomic_rcu_read(&qemu_logfile);
+ logfile = qatomic_rcu_read(&qemu_logfile);
if (logfile && logfile->fd != stderr) {
res = true;
}
{
QemuLogFile *logfile;
rcu_read_lock();
- logfile = atomic_rcu_read(&qemu_logfile);
+ logfile = qatomic_rcu_read(&qemu_logfile);
if (logfile) {
qemu_flockfile(logfile->fd);
return logfile->fd;
QemuLogFile *logfile;
rcu_read_lock();
- logfile = atomic_rcu_read(&qemu_logfile);
+ logfile = qatomic_rcu_read(&qemu_logfile);
if (logfile) {
vfprintf(logfile->fd, fmt, va);
}
typeof(elm) save_sle_next; \
do { \
save_sle_next = (elm)->field.sle_next = (head)->slh_first; \
- } while (atomic_cmpxchg(&(head)->slh_first, save_sle_next, (elm)) != \
+ } while (qatomic_cmpxchg(&(head)->slh_first, save_sle_next, (elm)) !=\
save_sle_next); \
} while (/*CONSTCOND*/0)
#define QSLIST_MOVE_ATOMIC(dest, src) do { \
- (dest)->slh_first = atomic_xchg(&(src)->slh_first, NULL); \
+ (dest)->slh_first = qatomic_xchg(&(src)->slh_first, NULL); \
} while (/*CONSTCOND*/0)
#define QSLIST_REMOVE_HEAD(head, field) do { \
/*
* Simple queue access methods.
*/
-#define QSIMPLEQ_EMPTY_ATOMIC(head) (atomic_read(&((head)->sqh_first)) == NULL)
+#define QSIMPLEQ_EMPTY_ATOMIC(head) \
+ (qatomic_read(&((head)->sqh_first)) == NULL)
#define QSIMPLEQ_EMPTY(head) ((head)->sqh_first == NULL)
#define QSIMPLEQ_FIRST(head) ((head)->sqh_first)
#define QSIMPLEQ_NEXT(elm, field) ((elm)->field.sqe_next)
return;
}
- ctr = atomic_read(&rcu_gp_ctr);
- atomic_set(&p_rcu_reader->ctr, ctr);
+ ctr = qatomic_read(&rcu_gp_ctr);
+ qatomic_set(&p_rcu_reader->ctr, ctr);
/* Write p_rcu_reader->ctr before reading RCU-protected pointers. */
smp_mb_placeholder();
* smp_mb_placeholder(), this ensures writes to p_rcu_reader->ctr
* are sequentially consistent.
*/
- atomic_store_release(&p_rcu_reader->ctr, 0);
+ qatomic_store_release(&p_rcu_reader->ctr, 0);
/* Write p_rcu_reader->ctr before reading p_rcu_reader->waiting. */
smp_mb_placeholder();
- if (unlikely(atomic_read(&p_rcu_reader->waiting))) {
- atomic_set(&p_rcu_reader->waiting, false);
+ if (unlikely(qatomic_read(&p_rcu_reader->waiting))) {
+ qatomic_set(&p_rcu_reader->waiting, false);
qemu_event_set(&rcu_gp_event);
}
}
/*
* List access methods.
*/
-#define QLIST_EMPTY_RCU(head) (atomic_read(&(head)->lh_first) == NULL)
-#define QLIST_FIRST_RCU(head) (atomic_rcu_read(&(head)->lh_first))
-#define QLIST_NEXT_RCU(elm, field) (atomic_rcu_read(&(elm)->field.le_next))
+#define QLIST_EMPTY_RCU(head) (qatomic_read(&(head)->lh_first) == NULL)
+#define QLIST_FIRST_RCU(head) (qatomic_rcu_read(&(head)->lh_first))
+#define QLIST_NEXT_RCU(elm, field) (qatomic_rcu_read(&(elm)->field.le_next))
/*
* List functions.
/*
- * The difference between atomic_read/set and atomic_rcu_read/set
+ * The difference between qatomic_read/set and qatomic_rcu_read/set
* is in the including of a read/write memory barrier to the volatile
* access. atomic_rcu_* macros include the memory barrier, the
* plain atomic macros do not. Therefore, it should be correct to
#define QLIST_INSERT_AFTER_RCU(listelm, elm, field) do { \
(elm)->field.le_next = (listelm)->field.le_next; \
(elm)->field.le_prev = &(listelm)->field.le_next; \
- atomic_rcu_set(&(listelm)->field.le_next, (elm)); \
+ qatomic_rcu_set(&(listelm)->field.le_next, (elm)); \
if ((elm)->field.le_next != NULL) { \
(elm)->field.le_next->field.le_prev = \
&(elm)->field.le_next; \
#define QLIST_INSERT_BEFORE_RCU(listelm, elm, field) do { \
(elm)->field.le_prev = (listelm)->field.le_prev; \
(elm)->field.le_next = (listelm); \
- atomic_rcu_set((listelm)->field.le_prev, (elm)); \
+ qatomic_rcu_set((listelm)->field.le_prev, (elm)); \
(listelm)->field.le_prev = &(elm)->field.le_next; \
} while (/*CONSTCOND*/0)
#define QLIST_INSERT_HEAD_RCU(head, elm, field) do { \
(elm)->field.le_prev = &(head)->lh_first; \
(elm)->field.le_next = (head)->lh_first; \
- atomic_rcu_set((&(head)->lh_first), (elm)); \
+ qatomic_rcu_set((&(head)->lh_first), (elm)); \
if ((elm)->field.le_next != NULL) { \
(elm)->field.le_next->field.le_prev = \
&(elm)->field.le_next; \
(elm)->field.le_next->field.le_prev = \
(elm)->field.le_prev; \
} \
- atomic_set((elm)->field.le_prev, (elm)->field.le_next); \
+ qatomic_set((elm)->field.le_prev, (elm)->field.le_next); \
} while (/*CONSTCOND*/0)
/* List traversal must occur within an RCU critical section. */
#define QLIST_FOREACH_RCU(var, head, field) \
- for ((var) = atomic_rcu_read(&(head)->lh_first); \
+ for ((var) = qatomic_rcu_read(&(head)->lh_first); \
(var); \
- (var) = atomic_rcu_read(&(var)->field.le_next))
+ (var) = qatomic_rcu_read(&(var)->field.le_next))
/* List traversal must occur within an RCU critical section. */
#define QLIST_FOREACH_SAFE_RCU(var, head, field, next_var) \
- for ((var) = (atomic_rcu_read(&(head)->lh_first)); \
+ for ((var) = (qatomic_rcu_read(&(head)->lh_first)); \
(var) && \
- ((next_var) = atomic_rcu_read(&(var)->field.le_next), 1); \
+ ((next_var) = qatomic_rcu_read(&(var)->field.le_next), 1); \
(var) = (next_var))
/*
*/
/* Simple queue access methods */
-#define QSIMPLEQ_EMPTY_RCU(head) (atomic_read(&(head)->sqh_first) == NULL)
-#define QSIMPLEQ_FIRST_RCU(head) atomic_rcu_read(&(head)->sqh_first)
-#define QSIMPLEQ_NEXT_RCU(elm, field) atomic_rcu_read(&(elm)->field.sqe_next)
+#define QSIMPLEQ_EMPTY_RCU(head) \
+ (qatomic_read(&(head)->sqh_first) == NULL)
+#define QSIMPLEQ_FIRST_RCU(head) qatomic_rcu_read(&(head)->sqh_first)
+#define QSIMPLEQ_NEXT_RCU(elm, field) qatomic_rcu_read(&(elm)->field.sqe_next)
/* Simple queue functions */
#define QSIMPLEQ_INSERT_HEAD_RCU(head, elm, field) do { \
if ((elm)->field.sqe_next == NULL) { \
(head)->sqh_last = &(elm)->field.sqe_next; \
} \
- atomic_rcu_set(&(head)->sqh_first, (elm)); \
+ qatomic_rcu_set(&(head)->sqh_first, (elm)); \
} while (/*CONSTCOND*/0)
#define QSIMPLEQ_INSERT_TAIL_RCU(head, elm, field) do { \
(elm)->field.sqe_next = NULL; \
- atomic_rcu_set((head)->sqh_last, (elm)); \
+ qatomic_rcu_set((head)->sqh_last, (elm)); \
(head)->sqh_last = &(elm)->field.sqe_next; \
} while (/*CONSTCOND*/0)
if ((elm)->field.sqe_next == NULL) { \
(head)->sqh_last = &(elm)->field.sqe_next; \
} \
- atomic_rcu_set(&(listelm)->field.sqe_next, (elm)); \
+ qatomic_rcu_set(&(listelm)->field.sqe_next, (elm)); \
} while (/*CONSTCOND*/0)
#define QSIMPLEQ_REMOVE_HEAD_RCU(head, field) do { \
- atomic_set(&(head)->sqh_first, (head)->sqh_first->field.sqe_next); \
+ qatomic_set(&(head)->sqh_first, (head)->sqh_first->field.sqe_next);\
if ((head)->sqh_first == NULL) { \
(head)->sqh_last = &(head)->sqh_first; \
} \
while (curr->field.sqe_next != (elm)) { \
curr = curr->field.sqe_next; \
} \
- atomic_set(&curr->field.sqe_next, \
+ qatomic_set(&curr->field.sqe_next, \
curr->field.sqe_next->field.sqe_next); \
if (curr->field.sqe_next == NULL) { \
(head)->sqh_last = &(curr)->field.sqe_next; \
} while (/*CONSTCOND*/0)
#define QSIMPLEQ_FOREACH_RCU(var, head, field) \
- for ((var) = atomic_rcu_read(&(head)->sqh_first); \
+ for ((var) = qatomic_rcu_read(&(head)->sqh_first); \
(var); \
- (var) = atomic_rcu_read(&(var)->field.sqe_next))
+ (var) = qatomic_rcu_read(&(var)->field.sqe_next))
#define QSIMPLEQ_FOREACH_SAFE_RCU(var, head, field, next) \
- for ((var) = atomic_rcu_read(&(head)->sqh_first); \
- (var) && ((next) = atomic_rcu_read(&(var)->field.sqe_next), 1); \
+ for ((var) = qatomic_rcu_read(&(head)->sqh_first); \
+ (var) && ((next) = qatomic_rcu_read(&(var)->field.sqe_next), 1);\
(var) = (next))
/*
*/
/* Tail queue access methods */
-#define QTAILQ_EMPTY_RCU(head) (atomic_read(&(head)->tqh_first) == NULL)
-#define QTAILQ_FIRST_RCU(head) atomic_rcu_read(&(head)->tqh_first)
-#define QTAILQ_NEXT_RCU(elm, field) atomic_rcu_read(&(elm)->field.tqe_next)
+#define QTAILQ_EMPTY_RCU(head) (qatomic_read(&(head)->tqh_first) == NULL)
+#define QTAILQ_FIRST_RCU(head) qatomic_rcu_read(&(head)->tqh_first)
+#define QTAILQ_NEXT_RCU(elm, field) qatomic_rcu_read(&(elm)->field.tqe_next)
/* Tail queue functions */
#define QTAILQ_INSERT_HEAD_RCU(head, elm, field) do { \
} else { \
(head)->tqh_circ.tql_prev = &(elm)->field.tqe_circ; \
} \
- atomic_rcu_set(&(head)->tqh_first, (elm)); \
+ qatomic_rcu_set(&(head)->tqh_first, (elm)); \
(elm)->field.tqe_circ.tql_prev = &(head)->tqh_circ; \
} while (/*CONSTCOND*/0)
#define QTAILQ_INSERT_TAIL_RCU(head, elm, field) do { \
(elm)->field.tqe_next = NULL; \
(elm)->field.tqe_circ.tql_prev = (head)->tqh_circ.tql_prev; \
- atomic_rcu_set(&(head)->tqh_circ.tql_prev->tql_next, (elm)); \
+ qatomic_rcu_set(&(head)->tqh_circ.tql_prev->tql_next, (elm)); \
(head)->tqh_circ.tql_prev = &(elm)->field.tqe_circ; \
} while (/*CONSTCOND*/0)
} else { \
(head)->tqh_circ.tql_prev = &(elm)->field.tqe_circ; \
} \
- atomic_rcu_set(&(listelm)->field.tqe_next, (elm)); \
+ qatomic_rcu_set(&(listelm)->field.tqe_next, (elm)); \
(elm)->field.tqe_circ.tql_prev = &(listelm)->field.tqe_circ; \
} while (/*CONSTCOND*/0)
#define QTAILQ_INSERT_BEFORE_RCU(listelm, elm, field) do { \
(elm)->field.tqe_circ.tql_prev = (listelm)->field.tqe_circ.tql_prev; \
(elm)->field.tqe_next = (listelm); \
- atomic_rcu_set(&(listelm)->field.tqe_circ.tql_prev->tql_next, (elm)); \
+ qatomic_rcu_set(&(listelm)->field.tqe_circ.tql_prev->tql_next, (elm));\
(listelm)->field.tqe_circ.tql_prev = &(elm)->field.tqe_circ; \
} while (/*CONSTCOND*/0)
} else { \
(head)->tqh_circ.tql_prev = (elm)->field.tqe_circ.tql_prev; \
} \
- atomic_set(&(elm)->field.tqe_circ.tql_prev->tql_next, (elm)->field.tqe_next); \
+ qatomic_set(&(elm)->field.tqe_circ.tql_prev->tql_next, \
+ (elm)->field.tqe_next); \
(elm)->field.tqe_circ.tql_prev = NULL; \
} while (/*CONSTCOND*/0)
#define QTAILQ_FOREACH_RCU(var, head, field) \
- for ((var) = atomic_rcu_read(&(head)->tqh_first); \
+ for ((var) = qatomic_rcu_read(&(head)->tqh_first); \
(var); \
- (var) = atomic_rcu_read(&(var)->field.tqe_next))
+ (var) = qatomic_rcu_read(&(var)->field.tqe_next))
#define QTAILQ_FOREACH_SAFE_RCU(var, head, field, next) \
- for ((var) = atomic_rcu_read(&(head)->tqh_first); \
- (var) && ((next) = atomic_rcu_read(&(var)->field.tqe_next), 1); \
+ for ((var) = qatomic_rcu_read(&(head)->tqh_first); \
+ (var) && ((next) = qatomic_rcu_read(&(var)->field.tqe_next), 1);\
(var) = (next))
/*
*/
/* Singly-linked list access methods */
-#define QSLIST_EMPTY_RCU(head) (atomic_read(&(head)->slh_first) == NULL)
-#define QSLIST_FIRST_RCU(head) atomic_rcu_read(&(head)->slh_first)
-#define QSLIST_NEXT_RCU(elm, field) atomic_rcu_read(&(elm)->field.sle_next)
+#define QSLIST_EMPTY_RCU(head) (qatomic_read(&(head)->slh_first) == NULL)
+#define QSLIST_FIRST_RCU(head) qatomic_rcu_read(&(head)->slh_first)
+#define QSLIST_NEXT_RCU(elm, field) qatomic_rcu_read(&(elm)->field.sle_next)
/* Singly-linked list functions */
#define QSLIST_INSERT_HEAD_RCU(head, elm, field) do { \
(elm)->field.sle_next = (head)->slh_first; \
- atomic_rcu_set(&(head)->slh_first, (elm)); \
+ qatomic_rcu_set(&(head)->slh_first, (elm)); \
} while (/*CONSTCOND*/0)
#define QSLIST_INSERT_AFTER_RCU(head, listelm, elm, field) do { \
(elm)->field.sle_next = (listelm)->field.sle_next; \
- atomic_rcu_set(&(listelm)->field.sle_next, (elm)); \
+ qatomic_rcu_set(&(listelm)->field.sle_next, (elm)); \
} while (/*CONSTCOND*/0)
#define QSLIST_REMOVE_HEAD_RCU(head, field) do { \
- atomic_set(&(head)->slh_first, (head)->slh_first->field.sle_next); \
+ qatomic_set(&(head)->slh_first, (head)->slh_first->field.sle_next);\
} while (/*CONSTCOND*/0)
#define QSLIST_REMOVE_RCU(head, elm, type, field) do { \
while (curr->field.sle_next != (elm)) { \
curr = curr->field.sle_next; \
} \
- atomic_set(&curr->field.sle_next, \
+ qatomic_set(&curr->field.sle_next, \
curr->field.sle_next->field.sle_next); \
} \
} while (/*CONSTCOND*/0)
#define QSLIST_FOREACH_RCU(var, head, field) \
- for ((var) = atomic_rcu_read(&(head)->slh_first); \
- (var); \
- (var) = atomic_rcu_read(&(var)->field.sle_next))
+ for ((var) = qatomic_rcu_read(&(head)->slh_first); \
+ (var); \
+ (var) = qatomic_rcu_read(&(var)->field.sle_next))
-#define QSLIST_FOREACH_SAFE_RCU(var, head, field, next) \
- for ((var) = atomic_rcu_read(&(head)->slh_first); \
- (var) && ((next) = atomic_rcu_read(&(var)->field.sle_next), 1); \
+#define QSLIST_FOREACH_SAFE_RCU(var, head, field, next) \
+ for ((var) = qatomic_rcu_read(&(head)->slh_first); \
+ (var) && ((next) = qatomic_rcu_read(&(var)->field.sle_next), 1); \
(var) = (next))
#ifdef __cplusplus
/* Lock out other writers and update the count. */
static inline void seqlock_write_begin(QemuSeqLock *sl)
{
- atomic_set(&sl->sequence, sl->sequence + 1);
+ qatomic_set(&sl->sequence, sl->sequence + 1);
/* Write sequence before updating other fields. */
smp_wmb();
/* Write other fields before finalizing sequence. */
smp_wmb();
- atomic_set(&sl->sequence, sl->sequence + 1);
+ qatomic_set(&sl->sequence, sl->sequence + 1);
}
/* Lock out other writers and update the count. */
static inline unsigned seqlock_read_begin(const QemuSeqLock *sl)
{
/* Always fail if a write is in progress. */
- unsigned ret = atomic_read(&sl->sequence);
+ unsigned ret = qatomic_read(&sl->sequence);
/* Read sequence before reading other fields. */
smp_rmb();
{
/* Read other fields before reading final sequence. */
smp_rmb();
- return unlikely(atomic_read(&sl->sequence) != start);
+ return unlikely(qatomic_read(&sl->sequence) != start);
}
#endif
static inline uint64_t stat64_get(const Stat64 *s)
{
- return atomic_read__nocheck(&s->value);
+ return qatomic_read__nocheck(&s->value);
}
static inline void stat64_add(Stat64 *s, uint64_t value)
{
- atomic_add(&s->value, value);
+ qatomic_add(&s->value, value);
}
static inline void stat64_min(Stat64 *s, uint64_t value)
{
- uint64_t orig = atomic_read__nocheck(&s->value);
+ uint64_t orig = qatomic_read__nocheck(&s->value);
while (orig > value) {
- orig = atomic_cmpxchg__nocheck(&s->value, orig, value);
+ orig = qatomic_cmpxchg__nocheck(&s->value, orig, value);
}
}
static inline void stat64_max(Stat64 *s, uint64_t value)
{
- uint64_t orig = atomic_read__nocheck(&s->value);
+ uint64_t orig = qatomic_read__nocheck(&s->value);
while (orig < value) {
- orig = atomic_cmpxchg__nocheck(&s->value, orig, value);
+ orig = qatomic_cmpxchg__nocheck(&s->value, orig, value);
}
}
#else
low = (uint32_t) value;
if (!low) {
if (high) {
- atomic_add(&s->high, high);
+ qatomic_add(&s->high, high);
}
return;
}
* the high 32 bits, so it can race just fine with stat64_add32_carry
* and even stat64_get!
*/
- old = atomic_cmpxchg(&s->low, orig, result);
+ old = qatomic_cmpxchg(&s->low, orig, result);
if (orig == old) {
return;
}
high = value >> 32;
low = (uint32_t) value;
do {
- orig_high = atomic_read(&s->high);
+ orig_high = qatomic_read(&s->high);
if (orig_high < high) {
return;
}
* the write barrier in stat64_min_slow.
*/
smp_rmb();
- orig_low = atomic_read(&s->low);
+ orig_low = qatomic_read(&s->low);
if (orig_low <= low) {
return;
}
* we may miss being lucky.
*/
smp_rmb();
- orig_high = atomic_read(&s->high);
+ orig_high = qatomic_read(&s->high);
if (orig_high < high) {
return;
}
high = value >> 32;
low = (uint32_t) value;
do {
- orig_high = atomic_read(&s->high);
+ orig_high = qatomic_read(&s->high);
if (orig_high > high) {
return;
}
* the write barrier in stat64_max_slow.
*/
smp_rmb();
- orig_low = atomic_read(&s->low);
+ orig_low = qatomic_read(&s->low);
if (orig_low >= low) {
return;
}
* we may miss being lucky.
*/
smp_rmb();
- orig_high = atomic_read(&s->high);
+ orig_high = qatomic_read(&s->high);
if (orig_high > high) {
return;
}
qemu_cond_timedwait_impl(c, m, ms, __FILE__, __LINE__)
#else
#define qemu_mutex_lock(m) ({ \
- QemuMutexLockFunc _f = atomic_read(&qemu_mutex_lock_func); \
+ QemuMutexLockFunc _f = qatomic_read(&qemu_mutex_lock_func); \
_f(m, __FILE__, __LINE__); \
})
-#define qemu_mutex_trylock(m) ({ \
- QemuMutexTrylockFunc _f = atomic_read(&qemu_mutex_trylock_func); \
- _f(m, __FILE__, __LINE__); \
+#define qemu_mutex_trylock(m) ({ \
+ QemuMutexTrylockFunc _f = qatomic_read(&qemu_mutex_trylock_func); \
+ _f(m, __FILE__, __LINE__); \
})
-#define qemu_rec_mutex_lock(m) ({ \
- QemuRecMutexLockFunc _f = atomic_read(&qemu_rec_mutex_lock_func); \
- _f(m, __FILE__, __LINE__); \
+#define qemu_rec_mutex_lock(m) ({ \
+ QemuRecMutexLockFunc _f = qatomic_read(&qemu_rec_mutex_lock_func);\
+ _f(m, __FILE__, __LINE__); \
})
#define qemu_rec_mutex_trylock(m) ({ \
QemuRecMutexTrylockFunc _f; \
- _f = atomic_read(&qemu_rec_mutex_trylock_func); \
+ _f = qatomic_read(&qemu_rec_mutex_trylock_func); \
_f(m, __FILE__, __LINE__); \
})
#define qemu_cond_wait(c, m) ({ \
- QemuCondWaitFunc _f = atomic_read(&qemu_cond_wait_func); \
+ QemuCondWaitFunc _f = qatomic_read(&qemu_cond_wait_func); \
_f(c, m, __FILE__, __LINE__); \
})
#define qemu_cond_timedwait(c, m, ms) ({ \
- QemuCondTimedWaitFunc _f = atomic_read(&qemu_cond_timedwait_func); \
+ QemuCondTimedWaitFunc _f = qatomic_read(&qemu_cond_timedwait_func);\
_f(c, m, ms, __FILE__, __LINE__); \
})
#endif
__tsan_mutex_pre_lock(spin, 0);
#endif
while (unlikely(__sync_lock_test_and_set(&spin->value, true))) {
- while (atomic_read(&spin->value)) {
+ while (qatomic_read(&spin->value)) {
cpu_relax();
}
}
static inline bool qemu_spin_locked(QemuSpin *spin)
{
- return atomic_read(&spin->value);
+ return qatomic_read(&spin->value);
}
static inline void qemu_spin_unlock(QemuSpin *spin)
static inline int32_t pvrdma_idx(int *var, uint32_t max_elems)
{
- const unsigned int idx = atomic_read(var);
+ const unsigned int idx = qatomic_read(var);
if (pvrdma_idx_valid(idx, max_elems))
return idx & (max_elems - 1);
static inline void pvrdma_idx_ring_inc(int *var, uint32_t max_elems)
{
- uint32_t idx = atomic_read(var) + 1; /* Increment. */
+ uint32_t idx = qatomic_read(var) + 1; /* Increment. */
idx &= (max_elems << 1) - 1; /* Modulo size, flip gen. */
- atomic_set(var, idx);
+ qatomic_set(var, idx);
}
static inline int32_t pvrdma_idx_ring_has_space(const struct pvrdma_ring *r,
uint32_t max_elems, uint32_t *out_tail)
{
- const uint32_t tail = atomic_read(&r->prod_tail);
- const uint32_t head = atomic_read(&r->cons_head);
+ const uint32_t tail = qatomic_read(&r->prod_tail);
+ const uint32_t head = qatomic_read(&r->cons_head);
if (pvrdma_idx_valid(tail, max_elems) &&
pvrdma_idx_valid(head, max_elems)) {
static inline int32_t pvrdma_idx_ring_has_data(const struct pvrdma_ring *r,
uint32_t max_elems, uint32_t *out_head)
{
- const uint32_t tail = atomic_read(&r->prod_tail);
- const uint32_t head = atomic_read(&r->cons_head);
+ const uint32_t tail = qatomic_read(&r->prod_tail);
+ const uint32_t head = qatomic_read(&r->cons_head);
if (pvrdma_idx_valid(tail, max_elems) &&
pvrdma_idx_valid(head, max_elems)) {
* We must check the running state again in case it was
* changed in previous aio_poll()
*/
- if (iothread->running && atomic_read(&iothread->run_gcontext)) {
+ if (iothread->running && qatomic_read(&iothread->run_gcontext)) {
g_main_loop_run(iothread->main_loop);
}
}
iothread->thread_id = -1;
qemu_sem_init(&iothread->init_done_sem, 0);
/* By default, we don't run gcontext */
- atomic_set(&iothread->run_gcontext, 0);
+ qatomic_set(&iothread->run_gcontext, 0);
}
static void iothread_instance_finalize(Object *obj)
GMainContext *iothread_get_g_main_context(IOThread *iothread)
{
- atomic_set(&iothread->run_gcontext, 1);
+ qatomic_set(&iothread->run_gcontext, 1);
aio_notify(iothread->ctx);
return iothread->worker_context;
}
}
old = tswap32(old);
new = tswap32(new);
- ret = atomic_cmpxchg((uint32_t *)g2h(addr), old, new);
+ ret = qatomic_cmpxchg((uint32_t *)g2h(addr), old, new);
ret = tswap32(ret);
break;
case 0:
old = *(uint8_t *)g2h(old);
new = *(uint8_t *)g2h(new);
- ret = atomic_cmpxchg((uint8_t *)g2h(addr), old, new);
+ ret = qatomic_cmpxchg((uint8_t *)g2h(addr), old, new);
ret = ret != old;
break;
case 1:
old = *(uint16_t *)g2h(old);
new = *(uint16_t *)g2h(new);
- ret = atomic_cmpxchg((uint16_t *)g2h(addr), old, new);
+ ret = qatomic_cmpxchg((uint16_t *)g2h(addr), old, new);
ret = ret != old;
break;
case 2:
old = *(uint32_t *)g2h(old);
new = *(uint32_t *)g2h(new);
- ret = atomic_cmpxchg((uint32_t *)g2h(addr), old, new);
+ ret = qatomic_cmpxchg((uint32_t *)g2h(addr), old, new);
ret = ret != old;
break;
case 3:
o64 = *(uint64_t *)g2h(old);
n64 = *(uint64_t *)g2h(new);
#ifdef CONFIG_ATOMIC64
- r64 = atomic_cmpxchg__nocheck((uint64_t *)g2h(addr), o64, n64);
+ r64 = qatomic_cmpxchg__nocheck((uint64_t *)g2h(addr),
+ o64, n64);
ret = r64 != o64;
#else
start_exclusive();
/* Nonzero if process_pending_signals() needs to do something (either
* handle a pending signal or unblock signals).
* This flag is written from a signal handler so should be accessed via
- * the atomic_read() and atomic_set() functions. (It is not accessed
+ * the qatomic_read() and qatomic_set() functions. (It is not accessed
* from multiple threads.)
*/
int signal_pending;
sigfillset(&set);
sigprocmask(SIG_SETMASK, &set, 0);
- return atomic_xchg(&ts->signal_pending, 1);
+ return qatomic_xchg(&ts->signal_pending, 1);
}
/* Wrapper for sigprocmask function
ts->sync_signal.info = *info;
ts->sync_signal.pending = sig;
/* signal that a new signal is pending */
- atomic_set(&ts->signal_pending, 1);
+ qatomic_set(&ts->signal_pending, 1);
return 1; /* indicates that the signal was queued */
}
sigset_t set;
sigset_t *blocked_set;
- while (atomic_read(&ts->signal_pending)) {
+ while (qatomic_read(&ts->signal_pending)) {
/* FIXME: This is not threadsafe. */
sigfillset(&set);
sigprocmask(SIG_SETMASK, &set, 0);
* of unblocking might cause us to take another host signal which
* will set signal_pending again).
*/
- atomic_set(&ts->signal_pending, 0);
+ qatomic_set(&ts->signal_pending, 0);
ts->in_sigsuspend = 0;
set = ts->signal_mask;
sigdelset(&set, SIGSEGV);
{
FailoverStatus old;
- old = atomic_cmpxchg(&failover_state, old_state, new_state);
+ old = qatomic_cmpxchg(&failover_state, old_state, new_state);
if (old == old_state) {
trace_colo_failover_set_state(FailoverStatus_str(new_state));
}
FailoverStatus failover_get_state(void)
{
- return atomic_read(&failover_state);
+ return qatomic_read(&failover_state);
}
void qmp_x_colo_lost_heartbeat(Error **errp)
* we don't error if migration has finished since that would be racy
* with issuing this command.
*/
- atomic_set(&s->start_postcopy, true);
+ qatomic_set(&s->start_postcopy, true);
}
/* shared migration helpers */
void migrate_set_state(int *state, int old_state, int new_state)
{
assert(new_state < MIGRATION_STATUS__MAX);
- if (atomic_cmpxchg(state, old_state, new_state) == old_state) {
+ if (qatomic_cmpxchg(state, old_state, new_state) == old_state) {
trace_migrate_set_state(MigrationStatus_str(new_state));
migrate_generate_event(new_state);
}
return;
}
- if (atomic_cmpxchg(&mis->postcopy_recover_triggered,
+ if (qatomic_cmpxchg(&mis->postcopy_recover_triggered,
false, true) == true) {
error_setg(errp, "Migrate recovery is triggered already");
return;
if (pending_size && pending_size >= s->threshold_size) {
/* Still a significant amount to transfer */
if (!in_postcopy && pend_pre <= s->threshold_size &&
- atomic_read(&s->start_postcopy)) {
+ qatomic_read(&s->start_postcopy)) {
if (postcopy_start(s)) {
error_report("%s: postcopy failed to start", __func__);
}
MultiFDPages_t *pages = multifd_send_state->pages;
uint64_t transferred;
- if (atomic_read(&multifd_send_state->exiting)) {
+ if (qatomic_read(&multifd_send_state->exiting)) {
return -1;
}
* threads at the same time, we can end calling this function
* twice.
*/
- if (atomic_xchg(&multifd_send_state->exiting, 1)) {
+ if (qatomic_xchg(&multifd_send_state->exiting, 1)) {
return;
}
while (true) {
qemu_sem_wait(&p->sem);
- if (atomic_read(&multifd_send_state->exiting)) {
+ if (qatomic_read(&multifd_send_state->exiting)) {
break;
}
qemu_mutex_lock(&p->mutex);
multifd_send_state->params = g_new0(MultiFDSendParams, thread_count);
multifd_send_state->pages = multifd_pages_init(page_count);
qemu_sem_init(&multifd_send_state->channels_ready, 0);
- atomic_set(&multifd_send_state->exiting, 0);
+ qatomic_set(&multifd_send_state->exiting, 0);
multifd_send_state->ops = multifd_ops[migrate_multifd_compression()];
for (i = 0; i < thread_count; i++) {
thread_count = migrate_multifd_channels();
multifd_recv_state = g_malloc0(sizeof(*multifd_recv_state));
multifd_recv_state->params = g_new0(MultiFDRecvParams, thread_count);
- atomic_set(&multifd_recv_state->count, 0);
+ qatomic_set(&multifd_recv_state->count, 0);
qemu_sem_init(&multifd_recv_state->sem_sync, 0);
multifd_recv_state->ops = multifd_ops[migrate_multifd_compression()];
return true;
}
- return thread_count == atomic_read(&multifd_recv_state->count);
+ return thread_count == qatomic_read(&multifd_recv_state->count);
}
/*
error_propagate_prepend(errp, local_err,
"failed to receive packet"
" via multifd channel %d: ",
- atomic_read(&multifd_recv_state->count));
+ qatomic_read(&multifd_recv_state->count));
return false;
}
trace_multifd_recv_new_channel(id);
p->running = true;
qemu_thread_create(&p->thread, p->name, multifd_recv_thread, p,
QEMU_THREAD_JOINABLE);
- atomic_inc(&multifd_recv_state->count);
- return atomic_read(&multifd_recv_state->count) ==
+ qatomic_inc(&multifd_recv_state->count);
+ return qatomic_read(&multifd_recv_state->count) ==
migrate_multifd_channels();
}
Error *local_err = NULL;
/* Let the fault thread quit */
- atomic_set(&mis->fault_thread_quit, 1);
+ qatomic_set(&mis->fault_thread_quit, 1);
postcopy_fault_thread_notify(mis);
trace_postcopy_ram_incoming_cleanup_join();
qemu_thread_join(&mis->fault_thread);
low_time_offset = get_low_time_offset(dc);
if (dc->vcpu_addr[cpu] == 0) {
- atomic_inc(&dc->smp_cpus_down);
+ qatomic_inc(&dc->smp_cpus_down);
}
- atomic_xchg(&dc->last_begin, low_time_offset);
- atomic_xchg(&dc->page_fault_vcpu_time[cpu], low_time_offset);
- atomic_xchg(&dc->vcpu_addr[cpu], addr);
+ qatomic_xchg(&dc->last_begin, low_time_offset);
+ qatomic_xchg(&dc->page_fault_vcpu_time[cpu], low_time_offset);
+ qatomic_xchg(&dc->vcpu_addr[cpu], addr);
/*
* check it here, not at the beginning of the function,
*/
already_received = ramblock_recv_bitmap_test(rb, (void *)addr);
if (already_received) {
- atomic_xchg(&dc->vcpu_addr[cpu], 0);
- atomic_xchg(&dc->page_fault_vcpu_time[cpu], 0);
- atomic_dec(&dc->smp_cpus_down);
+ qatomic_xchg(&dc->vcpu_addr[cpu], 0);
+ qatomic_xchg(&dc->page_fault_vcpu_time[cpu], 0);
+ qatomic_dec(&dc->smp_cpus_down);
}
trace_mark_postcopy_blocktime_begin(addr, dc, dc->page_fault_vcpu_time[cpu],
cpu, already_received);
for (i = 0; i < smp_cpus; i++) {
uint32_t vcpu_blocktime = 0;
- read_vcpu_time = atomic_fetch_add(&dc->page_fault_vcpu_time[i], 0);
- if (atomic_fetch_add(&dc->vcpu_addr[i], 0) != addr ||
+ read_vcpu_time = qatomic_fetch_add(&dc->page_fault_vcpu_time[i], 0);
+ if (qatomic_fetch_add(&dc->vcpu_addr[i], 0) != addr ||
read_vcpu_time == 0) {
continue;
}
- atomic_xchg(&dc->vcpu_addr[i], 0);
+ qatomic_xchg(&dc->vcpu_addr[i], 0);
vcpu_blocktime = low_time_offset - read_vcpu_time;
affected_cpu += 1;
/* we need to know is that mark_postcopy_end was due to
* faulted page, another possible case it's prefetched
* page and in that case we shouldn't be here */
if (!vcpu_total_blocktime &&
- atomic_fetch_add(&dc->smp_cpus_down, 0) == smp_cpus) {
+ qatomic_fetch_add(&dc->smp_cpus_down, 0) == smp_cpus) {
vcpu_total_blocktime = true;
}
/* continue cycle, due to one page could affect several vCPUs */
dc->vcpu_blocktime[i] += vcpu_blocktime;
}
- atomic_sub(&dc->smp_cpus_down, affected_cpu);
+ qatomic_sub(&dc->smp_cpus_down, affected_cpu);
if (vcpu_total_blocktime) {
- dc->total_blocktime += low_time_offset - atomic_fetch_add(
+ dc->total_blocktime += low_time_offset - qatomic_fetch_add(
&dc->last_begin, 0);
}
trace_mark_postcopy_blocktime_end(addr, dc, dc->total_blocktime,
error_report("%s: read() failed", __func__);
}
- if (atomic_read(&mis->fault_thread_quit)) {
+ if (qatomic_read(&mis->fault_thread_quit)) {
trace_postcopy_ram_fault_thread_quit();
break;
}
PostcopyState postcopy_state_get(void)
{
- return atomic_mb_read(&incoming_postcopy_state);
+ return qatomic_mb_read(&incoming_postcopy_state);
}
/* Set the state and return the old state */
PostcopyState postcopy_state_set(PostcopyState new_state)
{
- return atomic_xchg(&incoming_postcopy_state, new_state);
+ return qatomic_xchg(&incoming_postcopy_state, new_state);
}
/* Register a handler for external shared memory postcopy
size_t len = 0;
RCU_READ_LOCK_GUARD();
- rdma = atomic_rcu_read(&rioc->rdmaout);
+ rdma = qatomic_rcu_read(&rioc->rdmaout);
if (!rdma) {
return -EIO;
size_t done = 0;
RCU_READ_LOCK_GUARD();
- rdma = atomic_rcu_read(&rioc->rdmain);
+ rdma = qatomic_rcu_read(&rioc->rdmain);
if (!rdma) {
return -EIO;
RCU_READ_LOCK_GUARD();
if (rsource->condition == G_IO_IN) {
- rdma = atomic_rcu_read(&rsource->rioc->rdmain);
+ rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
} else {
- rdma = atomic_rcu_read(&rsource->rioc->rdmaout);
+ rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
}
if (!rdma) {
RCU_READ_LOCK_GUARD();
if (rsource->condition == G_IO_IN) {
- rdma = atomic_rcu_read(&rsource->rioc->rdmain);
+ rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
} else {
- rdma = atomic_rcu_read(&rsource->rioc->rdmaout);
+ rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
}
if (!rdma) {
RCU_READ_LOCK_GUARD();
if (rsource->condition == G_IO_IN) {
- rdma = atomic_rcu_read(&rsource->rioc->rdmain);
+ rdma = qatomic_rcu_read(&rsource->rioc->rdmain);
} else {
- rdma = atomic_rcu_read(&rsource->rioc->rdmaout);
+ rdma = qatomic_rcu_read(&rsource->rioc->rdmaout);
}
if (!rdma) {
rdmain = rioc->rdmain;
if (rdmain) {
- atomic_rcu_set(&rioc->rdmain, NULL);
+ qatomic_rcu_set(&rioc->rdmain, NULL);
}
rdmaout = rioc->rdmaout;
if (rdmaout) {
- atomic_rcu_set(&rioc->rdmaout, NULL);
+ qatomic_rcu_set(&rioc->rdmaout, NULL);
}
rcu->rdmain = rdmain;
RCU_READ_LOCK_GUARD();
- rdmain = atomic_rcu_read(&rioc->rdmain);
- rdmaout = atomic_rcu_read(&rioc->rdmain);
+ rdmain = qatomic_rcu_read(&rioc->rdmain);
+ rdmaout = qatomic_rcu_read(&rioc->rdmain);
switch (how) {
case QIO_CHANNEL_SHUTDOWN_READ:
int ret;
RCU_READ_LOCK_GUARD();
- rdma = atomic_rcu_read(&rioc->rdmaout);
+ rdma = qatomic_rcu_read(&rioc->rdmaout);
if (!rdma) {
return -EIO;
int i = 0;
RCU_READ_LOCK_GUARD();
- rdma = atomic_rcu_read(&rioc->rdmain);
+ rdma = qatomic_rcu_read(&rioc->rdmain);
if (!rdma) {
return -EIO;
int found = -1;
RCU_READ_LOCK_GUARD();
- rdma = atomic_rcu_read(&rioc->rdmain);
+ rdma = qatomic_rcu_read(&rioc->rdmain);
if (!rdma) {
return -EIO;
RDMAContext *rdma;
RCU_READ_LOCK_GUARD();
- rdma = atomic_rcu_read(&rioc->rdmaout);
+ rdma = qatomic_rcu_read(&rioc->rdmaout);
if (!rdma) {
return -EIO;
}
int ret = 0;
RCU_READ_LOCK_GUARD();
- rdma = atomic_rcu_read(&rioc->rdmaout);
+ rdma = qatomic_rcu_read(&rioc->rdmaout);
if (!rdma) {
return -EIO;
}
monitor_resume(mon);
monitor_flush(mon);
} else {
- atomic_mb_set(&mon->suspend_cnt, 0);
+ qatomic_mb_set(&mon->suspend_cnt, 0);
}
break;
case CHR_EVENT_MUX_OUT:
if (mon->reset_seen) {
- if (atomic_mb_read(&mon->suspend_cnt) == 0) {
+ if (qatomic_mb_read(&mon->suspend_cnt) == 0) {
monitor_printf(mon, "\n");
}
monitor_flush(mon);
monitor_suspend(mon);
} else {
- atomic_inc(&mon->suspend_cnt);
+ qatomic_inc(&mon->suspend_cnt);
}
qemu_mutex_lock(&mon->mon_lock);
mon->mux_out = 1;
}
/* Force copy-on-write if necessary. */
- atomic_add((uint8_t *)ptr, 0);
+ qatomic_add((uint8_t *)ptr, 0);
if (pread(fd, &pinfo, sizeof(pinfo), offset) != sizeof(pinfo)) {
error_setg_errno(errp, errno, "Cannot read pagemap");
return -ENOTTY;
}
- atomic_inc(&mon->suspend_cnt);
+ qatomic_inc(&mon->suspend_cnt);
if (mon->use_io_thread) {
/*
return;
}
- if (atomic_dec_fetch(&mon->suspend_cnt) == 0) {
+ if (qatomic_dec_fetch(&mon->suspend_cnt) == 0) {
AioContext *ctx;
if (mon->use_io_thread) {
{
Monitor *mon = opaque;
- return !atomic_mb_read(&mon->suspend_cnt);
+ return !qatomic_mb_read(&mon->suspend_cnt);
}
void monitor_list_append(Monitor *mon)
#if HAVE_NBD_DEVICE
static void termsig_handler(int signum)
{
- atomic_cmpxchg(&state, RUNNING, TERMINATE);
+ qatomic_cmpxchg(&state, RUNNING, TERMINATE);
qemu_notify_event();
}
#endif /* HAVE_NBD_DEVICE */
ges = g_new0(GuestExecStatus, 1);
- bool finished = atomic_mb_read(&gei->finished);
+ bool finished = qatomic_mb_read(&gei->finished);
/* need to wait till output channels are closed
* to be sure we captured all output at this point */
if (gei->has_output) {
- finished = finished && atomic_mb_read(&gei->out.closed);
- finished = finished && atomic_mb_read(&gei->err.closed);
+ finished = finished && qatomic_mb_read(&gei->out.closed);
+ finished = finished && qatomic_mb_read(&gei->err.closed);
}
ges->exited = finished;
(int32_t)gpid_to_int64(pid), (uint32_t)status);
gei->status = status;
- atomic_mb_set(&gei->finished, true);
+ qatomic_mb_set(&gei->finished, true);
g_spawn_close_pid(pid);
}
done:
g_io_channel_shutdown(ch, true, NULL);
g_io_channel_unref(ch);
- atomic_mb_set(&p->closed, true);
+ qatomic_mb_set(&p->closed, true);
g_free(p->data);
return false;
close:
g_io_channel_shutdown(ch, true, NULL);
g_io_channel_unref(ch);
- atomic_mb_set(&p->closed, true);
+ qatomic_mb_set(&p->closed, true);
return false;
}
Object *inst;
for (i = 0; obj && i < OBJECT_CLASS_CAST_CACHE; i++) {
- if (atomic_read(&obj->class->object_cast_cache[i]) == typename) {
+ if (qatomic_read(&obj->class->object_cast_cache[i]) == typename) {
goto out;
}
}
if (obj && obj == inst) {
for (i = 1; i < OBJECT_CLASS_CAST_CACHE; i++) {
- atomic_set(&obj->class->object_cast_cache[i - 1],
- atomic_read(&obj->class->object_cast_cache[i]));
+ qatomic_set(&obj->class->object_cast_cache[i - 1],
+ qatomic_read(&obj->class->object_cast_cache[i]));
}
- atomic_set(&obj->class->object_cast_cache[i - 1], typename);
+ qatomic_set(&obj->class->object_cast_cache[i - 1], typename);
}
out:
int i;
for (i = 0; class && i < OBJECT_CLASS_CAST_CACHE; i++) {
- if (atomic_read(&class->class_cast_cache[i]) == typename) {
+ if (qatomic_read(&class->class_cast_cache[i]) == typename) {
ret = class;
goto out;
}
#ifdef CONFIG_QOM_CAST_DEBUG
if (class && ret == class) {
for (i = 1; i < OBJECT_CLASS_CAST_CACHE; i++) {
- atomic_set(&class->class_cast_cache[i - 1],
- atomic_read(&class->class_cast_cache[i]));
+ qatomic_set(&class->class_cast_cache[i - 1],
+ qatomic_read(&class->class_cast_cache[i]));
}
- atomic_set(&class->class_cast_cache[i - 1], typename);
+ qatomic_set(&class->class_cast_cache[i - 1], typename);
}
out:
#endif
if (!obj) {
return NULL;
}
- atomic_inc(&obj->ref);
+ qatomic_inc(&obj->ref);
return obj;
}
g_assert(obj->ref > 0);
/* parent always holds a reference to its children */
- if (atomic_fetch_dec(&obj->ref) == 1) {
+ if (qatomic_fetch_dec(&obj->ref) == 1) {
object_finalize(obj);
}
}
# If you mess with these regexps, it's a good idea to check that
# the following functions' documentation still comes out right:
# - parport_register_device (function pointer parameters)
- # - atomic_set (macro)
+ # - qatomic_set (macro)
# - pci_match_device, __copy_to_user (long return type)
if ($define && $prototype =~ m/^()([a-zA-Z0-9_~:]+)\s+/) {
goto out;
}
- while (atomic_read(&state) == RUNNING) {
+ while (qatomic_read(&state) == RUNNING) {
PRHelperRequest req;
PRHelperResponse resp;
int sz;
static void termsig_handler(int signum)
{
- atomic_cmpxchg(&state, RUNNING, TERMINATE);
+ qatomic_cmpxchg(&state, RUNNING, TERMINATE);
qemu_notify_event();
}
}
sleeptime_ns = endtime_ns - qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
}
- atomic_set(&cpu->throttle_thread_scheduled, 0);
+ qatomic_set(&cpu->throttle_thread_scheduled, 0);
}
static void cpu_throttle_timer_tick(void *opaque)
return;
}
CPU_FOREACH(cpu) {
- if (!atomic_xchg(&cpu->throttle_thread_scheduled, 1)) {
+ if (!qatomic_xchg(&cpu->throttle_thread_scheduled, 1)) {
async_run_on_cpu(cpu, cpu_throttle_thread,
RUN_ON_CPU_NULL);
}
new_throttle_pct = MIN(new_throttle_pct, CPU_THROTTLE_PCT_MAX);
new_throttle_pct = MAX(new_throttle_pct, CPU_THROTTLE_PCT_MIN);
- atomic_set(&throttle_percentage, new_throttle_pct);
+ qatomic_set(&throttle_percentage, new_throttle_pct);
timer_mod(throttle_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT) +
CPU_THROTTLE_TIMESLICE_NS);
void cpu_throttle_stop(void)
{
- atomic_set(&throttle_percentage, 0);
+ qatomic_set(&throttle_percentage, 0);
}
bool cpu_throttle_active(void)
int cpu_throttle_get_percentage(void)
{
- return atomic_read(&throttle_percentage);
+ return qatomic_read(&throttle_percentage);
}
void cpu_throttle_init(void)
int64_t executed = cpu_get_icount_executed(cpu);
cpu->icount_budget -= executed;
- atomic_set_i64(&timers_state.qemu_icount,
+ qatomic_set_i64(&timers_state.qemu_icount,
timers_state.qemu_icount + executed);
}
cpu_update_icount_locked(cpu);
}
/* The read is protected by the seqlock, but needs atomic64 to avoid UB */
- return atomic_read_i64(&timers_state.qemu_icount);
+ return qatomic_read_i64(&timers_state.qemu_icount);
}
static int64_t cpu_get_icount_locked(void)
{
int64_t icount = cpu_get_icount_raw_locked();
- return atomic_read_i64(&timers_state.qemu_icount_bias) +
+ return qatomic_read_i64(&timers_state.qemu_icount_bias) +
cpu_icount_to_ns(icount);
}
int64_t cpu_icount_to_ns(int64_t icount)
{
- return icount << atomic_read(&timers_state.icount_time_shift);
+ return icount << qatomic_read(&timers_state.icount_time_shift);
}
static int64_t cpu_get_ticks_locked(void)
&& last_delta + ICOUNT_WOBBLE < delta * 2
&& timers_state.icount_time_shift > 0) {
/* The guest is getting too far ahead. Slow time down. */
- atomic_set(&timers_state.icount_time_shift,
+ qatomic_set(&timers_state.icount_time_shift,
timers_state.icount_time_shift - 1);
}
if (delta < 0
&& last_delta - ICOUNT_WOBBLE > delta * 2
&& timers_state.icount_time_shift < MAX_ICOUNT_SHIFT) {
/* The guest is getting too far behind. Speed time up. */
- atomic_set(&timers_state.icount_time_shift,
+ qatomic_set(&timers_state.icount_time_shift,
timers_state.icount_time_shift + 1);
}
last_delta = delta;
- atomic_set_i64(&timers_state.qemu_icount_bias,
+ qatomic_set_i64(&timers_state.qemu_icount_bias,
cur_icount - (timers_state.qemu_icount
<< timers_state.icount_time_shift));
seqlock_write_unlock(&timers_state.vm_clock_seqlock,
static int64_t qemu_icount_round(int64_t count)
{
- int shift = atomic_read(&timers_state.icount_time_shift);
+ int shift = qatomic_read(&timers_state.icount_time_shift);
return (count + (1 << shift) - 1) >> shift;
}
int64_t delta = clock - cur_icount;
warp_delta = MIN(warp_delta, delta);
}
- atomic_set_i64(&timers_state.qemu_icount_bias,
+ qatomic_set_i64(&timers_state.qemu_icount_bias,
timers_state.qemu_icount_bias + warp_delta);
}
timers_state.vm_clock_warp_start = -1;
seqlock_write_lock(&timers_state.vm_clock_seqlock,
&timers_state.vm_clock_lock);
- atomic_set_i64(&timers_state.qemu_icount_bias,
+ qatomic_set_i64(&timers_state.qemu_icount_bias,
timers_state.qemu_icount_bias + warp);
seqlock_write_unlock(&timers_state.vm_clock_seqlock,
&timers_state.vm_clock_lock);
*/
seqlock_write_lock(&timers_state.vm_clock_seqlock,
&timers_state.vm_clock_lock);
- atomic_set_i64(&timers_state.qemu_icount_bias,
+ qatomic_set_i64(&timers_state.qemu_icount_bias,
timers_state.qemu_icount_bias + deadline);
seqlock_write_unlock(&timers_state.vm_clock_seqlock,
&timers_state.vm_clock_lock);
{
CPUState *cpu;
do {
- cpu = atomic_mb_read(&tcg_current_rr_cpu);
+ cpu = qatomic_mb_read(&tcg_current_rr_cpu);
if (cpu) {
cpu_exit(cpu);
}
- } while (cpu != atomic_mb_read(&tcg_current_rr_cpu));
+ } while (cpu != qatomic_mb_read(&tcg_current_rr_cpu));
}
/* Kick all RR vCPUs */
static void qemu_wait_io_event_common(CPUState *cpu)
{
- atomic_mb_set(&cpu->thread_kicked, false);
+ qatomic_mb_set(&cpu->thread_kicked, false);
if (cpu->stop) {
qemu_cpu_stop(cpu, false);
}
ret = cpu_exec(cpu);
cpu_exec_end(cpu);
#ifdef CONFIG_PROFILER
- atomic_set(&tcg_ctx->prof.cpu_exec_time,
+ qatomic_set(&tcg_ctx->prof.cpu_exec_time,
tcg_ctx->prof.cpu_exec_time + profile_getclock() - ti);
#endif
return ret;
while (cpu && cpu_work_list_empty(cpu) && !cpu->exit_request) {
- atomic_mb_set(&tcg_current_rr_cpu, cpu);
+ qatomic_mb_set(&tcg_current_rr_cpu, cpu);
current_cpu = cpu;
qemu_clock_enable(QEMU_CLOCK_VIRTUAL,
cpu = CPU_NEXT(cpu);
} /* while (cpu && !cpu->exit_request).. */
- /* Does not need atomic_mb_set because a spurious wakeup is okay. */
- atomic_set(&tcg_current_rr_cpu, NULL);
+ /* Does not need qatomic_mb_set because a spurious wakeup is okay. */
+ qatomic_set(&tcg_current_rr_cpu, NULL);
if (cpu && cpu->exit_request) {
- atomic_mb_set(&cpu->exit_request, 0);
+ qatomic_mb_set(&cpu->exit_request, 0);
}
if (use_icount && all_cpu_threads_idle()) {
}
}
- atomic_mb_set(&cpu->exit_request, 0);
+ qatomic_mb_set(&cpu->exit_request, 0);
qemu_wait_io_event(cpu);
} while (!cpu->unplug || cpu_can_run(cpu));
*/
void qemu_mutex_lock_iothread_impl(const char *file, int line)
{
- QemuMutexLockFunc bql_lock = atomic_read(&qemu_bql_mutex_lock_func);
+ QemuMutexLockFunc bql_lock = qatomic_read(&qemu_bql_mutex_lock_func);
g_assert(!qemu_mutex_iothread_locked());
bql_lock(&qemu_global_mutex, file, line);
static bool flatview_ref(FlatView *view)
{
- return atomic_fetch_inc_nonzero(&view->ref) > 0;
+ return qatomic_fetch_inc_nonzero(&view->ref) > 0;
}
void flatview_unref(FlatView *view)
{
- if (atomic_fetch_dec(&view->ref) == 1) {
+ if (qatomic_fetch_dec(&view->ref) == 1) {
trace_flatview_destroy_rcu(view, view->root);
assert(view->root);
call_rcu(view, flatview_destroy, rcu);
}
/* Writes are protected by the BQL. */
- atomic_rcu_set(&as->current_map, new_view);
+ qatomic_rcu_set(&as->current_map, new_view);
if (old_view) {
flatview_unref(old_view);
}
static int qemu_shutdown_requested(void)
{
- return atomic_xchg(&shutdown_requested, SHUTDOWN_CAUSE_NONE);
+ return qatomic_xchg(&shutdown_requested, SHUTDOWN_CAUSE_NONE);
}
static void qemu_kill_report(void)
static void store_tag1_parallel(uint64_t ptr, uint8_t *mem, int tag)
{
int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
- uint8_t old = atomic_read(mem);
+ uint8_t old = qatomic_read(mem);
while (1) {
uint8_t new = deposit32(old, ofs, 4, tag);
- uint8_t cmp = atomic_cmpxchg(mem, old, new);
+ uint8_t cmp = qatomic_cmpxchg(mem, old, new);
if (likely(cmp == old)) {
return;
}
2 * TAG_GRANULE, MMU_DATA_STORE, 1, ra);
if (mem1) {
tag |= tag << 4;
- atomic_set(mem1, tag);
+ qatomic_set(mem1, tag);
}
}
}
old = *haddr;
while (1) {
new = (old & ~mask) | (val & mask);
- cmp = atomic_cmpxchg(haddr, old, new);
+ cmp = qatomic_cmpxchg(haddr, old, new);
if (cmp == old) {
return;
}
uint64_t *haddr = g2h(a0);
cmpv = cpu_to_le64(cmpv);
newv = cpu_to_le64(newv);
- oldv = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
+ oldv = qatomic_cmpxchg__nocheck(haddr, cmpv, newv);
oldv = le64_to_cpu(oldv);
}
#else
whpx_vcpu_process_async_events(cpu);
if (cpu->halted) {
cpu->exception_index = EXCP_HLT;
- atomic_set(&cpu->exit_request, false);
+ qatomic_set(&cpu->exit_request, false);
return 0;
}
whpx_vcpu_pre_run(cpu);
- if (atomic_read(&cpu->exit_request)) {
+ if (qatomic_read(&cpu->exit_request)) {
whpx_vcpu_kick(cpu);
}
qemu_mutex_lock_iothread();
current_cpu = cpu;
- atomic_set(&cpu->exit_request, false);
+ qatomic_set(&cpu->exit_request, false);
return ret < 0;
}
*pte_pa = pte = updated_pte;
#else
target_ulong old_pte =
- atomic_cmpxchg(pte_pa, pte, updated_pte);
+ qatomic_cmpxchg(pte_pa, pte, updated_pte);
if (old_pte != pte) {
goto restart;
} else {
if (parallel) {
#ifdef CONFIG_USER_ONLY
uint32_t *haddr = g2h(a1);
- ov = atomic_cmpxchg__nocheck(haddr, cv, nv);
+ ov = qatomic_cmpxchg__nocheck(haddr, cv, nv);
#else
TCGMemOpIdx oi = make_memop_idx(MO_TEUL | MO_ALIGN, mem_idx);
ov = helper_atomic_cmpxchgl_be_mmu(env, a1, cv, nv, oi, ra);
#ifdef CONFIG_ATOMIC64
# ifdef CONFIG_USER_ONLY
uint64_t *haddr = g2h(a1);
- ov = atomic_cmpxchg__nocheck(haddr, cv, nv);
+ ov = qatomic_cmpxchg__nocheck(haddr, cv, nv);
# else
TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN, mem_idx);
ov = helper_atomic_cmpxchgq_be_mmu(env, a1, cv, nv, oi, ra);
void HELPER(intset)(CPUXtensaState *env, uint32_t v)
{
- atomic_or(&env->sregs[INTSET],
+ qatomic_or(&env->sregs[INTSET],
v & env->config->inttype_mask[INTTYPE_SOFTWARE]);
}
static void intclear(CPUXtensaState *env, uint32_t v)
{
- atomic_and(&env->sregs[INTSET], ~v);
+ qatomic_and(&env->sregs[INTSET], ~v);
}
void HELPER(intclear)(CPUXtensaState *env, uint32_t v)
{
uint64_t dcc;
- atomic_and(&env->sregs[INTSET],
+ qatomic_and(&env->sregs[INTSET],
~(1u << env->config->timerint[i]));
HELPER(update_ccount)(env);
dcc = (uint64_t)(env->sregs[CCOMPARE + i] - env->sregs[CCOUNT] - 1) + 1;
i2 = I3401_ADDI | rt << 31 | (addr & 0xfff) << 10 | rd << 5 | rd;
}
pair = (uint64_t)i2 << 32 | i1;
- atomic_set((uint64_t *)jmp_addr, pair);
+ qatomic_set((uint64_t *)jmp_addr, pair);
flush_icache_range(jmp_addr, jmp_addr + 8);
}
uintptr_t jmp_addr, uintptr_t addr)
{
/* patch the branch destination */
- atomic_set((int32_t *)jmp_addr, addr - (jmp_addr + 4));
+ qatomic_set((int32_t *)jmp_addr, addr - (jmp_addr + 4));
/* no need to flush icache explicitly */
}
void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_addr,
uintptr_t addr)
{
- atomic_set((uint32_t *)jmp_addr, deposit32(OPC_J, 0, 26, addr >> 2));
+ qatomic_set((uint32_t *)jmp_addr, deposit32(OPC_J, 0, 26, addr >> 2));
flush_icache_range(jmp_addr, jmp_addr + 4);
}
#endif
/* As per the enclosing if, this is ppc64. Avoid the _Static_assert
- within atomic_set that would fail to build a ppc32 host. */
- atomic_set__nocheck((uint64_t *)jmp_addr, pair);
+ within qatomic_set that would fail to build a ppc32 host. */
+ qatomic_set__nocheck((uint64_t *)jmp_addr, pair);
flush_icache_range(jmp_addr, jmp_addr + 8);
} else {
intptr_t diff = addr - jmp_addr;
tcg_debug_assert(in_range_b(diff));
- atomic_set((uint32_t *)jmp_addr, B | (diff & 0x3fffffc));
+ qatomic_set((uint32_t *)jmp_addr, B | (diff & 0x3fffffc));
flush_icache_range(jmp_addr, jmp_addr + 4);
}
}
{
/* patch the branch destination */
intptr_t disp = addr - (jmp_addr - 2);
- atomic_set((int32_t *)jmp_addr, disp / 2);
+ qatomic_set((int32_t *)jmp_addr, disp / 2);
/* no need to flush icache explicitly */
}
tcg_debug_assert(br_disp == (int32_t)br_disp);
if (!USE_REG_TB) {
- atomic_set((uint32_t *)jmp_addr, deposit32(CALL, 0, 30, br_disp >> 2));
+ qatomic_set((uint32_t *)jmp_addr,
+ deposit32(CALL, 0, 30, br_disp >> 2));
flush_icache_range(jmp_addr, jmp_addr + 4);
return;
}
| INSN_IMM13((tb_disp & 0x3ff) | -0x400));
}
- atomic_set((uint64_t *)jmp_addr, deposit64(i2, 32, 32, i1));
+ qatomic_set((uint64_t *)jmp_addr, deposit64(i2, 32, 32, i1));
flush_icache_range(jmp_addr, jmp_addr + 8);
}
/* Call from a safe-work context */
void tcg_region_reset_all(void)
{
- unsigned int n_ctxs = atomic_read(&n_tcg_ctxs);
+ unsigned int n_ctxs = qatomic_read(&n_tcg_ctxs);
unsigned int i;
qemu_mutex_lock(®ion.lock);
region.agg_size_full = 0;
for (i = 0; i < n_ctxs; i++) {
- TCGContext *s = atomic_read(&tcg_ctxs[i]);
+ TCGContext *s = qatomic_read(&tcg_ctxs[i]);
bool err = tcg_region_initial_alloc__locked(s);
g_assert(!err);
}
/* Claim an entry in tcg_ctxs */
- n = atomic_fetch_inc(&n_tcg_ctxs);
+ n = qatomic_fetch_inc(&n_tcg_ctxs);
g_assert(n < ms->smp.max_cpus);
- atomic_set(&tcg_ctxs[n], s);
+ qatomic_set(&tcg_ctxs[n], s);
if (n > 0) {
alloc_tcg_plugin_context(s);
*/
size_t tcg_code_size(void)
{
- unsigned int n_ctxs = atomic_read(&n_tcg_ctxs);
+ unsigned int n_ctxs = qatomic_read(&n_tcg_ctxs);
unsigned int i;
size_t total;
qemu_mutex_lock(®ion.lock);
total = region.agg_size_full;
for (i = 0; i < n_ctxs; i++) {
- const TCGContext *s = atomic_read(&tcg_ctxs[i]);
+ const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
size_t size;
- size = atomic_read(&s->code_gen_ptr) - s->code_gen_buffer;
+ size = qatomic_read(&s->code_gen_ptr) - s->code_gen_buffer;
g_assert(size <= s->code_gen_buffer_size);
total += size;
}
size_t tcg_tb_phys_invalidate_count(void)
{
- unsigned int n_ctxs = atomic_read(&n_tcg_ctxs);
+ unsigned int n_ctxs = qatomic_read(&n_tcg_ctxs);
unsigned int i;
size_t total = 0;
for (i = 0; i < n_ctxs; i++) {
- const TCGContext *s = atomic_read(&tcg_ctxs[i]);
+ const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
- total += atomic_read(&s->tb_phys_invalidate_count);
+ total += qatomic_read(&s->tb_phys_invalidate_count);
}
return total;
}
}
goto retry;
}
- atomic_set(&s->code_gen_ptr, next);
+ qatomic_set(&s->code_gen_ptr, next);
s->data_gen_ptr = NULL;
return tb;
}
QemuLogFile *logfile;
rcu_read_lock();
- logfile = atomic_rcu_read(&qemu_logfile);
+ logfile = qatomic_rcu_read(&qemu_logfile);
if (logfile) {
for (; col < 40; ++col) {
putc(' ', logfile->fd);
s->nb_ops--;
#ifdef CONFIG_PROFILER
- atomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
+ qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
#endif
}
/* avoid copy/paste errors */
#define PROF_ADD(to, from, field) \
do { \
- (to)->field += atomic_read(&((from)->field)); \
+ (to)->field += qatomic_read(&((from)->field)); \
} while (0)
#define PROF_MAX(to, from, field) \
do { \
- typeof((from)->field) val__ = atomic_read(&((from)->field)); \
+ typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
if (val__ > (to)->field) { \
(to)->field = val__; \
} \
static inline
void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
{
- unsigned int n_ctxs = atomic_read(&n_tcg_ctxs);
+ unsigned int n_ctxs = qatomic_read(&n_tcg_ctxs);
unsigned int i;
for (i = 0; i < n_ctxs; i++) {
- TCGContext *s = atomic_read(&tcg_ctxs[i]);
+ TCGContext *s = qatomic_read(&tcg_ctxs[i]);
const TCGProfile *orig = &s->prof;
if (counters) {
int64_t tcg_cpu_exec_time(void)
{
- unsigned int n_ctxs = atomic_read(&n_tcg_ctxs);
+ unsigned int n_ctxs = qatomic_read(&n_tcg_ctxs);
unsigned int i;
int64_t ret = 0;
for (i = 0; i < n_ctxs; i++) {
- const TCGContext *s = atomic_read(&tcg_ctxs[i]);
+ const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
const TCGProfile *prof = &s->prof;
- ret += atomic_read(&prof->cpu_exec_time);
+ ret += qatomic_read(&prof->cpu_exec_time);
}
return ret;
}
QTAILQ_FOREACH(op, &s->ops, link) {
n++;
}
- atomic_set(&prof->op_count, prof->op_count + n);
+ qatomic_set(&prof->op_count, prof->op_count + n);
if (n > prof->op_count_max) {
- atomic_set(&prof->op_count_max, n);
+ qatomic_set(&prof->op_count_max, n);
}
n = s->nb_temps;
- atomic_set(&prof->temp_count, prof->temp_count + n);
+ qatomic_set(&prof->temp_count, prof->temp_count + n);
if (n > prof->temp_count_max) {
- atomic_set(&prof->temp_count_max, n);
+ qatomic_set(&prof->temp_count_max, n);
}
}
#endif
#endif
#ifdef CONFIG_PROFILER
- atomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
+ qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
#endif
#ifdef USE_TCG_OPTIMIZATIONS
#endif
#ifdef CONFIG_PROFILER
- atomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
- atomic_set(&prof->la_time, prof->la_time - profile_getclock());
+ qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
+ qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
#endif
reachable_code_pass(s);
}
#ifdef CONFIG_PROFILER
- atomic_set(&prof->la_time, prof->la_time + profile_getclock());
+ qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
#endif
#ifdef DEBUG_DISAS
TCGOpcode opc = op->opc;
#ifdef CONFIG_PROFILER
- atomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
+ qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
#endif
switch (opc) {
case INDEX_op_goto_tb:
/* Jump address is aligned */
tb_ptr = QEMU_ALIGN_PTR_UP(tb_ptr, 4);
- t0 = atomic_read((int32_t *)tb_ptr);
+ t0 = qatomic_read((int32_t *)tb_ptr);
tb_ptr += sizeof(int32_t);
tci_assert(tb_ptr == old_code_ptr + op_size);
tb_ptr += (int32_t)t0;
uintptr_t jmp_addr, uintptr_t addr)
{
/* patch the branch destination */
- atomic_set((int32_t *)jmp_addr, addr - (jmp_addr + 4));
+ qatomic_set((int32_t *)jmp_addr, addr - (jmp_addr + 4));
/* no need to flush icache explicitly */
}
{
struct thread_info *info = arg;
- atomic_inc(&n_ready_threads);
- while (!atomic_read(&test_start)) {
+ qatomic_inc(&n_ready_threads);
+ while (!qatomic_read(&test_start)) {
cpu_relax();
}
- while (!atomic_read(&test_stop)) {
+ while (!qatomic_read(&test_stop)) {
unsigned int index;
info->r = xorshift64star(info->r);
index = info->r & (range - 1);
- atomic_read_i64(&counts[index].i64);
+ qatomic_read_i64(&counts[index].i64);
info->accesses++;
}
return NULL;
{
unsigned int i;
- while (atomic_read(&n_ready_threads) != n_threads) {
+ while (qatomic_read(&n_ready_threads) != n_threads) {
cpu_relax();
}
- atomic_set(&test_start, true);
+ qatomic_set(&test_start, true);
g_usleep(duration * G_USEC_PER_SEC);
- atomic_set(&test_stop, true);
+ qatomic_set(&test_stop, true);
for (i = 0; i < n_threads; i++) {
qemu_thread_join(&threads[i]);
{
struct thread_info *info = arg;
- atomic_inc(&n_ready_threads);
- while (!atomic_read(&test_start)) {
+ qatomic_inc(&n_ready_threads);
+ while (!qatomic_read(&test_start)) {
cpu_relax();
}
- while (!atomic_read(&test_stop)) {
+ while (!qatomic_read(&test_stop)) {
unsigned int index;
info->r = xorshift64star(info->r);
counts[index].val += 1;
qemu_mutex_unlock(&counts[index].lock);
} else {
- atomic_inc(&counts[index].val);
+ qatomic_inc(&counts[index].val);
}
}
return NULL;
{
unsigned int i;
- while (atomic_read(&n_ready_threads) != n_threads) {
+ while (qatomic_read(&n_ready_threads) != n_threads) {
cpu_relax();
}
- atomic_set(&test_start, true);
+ qatomic_set(&test_start, true);
g_usleep(duration * G_USEC_PER_SEC);
- atomic_set(&test_stop, true);
+ qatomic_set(&test_stop, true);
for (i = 0; i < n_threads; i++) {
qemu_thread_join(&threads[i]);
qemu_cond_signal(&iothread->init_done_cond);
qemu_mutex_unlock(&iothread->init_done_lock);
- while (!atomic_read(&iothread->stopping)) {
+ while (!qatomic_read(&iothread->stopping)) {
aio_poll(iothread->ctx, true);
}
rcu_register_thread();
- atomic_inc(&n_ready_threads);
- while (!atomic_read(&test_start)) {
+ qatomic_inc(&n_ready_threads);
+ while (!qatomic_read(&test_start)) {
cpu_relax();
}
rcu_read_lock();
- while (!atomic_read(&test_stop)) {
+ while (!qatomic_read(&test_stop)) {
info->seed = xorshift64star(info->seed);
info->func(info);
}
{
int i;
- while (atomic_read(&n_ready_threads) != n_rw_threads + n_rz_threads) {
+ while (qatomic_read(&n_ready_threads) != n_rw_threads + n_rz_threads) {
cpu_relax();
}
- atomic_set(&test_start, true);
+ qatomic_set(&test_start, true);
g_usleep(duration * G_USEC_PER_SEC);
- atomic_set(&test_stop, true);
+ qatomic_set(&test_stop, true);
for (i = 0; i < n_rw_threads; i++) {
qemu_thread_join(&rw_threads[i]);
rcu_register_thread();
*(struct rcu_reader_data **)arg = &rcu_reader;
- atomic_inc(&nthreadsrunning);
+ qatomic_inc(&nthreadsrunning);
while (goflag == GOFLAG_INIT) {
g_usleep(1000);
}
rcu_register_thread();
*(struct rcu_reader_data **)arg = &rcu_reader;
- atomic_inc(&nthreadsrunning);
+ qatomic_inc(&nthreadsrunning);
while (goflag == GOFLAG_INIT) {
g_usleep(1000);
}
static void perftestrun(int nthreads, int duration, int nreaders, int nupdaters)
{
- while (atomic_read(&nthreadsrunning) < nthreads) {
+ while (qatomic_read(&nthreadsrunning) < nthreads) {
g_usleep(1000);
}
goflag = GOFLAG_RUN;
}
while (goflag == GOFLAG_RUN) {
rcu_read_lock();
- p = atomic_rcu_read(&rcu_stress_current);
- if (atomic_read(&p->mbtest) == 0) {
+ p = qatomic_rcu_read(&rcu_stress_current);
+ if (qatomic_read(&p->mbtest) == 0) {
n_mberror++;
}
rcu_read_lock();
garbage++;
}
rcu_read_unlock();
- pc = atomic_read(&p->age);
+ pc = qatomic_read(&p->age);
rcu_read_unlock();
if ((pc > RCU_STRESS_PIPE_LEN) || (pc < 0)) {
pc = RCU_STRESS_PIPE_LEN;
static void *rcu_update_stress_test(void *arg)
{
int i, rcu_stress_idx = 0;
- struct rcu_stress *cp = atomic_read(&rcu_stress_current);
+ struct rcu_stress *cp = qatomic_read(&rcu_stress_current);
rcu_register_thread();
*(struct rcu_reader_data **)arg = &rcu_reader;
p = &rcu_stress_array[rcu_stress_idx];
/* catching up with ourselves would be a bug */
assert(p != cp);
- atomic_set(&p->mbtest, 0);
+ qatomic_set(&p->mbtest, 0);
smp_mb();
- atomic_set(&p->age, 0);
- atomic_set(&p->mbtest, 1);
- atomic_rcu_set(&rcu_stress_current, p);
+ qatomic_set(&p->age, 0);
+ qatomic_set(&p->mbtest, 1);
+ qatomic_rcu_set(&rcu_stress_current, p);
cp = p;
/*
* New RCU structure is now live, update pipe counts on old
*/
for (i = 0; i < RCU_STRESS_PIPE_LEN; i++) {
if (i != rcu_stress_idx) {
- atomic_set(&rcu_stress_array[i].age,
+ qatomic_set(&rcu_stress_array[i].age,
rcu_stress_array[i].age + 1);
}
}
{
Coroutine *co;
- co = atomic_xchg(&to_schedule[n], NULL);
+ co = qatomic_xchg(&to_schedule[n], NULL);
if (!co) {
- atomic_inc(&count_retry);
+ qatomic_inc(&count_retry);
return false;
}
if (n == id) {
- atomic_inc(&count_here);
+ qatomic_inc(&count_here);
} else {
- atomic_inc(&count_other);
+ qatomic_inc(&count_other);
}
aio_co_schedule(ctx[n], co);
{
g_assert(to_schedule[id] == NULL);
- while (!atomic_mb_read(&now_stopping)) {
+ while (!qatomic_mb_read(&now_stopping)) {
int n;
n = g_test_rand_int_range(0, NUM_CONTEXTS);
schedule_next(n);
- atomic_mb_set(&to_schedule[id], qemu_coroutine_self());
+ qatomic_mb_set(&to_schedule[id], qemu_coroutine_self());
qemu_coroutine_yield();
g_assert(to_schedule[id] == NULL);
}
g_usleep(seconds * 1000000);
- atomic_mb_set(&now_stopping, true);
+ qatomic_mb_set(&now_stopping, true);
for (i = 0; i < NUM_CONTEXTS; i++) {
ctx_run(i, finish_cb, NULL);
to_schedule[i] = NULL;
static void coroutine_fn test_multi_co_mutex_entry(void *opaque)
{
- while (!atomic_mb_read(&now_stopping)) {
+ while (!qatomic_mb_read(&now_stopping)) {
qemu_co_mutex_lock(&comutex);
counter++;
qemu_co_mutex_unlock(&comutex);
* exits before the coroutine is woken up, causing a spurious
* assertion failure.
*/
- atomic_inc(&atomic_counter);
+ qatomic_inc(&atomic_counter);
}
- atomic_dec(&running);
+ qatomic_dec(&running);
}
static void test_multi_co_mutex(int threads, int seconds)
g_usleep(seconds * 1000000);
- atomic_mb_set(&now_stopping, true);
+ qatomic_mb_set(&now_stopping, true);
while (running > 0) {
g_usleep(100000);
}
nodes[id].next = -1;
nodes[id].locked = 1;
- prev = atomic_xchg(&mutex_head, id);
+ prev = qatomic_xchg(&mutex_head, id);
if (prev != -1) {
- atomic_set(&nodes[prev].next, id);
+ qatomic_set(&nodes[prev].next, id);
qemu_futex_wait(&nodes[id].locked, 1);
}
}
static void mcs_mutex_unlock(void)
{
int next;
- if (atomic_read(&nodes[id].next) == -1) {
- if (atomic_read(&mutex_head) == id &&
- atomic_cmpxchg(&mutex_head, id, -1) == id) {
+ if (qatomic_read(&nodes[id].next) == -1) {
+ if (qatomic_read(&mutex_head) == id &&
+ qatomic_cmpxchg(&mutex_head, id, -1) == id) {
/* Last item in the list, exit. */
return;
}
- while (atomic_read(&nodes[id].next) == -1) {
+ while (qatomic_read(&nodes[id].next) == -1) {
/* mcs_mutex_lock did the xchg, but has not updated
* nodes[prev].next yet.
*/
}
/* Wake up the next in line. */
- next = atomic_read(&nodes[id].next);
+ next = qatomic_read(&nodes[id].next);
nodes[next].locked = 0;
qemu_futex_wake(&nodes[next].locked, 1);
}
static void test_multi_fair_mutex_entry(void *opaque)
{
- while (!atomic_mb_read(&now_stopping)) {
+ while (!qatomic_mb_read(&now_stopping)) {
mcs_mutex_lock();
counter++;
mcs_mutex_unlock();
- atomic_inc(&atomic_counter);
+ qatomic_inc(&atomic_counter);
}
- atomic_dec(&running);
+ qatomic_dec(&running);
}
static void test_multi_fair_mutex(int threads, int seconds)
g_usleep(seconds * 1000000);
- atomic_mb_set(&now_stopping, true);
+ qatomic_mb_set(&now_stopping, true);
while (running > 0) {
g_usleep(100000);
}
static void test_multi_mutex_entry(void *opaque)
{
- while (!atomic_mb_read(&now_stopping)) {
+ while (!qatomic_mb_read(&now_stopping)) {
qemu_mutex_lock(&mutex);
counter++;
qemu_mutex_unlock(&mutex);
- atomic_inc(&atomic_counter);
+ qatomic_inc(&atomic_counter);
}
- atomic_dec(&running);
+ qatomic_dec(&running);
}
static void test_multi_mutex(int threads, int seconds)
g_usleep(seconds * 1000000);
- atomic_mb_set(&now_stopping, true);
+ qatomic_mb_set(&now_stopping, true);
while (running > 0) {
g_usleep(100000);
}
*/
qemu_set_log_filename(file_path, &error_abort);
rcu_read_lock();
- logfile = atomic_rcu_read(&qemu_logfile);
+ logfile = qatomic_rcu_read(&qemu_logfile);
orig_fd = logfile->fd;
g_assert(logfile && logfile->fd);
fprintf(logfile->fd, "%s 1st write to file\n", __func__);
/* Change the logfile and ensure that the handle is still valid. */
qemu_set_log_filename(file_path1, &error_abort);
- logfile2 = atomic_rcu_read(&qemu_logfile);
+ logfile2 = qatomic_rcu_read(&qemu_logfile);
g_assert(logfile->fd == orig_fd);
g_assert(logfile2->fd != logfile->fd);
fprintf(logfile->fd, "%s 2nd write to file\n", __func__);
struct list_element *el = container_of(prcu, struct list_element, rcu);
g_free(el);
/* Accessed only from call_rcu thread. */
- atomic_set_i64(&n_reclaims, n_reclaims + 1);
+ qatomic_set_i64(&n_reclaims, n_reclaims + 1);
}
#if TEST_LIST_TYPE == 1
rcu_register_thread();
*(struct rcu_reader_data **)arg = &rcu_reader;
- atomic_inc(&nthreadsrunning);
- while (atomic_read(&goflag) == GOFLAG_INIT) {
+ qatomic_inc(&nthreadsrunning);
+ while (qatomic_read(&goflag) == GOFLAG_INIT) {
g_usleep(1000);
}
- while (atomic_read(&goflag) == GOFLAG_RUN) {
+ while (qatomic_read(&goflag) == GOFLAG_RUN) {
rcu_read_lock();
TEST_LIST_FOREACH_RCU(el, &Q_list_head, entry) {
n_reads_local++;
- if (atomic_read(&goflag) == GOFLAG_STOP) {
+ if (qatomic_read(&goflag) == GOFLAG_STOP) {
break;
}
}
struct list_element *el, *prev_el;
*(struct rcu_reader_data **)arg = &rcu_reader;
- atomic_inc(&nthreadsrunning);
- while (atomic_read(&goflag) == GOFLAG_INIT) {
+ qatomic_inc(&nthreadsrunning);
+ while (qatomic_read(&goflag) == GOFLAG_INIT) {
g_usleep(1000);
}
- while (atomic_read(&goflag) == GOFLAG_RUN) {
+ while (qatomic_read(&goflag) == GOFLAG_RUN) {
target_el = select_random_el(RCU_Q_LEN);
j = 0;
/* FOREACH_RCU could work here but let's use both macros */
break;
}
}
- if (atomic_read(&goflag) == GOFLAG_STOP) {
+ if (qatomic_read(&goflag) == GOFLAG_STOP) {
break;
}
target_el = select_random_el(RCU_Q_LEN);
qemu_mutex_lock(&counts_mutex);
n_nodes += n_nodes_local;
n_updates += n_updates_local;
- atomic_set_i64(&n_nodes_removed, n_nodes_removed + n_removed_local);
+ qatomic_set_i64(&n_nodes_removed, n_nodes_removed + n_removed_local);
qemu_mutex_unlock(&counts_mutex);
return NULL;
}
static void rcu_qtest_run(int duration, int nreaders)
{
int nthreads = nreaders + 1;
- while (atomic_read(&nthreadsrunning) < nthreads) {
+ while (qatomic_read(&nthreadsrunning) < nthreads) {
g_usleep(1000);
}
- atomic_set(&goflag, GOFLAG_RUN);
+ qatomic_set(&goflag, GOFLAG_RUN);
sleep(duration);
- atomic_set(&goflag, GOFLAG_STOP);
+ qatomic_set(&goflag, GOFLAG_STOP);
wait_all_threads();
}
n_removed_local++;
}
qemu_mutex_lock(&counts_mutex);
- atomic_set_i64(&n_nodes_removed, n_nodes_removed + n_removed_local);
+ qatomic_set_i64(&n_nodes_removed, n_nodes_removed + n_removed_local);
qemu_mutex_unlock(&counts_mutex);
synchronize_rcu();
- while (atomic_read_i64(&n_nodes_removed) > atomic_read_i64(&n_reclaims)) {
+ while (qatomic_read_i64(&n_nodes_removed) >
+ qatomic_read_i64(&n_reclaims)) {
g_usleep(100);
synchronize_rcu();
}
if (g_test_in_charge) {
- g_assert_cmpint(atomic_read_i64(&n_nodes_removed), ==,
- atomic_read_i64(&n_reclaims));
+ g_assert_cmpint(qatomic_read_i64(&n_nodes_removed), ==,
+ qatomic_read_i64(&n_reclaims));
} else {
printf("%s: %d readers; 1 updater; nodes read: " \
"%lld, nodes removed: %"PRIi64"; nodes reclaimed: %"PRIi64"\n",
test, nthreadsrunning - 1, n_reads,
- atomic_read_i64(&n_nodes_removed), atomic_read_i64(&n_reclaims));
+ qatomic_read_i64(&n_nodes_removed),
+ qatomic_read_i64(&n_reclaims));
exit(0);
}
}
static int worker_cb(void *opaque)
{
WorkerTestData *data = opaque;
- return atomic_fetch_inc(&data->n);
+ return qatomic_fetch_inc(&data->n);
}
static int long_cb(void *opaque)
{
WorkerTestData *data = opaque;
- if (atomic_cmpxchg(&data->n, 0, 1) == 0) {
+ if (qatomic_cmpxchg(&data->n, 0, 1) == 0) {
g_usleep(2000000);
- atomic_or(&data->n, 2);
+ qatomic_or(&data->n, 2);
}
return 0;
}
/* Cancel the jobs that haven't been started yet. */
num_canceled = 0;
for (i = 0; i < 100; i++) {
- if (atomic_cmpxchg(&data[i].n, 0, 4) == 0) {
+ if (qatomic_cmpxchg(&data[i].n, 0, 4) == 0) {
data[i].ret = -ECANCELED;
if (sync) {
bdrv_aio_cancel(data[i].aiocb);
g_assert_cmpint(num_canceled, <, 100);
for (i = 0; i < 100; i++) {
- if (data[i].aiocb && atomic_read(&data[i].n) < 4) {
+ if (data[i].aiocb && qatomic_read(&data[i].n) < 4) {
if (sync) {
/* Canceling the others will be a blocking operation. */
bdrv_aio_cancel(data[i].aiocb);
bool aio_poll_disabled(AioContext *ctx)
{
- return atomic_read(&ctx->poll_disable_cnt);
+ return qatomic_read(&ctx->poll_disable_cnt);
}
void aio_add_ready_handler(AioHandlerList *ready_list,
* Changing handlers is a rare event, and a little wasted polling until
* the aio_notify below is not an issue.
*/
- atomic_set(&ctx->poll_disable_cnt,
- atomic_read(&ctx->poll_disable_cnt) + poll_disable_change);
+ qatomic_set(&ctx->poll_disable_cnt,
+ qatomic_read(&ctx->poll_disable_cnt) + poll_disable_change);
ctx->fdmon_ops->update(ctx, node, new_node);
if (node) {
*/
use_notify_me = timeout != 0;
if (use_notify_me) {
- atomic_set(&ctx->notify_me, atomic_read(&ctx->notify_me) + 2);
+ qatomic_set(&ctx->notify_me, qatomic_read(&ctx->notify_me) + 2);
/*
* Write ctx->notify_me before reading ctx->notified. Pairs with
* smp_mb in aio_notify().
smp_mb();
/* Don't block if aio_notify() was called */
- if (atomic_read(&ctx->notified)) {
+ if (qatomic_read(&ctx->notified)) {
timeout = 0;
}
}
if (use_notify_me) {
/* Finish the poll before clearing the flag. */
- atomic_store_release(&ctx->notify_me,
- atomic_read(&ctx->notify_me) - 2);
+ qatomic_store_release(&ctx->notify_me,
+ qatomic_read(&ctx->notify_me) - 2);
}
aio_notify_accept(ctx);
void aio_wait_kick(void)
{
/* The barrier (or an atomic op) is in the caller. */
- if (atomic_read(&global_aio_wait.num_waiters)) {
+ if (qatomic_read(&global_aio_wait.num_waiters)) {
aio_bh_schedule_oneshot(qemu_get_aio_context(), dummy_bh_cb, NULL);
}
}
* so disable the optimization now.
*/
if (blocking) {
- atomic_set(&ctx->notify_me, atomic_read(&ctx->notify_me) + 2);
+ qatomic_set(&ctx->notify_me, qatomic_read(&ctx->notify_me) + 2);
/*
* Write ctx->notify_me before computing the timeout
* (reading bottom half flags, etc.). Pairs with
ret = WaitForMultipleObjects(count, events, FALSE, timeout);
if (blocking) {
assert(first);
- atomic_store_release(&ctx->notify_me, atomic_read(&ctx->notify_me) - 2);
+ qatomic_store_release(&ctx->notify_me,
+ qatomic_read(&ctx->notify_me) - 2);
aio_notify_accept(ctx);
}
unsigned old_flags;
/*
- * The memory barrier implicit in atomic_fetch_or makes sure that:
+ * The memory barrier implicit in qatomic_fetch_or makes sure that:
* 1. idle & any writes needed by the callback are done before the
* locations are read in the aio_bh_poll.
* 2. ctx is loaded before the callback has a chance to execute and bh
* could be freed.
*/
- old_flags = atomic_fetch_or(&bh->flags, BH_PENDING | new_flags);
+ old_flags = qatomic_fetch_or(&bh->flags, BH_PENDING | new_flags);
if (!(old_flags & BH_PENDING)) {
QSLIST_INSERT_HEAD_ATOMIC(&ctx->bh_list, bh, next);
}
QSLIST_REMOVE_HEAD(head, next);
/*
- * The atomic_and is paired with aio_bh_enqueue(). The implicit memory
+ * The qatomic_and is paired with aio_bh_enqueue(). The implicit memory
* barrier ensures that the callback sees all writes done by the scheduling
* thread. It also ensures that the scheduling thread sees the cleared
* flag before bh->cb has run, and thus will call aio_notify again if
* necessary.
*/
- *flags = atomic_fetch_and(&bh->flags,
+ *flags = qatomic_fetch_and(&bh->flags,
~(BH_PENDING | BH_SCHEDULED | BH_IDLE));
return bh;
}
*/
void qemu_bh_cancel(QEMUBH *bh)
{
- atomic_and(&bh->flags, ~BH_SCHEDULED);
+ qatomic_and(&bh->flags, ~BH_SCHEDULED);
}
/* This func is async.The bottom half will do the delete action at the finial
{
AioContext *ctx = (AioContext *) source;
- atomic_set(&ctx->notify_me, atomic_read(&ctx->notify_me) | 1);
+ qatomic_set(&ctx->notify_me, qatomic_read(&ctx->notify_me) | 1);
/*
* Write ctx->notify_me before computing the timeout
BHListSlice *s;
/* Finish computing the timeout before clearing the flag. */
- atomic_store_release(&ctx->notify_me, atomic_read(&ctx->notify_me) & ~1);
+ qatomic_store_release(&ctx->notify_me, qatomic_read(&ctx->notify_me) & ~1);
aio_notify_accept(ctx);
QSLIST_FOREACH_RCU(bh, &ctx->bh_list, next) {
* aio_notify_accept.
*/
smp_wmb();
- atomic_set(&ctx->notified, true);
+ qatomic_set(&ctx->notified, true);
/*
* Write ctx->notified before reading ctx->notify_me. Pairs
* with smp_mb in aio_ctx_prepare or aio_poll.
*/
smp_mb();
- if (atomic_read(&ctx->notify_me)) {
+ if (qatomic_read(&ctx->notify_me)) {
event_notifier_set(&ctx->notifier);
}
}
void aio_notify_accept(AioContext *ctx)
{
- atomic_set(&ctx->notified, false);
+ qatomic_set(&ctx->notified, false);
/*
* Write ctx->notified before reading e.g. bh->flags. Pairs with smp_wmb
EventNotifier *e = opaque;
AioContext *ctx = container_of(e, AioContext, notifier);
- return atomic_read(&ctx->notified);
+ return qatomic_read(&ctx->notified);
}
static void co_schedule_bh_cb(void *opaque)
aio_context_acquire(ctx);
/* Protected by write barrier in qemu_aio_coroutine_enter */
- atomic_set(&co->scheduled, NULL);
+ qatomic_set(&co->scheduled, NULL);
qemu_aio_coroutine_enter(ctx, co);
aio_context_release(ctx);
}
void aio_co_schedule(AioContext *ctx, Coroutine *co)
{
trace_aio_co_schedule(ctx, co);
- const char *scheduled = atomic_cmpxchg(&co->scheduled, NULL,
+ const char *scheduled = qatomic_cmpxchg(&co->scheduled, NULL,
__func__);
if (scheduled) {
* qemu_coroutine_enter.
*/
smp_read_barrier_depends();
- ctx = atomic_read(&co->ctx);
+ ctx = qatomic_read(&co->ctx);
aio_co_enter(ctx, co);
}
return ret; \
}
-GEN_READ(atomic_read_i64, int64_t)
-GEN_READ(atomic_read_u64, uint64_t)
+GEN_READ(qatomic_read_i64, int64_t)
+GEN_READ(qatomic_read_u64, uint64_t)
#undef GEN_READ
#define GEN_SET(name, type) \
qemu_spin_unlock(lock); \
}
-GEN_SET(atomic_set_i64, int64_t)
-GEN_SET(atomic_set_u64, uint64_t)
+GEN_SET(qatomic_set_i64, int64_t)
+GEN_SET(qatomic_set_u64, uint64_t)
#undef GEN_SET
-void atomic64_init(void)
+void qatomic64_init(void)
{
int i;
/* First word */
if (nr - bits_to_set > 0) {
- atomic_or(p, mask_to_set);
+ qatomic_or(p, mask_to_set);
nr -= bits_to_set;
bits_to_set = BITS_PER_LONG;
mask_to_set = ~0UL;
/* Last word */
if (nr) {
mask_to_set &= BITMAP_LAST_WORD_MASK(size);
- atomic_or(p, mask_to_set);
+ qatomic_or(p, mask_to_set);
} else {
- /* If we avoided the full barrier in atomic_or(), issue a
+ /* If we avoided the full barrier in qatomic_or(), issue a
* barrier to account for the assignments in the while loop.
*/
smp_mb();
/* First word */
if (nr - bits_to_clear > 0) {
- old_bits = atomic_fetch_and(p, ~mask_to_clear);
+ old_bits = qatomic_fetch_and(p, ~mask_to_clear);
dirty |= old_bits & mask_to_clear;
nr -= bits_to_clear;
bits_to_clear = BITS_PER_LONG;
if (bits_to_clear == BITS_PER_LONG) {
while (nr >= BITS_PER_LONG) {
if (*p) {
- old_bits = atomic_xchg(p, 0);
+ old_bits = qatomic_xchg(p, 0);
dirty |= old_bits;
}
nr -= BITS_PER_LONG;
/* Last word */
if (nr) {
mask_to_clear &= BITMAP_LAST_WORD_MASK(size);
- old_bits = atomic_fetch_and(p, ~mask_to_clear);
+ old_bits = qatomic_fetch_and(p, ~mask_to_clear);
dirty |= old_bits & mask_to_clear;
} else {
if (!dirty) {
long nr)
{
while (nr > 0) {
- *dst = atomic_xchg(src, 0);
+ *dst = qatomic_xchg(src, 0);
dst++;
src++;
nr -= BITS_PER_LONG;
qemu_dcache_linesize = dsize;
qemu_dcache_linesize_log = ctz32(dsize);
- atomic64_init();
+ qatomic64_init();
}
struct epoll_event events[128];
/* Fall back while external clients are disabled */
- if (atomic_read(&ctx->external_disable_cnt)) {
+ if (qatomic_read(&ctx->external_disable_cnt)) {
return fdmon_poll_ops.wait(ctx, ready_list, timeout);
}
}
/* Do not upgrade while external clients are disabled */
- if (atomic_read(&ctx->external_disable_cnt)) {
+ if (qatomic_read(&ctx->external_disable_cnt)) {
return false;
}
{
unsigned old_flags;
- old_flags = atomic_fetch_or(&node->flags, FDMON_IO_URING_PENDING | flags);
+ old_flags = qatomic_fetch_or(&node->flags, FDMON_IO_URING_PENDING | flags);
if (!(old_flags & FDMON_IO_URING_PENDING)) {
QSLIST_INSERT_HEAD_ATOMIC(head, node, node_submitted);
}
* telling process_cqe() to delete the AioHandler when its
* IORING_OP_POLL_ADD completes.
*/
- *flags = atomic_fetch_and(&node->flags, ~(FDMON_IO_URING_PENDING |
+ *flags = qatomic_fetch_and(&node->flags, ~(FDMON_IO_URING_PENDING |
FDMON_IO_URING_ADD));
return node;
}
* with enqueue() here then we can safely clear the FDMON_IO_URING_REMOVE
* bit before IORING_OP_POLL_REMOVE is submitted.
*/
- flags = atomic_fetch_and(&node->flags, ~FDMON_IO_URING_REMOVE);
+ flags = qatomic_fetch_and(&node->flags, ~FDMON_IO_URING_REMOVE);
if (flags & FDMON_IO_URING_REMOVE) {
QLIST_INSERT_HEAD_RCU(&ctx->deleted_aio_handlers, node, node_deleted);
return false;
int ret;
/* Fall back while external clients are disabled */
- if (atomic_read(&ctx->external_disable_cnt)) {
+ if (qatomic_read(&ctx->external_disable_cnt)) {
return fdmon_poll_ops.wait(ctx, ready_list, timeout);
}
}
/* Are we falling back to fdmon-poll? */
- return atomic_read(&ctx->external_disable_cnt);
+ return qatomic_read(&ctx->external_disable_cnt);
}
static const FDMonOps fdmon_io_uring_ops = {
/* Move handlers due to be removed onto the deleted list */
while ((node = QSLIST_FIRST_RCU(&ctx->submit_list))) {
- unsigned flags = atomic_fetch_and(&node->flags,
+ unsigned flags = qatomic_fetch_and(&node->flags,
~(FDMON_IO_URING_PENDING |
FDMON_IO_URING_ADD |
FDMON_IO_URING_REMOVE));
int expected = *val;
trace_lockcnt_fast_path_attempt(lockcnt, expected, new_if_free);
- *val = atomic_cmpxchg(&lockcnt->count, expected, new_if_free);
+ *val = qatomic_cmpxchg(&lockcnt->count, expected, new_if_free);
if (*val == expected) {
trace_lockcnt_fast_path_success(lockcnt, expected, new_if_free);
*val = new_if_free;
int new = expected - QEMU_LOCKCNT_STATE_LOCKED + QEMU_LOCKCNT_STATE_WAITING;
trace_lockcnt_futex_wait_prepare(lockcnt, expected, new);
- *val = atomic_cmpxchg(&lockcnt->count, expected, new);
+ *val = qatomic_cmpxchg(&lockcnt->count, expected, new);
if (*val == expected) {
*val = new;
}
*waited = true;
trace_lockcnt_futex_wait(lockcnt, *val);
qemu_futex_wait(&lockcnt->count, *val);
- *val = atomic_read(&lockcnt->count);
+ *val = qatomic_read(&lockcnt->count);
trace_lockcnt_futex_wait_resume(lockcnt, *val);
continue;
}
void qemu_lockcnt_inc(QemuLockCnt *lockcnt)
{
- int val = atomic_read(&lockcnt->count);
+ int val = qatomic_read(&lockcnt->count);
bool waited = false;
for (;;) {
if (val >= QEMU_LOCKCNT_COUNT_STEP) {
int expected = val;
- val = atomic_cmpxchg(&lockcnt->count, val, val + QEMU_LOCKCNT_COUNT_STEP);
+ val = qatomic_cmpxchg(&lockcnt->count, val,
+ val + QEMU_LOCKCNT_COUNT_STEP);
if (val == expected) {
break;
}
void qemu_lockcnt_dec(QemuLockCnt *lockcnt)
{
- atomic_sub(&lockcnt->count, QEMU_LOCKCNT_COUNT_STEP);
+ qatomic_sub(&lockcnt->count, QEMU_LOCKCNT_COUNT_STEP);
}
/* Decrement a counter, and return locked if it is decremented to zero.
*/
bool qemu_lockcnt_dec_and_lock(QemuLockCnt *lockcnt)
{
- int val = atomic_read(&lockcnt->count);
+ int val = qatomic_read(&lockcnt->count);
int locked_state = QEMU_LOCKCNT_STATE_LOCKED;
bool waited = false;
for (;;) {
if (val >= 2 * QEMU_LOCKCNT_COUNT_STEP) {
int expected = val;
- val = atomic_cmpxchg(&lockcnt->count, val, val - QEMU_LOCKCNT_COUNT_STEP);
+ val = qatomic_cmpxchg(&lockcnt->count, val,
+ val - QEMU_LOCKCNT_COUNT_STEP);
if (val == expected) {
break;
}
*/
bool qemu_lockcnt_dec_if_lock(QemuLockCnt *lockcnt)
{
- int val = atomic_read(&lockcnt->count);
+ int val = qatomic_read(&lockcnt->count);
int locked_state = QEMU_LOCKCNT_STATE_LOCKED;
bool waited = false;
void qemu_lockcnt_lock(QemuLockCnt *lockcnt)
{
- int val = atomic_read(&lockcnt->count);
+ int val = qatomic_read(&lockcnt->count);
int step = QEMU_LOCKCNT_STATE_LOCKED;
bool waited = false;
{
int expected, new, val;
- val = atomic_read(&lockcnt->count);
+ val = qatomic_read(&lockcnt->count);
do {
expected = val;
new = (val + QEMU_LOCKCNT_COUNT_STEP) & ~QEMU_LOCKCNT_STATE_MASK;
trace_lockcnt_unlock_attempt(lockcnt, val, new);
- val = atomic_cmpxchg(&lockcnt->count, val, new);
+ val = qatomic_cmpxchg(&lockcnt->count, val, new);
} while (val != expected);
trace_lockcnt_unlock_success(lockcnt, val, new);
{
int expected, new, val;
- val = atomic_read(&lockcnt->count);
+ val = qatomic_read(&lockcnt->count);
do {
expected = val;
new = val & ~QEMU_LOCKCNT_STATE_MASK;
trace_lockcnt_unlock_attempt(lockcnt, val, new);
- val = atomic_cmpxchg(&lockcnt->count, val, new);
+ val = qatomic_cmpxchg(&lockcnt->count, val, new);
} while (val != expected);
trace_lockcnt_unlock_success(lockcnt, val, new);
unsigned qemu_lockcnt_count(QemuLockCnt *lockcnt)
{
- return atomic_read(&lockcnt->count) >> QEMU_LOCKCNT_COUNT_SHIFT;
+ return qatomic_read(&lockcnt->count) >> QEMU_LOCKCNT_COUNT_SHIFT;
}
#else
void qemu_lockcnt_init(QemuLockCnt *lockcnt)
{
int old;
for (;;) {
- old = atomic_read(&lockcnt->count);
+ old = qatomic_read(&lockcnt->count);
if (old == 0) {
qemu_lockcnt_lock(lockcnt);
qemu_lockcnt_inc_and_unlock(lockcnt);
return;
} else {
- if (atomic_cmpxchg(&lockcnt->count, old, old + 1) == old) {
+ if (qatomic_cmpxchg(&lockcnt->count, old, old + 1) == old) {
return;
}
}
void qemu_lockcnt_dec(QemuLockCnt *lockcnt)
{
- atomic_dec(&lockcnt->count);
+ qatomic_dec(&lockcnt->count);
}
/* Decrement a counter, and return locked if it is decremented to zero.
*/
bool qemu_lockcnt_dec_and_lock(QemuLockCnt *lockcnt)
{
- int val = atomic_read(&lockcnt->count);
+ int val = qatomic_read(&lockcnt->count);
while (val > 1) {
- int old = atomic_cmpxchg(&lockcnt->count, val, val - 1);
+ int old = qatomic_cmpxchg(&lockcnt->count, val, val - 1);
if (old != val) {
val = old;
continue;
}
qemu_lockcnt_lock(lockcnt);
- if (atomic_fetch_dec(&lockcnt->count) == 1) {
+ if (qatomic_fetch_dec(&lockcnt->count) == 1) {
return true;
}
bool qemu_lockcnt_dec_if_lock(QemuLockCnt *lockcnt)
{
/* No need for acquire semantics if we return false. */
- int val = atomic_read(&lockcnt->count);
+ int val = qatomic_read(&lockcnt->count);
if (val > 1) {
return false;
}
qemu_lockcnt_lock(lockcnt);
- if (atomic_fetch_dec(&lockcnt->count) == 1) {
+ if (qatomic_fetch_dec(&lockcnt->count) == 1) {
return true;
}
void qemu_lockcnt_inc_and_unlock(QemuLockCnt *lockcnt)
{
- atomic_inc(&lockcnt->count);
+ qatomic_inc(&lockcnt->count);
qemu_mutex_unlock(&lockcnt->mutex);
}
unsigned qemu_lockcnt_count(QemuLockCnt *lockcnt)
{
- return atomic_read(&lockcnt->count);
+ return qatomic_read(&lockcnt->count);
}
#endif
QemuLogFile *logfile;
rcu_read_lock();
- logfile = atomic_rcu_read(&qemu_logfile);
+ logfile = qatomic_rcu_read(&qemu_logfile);
if (logfile) {
va_list ap;
va_start(ap, fmt);
QEMU_LOCK_GUARD(&qemu_logfile_mutex);
if (qemu_logfile && !need_to_open_file) {
logfile = qemu_logfile;
- atomic_rcu_set(&qemu_logfile, NULL);
+ qatomic_rcu_set(&qemu_logfile, NULL);
call_rcu(logfile, qemu_logfile_free, rcu);
} else if (!qemu_logfile && need_to_open_file) {
logfile = g_new0(QemuLogFile, 1);
#endif
log_append = 1;
}
- atomic_rcu_set(&qemu_logfile, logfile);
+ qatomic_rcu_set(&qemu_logfile, logfile);
}
}
QemuLogFile *logfile;
rcu_read_lock();
- logfile = atomic_rcu_read(&qemu_logfile);
+ logfile = qatomic_rcu_read(&qemu_logfile);
if (logfile) {
fflush(logfile->fd);
}
logfile = qemu_logfile;
if (logfile) {
- atomic_rcu_set(&qemu_logfile, NULL);
+ qatomic_rcu_set(&qemu_logfile, NULL);
call_rcu(logfile, qemu_logfile_free, rcu);
}
qemu_mutex_unlock(&qemu_logfile_mutex);
/* This is the "Responsibility Hand-Off" protocol; a lock() picks from
* a concurrent unlock() the responsibility of waking somebody up.
*/
- old_handoff = atomic_mb_read(&mutex->handoff);
+ old_handoff = qatomic_mb_read(&mutex->handoff);
if (old_handoff &&
has_waiters(mutex) &&
- atomic_cmpxchg(&mutex->handoff, old_handoff, 0) == old_handoff) {
+ qatomic_cmpxchg(&mutex->handoff, old_handoff, 0) == old_handoff) {
/* There can be no concurrent pops, because there can be only
* one active handoff at a time.
*/
*/
i = 0;
retry_fast_path:
- waiters = atomic_cmpxchg(&mutex->locked, 0, 1);
+ waiters = qatomic_cmpxchg(&mutex->locked, 0, 1);
if (waiters != 0) {
while (waiters == 1 && ++i < 1000) {
- if (atomic_read(&mutex->ctx) == ctx) {
+ if (qatomic_read(&mutex->ctx) == ctx) {
break;
}
- if (atomic_read(&mutex->locked) == 0) {
+ if (qatomic_read(&mutex->locked) == 0) {
goto retry_fast_path;
}
cpu_relax();
}
- waiters = atomic_fetch_inc(&mutex->locked);
+ waiters = qatomic_fetch_inc(&mutex->locked);
}
if (waiters == 0) {
mutex->ctx = NULL;
mutex->holder = NULL;
self->locks_held--;
- if (atomic_fetch_dec(&mutex->locked) == 1) {
+ if (qatomic_fetch_dec(&mutex->locked) == 1) {
/* No waiting qemu_co_mutex_lock(). Pfew, that was easy! */
return;
}
}
our_handoff = mutex->sequence;
- atomic_mb_set(&mutex->handoff, our_handoff);
+ qatomic_mb_set(&mutex->handoff, our_handoff);
if (!has_waiters(mutex)) {
/* The concurrent lock has not added itself yet, so it
* will be able to pick our handoff.
/* Try to do the handoff protocol ourselves; if somebody else has
* already taken it, however, we're done and they're responsible.
*/
- if (atomic_cmpxchg(&mutex->handoff, our_handoff, 0) != our_handoff) {
+ if (qatomic_cmpxchg(&mutex->handoff, our_handoff, 0) != our_handoff) {
break;
}
}
void qemu_co_sleep_wake(QemuCoSleepState *sleep_state)
{
/* Write of schedule protected by barrier write in aio_co_schedule */
- const char *scheduled = atomic_cmpxchg(&sleep_state->co->scheduled,
+ const char *scheduled = qatomic_cmpxchg(&sleep_state->co->scheduled,
qemu_co_sleep_ns__scheduled, NULL);
assert(scheduled == qemu_co_sleep_ns__scheduled);
.user_state_pointer = sleep_state,
};
- const char *scheduled = atomic_cmpxchg(&state.co->scheduled, NULL,
+ const char *scheduled = qatomic_cmpxchg(&state.co->scheduled, NULL,
qemu_co_sleep_ns__scheduled);
if (scheduled) {
fprintf(stderr,
* release_pool_size and the actual size of release_pool. But
* it is just a heuristic, it does not need to be perfect.
*/
- alloc_pool_size = atomic_xchg(&release_pool_size, 0);
+ alloc_pool_size = qatomic_xchg(&release_pool_size, 0);
QSLIST_MOVE_ATOMIC(&alloc_pool, &release_pool);
co = QSLIST_FIRST(&alloc_pool);
}
if (CONFIG_COROUTINE_POOL) {
if (release_pool_size < POOL_BATCH_SIZE * 2) {
QSLIST_INSERT_HEAD_ATOMIC(&release_pool, co, pool_next);
- atomic_inc(&release_pool_size);
+ qatomic_inc(&release_pool_size);
return;
}
if (alloc_pool_size < POOL_BATCH_SIZE) {
/* Cannot rely on the read barrier for to in aio_co_wake(), as there are
* callers outside of aio_co_wake() */
- const char *scheduled = atomic_mb_read(&to->scheduled);
+ const char *scheduled = qatomic_mb_read(&to->scheduled);
QSIMPLEQ_REMOVE_HEAD(&pending, co_queue_next);
memset(&ai, 0, sizeof(ai));
ai.ai_flags = AI_CANONNAME | AI_ADDRCONFIG;
- if (atomic_read(&useV4Mapped)) {
+ if (qatomic_read(&useV4Mapped)) {
ai.ai_flags |= AI_V4MAPPED;
}
ai.ai_family = inet_ai_family_from_address(saddr, &err);
*/
if (rc == EAI_BADFLAGS &&
(ai.ai_flags & AI_V4MAPPED)) {
- atomic_set(&useV4Mapped, 0);
+ qatomic_set(&useV4Mapped, 0);
ai.ai_flags &= ~AI_V4MAPPED;
rc = getaddrinfo(saddr->host, saddr->port, &ai, &res);
}
*/
assert(ev->initialized);
smp_mb();
- if (atomic_read(&ev->value) != EV_SET) {
- if (atomic_xchg(&ev->value, EV_SET) == EV_BUSY) {
+ if (qatomic_read(&ev->value) != EV_SET) {
+ if (qatomic_xchg(&ev->value, EV_SET) == EV_BUSY) {
/* There were waiters, wake them up. */
qemu_futex_wake(ev, INT_MAX);
}
unsigned value;
assert(ev->initialized);
- value = atomic_read(&ev->value);
+ value = qatomic_read(&ev->value);
smp_mb_acquire();
if (value == EV_SET) {
/*
* If there was a concurrent reset (or even reset+wait),
* do nothing. Otherwise change EV_SET->EV_FREE.
*/
- atomic_or(&ev->value, EV_FREE);
+ qatomic_or(&ev->value, EV_FREE);
}
}
unsigned value;
assert(ev->initialized);
- value = atomic_read(&ev->value);
+ value = qatomic_read(&ev->value);
smp_mb_acquire();
if (value != EV_SET) {
if (value == EV_FREE) {
* a concurrent busy->free transition. After the CAS, the
* event will be either set or busy.
*/
- if (atomic_cmpxchg(&ev->value, EV_FREE, EV_BUSY) == EV_SET) {
+ if (qatomic_cmpxchg(&ev->value, EV_FREE, EV_BUSY) == EV_SET) {
return;
}
}
* ev->value we need a full memory barrier here.
*/
smp_mb();
- if (atomic_read(&ev->value) != EV_SET) {
- if (atomic_xchg(&ev->value, EV_SET) == EV_BUSY) {
+ if (qatomic_read(&ev->value) != EV_SET) {
+ if (qatomic_xchg(&ev->value, EV_SET) == EV_BUSY) {
/* There were waiters, wake them up. */
SetEvent(ev->event);
}
unsigned value;
assert(ev->initialized);
- value = atomic_read(&ev->value);
+ value = qatomic_read(&ev->value);
smp_mb_acquire();
if (value == EV_SET) {
/* If there was a concurrent reset (or even reset+wait),
* do nothing. Otherwise change EV_SET->EV_FREE.
*/
- atomic_or(&ev->value, EV_FREE);
+ qatomic_or(&ev->value, EV_FREE);
}
}
unsigned value;
assert(ev->initialized);
- value = atomic_read(&ev->value);
+ value = qatomic_read(&ev->value);
smp_mb_acquire();
if (value != EV_SET) {
if (value == EV_FREE) {
* because there cannot be a concurrent busy->free transition.
* After the CAS, the event will be either set or busy.
*/
- if (atomic_cmpxchg(&ev->value, EV_FREE, EV_BUSY) == EV_SET) {
+ if (qatomic_cmpxchg(&ev->value, EV_FREE, EV_BUSY) == EV_SET) {
value = EV_SET;
} else {
value = EV_BUSY;
bool timerlist_has_timers(QEMUTimerList *timer_list)
{
- return !!atomic_read(&timer_list->active_timers);
+ return !!qatomic_read(&timer_list->active_timers);
}
bool qemu_clock_has_timers(QEMUClockType type)
{
int64_t expire_time;
- if (!atomic_read(&timer_list->active_timers)) {
+ if (!qatomic_read(&timer_list->active_timers)) {
return false;
}
int64_t delta;
int64_t expire_time;
- if (!atomic_read(&timer_list->active_timers)) {
+ if (!qatomic_read(&timer_list->active_timers)) {
return -1;
}
if (!t)
break;
if (t == ts) {
- atomic_set(pt, t->next);
+ qatomic_set(pt, t->next);
break;
}
pt = &t->next;
}
ts->expire_time = MAX(expire_time, 0);
ts->next = *pt;
- atomic_set(pt, ts);
+ qatomic_set(pt, ts);
return pt == &timer_list->active_timers;
}
QEMUTimerCB *cb;
void *opaque;
- if (!atomic_read(&timer_list->active_timers)) {
+ if (!qatomic_read(&timer_list->active_timers)) {
return false;
}
/*
* Note: reading partially-updated pointers in @pointers could lead to
- * segfaults. We thus access them with atomic_read/set; this guarantees
+ * segfaults. We thus access them with qatomic_read/set; this guarantees
* that the compiler makes all those accesses atomic. We also need the
- * volatile-like behavior in atomic_read, since otherwise the compiler
+ * volatile-like behavior in qatomic_read, since otherwise the compiler
* might refetch the pointer.
- * atomic_read's are of course not necessary when the bucket lock is held.
+ * qatomic_read's are of course not necessary when the bucket lock is held.
*
* If both ht->lock and b->lock are grabbed, ht->lock should always
* be grabbed first.
{
struct qht_map *map;
- map = atomic_rcu_read(&ht->map);
+ map = qatomic_rcu_read(&ht->map);
qht_map_lock_buckets(map);
if (likely(!qht_map_is_stale__locked(ht, map))) {
*pmap = map;
struct qht_bucket *b;
struct qht_map *map;
- map = atomic_rcu_read(&ht->map);
+ map = qatomic_rcu_read(&ht->map);
b = qht_map_to_bucket(map, hash);
qemu_spin_lock(&b->lock);
static inline bool qht_map_needs_resize(const struct qht_map *map)
{
- return atomic_read(&map->n_added_buckets) > map->n_added_buckets_threshold;
+ return qatomic_read(&map->n_added_buckets) >
+ map->n_added_buckets_threshold;
}
static inline void qht_chain_destroy(const struct qht_bucket *head)
ht->mode = mode;
qemu_mutex_init(&ht->lock);
map = qht_map_create(n_buckets);
- atomic_rcu_set(&ht->map, map);
+ qatomic_rcu_set(&ht->map, map);
}
/* call only when there are no readers/writers left */
if (b->pointers[i] == NULL) {
goto done;
}
- atomic_set(&b->hashes[i], 0);
- atomic_set(&b->pointers[i], NULL);
+ qatomic_set(&b->hashes[i], 0);
+ qatomic_set(&b->pointers[i], NULL);
}
b = b->next;
} while (b);
do {
for (i = 0; i < QHT_BUCKET_ENTRIES; i++) {
- if (atomic_read(&b->hashes[i]) == hash) {
+ if (qatomic_read(&b->hashes[i]) == hash) {
/* The pointer is dereferenced before seqlock_read_retry,
* so (unlike qht_insert__locked) we need to use
- * atomic_rcu_read here.
+ * qatomic_rcu_read here.
*/
- void *p = atomic_rcu_read(&b->pointers[i]);
+ void *p = qatomic_rcu_read(&b->pointers[i]);
if (likely(p) && likely(func(p, userp))) {
return p;
}
}
}
- b = atomic_rcu_read(&b->next);
+ b = qatomic_rcu_read(&b->next);
} while (b);
return NULL;
unsigned int version;
void *ret;
- map = atomic_rcu_read(&ht->map);
+ map = qatomic_rcu_read(&ht->map);
b = qht_map_to_bucket(map, hash);
version = seqlock_read_begin(&b->sequence);
memset(b, 0, sizeof(*b));
new = b;
i = 0;
- atomic_inc(&map->n_added_buckets);
+ qatomic_inc(&map->n_added_buckets);
if (unlikely(qht_map_needs_resize(map)) && needs_resize) {
*needs_resize = true;
}
/* found an empty key: acquire the seqlock and write */
seqlock_write_begin(&head->sequence);
if (new) {
- atomic_rcu_set(&prev->next, b);
+ qatomic_rcu_set(&prev->next, b);
}
/* smp_wmb() implicit in seqlock_write_begin. */
- atomic_set(&b->hashes[i], hash);
- atomic_set(&b->pointers[i], p);
+ qatomic_set(&b->hashes[i], hash);
+ qatomic_set(&b->pointers[i], p);
seqlock_write_end(&head->sequence);
return NULL;
}
qht_debug_assert(to->pointers[i]);
qht_debug_assert(from->pointers[j]);
- atomic_set(&to->hashes[i], from->hashes[j]);
- atomic_set(&to->pointers[i], from->pointers[j]);
+ qatomic_set(&to->hashes[i], from->hashes[j]);
+ qatomic_set(&to->pointers[i], from->pointers[j]);
- atomic_set(&from->hashes[j], 0);
- atomic_set(&from->pointers[j], NULL);
+ qatomic_set(&from->hashes[j], 0);
+ qatomic_set(&from->pointers[j], NULL);
}
/*
if (qht_entry_is_last(orig, pos)) {
orig->hashes[pos] = 0;
- atomic_set(&orig->pointers[pos], NULL);
+ qatomic_set(&orig->pointers[pos], NULL);
return;
}
do {
{
struct qht_map *map;
- map = atomic_rcu_read(&ht->map);
+ map = qatomic_rcu_read(&ht->map);
qht_map_lock_buckets(map);
qht_map_iter__all_locked(map, iter, userp);
qht_map_unlock_buckets(map);
qht_map_iter__all_locked(old, &iter, &data);
qht_map_debug__all_locked(new);
- atomic_rcu_set(&ht->map, new);
+ qatomic_rcu_set(&ht->map, new);
qht_map_unlock_buckets(old);
call_rcu(old, qht_map_destroy, rcu);
}
const struct qht_map *map;
int i;
- map = atomic_rcu_read(&ht->map);
+ map = qatomic_rcu_read(&ht->map);
stats->used_head_buckets = 0;
stats->entries = 0;
b = head;
do {
for (j = 0; j < QHT_BUCKET_ENTRIES; j++) {
- if (atomic_read(&b->pointers[j]) == NULL) {
+ if (qatomic_read(&b->pointers[j]) == NULL) {
break;
}
entries++;
}
buckets++;
- b = atomic_rcu_read(&b->next);
+ b = qatomic_rcu_read(&b->next);
} while (b);
} while (seqlock_read_retry(&head->sequence, version));
static __attribute__((noinline)) void qsp_init__slowpath(void)
{
- if (atomic_cmpxchg(&qsp_initializing, false, true) == false) {
+ if (qatomic_cmpxchg(&qsp_initializing, false, true) == false) {
qsp_do_init();
- atomic_set(&qsp_initialized, true);
+ qatomic_set(&qsp_initialized, true);
} else {
- while (!atomic_read(&qsp_initialized)) {
+ while (!qatomic_read(&qsp_initialized)) {
cpu_relax();
}
}
/* qsp_init() must be called from _all_ exported functions */
static inline void qsp_init(void)
{
- if (likely(atomic_read(&qsp_initialized))) {
+ if (likely(qatomic_read(&qsp_initialized))) {
return;
}
qsp_init__slowpath();
*/
static inline void do_qsp_entry_record(QSPEntry *e, int64_t delta, bool acq)
{
- atomic_set_u64(&e->ns, e->ns + delta);
+ qatomic_set_u64(&e->ns, e->ns + delta);
if (acq) {
- atomic_set_u64(&e->n_acqs, e->n_acqs + 1);
+ qatomic_set_u64(&e->n_acqs, e->n_acqs + 1);
}
}
bool qsp_is_enabled(void)
{
- return atomic_read(&qemu_mutex_lock_func) == qsp_mutex_lock;
+ return qatomic_read(&qemu_mutex_lock_func) == qsp_mutex_lock;
}
void qsp_enable(void)
{
- atomic_set(&qemu_mutex_lock_func, qsp_mutex_lock);
- atomic_set(&qemu_mutex_trylock_func, qsp_mutex_trylock);
- atomic_set(&qemu_bql_mutex_lock_func, qsp_bql_mutex_lock);
- atomic_set(&qemu_rec_mutex_lock_func, qsp_rec_mutex_lock);
- atomic_set(&qemu_rec_mutex_trylock_func, qsp_rec_mutex_trylock);
- atomic_set(&qemu_cond_wait_func, qsp_cond_wait);
- atomic_set(&qemu_cond_timedwait_func, qsp_cond_timedwait);
+ qatomic_set(&qemu_mutex_lock_func, qsp_mutex_lock);
+ qatomic_set(&qemu_mutex_trylock_func, qsp_mutex_trylock);
+ qatomic_set(&qemu_bql_mutex_lock_func, qsp_bql_mutex_lock);
+ qatomic_set(&qemu_rec_mutex_lock_func, qsp_rec_mutex_lock);
+ qatomic_set(&qemu_rec_mutex_trylock_func, qsp_rec_mutex_trylock);
+ qatomic_set(&qemu_cond_wait_func, qsp_cond_wait);
+ qatomic_set(&qemu_cond_timedwait_func, qsp_cond_timedwait);
}
void qsp_disable(void)
{
- atomic_set(&qemu_mutex_lock_func, qemu_mutex_lock_impl);
- atomic_set(&qemu_mutex_trylock_func, qemu_mutex_trylock_impl);
- atomic_set(&qemu_bql_mutex_lock_func, qemu_mutex_lock_impl);
- atomic_set(&qemu_rec_mutex_lock_func, qemu_rec_mutex_lock_impl);
- atomic_set(&qemu_rec_mutex_trylock_func, qemu_rec_mutex_trylock_impl);
- atomic_set(&qemu_cond_wait_func, qemu_cond_wait_impl);
- atomic_set(&qemu_cond_timedwait_func, qemu_cond_timedwait_impl);
+ qatomic_set(&qemu_mutex_lock_func, qemu_mutex_lock_impl);
+ qatomic_set(&qemu_mutex_trylock_func, qemu_mutex_trylock_impl);
+ qatomic_set(&qemu_bql_mutex_lock_func, qemu_mutex_lock_impl);
+ qatomic_set(&qemu_rec_mutex_lock_func, qemu_rec_mutex_lock_impl);
+ qatomic_set(&qemu_rec_mutex_trylock_func, qemu_rec_mutex_trylock_impl);
+ qatomic_set(&qemu_cond_wait_func, qemu_cond_wait_impl);
+ qatomic_set(&qemu_cond_timedwait_func, qemu_cond_timedwait_impl);
}
static gint qsp_tree_cmp(gconstpointer ap, gconstpointer bp, gpointer up)
* The entry is in the global hash table; read from it atomically (as in
* "read once").
*/
- agg->ns += atomic_read_u64(&e->ns);
- agg->n_acqs += atomic_read_u64(&e->n_acqs);
+ agg->ns += qatomic_read_u64(&e->ns);
+ agg->n_acqs += qatomic_read_u64(&e->n_acqs);
}
static void qsp_iter_diff(void *p, uint32_t hash, void *htp)
* with the snapshot.
*/
WITH_RCU_READ_LOCK_GUARD() {
- QSPSnapshot *snap = atomic_rcu_read(&qsp_snapshot);
+ QSPSnapshot *snap = qatomic_rcu_read(&qsp_snapshot);
/* Aggregate all results from the global hash table into a local one */
qht_init(&ht, qsp_entry_no_thread_cmp, QSP_INITIAL_SIZE,
qht_iter(&qsp_ht, qsp_aggregate, &new->ht);
/* replace the previous snapshot, if any */
- old = atomic_xchg(&qsp_snapshot, new);
+ old = qatomic_xchg(&qsp_snapshot, new);
if (old) {
call_rcu(old, qsp_snapshot_destroy, rcu);
}
{
unsigned long v;
- v = atomic_read(ctr);
+ v = qatomic_read(ctr);
return v && (v != rcu_gp_ctr);
}
*/
qemu_event_reset(&rcu_gp_event);
- /* Instead of using atomic_mb_set for index->waiting, and
- * atomic_mb_read for index->ctr, memory barriers are placed
+ /* Instead of using qatomic_mb_set for index->waiting, and
+ * qatomic_mb_read for index->ctr, memory barriers are placed
* manually since writes to different threads are independent.
* qemu_event_reset has acquire semantics, so no memory barrier
* is needed here.
*/
QLIST_FOREACH(index, ®istry, node) {
- atomic_set(&index->waiting, true);
+ qatomic_set(&index->waiting, true);
}
/* Here, order the stores to index->waiting before the loads of
/* No need for mb_set here, worst of all we
* get some extra futex wakeups.
*/
- atomic_set(&index->waiting, false);
+ qatomic_set(&index->waiting, false);
}
}
QEMU_LOCK_GUARD(&rcu_registry_lock);
if (!QLIST_EMPTY(®istry)) {
- /* In either case, the atomic_mb_set below blocks stores that free
+ /* In either case, the qatomic_mb_set below blocks stores that free
* old RCU-protected pointers.
*/
if (sizeof(rcu_gp_ctr) < 8) {
*
* Switch parity: 0 -> 1, 1 -> 0.
*/
- atomic_mb_set(&rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR);
+ qatomic_mb_set(&rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR);
wait_for_readers();
- atomic_mb_set(&rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR);
+ qatomic_mb_set(&rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR);
} else {
/* Increment current grace period. */
- atomic_mb_set(&rcu_gp_ctr, rcu_gp_ctr + RCU_GP_CTR);
+ qatomic_mb_set(&rcu_gp_ctr, rcu_gp_ctr + RCU_GP_CTR);
}
wait_for_readers();
struct rcu_head **old_tail;
node->next = NULL;
- old_tail = atomic_xchg(&tail, &node->next);
- atomic_mb_set(old_tail, node);
+ old_tail = qatomic_xchg(&tail, &node->next);
+ qatomic_mb_set(old_tail, node);
}
static struct rcu_head *try_dequeue(void)
* The tail, because it is the first step in the enqueuing.
* It is only the next pointers that might be inconsistent.
*/
- if (head == &dummy && atomic_mb_read(&tail) == &dummy.next) {
+ if (head == &dummy && qatomic_mb_read(&tail) == &dummy.next) {
abort();
}
* wrong and we need to wait until its enqueuer finishes the update.
*/
node = head;
- next = atomic_mb_read(&head->next);
+ next = qatomic_mb_read(&head->next);
if (!next) {
return NULL;
}
for (;;) {
int tries = 0;
- int n = atomic_read(&rcu_call_count);
+ int n = qatomic_read(&rcu_call_count);
/* Heuristically wait for a decent number of callbacks to pile up.
* Fetch rcu_call_count now, we only must process elements that were
g_usleep(10000);
if (n == 0) {
qemu_event_reset(&rcu_call_ready_event);
- n = atomic_read(&rcu_call_count);
+ n = qatomic_read(&rcu_call_count);
if (n == 0) {
#if defined(CONFIG_MALLOC_TRIM)
malloc_trim(4 * 1024 * 1024);
qemu_event_wait(&rcu_call_ready_event);
}
}
- n = atomic_read(&rcu_call_count);
+ n = qatomic_read(&rcu_call_count);
}
- atomic_sub(&rcu_call_count, n);
+ qatomic_sub(&rcu_call_count, n);
synchronize_rcu();
qemu_mutex_lock_iothread();
while (n > 0) {
{
node->func = func;
enqueue(node);
- atomic_inc(&rcu_call_count);
+ qatomic_inc(&rcu_call_count);
qemu_event_set(&rcu_call_ready_event);
}
static inline void stat64_rdlock(Stat64 *s)
{
/* Keep out incoming writers to avoid them starving us. */
- atomic_add(&s->lock, 2);
+ qatomic_add(&s->lock, 2);
/* If there is a concurrent writer, wait for it. */
- while (atomic_read(&s->lock) & 1) {
+ while (qatomic_read(&s->lock) & 1) {
cpu_relax();
}
}
static inline void stat64_rdunlock(Stat64 *s)
{
- atomic_sub(&s->lock, 2);
+ qatomic_sub(&s->lock, 2);
}
static inline bool stat64_wrtrylock(Stat64 *s)
{
- return atomic_cmpxchg(&s->lock, 0, 1) == 0;
+ return qatomic_cmpxchg(&s->lock, 0, 1) == 0;
}
static inline void stat64_wrunlock(Stat64 *s)
{
- atomic_dec(&s->lock);
+ qatomic_dec(&s->lock);
}
uint64_t stat64_get(const Stat64 *s)
/* 64-bit writes always take the lock, so we can read in
* any order.
*/
- high = atomic_read(&s->high);
- low = atomic_read(&s->low);
+ high = qatomic_read(&s->high);
+ low = qatomic_read(&s->low);
stat64_rdunlock((Stat64 *)s);
return ((uint64_t)high << 32) | low;
* order of our update. By updating s->low first, we can check
* whether we have to carry into s->high.
*/
- old = atomic_fetch_add(&s->low, low);
+ old = qatomic_fetch_add(&s->low, low);
high += (old + low) < old;
- atomic_add(&s->high, high);
+ qatomic_add(&s->high, high);
stat64_wrunlock(s);
return true;
}
return false;
}
- high = atomic_read(&s->high);
- low = atomic_read(&s->low);
+ high = qatomic_read(&s->high);
+ low = qatomic_read(&s->low);
orig = ((uint64_t)high << 32) | low;
if (value < orig) {
* effect on stat64_min is that the slow path may be triggered
* unnecessarily.
*/
- atomic_set(&s->low, (uint32_t)value);
+ qatomic_set(&s->low, (uint32_t)value);
smp_wmb();
- atomic_set(&s->high, value >> 32);
+ qatomic_set(&s->high, value >> 32);
}
stat64_wrunlock(s);
return true;
return false;
}
- high = atomic_read(&s->high);
- low = atomic_read(&s->low);
+ high = qatomic_read(&s->high);
+ low = qatomic_read(&s->low);
orig = ((uint64_t)high << 32) | low;
if (value > orig) {
* effect on stat64_max is that the slow path may be triggered
* unnecessarily.
*/
- atomic_set(&s->low, (uint32_t)value);
+ qatomic_set(&s->low, (uint32_t)value);
smp_wmb();
- atomic_set(&s->high, value >> 32);
+ qatomic_set(&s->high, value >> 32);
}
stat64_wrunlock(s);
return true;