[mirror_ovs.git] / lib / ovs-atomic-gcc4+.h

/*
 * Copyright (c) 2013, 2014 Nicira, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/* This header implements atomic operation primitives on GCC 4.x. */
#ifndef IN_OVS_ATOMIC_H
#error "This header should only be included indirectly via ovs-atomic.h."
#endif

#include "ovs-atomic-locked.h"
#define OVS_ATOMIC_GCC4P_IMPL 1

#define ATOMIC(TYPE) TYPE

#define ATOMIC_BOOL_LOCK_FREE 2
#define ATOMIC_CHAR_LOCK_FREE 2
#define ATOMIC_SHORT_LOCK_FREE 2
#define ATOMIC_INT_LOCK_FREE 2
#define ATOMIC_LONG_LOCK_FREE (ULONG_MAX <= UINTPTR_MAX ? 2 : 0)
#define ATOMIC_LLONG_LOCK_FREE (ULLONG_MAX <= UINTPTR_MAX ? 2 : 0)
#define ATOMIC_POINTER_LOCK_FREE 2

typedef enum {
    memory_order_relaxed,
    memory_order_consume,
    memory_order_acquire,
    memory_order_release,
    memory_order_acq_rel,
    memory_order_seq_cst
} memory_order;
\f
#define IS_LOCKLESS_ATOMIC(OBJECT) (sizeof(OBJECT) <= sizeof(void *))
\f
#define ATOMIC_VAR_INIT(VALUE) VALUE
#define atomic_init(OBJECT, VALUE) (*(OBJECT) = (VALUE), (void) 0)

static inline void
atomic_thread_fence(memory_order order)
{
    if (order != memory_order_relaxed) {
        __sync_synchronize();
    }
}

static inline void
atomic_thread_fence_if_seq_cst(memory_order order)
{
    if (order == memory_order_seq_cst) {
        __sync_synchronize();
    }
}

static inline void
atomic_signal_fence(memory_order order)
{
    if (order != memory_order_relaxed) {
        asm volatile("" : : : "memory");
    }
}

#define atomic_is_lock_free(OBJ)                \
    ((void) *(OBJ),                             \
     IS_LOCKLESS_ATOMIC(*(OBJ)) ? 2 : 0)

#define atomic_store(DST, SRC) \
    atomic_store_explicit(DST, SRC, memory_order_seq_cst)
#define atomic_store_explicit(DST, SRC, ORDER)          \
    ({                                                  \
        typeof(DST) dst__ = (DST);                      \
        typeof(SRC) src__ = (SRC);                      \
                                                        \
        if (IS_LOCKLESS_ATOMIC(*dst__)) {               \
            atomic_thread_fence(ORDER);                 \
            *(typeof(*(DST)) volatile *)dst__ = src__;  \
            atomic_thread_fence_if_seq_cst(ORDER);      \
        } else {                                        \
            atomic_store_locked(dst__, src__);          \
        }                                               \
        (void) 0;                                       \
    })
#define atomic_read(SRC, DST) \
    atomic_read_explicit(SRC, DST, memory_order_seq_cst)
#define atomic_read_explicit(SRC, DST, ORDER)           \
    ({                                                  \
        typeof(DST) dst__ = (DST);                      \
        typeof(SRC) src__ = (SRC);                      \
                                                        \
        if (IS_LOCKLESS_ATOMIC(*src__)) {               \
            atomic_thread_fence_if_seq_cst(ORDER);      \
            *dst__ = *(typeof(*(SRC)) volatile *)src__; \
        } else {                                        \
            atomic_read_locked(src__, dst__);           \
        }                                               \
        (void) 0;                                       \
    })

#define atomic_compare_exchange_strong(DST, EXP, SRC)   \
    ({                                                  \
        typeof(DST) dst__ = (DST);                      \
        typeof(EXP) expp__ = (EXP);                     \
        typeof(SRC) src__ = (SRC);                      \
        typeof(SRC) exp__ = *expp__;                    \
        typeof(SRC) ret__;                              \
                                                        \
        ret__ = __sync_val_compare_and_swap(dst__, exp__, src__); \
        if (ret__ != exp__) {                                     \
            *expp__ = ret__;                                      \
        }                                                         \
        ret__ == exp__;                                           \
    })
#define atomic_compare_exchange_strong_explicit(DST, EXP, SRC, ORD1, ORD2) \
    ((void) (ORD1), (void) (ORD2), \
     atomic_compare_exchange_strong(DST, EXP, SRC))
#define atomic_compare_exchange_weak            \
    atomic_compare_exchange_strong
#define atomic_compare_exchange_weak_explicit   \
    atomic_compare_exchange_strong_explicit

#define atomic_op__(RMW, OP, ARG, ORIG)                     \
    ({                                                      \
        typeof(RMW) rmw__ = (RMW);                          \
        typeof(ARG) arg__ = (ARG);                          \
        typeof(ORIG) orig__ = (ORIG);                       \
                                                            \
        if (IS_LOCKLESS_ATOMIC(*rmw__)) {                   \
            *orig__ = __sync_fetch_and_##OP(rmw__, arg__);  \
        } else {                                            \
            atomic_op_locked(rmw__, OP, arg__, orig__);     \
        }                                                   \
        (void) 0;                                           \
    })

#define atomic_add(RMW, ARG, ORIG) atomic_op__(RMW, add, ARG, ORIG)
#define atomic_sub(RMW, ARG, ORIG) atomic_op__(RMW, sub, ARG, ORIG)
#define atomic_or(RMW, ARG, ORIG) atomic_op__(RMW, or,  ARG, ORIG)
#define atomic_xor(RMW, ARG, ORIG) atomic_op__(RMW, xor, ARG, ORIG)
#define atomic_and(RMW, ARG, ORIG) atomic_op__(RMW, and, ARG, ORIG)

#define atomic_add_explicit(RMW, OPERAND, ORIG, ORDER)  \
    ((void) (ORDER), atomic_add(RMW, OPERAND, ORIG))
#define atomic_sub_explicit(RMW, OPERAND, ORIG, ORDER)  \
    ((void) (ORDER), atomic_sub(RMW, OPERAND, ORIG))
#define atomic_or_explicit(RMW, OPERAND, ORIG, ORDER)   \
    ((void) (ORDER), atomic_or(RMW, OPERAND, ORIG))
#define atomic_xor_explicit(RMW, OPERAND, ORIG, ORDER)  \
    ((void) (ORDER), atomic_xor(RMW, OPERAND, ORIG))
#define atomic_and_explicit(RMW, OPERAND, ORIG, ORDER)  \
    ((void) (ORDER), atomic_and(RMW, OPERAND, ORIG))
\f
/* atomic_flag */

typedef struct {
    int b;
} atomic_flag;
#define ATOMIC_FLAG_INIT { false }

static inline bool
atomic_flag_test_and_set_explicit(volatile atomic_flag *object,
                                  memory_order order)
{
    bool old;

    /* __sync_lock_test_and_set() by itself is an acquire barrier.
     * For anything higher additional barriers are needed. */
    if (order > memory_order_acquire) {
        atomic_thread_fence(order);
    }
    old = __sync_lock_test_and_set(&object->b, 1);
    atomic_thread_fence_if_seq_cst(order);

    return old;
}

#define atomic_flag_test_and_set(FLAG)                                  \
    atomic_flag_test_and_set_explicit(FLAG, memory_order_seq_cst)

static inline void
atomic_flag_clear_explicit(volatile atomic_flag *object,
                           memory_order order)
{
    /* __sync_lock_release() by itself is a release barrier.  For
     * anything else additional barrier may be needed. */
    if (order != memory_order_release) {
        atomic_thread_fence(order);
    }
    __sync_lock_release(&object->b);
    atomic_thread_fence_if_seq_cst(order);
}

#define atomic_flag_clear(FLAG)                                 \
    atomic_flag_clear_explicit(FLAG, memory_order_seq_cst)
Commit	Line	Data
31a3fc6e	1	/*
1bd2c9ed	2	* Copyright (c) 2013, 2014 Nicira, Inc.
31a3fc6e BP	3	*
	4	* Licensed under the Apache License, Version 2.0 (the "License");
	5	* you may not use this file except in compliance with the License.
	6	* You may obtain a copy of the License at:
	7	*
	8	* http://www.apache.org/licenses/LICENSE-2.0
	9	*
	10	* Unless required by applicable law or agreed to in writing, software
	11	* distributed under the License is distributed on an "AS IS" BASIS,
	12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	* See the License for the specific language governing permissions and
	14	* limitations under the License.
	15	*/
	16
	17	/* This header implements atomic operation primitives on GCC 4.x. */
	18	#ifndef IN_OVS_ATOMIC_H
	19	#error "This header should only be included indirectly via ovs-atomic.h."
	20	#endif
	21
1bd2c9ed	22	#include "ovs-atomic-locked.h"
31a3fc6e BP	23	#define OVS_ATOMIC_GCC4P_IMPL 1
31a3fc6e BP	24
1bd2c9ed	25	#define ATOMIC(TYPE) TYPE
31a3fc6e BP	26
31a3fc6e BP	27	#define ATOMIC_BOOL_LOCK_FREE 2
31a3fc6e	28	#define ATOMIC_CHAR_LOCK_FREE 2
31a3fc6e	29	#define ATOMIC_SHORT_LOCK_FREE 2
31a3fc6e	30	#define ATOMIC_INT_LOCK_FREE 2
1bd2c9ed BP	31	#define ATOMIC_LONG_LOCK_FREE (ULONG_MAX <= UINTPTR_MAX ? 2 : 0)
1bd2c9ed BP	32	#define ATOMIC_LLONG_LOCK_FREE (ULLONG_MAX <= UINTPTR_MAX ? 2 : 0)
31a3fc6e	33	#define ATOMIC_POINTER_LOCK_FREE 2
31a3fc6e BP	34
	35	typedef enum {
	36	memory_order_relaxed,
	37	memory_order_consume,
	38	memory_order_acquire,
	39	memory_order_release,
	40	memory_order_acq_rel,
	41	memory_order_seq_cst
	42	} memory_order;
	43	\f
1bd2c9ed	44	#define IS_LOCKLESS_ATOMIC(OBJECT) (sizeof(OBJECT) <= sizeof(void *))
31a3fc6e	45	\f
1bd2c9ed BP	46	#define ATOMIC_VAR_INIT(VALUE) VALUE
1bd2c9ed BP	47	#define atomic_init(OBJECT, VALUE) (*(OBJECT) = (VALUE), (void) 0)
31a3fc6e BP	48
	49	static inline void
	50	atomic_thread_fence(memory_order order)
	51	{
	52	if (order != memory_order_relaxed) {
	53	__sync_synchronize();
	54	}
	55	}
	56
	57	static inline void
	58	atomic_thread_fence_if_seq_cst(memory_order order)
	59	{
	60	if (order == memory_order_seq_cst) {
	61	__sync_synchronize();
	62	}
	63	}
	64
	65	static inline void
15ba057e	66	atomic_signal_fence(memory_order order)
31a3fc6e BP	67	{
	68	if (order != memory_order_relaxed) {
	69	asm volatile("" : : : "memory");
	70	}
	71	}
	72
31a3fc6e	73	#define atomic_is_lock_free(OBJ) \
1bd2c9ed	74	((void) *(OBJ), \
3e0b991e	75	IS_LOCKLESS_ATOMIC(*(OBJ)) ? 2 : 0)
31a3fc6e BP	76
	77	#define atomic_store(DST, SRC) \
	78	atomic_store_explicit(DST, SRC, memory_order_seq_cst)
1bd2c9ed BP	79	#define atomic_store_explicit(DST, SRC, ORDER) \
	80	({ \
	81	typeof(DST) dst__ = (DST); \
	82	typeof(SRC) src__ = (SRC); \
1bd2c9ed BP	83	\
1bd2c9ed BP	84	if (IS_LOCKLESS_ATOMIC(*dst__)) { \
15ba057e	85	atomic_thread_fence(ORDER); \
0b83904f	86	(typeof((DST)) volatile *)dst__ = src__; \
15ba057e	87	atomic_thread_fence_if_seq_cst(ORDER); \
1bd2c9ed	88	} else { \
3e0b991e	89	atomic_store_locked(dst__, src__); \
1bd2c9ed BP	90	} \
	91	(void) 0; \
	92	})
31a3fc6e BP	93	#define atomic_read(SRC, DST) \
31a3fc6e BP	94	atomic_read_explicit(SRC, DST, memory_order_seq_cst)
1bd2c9ed BP	95	#define atomic_read_explicit(SRC, DST, ORDER) \
	96	({ \
	97	typeof(DST) dst__ = (DST); \
	98	typeof(SRC) src__ = (SRC); \
1bd2c9ed BP	99	\
1bd2c9ed BP	100	if (IS_LOCKLESS_ATOMIC(*src__)) { \
15ba057e	101	atomic_thread_fence_if_seq_cst(ORDER); \
0b83904f	102	dst__ = (typeof((SRC)) volatile )src__; \
1bd2c9ed	103	} else { \
3e0b991e	104	atomic_read_locked(src__, dst__); \
1bd2c9ed BP	105	} \
	106	(void) 0; \
	107	})
	108
25045d75 JR	109	#define atomic_compare_exchange_strong(DST, EXP, SRC) \
	110	({ \
	111	typeof(DST) dst__ = (DST); \
	112	typeof(EXP) expp__ = (EXP); \
	113	typeof(SRC) src__ = (SRC); \
	114	typeof(SRC) exp__ = *expp__; \
	115	typeof(SRC) ret__; \
	116	\
	117	ret__ = __sync_val_compare_and_swap(dst__, exp__, src__); \
	118	if (ret__ != exp__) { \
	119	*expp__ = ret__; \
	120	} \
	121	ret__ == exp__; \
	122	})
	123	#define atomic_compare_exchange_strong_explicit(DST, EXP, SRC, ORD1, ORD2) \
	124	((void) (ORD1), (void) (ORD2), \
	125	atomic_compare_exchange_strong(DST, EXP, SRC))
	126	#define atomic_compare_exchange_weak \
	127	atomic_compare_exchange_strong
	128	#define atomic_compare_exchange_weak_explicit \
	129	atomic_compare_exchange_strong_explicit
	130
1bd2c9ed BP	131	#define atomic_op__(RMW, OP, ARG, ORIG) \
	132	({ \
	133	typeof(RMW) rmw__ = (RMW); \
	134	typeof(ARG) arg__ = (ARG); \
	135	typeof(ORIG) orig__ = (ORIG); \
	136	\
	137	if (IS_LOCKLESS_ATOMIC(*rmw__)) { \
	138	*orig__ = __sync_fetch_and_##OP(rmw__, arg__); \
	139	} else { \
3e0b991e	140	atomic_op_locked(rmw__, OP, arg__, orig__); \
1bd2c9ed	141	} \
0b83904f	142	(void) 0; \
1bd2c9ed	143	})
31a3fc6e BP	144
	145	#define atomic_add(RMW, ARG, ORIG) atomic_op__(RMW, add, ARG, ORIG)
	146	#define atomic_sub(RMW, ARG, ORIG) atomic_op__(RMW, sub, ARG, ORIG)
0b83904f	147	#define atomic_or(RMW, ARG, ORIG) atomic_op__(RMW, or, ARG, ORIG)
31a3fc6e BP	148	#define atomic_xor(RMW, ARG, ORIG) atomic_op__(RMW, xor, ARG, ORIG)
	149	#define atomic_and(RMW, ARG, ORIG) atomic_op__(RMW, and, ARG, ORIG)
	150
	151	#define atomic_add_explicit(RMW, OPERAND, ORIG, ORDER) \
	152	((void) (ORDER), atomic_add(RMW, OPERAND, ORIG))
	153	#define atomic_sub_explicit(RMW, OPERAND, ORIG, ORDER) \
	154	((void) (ORDER), atomic_sub(RMW, OPERAND, ORIG))
	155	#define atomic_or_explicit(RMW, OPERAND, ORIG, ORDER) \
	156	((void) (ORDER), atomic_or(RMW, OPERAND, ORIG))
	157	#define atomic_xor_explicit(RMW, OPERAND, ORIG, ORDER) \
	158	((void) (ORDER), atomic_xor(RMW, OPERAND, ORIG))
	159	#define atomic_and_explicit(RMW, OPERAND, ORIG, ORDER) \
	160	((void) (ORDER), atomic_and(RMW, OPERAND, ORIG))
	161	\f
	162	/* atomic_flag */
	163
	164	typedef struct {
	165	int b;
	166	} atomic_flag;
	167	#define ATOMIC_FLAG_INIT { false }
	168
31a3fc6e BP	169	static inline bool
31a3fc6e BP	170	atomic_flag_test_and_set_explicit(volatile atomic_flag *object,
91645580	171	memory_order order)
31a3fc6e	172	{
91645580	173	bool old;
31a3fc6e	174
91645580 JR	175	/* __sync_lock_test_and_set() by itself is an acquire barrier.
	176	* For anything higher additional barriers are needed. */
	177	if (order > memory_order_acquire) {
	178	atomic_thread_fence(order);
	179	}
	180	old = __sync_lock_test_and_set(&object->b, 1);
	181	atomic_thread_fence_if_seq_cst(order);
	182
	183	return old;
31a3fc6e BP	184	}
31a3fc6e BP	185
91645580 JR	186	#define atomic_flag_test_and_set(FLAG) \
	187	atomic_flag_test_and_set_explicit(FLAG, memory_order_seq_cst)
	188
31a3fc6e BP	189	static inline void
31a3fc6e BP	190	atomic_flag_clear_explicit(volatile atomic_flag *object,
91645580	191	memory_order order)
31a3fc6e	192	{
91645580 JR	193	/* __sync_lock_release() by itself is a release barrier. For
	194	* anything else additional barrier may be needed. */
	195	if (order != memory_order_release) {
	196	atomic_thread_fence(order);
	197	}
	198	__sync_lock_release(&object->b);
	199	atomic_thread_fence_if_seq_cst(order);
31a3fc6e	200	}
91645580 JR	201
	202	#define atomic_flag_clear(FLAG) \
	203	atomic_flag_clear_explicit(FLAG, memory_order_seq_cst)