]>
git.proxmox.com Git - ceph.git/blob - ceph/src/seastar/dpdk/lib/librte_eal/common/include/arch/x86/rte_atomic.h
1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
5 #ifndef _RTE_ATOMIC_X86_H_
6 #define _RTE_ATOMIC_X86_H_
13 #include <rte_common.h>
14 #include <rte_config.h>
15 #include <emmintrin.h>
16 #include "generic/rte_atomic.h"
18 #if RTE_MAX_LCORE == 1
19 #define MPLOCKED /**< No need to insert MP lock prefix. */
21 #define MPLOCKED "lock ; " /**< Insert MP lock prefix. */
24 #define rte_mb() _mm_mfence()
26 #define rte_wmb() _mm_sfence()
28 #define rte_rmb() _mm_lfence()
30 #define rte_smp_wmb() rte_compiler_barrier()
32 #define rte_smp_rmb() rte_compiler_barrier()
35 * From Intel Software Development Manual; Vol 3;
36 * 8.2.2 Memory Ordering in P6 and More Recent Processor Families:
38 * . Reads are not reordered with other reads.
39 * . Writes are not reordered with older reads.
40 * . Writes to memory are not reordered with other writes,
41 * with the following exceptions:
42 * . streaming stores (writes) executed with the non-temporal move
43 * instructions (MOVNTI, MOVNTQ, MOVNTDQ, MOVNTPS, and MOVNTPD); and
44 * . string operations (see Section 8.2.4.1).
46 * . Reads may be reordered with older writes to different locations but not
47 * with older writes to the same location.
48 * . Reads or writes cannot be reordered with I/O instructions,
49 * locked instructions, or serializing instructions.
50 * . Reads cannot pass earlier LFENCE and MFENCE instructions.
51 * . Writes ... cannot pass earlier LFENCE, SFENCE, and MFENCE instructions.
52 * . LFENCE instructions cannot pass earlier reads.
53 * . SFENCE instructions cannot pass earlier writes ...
54 * . MFENCE instructions cannot pass earlier reads, writes ...
56 * As pointed by Java guys, that makes possible to use lock-prefixed
57 * instructions to get the same effect as mfence and on most modern HW
58 * that gives a better perfomance then using mfence:
59 * https://shipilev.net/blog/2014/on-the-fence-with-dependencies/
60 * Basic idea is to use lock prefixed add with some dummy memory location
61 * as the destination. From their experiments 128B(2 cache lines) below
62 * current stack pointer looks like a good candidate.
63 * So below we use that techinque for rte_smp_mb() implementation.
66 static __rte_always_inline
void
70 asm volatile("lock addl $0, -128(%%esp); " ::: "memory");
72 asm volatile("lock addl $0, -128(%%rsp); " ::: "memory");
76 #define rte_io_mb() rte_mb()
78 #define rte_io_wmb() rte_compiler_barrier()
80 #define rte_io_rmb() rte_compiler_barrier()
82 #define rte_cio_wmb() rte_compiler_barrier()
84 #define rte_cio_rmb() rte_compiler_barrier()
86 /*------------------------- 16 bit atomic operations -------------------------*/
88 #ifndef RTE_FORCE_INTRINSICS
90 rte_atomic16_cmpset(volatile uint16_t *dst
, uint16_t exp
, uint16_t src
)
96 "cmpxchgw %[src], %[dst];"
98 : [res
] "=a" (res
), /* output */
100 : [src
] "r" (src
), /* input */
103 : "memory"); /* no-clobber list */
107 static inline uint16_t
108 rte_atomic16_exchange(volatile uint16_t *dst
, uint16_t val
)
113 : "=r" (val
), "=m" (*dst
)
114 : "0" (val
), "m" (*dst
)
115 : "memory"); /* no-clobber list */
119 static inline int rte_atomic16_test_and_set(rte_atomic16_t
*v
)
121 return rte_atomic16_cmpset((volatile uint16_t *)&v
->cnt
, 0, 1);
125 rte_atomic16_inc(rte_atomic16_t
*v
)
130 : [cnt
] "=m" (v
->cnt
) /* output */
131 : "m" (v
->cnt
) /* input */
136 rte_atomic16_dec(rte_atomic16_t
*v
)
141 : [cnt
] "=m" (v
->cnt
) /* output */
142 : "m" (v
->cnt
) /* input */
146 static inline int rte_atomic16_inc_and_test(rte_atomic16_t
*v
)
154 : [cnt
] "+m" (v
->cnt
), /* output */
160 static inline int rte_atomic16_dec_and_test(rte_atomic16_t
*v
)
164 asm volatile(MPLOCKED
167 : [cnt
] "+m" (v
->cnt
), /* output */
173 /*------------------------- 32 bit atomic operations -------------------------*/
176 rte_atomic32_cmpset(volatile uint32_t *dst
, uint32_t exp
, uint32_t src
)
182 "cmpxchgl %[src], %[dst];"
184 : [res
] "=a" (res
), /* output */
186 : [src
] "r" (src
), /* input */
189 : "memory"); /* no-clobber list */
193 static inline uint32_t
194 rte_atomic32_exchange(volatile uint32_t *dst
, uint32_t val
)
199 : "=r" (val
), "=m" (*dst
)
200 : "0" (val
), "m" (*dst
)
201 : "memory"); /* no-clobber list */
205 static inline int rte_atomic32_test_and_set(rte_atomic32_t
*v
)
207 return rte_atomic32_cmpset((volatile uint32_t *)&v
->cnt
, 0, 1);
211 rte_atomic32_inc(rte_atomic32_t
*v
)
216 : [cnt
] "=m" (v
->cnt
) /* output */
217 : "m" (v
->cnt
) /* input */
222 rte_atomic32_dec(rte_atomic32_t
*v
)
227 : [cnt
] "=m" (v
->cnt
) /* output */
228 : "m" (v
->cnt
) /* input */
232 static inline int rte_atomic32_inc_and_test(rte_atomic32_t
*v
)
240 : [cnt
] "+m" (v
->cnt
), /* output */
246 static inline int rte_atomic32_dec_and_test(rte_atomic32_t
*v
)
250 asm volatile(MPLOCKED
253 : [cnt
] "+m" (v
->cnt
), /* output */
261 #include "rte_atomic_32.h"
263 #include "rte_atomic_64.h"
270 #endif /* _RTE_ATOMIC_X86_H_ */