]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | /* |
2 | * Distributed under the Boost Software License, Version 1.0. | |
3 | * (See accompanying file LICENSE_1_0.txt or copy at | |
4 | * http://www.boost.org/LICENSE_1_0.txt) | |
5 | * | |
6 | * Copyright (c) 2011 Helge Bahmann | |
7 | * Copyright (c) 2013 Tim Blechmann | |
8 | * Copyright (c) 2014 Andrey Semashev | |
9 | */ | |
10 | /*! | |
11 | * \file atomic/detail/ops_gcc_sync.hpp | |
12 | * | |
13 | * This header contains implementation of the \c operations template. | |
14 | */ | |
15 | ||
16 | #ifndef BOOST_ATOMIC_DETAIL_OPS_GCC_SYNC_HPP_INCLUDED_ | |
17 | #define BOOST_ATOMIC_DETAIL_OPS_GCC_SYNC_HPP_INCLUDED_ | |
18 | ||
19 | #include <boost/memory_order.hpp> | |
20 | #include <boost/atomic/detail/config.hpp> | |
21 | #include <boost/atomic/detail/storage_type.hpp> | |
22 | #include <boost/atomic/detail/operations_fwd.hpp> | |
23 | #include <boost/atomic/detail/ops_extending_cas_based.hpp> | |
24 | #include <boost/atomic/capabilities.hpp> | |
25 | ||
26 | #ifdef BOOST_HAS_PRAGMA_ONCE | |
27 | #pragma once | |
28 | #endif | |
29 | ||
30 | namespace boost { | |
31 | namespace atomics { | |
32 | namespace detail { | |
33 | ||
34 | struct gcc_sync_operations_base | |
35 | { | |
36 | static BOOST_CONSTEXPR_OR_CONST bool is_always_lock_free = true; | |
37 | ||
38 | static BOOST_FORCEINLINE void fence_before_store(memory_order order) BOOST_NOEXCEPT | |
39 | { | |
40 | if ((order & memory_order_release) != 0) | |
41 | __sync_synchronize(); | |
42 | } | |
43 | ||
44 | static BOOST_FORCEINLINE void fence_after_store(memory_order order) BOOST_NOEXCEPT | |
45 | { | |
46 | if (order == memory_order_seq_cst) | |
47 | __sync_synchronize(); | |
48 | } | |
49 | ||
50 | static BOOST_FORCEINLINE void fence_after_load(memory_order order) BOOST_NOEXCEPT | |
51 | { | |
52 | if ((order & (memory_order_acquire | memory_order_consume)) != 0) | |
53 | __sync_synchronize(); | |
54 | } | |
55 | }; | |
56 | ||
57 | template< typename T > | |
58 | struct gcc_sync_operations : | |
59 | public gcc_sync_operations_base | |
60 | { | |
61 | typedef T storage_type; | |
62 | ||
63 | static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
64 | { | |
65 | fence_before_store(order); | |
66 | storage = v; | |
67 | fence_after_store(order); | |
68 | } | |
69 | ||
70 | static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT | |
71 | { | |
72 | storage_type v = storage; | |
73 | fence_after_load(order); | |
74 | return v; | |
75 | } | |
76 | ||
77 | static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
78 | { | |
79 | return __sync_fetch_and_add(&storage, v); | |
80 | } | |
81 | ||
82 | static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
83 | { | |
84 | return __sync_fetch_and_sub(&storage, v); | |
85 | } | |
86 | ||
87 | static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
88 | { | |
89 | // GCC docs mention that not all architectures may support full exchange semantics for this intrinsic. However, GCC's implementation of | |
90 | // std::atomic<> uses this intrinsic unconditionally. We do so as well. In case if some architectures actually don't support this, we can always | |
91 | // add a check here and fall back to a CAS loop. | |
92 | if ((order & memory_order_release) != 0) | |
93 | __sync_synchronize(); | |
94 | return __sync_lock_test_and_set(&storage, v); | |
95 | } | |
96 | ||
97 | static BOOST_FORCEINLINE bool compare_exchange_strong( | |
98 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT | |
99 | { | |
100 | storage_type expected2 = expected; | |
101 | storage_type old_val = __sync_val_compare_and_swap(&storage, expected2, desired); | |
102 | ||
103 | if (old_val == expected2) | |
104 | { | |
105 | return true; | |
106 | } | |
107 | else | |
108 | { | |
109 | expected = old_val; | |
110 | return false; | |
111 | } | |
112 | } | |
113 | ||
114 | static BOOST_FORCEINLINE bool compare_exchange_weak( | |
115 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT | |
116 | { | |
117 | return compare_exchange_strong(storage, expected, desired, success_order, failure_order); | |
118 | } | |
119 | ||
120 | static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
121 | { | |
122 | return __sync_fetch_and_and(&storage, v); | |
123 | } | |
124 | ||
125 | static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
126 | { | |
127 | return __sync_fetch_and_or(&storage, v); | |
128 | } | |
129 | ||
130 | static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
131 | { | |
132 | return __sync_fetch_and_xor(&storage, v); | |
133 | } | |
134 | ||
135 | static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT | |
136 | { | |
137 | if ((order & memory_order_release) != 0) | |
138 | __sync_synchronize(); | |
139 | return !!__sync_lock_test_and_set(&storage, 1); | |
140 | } | |
141 | ||
142 | static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT | |
143 | { | |
144 | __sync_lock_release(&storage); | |
145 | if (order == memory_order_seq_cst) | |
146 | __sync_synchronize(); | |
147 | } | |
148 | ||
149 | static BOOST_FORCEINLINE bool is_lock_free(storage_type const volatile&) BOOST_NOEXCEPT | |
150 | { | |
151 | return true; | |
152 | } | |
153 | }; | |
154 | ||
155 | #if BOOST_ATOMIC_INT8_LOCK_FREE > 0 | |
156 | template< bool Signed > | |
157 | struct operations< 1u, Signed > : | |
158 | #if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1) | |
159 | public gcc_sync_operations< typename make_storage_type< 1u, Signed >::type > | |
160 | #elif defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2) | |
161 | public extending_cas_based_operations< gcc_sync_operations< typename make_storage_type< 2u, Signed >::type >, 1u, Signed > | |
162 | #elif defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) | |
163 | public extending_cas_based_operations< gcc_sync_operations< typename make_storage_type< 4u, Signed >::type >, 1u, Signed > | |
164 | #elif defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) | |
165 | public extending_cas_based_operations< gcc_sync_operations< typename make_storage_type< 8u, Signed >::type >, 1u, Signed > | |
166 | #else | |
167 | public extending_cas_based_operations< gcc_sync_operations< typename make_storage_type< 16u, Signed >::type >, 1u, Signed > | |
168 | #endif | |
169 | { | |
170 | #if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1) | |
171 | typedef typename make_storage_type< 1u, Signed >::aligned aligned_storage_type; | |
172 | #elif defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2) | |
173 | typedef typename make_storage_type< 2u, Signed >::aligned aligned_storage_type; | |
174 | #elif defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) | |
175 | typedef typename make_storage_type< 4u, Signed >::aligned aligned_storage_type; | |
176 | #elif defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) | |
177 | typedef typename make_storage_type< 8u, Signed >::aligned aligned_storage_type; | |
178 | #else | |
179 | typedef typename make_storage_type< 16u, Signed >::aligned aligned_storage_type; | |
180 | #endif | |
181 | }; | |
182 | #endif | |
183 | ||
184 | #if BOOST_ATOMIC_INT16_LOCK_FREE > 0 | |
185 | template< bool Signed > | |
186 | struct operations< 2u, Signed > : | |
187 | #if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2) | |
188 | public gcc_sync_operations< typename make_storage_type< 2u, Signed >::type > | |
189 | #elif defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) | |
190 | public extending_cas_based_operations< gcc_sync_operations< typename make_storage_type< 4u, Signed >::type >, 2u, Signed > | |
191 | #elif defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) | |
192 | public extending_cas_based_operations< gcc_sync_operations< typename make_storage_type< 8u, Signed >::type >, 2u, Signed > | |
193 | #else | |
194 | public extending_cas_based_operations< gcc_sync_operations< typename make_storage_type< 16u, Signed >::type >, 2u, Signed > | |
195 | #endif | |
196 | { | |
197 | #if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2) | |
198 | typedef typename make_storage_type< 2u, Signed >::aligned aligned_storage_type; | |
199 | #elif defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) | |
200 | typedef typename make_storage_type< 4u, Signed >::aligned aligned_storage_type; | |
201 | #elif defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) | |
202 | typedef typename make_storage_type< 8u, Signed >::aligned aligned_storage_type; | |
203 | #else | |
204 | typedef typename make_storage_type< 16u, Signed >::aligned aligned_storage_type; | |
205 | #endif | |
206 | }; | |
207 | #endif | |
208 | ||
209 | #if BOOST_ATOMIC_INT32_LOCK_FREE > 0 | |
210 | template< bool Signed > | |
211 | struct operations< 4u, Signed > : | |
212 | #if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) | |
213 | public gcc_sync_operations< typename make_storage_type< 4u, Signed >::type > | |
214 | #elif defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) | |
215 | public extending_cas_based_operations< gcc_sync_operations< typename make_storage_type< 8u, Signed >::type >, 4u, Signed > | |
216 | #else | |
217 | public extending_cas_based_operations< gcc_sync_operations< typename make_storage_type< 16u, Signed >::type >, 4u, Signed > | |
218 | #endif | |
219 | { | |
220 | #if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) | |
221 | typedef typename make_storage_type< 4u, Signed >::aligned aligned_storage_type; | |
222 | #elif defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) | |
223 | typedef typename make_storage_type< 8u, Signed >::aligned aligned_storage_type; | |
224 | #else | |
225 | typedef typename make_storage_type< 16u, Signed >::aligned aligned_storage_type; | |
226 | #endif | |
227 | }; | |
228 | #endif | |
229 | ||
230 | #if BOOST_ATOMIC_INT64_LOCK_FREE > 0 | |
231 | template< bool Signed > | |
232 | struct operations< 8u, Signed > : | |
233 | #if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) | |
234 | public gcc_sync_operations< typename make_storage_type< 8u, Signed >::type > | |
235 | #else | |
236 | public extending_cas_based_operations< gcc_sync_operations< typename make_storage_type< 16u, Signed >::type >, 8u, Signed > | |
237 | #endif | |
238 | { | |
239 | #if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) | |
240 | typedef typename make_storage_type< 8u, Signed >::aligned aligned_storage_type; | |
241 | #else | |
242 | typedef typename make_storage_type< 16u, Signed >::aligned aligned_storage_type; | |
243 | #endif | |
244 | }; | |
245 | #endif | |
246 | ||
247 | #if BOOST_ATOMIC_INT128_LOCK_FREE > 0 | |
248 | template< bool Signed > | |
249 | struct operations< 16u, Signed > : | |
250 | public gcc_sync_operations< typename make_storage_type< 16u, Signed >::type > | |
251 | { | |
252 | typedef typename make_storage_type< 16u, Signed >::aligned aligned_storage_type; | |
253 | }; | |
254 | #endif | |
255 | ||
256 | BOOST_FORCEINLINE void thread_fence(memory_order order) BOOST_NOEXCEPT | |
257 | { | |
258 | if (order != memory_order_relaxed) | |
259 | __sync_synchronize(); | |
260 | } | |
261 | ||
262 | BOOST_FORCEINLINE void signal_fence(memory_order order) BOOST_NOEXCEPT | |
263 | { | |
264 | if (order != memory_order_relaxed) | |
265 | __asm__ __volatile__ ("" ::: "memory"); | |
266 | } | |
267 | ||
268 | } // namespace detail | |
269 | } // namespace atomics | |
270 | } // namespace boost | |
271 | ||
272 | #endif // BOOST_ATOMIC_DETAIL_OPS_GCC_SYNC_HPP_INCLUDED_ |