]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | /* |
2 | * Distributed under the Boost Software License, Version 1.0. | |
3 | * (See accompanying file LICENSE_1_0.txt or copy at | |
4 | * http://www.boost.org/LICENSE_1_0.txt) | |
5 | * | |
6 | * Copyright (c) 2009 Helge Bahmann | |
7 | * Copyright (c) 2012 Tim Blechmann | |
8 | * Copyright (c) 2014 Andrey Semashev | |
9 | */ | |
10 | /*! | |
20effc67 | 11 | * \file atomic/detail/core_arch_ops_msvc_x86.hpp |
7c673cae | 12 | * |
20effc67 | 13 | * This header contains implementation of the \c core_arch_operations template. |
7c673cae FG |
14 | */ |
15 | ||
20effc67 TL |
16 | #ifndef BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_MSVC_X86_HPP_INCLUDED_ |
17 | #define BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_MSVC_X86_HPP_INCLUDED_ | |
7c673cae | 18 | |
b32b8144 | 19 | #include <cstddef> |
20effc67 | 20 | #include <boost/cstdint.hpp> |
7c673cae | 21 | #include <boost/memory_order.hpp> |
7c673cae | 22 | #include <boost/atomic/detail/config.hpp> |
20effc67 | 23 | #include <boost/atomic/detail/intptr.hpp> |
7c673cae | 24 | #include <boost/atomic/detail/interlocked.hpp> |
f67539c2 | 25 | #include <boost/atomic/detail/storage_traits.hpp> |
20effc67 | 26 | #include <boost/atomic/detail/core_arch_operations_fwd.hpp> |
b32b8144 | 27 | #include <boost/atomic/detail/type_traits/make_signed.hpp> |
20effc67 | 28 | #include <boost/atomic/detail/capabilities.hpp> |
7c673cae FG |
29 | #if defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG8B) || defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG16B) |
30 | #include <boost/cstdint.hpp> | |
20effc67 TL |
31 | #include <boost/atomic/detail/cas_based_exchange.hpp> |
32 | #include <boost/atomic/detail/core_ops_cas_based.hpp> | |
7c673cae FG |
33 | #endif |
34 | #include <boost/atomic/detail/ops_msvc_common.hpp> | |
35 | #if !defined(_M_IX86) && !(defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE8) && defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE16)) | |
20effc67 | 36 | #include <boost/atomic/detail/extending_cas_based_arithmetic.hpp> |
7c673cae | 37 | #endif |
20effc67 | 38 | #include <boost/atomic/detail/header.hpp> |
7c673cae FG |
39 | |
40 | #ifdef BOOST_HAS_PRAGMA_ONCE | |
41 | #pragma once | |
42 | #endif | |
43 | ||
7c673cae FG |
44 | namespace boost { |
45 | namespace atomics { | |
46 | namespace detail { | |
47 | ||
48 | /* | |
49 | * Implementation note for asm blocks. | |
50 | * | |
51 | * http://msdn.microsoft.com/en-us/data/k1a8ss06%28v=vs.105%29 | |
52 | * | |
53 | * Some SSE types require eight-byte stack alignment, forcing the compiler to emit dynamic stack-alignment code. | |
54 | * To be able to access both the local variables and the function parameters after the alignment, the compiler | |
55 | * maintains two frame pointers. If the compiler performs frame pointer omission (FPO), it will use EBP and ESP. | |
56 | * If the compiler does not perform FPO, it will use EBX and EBP. To ensure code runs correctly, do not modify EBX | |
57 | * in asm code if the function requires dynamic stack alignment as it could modify the frame pointer. | |
58 | * Either move the eight-byte aligned types out of the function, or avoid using EBX. | |
59 | * | |
60 | * Since we have no way of knowing that the compiler uses FPO, we have to always save and restore ebx | |
20effc67 | 61 | * whenever we have to clobber it. Additionally, we disable warning C4731 in header.hpp so that the compiler |
7c673cae FG |
62 | * doesn't spam about ebx use. |
63 | */ | |
64 | ||
20effc67 | 65 | struct core_arch_operations_msvc_x86_base |
7c673cae | 66 | { |
11fdf7f2 | 67 | static BOOST_CONSTEXPR_OR_CONST bool full_cas_based = false; |
7c673cae FG |
68 | static BOOST_CONSTEXPR_OR_CONST bool is_always_lock_free = true; |
69 | ||
7c673cae FG |
70 | static BOOST_FORCEINLINE void fence_before(memory_order) BOOST_NOEXCEPT |
71 | { | |
72 | BOOST_ATOMIC_DETAIL_COMPILER_BARRIER(); | |
73 | } | |
74 | ||
75 | static BOOST_FORCEINLINE void fence_after(memory_order) BOOST_NOEXCEPT | |
76 | { | |
77 | BOOST_ATOMIC_DETAIL_COMPILER_BARRIER(); | |
78 | } | |
79 | ||
80 | static BOOST_FORCEINLINE void fence_after_load(memory_order) BOOST_NOEXCEPT | |
81 | { | |
82 | BOOST_ATOMIC_DETAIL_COMPILER_BARRIER(); | |
83 | ||
84 | // On x86 and x86_64 there is no need for a hardware barrier, | |
85 | // even if seq_cst memory order is requested, because all | |
86 | // seq_cst writes are implemented with lock-prefixed operations | |
87 | // or xchg which has implied lock prefix. Therefore normal loads | |
88 | // are already ordered with seq_cst stores on these architectures. | |
89 | } | |
90 | }; | |
91 | ||
20effc67 TL |
92 | template< std::size_t Size, bool Signed, bool Interprocess, typename Derived > |
93 | struct core_arch_operations_msvc_x86 : | |
94 | public core_arch_operations_msvc_x86_base | |
7c673cae | 95 | { |
f67539c2 | 96 | typedef typename storage_traits< Size >::type storage_type; |
11fdf7f2 TL |
97 | |
98 | static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = Size; | |
f67539c2 | 99 | static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = storage_traits< Size >::alignment; |
11fdf7f2 | 100 | static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed; |
20effc67 | 101 | static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess; |
7c673cae FG |
102 | |
103 | static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
104 | { | |
105 | if (order != memory_order_seq_cst) | |
106 | { | |
107 | fence_before(order); | |
108 | storage = v; | |
109 | fence_after(order); | |
110 | } | |
111 | else | |
112 | { | |
113 | Derived::exchange(storage, v, order); | |
114 | } | |
115 | } | |
116 | ||
117 | static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT | |
118 | { | |
119 | storage_type v = storage; | |
120 | fence_after_load(order); | |
121 | return v; | |
122 | } | |
123 | ||
124 | static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
125 | { | |
b32b8144 | 126 | typedef typename boost::atomics::detail::make_signed< storage_type >::type signed_storage_type; |
7c673cae FG |
127 | return Derived::fetch_add(storage, static_cast< storage_type >(-static_cast< signed_storage_type >(v)), order); |
128 | } | |
129 | ||
130 | static BOOST_FORCEINLINE bool compare_exchange_weak( | |
131 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT | |
132 | { | |
133 | return Derived::compare_exchange_strong(storage, expected, desired, success_order, failure_order); | |
134 | } | |
135 | ||
136 | static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT | |
137 | { | |
138 | return !!Derived::exchange(storage, (storage_type)1, order); | |
139 | } | |
140 | ||
141 | static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT | |
142 | { | |
143 | store(storage, (storage_type)0, order); | |
144 | } | |
7c673cae FG |
145 | }; |
146 | ||
20effc67 TL |
147 | template< bool Signed, bool Interprocess > |
148 | struct core_arch_operations< 4u, Signed, Interprocess > : | |
149 | public core_arch_operations_msvc_x86< 4u, Signed, Interprocess, core_arch_operations< 4u, Signed, Interprocess > > | |
7c673cae | 150 | { |
20effc67 | 151 | typedef core_arch_operations_msvc_x86< 4u, Signed, Interprocess, core_arch_operations< 4u, Signed, Interprocess > > base_type; |
7c673cae | 152 | typedef typename base_type::storage_type storage_type; |
b32b8144 | 153 | |
7c673cae FG |
154 | static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT |
155 | { | |
156 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD(&storage, v)); | |
157 | } | |
158 | ||
159 | static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
160 | { | |
161 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE(&storage, v)); | |
162 | } | |
163 | ||
164 | static BOOST_FORCEINLINE bool compare_exchange_strong( | |
165 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT | |
166 | { | |
167 | storage_type previous = expected; | |
168 | storage_type old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE(&storage, desired, previous)); | |
169 | expected = old_val; | |
170 | return (previous == old_val); | |
171 | } | |
172 | ||
173 | #if defined(BOOST_ATOMIC_INTERLOCKED_AND) | |
174 | static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
175 | { | |
176 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND(&storage, v)); | |
177 | } | |
178 | #else | |
179 | static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
180 | { | |
181 | storage_type res = storage; | |
182 | while (!compare_exchange_strong(storage, res, res & v, order, memory_order_relaxed)) {} | |
183 | return res; | |
184 | } | |
185 | #endif | |
186 | ||
187 | #if defined(BOOST_ATOMIC_INTERLOCKED_OR) | |
188 | static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
189 | { | |
190 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR(&storage, v)); | |
191 | } | |
192 | #else | |
193 | static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
194 | { | |
195 | storage_type res = storage; | |
196 | while (!compare_exchange_strong(storage, res, res | v, order, memory_order_relaxed)) {} | |
197 | return res; | |
198 | } | |
199 | #endif | |
200 | ||
201 | #if defined(BOOST_ATOMIC_INTERLOCKED_XOR) | |
202 | static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
203 | { | |
204 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR(&storage, v)); | |
205 | } | |
206 | #else | |
207 | static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
208 | { | |
209 | storage_type res = storage; | |
210 | while (!compare_exchange_strong(storage, res, res ^ v, order, memory_order_relaxed)) {} | |
211 | return res; | |
212 | } | |
213 | #endif | |
214 | }; | |
215 | ||
216 | #if defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE8) | |
217 | ||
20effc67 TL |
218 | template< bool Signed, bool Interprocess > |
219 | struct core_arch_operations< 1u, Signed, Interprocess > : | |
220 | public core_arch_operations_msvc_x86< 1u, Signed, Interprocess, core_arch_operations< 1u, Signed, Interprocess > > | |
7c673cae | 221 | { |
20effc67 | 222 | typedef core_arch_operations_msvc_x86< 1u, Signed, Interprocess, core_arch_operations< 1u, Signed, Interprocess > > base_type; |
7c673cae | 223 | typedef typename base_type::storage_type storage_type; |
b32b8144 | 224 | |
7c673cae FG |
225 | static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT |
226 | { | |
227 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD8(&storage, v)); | |
228 | } | |
229 | ||
230 | static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
231 | { | |
232 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE8(&storage, v)); | |
233 | } | |
234 | ||
235 | static BOOST_FORCEINLINE bool compare_exchange_strong( | |
236 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT | |
237 | { | |
238 | storage_type previous = expected; | |
239 | storage_type old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE8(&storage, desired, previous)); | |
240 | expected = old_val; | |
241 | return (previous == old_val); | |
242 | } | |
243 | ||
244 | static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
245 | { | |
246 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND8(&storage, v)); | |
247 | } | |
248 | ||
249 | static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
250 | { | |
251 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR8(&storage, v)); | |
252 | } | |
253 | ||
254 | static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
255 | { | |
256 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR8(&storage, v)); | |
257 | } | |
258 | }; | |
259 | ||
260 | #elif defined(_M_IX86) | |
261 | ||
20effc67 TL |
262 | template< bool Signed, bool Interprocess > |
263 | struct core_arch_operations< 1u, Signed, Interprocess > : | |
264 | public core_arch_operations_msvc_x86< 1u, Signed, Interprocess, core_arch_operations< 1u, Signed, Interprocess > > | |
7c673cae | 265 | { |
20effc67 | 266 | typedef core_arch_operations_msvc_x86< 1u, Signed, Interprocess, core_arch_operations< 1u, Signed, Interprocess > > base_type; |
7c673cae | 267 | typedef typename base_type::storage_type storage_type; |
b32b8144 | 268 | |
7c673cae FG |
269 | static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT |
270 | { | |
271 | base_type::fence_before(order); | |
272 | __asm | |
273 | { | |
274 | mov edx, storage | |
275 | movzx eax, v | |
276 | lock xadd byte ptr [edx], al | |
277 | mov v, al | |
278 | }; | |
279 | base_type::fence_after(order); | |
280 | return v; | |
281 | } | |
282 | ||
283 | static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
284 | { | |
285 | base_type::fence_before(order); | |
286 | __asm | |
287 | { | |
288 | mov edx, storage | |
289 | movzx eax, v | |
290 | xchg byte ptr [edx], al | |
291 | mov v, al | |
292 | }; | |
293 | base_type::fence_after(order); | |
294 | return v; | |
295 | } | |
296 | ||
297 | static BOOST_FORCEINLINE bool compare_exchange_strong( | |
298 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order) BOOST_NOEXCEPT | |
299 | { | |
300 | base_type::fence_before(success_order); | |
301 | bool success; | |
302 | __asm | |
303 | { | |
304 | mov esi, expected | |
305 | mov edi, storage | |
306 | movzx eax, byte ptr [esi] | |
307 | movzx edx, desired | |
308 | lock cmpxchg byte ptr [edi], dl | |
309 | mov byte ptr [esi], al | |
310 | sete success | |
311 | }; | |
312 | // The success and failure fences are equivalent anyway | |
313 | base_type::fence_after(success_order); | |
314 | return success; | |
315 | } | |
316 | ||
317 | static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
318 | { | |
319 | base_type::fence_before(order); | |
7c673cae FG |
320 | __asm |
321 | { | |
7c673cae | 322 | mov edi, storage |
11fdf7f2 TL |
323 | movzx ecx, v |
324 | xor edx, edx | |
7c673cae FG |
325 | movzx eax, byte ptr [edi] |
326 | align 16 | |
327 | again: | |
328 | mov dl, al | |
11fdf7f2 | 329 | and dl, cl |
7c673cae FG |
330 | lock cmpxchg byte ptr [edi], dl |
331 | jne again | |
332 | mov v, al | |
7c673cae FG |
333 | }; |
334 | base_type::fence_after(order); | |
335 | return v; | |
336 | } | |
337 | ||
338 | static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
339 | { | |
340 | base_type::fence_before(order); | |
7c673cae FG |
341 | __asm |
342 | { | |
7c673cae | 343 | mov edi, storage |
11fdf7f2 TL |
344 | movzx ecx, v |
345 | xor edx, edx | |
7c673cae FG |
346 | movzx eax, byte ptr [edi] |
347 | align 16 | |
348 | again: | |
349 | mov dl, al | |
11fdf7f2 | 350 | or dl, cl |
7c673cae FG |
351 | lock cmpxchg byte ptr [edi], dl |
352 | jne again | |
353 | mov v, al | |
7c673cae FG |
354 | }; |
355 | base_type::fence_after(order); | |
356 | return v; | |
357 | } | |
358 | ||
359 | static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
360 | { | |
361 | base_type::fence_before(order); | |
7c673cae FG |
362 | __asm |
363 | { | |
7c673cae | 364 | mov edi, storage |
11fdf7f2 TL |
365 | movzx ecx, v |
366 | xor edx, edx | |
7c673cae FG |
367 | movzx eax, byte ptr [edi] |
368 | align 16 | |
369 | again: | |
370 | mov dl, al | |
11fdf7f2 | 371 | xor dl, cl |
7c673cae FG |
372 | lock cmpxchg byte ptr [edi], dl |
373 | jne again | |
374 | mov v, al | |
7c673cae FG |
375 | }; |
376 | base_type::fence_after(order); | |
377 | return v; | |
378 | } | |
379 | }; | |
380 | ||
381 | #else | |
382 | ||
20effc67 TL |
383 | template< bool Signed, bool Interprocess > |
384 | struct core_arch_operations< 1u, Signed, Interprocess > : | |
385 | public extending_cas_based_arithmetic< core_arch_operations< 4u, Signed, Interprocess >, 1u, Signed > | |
7c673cae FG |
386 | { |
387 | }; | |
388 | ||
389 | #endif | |
390 | ||
391 | #if defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE16) | |
392 | ||
20effc67 TL |
393 | template< bool Signed, bool Interprocess > |
394 | struct core_arch_operations< 2u, Signed, Interprocess > : | |
395 | public core_arch_operations_msvc_x86< 2u, Signed, Interprocess, core_arch_operations< 2u, Signed, Interprocess > > | |
7c673cae | 396 | { |
20effc67 | 397 | typedef core_arch_operations_msvc_x86< 2u, Signed, Interprocess, core_arch_operations< 2u, Signed, Interprocess > > base_type; |
7c673cae | 398 | typedef typename base_type::storage_type storage_type; |
b32b8144 | 399 | |
7c673cae FG |
400 | static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT |
401 | { | |
402 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD16(&storage, v)); | |
403 | } | |
404 | ||
405 | static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
406 | { | |
407 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE16(&storage, v)); | |
408 | } | |
409 | ||
410 | static BOOST_FORCEINLINE bool compare_exchange_strong( | |
411 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT | |
412 | { | |
413 | storage_type previous = expected; | |
414 | storage_type old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE16(&storage, desired, previous)); | |
415 | expected = old_val; | |
416 | return (previous == old_val); | |
417 | } | |
418 | ||
419 | static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
420 | { | |
421 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND16(&storage, v)); | |
422 | } | |
423 | ||
424 | static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
425 | { | |
426 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR16(&storage, v)); | |
427 | } | |
428 | ||
429 | static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
430 | { | |
431 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR16(&storage, v)); | |
432 | } | |
433 | }; | |
434 | ||
435 | #elif defined(_M_IX86) | |
436 | ||
20effc67 TL |
437 | template< bool Signed, bool Interprocess > |
438 | struct core_arch_operations< 2u, Signed, Interprocess > : | |
439 | public core_arch_operations_msvc_x86< 2u, Signed, Interprocess, core_arch_operations< 2u, Signed, Interprocess > > | |
7c673cae | 440 | { |
20effc67 | 441 | typedef core_arch_operations_msvc_x86< 2u, Signed, Interprocess, core_arch_operations< 2u, Signed, Interprocess > > base_type; |
7c673cae | 442 | typedef typename base_type::storage_type storage_type; |
b32b8144 | 443 | |
7c673cae FG |
444 | static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT |
445 | { | |
446 | base_type::fence_before(order); | |
447 | __asm | |
448 | { | |
449 | mov edx, storage | |
450 | movzx eax, v | |
451 | lock xadd word ptr [edx], ax | |
452 | mov v, ax | |
453 | }; | |
454 | base_type::fence_after(order); | |
455 | return v; | |
456 | } | |
457 | ||
458 | static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
459 | { | |
460 | base_type::fence_before(order); | |
461 | __asm | |
462 | { | |
463 | mov edx, storage | |
464 | movzx eax, v | |
465 | xchg word ptr [edx], ax | |
466 | mov v, ax | |
467 | }; | |
468 | base_type::fence_after(order); | |
469 | return v; | |
470 | } | |
471 | ||
472 | static BOOST_FORCEINLINE bool compare_exchange_strong( | |
473 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order) BOOST_NOEXCEPT | |
474 | { | |
475 | base_type::fence_before(success_order); | |
476 | bool success; | |
477 | __asm | |
478 | { | |
479 | mov esi, expected | |
480 | mov edi, storage | |
481 | movzx eax, word ptr [esi] | |
482 | movzx edx, desired | |
483 | lock cmpxchg word ptr [edi], dx | |
484 | mov word ptr [esi], ax | |
485 | sete success | |
486 | }; | |
487 | // The success and failure fences are equivalent anyway | |
488 | base_type::fence_after(success_order); | |
489 | return success; | |
490 | } | |
491 | ||
492 | static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
493 | { | |
494 | base_type::fence_before(order); | |
7c673cae FG |
495 | __asm |
496 | { | |
7c673cae | 497 | mov edi, storage |
11fdf7f2 TL |
498 | movzx ecx, v |
499 | xor edx, edx | |
7c673cae FG |
500 | movzx eax, word ptr [edi] |
501 | align 16 | |
502 | again: | |
503 | mov dx, ax | |
11fdf7f2 | 504 | and dx, cx |
7c673cae FG |
505 | lock cmpxchg word ptr [edi], dx |
506 | jne again | |
507 | mov v, ax | |
7c673cae FG |
508 | }; |
509 | base_type::fence_after(order); | |
510 | return v; | |
511 | } | |
512 | ||
513 | static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
514 | { | |
515 | base_type::fence_before(order); | |
7c673cae FG |
516 | __asm |
517 | { | |
7c673cae | 518 | mov edi, storage |
11fdf7f2 TL |
519 | movzx ecx, v |
520 | xor edx, edx | |
7c673cae FG |
521 | movzx eax, word ptr [edi] |
522 | align 16 | |
523 | again: | |
524 | mov dx, ax | |
11fdf7f2 | 525 | or dx, cx |
7c673cae FG |
526 | lock cmpxchg word ptr [edi], dx |
527 | jne again | |
528 | mov v, ax | |
7c673cae FG |
529 | }; |
530 | base_type::fence_after(order); | |
531 | return v; | |
532 | } | |
533 | ||
534 | static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
535 | { | |
536 | base_type::fence_before(order); | |
7c673cae FG |
537 | __asm |
538 | { | |
7c673cae | 539 | mov edi, storage |
11fdf7f2 TL |
540 | movzx ecx, v |
541 | xor edx, edx | |
7c673cae FG |
542 | movzx eax, word ptr [edi] |
543 | align 16 | |
544 | again: | |
545 | mov dx, ax | |
11fdf7f2 | 546 | xor dx, cx |
7c673cae FG |
547 | lock cmpxchg word ptr [edi], dx |
548 | jne again | |
549 | mov v, ax | |
7c673cae FG |
550 | }; |
551 | base_type::fence_after(order); | |
552 | return v; | |
553 | } | |
554 | }; | |
555 | ||
556 | #else | |
557 | ||
20effc67 TL |
558 | template< bool Signed, bool Interprocess > |
559 | struct core_arch_operations< 2u, Signed, Interprocess > : | |
560 | public extending_cas_based_arithmetic< core_arch_operations< 4u, Signed, Interprocess >, 2u, Signed > | |
7c673cae FG |
561 | { |
562 | }; | |
563 | ||
564 | #endif | |
565 | ||
566 | ||
567 | #if defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG8B) | |
568 | ||
20effc67 | 569 | template< bool Signed, bool Interprocess > |
7c673cae FG |
570 | struct msvc_dcas_x86 |
571 | { | |
f67539c2 | 572 | typedef typename storage_traits< 8u >::type storage_type; |
7c673cae | 573 | |
20effc67 | 574 | static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess; |
11fdf7f2 | 575 | static BOOST_CONSTEXPR_OR_CONST bool full_cas_based = true; |
7c673cae FG |
576 | static BOOST_CONSTEXPR_OR_CONST bool is_always_lock_free = true; |
577 | ||
b32b8144 | 578 | static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 8u; |
f67539c2 | 579 | static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 8u; |
b32b8144 FG |
580 | static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed; |
581 | ||
7c673cae FG |
582 | // Intel 64 and IA-32 Architectures Software Developer's Manual, Volume 3A, 8.1.1. Guaranteed Atomic Operations: |
583 | // | |
584 | // The Pentium processor (and newer processors since) guarantees that the following additional memory operations will always be carried out atomically: | |
585 | // * Reading or writing a quadword aligned on a 64-bit boundary | |
586 | // | |
587 | // Luckily, the memory is almost always 8-byte aligned in our case because atomic<> uses 64 bit native types for storage and dynamic memory allocations | |
588 | // have at least 8 byte alignment. The only unfortunate case is when atomic is placed on the stack and it is not 8-byte aligned (like on 32 bit Windows). | |
589 | ||
590 | static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
591 | { | |
592 | BOOST_ATOMIC_DETAIL_COMPILER_BARRIER(); | |
593 | ||
594 | storage_type volatile* p = &storage; | |
20effc67 | 595 | if (((uintptr_t)p & 0x00000007) == 0) |
7c673cae FG |
596 | { |
597 | #if defined(_M_IX86_FP) && _M_IX86_FP >= 2 | |
598 | #if defined(__AVX__) | |
599 | __asm | |
600 | { | |
601 | mov edx, p | |
602 | vmovq xmm4, v | |
603 | vmovq qword ptr [edx], xmm4 | |
604 | }; | |
605 | #else | |
606 | __asm | |
607 | { | |
608 | mov edx, p | |
609 | movq xmm4, v | |
610 | movq qword ptr [edx], xmm4 | |
611 | }; | |
612 | #endif | |
613 | #else | |
614 | __asm | |
615 | { | |
616 | mov edx, p | |
617 | fild v | |
618 | fistp qword ptr [edx] | |
619 | }; | |
620 | #endif | |
621 | } | |
622 | else | |
623 | { | |
11fdf7f2 | 624 | uint32_t backup; |
7c673cae FG |
625 | __asm |
626 | { | |
627 | mov backup, ebx | |
628 | mov edi, p | |
629 | mov ebx, dword ptr [v] | |
630 | mov ecx, dword ptr [v + 4] | |
631 | mov eax, dword ptr [edi] | |
632 | mov edx, dword ptr [edi + 4] | |
633 | align 16 | |
634 | again: | |
635 | lock cmpxchg8b qword ptr [edi] | |
636 | jne again | |
637 | mov ebx, backup | |
638 | }; | |
639 | } | |
640 | ||
641 | BOOST_ATOMIC_DETAIL_COMPILER_BARRIER(); | |
642 | } | |
643 | ||
644 | static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order) BOOST_NOEXCEPT | |
645 | { | |
646 | BOOST_ATOMIC_DETAIL_COMPILER_BARRIER(); | |
647 | ||
648 | storage_type const volatile* p = &storage; | |
649 | storage_type value; | |
650 | ||
20effc67 | 651 | if (((uintptr_t)p & 0x00000007) == 0) |
7c673cae FG |
652 | { |
653 | #if defined(_M_IX86_FP) && _M_IX86_FP >= 2 | |
654 | #if defined(__AVX__) | |
655 | __asm | |
656 | { | |
657 | mov edx, p | |
658 | vmovq xmm4, qword ptr [edx] | |
659 | vmovq value, xmm4 | |
660 | }; | |
661 | #else | |
662 | __asm | |
663 | { | |
664 | mov edx, p | |
665 | movq xmm4, qword ptr [edx] | |
666 | movq value, xmm4 | |
667 | }; | |
668 | #endif | |
669 | #else | |
670 | __asm | |
671 | { | |
672 | mov edx, p | |
673 | fild qword ptr [edx] | |
674 | fistp value | |
675 | }; | |
676 | #endif | |
677 | } | |
678 | else | |
679 | { | |
680 | // We don't care for comparison result here; the previous value will be stored into value anyway. | |
681 | // Also we don't care for ebx and ecx values, they just have to be equal to eax and edx before cmpxchg8b. | |
682 | __asm | |
683 | { | |
684 | mov edi, p | |
685 | mov eax, ebx | |
686 | mov edx, ecx | |
687 | lock cmpxchg8b qword ptr [edi] | |
688 | mov dword ptr [value], eax | |
689 | mov dword ptr [value + 4], edx | |
690 | }; | |
691 | } | |
692 | ||
693 | BOOST_ATOMIC_DETAIL_COMPILER_BARRIER(); | |
694 | ||
695 | return value; | |
696 | } | |
697 | ||
698 | static BOOST_FORCEINLINE bool compare_exchange_strong( | |
699 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT | |
700 | { | |
701 | // MSVC-11 in 32-bit mode sometimes generates messed up code without compiler barriers, | |
702 | // even though the _InterlockedCompareExchange64 intrinsic already provides one. | |
703 | BOOST_ATOMIC_DETAIL_COMPILER_BARRIER(); | |
704 | ||
705 | storage_type volatile* p = &storage; | |
706 | #if defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64) | |
707 | const storage_type old_val = (storage_type)BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64(p, desired, expected); | |
708 | const bool result = (old_val == expected); | |
709 | expected = old_val; | |
710 | #else | |
711 | bool result; | |
11fdf7f2 | 712 | uint32_t backup; |
7c673cae FG |
713 | __asm |
714 | { | |
715 | mov backup, ebx | |
716 | mov edi, p | |
717 | mov esi, expected | |
718 | mov ebx, dword ptr [desired] | |
719 | mov ecx, dword ptr [desired + 4] | |
720 | mov eax, dword ptr [esi] | |
721 | mov edx, dword ptr [esi + 4] | |
722 | lock cmpxchg8b qword ptr [edi] | |
723 | mov dword ptr [esi], eax | |
724 | mov dword ptr [esi + 4], edx | |
725 | mov ebx, backup | |
726 | sete result | |
727 | }; | |
728 | #endif | |
729 | BOOST_ATOMIC_DETAIL_COMPILER_BARRIER(); | |
730 | ||
731 | return result; | |
732 | } | |
733 | ||
734 | static BOOST_FORCEINLINE bool compare_exchange_weak( | |
735 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT | |
736 | { | |
737 | return compare_exchange_strong(storage, expected, desired, success_order, failure_order); | |
738 | } | |
739 | ||
b32b8144 | 740 | static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT |
7c673cae FG |
741 | { |
742 | BOOST_ATOMIC_DETAIL_COMPILER_BARRIER(); | |
743 | ||
744 | storage_type volatile* p = &storage; | |
11fdf7f2 | 745 | uint32_t backup; |
7c673cae FG |
746 | __asm |
747 | { | |
748 | mov backup, ebx | |
749 | mov edi, p | |
750 | mov ebx, dword ptr [v] | |
751 | mov ecx, dword ptr [v + 4] | |
752 | mov eax, dword ptr [edi] | |
753 | mov edx, dword ptr [edi + 4] | |
754 | align 16 | |
755 | again: | |
756 | lock cmpxchg8b qword ptr [edi] | |
757 | jne again | |
758 | mov ebx, backup | |
759 | mov dword ptr [v], eax | |
760 | mov dword ptr [v + 4], edx | |
761 | }; | |
762 | ||
763 | BOOST_ATOMIC_DETAIL_COMPILER_BARRIER(); | |
764 | ||
765 | return v; | |
766 | } | |
7c673cae FG |
767 | }; |
768 | ||
20effc67 TL |
769 | template< bool Signed, bool Interprocess > |
770 | struct core_arch_operations< 8u, Signed, Interprocess > : | |
771 | public core_operations_cas_based< msvc_dcas_x86< Signed, Interprocess > > | |
7c673cae FG |
772 | { |
773 | }; | |
774 | ||
775 | #elif defined(_M_AMD64) | |
776 | ||
20effc67 TL |
777 | template< bool Signed, bool Interprocess > |
778 | struct core_arch_operations< 8u, Signed, Interprocess > : | |
779 | public core_arch_operations_msvc_x86< 8u, Signed, Interprocess, core_arch_operations< 8u, Signed, Interprocess > > | |
7c673cae | 780 | { |
20effc67 | 781 | typedef core_arch_operations_msvc_x86< 8u, Signed, Interprocess, core_arch_operations< 8u, Signed, Interprocess > > base_type; |
7c673cae | 782 | typedef typename base_type::storage_type storage_type; |
b32b8144 | 783 | |
7c673cae FG |
784 | static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT |
785 | { | |
786 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD64(&storage, v)); | |
787 | } | |
788 | ||
789 | static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
790 | { | |
791 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE64(&storage, v)); | |
792 | } | |
793 | ||
794 | static BOOST_FORCEINLINE bool compare_exchange_strong( | |
795 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT | |
796 | { | |
797 | storage_type previous = expected; | |
798 | storage_type old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64(&storage, desired, previous)); | |
799 | expected = old_val; | |
800 | return (previous == old_val); | |
801 | } | |
802 | ||
803 | static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
804 | { | |
805 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND64(&storage, v)); | |
806 | } | |
807 | ||
808 | static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
809 | { | |
810 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR64(&storage, v)); | |
811 | } | |
812 | ||
813 | static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
814 | { | |
815 | return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR64(&storage, v)); | |
816 | } | |
817 | }; | |
818 | ||
819 | #endif | |
820 | ||
821 | #if defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG16B) | |
822 | ||
20effc67 | 823 | template< bool Signed, bool Interprocess > |
7c673cae FG |
824 | struct msvc_dcas_x86_64 |
825 | { | |
f67539c2 | 826 | typedef typename storage_traits< 16u >::type storage_type; |
7c673cae | 827 | |
20effc67 | 828 | static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess; |
11fdf7f2 | 829 | static BOOST_CONSTEXPR_OR_CONST bool full_cas_based = true; |
7c673cae FG |
830 | static BOOST_CONSTEXPR_OR_CONST bool is_always_lock_free = true; |
831 | ||
b32b8144 | 832 | static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 16u; |
f67539c2 | 833 | static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 16u; |
b32b8144 FG |
834 | static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed; |
835 | ||
7c673cae FG |
836 | static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT |
837 | { | |
838 | storage_type value = const_cast< storage_type& >(storage); | |
839 | while (!BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE128(&storage, v, &value)) {} | |
840 | } | |
841 | ||
842 | static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order) BOOST_NOEXCEPT | |
843 | { | |
844 | storage_type value = storage_type(); | |
845 | BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE128(&storage, value, &value); | |
846 | return value; | |
847 | } | |
848 | ||
849 | static BOOST_FORCEINLINE bool compare_exchange_strong( | |
850 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT | |
851 | { | |
852 | return !!BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE128(&storage, desired, &expected); | |
853 | } | |
854 | ||
855 | static BOOST_FORCEINLINE bool compare_exchange_weak( | |
856 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT | |
857 | { | |
858 | return compare_exchange_strong(storage, expected, desired, success_order, failure_order); | |
859 | } | |
7c673cae FG |
860 | }; |
861 | ||
20effc67 TL |
862 | template< bool Signed, bool Interprocess > |
863 | struct core_arch_operations< 16u, Signed, Interprocess > : | |
864 | public core_operations_cas_based< cas_based_exchange< msvc_dcas_x86_64< Signed, Interprocess > > > | |
7c673cae FG |
865 | { |
866 | }; | |
867 | ||
868 | #endif // defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG16B) | |
869 | ||
7c673cae FG |
870 | } // namespace detail |
871 | } // namespace atomics | |
872 | } // namespace boost | |
873 | ||
20effc67 | 874 | #include <boost/atomic/detail/footer.hpp> |
7c673cae | 875 | |
20effc67 | 876 | #endif // BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_MSVC_X86_HPP_INCLUDED_ |