ceph/src/boost/boost/atomic/detail/ops_msvc_x86.hpp

   1 /*
   2  * Distributed under the Boost Software License, Version 1.0.
   3  * (See accompanying file LICENSE_1_0.txt or copy at
   4  * http://www.boost.org/LICENSE_1_0.txt)
   5  *
   6  * Copyright (c) 2009 Helge Bahmann
   7  * Copyright (c) 2012 Tim Blechmann
   8  * Copyright (c) 2014 Andrey Semashev
   9  */
  10 /*!
  11  * \file   atomic/detail/ops_msvc_x86.hpp
  12  *
  13  * This header contains implementation of the \c operations template.
  14  */
  15
  16 #ifndef BOOST_ATOMIC_DETAIL_OPS_MSVC_X86_HPP_INCLUDED_
  17 #define BOOST_ATOMIC_DETAIL_OPS_MSVC_X86_HPP_INCLUDED_
  18
  19 #include <cstddef>
  20 #include <boost/memory_order.hpp>
  21 #include <boost/atomic/detail/config.hpp>
  22 #include <boost/atomic/detail/interlocked.hpp>
  23 #include <boost/atomic/detail/storage_type.hpp>
  24 #include <boost/atomic/detail/operations_fwd.hpp>
  25 #include <boost/atomic/detail/type_traits/make_signed.hpp>
  26 #include <boost/atomic/capabilities.hpp>
  27 #if defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG8B) || defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG16B)
  28 #include <boost/cstdint.hpp>
  29 #include <boost/atomic/detail/ops_cas_based.hpp>
  30 #endif
  31 #include <boost/atomic/detail/ops_msvc_common.hpp>
  32 #if !defined(_M_IX86) && !(defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE8) && defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE16))
  33 #include <boost/atomic/detail/ops_extending_cas_based.hpp>
  34 #endif
  35
  36 #ifdef BOOST_HAS_PRAGMA_ONCE
  37 #pragma once
  38 #endif
  39
  40 #if defined(BOOST_MSVC)
  41 #pragma warning(push)
  42 // frame pointer register 'ebx' modified by inline assembly code. See the note below.
  43 #pragma warning(disable: 4731)
  44 #endif
  45
  46 #if defined(BOOST_ATOMIC_DETAIL_X86_HAS_MFENCE)
  47 extern "C" void _mm_mfence(void);
  48 #if defined(BOOST_MSVC)
  49 #pragma intrinsic(_mm_mfence)
  50 #endif
  51 #endif
  52
  53 namespace boost {
  54 namespace atomics {
  55 namespace detail {
  56
  57 /*
  58  * Implementation note for asm blocks.
  59  *
  60  * http://msdn.microsoft.com/en-us/data/k1a8ss06%28v=vs.105%29
  61  *
  62  * Some SSE types require eight-byte stack alignment, forcing the compiler to emit dynamic stack-alignment code.
  63  * To be able to access both the local variables and the function parameters after the alignment, the compiler
  64  * maintains two frame pointers. If the compiler performs frame pointer omission (FPO), it will use EBP and ESP.
  65  * If the compiler does not perform FPO, it will use EBX and EBP. To ensure code runs correctly, do not modify EBX
  66  * in asm code if the function requires dynamic stack alignment as it could modify the frame pointer.
  67  * Either move the eight-byte aligned types out of the function, or avoid using EBX.
  68  *
  69  * Since we have no way of knowing that the compiler uses FPO, we have to always save and restore ebx
  70  * whenever we have to clobber it. Additionally, we disable warning C4731 above so that the compiler
  71  * doesn't spam about ebx use.
  72  */
  73
  74 struct msvc_x86_operations_base
  75 {
  76     static BOOST_CONSTEXPR_OR_CONST bool is_always_lock_free = true;
  77
  78     static BOOST_FORCEINLINE void hardware_full_fence() BOOST_NOEXCEPT
  79     {
  80 #if defined(BOOST_ATOMIC_DETAIL_X86_HAS_MFENCE)
  81         _mm_mfence();
  82 #else
  83         long tmp;
  84         BOOST_ATOMIC_INTERLOCKED_EXCHANGE(&tmp, 0);
  85 #endif
  86     }
  87
  88     static BOOST_FORCEINLINE void fence_before(memory_order) BOOST_NOEXCEPT
  89     {
  90         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
  91     }
  92
  93     static BOOST_FORCEINLINE void fence_after(memory_order) BOOST_NOEXCEPT
  94     {
  95         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
  96     }
  97
  98     static BOOST_FORCEINLINE void fence_after_load(memory_order) BOOST_NOEXCEPT
  99     {
 100         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
 101
 102         // On x86 and x86_64 there is no need for a hardware barrier,
 103         // even if seq_cst memory order is requested, because all
 104         // seq_cst writes are implemented with lock-prefixed operations
 105         // or xchg which has implied lock prefix. Therefore normal loads
 106         // are already ordered with seq_cst stores on these architectures.
 107     }
 108 };
 109
 110 template< typename T, typename Derived >
 111 struct msvc_x86_operations :
 112     public msvc_x86_operations_base
 113 {
 114     typedef T storage_type;
 115
 116     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 117     {
 118         if (order != memory_order_seq_cst)
 119         {
 120             fence_before(order);
 121             storage = v;
 122             fence_after(order);
 123         }
 124         else
 125         {
 126             Derived::exchange(storage, v, order);
 127         }
 128     }
 129
 130     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
 131     {
 132         storage_type v = storage;
 133         fence_after_load(order);
 134         return v;
 135     }
 136
 137     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 138     {
 139         typedef typename boost::atomics::detail::make_signed< storage_type >::type signed_storage_type;
 140         return Derived::fetch_add(storage, static_cast< storage_type >(-static_cast< signed_storage_type >(v)), order);
 141     }
 142
 143     static BOOST_FORCEINLINE bool compare_exchange_weak(
 144         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
 145     {
 146         return Derived::compare_exchange_strong(storage, expected, desired, success_order, failure_order);
 147     }
 148
 149     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 150     {
 151         return !!Derived::exchange(storage, (storage_type)1, order);
 152     }
 153
 154     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 155     {
 156         store(storage, (storage_type)0, order);
 157     }
 158 };
 159
 160 template< bool Signed >
 161 struct operations< 4u, Signed > :
 162     public msvc_x86_operations< typename make_storage_type< 4u, Signed >::type, operations< 4u, Signed > >
 163 {
 164     typedef msvc_x86_operations< typename make_storage_type< 4u, Signed >::type, operations< 4u, Signed > > base_type;
 165     typedef typename base_type::storage_type storage_type;
 166     typedef typename make_storage_type< 4u, Signed >::aligned aligned_storage_type;
 167
 168     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 4u;
 169     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
 170
 171     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 172     {
 173         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD(&storage, v));
 174     }
 175
 176     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 177     {
 178         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE(&storage, v));
 179     }
 180
 181     static BOOST_FORCEINLINE bool compare_exchange_strong(
 182         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
 183     {
 184         storage_type previous = expected;
 185         storage_type old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE(&storage, desired, previous));
 186         expected = old_val;
 187         return (previous == old_val);
 188     }
 189
 190 #if defined(BOOST_ATOMIC_INTERLOCKED_AND)
 191     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 192     {
 193         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND(&storage, v));
 194     }
 195 #else
 196     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 197     {
 198         storage_type res = storage;
 199         while (!compare_exchange_strong(storage, res, res & v, order, memory_order_relaxed)) {}
 200         return res;
 201     }
 202 #endif
 203
 204 #if defined(BOOST_ATOMIC_INTERLOCKED_OR)
 205     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 206     {
 207         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR(&storage, v));
 208     }
 209 #else
 210     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 211     {
 212         storage_type res = storage;
 213         while (!compare_exchange_strong(storage, res, res | v, order, memory_order_relaxed)) {}
 214         return res;
 215     }
 216 #endif
 217
 218 #if defined(BOOST_ATOMIC_INTERLOCKED_XOR)
 219     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 220     {
 221         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR(&storage, v));
 222     }
 223 #else
 224     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 225     {
 226         storage_type res = storage;
 227         while (!compare_exchange_strong(storage, res, res ^ v, order, memory_order_relaxed)) {}
 228         return res;
 229     }
 230 #endif
 231 };
 232
 233 #if defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE8)
 234
 235 template< bool Signed >
 236 struct operations< 1u, Signed > :
 237     public msvc_x86_operations< typename make_storage_type< 1u, Signed >::type, operations< 1u, Signed > >
 238 {
 239     typedef msvc_x86_operations< typename make_storage_type< 1u, Signed >::type, operations< 1u, Signed > > base_type;
 240     typedef typename base_type::storage_type storage_type;
 241     typedef typename make_storage_type< 1u, Signed >::aligned aligned_storage_type;
 242
 243     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 1u;
 244     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
 245
 246     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 247     {
 248         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD8(&storage, v));
 249     }
 250
 251     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 252     {
 253         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE8(&storage, v));
 254     }
 255
 256     static BOOST_FORCEINLINE bool compare_exchange_strong(
 257         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
 258     {
 259         storage_type previous = expected;
 260         storage_type old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE8(&storage, desired, previous));
 261         expected = old_val;
 262         return (previous == old_val);
 263     }
 264
 265     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 266     {
 267         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND8(&storage, v));
 268     }
 269
 270     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 271     {
 272         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR8(&storage, v));
 273     }
 274
 275     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 276     {
 277         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR8(&storage, v));
 278     }
 279 };
 280
 281 #elif defined(_M_IX86)
 282
 283 template< bool Signed >
 284 struct operations< 1u, Signed > :
 285     public msvc_x86_operations< typename make_storage_type< 1u, Signed >::type, operations< 1u, Signed > >
 286 {
 287     typedef msvc_x86_operations< typename make_storage_type< 1u, Signed >::type, operations< 1u, Signed > > base_type;
 288     typedef typename base_type::storage_type storage_type;
 289     typedef typename make_storage_type< 1u, Signed >::aligned aligned_storage_type;
 290
 291     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 1u;
 292     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
 293
 294     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 295     {
 296         base_type::fence_before(order);
 297         __asm
 298         {
 299             mov edx, storage
 300             movzx eax, v
 301             lock xadd byte ptr [edx], al
 302             mov v, al
 303         };
 304         base_type::fence_after(order);
 305         return v;
 306     }
 307
 308     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 309     {
 310         base_type::fence_before(order);
 311         __asm
 312         {
 313             mov edx, storage
 314             movzx eax, v
 315             xchg byte ptr [edx], al
 316             mov v, al
 317         };
 318         base_type::fence_after(order);
 319         return v;
 320     }
 321
 322     static BOOST_FORCEINLINE bool compare_exchange_strong(
 323         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order) BOOST_NOEXCEPT
 324     {
 325         base_type::fence_before(success_order);
 326         bool success;
 327         __asm
 328         {
 329             mov esi, expected
 330             mov edi, storage
 331             movzx eax, byte ptr [esi]
 332             movzx edx, desired
 333             lock cmpxchg byte ptr [edi], dl
 334             mov byte ptr [esi], al
 335             sete success
 336         };
 337         // The success and failure fences are equivalent anyway
 338         base_type::fence_after(success_order);
 339         return success;
 340     }
 341
 342     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 343     {
 344         base_type::fence_before(order);
 345         int backup;
 346         __asm
 347         {
 348             mov backup, ebx
 349             xor edx, edx
 350             mov edi, storage
 351             movzx ebx, v
 352             movzx eax, byte ptr [edi]
 353             align 16
 354         again:
 355             mov dl, al
 356             and dl, bl
 357             lock cmpxchg byte ptr [edi], dl
 358             jne again
 359             mov v, al
 360             mov ebx, backup
 361         };
 362         base_type::fence_after(order);
 363         return v;
 364     }
 365
 366     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 367     {
 368         base_type::fence_before(order);
 369         int backup;
 370         __asm
 371         {
 372             mov backup, ebx
 373             xor edx, edx
 374             mov edi, storage
 375             movzx ebx, v
 376             movzx eax, byte ptr [edi]
 377             align 16
 378         again:
 379             mov dl, al
 380             or dl, bl
 381             lock cmpxchg byte ptr [edi], dl
 382             jne again
 383             mov v, al
 384             mov ebx, backup
 385         };
 386         base_type::fence_after(order);
 387         return v;
 388     }
 389
 390     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 391     {
 392         base_type::fence_before(order);
 393         int backup;
 394         __asm
 395         {
 396             mov backup, ebx
 397             xor edx, edx
 398             mov edi, storage
 399             movzx ebx, v
 400             movzx eax, byte ptr [edi]
 401             align 16
 402         again:
 403             mov dl, al
 404             xor dl, bl
 405             lock cmpxchg byte ptr [edi], dl
 406             jne again
 407             mov v, al
 408             mov ebx, backup
 409         };
 410         base_type::fence_after(order);
 411         return v;
 412     }
 413 };
 414
 415 #else
 416
 417 template< bool Signed >
 418 struct operations< 1u, Signed > :
 419     public extending_cas_based_operations< operations< 4u, Signed >, 1u, Signed >
 420 {
 421 };
 422
 423 #endif
 424
 425 #if defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE16)
 426
 427 template< bool Signed >
 428 struct operations< 2u, Signed > :
 429     public msvc_x86_operations< typename make_storage_type< 2u, Signed >::type, operations< 2u, Signed > >
 430 {
 431     typedef msvc_x86_operations< typename make_storage_type< 2u, Signed >::type, operations< 2u, Signed > > base_type;
 432     typedef typename base_type::storage_type storage_type;
 433     typedef typename make_storage_type< 2u, Signed >::aligned aligned_storage_type;
 434
 435     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 2u;
 436     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
 437
 438     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 439     {
 440         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD16(&storage, v));
 441     }
 442
 443     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 444     {
 445         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE16(&storage, v));
 446     }
 447
 448     static BOOST_FORCEINLINE bool compare_exchange_strong(
 449         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
 450     {
 451         storage_type previous = expected;
 452         storage_type old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE16(&storage, desired, previous));
 453         expected = old_val;
 454         return (previous == old_val);
 455     }
 456
 457     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 458     {
 459         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND16(&storage, v));
 460     }
 461
 462     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 463     {
 464         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR16(&storage, v));
 465     }
 466
 467     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 468     {
 469         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR16(&storage, v));
 470     }
 471 };
 472
 473 #elif defined(_M_IX86)
 474
 475 template< bool Signed >
 476 struct operations< 2u, Signed > :
 477     public msvc_x86_operations< typename make_storage_type< 2u, Signed >::type, operations< 2u, Signed > >
 478 {
 479     typedef msvc_x86_operations< typename make_storage_type< 2u, Signed >::type, operations< 2u, Signed > > base_type;
 480     typedef typename base_type::storage_type storage_type;
 481     typedef typename make_storage_type< 2u, Signed >::aligned aligned_storage_type;
 482
 483     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 2u;
 484     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
 485
 486     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 487     {
 488         base_type::fence_before(order);
 489         __asm
 490         {
 491             mov edx, storage
 492             movzx eax, v
 493             lock xadd word ptr [edx], ax
 494             mov v, ax
 495         };
 496         base_type::fence_after(order);
 497         return v;
 498     }
 499
 500     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 501     {
 502         base_type::fence_before(order);
 503         __asm
 504         {
 505             mov edx, storage
 506             movzx eax, v
 507             xchg word ptr [edx], ax
 508             mov v, ax
 509         };
 510         base_type::fence_after(order);
 511         return v;
 512     }
 513
 514     static BOOST_FORCEINLINE bool compare_exchange_strong(
 515         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order) BOOST_NOEXCEPT
 516     {
 517         base_type::fence_before(success_order);
 518         bool success;
 519         __asm
 520         {
 521             mov esi, expected
 522             mov edi, storage
 523             movzx eax, word ptr [esi]
 524             movzx edx, desired
 525             lock cmpxchg word ptr [edi], dx
 526             mov word ptr [esi], ax
 527             sete success
 528         };
 529         // The success and failure fences are equivalent anyway
 530         base_type::fence_after(success_order);
 531         return success;
 532     }
 533
 534     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 535     {
 536         base_type::fence_before(order);
 537         int backup;
 538         __asm
 539         {
 540             mov backup, ebx
 541             xor edx, edx
 542             mov edi, storage
 543             movzx ebx, v
 544             movzx eax, word ptr [edi]
 545             align 16
 546         again:
 547             mov dx, ax
 548             and dx, bx
 549             lock cmpxchg word ptr [edi], dx
 550             jne again
 551             mov v, ax
 552             mov ebx, backup
 553         };
 554         base_type::fence_after(order);
 555         return v;
 556     }
 557
 558     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 559     {
 560         base_type::fence_before(order);
 561         int backup;
 562         __asm
 563         {
 564             mov backup, ebx
 565             xor edx, edx
 566             mov edi, storage
 567             movzx ebx, v
 568             movzx eax, word ptr [edi]
 569             align 16
 570         again:
 571             mov dx, ax
 572             or dx, bx
 573             lock cmpxchg word ptr [edi], dx
 574             jne again
 575             mov v, ax
 576             mov ebx, backup
 577         };
 578         base_type::fence_after(order);
 579         return v;
 580     }
 581
 582     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 583     {
 584         base_type::fence_before(order);
 585         int backup;
 586         __asm
 587         {
 588             mov backup, ebx
 589             xor edx, edx
 590             mov edi, storage
 591             movzx ebx, v
 592             movzx eax, word ptr [edi]
 593             align 16
 594         again:
 595             mov dx, ax
 596             xor dx, bx
 597             lock cmpxchg word ptr [edi], dx
 598             jne again
 599             mov v, ax
 600             mov ebx, backup
 601         };
 602         base_type::fence_after(order);
 603         return v;
 604     }
 605 };
 606
 607 #else
 608
 609 template< bool Signed >
 610 struct operations< 2u, Signed > :
 611     public extending_cas_based_operations< operations< 4u, Signed >, 2u, Signed >
 612 {
 613 };
 614
 615 #endif
 616
 617
 618 #if defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG8B)
 619
 620 template< bool Signed >
 621 struct msvc_dcas_x86
 622 {
 623     typedef typename make_storage_type< 8u, Signed >::type storage_type;
 624     typedef typename make_storage_type< 8u, Signed >::aligned aligned_storage_type;
 625
 626     static BOOST_CONSTEXPR_OR_CONST bool is_always_lock_free = true;
 627
 628     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 8u;
 629     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
 630
 631     // Intel 64 and IA-32 Architectures Software Developer's Manual, Volume 3A, 8.1.1. Guaranteed Atomic Operations:
 632     //
 633     // The Pentium processor (and newer processors since) guarantees that the following additional memory operations will always be carried out atomically:
 634     // * Reading or writing a quadword aligned on a 64-bit boundary
 635     //
 636     // Luckily, the memory is almost always 8-byte aligned in our case because atomic<> uses 64 bit native types for storage and dynamic memory allocations
 637     // have at least 8 byte alignment. The only unfortunate case is when atomic is placed on the stack and it is not 8-byte aligned (like on 32 bit Windows).
 638
 639     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 640     {
 641         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
 642
 643         storage_type volatile* p = &storage;
 644         if (((uint32_t)p & 0x00000007) == 0)
 645         {
 646 #if defined(_M_IX86_FP) && _M_IX86_FP >= 2
 647 #if defined(__AVX__)
 648             __asm
 649             {
 650                 mov edx, p
 651                 vmovq xmm4, v
 652                 vmovq qword ptr [edx], xmm4
 653             };
 654 #else
 655             __asm
 656             {
 657                 mov edx, p
 658                 movq xmm4, v
 659                 movq qword ptr [edx], xmm4
 660             };
 661 #endif
 662 #else
 663             __asm
 664             {
 665                 mov edx, p
 666                 fild v
 667                 fistp qword ptr [edx]
 668             };
 669 #endif
 670         }
 671         else
 672         {
 673             int backup;
 674             __asm
 675             {
 676                 mov backup, ebx
 677                 mov edi, p
 678                 mov ebx, dword ptr [v]
 679                 mov ecx, dword ptr [v + 4]
 680                 mov eax, dword ptr [edi]
 681                 mov edx, dword ptr [edi + 4]
 682                 align 16
 683             again:
 684                 lock cmpxchg8b qword ptr [edi]
 685                 jne again
 686                 mov ebx, backup
 687             };
 688         }
 689
 690         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
 691     }
 692
 693     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order) BOOST_NOEXCEPT
 694     {
 695         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
 696
 697         storage_type const volatile* p = &storage;
 698         storage_type value;
 699
 700         if (((uint32_t)p & 0x00000007) == 0)
 701         {
 702 #if defined(_M_IX86_FP) && _M_IX86_FP >= 2
 703 #if defined(__AVX__)
 704             __asm
 705             {
 706                 mov edx, p
 707                 vmovq xmm4, qword ptr [edx]
 708                 vmovq value, xmm4
 709             };
 710 #else
 711             __asm
 712             {
 713                 mov edx, p
 714                 movq xmm4, qword ptr [edx]
 715                 movq value, xmm4
 716             };
 717 #endif
 718 #else
 719             __asm
 720             {
 721                 mov edx, p
 722                 fild qword ptr [edx]
 723                 fistp value
 724             };
 725 #endif
 726         }
 727         else
 728         {
 729             // We don't care for comparison result here; the previous value will be stored into value anyway.
 730             // Also we don't care for ebx and ecx values, they just have to be equal to eax and edx before cmpxchg8b.
 731             __asm
 732             {
 733                 mov edi, p
 734                 mov eax, ebx
 735                 mov edx, ecx
 736                 lock cmpxchg8b qword ptr [edi]
 737                 mov dword ptr [value], eax
 738                 mov dword ptr [value + 4], edx
 739             };
 740         }
 741
 742         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
 743
 744         return value;
 745     }
 746
 747     static BOOST_FORCEINLINE bool compare_exchange_strong(
 748         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
 749     {
 750         // MSVC-11 in 32-bit mode sometimes generates messed up code without compiler barriers,
 751         // even though the _InterlockedCompareExchange64 intrinsic already provides one.
 752         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
 753
 754         storage_type volatile* p = &storage;
 755 #if defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64)
 756         const storage_type old_val = (storage_type)BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64(p, desired, expected);
 757         const bool result = (old_val == expected);
 758         expected = old_val;
 759 #else
 760         bool result;
 761         int backup;
 762         __asm
 763         {
 764             mov backup, ebx
 765             mov edi, p
 766             mov esi, expected
 767             mov ebx, dword ptr [desired]
 768             mov ecx, dword ptr [desired + 4]
 769             mov eax, dword ptr [esi]
 770             mov edx, dword ptr [esi + 4]
 771             lock cmpxchg8b qword ptr [edi]
 772             mov dword ptr [esi], eax
 773             mov dword ptr [esi + 4], edx
 774             mov ebx, backup
 775             sete result
 776         };
 777 #endif
 778         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
 779
 780         return result;
 781     }
 782
 783     static BOOST_FORCEINLINE bool compare_exchange_weak(
 784         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
 785     {
 786         return compare_exchange_strong(storage, expected, desired, success_order, failure_order);
 787     }
 788
 789     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 790     {
 791         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
 792
 793         storage_type volatile* p = &storage;
 794         int backup;
 795         __asm
 796         {
 797             mov backup, ebx
 798             mov edi, p
 799             mov ebx, dword ptr [v]
 800             mov ecx, dword ptr [v + 4]
 801             mov eax, dword ptr [edi]
 802             mov edx, dword ptr [edi + 4]
 803             align 16
 804         again:
 805             lock cmpxchg8b qword ptr [edi]
 806             jne again
 807             mov ebx, backup
 808             mov dword ptr [v], eax
 809             mov dword ptr [v + 4], edx
 810         };
 811
 812         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
 813
 814         return v;
 815     }
 816 };
 817
 818 template< bool Signed >
 819 struct operations< 8u, Signed > :
 820     public cas_based_operations< msvc_dcas_x86< Signed > >
 821 {
 822 };
 823
 824 #elif defined(_M_AMD64)
 825
 826 template< bool Signed >
 827 struct operations< 8u, Signed > :
 828     public msvc_x86_operations< typename make_storage_type< 8u, Signed >::type, operations< 8u, Signed > >
 829 {
 830     typedef msvc_x86_operations< typename make_storage_type< 8u, Signed >::type, operations< 8u, Signed > > base_type;
 831     typedef typename base_type::storage_type storage_type;
 832     typedef typename make_storage_type< 8u, Signed >::aligned aligned_storage_type;
 833
 834     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 8u;
 835     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
 836
 837     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 838     {
 839         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE_ADD64(&storage, v));
 840     }
 841
 842     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 843     {
 844         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_EXCHANGE64(&storage, v));
 845     }
 846
 847     static BOOST_FORCEINLINE bool compare_exchange_strong(
 848         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
 849     {
 850         storage_type previous = expected;
 851         storage_type old_val = static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE64(&storage, desired, previous));
 852         expected = old_val;
 853         return (previous == old_val);
 854     }
 855
 856     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 857     {
 858         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_AND64(&storage, v));
 859     }
 860
 861     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 862     {
 863         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_OR64(&storage, v));
 864     }
 865
 866     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 867     {
 868         return static_cast< storage_type >(BOOST_ATOMIC_INTERLOCKED_XOR64(&storage, v));
 869     }
 870 };
 871
 872 #endif
 873
 874 #if defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG16B)
 875
 876 template< bool Signed >
 877 struct msvc_dcas_x86_64
 878 {
 879     typedef typename make_storage_type< 16u, Signed >::type storage_type;
 880     typedef typename make_storage_type< 16u, Signed >::aligned aligned_storage_type;
 881
 882     static BOOST_CONSTEXPR_OR_CONST bool is_always_lock_free = true;
 883
 884     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 16u;
 885     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
 886
 887     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT
 888     {
 889         storage_type value = const_cast< storage_type& >(storage);
 890         while (!BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE128(&storage, v, &value)) {}
 891     }
 892
 893     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order) BOOST_NOEXCEPT
 894     {
 895         storage_type value = storage_type();
 896         BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE128(&storage, value, &value);
 897         return value;
 898     }
 899
 900     static BOOST_FORCEINLINE bool compare_exchange_strong(
 901         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order, memory_order) BOOST_NOEXCEPT
 902     {
 903         return !!BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE128(&storage, desired, &expected);
 904     }
 905
 906     static BOOST_FORCEINLINE bool compare_exchange_weak(
 907         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
 908     {
 909         return compare_exchange_strong(storage, expected, desired, success_order, failure_order);
 910     }
 911 };
 912
 913 template< bool Signed >
 914 struct operations< 16u, Signed > :
 915     public cas_based_operations< cas_based_exchange< msvc_dcas_x86_64< Signed > > >
 916 {
 917 };
 918
 919 #endif // defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG16B)
 920
 921 BOOST_FORCEINLINE void thread_fence(memory_order order) BOOST_NOEXCEPT
 922 {
 923     BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
 924     if (order == memory_order_seq_cst)
 925         msvc_x86_operations_base::hardware_full_fence();
 926     BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
 927 }
 928
 929 BOOST_FORCEINLINE void signal_fence(memory_order order) BOOST_NOEXCEPT
 930 {
 931     if (order != memory_order_relaxed)
 932         BOOST_ATOMIC_DETAIL_COMPILER_BARRIER();
 933 }
 934
 935 } // namespace detail
 936 } // namespace atomics
 937 } // namespace boost
 938
 939 #if defined(BOOST_MSVC)
 940 #pragma warning(pop)
 941 #endif
 942
 943 #endif // BOOST_ATOMIC_DETAIL_OPS_MSVC_X86_HPP_INCLUDED_