ceph/src/boost/boost/atomic/detail/extra_ops_msvc_x86.hpp

   1 /*
   2  * Distributed under the Boost Software License, Version 1.0.
   3  * (See accompanying file LICENSE_1_0.txt or copy at
   4  * http://www.boost.org/LICENSE_1_0.txt)
   5  *
   6  * Copyright (c) 2017 Andrey Semashev
   7  */
   8 /*!
   9  * \file   atomic/detail/extra_ops_msvc_x86.hpp
  10  *
  11  * This header contains implementation of the extra atomic operations for x86.
  12  */
  13
  14 #ifndef BOOST_ATOMIC_DETAIL_EXTRA_OPS_MSVC_X86_HPP_INCLUDED_
  15 #define BOOST_ATOMIC_DETAIL_EXTRA_OPS_MSVC_X86_HPP_INCLUDED_
  16
  17 #include <cstddef>
  18 #include <boost/memory_order.hpp>
  19 #include <boost/atomic/detail/config.hpp>
  20 #include <boost/atomic/detail/interlocked.hpp>
  21 #include <boost/atomic/detail/storage_traits.hpp>
  22 #include <boost/atomic/detail/extra_operations_fwd.hpp>
  23 #include <boost/atomic/detail/extra_ops_generic.hpp>
  24 #include <boost/atomic/detail/header.hpp>
  25
  26 #ifdef BOOST_HAS_PRAGMA_ONCE
  27 #pragma once
  28 #endif
  29
  30 namespace boost {
  31 namespace atomics {
  32 namespace detail {
  33
  34 #if defined(_M_IX86)
  35
  36 template< typename Base, bool Signed >
  37 struct extra_operations< Base, 1u, Signed, true > :
  38     public extra_operations_generic< Base, 1u, Signed >
  39 {
  40     typedef extra_operations_generic< Base, 1u, Signed > base_type;
  41     typedef typename base_type::storage_type storage_type;
  42
  43     static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  44     {
  45         base_type::fence_before(order);
  46         storage_type old_val;
  47         __asm
  48         {
  49             mov ecx, storage
  50             movzx eax, byte ptr [ecx]
  51             align 16
  52         again:
  53             mov edx, eax
  54             neg dl
  55             lock cmpxchg byte ptr [ecx], dl
  56             jne again
  57             mov old_val, al
  58         };
  59         base_type::fence_after(order);
  60         return old_val;
  61     }
  62
  63     static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  64     {
  65         base_type::fence_before(order);
  66         storage_type new_val;
  67         __asm
  68         {
  69             mov ecx, storage
  70             movzx eax, byte ptr [ecx]
  71             align 16
  72         again:
  73             mov edx, eax
  74             neg dl
  75             lock cmpxchg byte ptr [ecx], dl
  76             jne again
  77             mov new_val, dl
  78         };
  79         base_type::fence_after(order);
  80         return new_val;
  81     }
  82
  83     static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  84     {
  85         base_type::fence_before(order);
  86         bool result;
  87         __asm
  88         {
  89             mov ecx, storage
  90             movzx eax, byte ptr [ecx]
  91             align 16
  92         again:
  93             mov edx, eax
  94             neg dl
  95             lock cmpxchg byte ptr [ecx], dl
  96             jne again
  97             test dl, dl
  98             setnz result
  99         };
 100         base_type::fence_after(order);
 101         return result;
 102     }
 103
 104     static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 105     {
 106         base_type::fence_before(order);
 107         __asm
 108         {
 109             mov ecx, storage
 110             movzx eax, byte ptr [ecx]
 111             align 16
 112         again:
 113             mov edx, eax
 114             neg dl
 115             lock cmpxchg byte ptr [ecx], dl
 116             jne again
 117         };
 118         base_type::fence_after(order);
 119     }
 120
 121     static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 122     {
 123         base_type::fence_before(order);
 124         __asm
 125         {
 126             mov edi, storage
 127             movzx ecx, v
 128             xor edx, edx
 129             movzx eax, byte ptr [edi]
 130             align 16
 131         again:
 132             mov dl, al
 133             and dl, cl
 134             lock cmpxchg byte ptr [edi], dl
 135             jne again
 136             mov v, dl
 137         };
 138         base_type::fence_after(order);
 139         return v;
 140     }
 141
 142     static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 143     {
 144         base_type::fence_before(order);
 145         __asm
 146         {
 147             mov edi, storage
 148             movzx ecx, v
 149             xor edx, edx
 150             movzx eax, byte ptr [edi]
 151             align 16
 152         again:
 153             mov dl, al
 154             or dl, cl
 155             lock cmpxchg byte ptr [edi], dl
 156             jne again
 157             mov v, dl
 158         };
 159         base_type::fence_after(order);
 160         return v;
 161     }
 162
 163     static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 164     {
 165         base_type::fence_before(order);
 166         __asm
 167         {
 168             mov edi, storage
 169             movzx ecx, v
 170             xor edx, edx
 171             movzx eax, byte ptr [edi]
 172             align 16
 173         again:
 174             mov dl, al
 175             xor dl, cl
 176             lock cmpxchg byte ptr [edi], dl
 177             jne again
 178             mov v, dl
 179         };
 180         base_type::fence_after(order);
 181         return v;
 182     }
 183
 184     static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 185     {
 186         base_type::fence_before(order);
 187         storage_type old_val;
 188         __asm
 189         {
 190             mov ecx, storage
 191             movzx eax, byte ptr [ecx]
 192             align 16
 193         again:
 194             mov edx, eax
 195             not dl
 196             lock cmpxchg byte ptr [ecx], dl
 197             jne again
 198             mov old_val, al
 199         };
 200         base_type::fence_after(order);
 201         return old_val;
 202     }
 203
 204     static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 205     {
 206         base_type::fence_before(order);
 207         storage_type new_val;
 208         __asm
 209         {
 210             mov ecx, storage
 211             movzx eax, byte ptr [ecx]
 212             align 16
 213         again:
 214             mov edx, eax
 215             not dl
 216             lock cmpxchg byte ptr [ecx], dl
 217             jne again
 218             mov new_val, dl
 219         };
 220         base_type::fence_after(order);
 221         return new_val;
 222     }
 223
 224     static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 225     {
 226         base_type::fence_before(order);
 227         bool result;
 228         __asm
 229         {
 230             mov ecx, storage
 231             movzx eax, byte ptr [ecx]
 232             align 16
 233         again:
 234             mov edx, eax
 235             not dl
 236             lock cmpxchg byte ptr [ecx], dl
 237             jne again
 238             test dl, dl
 239             setnz result
 240         };
 241         base_type::fence_after(order);
 242         return result;
 243     }
 244
 245     static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 246     {
 247         base_type::fence_before(order);
 248         __asm
 249         {
 250             mov ecx, storage
 251             movzx eax, byte ptr [ecx]
 252             align 16
 253         again:
 254             mov edx, eax
 255             not dl
 256             lock cmpxchg byte ptr [ecx], dl
 257             jne again
 258         };
 259         base_type::fence_after(order);
 260     }
 261
 262     static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 263     {
 264         base_type::fence_before(order);
 265         __asm
 266         {
 267             mov edx, storage
 268             movzx eax, v
 269             lock add byte ptr [edx], al
 270         };
 271         base_type::fence_after(order);
 272     }
 273
 274     static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 275     {
 276         base_type::fence_before(order);
 277         __asm
 278         {
 279             mov edx, storage
 280             movzx eax, v
 281             lock sub byte ptr [edx], al
 282         };
 283         base_type::fence_after(order);
 284     }
 285
 286     static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 287     {
 288         base_type::fence_before(order);
 289         __asm
 290         {
 291             mov edx, storage
 292             lock neg byte ptr [edx]
 293         };
 294         base_type::fence_after(order);
 295     }
 296
 297     static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 298     {
 299         base_type::fence_before(order);
 300         __asm
 301         {
 302             mov edx, storage
 303             movzx eax, v
 304             lock and byte ptr [edx], al
 305         };
 306         base_type::fence_after(order);
 307     }
 308
 309     static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 310     {
 311         base_type::fence_before(order);
 312         __asm
 313         {
 314             mov edx, storage
 315             movzx eax, v
 316             lock or byte ptr [edx], al
 317         };
 318         base_type::fence_after(order);
 319     }
 320
 321     static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 322     {
 323         base_type::fence_before(order);
 324         __asm
 325         {
 326             mov edx, storage
 327             movzx eax, v
 328             lock xor byte ptr [edx], al
 329         };
 330         base_type::fence_after(order);
 331     }
 332
 333     static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 334     {
 335         base_type::fence_before(order);
 336         __asm
 337         {
 338             mov edx, storage
 339             lock not byte ptr [edx]
 340         };
 341         base_type::fence_after(order);
 342     }
 343
 344     static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 345     {
 346         base_type::fence_before(order);
 347         bool result;
 348         __asm
 349         {
 350             mov edx, storage
 351             movzx eax, v
 352             lock add byte ptr [edx], al
 353             setnz result
 354         };
 355         base_type::fence_after(order);
 356         return result;
 357     }
 358
 359     static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 360     {
 361         base_type::fence_before(order);
 362         bool result;
 363         __asm
 364         {
 365             mov edx, storage
 366             movzx eax, v
 367             lock sub byte ptr [edx], al
 368             setnz result
 369         };
 370         base_type::fence_after(order);
 371         return result;
 372     }
 373
 374     static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 375     {
 376         base_type::fence_before(order);
 377         bool result;
 378         __asm
 379         {
 380             mov edx, storage
 381             movzx eax, v
 382             lock and byte ptr [edx], al
 383             setnz result
 384         };
 385         base_type::fence_after(order);
 386         return result;
 387     }
 388
 389     static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 390     {
 391         base_type::fence_before(order);
 392         bool result;
 393         __asm
 394         {
 395             mov edx, storage
 396             movzx eax, v
 397             lock or byte ptr [edx], al
 398             setnz result
 399         };
 400         base_type::fence_after(order);
 401         return result;
 402     }
 403
 404     static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 405     {
 406         base_type::fence_before(order);
 407         bool result;
 408         __asm
 409         {
 410             mov edx, storage
 411             movzx eax, v
 412             lock xor byte ptr [edx], al
 413             setnz result
 414         };
 415         base_type::fence_after(order);
 416         return result;
 417     }
 418 };
 419
 420 template< typename Base, bool Signed >
 421 struct extra_operations< Base, 2u, Signed, true > :
 422     public extra_operations_generic< Base, 2u, Signed >
 423 {
 424     typedef extra_operations_generic< Base, 2u, Signed > base_type;
 425     typedef typename base_type::storage_type storage_type;
 426
 427     static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 428     {
 429         base_type::fence_before(order);
 430         storage_type old_val;
 431         __asm
 432         {
 433             mov ecx, storage
 434             movzx eax, word ptr [ecx]
 435             align 16
 436         again:
 437             mov edx, eax
 438             neg dx
 439             lock cmpxchg word ptr [ecx], dx
 440             jne again
 441             mov old_val, ax
 442         };
 443         base_type::fence_after(order);
 444         return old_val;
 445     }
 446
 447     static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 448     {
 449         base_type::fence_before(order);
 450         storage_type new_val;
 451         __asm
 452         {
 453             mov ecx, storage
 454             movzx eax, word ptr [ecx]
 455             align 16
 456         again:
 457             mov edx, eax
 458             neg dx
 459             lock cmpxchg word ptr [ecx], dx
 460             jne again
 461             mov new_val, dx
 462         };
 463         base_type::fence_after(order);
 464         return new_val;
 465     }
 466
 467     static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 468     {
 469         base_type::fence_before(order);
 470         bool result;
 471         __asm
 472         {
 473             mov ecx, storage
 474             movzx eax, word ptr [ecx]
 475             align 16
 476         again:
 477             mov edx, eax
 478             neg dx
 479             lock cmpxchg word ptr [ecx], dx
 480             jne again
 481             test dx, dx
 482             setnz result
 483         };
 484         base_type::fence_after(order);
 485         return result;
 486     }
 487
 488     static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 489     {
 490         base_type::fence_before(order);
 491         __asm
 492         {
 493             mov ecx, storage
 494             movzx eax, word ptr [ecx]
 495             align 16
 496         again:
 497             mov edx, eax
 498             neg dx
 499             lock cmpxchg word ptr [ecx], dx
 500             jne again
 501         };
 502         base_type::fence_after(order);
 503     }
 504
 505     static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 506     {
 507         base_type::fence_before(order);
 508         __asm
 509         {
 510             mov edi, storage
 511             movzx ecx, v
 512             xor edx, edx
 513             movzx eax, word ptr [edi]
 514             align 16
 515         again:
 516             mov dx, ax
 517             and dx, cx
 518             lock cmpxchg word ptr [edi], dx
 519             jne again
 520             mov v, dx
 521         };
 522         base_type::fence_after(order);
 523         return v;
 524     }
 525
 526     static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 527     {
 528         base_type::fence_before(order);
 529         __asm
 530         {
 531             mov edi, storage
 532             movzx ecx, v
 533             xor edx, edx
 534             movzx eax, word ptr [edi]
 535             align 16
 536         again:
 537             mov dx, ax
 538             or dx, cx
 539             lock cmpxchg word ptr [edi], dx
 540             jne again
 541             mov v, dx
 542         };
 543         base_type::fence_after(order);
 544         return v;
 545     }
 546
 547     static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 548     {
 549         base_type::fence_before(order);
 550         __asm
 551         {
 552             mov edi, storage
 553             movzx ecx, v
 554             xor edx, edx
 555             movzx eax, word ptr [edi]
 556             align 16
 557         again:
 558             mov dx, ax
 559             xor dx, cx
 560             lock cmpxchg word ptr [edi], dx
 561             jne again
 562             mov v, dx
 563         };
 564         base_type::fence_after(order);
 565         return v;
 566     }
 567
 568     static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 569     {
 570         base_type::fence_before(order);
 571         storage_type old_val;
 572         __asm
 573         {
 574             mov ecx, storage
 575             movzx eax, word ptr [ecx]
 576             align 16
 577         again:
 578             mov edx, eax
 579             not dx
 580             lock cmpxchg word ptr [ecx], dx
 581             jne again
 582             mov old_val, ax
 583         };
 584         base_type::fence_after(order);
 585         return old_val;
 586     }
 587
 588     static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 589     {
 590         base_type::fence_before(order);
 591         storage_type new_val;
 592         __asm
 593         {
 594             mov ecx, storage
 595             movzx eax, word ptr [ecx]
 596             align 16
 597         again:
 598             mov edx, eax
 599             not dx
 600             lock cmpxchg word ptr [ecx], dx
 601             jne again
 602             mov new_val, dx
 603         };
 604         base_type::fence_after(order);
 605         return new_val;
 606     }
 607
 608     static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 609     {
 610         base_type::fence_before(order);
 611         bool result;
 612         __asm
 613         {
 614             mov ecx, storage
 615             movzx eax, word ptr [ecx]
 616             align 16
 617         again:
 618             mov edx, eax
 619             not dx
 620             lock cmpxchg word ptr [ecx], dx
 621             jne again
 622             test dx, dx
 623             setnz result
 624         };
 625         base_type::fence_after(order);
 626         return result;
 627     }
 628
 629     static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 630     {
 631         base_type::fence_before(order);
 632         __asm
 633         {
 634             mov ecx, storage
 635             movzx eax, word ptr [ecx]
 636             align 16
 637         again:
 638             mov edx, eax
 639             not dx
 640             lock cmpxchg word ptr [ecx], dx
 641             jne again
 642         };
 643         base_type::fence_after(order);
 644     }
 645
 646     static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 647     {
 648         base_type::fence_before(order);
 649         __asm
 650         {
 651             mov edx, storage
 652             movzx eax, v
 653             lock add word ptr [edx], ax
 654         };
 655         base_type::fence_after(order);
 656     }
 657
 658     static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 659     {
 660         base_type::fence_before(order);
 661         __asm
 662         {
 663             mov edx, storage
 664             movzx eax, v
 665             lock sub word ptr [edx], ax
 666         };
 667         base_type::fence_after(order);
 668     }
 669
 670     static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 671     {
 672         base_type::fence_before(order);
 673         __asm
 674         {
 675             mov edx, storage
 676             lock neg word ptr [edx]
 677         };
 678         base_type::fence_after(order);
 679     }
 680
 681     static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 682     {
 683         base_type::fence_before(order);
 684         __asm
 685         {
 686             mov edx, storage
 687             movzx eax, v
 688             lock and word ptr [edx], ax
 689         };
 690         base_type::fence_after(order);
 691     }
 692
 693     static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 694     {
 695         base_type::fence_before(order);
 696         __asm
 697         {
 698             mov edx, storage
 699             movzx eax, v
 700             lock or word ptr [edx], ax
 701         };
 702         base_type::fence_after(order);
 703     }
 704
 705     static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 706     {
 707         base_type::fence_before(order);
 708         __asm
 709         {
 710             mov edx, storage
 711             movzx eax, v
 712             lock xor word ptr [edx], ax
 713         };
 714         base_type::fence_after(order);
 715     }
 716
 717     static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 718     {
 719         base_type::fence_before(order);
 720         __asm
 721         {
 722             mov edx, storage
 723             lock not word ptr [edx]
 724         };
 725         base_type::fence_after(order);
 726     }
 727
 728     static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 729     {
 730         base_type::fence_before(order);
 731         bool result;
 732         __asm
 733         {
 734             mov edx, storage
 735             movzx eax, v
 736             lock add word ptr [edx], ax
 737             setnz result
 738         };
 739         base_type::fence_after(order);
 740         return result;
 741     }
 742
 743     static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 744     {
 745         base_type::fence_before(order);
 746         bool result;
 747         __asm
 748         {
 749             mov edx, storage
 750             movzx eax, v
 751             lock sub word ptr [edx], ax
 752             setnz result
 753         };
 754         base_type::fence_after(order);
 755         return result;
 756     }
 757
 758     static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 759     {
 760         base_type::fence_before(order);
 761         bool result;
 762         __asm
 763         {
 764             mov edx, storage
 765             movzx eax, v
 766             lock and word ptr [edx], ax
 767             setnz result
 768         };
 769         base_type::fence_after(order);
 770         return result;
 771     }
 772
 773     static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 774     {
 775         base_type::fence_before(order);
 776         bool result;
 777         __asm
 778         {
 779             mov edx, storage
 780             movzx eax, v
 781             lock or word ptr [edx], ax
 782             setnz result
 783         };
 784         base_type::fence_after(order);
 785         return result;
 786     }
 787
 788     static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 789     {
 790         base_type::fence_before(order);
 791         bool result;
 792         __asm
 793         {
 794             mov edx, storage
 795             movzx eax, v
 796             lock xor word ptr [edx], ax
 797             setnz result
 798         };
 799         base_type::fence_after(order);
 800         return result;
 801     }
 802
 803     static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
 804     {
 805         base_type::fence_before(order);
 806         bool result;
 807         __asm
 808         {
 809             mov edx, storage
 810             mov eax, bit_number
 811             lock bts word ptr [edx], ax
 812             setc result
 813         };
 814         base_type::fence_after(order);
 815         return result;
 816     }
 817
 818     static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
 819     {
 820         base_type::fence_before(order);
 821         bool result;
 822         __asm
 823         {
 824             mov edx, storage
 825             mov eax, bit_number
 826             lock btr word ptr [edx], ax
 827             setc result
 828         };
 829         base_type::fence_after(order);
 830         return result;
 831     }
 832
 833     static BOOST_FORCEINLINE bool bit_test_and_complement(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
 834     {
 835         base_type::fence_before(order);
 836         bool result;
 837         __asm
 838         {
 839             mov edx, storage
 840             mov eax, bit_number
 841             lock btc word ptr [edx], ax
 842             setc result
 843         };
 844         base_type::fence_after(order);
 845         return result;
 846     }
 847 };
 848
 849 #endif // defined(_M_IX86)
 850
 851 #if defined(_M_IX86) || (defined(BOOST_ATOMIC_INTERLOCKED_BTS) && defined(BOOST_ATOMIC_INTERLOCKED_BTR))
 852
 853 template< typename Base, bool Signed >
 854 struct extra_operations< Base, 4u, Signed, true > :
 855     public extra_operations_generic< Base, 4u, Signed >
 856 {
 857     typedef extra_operations_generic< Base, 4u, Signed > base_type;
 858     typedef typename base_type::storage_type storage_type;
 859
 860 #if defined(_M_IX86)
 861     static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 862     {
 863         base_type::fence_before(order);
 864         storage_type old_val;
 865         __asm
 866         {
 867             mov ecx, storage
 868             mov eax, dword ptr [ecx]
 869             align 16
 870         again:
 871             mov edx, eax
 872             neg edx
 873             lock cmpxchg dword ptr [ecx], edx
 874             jne again
 875             mov old_val, eax
 876         };
 877         base_type::fence_after(order);
 878         return old_val;
 879     }
 880
 881     static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 882     {
 883         base_type::fence_before(order);
 884         storage_type new_val;
 885         __asm
 886         {
 887             mov ecx, storage
 888             mov eax, dword ptr [ecx]
 889             align 16
 890         again:
 891             mov edx, eax
 892             neg edx
 893             lock cmpxchg dword ptr [ecx], edx
 894             jne again
 895             mov new_val, edx
 896         };
 897         base_type::fence_after(order);
 898         return new_val;
 899     }
 900
 901     static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 902     {
 903         base_type::fence_before(order);
 904         bool result;
 905         __asm
 906         {
 907             mov ecx, storage
 908             mov eax, dword ptr [ecx]
 909             align 16
 910         again:
 911             mov edx, eax
 912             neg edx
 913             lock cmpxchg dword ptr [ecx], edx
 914             jne again
 915             test edx, edx
 916             setnz result
 917         };
 918         base_type::fence_after(order);
 919         return result;
 920     }
 921
 922     static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 923     {
 924         base_type::fence_before(order);
 925         __asm
 926         {
 927             mov ecx, storage
 928             mov eax, dword ptr [ecx]
 929             align 16
 930         again:
 931             mov edx, eax
 932             neg edx
 933             lock cmpxchg dword ptr [ecx], edx
 934             jne again
 935         };
 936         base_type::fence_after(order);
 937     }
 938
 939     static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 940     {
 941         base_type::fence_before(order);
 942         __asm
 943         {
 944             mov edi, storage
 945             mov ecx, v
 946             xor edx, edx
 947             mov eax, dword ptr [edi]
 948             align 16
 949         again:
 950             mov edx, eax
 951             and edx, ecx
 952             lock cmpxchg dword ptr [edi], edx
 953             jne again
 954             mov v, edx
 955         };
 956         base_type::fence_after(order);
 957         return v;
 958     }
 959
 960     static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 961     {
 962         base_type::fence_before(order);
 963         __asm
 964         {
 965             mov edi, storage
 966             mov ecx, v
 967             xor edx, edx
 968             mov eax, dword ptr [edi]
 969             align 16
 970         again:
 971             mov edx, eax
 972             or edx, ecx
 973             lock cmpxchg dword ptr [edi], edx
 974             jne again
 975             mov v, edx
 976         };
 977         base_type::fence_after(order);
 978         return v;
 979     }
 980
 981     static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 982     {
 983         base_type::fence_before(order);
 984         __asm
 985         {
 986             mov edi, storage
 987             mov ecx, v
 988             xor edx, edx
 989             mov eax, dword ptr [edi]
 990             align 16
 991         again:
 992             mov edx, eax
 993             xor edx, ecx
 994             lock cmpxchg dword ptr [edi], edx
 995             jne again
 996             mov v, edx
 997         };
 998         base_type::fence_after(order);
 999         return v;
1000     }
1001
1002     static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1003     {
1004         base_type::fence_before(order);
1005         storage_type old_val;
1006         __asm
1007         {
1008             mov ecx, storage
1009             mov eax, dword ptr [ecx]
1010             align 16
1011         again:
1012             mov edx, eax
1013             not edx
1014             lock cmpxchg dword ptr [ecx], edx
1015             jne again
1016             mov old_val, eax
1017         };
1018         base_type::fence_after(order);
1019         return old_val;
1020     }
1021
1022     static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1023     {
1024         base_type::fence_before(order);
1025         storage_type new_val;
1026         __asm
1027         {
1028             mov ecx, storage
1029             mov eax, dword ptr [ecx]
1030             align 16
1031         again:
1032             mov edx, eax
1033             not edx
1034             lock cmpxchg dword ptr [ecx], edx
1035             jne again
1036             mov new_val, edx
1037         };
1038         base_type::fence_after(order);
1039         return new_val;
1040     }
1041
1042     static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1043     {
1044         base_type::fence_before(order);
1045         bool result;
1046         __asm
1047         {
1048             mov ecx, storage
1049             mov eax, dword ptr [ecx]
1050             align 16
1051         again:
1052             mov edx, eax
1053             not edx
1054             lock cmpxchg dword ptr [ecx], edx
1055             jne again
1056             test edx, edx
1057             setnz result
1058         };
1059         base_type::fence_after(order);
1060         return result;
1061     }
1062
1063     static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1064     {
1065         base_type::fence_before(order);
1066         __asm
1067         {
1068             mov ecx, storage
1069             mov eax, dword ptr [ecx]
1070             align 16
1071         again:
1072             mov edx, eax
1073             not edx
1074             lock cmpxchg dword ptr [ecx], edx
1075             jne again
1076         };
1077         base_type::fence_after(order);
1078     }
1079
1080     static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1081     {
1082         base_type::fence_before(order);
1083         __asm
1084         {
1085             mov edx, storage
1086             mov eax, v
1087             lock add dword ptr [edx], eax
1088         };
1089         base_type::fence_after(order);
1090     }
1091
1092     static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1093     {
1094         base_type::fence_before(order);
1095         __asm
1096         {
1097             mov edx, storage
1098             mov eax, v
1099             lock sub dword ptr [edx], eax
1100         };
1101         base_type::fence_after(order);
1102     }
1103
1104     static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1105     {
1106         base_type::fence_before(order);
1107         __asm
1108         {
1109             mov edx, storage
1110             lock neg dword ptr [edx]
1111         };
1112         base_type::fence_after(order);
1113     }
1114
1115     static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1116     {
1117         base_type::fence_before(order);
1118         __asm
1119         {
1120             mov edx, storage
1121             mov eax, v
1122             lock and dword ptr [edx], eax
1123         };
1124         base_type::fence_after(order);
1125     }
1126
1127     static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1128     {
1129         base_type::fence_before(order);
1130         __asm
1131         {
1132             mov edx, storage
1133             mov eax, v
1134             lock or dword ptr [edx], eax
1135         };
1136         base_type::fence_after(order);
1137     }
1138
1139     static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1140     {
1141         base_type::fence_before(order);
1142         __asm
1143         {
1144             mov edx, storage
1145             mov eax, v
1146             lock xor dword ptr [edx], eax
1147         };
1148         base_type::fence_after(order);
1149     }
1150
1151     static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1152     {
1153         base_type::fence_before(order);
1154         __asm
1155         {
1156             mov edx, storage
1157             lock not dword ptr [edx]
1158         };
1159         base_type::fence_after(order);
1160     }
1161
1162     static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1163     {
1164         base_type::fence_before(order);
1165         bool result;
1166         __asm
1167         {
1168             mov edx, storage
1169             mov eax, v
1170             lock add dword ptr [edx], eax
1171             setnz result
1172         };
1173         base_type::fence_after(order);
1174         return result;
1175     }
1176
1177     static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1178     {
1179         base_type::fence_before(order);
1180         bool result;
1181         __asm
1182         {
1183             mov edx, storage
1184             mov eax, v
1185             lock sub dword ptr [edx], eax
1186             setnz result
1187         };
1188         base_type::fence_after(order);
1189         return result;
1190     }
1191
1192     static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1193     {
1194         base_type::fence_before(order);
1195         bool result;
1196         __asm
1197         {
1198             mov edx, storage
1199             mov eax, v
1200             lock and dword ptr [edx], eax
1201             setnz result
1202         };
1203         base_type::fence_after(order);
1204         return result;
1205     }
1206
1207     static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1208     {
1209         base_type::fence_before(order);
1210         bool result;
1211         __asm
1212         {
1213             mov edx, storage
1214             mov eax, v
1215             lock or dword ptr [edx], eax
1216             setnz result
1217         };
1218         base_type::fence_after(order);
1219         return result;
1220     }
1221
1222     static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1223     {
1224         base_type::fence_before(order);
1225         bool result;
1226         __asm
1227         {
1228             mov edx, storage
1229             mov eax, v
1230             lock xor dword ptr [edx], eax
1231             setnz result
1232         };
1233         base_type::fence_after(order);
1234         return result;
1235     }
1236
1237     static BOOST_FORCEINLINE bool bit_test_and_complement(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
1238     {
1239         base_type::fence_before(order);
1240         bool result;
1241         __asm
1242         {
1243             mov edx, storage
1244             mov eax, bit_number
1245             lock btc dword ptr [edx], eax
1246             setc result
1247         };
1248         base_type::fence_after(order);
1249         return result;
1250     }
1251 #endif // defined(_M_IX86)
1252
1253 #if defined(BOOST_ATOMIC_INTERLOCKED_BTS)
1254     static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT
1255     {
1256         return !!BOOST_ATOMIC_INTERLOCKED_BTS(&storage, bit_number);
1257     }
1258 #elif defined(_M_IX86)
1259     static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
1260     {
1261         base_type::fence_before(order);
1262         bool result;
1263         __asm
1264         {
1265             mov edx, storage
1266             mov eax, bit_number
1267             lock bts dword ptr [edx], eax
1268             setc result
1269         };
1270         base_type::fence_after(order);
1271         return result;
1272     }
1273 #endif
1274
1275 #if defined(BOOST_ATOMIC_INTERLOCKED_BTR)
1276     static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT
1277     {
1278         return !!BOOST_ATOMIC_INTERLOCKED_BTR(&storage, bit_number);
1279     }
1280 #elif defined(_M_IX86)
1281     static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
1282     {
1283         base_type::fence_before(order);
1284         bool result;
1285         __asm
1286         {
1287             mov edx, storage
1288             mov eax, bit_number
1289             lock btr dword ptr [edx], eax
1290             setc result
1291         };
1292         base_type::fence_after(order);
1293         return result;
1294     }
1295 #endif
1296 };
1297
1298 #endif // defined(_M_IX86) || (defined(BOOST_ATOMIC_INTERLOCKED_BTS) && defined(BOOST_ATOMIC_INTERLOCKED_BTR))
1299
1300 #if defined(BOOST_ATOMIC_INTERLOCKED_BTS64) && defined(BOOST_ATOMIC_INTERLOCKED_BTR64)
1301
1302 template< typename Base, bool Signed >
1303 struct extra_operations< Base, 8u, Signed, true > :
1304     public extra_operations_generic< Base, 8u, Signed >
1305 {
1306     typedef extra_operations_generic< Base, 8u, Signed > base_type;
1307     typedef typename base_type::storage_type storage_type;
1308
1309     static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
1310     {
1311         return !!BOOST_ATOMIC_INTERLOCKED_BTS64(&storage, bit_number);
1312     }
1313
1314     static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
1315     {
1316         return !!BOOST_ATOMIC_INTERLOCKED_BTR64(&storage, bit_number);
1317     }
1318 };
1319
1320 #endif // defined(BOOST_ATOMIC_INTERLOCKED_BTS64) && defined(BOOST_ATOMIC_INTERLOCKED_BTR64)
1321
1322 } // namespace detail
1323 } // namespace atomics
1324 } // namespace boost
1325
1326 #include <boost/atomic/detail/footer.hpp>
1327
1328 #endif // BOOST_ATOMIC_DETAIL_EXTRA_OPS_MSVC_X86_HPP_INCLUDED_