ceph/src/boost/boost/atomic/detail/core_arch_ops_gcc_ppc.hpp

   1 /*
   2  * Distributed under the Boost Software License, Version 1.0.
   3  * (See accompanying file LICENSE_1_0.txt or copy at
   4  * http://www.boost.org/LICENSE_1_0.txt)
   5  *
   6  * Copyright (c) 2009 Helge Bahmann
   7  * Copyright (c) 2013 Tim Blechmann
   8  * Copyright (c) 2014 Andrey Semashev
   9  */
  10 /*!
  11  * \file   atomic/detail/core_arch_ops_gcc_ppc.hpp
  12  *
  13  * This header contains implementation of the \c core_arch_operations template.
  14  */
  15
  16 #ifndef BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_PPC_HPP_INCLUDED_
  17 #define BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_PPC_HPP_INCLUDED_
  18
  19 #include <cstddef>
  20 #include <boost/memory_order.hpp>
  21 #include <boost/atomic/detail/config.hpp>
  22 #include <boost/atomic/detail/storage_traits.hpp>
  23 #include <boost/atomic/detail/core_arch_operations_fwd.hpp>
  24 #include <boost/atomic/detail/ops_gcc_ppc_common.hpp>
  25 #include <boost/atomic/detail/capabilities.hpp>
  26 #include <boost/atomic/detail/header.hpp>
  27
  28 #ifdef BOOST_HAS_PRAGMA_ONCE
  29 #pragma once
  30 #endif
  31
  32 namespace boost {
  33 namespace atomics {
  34 namespace detail {
  35
  36 // The implementation below uses information from this document:
  37 // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2010.02.19a.html
  38
  39 /*
  40     Refer to: Motorola: "Programming Environments Manual for 32-Bit
  41     Implementations of the PowerPC Architecture", Appendix E:
  42     "Synchronization Programming Examples" for an explanation of what is
  43     going on here (can be found on the web at various places by the
  44     name "MPCFPE32B.pdf", Google is your friend...)
  45
  46     Most of the atomic operations map to instructions in a relatively
  47     straight-forward fashion, but "load"s may at first glance appear
  48     a bit strange as they map to:
  49
  50             lwz %rX, addr
  51             cmpw %rX, %rX
  52             bne- 1f
  53         1:
  54
  55     That is, the CPU is forced to perform a branch that "formally" depends
  56     on the value retrieved from memory. This scheme has an overhead of
  57     about 1-2 clock cycles per load, but it allows to map "acquire" to
  58     the "isync" instruction instead of "sync" uniformly and for all type
  59     of atomic operations. Since "isync" has a cost of about 15 clock
  60     cycles, while "sync" hast a cost of about 50 clock cycles, the small
  61     penalty to atomic loads more than compensates for this.
  62
  63     Byte- and halfword-sized atomic values are implemented in two ways.
  64     When 8 and 16-bit instructions are available (in Power8 and later),
  65     they are used. Otherwise operations are realized by encoding the
  66     value to be represented into a word, performing sign/zero extension
  67     as appropriate. This means that after add/sub operations the value
  68     needs fixing up to accurately preserve the wrap-around semantic of
  69     the smaller type. (Nothing special needs to be done for the bit-wise
  70     and the "exchange type" operators as the compiler already sees to
  71     it that values carried in registers are extended appropriately and
  72     everything falls into place naturally).
  73
  74     The register constraint "b"  instructs gcc to use any register
  75     except r0; this is sometimes required because the encoding for
  76     r0 is used to signify "constant zero" in a number of instructions,
  77     making r0 unusable in this place. For simplicity this constraint
  78     is used everywhere since I am to lazy to look this up on a
  79     per-instruction basis, and ppc has enough registers for this not
  80     to pose a problem.
  81 */
  82
  83 template< bool Signed, bool Interprocess >
  84 struct core_arch_operations< 4u, Signed, Interprocess > :
  85     public core_arch_operations_gcc_ppc_base
  86 {
  87     typedef typename storage_traits< 4u >::type storage_type;
  88
  89     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 4u;
  90     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 4u;
  91     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
  92     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
  93
  94     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  95     {
  96         fence_before(order);
  97         __asm__ __volatile__
  98         (
  99             "stw %1, %0\n\t"
 100             : "+m" (storage)
 101             : "r" (v)
 102         );
 103     }
 104
 105     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
 106     {
 107         storage_type v;
 108         if (order == memory_order_seq_cst)
 109             __asm__ __volatile__ ("sync" ::: "memory");
 110         if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u)
 111         {
 112             __asm__ __volatile__
 113             (
 114                 "lwz %0, %1\n\t"
 115                 "cmpw %0, %0\n\t"
 116                 "bne- 1f\n\t"
 117                 "1:\n\t"
 118                 "isync\n\t"
 119                 : "=&r" (v)
 120                 : "m" (storage)
 121                 : "cr0", "memory"
 122             );
 123         }
 124         else
 125         {
 126             __asm__ __volatile__
 127             (
 128                 "lwz %0, %1\n\t"
 129                 : "=&r" (v)
 130                 : "m" (storage)
 131             );
 132         }
 133         return v;
 134     }
 135
 136     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 137     {
 138         storage_type original;
 139         fence_before(order);
 140         __asm__ __volatile__
 141         (
 142             "1:\n\t"
 143             "lwarx %0,%y1\n\t"
 144             "stwcx. %2,%y1\n\t"
 145             "bne- 1b\n\t"
 146             : "=&b" (original), "+Z" (storage)
 147             : "b" (v)
 148             : "cr0"
 149         );
 150         fence_after(order);
 151         return original;
 152     }
 153
 154     static BOOST_FORCEINLINE bool compare_exchange_weak(
 155         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
 156     {
 157         int success;
 158         fence_before(success_order);
 159         __asm__ __volatile__
 160         (
 161             "li %1, 0\n\t"
 162             "lwarx %0,%y2\n\t"
 163             "cmpw %0, %3\n\t"
 164             "bne- 1f\n\t"
 165             "stwcx. %4,%y2\n\t"
 166             "bne- 1f\n\t"
 167             "li %1, 1\n\t"
 168             "1:\n\t"
 169             : "=&b" (expected), "=&b" (success), "+Z" (storage)
 170             : "b" (expected), "b" (desired)
 171             : "cr0"
 172         );
 173         if (success)
 174             fence_after(success_order);
 175         else
 176             fence_after(failure_order);
 177         return !!success;
 178     }
 179
 180     static BOOST_FORCEINLINE bool compare_exchange_strong(
 181         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
 182     {
 183         int success;
 184         fence_before(success_order);
 185         __asm__ __volatile__
 186         (
 187             "li %1, 0\n\t"
 188             "0: lwarx %0,%y2\n\t"
 189             "cmpw %0, %3\n\t"
 190             "bne- 1f\n\t"
 191             "stwcx. %4,%y2\n\t"
 192             "bne- 0b\n\t"
 193             "li %1, 1\n\t"
 194             "1:\n\t"
 195             : "=&b" (expected), "=&b" (success), "+Z" (storage)
 196             : "b" (expected), "b" (desired)
 197             : "cr0"
 198         );
 199         if (success)
 200             fence_after(success_order);
 201         else
 202             fence_after(failure_order);
 203         return !!success;
 204     }
 205
 206     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 207     {
 208         storage_type original, result;
 209         fence_before(order);
 210         __asm__ __volatile__
 211         (
 212             "1:\n\t"
 213             "lwarx %0,%y2\n\t"
 214             "add %1,%0,%3\n\t"
 215             "stwcx. %1,%y2\n\t"
 216             "bne- 1b\n\t"
 217             : "=&b" (original), "=&b" (result), "+Z" (storage)
 218             : "b" (v)
 219             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 220         );
 221         fence_after(order);
 222         return original;
 223     }
 224
 225     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 226     {
 227         storage_type original, result;
 228         fence_before(order);
 229         __asm__ __volatile__
 230         (
 231             "1:\n\t"
 232             "lwarx %0,%y2\n\t"
 233             "sub %1,%0,%3\n\t"
 234             "stwcx. %1,%y2\n\t"
 235             "bne- 1b\n\t"
 236             : "=&b" (original), "=&b" (result), "+Z" (storage)
 237             : "b" (v)
 238             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 239         );
 240         fence_after(order);
 241         return original;
 242     }
 243
 244     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 245     {
 246         storage_type original, result;
 247         fence_before(order);
 248         __asm__ __volatile__
 249         (
 250             "1:\n\t"
 251             "lwarx %0,%y2\n\t"
 252             "and %1,%0,%3\n\t"
 253             "stwcx. %1,%y2\n\t"
 254             "bne- 1b\n\t"
 255             : "=&b" (original), "=&b" (result), "+Z" (storage)
 256             : "b" (v)
 257             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 258         );
 259         fence_after(order);
 260         return original;
 261     }
 262
 263     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 264     {
 265         storage_type original, result;
 266         fence_before(order);
 267         __asm__ __volatile__
 268         (
 269             "1:\n\t"
 270             "lwarx %0,%y2\n\t"
 271             "or %1,%0,%3\n\t"
 272             "stwcx. %1,%y2\n\t"
 273             "bne- 1b\n\t"
 274             : "=&b" (original), "=&b" (result), "+Z" (storage)
 275             : "b" (v)
 276             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 277         );
 278         fence_after(order);
 279         return original;
 280     }
 281
 282     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 283     {
 284         storage_type original, result;
 285         fence_before(order);
 286         __asm__ __volatile__
 287         (
 288             "1:\n\t"
 289             "lwarx %0,%y2\n\t"
 290             "xor %1,%0,%3\n\t"
 291             "stwcx. %1,%y2\n\t"
 292             "bne- 1b\n\t"
 293             : "=&b" (original), "=&b" (result), "+Z" (storage)
 294             : "b" (v)
 295             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 296         );
 297         fence_after(order);
 298         return original;
 299     }
 300
 301     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 302     {
 303         return !!exchange(storage, (storage_type)1, order);
 304     }
 305
 306     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 307     {
 308         store(storage, (storage_type)0, order);
 309     }
 310 };
 311
 312 #if defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LBARX_STBCX)
 313
 314 template< bool Signed, bool Interprocess >
 315 struct core_arch_operations< 1u, Signed, Interprocess > :
 316     public core_arch_operations_gcc_ppc_base
 317 {
 318     typedef typename storage_traits< 1u >::type storage_type;
 319
 320     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 1u;
 321     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 1u;
 322     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
 323     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
 324
 325     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 326     {
 327         fence_before(order);
 328         __asm__ __volatile__
 329         (
 330             "stb %1, %0\n\t"
 331             : "+m" (storage)
 332             : "r" (v)
 333         );
 334     }
 335
 336     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
 337     {
 338         storage_type v;
 339         if (order == memory_order_seq_cst)
 340             __asm__ __volatile__ ("sync" ::: "memory");
 341         if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u)
 342         {
 343             __asm__ __volatile__
 344             (
 345                 "lbz %0, %1\n\t"
 346                 "cmpw %0, %0\n\t"
 347                 "bne- 1f\n\t"
 348                 "1:\n\t"
 349                 "isync\n\t"
 350                 : "=&r" (v)
 351                 : "m" (storage)
 352                 : "cr0", "memory"
 353             );
 354         }
 355         else
 356         {
 357             __asm__ __volatile__
 358             (
 359                 "lbz %0, %1\n\t"
 360                 : "=&r" (v)
 361                 : "m" (storage)
 362             );
 363         }
 364         return v;
 365     }
 366
 367     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 368     {
 369         storage_type original;
 370         fence_before(order);
 371         __asm__ __volatile__
 372         (
 373             "1:\n\t"
 374             "lbarx %0,%y1\n\t"
 375             "stbcx. %2,%y1\n\t"
 376             "bne- 1b\n\t"
 377             : "=&b" (original), "+Z" (storage)
 378             : "b" (v)
 379             : "cr0"
 380         );
 381         fence_after(order);
 382         return original;
 383     }
 384
 385     static BOOST_FORCEINLINE bool compare_exchange_weak(
 386         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
 387     {
 388         int success;
 389         fence_before(success_order);
 390         __asm__ __volatile__
 391         (
 392             "li %1, 0\n\t"
 393             "lbarx %0,%y2\n\t"
 394             "cmpw %0, %3\n\t"
 395             "bne- 1f\n\t"
 396             "stbcx. %4,%y2\n\t"
 397             "bne- 1f\n\t"
 398             "li %1, 1\n\t"
 399             "1:\n\t"
 400             : "=&b" (expected), "=&b" (success), "+Z" (storage)
 401             : "b" (expected), "b" (desired)
 402             : "cr0"
 403         );
 404         if (success)
 405             fence_after(success_order);
 406         else
 407             fence_after(failure_order);
 408         return !!success;
 409     }
 410
 411     static BOOST_FORCEINLINE bool compare_exchange_strong(
 412         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
 413     {
 414         int success;
 415         fence_before(success_order);
 416         __asm__ __volatile__
 417         (
 418             "li %1, 0\n\t"
 419             "0: lbarx %0,%y2\n\t"
 420             "cmpw %0, %3\n\t"
 421             "bne- 1f\n\t"
 422             "stbcx. %4,%y2\n\t"
 423             "bne- 0b\n\t"
 424             "li %1, 1\n\t"
 425             "1:\n\t"
 426             : "=&b" (expected), "=&b" (success), "+Z" (storage)
 427             : "b" (expected), "b" (desired)
 428             : "cr0"
 429         );
 430         if (success)
 431             fence_after(success_order);
 432         else
 433             fence_after(failure_order);
 434         return !!success;
 435     }
 436
 437     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 438     {
 439         storage_type original, result;
 440         fence_before(order);
 441         __asm__ __volatile__
 442         (
 443             "1:\n\t"
 444             "lbarx %0,%y2\n\t"
 445             "add %1,%0,%3\n\t"
 446             "stbcx. %1,%y2\n\t"
 447             "bne- 1b\n\t"
 448             : "=&b" (original), "=&b" (result), "+Z" (storage)
 449             : "b" (v)
 450             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 451         );
 452         fence_after(order);
 453         return original;
 454     }
 455
 456     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 457     {
 458         storage_type original, result;
 459         fence_before(order);
 460         __asm__ __volatile__
 461         (
 462             "1:\n\t"
 463             "lbarx %0,%y2\n\t"
 464             "sub %1,%0,%3\n\t"
 465             "stbcx. %1,%y2\n\t"
 466             "bne- 1b\n\t"
 467             : "=&b" (original), "=&b" (result), "+Z" (storage)
 468             : "b" (v)
 469             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 470         );
 471         fence_after(order);
 472         return original;
 473     }
 474
 475     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 476     {
 477         storage_type original, result;
 478         fence_before(order);
 479         __asm__ __volatile__
 480         (
 481             "1:\n\t"
 482             "lbarx %0,%y2\n\t"
 483             "and %1,%0,%3\n\t"
 484             "stbcx. %1,%y2\n\t"
 485             "bne- 1b\n\t"
 486             : "=&b" (original), "=&b" (result), "+Z" (storage)
 487             : "b" (v)
 488             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 489         );
 490         fence_after(order);
 491         return original;
 492     }
 493
 494     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 495     {
 496         storage_type original, result;
 497         fence_before(order);
 498         __asm__ __volatile__
 499         (
 500             "1:\n\t"
 501             "lbarx %0,%y2\n\t"
 502             "or %1,%0,%3\n\t"
 503             "stbcx. %1,%y2\n\t"
 504             "bne- 1b\n\t"
 505             : "=&b" (original), "=&b" (result), "+Z" (storage)
 506             : "b" (v)
 507             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 508         );
 509         fence_after(order);
 510         return original;
 511     }
 512
 513     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 514     {
 515         storage_type original, result;
 516         fence_before(order);
 517         __asm__ __volatile__
 518         (
 519             "1:\n\t"
 520             "lbarx %0,%y2\n\t"
 521             "xor %1,%0,%3\n\t"
 522             "stbcx. %1,%y2\n\t"
 523             "bne- 1b\n\t"
 524             : "=&b" (original), "=&b" (result), "+Z" (storage)
 525             : "b" (v)
 526             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 527         );
 528         fence_after(order);
 529         return original;
 530     }
 531
 532     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 533     {
 534         return !!exchange(storage, (storage_type)1, order);
 535     }
 536
 537     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 538     {
 539         store(storage, (storage_type)0, order);
 540     }
 541 };
 542
 543 #else // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LBARX_STBCX)
 544
 545 template< bool Interprocess >
 546 struct core_arch_operations< 1u, false, Interprocess > :
 547     public core_arch_operations< 4u, false, Interprocess >
 548 {
 549     typedef core_arch_operations< 4u, false, Interprocess > base_type;
 550     typedef typename base_type::storage_type storage_type;
 551
 552     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 553     {
 554         storage_type original, result;
 555         base_type::fence_before(order);
 556         __asm__ __volatile__
 557         (
 558             "1:\n\t"
 559             "lwarx %0,%y2\n\t"
 560             "add %1,%0,%3\n\t"
 561             "rlwinm %1, %1, 0, 0xff\n\t"
 562             "stwcx. %1,%y2\n\t"
 563             "bne- 1b\n\t"
 564             : "=&b" (original), "=&b" (result), "+Z" (storage)
 565             : "b" (v)
 566             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 567         );
 568         base_type::fence_after(order);
 569         return original;
 570     }
 571
 572     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 573     {
 574         storage_type original, result;
 575         base_type::fence_before(order);
 576         __asm__ __volatile__
 577         (
 578             "1:\n\t"
 579             "lwarx %0,%y2\n\t"
 580             "sub %1,%0,%3\n\t"
 581             "rlwinm %1, %1, 0, 0xff\n\t"
 582             "stwcx. %1,%y2\n\t"
 583             "bne- 1b\n\t"
 584             : "=&b" (original), "=&b" (result), "+Z" (storage)
 585             : "b" (v)
 586             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 587         );
 588         base_type::fence_after(order);
 589         return original;
 590     }
 591 };
 592
 593 template< bool Interprocess >
 594 struct core_arch_operations< 1u, true, Interprocess > :
 595     public core_arch_operations< 4u, true, Interprocess >
 596 {
 597     typedef core_arch_operations< 4u, true, Interprocess > base_type;
 598     typedef typename base_type::storage_type storage_type;
 599
 600     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 601     {
 602         storage_type original, result;
 603         base_type::fence_before(order);
 604         __asm__ __volatile__
 605         (
 606             "1:\n\t"
 607             "lwarx %0,%y2\n\t"
 608             "add %1,%0,%3\n\t"
 609             "extsb %1, %1\n\t"
 610             "stwcx. %1,%y2\n\t"
 611             "bne- 1b\n\t"
 612             : "=&b" (original), "=&b" (result), "+Z" (storage)
 613             : "b" (v)
 614             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 615         );
 616         base_type::fence_after(order);
 617         return original;
 618     }
 619
 620     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 621     {
 622         storage_type original, result;
 623         base_type::fence_before(order);
 624         __asm__ __volatile__
 625         (
 626             "1:\n\t"
 627             "lwarx %0,%y2\n\t"
 628             "sub %1,%0,%3\n\t"
 629             "extsb %1, %1\n\t"
 630             "stwcx. %1,%y2\n\t"
 631             "bne- 1b\n\t"
 632             : "=&b" (original), "=&b" (result), "+Z" (storage)
 633             : "b" (v)
 634             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 635         );
 636         base_type::fence_after(order);
 637         return original;
 638     }
 639 };
 640
 641 #endif // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LBARX_STBCX)
 642
 643 #if defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LHARX_STHCX)
 644
 645 template< bool Signed, bool Interprocess >
 646 struct core_arch_operations< 2u, Signed, Interprocess > :
 647     public core_arch_operations_gcc_ppc_base
 648 {
 649     typedef typename storage_traits< 2u >::type storage_type;
 650
 651     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 2u;
 652     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 2u;
 653     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
 654     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
 655
 656     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 657     {
 658         fence_before(order);
 659         __asm__ __volatile__
 660         (
 661             "sth %1, %0\n\t"
 662             : "+m" (storage)
 663             : "r" (v)
 664         );
 665     }
 666
 667     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
 668     {
 669         storage_type v;
 670         if (order == memory_order_seq_cst)
 671             __asm__ __volatile__ ("sync" ::: "memory");
 672         if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u)
 673         {
 674             __asm__ __volatile__
 675             (
 676                 "lhz %0, %1\n\t"
 677                 "cmpw %0, %0\n\t"
 678                 "bne- 1f\n\t"
 679                 "1:\n\t"
 680                 "isync\n\t"
 681                 : "=&r" (v)
 682                 : "m" (storage)
 683                 : "cr0", "memory"
 684             );
 685         }
 686         else
 687         {
 688             __asm__ __volatile__
 689             (
 690                 "lhz %0, %1\n\t"
 691                 : "=&r" (v)
 692                 : "m" (storage)
 693             );
 694         }
 695         return v;
 696     }
 697
 698     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 699     {
 700         storage_type original;
 701         fence_before(order);
 702         __asm__ __volatile__
 703         (
 704             "1:\n\t"
 705             "lharx %0,%y1\n\t"
 706             "sthcx. %2,%y1\n\t"
 707             "bne- 1b\n\t"
 708             : "=&b" (original), "+Z" (storage)
 709             : "b" (v)
 710             : "cr0"
 711         );
 712         fence_after(order);
 713         return original;
 714     }
 715
 716     static BOOST_FORCEINLINE bool compare_exchange_weak(
 717         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
 718     {
 719         int success;
 720         fence_before(success_order);
 721         __asm__ __volatile__
 722         (
 723             "li %1, 0\n\t"
 724             "lharx %0,%y2\n\t"
 725             "cmpw %0, %3\n\t"
 726             "bne- 1f\n\t"
 727             "sthcx. %4,%y2\n\t"
 728             "bne- 1f\n\t"
 729             "li %1, 1\n\t"
 730             "1:\n\t"
 731             : "=&b" (expected), "=&b" (success), "+Z" (storage)
 732             : "b" (expected), "b" (desired)
 733             : "cr0"
 734         );
 735         if (success)
 736             fence_after(success_order);
 737         else
 738             fence_after(failure_order);
 739         return !!success;
 740     }
 741
 742     static BOOST_FORCEINLINE bool compare_exchange_strong(
 743         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
 744     {
 745         int success;
 746         fence_before(success_order);
 747         __asm__ __volatile__
 748         (
 749             "li %1, 0\n\t"
 750             "0: lharx %0,%y2\n\t"
 751             "cmpw %0, %3\n\t"
 752             "bne- 1f\n\t"
 753             "sthcx. %4,%y2\n\t"
 754             "bne- 0b\n\t"
 755             "li %1, 1\n\t"
 756             "1:\n\t"
 757             : "=&b" (expected), "=&b" (success), "+Z" (storage)
 758             : "b" (expected), "b" (desired)
 759             : "cr0"
 760         );
 761         if (success)
 762             fence_after(success_order);
 763         else
 764             fence_after(failure_order);
 765         return !!success;
 766     }
 767
 768     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 769     {
 770         storage_type original, result;
 771         fence_before(order);
 772         __asm__ __volatile__
 773         (
 774             "1:\n\t"
 775             "lharx %0,%y2\n\t"
 776             "add %1,%0,%3\n\t"
 777             "sthcx. %1,%y2\n\t"
 778             "bne- 1b\n\t"
 779             : "=&b" (original), "=&b" (result), "+Z" (storage)
 780             : "b" (v)
 781             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 782         );
 783         fence_after(order);
 784         return original;
 785     }
 786
 787     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 788     {
 789         storage_type original, result;
 790         fence_before(order);
 791         __asm__ __volatile__
 792         (
 793             "1:\n\t"
 794             "lharx %0,%y2\n\t"
 795             "sub %1,%0,%3\n\t"
 796             "sthcx. %1,%y2\n\t"
 797             "bne- 1b\n\t"
 798             : "=&b" (original), "=&b" (result), "+Z" (storage)
 799             : "b" (v)
 800             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 801         );
 802         fence_after(order);
 803         return original;
 804     }
 805
 806     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 807     {
 808         storage_type original, result;
 809         fence_before(order);
 810         __asm__ __volatile__
 811         (
 812             "1:\n\t"
 813             "lharx %0,%y2\n\t"
 814             "and %1,%0,%3\n\t"
 815             "sthcx. %1,%y2\n\t"
 816             "bne- 1b\n\t"
 817             : "=&b" (original), "=&b" (result), "+Z" (storage)
 818             : "b" (v)
 819             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 820         );
 821         fence_after(order);
 822         return original;
 823     }
 824
 825     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 826     {
 827         storage_type original, result;
 828         fence_before(order);
 829         __asm__ __volatile__
 830         (
 831             "1:\n\t"
 832             "lharx %0,%y2\n\t"
 833             "or %1,%0,%3\n\t"
 834             "sthcx. %1,%y2\n\t"
 835             "bne- 1b\n\t"
 836             : "=&b" (original), "=&b" (result), "+Z" (storage)
 837             : "b" (v)
 838             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 839         );
 840         fence_after(order);
 841         return original;
 842     }
 843
 844     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 845     {
 846         storage_type original, result;
 847         fence_before(order);
 848         __asm__ __volatile__
 849         (
 850             "1:\n\t"
 851             "lharx %0,%y2\n\t"
 852             "xor %1,%0,%3\n\t"
 853             "sthcx. %1,%y2\n\t"
 854             "bne- 1b\n\t"
 855             : "=&b" (original), "=&b" (result), "+Z" (storage)
 856             : "b" (v)
 857             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 858         );
 859         fence_after(order);
 860         return original;
 861     }
 862
 863     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 864     {
 865         return !!exchange(storage, (storage_type)1, order);
 866     }
 867
 868     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 869     {
 870         store(storage, (storage_type)0, order);
 871     }
 872 };
 873
 874 #else // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LHARX_STHCX)
 875
 876 template< bool Interprocess >
 877 struct core_arch_operations< 2u, false, Interprocess > :
 878     public core_arch_operations< 4u, false, Interprocess >
 879 {
 880     typedef core_arch_operations< 4u, false, Interprocess > base_type;
 881     typedef typename base_type::storage_type storage_type;
 882
 883     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 884     {
 885         storage_type original, result;
 886         base_type::fence_before(order);
 887         __asm__ __volatile__
 888         (
 889             "1:\n\t"
 890             "lwarx %0,%y2\n\t"
 891             "add %1,%0,%3\n\t"
 892             "rlwinm %1, %1, 0, 0xffff\n\t"
 893             "stwcx. %1,%y2\n\t"
 894             "bne- 1b\n\t"
 895             : "=&b" (original), "=&b" (result), "+Z" (storage)
 896             : "b" (v)
 897             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 898         );
 899         base_type::fence_after(order);
 900         return original;
 901     }
 902
 903     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 904     {
 905         storage_type original, result;
 906         base_type::fence_before(order);
 907         __asm__ __volatile__
 908         (
 909             "1:\n\t"
 910             "lwarx %0,%y2\n\t"
 911             "sub %1,%0,%3\n\t"
 912             "rlwinm %1, %1, 0, 0xffff\n\t"
 913             "stwcx. %1,%y2\n\t"
 914             "bne- 1b\n\t"
 915             : "=&b" (original), "=&b" (result), "+Z" (storage)
 916             : "b" (v)
 917             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 918         );
 919         base_type::fence_after(order);
 920         return original;
 921     }
 922 };
 923
 924 template< bool Interprocess >
 925 struct core_arch_operations< 2u, true, Interprocess > :
 926     public core_arch_operations< 4u, true, Interprocess >
 927 {
 928     typedef core_arch_operations< 4u, true, Interprocess > base_type;
 929     typedef typename base_type::storage_type storage_type;
 930
 931     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 932     {
 933         storage_type original, result;
 934         base_type::fence_before(order);
 935         __asm__ __volatile__
 936         (
 937             "1:\n\t"
 938             "lwarx %0,%y2\n\t"
 939             "add %1,%0,%3\n\t"
 940             "extsh %1, %1\n\t"
 941             "stwcx. %1,%y2\n\t"
 942             "bne- 1b\n\t"
 943             : "=&b" (original), "=&b" (result), "+Z" (storage)
 944             : "b" (v)
 945             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 946         );
 947         base_type::fence_after(order);
 948         return original;
 949     }
 950
 951     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 952     {
 953         storage_type original, result;
 954         base_type::fence_before(order);
 955         __asm__ __volatile__
 956         (
 957             "1:\n\t"
 958             "lwarx %0,%y2\n\t"
 959             "sub %1,%0,%3\n\t"
 960             "extsh %1, %1\n\t"
 961             "stwcx. %1,%y2\n\t"
 962             "bne- 1b\n\t"
 963             : "=&b" (original), "=&b" (result), "+Z" (storage)
 964             : "b" (v)
 965             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 966         );
 967         base_type::fence_after(order);
 968         return original;
 969     }
 970 };
 971
 972 #endif // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LHARX_STHCX)
 973
 974 #if defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LDARX_STDCX)
 975
 976 template< bool Signed, bool Interprocess >
 977 struct core_arch_operations< 8u, Signed, Interprocess > :
 978     public core_arch_operations_gcc_ppc_base
 979 {
 980     typedef typename storage_traits< 8u >::type storage_type;
 981
 982     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 8u;
 983     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 8u;
 984     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
 985     static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
 986
 987     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 988     {
 989         fence_before(order);
 990         __asm__ __volatile__
 991         (
 992             "std %1, %0\n\t"
 993             : "+m" (storage)
 994             : "r" (v)
 995         );
 996     }
 997
 998     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
 999     {
1000         storage_type v;
1001         if (order == memory_order_seq_cst)
1002             __asm__ __volatile__ ("sync" ::: "memory");
1003         if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u)
1004         {
1005             __asm__ __volatile__
1006             (
1007                 "ld %0, %1\n\t"
1008                 "cmpd %0, %0\n\t"
1009                 "bne- 1f\n\t"
1010                 "1:\n\t"
1011                 "isync\n\t"
1012                 : "=&b" (v)
1013                 : "m" (storage)
1014                 : "cr0", "memory"
1015             );
1016         }
1017         else
1018         {
1019             __asm__ __volatile__
1020             (
1021                 "ld %0, %1\n\t"
1022                 : "=&b" (v)
1023                 : "m" (storage)
1024             );
1025         }
1026         return v;
1027     }
1028
1029     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1030     {
1031         storage_type original;
1032         fence_before(order);
1033         __asm__ __volatile__
1034         (
1035             "1:\n\t"
1036             "ldarx %0,%y1\n\t"
1037             "stdcx. %2,%y1\n\t"
1038             "bne- 1b\n\t"
1039             : "=&b" (original), "+Z" (storage)
1040             : "b" (v)
1041             : "cr0"
1042         );
1043         fence_after(order);
1044         return original;
1045     }
1046
1047     static BOOST_FORCEINLINE bool compare_exchange_weak(
1048         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
1049     {
1050         int success;
1051         fence_before(success_order);
1052         __asm__ __volatile__
1053         (
1054             "li %1, 0\n\t"
1055             "ldarx %0,%y2\n\t"
1056             "cmpd %0, %3\n\t"
1057             "bne- 1f\n\t"
1058             "stdcx. %4,%y2\n\t"
1059             "bne- 1f\n\t"
1060             "li %1, 1\n\t"
1061             "1:"
1062             : "=&b" (expected), "=&b" (success), "+Z" (storage)
1063             : "b" (expected), "b" (desired)
1064             : "cr0"
1065         );
1066         if (success)
1067             fence_after(success_order);
1068         else
1069             fence_after(failure_order);
1070         return !!success;
1071     }
1072
1073     static BOOST_FORCEINLINE bool compare_exchange_strong(
1074         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
1075     {
1076         int success;
1077         fence_before(success_order);
1078         __asm__ __volatile__
1079         (
1080             "li %1, 0\n\t"
1081             "0: ldarx %0,%y2\n\t"
1082             "cmpd %0, %3\n\t"
1083             "bne- 1f\n\t"
1084             "stdcx. %4,%y2\n\t"
1085             "bne- 0b\n\t"
1086             "li %1, 1\n\t"
1087             "1:\n\t"
1088             : "=&b" (expected), "=&b" (success), "+Z" (storage)
1089             : "b" (expected), "b" (desired)
1090             : "cr0"
1091         );
1092         if (success)
1093             fence_after(success_order);
1094         else
1095             fence_after(failure_order);
1096         return !!success;
1097     }
1098
1099     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1100     {
1101         storage_type original, result;
1102         fence_before(order);
1103         __asm__ __volatile__
1104         (
1105             "1:\n\t"
1106             "ldarx %0,%y2\n\t"
1107             "add %1,%0,%3\n\t"
1108             "stdcx. %1,%y2\n\t"
1109             "bne- 1b\n\t"
1110             : "=&b" (original), "=&b" (result), "+Z" (storage)
1111             : "b" (v)
1112             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1113         );
1114         fence_after(order);
1115         return original;
1116     }
1117
1118     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1119     {
1120         storage_type original, result;
1121         fence_before(order);
1122         __asm__ __volatile__
1123         (
1124             "1:\n\t"
1125             "ldarx %0,%y2\n\t"
1126             "sub %1,%0,%3\n\t"
1127             "stdcx. %1,%y2\n\t"
1128             "bne- 1b\n\t"
1129             : "=&b" (original), "=&b" (result), "+Z" (storage)
1130             : "b" (v)
1131             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1132         );
1133         fence_after(order);
1134         return original;
1135     }
1136
1137     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1138     {
1139         storage_type original, result;
1140         fence_before(order);
1141         __asm__ __volatile__
1142         (
1143             "1:\n\t"
1144             "ldarx %0,%y2\n\t"
1145             "and %1,%0,%3\n\t"
1146             "stdcx. %1,%y2\n\t"
1147             "bne- 1b\n\t"
1148             : "=&b" (original), "=&b" (result), "+Z" (storage)
1149             : "b" (v)
1150             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1151         );
1152         fence_after(order);
1153         return original;
1154     }
1155
1156     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1157     {
1158         storage_type original, result;
1159         fence_before(order);
1160         __asm__ __volatile__
1161         (
1162             "1:\n\t"
1163             "ldarx %0,%y2\n\t"
1164             "or %1,%0,%3\n\t"
1165             "stdcx. %1,%y2\n\t"
1166             "bne- 1b\n\t"
1167             : "=&b" (original), "=&b" (result), "+Z" (storage)
1168             : "b" (v)
1169             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1170         );
1171         fence_after(order);
1172         return original;
1173     }
1174
1175     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1176     {
1177         storage_type original, result;
1178         fence_before(order);
1179         __asm__ __volatile__
1180         (
1181             "1:\n\t"
1182             "ldarx %0,%y2\n\t"
1183             "xor %1,%0,%3\n\t"
1184             "stdcx. %1,%y2\n\t"
1185             "bne- 1b\n\t"
1186             : "=&b" (original), "=&b" (result), "+Z" (storage)
1187             : "b" (v)
1188             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1189         );
1190         fence_after(order);
1191         return original;
1192     }
1193
1194     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1195     {
1196         return !!exchange(storage, (storage_type)1, order);
1197     }
1198
1199     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1200     {
1201         store(storage, (storage_type)0, order);
1202     }
1203 };
1204
1205 #endif // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LDARX_STDCX)
1206
1207 } // namespace detail
1208 } // namespace atomics
1209 } // namespace boost
1210
1211 #include <boost/atomic/detail/footer.hpp>
1212
1213 #endif // BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_PPC_HPP_INCLUDED_