ceph/src/boost/boost/atomic/detail/ops_gcc_ppc.hpp

   1 /*
   2  * Distributed under the Boost Software License, Version 1.0.
   3  * (See accompanying file LICENSE_1_0.txt or copy at
   4  * http://www.boost.org/LICENSE_1_0.txt)
   5  *
   6  * Copyright (c) 2009 Helge Bahmann
   7  * Copyright (c) 2013 Tim Blechmann
   8  * Copyright (c) 2014 Andrey Semashev
   9  */
  10 /*!
  11  * \file   atomic/detail/ops_gcc_ppc.hpp
  12  *
  13  * This header contains implementation of the \c operations template.
  14  */
  15
  16 #ifndef BOOST_ATOMIC_DETAIL_OPS_GCC_PPC_HPP_INCLUDED_
  17 #define BOOST_ATOMIC_DETAIL_OPS_GCC_PPC_HPP_INCLUDED_
  18
  19 #include <cstddef>
  20 #include <boost/memory_order.hpp>
  21 #include <boost/atomic/detail/config.hpp>
  22 #include <boost/atomic/detail/storage_type.hpp>
  23 #include <boost/atomic/detail/operations_fwd.hpp>
  24 #include <boost/atomic/detail/ops_gcc_ppc_common.hpp>
  25 #include <boost/atomic/capabilities.hpp>
  26
  27 #ifdef BOOST_HAS_PRAGMA_ONCE
  28 #pragma once
  29 #endif
  30
  31 namespace boost {
  32 namespace atomics {
  33 namespace detail {
  34
  35 // The implementation below uses information from this document:
  36 // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2010.02.19a.html
  37
  38 /*
  39     Refer to: Motorola: "Programming Environments Manual for 32-Bit
  40     Implementations of the PowerPC Architecture", Appendix E:
  41     "Synchronization Programming Examples" for an explanation of what is
  42     going on here (can be found on the web at various places by the
  43     name "MPCFPE32B.pdf", Google is your friend...)
  44
  45     Most of the atomic operations map to instructions in a relatively
  46     straight-forward fashion, but "load"s may at first glance appear
  47     a bit strange as they map to:
  48
  49             lwz %rX, addr
  50             cmpw %rX, %rX
  51             bne- 1f
  52         1:
  53
  54     That is, the CPU is forced to perform a branch that "formally" depends
  55     on the value retrieved from memory. This scheme has an overhead of
  56     about 1-2 clock cycles per load, but it allows to map "acquire" to
  57     the "isync" instruction instead of "sync" uniformly and for all type
  58     of atomic operations. Since "isync" has a cost of about 15 clock
  59     cycles, while "sync" hast a cost of about 50 clock cycles, the small
  60     penalty to atomic loads more than compensates for this.
  61
  62     Byte- and halfword-sized atomic values are implemented in two ways.
  63     When 8 and 16-bit instructions are available (in Power8 and later),
  64     they are used. Otherwise operations are realized by encoding the
  65     value to be represented into a word, performing sign/zero extension
  66     as appropriate. This means that after add/sub operations the value
  67     needs fixing up to accurately preserve the wrap-around semantic of
  68     the smaller type. (Nothing special needs to be done for the bit-wise
  69     and the "exchange type" operators as the compiler already sees to
  70     it that values carried in registers are extended appropriately and
  71     everything falls into place naturally).
  72
  73     The register constraint "b"  instructs gcc to use any register
  74     except r0; this is sometimes required because the encoding for
  75     r0 is used to signify "constant zero" in a number of instructions,
  76     making r0 unusable in this place. For simplicity this constraint
  77     is used everywhere since I am to lazy to look this up on a
  78     per-instruction basis, and ppc has enough registers for this not
  79     to pose a problem.
  80 */
  81
  82 template< bool Signed >
  83 struct operations< 4u, Signed > :
  84     public gcc_ppc_operations_base
  85 {
  86     typedef typename make_storage_type< 4u, Signed >::type storage_type;
  87     typedef typename make_storage_type< 4u, Signed >::aligned aligned_storage_type;
  88
  89     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 4u;
  90     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
  91
  92     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  93     {
  94         fence_before(order);
  95         __asm__ __volatile__
  96         (
  97             "stw %1, %0\n\t"
  98             : "+m" (storage)
  99             : "r" (v)
 100         );
 101     }
 102
 103     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
 104     {
 105         storage_type v;
 106         if (order == memory_order_seq_cst)
 107             __asm__ __volatile__ ("sync" ::: "memory");
 108         if ((order & (memory_order_consume | memory_order_acquire)) != 0)
 109         {
 110             __asm__ __volatile__
 111             (
 112                 "lwz %0, %1\n\t"
 113                 "cmpw %0, %0\n\t"
 114                 "bne- 1f\n\t"
 115                 "1:\n\t"
 116                 "isync\n\t"
 117                 : "=&r" (v)
 118                 : "m" (storage)
 119                 : "cr0", "memory"
 120             );
 121         }
 122         else
 123         {
 124             __asm__ __volatile__
 125             (
 126                 "lwz %0, %1\n\t"
 127                 : "=&r" (v)
 128                 : "m" (storage)
 129             );
 130         }
 131         return v;
 132     }
 133
 134     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 135     {
 136         storage_type original;
 137         fence_before(order);
 138         __asm__ __volatile__
 139         (
 140             "1:\n\t"
 141             "lwarx %0,%y1\n\t"
 142             "stwcx. %2,%y1\n\t"
 143             "bne- 1b\n\t"
 144             : "=&b" (original), "+Z" (storage)
 145             : "b" (v)
 146             : "cr0"
 147         );
 148         fence_after(order);
 149         return original;
 150     }
 151
 152     static BOOST_FORCEINLINE bool compare_exchange_weak(
 153         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
 154     {
 155         int success;
 156         fence_before(success_order);
 157         __asm__ __volatile__
 158         (
 159             "li %1, 0\n\t"
 160             "lwarx %0,%y2\n\t"
 161             "cmpw %0, %3\n\t"
 162             "bne- 1f\n\t"
 163             "stwcx. %4,%y2\n\t"
 164             "bne- 1f\n\t"
 165             "li %1, 1\n\t"
 166             "1:\n\t"
 167             : "=&b" (expected), "=&b" (success), "+Z" (storage)
 168             : "b" (expected), "b" (desired)
 169             : "cr0"
 170         );
 171         if (success)
 172             fence_after(success_order);
 173         else
 174             fence_after(failure_order);
 175         return !!success;
 176     }
 177
 178     static BOOST_FORCEINLINE bool compare_exchange_strong(
 179         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
 180     {
 181         int success;
 182         fence_before(success_order);
 183         __asm__ __volatile__
 184         (
 185             "li %1, 0\n\t"
 186             "0: lwarx %0,%y2\n\t"
 187             "cmpw %0, %3\n\t"
 188             "bne- 1f\n\t"
 189             "stwcx. %4,%y2\n\t"
 190             "bne- 0b\n\t"
 191             "li %1, 1\n\t"
 192             "1:\n\t"
 193             : "=&b" (expected), "=&b" (success), "+Z" (storage)
 194             : "b" (expected), "b" (desired)
 195             : "cr0"
 196         );
 197         if (success)
 198             fence_after(success_order);
 199         else
 200             fence_after(failure_order);
 201         return !!success;
 202     }
 203
 204     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 205     {
 206         storage_type original, tmp;
 207         fence_before(order);
 208         __asm__ __volatile__
 209         (
 210             "1:\n\t"
 211             "lwarx %0,%y2\n\t"
 212             "add %1,%0,%3\n\t"
 213             "stwcx. %1,%y2\n\t"
 214             "bne- 1b\n\t"
 215             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 216             : "b" (v)
 217             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 218         );
 219         fence_after(order);
 220         return original;
 221     }
 222
 223     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 224     {
 225         storage_type original, tmp;
 226         fence_before(order);
 227         __asm__ __volatile__
 228         (
 229             "1:\n\t"
 230             "lwarx %0,%y2\n\t"
 231             "sub %1,%0,%3\n\t"
 232             "stwcx. %1,%y2\n\t"
 233             "bne- 1b\n\t"
 234             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 235             : "b" (v)
 236             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 237         );
 238         fence_after(order);
 239         return original;
 240     }
 241
 242     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 243     {
 244         storage_type original, tmp;
 245         fence_before(order);
 246         __asm__ __volatile__
 247         (
 248             "1:\n\t"
 249             "lwarx %0,%y2\n\t"
 250             "and %1,%0,%3\n\t"
 251             "stwcx. %1,%y2\n\t"
 252             "bne- 1b\n\t"
 253             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 254             : "b" (v)
 255             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 256         );
 257         fence_after(order);
 258         return original;
 259     }
 260
 261     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 262     {
 263         storage_type original, tmp;
 264         fence_before(order);
 265         __asm__ __volatile__
 266         (
 267             "1:\n\t"
 268             "lwarx %0,%y2\n\t"
 269             "or %1,%0,%3\n\t"
 270             "stwcx. %1,%y2\n\t"
 271             "bne- 1b\n\t"
 272             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 273             : "b" (v)
 274             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 275         );
 276         fence_after(order);
 277         return original;
 278     }
 279
 280     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 281     {
 282         storage_type original, tmp;
 283         fence_before(order);
 284         __asm__ __volatile__
 285         (
 286             "1:\n\t"
 287             "lwarx %0,%y2\n\t"
 288             "xor %1,%0,%3\n\t"
 289             "stwcx. %1,%y2\n\t"
 290             "bne- 1b\n\t"
 291             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 292             : "b" (v)
 293             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 294         );
 295         fence_after(order);
 296         return original;
 297     }
 298
 299     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 300     {
 301         return !!exchange(storage, (storage_type)1, order);
 302     }
 303
 304     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 305     {
 306         store(storage, 0, order);
 307     }
 308 };
 309
 310 #if defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LBARX_STBCX)
 311
 312 template< bool Signed >
 313 struct operations< 1u, Signed > :
 314     public gcc_ppc_operations_base
 315 {
 316     typedef typename make_storage_type< 1u, Signed >::type storage_type;
 317     typedef typename make_storage_type< 1u, Signed >::aligned aligned_storage_type;
 318     typedef typename make_storage_type< 1u, false >::type unsigned_storage_type;
 319
 320     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 1u;
 321     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
 322
 323     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 324     {
 325         fence_before(order);
 326         __asm__ __volatile__
 327         (
 328             "stb %1, %0\n\t"
 329             : "+m" (storage)
 330             : "r" (v)
 331         );
 332     }
 333
 334     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
 335     {
 336         storage_type v;
 337         if (order == memory_order_seq_cst)
 338             __asm__ __volatile__ ("sync" ::: "memory");
 339         if ((order & (memory_order_consume | memory_order_acquire)) != 0)
 340         {
 341             __asm__ __volatile__
 342             (
 343                 "lbz %0, %1\n\t"
 344                 "cmpw %0, %0\n\t"
 345                 "bne- 1f\n\t"
 346                 "1:\n\t"
 347                 "isync\n\t"
 348                 : "=&r" (v)
 349                 : "m" (storage)
 350                 : "cr0", "memory"
 351             );
 352         }
 353         else
 354         {
 355             __asm__ __volatile__
 356             (
 357                 "lbz %0, %1\n\t"
 358                 : "=&r" (v)
 359                 : "m" (storage)
 360             );
 361         }
 362         return v;
 363     }
 364
 365     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 366     {
 367         storage_type original;
 368         fence_before(order);
 369         __asm__ __volatile__
 370         (
 371             "1:\n\t"
 372             "lbarx %0,%y1\n\t"
 373             "stbcx. %2,%y1\n\t"
 374             "bne- 1b\n\t"
 375             : "=&b" (original), "+Z" (storage)
 376             : "b" (v)
 377             : "cr0"
 378         );
 379         fence_after(order);
 380         return original;
 381     }
 382
 383     static BOOST_FORCEINLINE bool compare_exchange_weak(
 384         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
 385     {
 386         int success;
 387         fence_before(success_order);
 388         __asm__ __volatile__
 389         (
 390             "li %1, 0\n\t"
 391             "lbarx %0,%y2\n\t"
 392             "cmpw %0, %3\n\t"
 393             "bne- 1f\n\t"
 394             "stbcx. %4,%y2\n\t"
 395             "bne- 1f\n\t"
 396             "li %1, 1\n\t"
 397             "1:\n\t"
 398             : "=&b" (expected), "=&b" (success), "+Z" (storage)
 399             : "b" ((unsigned_storage_type)expected), "b" (desired)
 400             : "cr0"
 401         );
 402         if (success)
 403             fence_after(success_order);
 404         else
 405             fence_after(failure_order);
 406         return !!success;
 407     }
 408
 409     static BOOST_FORCEINLINE bool compare_exchange_strong(
 410         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
 411     {
 412         int success;
 413         fence_before(success_order);
 414         __asm__ __volatile__
 415         (
 416             "li %1, 0\n\t"
 417             "0: lbarx %0,%y2\n\t"
 418             "cmpw %0, %3\n\t"
 419             "bne- 1f\n\t"
 420             "stbcx. %4,%y2\n\t"
 421             "bne- 0b\n\t"
 422             "li %1, 1\n\t"
 423             "1:\n\t"
 424             : "=&b" (expected), "=&b" (success), "+Z" (storage)
 425             : "b" ((unsigned_storage_type)expected), "b" (desired)
 426             : "cr0"
 427         );
 428         if (success)
 429             fence_after(success_order);
 430         else
 431             fence_after(failure_order);
 432         return !!success;
 433     }
 434
 435     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 436     {
 437         storage_type original, tmp;
 438         fence_before(order);
 439         __asm__ __volatile__
 440         (
 441             "1:\n\t"
 442             "lbarx %0,%y2\n\t"
 443             "add %1,%0,%3\n\t"
 444             "stbcx. %1,%y2\n\t"
 445             "bne- 1b\n\t"
 446             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 447             : "b" (v)
 448             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 449         );
 450         fence_after(order);
 451         return original;
 452     }
 453
 454     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 455     {
 456         storage_type original, tmp;
 457         fence_before(order);
 458         __asm__ __volatile__
 459         (
 460             "1:\n\t"
 461             "lbarx %0,%y2\n\t"
 462             "sub %1,%0,%3\n\t"
 463             "stbcx. %1,%y2\n\t"
 464             "bne- 1b\n\t"
 465             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 466             : "b" (v)
 467             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 468         );
 469         fence_after(order);
 470         return original;
 471     }
 472
 473     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 474     {
 475         storage_type original, tmp;
 476         fence_before(order);
 477         __asm__ __volatile__
 478         (
 479             "1:\n\t"
 480             "lbarx %0,%y2\n\t"
 481             "and %1,%0,%3\n\t"
 482             "stbcx. %1,%y2\n\t"
 483             "bne- 1b\n\t"
 484             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 485             : "b" (v)
 486             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 487         );
 488         fence_after(order);
 489         return original;
 490     }
 491
 492     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 493     {
 494         storage_type original, tmp;
 495         fence_before(order);
 496         __asm__ __volatile__
 497         (
 498             "1:\n\t"
 499             "lbarx %0,%y2\n\t"
 500             "or %1,%0,%3\n\t"
 501             "stbcx. %1,%y2\n\t"
 502             "bne- 1b\n\t"
 503             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 504             : "b" (v)
 505             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 506         );
 507         fence_after(order);
 508         return original;
 509     }
 510
 511     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 512     {
 513         storage_type original, tmp;
 514         fence_before(order);
 515         __asm__ __volatile__
 516         (
 517             "1:\n\t"
 518             "lbarx %0,%y2\n\t"
 519             "xor %1,%0,%3\n\t"
 520             "stbcx. %1,%y2\n\t"
 521             "bne- 1b\n\t"
 522             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 523             : "b" (v)
 524             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 525         );
 526         fence_after(order);
 527         return original;
 528     }
 529
 530     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 531     {
 532         return !!exchange(storage, (storage_type)1, order);
 533     }
 534
 535     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 536     {
 537         store(storage, 0, order);
 538     }
 539 };
 540
 541 #else // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LBARX_STBCX)
 542
 543 template< >
 544 struct operations< 1u, false > :
 545     public operations< 4u, false >
 546 {
 547     typedef operations< 4u, false > base_type;
 548     typedef base_type::storage_type storage_type;
 549
 550     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 551     {
 552         storage_type original, tmp;
 553         fence_before(order);
 554         __asm__ __volatile__
 555         (
 556             "1:\n\t"
 557             "lwarx %0,%y2\n\t"
 558             "add %1,%0,%3\n\t"
 559             "rlwinm %1, %1, 0, 0xff\n\t"
 560             "stwcx. %1,%y2\n\t"
 561             "bne- 1b\n\t"
 562             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 563             : "b" (v)
 564             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 565         );
 566         fence_after(order);
 567         return original;
 568     }
 569
 570     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 571     {
 572         storage_type original, tmp;
 573         fence_before(order);
 574         __asm__ __volatile__
 575         (
 576             "1:\n\t"
 577             "lwarx %0,%y2\n\t"
 578             "sub %1,%0,%3\n\t"
 579             "rlwinm %1, %1, 0, 0xff\n\t"
 580             "stwcx. %1,%y2\n\t"
 581             "bne- 1b\n\t"
 582             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 583             : "b" (v)
 584             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 585         );
 586         fence_after(order);
 587         return original;
 588     }
 589 };
 590
 591 template< >
 592 struct operations< 1u, true > :
 593     public operations< 4u, true >
 594 {
 595     typedef operations< 4u, true > base_type;
 596     typedef base_type::storage_type storage_type;
 597
 598     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 599     {
 600         storage_type original, tmp;
 601         fence_before(order);
 602         __asm__ __volatile__
 603         (
 604             "1:\n\t"
 605             "lwarx %0,%y2\n\t"
 606             "add %1,%0,%3\n\t"
 607             "extsb %1, %1\n\t"
 608             "stwcx. %1,%y2\n\t"
 609             "bne- 1b\n\t"
 610             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 611             : "b" (v)
 612             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 613         );
 614         fence_after(order);
 615         return original;
 616     }
 617
 618     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 619     {
 620         storage_type original, tmp;
 621         fence_before(order);
 622         __asm__ __volatile__
 623         (
 624             "1:\n\t"
 625             "lwarx %0,%y2\n\t"
 626             "sub %1,%0,%3\n\t"
 627             "extsb %1, %1\n\t"
 628             "stwcx. %1,%y2\n\t"
 629             "bne- 1b\n\t"
 630             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 631             : "b" (v)
 632             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 633         );
 634         fence_after(order);
 635         return original;
 636     }
 637 };
 638
 639 #endif // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LBARX_STBCX)
 640
 641 #if defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LHARX_STHCX)
 642
 643 template< bool Signed >
 644 struct operations< 2u, Signed > :
 645     public gcc_ppc_operations_base
 646 {
 647     typedef typename make_storage_type< 2u, Signed >::type storage_type;
 648     typedef typename make_storage_type< 2u, Signed >::aligned aligned_storage_type;
 649     typedef typename make_storage_type< 2u, false >::type unsigned_storage_type;
 650
 651     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 2u;
 652     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
 653
 654     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 655     {
 656         fence_before(order);
 657         __asm__ __volatile__
 658         (
 659             "sth %1, %0\n\t"
 660             : "+m" (storage)
 661             : "r" (v)
 662         );
 663     }
 664
 665     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
 666     {
 667         storage_type v;
 668         if (order == memory_order_seq_cst)
 669             __asm__ __volatile__ ("sync" ::: "memory");
 670         if ((order & (memory_order_consume | memory_order_acquire)) != 0)
 671         {
 672             __asm__ __volatile__
 673             (
 674                 "lhz %0, %1\n\t"
 675                 "cmpw %0, %0\n\t"
 676                 "bne- 1f\n\t"
 677                 "1:\n\t"
 678                 "isync\n\t"
 679                 : "=&r" (v)
 680                 : "m" (storage)
 681                 : "cr0", "memory"
 682             );
 683         }
 684         else
 685         {
 686             __asm__ __volatile__
 687             (
 688                 "lhz %0, %1\n\t"
 689                 : "=&r" (v)
 690                 : "m" (storage)
 691             );
 692         }
 693         return v;
 694     }
 695
 696     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 697     {
 698         storage_type original;
 699         fence_before(order);
 700         __asm__ __volatile__
 701         (
 702             "1:\n\t"
 703             "lharx %0,%y1\n\t"
 704             "sthcx. %2,%y1\n\t"
 705             "bne- 1b\n\t"
 706             : "=&b" (original), "+Z" (storage)
 707             : "b" (v)
 708             : "cr0"
 709         );
 710         fence_after(order);
 711         return original;
 712     }
 713
 714     static BOOST_FORCEINLINE bool compare_exchange_weak(
 715         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
 716     {
 717         int success;
 718         fence_before(success_order);
 719         __asm__ __volatile__
 720         (
 721             "li %1, 0\n\t"
 722             "lharx %0,%y2\n\t"
 723             "cmpw %0, %3\n\t"
 724             "bne- 1f\n\t"
 725             "sthcx. %4,%y2\n\t"
 726             "bne- 1f\n\t"
 727             "li %1, 1\n\t"
 728             "1:\n\t"
 729             : "=&b" (expected), "=&b" (success), "+Z" (storage)
 730             : "b" ((unsigned_storage_type)expected), "b" (desired)
 731             : "cr0"
 732         );
 733         if (success)
 734             fence_after(success_order);
 735         else
 736             fence_after(failure_order);
 737         return !!success;
 738     }
 739
 740     static BOOST_FORCEINLINE bool compare_exchange_strong(
 741         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
 742     {
 743         int success;
 744         fence_before(success_order);
 745         __asm__ __volatile__
 746         (
 747             "li %1, 0\n\t"
 748             "0: lharx %0,%y2\n\t"
 749             "cmpw %0, %3\n\t"
 750             "bne- 1f\n\t"
 751             "sthcx. %4,%y2\n\t"
 752             "bne- 0b\n\t"
 753             "li %1, 1\n\t"
 754             "1:\n\t"
 755             : "=&b" (expected), "=&b" (success), "+Z" (storage)
 756             : "b" ((unsigned_storage_type)expected), "b" (desired)
 757             : "cr0"
 758         );
 759         if (success)
 760             fence_after(success_order);
 761         else
 762             fence_after(failure_order);
 763         return !!success;
 764     }
 765
 766     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 767     {
 768         storage_type original, tmp;
 769         fence_before(order);
 770         __asm__ __volatile__
 771         (
 772             "1:\n\t"
 773             "lharx %0,%y2\n\t"
 774             "add %1,%0,%3\n\t"
 775             "sthcx. %1,%y2\n\t"
 776             "bne- 1b\n\t"
 777             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 778             : "b" (v)
 779             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 780         );
 781         fence_after(order);
 782         return original;
 783     }
 784
 785     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 786     {
 787         storage_type original, tmp;
 788         fence_before(order);
 789         __asm__ __volatile__
 790         (
 791             "1:\n\t"
 792             "lharx %0,%y2\n\t"
 793             "sub %1,%0,%3\n\t"
 794             "sthcx. %1,%y2\n\t"
 795             "bne- 1b\n\t"
 796             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 797             : "b" (v)
 798             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 799         );
 800         fence_after(order);
 801         return original;
 802     }
 803
 804     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 805     {
 806         storage_type original, tmp;
 807         fence_before(order);
 808         __asm__ __volatile__
 809         (
 810             "1:\n\t"
 811             "lharx %0,%y2\n\t"
 812             "and %1,%0,%3\n\t"
 813             "sthcx. %1,%y2\n\t"
 814             "bne- 1b\n\t"
 815             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 816             : "b" (v)
 817             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 818         );
 819         fence_after(order);
 820         return original;
 821     }
 822
 823     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 824     {
 825         storage_type original, tmp;
 826         fence_before(order);
 827         __asm__ __volatile__
 828         (
 829             "1:\n\t"
 830             "lharx %0,%y2\n\t"
 831             "or %1,%0,%3\n\t"
 832             "sthcx. %1,%y2\n\t"
 833             "bne- 1b\n\t"
 834             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 835             : "b" (v)
 836             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 837         );
 838         fence_after(order);
 839         return original;
 840     }
 841
 842     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 843     {
 844         storage_type original, tmp;
 845         fence_before(order);
 846         __asm__ __volatile__
 847         (
 848             "1:\n\t"
 849             "lharx %0,%y2\n\t"
 850             "xor %1,%0,%3\n\t"
 851             "sthcx. %1,%y2\n\t"
 852             "bne- 1b\n\t"
 853             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 854             : "b" (v)
 855             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 856         );
 857         fence_after(order);
 858         return original;
 859     }
 860
 861     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 862     {
 863         return !!exchange(storage, (storage_type)1, order);
 864     }
 865
 866     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 867     {
 868         store(storage, 0, order);
 869     }
 870 };
 871
 872 #else // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LHARX_STHCX)
 873
 874 template< >
 875 struct operations< 2u, false > :
 876     public operations< 4u, false >
 877 {
 878     typedef operations< 4u, false > base_type;
 879     typedef base_type::storage_type storage_type;
 880
 881     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 882     {
 883         storage_type original, tmp;
 884         fence_before(order);
 885         __asm__ __volatile__
 886         (
 887             "1:\n\t"
 888             "lwarx %0,%y2\n\t"
 889             "add %1,%0,%3\n\t"
 890             "rlwinm %1, %1, 0, 0xffff\n\t"
 891             "stwcx. %1,%y2\n\t"
 892             "bne- 1b\n\t"
 893             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 894             : "b" (v)
 895             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 896         );
 897         fence_after(order);
 898         return original;
 899     }
 900
 901     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 902     {
 903         storage_type original, tmp;
 904         fence_before(order);
 905         __asm__ __volatile__
 906         (
 907             "1:\n\t"
 908             "lwarx %0,%y2\n\t"
 909             "sub %1,%0,%3\n\t"
 910             "rlwinm %1, %1, 0, 0xffff\n\t"
 911             "stwcx. %1,%y2\n\t"
 912             "bne- 1b\n\t"
 913             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 914             : "b" (v)
 915             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 916         );
 917         fence_after(order);
 918         return original;
 919     }
 920 };
 921
 922 template< >
 923 struct operations< 2u, true > :
 924     public operations< 4u, true >
 925 {
 926     typedef operations< 4u, true > base_type;
 927     typedef base_type::storage_type storage_type;
 928
 929     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 930     {
 931         storage_type original, tmp;
 932         fence_before(order);
 933         __asm__ __volatile__
 934         (
 935             "1:\n\t"
 936             "lwarx %0,%y2\n\t"
 937             "add %1,%0,%3\n\t"
 938             "extsh %1, %1\n\t"
 939             "stwcx. %1,%y2\n\t"
 940             "bne- 1b\n\t"
 941             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 942             : "b" (v)
 943             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 944         );
 945         fence_after(order);
 946         return original;
 947     }
 948
 949     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 950     {
 951         storage_type original, tmp;
 952         fence_before(order);
 953         __asm__ __volatile__
 954         (
 955             "1:\n\t"
 956             "lwarx %0,%y2\n\t"
 957             "sub %1,%0,%3\n\t"
 958             "extsh %1, %1\n\t"
 959             "stwcx. %1,%y2\n\t"
 960             "bne- 1b\n\t"
 961             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
 962             : "b" (v)
 963             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 964         );
 965         fence_after(order);
 966         return original;
 967     }
 968 };
 969
 970 #endif // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LHARX_STHCX)
 971
 972 #if defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LDARX_STDCX)
 973
 974 template< bool Signed >
 975 struct operations< 8u, Signed > :
 976     public gcc_ppc_operations_base
 977 {
 978     typedef typename make_storage_type< 8u, Signed >::type storage_type;
 979     typedef typename make_storage_type< 8u, Signed >::aligned aligned_storage_type;
 980
 981     static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 8u;
 982     static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
 983
 984     static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
 985     {
 986         fence_before(order);
 987         __asm__ __volatile__
 988         (
 989             "std %1, %0\n\t"
 990             : "+m" (storage)
 991             : "r" (v)
 992         );
 993     }
 994
 995     static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
 996     {
 997         storage_type v;
 998         if (order == memory_order_seq_cst)
 999             __asm__ __volatile__ ("sync" ::: "memory");
1000         if ((order & (memory_order_consume | memory_order_acquire)) != 0)
1001         {
1002             __asm__ __volatile__
1003             (
1004                 "ld %0, %1\n\t"
1005                 "cmpd %0, %0\n\t"
1006                 "bne- 1f\n\t"
1007                 "1:\n\t"
1008                 "isync\n\t"
1009                 : "=&b" (v)
1010                 : "m" (storage)
1011                 : "cr0", "memory"
1012             );
1013         }
1014         else
1015         {
1016             __asm__ __volatile__
1017             (
1018                 "ld %0, %1\n\t"
1019                 : "=&b" (v)
1020                 : "m" (storage)
1021             );
1022         }
1023         return v;
1024     }
1025
1026     static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1027     {
1028         storage_type original;
1029         fence_before(order);
1030         __asm__ __volatile__
1031         (
1032             "1:\n\t"
1033             "ldarx %0,%y1\n\t"
1034             "stdcx. %2,%y1\n\t"
1035             "bne- 1b\n\t"
1036             : "=&b" (original), "+Z" (storage)
1037             : "b" (v)
1038             : "cr0"
1039         );
1040         fence_after(order);
1041         return original;
1042     }
1043
1044     static BOOST_FORCEINLINE bool compare_exchange_weak(
1045         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
1046     {
1047         int success;
1048         fence_before(success_order);
1049         __asm__ __volatile__
1050         (
1051             "li %1, 0\n\t"
1052             "ldarx %0,%y2\n\t"
1053             "cmpd %0, %3\n\t"
1054             "bne- 1f\n\t"
1055             "stdcx. %4,%y2\n\t"
1056             "bne- 1f\n\t"
1057             "li %1, 1\n\t"
1058             "1:"
1059             : "=&b" (expected), "=&b" (success), "+Z" (storage)
1060             : "b" (expected), "b" (desired)
1061             : "cr0"
1062         );
1063         if (success)
1064             fence_after(success_order);
1065         else
1066             fence_after(failure_order);
1067         return !!success;
1068     }
1069
1070     static BOOST_FORCEINLINE bool compare_exchange_strong(
1071         storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
1072     {
1073         int success;
1074         fence_before(success_order);
1075         __asm__ __volatile__
1076         (
1077             "li %1, 0\n\t"
1078             "0: ldarx %0,%y2\n\t"
1079             "cmpd %0, %3\n\t"
1080             "bne- 1f\n\t"
1081             "stdcx. %4,%y2\n\t"
1082             "bne- 0b\n\t"
1083             "li %1, 1\n\t"
1084             "1:\n\t"
1085             : "=&b" (expected), "=&b" (success), "+Z" (storage)
1086             : "b" (expected), "b" (desired)
1087             : "cr0"
1088         );
1089         if (success)
1090             fence_after(success_order);
1091         else
1092             fence_after(failure_order);
1093         return !!success;
1094     }
1095
1096     static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1097     {
1098         storage_type original, tmp;
1099         fence_before(order);
1100         __asm__ __volatile__
1101         (
1102             "1:\n\t"
1103             "ldarx %0,%y2\n\t"
1104             "add %1,%0,%3\n\t"
1105             "stdcx. %1,%y2\n\t"
1106             "bne- 1b\n\t"
1107             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
1108             : "b" (v)
1109             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1110         );
1111         fence_after(order);
1112         return original;
1113     }
1114
1115     static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1116     {
1117         storage_type original, tmp;
1118         fence_before(order);
1119         __asm__ __volatile__
1120         (
1121             "1:\n\t"
1122             "ldarx %0,%y2\n\t"
1123             "sub %1,%0,%3\n\t"
1124             "stdcx. %1,%y2\n\t"
1125             "bne- 1b\n\t"
1126             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
1127             : "b" (v)
1128             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1129         );
1130         fence_after(order);
1131         return original;
1132     }
1133
1134     static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1135     {
1136         storage_type original, tmp;
1137         fence_before(order);
1138         __asm__ __volatile__
1139         (
1140             "1:\n\t"
1141             "ldarx %0,%y2\n\t"
1142             "and %1,%0,%3\n\t"
1143             "stdcx. %1,%y2\n\t"
1144             "bne- 1b\n\t"
1145             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
1146             : "b" (v)
1147             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1148         );
1149         fence_after(order);
1150         return original;
1151     }
1152
1153     static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1154     {
1155         storage_type original, tmp;
1156         fence_before(order);
1157         __asm__ __volatile__
1158         (
1159             "1:\n\t"
1160             "ldarx %0,%y2\n\t"
1161             "or %1,%0,%3\n\t"
1162             "stdcx. %1,%y2\n\t"
1163             "bne- 1b\n\t"
1164             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
1165             : "b" (v)
1166             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1167         );
1168         fence_after(order);
1169         return original;
1170     }
1171
1172     static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1173     {
1174         storage_type original, tmp;
1175         fence_before(order);
1176         __asm__ __volatile__
1177         (
1178             "1:\n\t"
1179             "ldarx %0,%y2\n\t"
1180             "xor %1,%0,%3\n\t"
1181             "stdcx. %1,%y2\n\t"
1182             "bne- 1b\n\t"
1183             : "=&b" (original), "=&b" (tmp), "+Z" (storage)
1184             : "b" (v)
1185             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1186         );
1187         fence_after(order);
1188         return original;
1189     }
1190
1191     static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1192     {
1193         return !!exchange(storage, (storage_type)1, order);
1194     }
1195
1196     static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1197     {
1198         store(storage, 0, order);
1199     }
1200 };
1201
1202 #endif // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LDARX_STDCX)
1203
1204
1205 BOOST_FORCEINLINE void thread_fence(memory_order order) BOOST_NOEXCEPT
1206 {
1207     switch (order)
1208     {
1209     case memory_order_consume:
1210     case memory_order_acquire:
1211     case memory_order_release:
1212     case memory_order_acq_rel:
1213 #if defined(__powerpc64__) || defined(__PPC64__)
1214         __asm__ __volatile__ ("lwsync" ::: "memory");
1215         break;
1216 #endif
1217     case memory_order_seq_cst:
1218         __asm__ __volatile__ ("sync" ::: "memory");
1219         break;
1220     default:;
1221     }
1222 }
1223
1224 BOOST_FORCEINLINE void signal_fence(memory_order order) BOOST_NOEXCEPT
1225 {
1226     if (order != memory_order_relaxed)
1227 #if defined(__ibmxl__) || defined(__IBMCPP__)
1228         __fence();
1229 #else
1230         __asm__ __volatile__ ("" ::: "memory");
1231 #endif
1232 }
1233
1234 } // namespace detail
1235 } // namespace atomics
1236 } // namespace boost
1237
1238 #endif // BOOST_ATOMIC_DETAIL_OPS_GCC_PPC_HPP_INCLUDED_