ceph/src/boost/boost/atomic/detail/extra_ops_gcc_arm.hpp

   1 /*
   2  * Distributed under the Boost Software License, Version 1.0.
   3  * (See accompanying file LICENSE_1_0.txt or copy at
   4  * http://www.boost.org/LICENSE_1_0.txt)
   5  *
   6  * Copyright (c) 2017 Andrey Semashev
   7  */
   8 /*!
   9  * \file   atomic/detail/extra_ops_gcc_arm.hpp
  10  *
  11  * This header contains implementation of the extra atomic operations for ARM.
  12  */
  13
  14 #ifndef BOOST_ATOMIC_DETAIL_EXTRA_OPS_GCC_ARM_HPP_INCLUDED_
  15 #define BOOST_ATOMIC_DETAIL_EXTRA_OPS_GCC_ARM_HPP_INCLUDED_
  16
  17 #include <cstddef>
  18 #include <boost/cstdint.hpp>
  19 #include <boost/memory_order.hpp>
  20 #include <boost/atomic/detail/config.hpp>
  21 #include <boost/atomic/detail/platform.hpp>
  22 #include <boost/atomic/detail/storage_type.hpp>
  23 #include <boost/atomic/detail/extra_operations_fwd.hpp>
  24 #include <boost/atomic/detail/extra_ops_generic.hpp>
  25 #include <boost/atomic/detail/ops_gcc_arm_common.hpp>
  26 #include <boost/atomic/capabilities.hpp>
  27
  28 #ifdef BOOST_HAS_PRAGMA_ONCE
  29 #pragma once
  30 #endif
  31
  32 namespace boost {
  33 namespace atomics {
  34 namespace detail {
  35
  36 #if defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXB_STREXB)
  37
  38 template< typename Base, bool Signed >
  39 struct extra_operations< Base, 1u, Signed > :
  40     public generic_extra_operations< Base, 1u, Signed >
  41 {
  42     typedef generic_extra_operations< Base, 1u, Signed > base_type;
  43     typedef typename base_type::storage_type storage_type;
  44     typedef typename make_storage_type< 4u, Signed >::type extended_storage_type;
  45
  46     static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  47     {
  48         gcc_arm_operations_base::fence_before(order);
  49         uint32_t tmp;
  50         extended_storage_type original, result;
  51         __asm__ __volatile__
  52         (
  53             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
  54             "1:\n"
  55             "ldrexb   %[original], %[storage]\n"           // original = *(&storage)
  56             "rsb      %[result], %[original], #0\n"        // result = 0 - original
  57             "strexb   %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
  58             "teq      %[tmp], #0\n"                        // flags = tmp==0
  59             "bne      1b\n"                                // if (!flags.equal) goto retry
  60             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
  61             : [original] "=&r" (original),  // %0
  62               [result] "=&r" (result),      // %1
  63               [tmp] "=&l" (tmp),            // %2
  64               [storage] "+Q" (storage)      // %3
  65             :
  66             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
  67         );
  68         gcc_arm_operations_base::fence_after(order);
  69         return static_cast< storage_type >(original);
  70     }
  71
  72     static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  73     {
  74         gcc_arm_operations_base::fence_before(order);
  75         uint32_t tmp;
  76         extended_storage_type original, result;
  77         __asm__ __volatile__
  78         (
  79             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
  80             "1:\n"
  81             "ldrexb   %[original], %[storage]\n"           // original = *(&storage)
  82             "mvn      %[result], %[original]\n"            // result = NOT original
  83             "strexb   %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
  84             "teq      %[tmp], #0\n"                        // flags = tmp==0
  85             "bne      1b\n"                                // if (!flags.equal) goto retry
  86             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
  87             : [original] "=&r" (original),  // %0
  88               [result] "=&r" (result),      // %1
  89               [tmp] "=&l" (tmp),            // %2
  90               [storage] "+Q" (storage)      // %3
  91             :
  92             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
  93         );
  94         gcc_arm_operations_base::fence_after(order);
  95         return static_cast< storage_type >(original);
  96     }
  97
  98     static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  99     {
 100         fetch_negate(storage, order);
 101     }
 102
 103     static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 104     {
 105         fetch_complement(storage, order);
 106     }
 107 };
 108
 109 #endif // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXB_STREXB)
 110
 111 #if defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXH_STREXH)
 112
 113 template< typename Base, bool Signed >
 114 struct extra_operations< Base, 2u, Signed > :
 115     public generic_extra_operations< Base, 2u, Signed >
 116 {
 117     typedef generic_extra_operations< Base, 2u, Signed > base_type;
 118     typedef typename base_type::storage_type storage_type;
 119     typedef typename make_storage_type< 4u, Signed >::type extended_storage_type;
 120
 121     static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 122     {
 123         gcc_arm_operations_base::fence_before(order);
 124         uint32_t tmp;
 125         extended_storage_type original, result;
 126         __asm__ __volatile__
 127         (
 128             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
 129             "1:\n"
 130             "ldrexh   %[original], %[storage]\n"           // original = *(&storage)
 131             "rsb      %[result], %[original], #0\n"        // result = 0 - original
 132             "strexh   %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
 133             "teq      %[tmp], #0\n"                        // flags = tmp==0
 134             "bne      1b\n"                                // if (!flags.equal) goto retry
 135             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
 136             : [original] "=&r" (original),  // %0
 137               [result] "=&r" (result),      // %1
 138               [tmp] "=&l" (tmp),            // %2
 139               [storage] "+Q" (storage)      // %3
 140             :
 141             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 142         );
 143         gcc_arm_operations_base::fence_after(order);
 144         return static_cast< storage_type >(original);
 145     }
 146
 147     static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 148     {
 149         gcc_arm_operations_base::fence_before(order);
 150         uint32_t tmp;
 151         extended_storage_type original, result;
 152         __asm__ __volatile__
 153         (
 154             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
 155             "1:\n"
 156             "ldrexh   %[original], %[storage]\n"           // original = *(&storage)
 157             "mvn      %[result], %[original]\n"            // result = NOT original
 158             "strexh   %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
 159             "teq      %[tmp], #0\n"                        // flags = tmp==0
 160             "bne      1b\n"                                // if (!flags.equal) goto retry
 161             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
 162             : [original] "=&r" (original),  // %0
 163               [result] "=&r" (result),      // %1
 164               [tmp] "=&l" (tmp),            // %2
 165               [storage] "+Q" (storage)      // %3
 166             :
 167             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 168         );
 169         gcc_arm_operations_base::fence_after(order);
 170         return static_cast< storage_type >(original);
 171     }
 172
 173     static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 174     {
 175         fetch_negate(storage, order);
 176     }
 177
 178     static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 179     {
 180         fetch_complement(storage, order);
 181     }
 182 };
 183
 184 #endif // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXH_STREXH)
 185
 186 template< typename Base, bool Signed >
 187 struct extra_operations< Base, 4u, Signed > :
 188     public generic_extra_operations< Base, 4u, Signed >
 189 {
 190     typedef generic_extra_operations< Base, 4u, Signed > base_type;
 191     typedef typename base_type::storage_type storage_type;
 192
 193     static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 194     {
 195         gcc_arm_operations_base::fence_before(order);
 196         uint32_t tmp;
 197         storage_type original, result;
 198         __asm__ __volatile__
 199         (
 200             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
 201             "1:\n"
 202             "ldrex    %[original], %[storage]\n"           // original = *(&storage)
 203             "rsb      %[result], %[original], #0\n"        // result = 0 - original
 204             "strex    %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
 205             "teq      %[tmp], #0\n"                        // flags = tmp==0
 206             "bne      1b\n"                                // if (!flags.equal) goto retry
 207             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
 208             : [original] "=&r" (original),  // %0
 209               [result] "=&r" (result),      // %1
 210               [tmp] "=&l" (tmp),            // %2
 211               [storage] "+Q" (storage)      // %3
 212             :
 213             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 214         );
 215         gcc_arm_operations_base::fence_after(order);
 216         return original;
 217     }
 218
 219     static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 220     {
 221         gcc_arm_operations_base::fence_before(order);
 222         uint32_t tmp;
 223         storage_type original, result;
 224         __asm__ __volatile__
 225         (
 226             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
 227             "1:\n"
 228             "ldrex    %[original], %[storage]\n"           // original = *(&storage)
 229             "mvn      %[result], %[original]\n"            // result = NOT original
 230             "strex    %[tmp], %[result], %[storage]\n"     // *(&storage) = result, tmp = store failed
 231             "teq      %[tmp], #0\n"                        // flags = tmp==0
 232             "bne      1b\n"                                // if (!flags.equal) goto retry
 233             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
 234             : [original] "=&r" (original),  // %0
 235               [result] "=&r" (result),      // %1
 236               [tmp] "=&l" (tmp),            // %2
 237               [storage] "+Q" (storage)      // %3
 238             :
 239             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
 240         );
 241         gcc_arm_operations_base::fence_after(order);
 242         return original;
 243     }
 244
 245     static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 246     {
 247         fetch_negate(storage, order);
 248     }
 249
 250     static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 251     {
 252         fetch_complement(storage, order);
 253     }
 254 };
 255
 256 #if defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXD_STREXD)
 257
 258 template< typename Base, bool Signed >
 259 struct extra_operations< Base, 8u, Signed > :
 260     public generic_extra_operations< Base, 8u, Signed >
 261 {
 262     typedef generic_extra_operations< Base, 8u, Signed > base_type;
 263     typedef typename base_type::storage_type storage_type;
 264
 265     static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 266     {
 267         gcc_arm_operations_base::fence_before(order);
 268         storage_type original, result;
 269         uint32_t tmp;
 270         __asm__ __volatile__
 271         (
 272             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
 273             "1:\n"
 274             "ldrexd  %1, %H1, [%3]\n"               // original = *(&storage)
 275             "mvn     %2, %1\n"                      // result = NOT original
 276             "mvn     %H2, %H1\n"
 277             "adds    %2, %2, #1\n"                  // result = result + 1
 278             "adc     %H2, %H2, #0\n"
 279             "strexd  %0, %2, %H2, [%3]\n"           // *(&storage) = result, tmp = store failed
 280             "teq     %0, #0\n"                      // flags = tmp==0
 281             "bne     1b\n"                          // if (!flags.equal) goto retry
 282             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
 283             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
 284               "=&r" (original),  // %1
 285               "=&r" (result)     // %2
 286             : "r" (&storage)     // %3
 287             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
 288         );
 289         gcc_arm_operations_base::fence_after(order);
 290         return original;
 291     }
 292
 293     static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 294     {
 295         gcc_arm_operations_base::fence_before(order);
 296         storage_type original, result;
 297         uint32_t tmp;
 298         __asm__ __volatile__
 299         (
 300             BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
 301             "1:\n"
 302             "ldrexd  %1, %H1, [%3]\n"               // original = *(&storage)
 303             "mvn     %2, %1\n"                      // result = NOT original
 304             "mvn     %H2, %H1\n"
 305             "strexd  %0, %2, %H2, [%3]\n"           // *(&storage) = result, tmp = store failed
 306             "teq     %0, #0\n"                      // flags = tmp==0
 307             "bne     1b\n"                          // if (!flags.equal) goto retry
 308             BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
 309             : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
 310               "=&r" (original),  // %1
 311               "=&r" (result)     // %2
 312             : "r" (&storage)     // %3
 313             : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
 314         );
 315         gcc_arm_operations_base::fence_after(order);
 316         return original;
 317     }
 318
 319     static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 320     {
 321         fetch_negate(storage, order);
 322     }
 323
 324     static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
 325     {
 326         fetch_complement(storage, order);
 327     }
 328 };
 329
 330 #endif // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXD_STREXD)
 331
 332 } // namespace detail
 333 } // namespace atomics
 334 } // namespace boost
 335
 336 #endif // BOOST_ATOMIC_DETAIL_EXTRA_OPS_GCC_ARM_HPP_INCLUDED_