]> git.proxmox.com Git - ceph.git/blame - ceph/src/boost/boost/atomic/detail/core_arch_ops_gcc_arm.hpp
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
import quincy beta 17.1.0
[ceph.git] / ceph / src / boost / boost / atomic / detail / core_arch_ops_gcc_arm.hpp
CommitLineData
7c673cae
FG		 1/*
2 * Distributed under the Boost Software License, Version 1.0.
3 * (See accompanying file LICENSE_1_0.txt or copy at
4 * http://www.boost.org/LICENSE_1_0.txt)
5 *
6 * Copyright (c) 2009 Helge Bahmann
7 * Copyright (c) 2013 Tim Blechmann
20effc67 8 * Copyright (c) 2014, 2020 Andrey Semashev
7c673cae
FG		 9 */
10/*!
20effc67 11 * \file atomic/detail/core_arch_ops_gcc_arm.hpp
7c673cae 12 *
20effc67 13 * This header contains implementation of the \c core_arch_operations template.
7c673cae
FG		 14 */
15
20effc67
TL		 16#ifndef BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_ARM_HPP_INCLUDED_
17#define BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_ARM_HPP_INCLUDED_
7c673cae 18
b32b8144 19#include <cstddef>
7c673cae
FG		 20#include <boost/cstdint.hpp>
21#include <boost/memory_order.hpp>
22#include <boost/atomic/detail/config.hpp>
f67539c2
TL		 23#include <boost/atomic/detail/storage_traits.hpp>
24#include <boost/atomic/detail/integral_conversions.hpp>
20effc67 25#include <boost/atomic/detail/core_arch_operations_fwd.hpp>
b32b8144 26#include <boost/atomic/detail/ops_gcc_arm_common.hpp>
20effc67
TL		 27#include <boost/atomic/detail/gcc_arm_asm_common.hpp>
28#include <boost/atomic/detail/capabilities.hpp>
29#include <boost/atomic/detail/header.hpp>
7c673cae
FG		 30
31#ifdef BOOST_HAS_PRAGMA_ONCE
32#pragma once
33#endif
34
35namespace boost {
36namespace atomics {
37namespace detail {
38
39// From the ARM Architecture Reference Manual for architecture v6:
40//
41// LDREX{<cond>} <Rd>, [<Rn>]
42// <Rd> Specifies the destination register for the memory word addressed by <Rd>
43// <Rn> Specifies the register containing the address.
44//
45// STREX{<cond>} <Rd>, <Rm>, [<Rn>]
46// <Rd> Specifies the destination register for the returned status value.
47// 0 if the operation updates memory
48// 1 if the operation fails to update memory
49// <Rm> Specifies the register containing the word to be stored to memory.
50// <Rn> Specifies the register containing the address.
51// Rd must not be the same register as Rm or Rn.
52//
53// ARM v7 is like ARM v6 plus:
54// There are half-word and byte versions of the LDREX and STREX instructions,
55// LDREXH, LDREXB, STREXH and STREXB.
56// There are also double-word versions, LDREXD and STREXD.
57// (Actually it looks like these are available from version 6k onwards.)
7c673cae 58
20effc67
TL		 59template< bool Signed, bool Interprocess >
60struct core_arch_operations< 4u, Signed, Interprocess > :
61 public core_arch_operations_gcc_arm_base
7c673cae 62{
f67539c2 63 typedef typename storage_traits< 4u >::type storage_type;
7c673cae 64
b32b8144 65 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 4u;
f67539c2 66 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 4u;
b32b8144 67 static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
20effc67 68 static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
b32b8144 69
7c673cae
FG		 70 static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
71 {
72 fence_before(order);
73 storage = v;
74 fence_after_store(order);
75 }
76
77 static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
78 {
79 storage_type v = storage;
80 fence_after(order);
81 return v;
82 }
83
84 static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
85 {
7c673cae 86 fence_before(order);
20effc67 87 storage_type original;
7c673cae
FG		 88 uint32_t tmp;
89 __asm__ __volatile__
90 (
91 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 92 "1:\n\t"
93 "ldrex %[original], %[storage]\n\t" // load the original value
94 "strex %[tmp], %[value], %[storage]\n\t" // store the replacement, tmp = store failed
95 "teq %[tmp], #0\n\t" // check if store succeeded
96 "bne 1b\n\t"
7c673cae
FG		 97 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
98 : [tmp] "=&l" (tmp), [original] "=&r" (original), [storage] "+Q" (storage)
99 : [value] "r" (v)
100 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
101 );
102 fence_after(order);
103 return original;
104 }
105
106 static BOOST_FORCEINLINE bool compare_exchange_weak(
107 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
108 {
109 fence_before(success_order);
20effc67
TL		 110 bool success = false;
111#if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
7c673cae 112 uint32_t tmp;
20effc67 113#endif
7c673cae
FG		 114 storage_type original;
115 __asm__ __volatile__
116 (
117 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 118 "ldrex %[original], %[storage]\n\t" // original = *(&storage)
119 "cmp %[original], %[expected]\n\t" // flags = original==expected
120 "itt eq\n\t" // [hint that the following 2 instructions are conditional on flags.equal]
121 "strexeq %[success], %[desired], %[storage]\n\t" // if (flags.equal) *(&storage) = desired, success = store failed
122 "eoreq %[success], %[success], #1\n\t" // if (flags.equal) success ^= 1 (i.e. make it 1 if store succeeded)
7c673cae 123 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
20effc67
TL		 124 : [original] "=&r" (original),
125 [success] "+r" (success),
126#if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
127 [tmp] "=&l" (tmp),
128#endif
129 [storage] "+Q" (storage)
130 : [expected] "Ir" (expected),
131 [desired] "r" (desired)
7c673cae
FG		 132 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
133 );
134 if (success)
135 fence_after(success_order);
136 else
137 fence_after(failure_order);
138 expected = original;
20effc67 139 return success;
7c673cae
FG		 140 }
141
142 static BOOST_FORCEINLINE bool compare_exchange_strong(
143 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
144 {
145 fence_before(success_order);
20effc67
TL		 146 bool success = false;
147#if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
7c673cae 148 uint32_t tmp;
20effc67 149#endif
7c673cae
FG		 150 storage_type original;
151 __asm__ __volatile__
152 (
153 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 154 "1:\n\t"
155 "ldrex %[original], %[storage]\n\t" // original = *(&storage)
156 "cmp %[original], %[expected]\n\t" // flags = original==expected
157 "bne 2f\n\t" // if (!flags.equal) goto end
158 "strex %[success], %[desired], %[storage]\n\t" // *(&storage) = desired, success = store failed
159 "eors %[success], %[success], #1\n\t" // success ^= 1 (i.e. make it 1 if store succeeded); flags.equal = success == 0
160 "beq 1b\n\t" // if (flags.equal) goto retry
161 "2:\n\t"
7c673cae 162 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
20effc67
TL		 163 : [original] "=&r" (original),
164 [success] "+r" (success),
165#if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
166 [tmp] "=&l" (tmp),
167#endif
168 [storage] "+Q" (storage)
169 : [expected] "Ir" (expected),
170 [desired] "r" (desired)
7c673cae
FG		 171 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
172 );
173 if (success)
174 fence_after(success_order);
175 else
176 fence_after(failure_order);
177 expected = original;
20effc67 178 return success;
7c673cae
FG		 179 }
180
181 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
182 {
183 fence_before(order);
184 uint32_t tmp;
185 storage_type original, result;
186 __asm__ __volatile__
187 (
188 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 189 "1:\n\t"
190 "ldrex %[original], %[storage]\n\t" // original = *(&storage)
191 "add %[result], %[original], %[value]\n\t" // result = original + value
192 "strex %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
193 "teq %[tmp], #0\n\t" // flags = tmp==0
194 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 195 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
196 : [original] "=&r" (original), // %0
197 [result] "=&r" (result), // %1
198 [tmp] "=&l" (tmp), // %2
199 [storage] "+Q" (storage) // %3
b32b8144 200 : [value] "Ir" (v) // %4
7c673cae
FG		 201 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
202 );
203 fence_after(order);
204 return original;
205 }
206
207 static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
208 {
209 fence_before(order);
210 uint32_t tmp;
211 storage_type original, result;
212 __asm__ __volatile__
213 (
214 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 215 "1:\n\t"
216 "ldrex %[original], %[storage]\n\t" // original = *(&storage)
217 "sub %[result], %[original], %[value]\n\t" // result = original - value
218 "strex %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
219 "teq %[tmp], #0\n\t" // flags = tmp==0
220 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 221 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
222 : [original] "=&r" (original), // %0
223 [result] "=&r" (result), // %1
224 [tmp] "=&l" (tmp), // %2
225 [storage] "+Q" (storage) // %3
b32b8144 226 : [value] "Ir" (v) // %4
7c673cae
FG		 227 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
228 );
229 fence_after(order);
230 return original;
231 }
232
233 static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
234 {
235 fence_before(order);
236 uint32_t tmp;
237 storage_type original, result;
238 __asm__ __volatile__
239 (
240 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 241 "1:\n\t"
242 "ldrex %[original], %[storage]\n\t" // original = *(&storage)
243 "and %[result], %[original], %[value]\n\t" // result = original & value
244 "strex %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
245 "teq %[tmp], #0\n\t" // flags = tmp==0
246 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 247 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
248 : [original] "=&r" (original), // %0
249 [result] "=&r" (result), // %1
250 [tmp] "=&l" (tmp), // %2
251 [storage] "+Q" (storage) // %3
b32b8144 252 : [value] "Ir" (v) // %4
7c673cae
FG		 253 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
254 );
255 fence_after(order);
256 return original;
257 }
258
259 static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
260 {
261 fence_before(order);
262 uint32_t tmp;
263 storage_type original, result;
264 __asm__ __volatile__
265 (
266 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 267 "1:\n\t"
268 "ldrex %[original], %[storage]\n\t" // original = *(&storage)
269 "orr %[result], %[original], %[value]\n\t" // result = original | value
270 "strex %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
271 "teq %[tmp], #0\n\t" // flags = tmp==0
272 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 273 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
274 : [original] "=&r" (original), // %0
275 [result] "=&r" (result), // %1
276 [tmp] "=&l" (tmp), // %2
277 [storage] "+Q" (storage) // %3
b32b8144 278 : [value] "Ir" (v) // %4
7c673cae
FG		 279 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
280 );
281 fence_after(order);
282 return original;
283 }
284
285 static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
286 {
287 fence_before(order);
288 uint32_t tmp;
289 storage_type original, result;
290 __asm__ __volatile__
291 (
292 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 293 "1:\n\t"
294 "ldrex %[original], %[storage]\n\t" // original = *(&storage)
295 "eor %[result], %[original], %[value]\n\t" // result = original ^ value
296 "strex %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
297 "teq %[tmp], #0\n\t" // flags = tmp==0
298 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 299 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
300 : [original] "=&r" (original), // %0
301 [result] "=&r" (result), // %1
302 [tmp] "=&l" (tmp), // %2
303 [storage] "+Q" (storage) // %3
b32b8144 304 : [value] "Ir" (v) // %4
7c673cae
FG		 305 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
306 );
307 fence_after(order);
308 return original;
309 }
310
311 static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
312 {
313 return !!exchange(storage, (storage_type)1, order);
314 }
315
316 static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
317 {
f67539c2 318 store(storage, (storage_type)0, order);
7c673cae 319 }
b32b8144
FG		 320};
321
322#if defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXB_STREXB)
323
20effc67
TL		 324template< bool Signed, bool Interprocess >
325struct core_arch_operations< 1u, Signed, Interprocess > :
326 public core_arch_operations_gcc_arm_base
b32b8144 327{
f67539c2
TL		 328 typedef typename storage_traits< 1u >::type storage_type;
329 typedef typename storage_traits< 4u >::type extended_storage_type;
b32b8144
FG		 330
331 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 1u;
f67539c2 332 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 1u;
b32b8144 333 static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
20effc67 334 static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
b32b8144
FG		 335
336 static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
337 {
338 fence_before(order);
339 storage = v;
340 fence_after_store(order);
341 }
342
343 static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
344 {
345 storage_type v = storage;
346 fence_after(order);
347 return v;
348 }
349
350 static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
351 {
b32b8144 352 fence_before(order);
20effc67 353 extended_storage_type original;
b32b8144
FG		 354 uint32_t tmp;
355 __asm__ __volatile__
356 (
357 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 358 "1:\n\t"
359 "ldrexb %[original], %[storage]\n\t" // load the original value and zero-extend to 32 bits
360 "strexb %[tmp], %[value], %[storage]\n\t" // store the replacement, tmp = store failed
361 "teq %[tmp], #0\n\t" // check if store succeeded
362 "bne 1b\n\t"
b32b8144
FG		 363 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
364 : [tmp] "=&l" (tmp), [original] "=&r" (original), [storage] "+Q" (storage)
11fdf7f2 365 : [value] "r" (v)
b32b8144
FG		 366 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
367 );
368 fence_after(order);
369 return static_cast< storage_type >(original);
370 }
371
372 static BOOST_FORCEINLINE bool compare_exchange_weak(
373 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
374 {
375 fence_before(success_order);
20effc67
TL		 376 bool success = false;
377#if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
b32b8144 378 uint32_t tmp;
20effc67 379#endif
b32b8144
FG		 380 extended_storage_type original;
381 __asm__ __volatile__
382 (
383 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 384 "ldrexb %[original], %[storage]\n\t" // original = zero_extend(*(&storage))
385 "cmp %[original], %[expected]\n\t" // flags = original==expected
386 "itt eq\n\t" // [hint that the following 2 instructions are conditional on flags.equal]
387 "strexbeq %[success], %[desired], %[storage]\n\t" // if (flags.equal) *(&storage) = desired, success = store failed
388 "eoreq %[success], %[success], #1\n\t" // if (flags.equal) success ^= 1 (i.e. make it 1 if store succeeded)
b32b8144 389 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
20effc67
TL		 390 : [original] "=&r" (original),
391 [success] "+r" (success),
392#if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
393 [tmp] "=&l" (tmp),
394#endif
395 [storage] "+Q" (storage)
396 : [expected] "Ir" (atomics::detail::zero_extend< extended_storage_type >(expected)),
397 [desired] "r" (desired)
b32b8144
FG		 398 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
399 );
400 if (success)
401 fence_after(success_order);
402 else
403 fence_after(failure_order);
404 expected = static_cast< storage_type >(original);
20effc67 405 return success;
b32b8144
FG		 406 }
407
408 static BOOST_FORCEINLINE bool compare_exchange_strong(
409 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
410 {
411 fence_before(success_order);
20effc67
TL		 412 bool success = false;
413#if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
b32b8144 414 uint32_t tmp;
20effc67 415#endif
b32b8144
FG		 416 extended_storage_type original;
417 __asm__ __volatile__
418 (
419 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 420 "1:\n\t"
421 "ldrexb %[original], %[storage]\n\t" // original = zero_extend(*(&storage))
422 "cmp %[original], %[expected]\n\t" // flags = original==expected
423 "bne 2f\n\t" // if (!flags.equal) goto end
424 "strexb %[success], %[desired], %[storage]\n\t" // *(&storage) = desired, success = store failed
425 "eors %[success], %[success], #1\n\t" // success ^= 1 (i.e. make it 1 if store succeeded); flags.equal = success == 0
426 "beq 1b\n\t" // if (flags.equal) goto retry
427 "2:\n\t"
b32b8144 428 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
20effc67
TL		 429 : [original] "=&r" (original),
430 [success] "+r" (success),
431#if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
432 [tmp] "=&l" (tmp),
433#endif
434 [storage] "+Q" (storage)
435 : [expected] "Ir" (atomics::detail::zero_extend< extended_storage_type >(expected)),
436 [desired] "r" (desired)
b32b8144
FG		 437 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
438 );
439 if (success)
440 fence_after(success_order);
441 else
442 fence_after(failure_order);
443 expected = static_cast< storage_type >(original);
20effc67 444 return success;
b32b8144
FG		 445 }
446
447 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
448 {
449 fence_before(order);
450 uint32_t tmp;
451 extended_storage_type original, result;
452 __asm__ __volatile__
453 (
454 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 455 "1:\n\t"
456 "ldrexb %[original], %[storage]\n\t" // original = zero_extend(*(&storage))
457 "add %[result], %[original], %[value]\n\t" // result = original + value
458 "strexb %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
459 "teq %[tmp], #0\n\t" // flags = tmp==0
460 "bne 1b\n\t" // if (!flags.equal) goto retry
b32b8144
FG		 461 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
462 : [original] "=&r" (original), // %0
463 [result] "=&r" (result), // %1
464 [tmp] "=&l" (tmp), // %2
465 [storage] "+Q" (storage) // %3
11fdf7f2 466 : [value] "Ir" (v) // %4
b32b8144
FG		 467 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
468 );
469 fence_after(order);
470 return static_cast< storage_type >(original);
471 }
472
473 static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
474 {
475 fence_before(order);
476 uint32_t tmp;
477 extended_storage_type original, result;
478 __asm__ __volatile__
479 (
480 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 481 "1:\n\t"
482 "ldrexb %[original], %[storage]\n\t" // original = zero_extend(*(&storage))
483 "sub %[result], %[original], %[value]\n\t" // result = original - value
484 "strexb %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
485 "teq %[tmp], #0\n\t" // flags = tmp==0
486 "bne 1b\n\t" // if (!flags.equal) goto retry
b32b8144
FG		 487 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
488 : [original] "=&r" (original), // %0
489 [result] "=&r" (result), // %1
490 [tmp] "=&l" (tmp), // %2
491 [storage] "+Q" (storage) // %3
11fdf7f2 492 : [value] "Ir" (v) // %4
b32b8144
FG		 493 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
494 );
495 fence_after(order);
496 return static_cast< storage_type >(original);
497 }
498
499 static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
500 {
501 fence_before(order);
502 uint32_t tmp;
503 extended_storage_type original, result;
504 __asm__ __volatile__
505 (
506 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 507 "1:\n\t"
508 "ldrexb %[original], %[storage]\n\t" // original = zero_extend(*(&storage))
509 "and %[result], %[original], %[value]\n\t" // result = original & value
510 "strexb %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
511 "teq %[tmp], #0\n\t" // flags = tmp==0
512 "bne 1b\n\t" // if (!flags.equal) goto retry
b32b8144
FG		 513 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
514 : [original] "=&r" (original), // %0
515 [result] "=&r" (result), // %1
516 [tmp] "=&l" (tmp), // %2
517 [storage] "+Q" (storage) // %3
11fdf7f2 518 : [value] "Ir" (v) // %4
b32b8144
FG		 519 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
520 );
521 fence_after(order);
522 return static_cast< storage_type >(original);
523 }
524
525 static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
526 {
527 fence_before(order);
528 uint32_t tmp;
529 extended_storage_type original, result;
530 __asm__ __volatile__
531 (
532 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 533 "1:\n\t"
534 "ldrexb %[original], %[storage]\n\t" // original = zero_extend(*(&storage))
535 "orr %[result], %[original], %[value]\n\t" // result = original | value
536 "strexb %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
537 "teq %[tmp], #0\n\t" // flags = tmp==0
538 "bne 1b\n\t" // if (!flags.equal) goto retry
b32b8144
FG		 539 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
540 : [original] "=&r" (original), // %0
541 [result] "=&r" (result), // %1
542 [tmp] "=&l" (tmp), // %2
543 [storage] "+Q" (storage) // %3
11fdf7f2 544 : [value] "Ir" (v) // %4
b32b8144
FG		 545 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
546 );
547 fence_after(order);
548 return static_cast< storage_type >(original);
549 }
550
551 static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
552 {
553 fence_before(order);
554 uint32_t tmp;
555 extended_storage_type original, result;
556 __asm__ __volatile__
557 (
558 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 559 "1:\n\t"
560 "ldrexb %[original], %[storage]\n\t" // original = zero_extend(*(&storage))
561 "eor %[result], %[original], %[value]\n\t" // result = original ^ value
562 "strexb %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
563 "teq %[tmp], #0\n\t" // flags = tmp==0
564 "bne 1b\n\t" // if (!flags.equal) goto retry
b32b8144
FG		 565 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
566 : [original] "=&r" (original), // %0
567 [result] "=&r" (result), // %1
568 [tmp] "=&l" (tmp), // %2
569 [storage] "+Q" (storage) // %3
11fdf7f2 570 : [value] "Ir" (v) // %4
b32b8144
FG		 571 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
572 );
573 fence_after(order);
574 return static_cast< storage_type >(original);
575 }
7c673cae 576
b32b8144 577 static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
7c673cae 578 {
b32b8144
FG		 579 return !!exchange(storage, (storage_type)1, order);
580 }
581
582 static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
583 {
f67539c2 584 store(storage, (storage_type)0, order);
7c673cae
FG		 585 }
586};
587
b32b8144 588#else // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXB_STREXB)
7c673cae 589
20effc67
TL		 590template< bool Interprocess >
591struct core_arch_operations< 1u, false, Interprocess > :
592 public core_arch_operations< 4u, false, Interprocess >
7c673cae 593{
20effc67
TL		 594 typedef core_arch_operations< 4u, false, Interprocess > base_type;
595 typedef typename base_type::storage_type storage_type;
7c673cae
FG		 596
597 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
598 {
20effc67 599 base_type::fence_before(order);
7c673cae
FG		 600 uint32_t tmp;
601 storage_type original, result;
602 __asm__ __volatile__
603 (
604 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 605 "1:\n\t"
606 "ldrex %[original], %[storage]\n\t" // original = *(&storage)
607 "add %[result], %[original], %[value]\n\t" // result = original + value
608 "uxtb %[result], %[result]\n\t" // zero extend result from 8 to 32 bits
609 "strex %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
610 "teq %[tmp], #0\n\t" // flags = tmp==0
611 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 612 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
613 : [original] "=&r" (original), // %0
614 [result] "=&r" (result), // %1
615 [tmp] "=&l" (tmp), // %2
616 [storage] "+Q" (storage) // %3
b32b8144 617 : [value] "Ir" (v) // %4
7c673cae
FG		 618 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
619 );
20effc67 620 base_type::fence_after(order);
7c673cae
FG		 621 return original;
622 }
623
624 static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
625 {
20effc67 626 base_type::fence_before(order);
7c673cae
FG		 627 uint32_t tmp;
628 storage_type original, result;
629 __asm__ __volatile__
630 (
631 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 632 "1:\n\t"
633 "ldrex %[original], %[storage]\n\t" // original = *(&storage)
634 "sub %[result], %[original], %[value]\n\t" // result = original - value
635 "uxtb %[result], %[result]\n\t" // zero extend result from 8 to 32 bits
636 "strex %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
637 "teq %[tmp], #0\n\t" // flags = tmp==0
638 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 639 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
640 : [original] "=&r" (original), // %0
641 [result] "=&r" (result), // %1
642 [tmp] "=&l" (tmp), // %2
643 [storage] "+Q" (storage) // %3
b32b8144 644 : [value] "Ir" (v) // %4
7c673cae
FG		 645 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
646 );
20effc67 647 base_type::fence_after(order);
7c673cae
FG		 648 return original;
649 }
650};
651
20effc67
TL		 652template< bool Interprocess >
653struct core_arch_operations< 1u, true, Interprocess > :
654 public core_arch_operations< 4u, true, Interprocess >
7c673cae 655{
20effc67
TL		 656 typedef core_arch_operations< 4u, true, Interprocess > base_type;
657 typedef typename base_type::storage_type storage_type;
7c673cae
FG		 658
659 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
660 {
20effc67 661 base_type::fence_before(order);
7c673cae
FG		 662 uint32_t tmp;
663 storage_type original, result;
664 __asm__ __volatile__
665 (
666 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 667 "1:\n\t"
668 "ldrex %[original], %[storage]\n\t" // original = *(&storage)
669 "add %[result], %[original], %[value]\n\t" // result = original + value
670 "sxtb %[result], %[result]\n\t" // sign extend result from 8 to 32 bits
671 "strex %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
672 "teq %[tmp], #0\n\t" // flags = tmp==0
673 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 674 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
675 : [original] "=&r" (original), // %0
676 [result] "=&r" (result), // %1
677 [tmp] "=&l" (tmp), // %2
678 [storage] "+Q" (storage) // %3
b32b8144 679 : [value] "Ir" (v) // %4
7c673cae
FG		 680 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
681 );
20effc67 682 base_type::fence_after(order);
7c673cae
FG		 683 return original;
684 }
685
686 static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
687 {
20effc67 688 base_type::fence_before(order);
7c673cae
FG		 689 uint32_t tmp;
690 storage_type original, result;
691 __asm__ __volatile__
692 (
693 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 694 "1:\n\t"
695 "ldrex %[original], %[storage]\n\t" // original = *(&storage)
696 "sub %[result], %[original], %[value]\n\t" // result = original - value
697 "sxtb %[result], %[result]\n\t" // sign extend result from 8 to 32 bits
698 "strex %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
699 "teq %[tmp], #0\n\t" // flags = tmp==0
700 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 701 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
702 : [original] "=&r" (original), // %0
703 [result] "=&r" (result), // %1
704 [tmp] "=&l" (tmp), // %2
705 [storage] "+Q" (storage) // %3
b32b8144 706 : [value] "Ir" (v) // %4
7c673cae
FG		 707 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
708 );
20effc67 709 base_type::fence_after(order);
7c673cae
FG		 710 return original;
711 }
712};
713
b32b8144
FG		 714#endif // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXB_STREXB)
715
716#if defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXH_STREXH)
717
20effc67
TL		 718template< bool Signed, bool Interprocess >
719struct core_arch_operations< 2u, Signed, Interprocess > :
720 public core_arch_operations_gcc_arm_base
b32b8144 721{
f67539c2
TL		 722 typedef typename storage_traits< 2u >::type storage_type;
723 typedef typename storage_traits< 4u >::type extended_storage_type;
b32b8144
FG		 724
725 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 2u;
f67539c2 726 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 2u;
b32b8144 727 static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
20effc67 728 static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
b32b8144
FG		 729
730 static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
731 {
732 fence_before(order);
733 storage = v;
734 fence_after_store(order);
735 }
736
737 static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
738 {
739 storage_type v = storage;
740 fence_after(order);
741 return v;
742 }
743
744 static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
745 {
b32b8144 746 fence_before(order);
20effc67 747 extended_storage_type original;
b32b8144
FG		 748 uint32_t tmp;
749 __asm__ __volatile__
750 (
751 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 752 "1:\n\t"
753 "ldrexh %[original], %[storage]\n\t" // load the original value and zero-extend to 32 bits
754 "strexh %[tmp], %[value], %[storage]\n\t" // store the replacement, tmp = store failed
755 "teq %[tmp], #0\n\t" // check if store succeeded
756 "bne 1b\n\t"
b32b8144
FG		 757 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
758 : [tmp] "=&l" (tmp), [original] "=&r" (original), [storage] "+Q" (storage)
11fdf7f2 759 : [value] "r" (v)
b32b8144
FG		 760 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
761 );
762 fence_after(order);
763 return static_cast< storage_type >(original);
764 }
765
766 static BOOST_FORCEINLINE bool compare_exchange_weak(
767 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
768 {
769 fence_before(success_order);
20effc67
TL		 770 bool success = false;
771#if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
b32b8144 772 uint32_t tmp;
20effc67 773#endif
b32b8144
FG		 774 extended_storage_type original;
775 __asm__ __volatile__
776 (
777 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 778 "ldrexh %[original], %[storage]\n\t" // original = zero_extend(*(&storage))
779 "cmp %[original], %[expected]\n\t" // flags = original==expected
780 "itt eq\n\t" // [hint that the following 2 instructions are conditional on flags.equal]
781 "strexheq %[success], %[desired], %[storage]\n\t" // if (flags.equal) *(&storage) = desired, success = store failed
782 "eoreq %[success], %[success], #1\n\t" // if (flags.equal) success ^= 1 (i.e. make it 1 if store succeeded)
b32b8144 783 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
20effc67
TL		 784 : [original] "=&r" (original),
785 [success] "+r" (success),
786#if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
787 [tmp] "=&l" (tmp),
788#endif
789 [storage] "+Q" (storage)
790 : [expected] "Ir" (atomics::detail::zero_extend< extended_storage_type >(expected)),
791 [desired] "r" (desired)
b32b8144
FG		 792 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
793 );
794 if (success)
795 fence_after(success_order);
796 else
797 fence_after(failure_order);
798 expected = static_cast< storage_type >(original);
20effc67 799 return success;
b32b8144
FG		 800 }
801
802 static BOOST_FORCEINLINE bool compare_exchange_strong(
803 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
804 {
805 fence_before(success_order);
20effc67
TL		 806 bool success = false;
807#if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
b32b8144 808 uint32_t tmp;
20effc67 809#endif
b32b8144
FG		 810 extended_storage_type original;
811 __asm__ __volatile__
812 (
813 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 814 "1:\n\t"
815 "ldrexh %[original], %[storage]\n\t" // original = zero_extend(*(&storage))
816 "cmp %[original], %[expected]\n\t" // flags = original==expected
817 "bne 2f\n\t" // if (!flags.equal) goto end
818 "strexh %[success], %[desired], %[storage]\n\t" // *(&storage) = desired, success = store failed
819 "eors %[success], %[success], #1\n\t" // success ^= 1 (i.e. make it 1 if store succeeded); flags.equal = success == 0
820 "beq 1b\n\t" // if (flags.equal) goto retry
821 "2:\n\t"
b32b8144 822 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
20effc67
TL		 823 : [original] "=&r" (original),
824 [success] "+r" (success),
825#if !defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
826 [tmp] "=&l" (tmp),
827#endif
828 [storage] "+Q" (storage)
829 : [expected] "Ir" (atomics::detail::zero_extend< extended_storage_type >(expected)),
830 [desired] "r" (desired)
b32b8144
FG		 831 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
832 );
833 if (success)
834 fence_after(success_order);
835 else
836 fence_after(failure_order);
837 expected = static_cast< storage_type >(original);
20effc67 838 return success;
b32b8144
FG		 839 }
840
841 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
842 {
843 fence_before(order);
844 uint32_t tmp;
845 extended_storage_type original, result;
846 __asm__ __volatile__
847 (
848 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 849 "1:\n\t"
850 "ldrexh %[original], %[storage]\n\t" // original = zero_extend(*(&storage))
851 "add %[result], %[original], %[value]\n\t" // result = original + value
852 "strexh %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
853 "teq %[tmp], #0\n\t" // flags = tmp==0
854 "bne 1b\n\t" // if (!flags.equal) goto retry
b32b8144
FG		 855 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
856 : [original] "=&r" (original), // %0
857 [result] "=&r" (result), // %1
858 [tmp] "=&l" (tmp), // %2
859 [storage] "+Q" (storage) // %3
11fdf7f2 860 : [value] "Ir" (v) // %4
b32b8144
FG		 861 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
862 );
863 fence_after(order);
864 return static_cast< storage_type >(original);
865 }
866
867 static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
868 {
869 fence_before(order);
870 uint32_t tmp;
871 extended_storage_type original, result;
872 __asm__ __volatile__
873 (
874 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 875 "1:\n\t"
876 "ldrexh %[original], %[storage]\n\t" // original = zero_extend(*(&storage))
877 "sub %[result], %[original], %[value]\n\t" // result = original - value
878 "strexh %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
879 "teq %[tmp], #0\n\t" // flags = tmp==0
880 "bne 1b\n\t" // if (!flags.equal) goto retry
b32b8144
FG		 881 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
882 : [original] "=&r" (original), // %0
883 [result] "=&r" (result), // %1
884 [tmp] "=&l" (tmp), // %2
885 [storage] "+Q" (storage) // %3
11fdf7f2 886 : [value] "Ir" (v) // %4
b32b8144
FG		 887 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
888 );
889 fence_after(order);
890 return static_cast< storage_type >(original);
891 }
892
893 static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
894 {
895 fence_before(order);
896 uint32_t tmp;
897 extended_storage_type original, result;
898 __asm__ __volatile__
899 (
900 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 901 "1:\n\t"
902 "ldrexh %[original], %[storage]\n\t" // original = zero_extend(*(&storage))
903 "and %[result], %[original], %[value]\n\t" // result = original & value
904 "strexh %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
905 "teq %[tmp], #0\n\t" // flags = tmp==0
906 "bne 1b\n\t" // if (!flags.equal) goto retry
b32b8144
FG		 907 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
908 : [original] "=&r" (original), // %0
909 [result] "=&r" (result), // %1
910 [tmp] "=&l" (tmp), // %2
911 [storage] "+Q" (storage) // %3
11fdf7f2 912 : [value] "Ir" (v) // %4
b32b8144
FG		 913 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
914 );
915 fence_after(order);
916 return static_cast< storage_type >(original);
917 }
918
919 static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
920 {
921 fence_before(order);
922 uint32_t tmp;
923 extended_storage_type original, result;
924 __asm__ __volatile__
925 (
926 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 927 "1:\n\t"
928 "ldrexh %[original], %[storage]\n\t" // original = zero_extend(*(&storage))
929 "orr %[result], %[original], %[value]\n\t" // result = original | value
930 "strexh %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
931 "teq %[tmp], #0\n\t" // flags = tmp==0
932 "bne 1b\n\t" // if (!flags.equal) goto retry
b32b8144
FG		 933 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
934 : [original] "=&r" (original), // %0
935 [result] "=&r" (result), // %1
936 [tmp] "=&l" (tmp), // %2
937 [storage] "+Q" (storage) // %3
11fdf7f2 938 : [value] "Ir" (v) // %4
b32b8144
FG		 939 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
940 );
941 fence_after(order);
942 return static_cast< storage_type >(original);
943 }
944
945 static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
946 {
947 fence_before(order);
948 uint32_t tmp;
949 extended_storage_type original, result;
950 __asm__ __volatile__
951 (
952 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 953 "1:\n\t"
954 "ldrexh %[original], %[storage]\n\t" // original = zero_extend(*(&storage))
955 "eor %[result], %[original], %[value]\n\t" // result = original ^ value
956 "strexh %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
957 "teq %[tmp], #0\n\t" // flags = tmp==0
958 "bne 1b\n\t" // if (!flags.equal) goto retry
b32b8144
FG		 959 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
960 : [original] "=&r" (original), // %0
961 [result] "=&r" (result), // %1
962 [tmp] "=&l" (tmp), // %2
963 [storage] "+Q" (storage) // %3
11fdf7f2 964 : [value] "Ir" (v) // %4
b32b8144
FG		 965 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
966 );
967 fence_after(order);
968 return static_cast< storage_type >(original);
969 }
970
971 static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
972 {
973 return !!exchange(storage, (storage_type)1, order);
974 }
975
976 static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
977 {
f67539c2 978 store(storage, (storage_type)0, order);
b32b8144
FG		 979 }
980};
981
982#else // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXH_STREXH)
7c673cae 983
20effc67
TL		 984template< bool Interprocess >
985struct core_arch_operations< 2u, false, Interprocess > :
986 public core_arch_operations< 4u, false, Interprocess >
7c673cae 987{
20effc67
TL		 988 typedef core_arch_operations< 4u, false, Interprocess > base_type;
989 typedef typename base_type::storage_type storage_type;
7c673cae
FG		 990
991 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
992 {
20effc67 993 base_type::fence_before(order);
7c673cae
FG		 994 uint32_t tmp;
995 storage_type original, result;
996 __asm__ __volatile__
997 (
998 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 999 "1:\n\t"
1000 "ldrex %[original], %[storage]\n\t" // original = *(&storage)
1001 "add %[result], %[original], %[value]\n\t" // result = original + value
1002 "uxth %[result], %[result]\n\t" // zero extend result from 16 to 32 bits
1003 "strex %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
1004 "teq %[tmp], #0\n\t" // flags = tmp==0
1005 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 1006 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
1007 : [original] "=&r" (original), // %0
1008 [result] "=&r" (result), // %1
1009 [tmp] "=&l" (tmp), // %2
1010 [storage] "+Q" (storage) // %3
b32b8144 1011 : [value] "Ir" (v) // %4
7c673cae
FG		 1012 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1013 );
20effc67 1014 base_type::fence_after(order);
7c673cae
FG		 1015 return original;
1016 }
1017
1018 static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1019 {
20effc67 1020 base_type::fence_before(order);
7c673cae
FG		 1021 uint32_t tmp;
1022 storage_type original, result;
1023 __asm__ __volatile__
1024 (
1025 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 1026 "1:\n\t"
1027 "ldrex %[original], %[storage]\n\t" // original = *(&storage)
1028 "sub %[result], %[original], %[value]\n\t" // result = original - value
1029 "uxth %[result], %[result]\n\t" // zero extend result from 16 to 32 bits
1030 "strex %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
1031 "teq %[tmp], #0\n\t" // flags = tmp==0
1032 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 1033 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
1034 : [original] "=&r" (original), // %0
1035 [result] "=&r" (result), // %1
1036 [tmp] "=&l" (tmp), // %2
1037 [storage] "+Q" (storage) // %3
b32b8144 1038 : [value] "Ir" (v) // %4
7c673cae
FG		 1039 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1040 );
20effc67 1041 base_type::fence_after(order);
7c673cae
FG		 1042 return original;
1043 }
1044};
1045
20effc67
TL		 1046template< bool Interprocess >
1047struct core_arch_operations< 2u, true, Interprocess > :
1048 public core_arch_operations< 4u, true, Interprocess >
7c673cae 1049{
20effc67
TL		 1050 typedef core_arch_operations< 4u, true, Interprocess > base_type;
1051 typedef typename base_type::storage_type storage_type;
7c673cae
FG		 1052
1053 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1054 {
20effc67 1055 base_type::fence_before(order);
7c673cae
FG		 1056 uint32_t tmp;
1057 storage_type original, result;
1058 __asm__ __volatile__
1059 (
1060 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 1061 "1:\n\t"
1062 "ldrex %[original], %[storage]\n\t" // original = *(&storage)
1063 "add %[result], %[original], %[value]\n\t" // result = original + value
1064 "sxth %[result], %[result]\n\t" // sign extend result from 16 to 32 bits
1065 "strex %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
1066 "teq %[tmp], #0\n\t" // flags = tmp==0
1067 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 1068 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
1069 : [original] "=&r" (original), // %0
1070 [result] "=&r" (result), // %1
1071 [tmp] "=&l" (tmp), // %2
1072 [storage] "+Q" (storage) // %3
b32b8144 1073 : [value] "Ir" (v) // %4
7c673cae
FG		 1074 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1075 );
20effc67 1076 base_type::fence_after(order);
7c673cae
FG		 1077 return original;
1078 }
1079
1080 static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1081 {
20effc67 1082 base_type::fence_before(order);
7c673cae
FG		 1083 uint32_t tmp;
1084 storage_type original, result;
1085 __asm__ __volatile__
1086 (
1087 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
20effc67
TL		 1088 "1:\n\t"
1089 "ldrex %[original], %[storage]\n\t" // original = *(&storage)
1090 "sub %[result], %[original], %[value]\n\t" // result = original - value
1091 "sxth %[result], %[result]\n\t" // sign extend result from 16 to 32 bits
1092 "strex %[tmp], %[result], %[storage]\n\t" // *(&storage) = result, tmp = store failed
1093 "teq %[tmp], #0\n\t" // flags = tmp==0
1094 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 1095 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
1096 : [original] "=&r" (original), // %0
1097 [result] "=&r" (result), // %1
1098 [tmp] "=&l" (tmp), // %2
1099 [storage] "+Q" (storage) // %3
b32b8144 1100 : [value] "Ir" (v) // %4
7c673cae
FG		 1101 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
1102 );
20effc67 1103 base_type::fence_after(order);
7c673cae
FG		 1104 return original;
1105 }
1106};
1107
b32b8144 1108#endif // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXH_STREXH)
7c673cae
FG		 1109
1110#if defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXD_STREXD)
1111
1112// Unlike 32-bit operations, for 64-bit loads and stores we must use ldrexd/strexd.
1113// Any other instructions result in a non-atomic sequence of 32-bit accesses.
1114// See "ARM Architecture Reference Manual ARMv7-A and ARMv7-R edition",
1115// Section A3.5.3 "Atomicity in the ARM architecture".
1116
1117// In the asm blocks below we have to use 32-bit register pairs to compose 64-bit values.
1118// In order to pass the 64-bit operands to/from asm blocks, we use undocumented gcc feature:
1119// the lower half (Rt) of the operand is accessible normally, via the numbered placeholder (e.g. %0),
1120// and the upper half (Rt2) - via the same placeholder with an 'H' after the '%' sign (e.g. %H0).
1121// See: http://hardwarebug.org/2010/07/06/arm-inline-asm-secrets/
1122
20effc67
TL		 1123template< bool Signed, bool Interprocess >
1124struct core_arch_operations< 8u, Signed, Interprocess > :
1125 public core_arch_operations_gcc_arm_base
7c673cae 1126{
f67539c2 1127 typedef typename storage_traits< 8u >::type storage_type;
7c673cae 1128
b32b8144 1129 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 8u;
f67539c2 1130 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 8u;
b32b8144 1131 static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed;
20effc67 1132 static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess;
b32b8144 1133
7c673cae
FG		 1134 static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1135 {
1136 exchange(storage, v, order);
1137 }
1138
1139 static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT
1140 {
20effc67
TL		 1141 // ARMv7 says ldrex (and other load-exclusive instructions) can be used without a matching strex, see
1142 // "ARM Architecture Reference Manual ARMv7-A and ARMv7-R edition", Section A3.4.5 "Load-Exclusive and Store-Exclusive usage restrictions".
7c673cae 1143 storage_type original;
20effc67
TL		 1144#if defined(BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_UNUSED)
1145 __asm__ __volatile__
1146 (
1147 "ldrexd %0, %H0, %1\n\t"
1148 : "=&r" (original) // %0
1149 : "Q" (storage) // %1
1150 );
1151#else
7c673cae
FG		 1152 uint32_t tmp;
1153 __asm__ __volatile__
1154 (
1155 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
20effc67 1156 "ldrexd %1, %H1, %2\n\t"
7c673cae
FG		 1157 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1158 : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1159 "=&r" (original) // %1
20effc67 1160 : "Q" (storage) // %2
7c673cae 1161 );
20effc67 1162#endif
7c673cae
FG		 1163 fence_after(order);
1164 return original;
1165 }
1166
1167 static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1168 {
7c673cae 1169 fence_before(order);
20effc67 1170 storage_type original;
7c673cae
FG		 1171 uint32_t tmp;
1172 __asm__ __volatile__
1173 (
1174 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
20effc67
TL		 1175 "1:\n\t"
1176 "ldrexd %1, %H1, %2\n\t" // load the original value
1177 "strexd %0, %3, %H3, %2\n\t" // store the replacement, tmp = store failed
1178 "teq %0, #0\n\t" // check if store succeeded
1179 "bne 1b\n\t"
7c673cae
FG		 1180 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1181 : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
20effc67
TL		 1182 "=&r" (original), // %1
1183 "+Q" (storage) // %2
1184 : "r" (v) // %3
1185 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
7c673cae
FG		 1186 );
1187 fence_after(order);
1188 return original;
1189 }
1190
1191 static BOOST_FORCEINLINE bool compare_exchange_weak(
1192 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
1193 {
1194 fence_before(success_order);
20effc67
TL		 1195 storage_type original;
1196 bool success = false;
7c673cae 1197 uint32_t tmp;
7c673cae
FG		 1198 __asm__ __volatile__
1199 (
1200 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
20effc67
TL		 1201 "ldrexd %1, %H1, %3\n\t" // original = *(&storage)
1202 "cmp %1, %4\n\t" // flags = original.lo==expected.lo
1203 "it eq\n\t" // [hint that the following 1 instruction is conditional on flags.equal]
1204 "cmpeq %H1, %H4\n\t" // if (flags.equal) flags = original.hi==expected.hi
1205 "bne 1f\n\t"
1206 "strexd %2, %5, %H5, %3\n\t" // *(&storage) = desired, success = store failed
1207 "eor %2, %2, #1\n\t" // success ^= 1 (i.e. make it 1 if store succeeded)
1208 "1:\n\t"
7c673cae
FG		 1209 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1210 : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1211 "=&r" (original), // %1
20effc67
TL		 1212 "+r" (success), // %2
1213 "+Q" (storage) // %3
1214 : "r" (expected), // %4
1215 "r" (desired) // %5
1216 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
7c673cae 1217 );
7c673cae
FG		 1218 if (success)
1219 fence_after(success_order);
1220 else
1221 fence_after(failure_order);
1222 expected = original;
20effc67 1223 return success;
7c673cae
FG		 1224 }
1225
1226 static BOOST_FORCEINLINE bool compare_exchange_strong(
1227 storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT
1228 {
1229 fence_before(success_order);
20effc67
TL		 1230 storage_type original;
1231 bool success = false;
7c673cae 1232 uint32_t tmp;
7c673cae
FG		 1233 __asm__ __volatile__
1234 (
1235 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
20effc67
TL		 1236 "1:\n\t"
1237 "ldrexd %1, %H1, %3\n\t" // original = *(&storage)
1238 "cmp %1, %4\n\t" // flags = original.lo==expected.lo
1239 "it eq\n\t" // [hint that the following 1 instruction is conditional on flags.equal]
1240 "cmpeq %H1, %H4\n\t" // if (flags.equal) flags = original.hi==expected.hi
1241 "bne 2f\n\t"
1242 "strexd %2, %5, %H5, %3\n\t" // *(&storage) = desired, success = store failed
1243 "eors %2, %2, #1\n\t" // success ^= 1 (i.e. make it 1 if store succeeded), flags.equal = success == 0
1244 "beq 1b\n\t" // if (flags.equal) goto retry
1245 "2:\n\t"
7c673cae
FG		 1246 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1247 : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1248 "=&r" (original), // %1
20effc67
TL		 1249 "+r" (success), // %2
1250 "+Q" (storage) // %3
1251 : "r" (expected), // %4
1252 "r" (desired) // %5
1253 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
7c673cae 1254 );
7c673cae
FG		 1255 if (success)
1256 fence_after(success_order);
1257 else
1258 fence_after(failure_order);
1259 expected = original;
20effc67 1260 return success;
7c673cae
FG		 1261 }
1262
1263 static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1264 {
1265 fence_before(order);
1266 storage_type original, result;
1267 uint32_t tmp;
1268 __asm__ __volatile__
1269 (
1270 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
20effc67
TL		 1271 "1:\n\t"
1272 "ldrexd %1, %H1, %3\n\t" // original = *(&storage)
1273 "adds " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(2) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(1) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(4) "\n\t" // result = original + value
1274 "adc " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(2) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(1) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(4) "\n\t"
1275 "strexd %0, %2, %H2, %3\n\t" // *(&storage) = result, tmp = store failed
1276 "teq %0, #0\n\t" // flags = tmp==0
1277 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 1278 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1279 : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1280 "=&r" (original), // %1
20effc67
TL		 1281 "=&r" (result), // %2
1282 "+Q" (storage) // %3
1283 : "r" (v) // %4
1284 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
7c673cae
FG		 1285 );
1286 fence_after(order);
1287 return original;
1288 }
1289
1290 static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1291 {
1292 fence_before(order);
1293 storage_type original, result;
1294 uint32_t tmp;
1295 __asm__ __volatile__
1296 (
1297 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
20effc67
TL		 1298 "1:\n\t"
1299 "ldrexd %1, %H1, %3\n\t" // original = *(&storage)
1300 "subs " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(2) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(1) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_LO(4) "\n\t" // result = original - value
1301 "sbc " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(2) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(1) ", " BOOST_ATOMIC_DETAIL_ARM_ASM_ARG_HI(4) "\n\t"
1302 "strexd %0, %2, %H2, %3\n\t" // *(&storage) = result, tmp = store failed
1303 "teq %0, #0\n\t" // flags = tmp==0
1304 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 1305 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1306 : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1307 "=&r" (original), // %1
20effc67
TL		 1308 "=&r" (result), // %2
1309 "+Q" (storage) // %3
1310 : "r" (v) // %4
1311 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
7c673cae
FG		 1312 );
1313 fence_after(order);
1314 return original;
1315 }
1316
1317 static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1318 {
1319 fence_before(order);
1320 storage_type original, result;
1321 uint32_t tmp;
1322 __asm__ __volatile__
1323 (
1324 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
20effc67
TL		 1325 "1:\n\t"
1326 "ldrexd %1, %H1, %3\n\t" // original = *(&storage)
1327 "and %2, %1, %4\n\t" // result = original & value
1328 "and %H2, %H1, %H4\n\t"
1329 "strexd %0, %2, %H2, %3\n\t" // *(&storage) = result, tmp = store failed
1330 "teq %0, #0\n\t" // flags = tmp==0
1331 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 1332 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1333 : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1334 "=&r" (original), // %1
20effc67
TL		 1335 "=&r" (result), // %2
1336 "+Q" (storage) // %3
1337 : "r" (v) // %4
1338 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
7c673cae
FG		 1339 );
1340 fence_after(order);
1341 return original;
1342 }
1343
1344 static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1345 {
1346 fence_before(order);
1347 storage_type original, result;
1348 uint32_t tmp;
1349 __asm__ __volatile__
1350 (
1351 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
20effc67
TL		 1352 "1:\n\t"
1353 "ldrexd %1, %H1, %3\n\t" // original = *(&storage)
1354 "orr %2, %1, %4\n\t" // result = original | value
1355 "orr %H2, %H1, %H4\n\t"
1356 "strexd %0, %2, %H2, %3\n\t" // *(&storage) = result, tmp = store failed
1357 "teq %0, #0\n\t" // flags = tmp==0
1358 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 1359 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1360 : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1361 "=&r" (original), // %1
20effc67
TL		 1362 "=&r" (result), // %2
1363 "+Q" (storage) // %3
1364 : "r" (v) // %4
1365 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
7c673cae
FG		 1366 );
1367 fence_after(order);
1368 return original;
1369 }
1370
1371 static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
1372 {
1373 fence_before(order);
1374 storage_type original, result;
1375 uint32_t tmp;
1376 __asm__ __volatile__
1377 (
1378 BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
20effc67
TL		 1379 "1:\n\t"
1380 "ldrexd %1, %H1, %3\n\t" // original = *(&storage)
1381 "eor %2, %1, %4\n\t" // result = original ^ value
1382 "eor %H2, %H1, %H4\n\t"
1383 "strexd %0, %2, %H2, %3\n\t" // *(&storage) = result, tmp = store failed
1384 "teq %0, #0\n\t" // flags = tmp==0
1385 "bne 1b\n\t" // if (!flags.equal) goto retry
7c673cae
FG		 1386 BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
1387 : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
1388 "=&r" (original), // %1
20effc67
TL		 1389 "=&r" (result), // %2
1390 "+Q" (storage) // %3
1391 : "r" (v) // %4
1392 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
7c673cae
FG		 1393 );
1394 fence_after(order);
1395 return original;
1396 }
1397
1398 static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1399 {
1400 return !!exchange(storage, (storage_type)1, order);
1401 }
1402
1403 static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
1404 {
f67539c2 1405 store(storage, (storage_type)0, order);
7c673cae 1406 }
7c673cae
FG		 1407};
1408
1409#endif // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXD_STREXD)
1410
7c673cae
FG		 1411} // namespace detail
1412} // namespace atomics
1413} // namespace boost
1414
20effc67
TL		 1415#include <boost/atomic/detail/footer.hpp>
1416
1417#endif // BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_ARM_HPP_INCLUDED_
Ceph