]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | /* |
2 | * Distributed under the Boost Software License, Version 1.0. | |
3 | * (See accompanying file LICENSE_1_0.txt or copy at | |
4 | * http://www.boost.org/LICENSE_1_0.txt) | |
5 | * | |
6 | * Copyright (c) 2009 Helge Bahmann | |
7 | * Copyright (c) 2013 Tim Blechmann | |
8 | * Copyright (c) 2014 Andrey Semashev | |
9 | */ | |
10 | /*! | |
20effc67 | 11 | * \file atomic/detail/core_arch_ops_gcc_ppc.hpp |
7c673cae | 12 | * |
20effc67 | 13 | * This header contains implementation of the \c core_arch_operations template. |
7c673cae FG |
14 | */ |
15 | ||
20effc67 TL |
16 | #ifndef BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_PPC_HPP_INCLUDED_ |
17 | #define BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_PPC_HPP_INCLUDED_ | |
7c673cae | 18 | |
b32b8144 | 19 | #include <cstddef> |
7c673cae FG |
20 | #include <boost/memory_order.hpp> |
21 | #include <boost/atomic/detail/config.hpp> | |
f67539c2 | 22 | #include <boost/atomic/detail/storage_traits.hpp> |
20effc67 | 23 | #include <boost/atomic/detail/core_arch_operations_fwd.hpp> |
b32b8144 | 24 | #include <boost/atomic/detail/ops_gcc_ppc_common.hpp> |
1e59de90 | 25 | #include <boost/atomic/detail/gcc_ppc_asm_common.hpp> |
20effc67 TL |
26 | #include <boost/atomic/detail/capabilities.hpp> |
27 | #include <boost/atomic/detail/header.hpp> | |
7c673cae FG |
28 | |
29 | #ifdef BOOST_HAS_PRAGMA_ONCE | |
30 | #pragma once | |
31 | #endif | |
32 | ||
33 | namespace boost { | |
34 | namespace atomics { | |
35 | namespace detail { | |
36 | ||
37 | // The implementation below uses information from this document: | |
38 | // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2010.02.19a.html | |
39 | ||
40 | /* | |
41 | Refer to: Motorola: "Programming Environments Manual for 32-Bit | |
42 | Implementations of the PowerPC Architecture", Appendix E: | |
43 | "Synchronization Programming Examples" for an explanation of what is | |
44 | going on here (can be found on the web at various places by the | |
45 | name "MPCFPE32B.pdf", Google is your friend...) | |
46 | ||
47 | Most of the atomic operations map to instructions in a relatively | |
48 | straight-forward fashion, but "load"s may at first glance appear | |
49 | a bit strange as they map to: | |
50 | ||
51 | lwz %rX, addr | |
52 | cmpw %rX, %rX | |
53 | bne- 1f | |
54 | 1: | |
55 | ||
56 | That is, the CPU is forced to perform a branch that "formally" depends | |
57 | on the value retrieved from memory. This scheme has an overhead of | |
58 | about 1-2 clock cycles per load, but it allows to map "acquire" to | |
59 | the "isync" instruction instead of "sync" uniformly and for all type | |
60 | of atomic operations. Since "isync" has a cost of about 15 clock | |
61 | cycles, while "sync" hast a cost of about 50 clock cycles, the small | |
62 | penalty to atomic loads more than compensates for this. | |
63 | ||
b32b8144 FG |
64 | Byte- and halfword-sized atomic values are implemented in two ways. |
65 | When 8 and 16-bit instructions are available (in Power8 and later), | |
66 | they are used. Otherwise operations are realized by encoding the | |
7c673cae FG |
67 | value to be represented into a word, performing sign/zero extension |
68 | as appropriate. This means that after add/sub operations the value | |
69 | needs fixing up to accurately preserve the wrap-around semantic of | |
70 | the smaller type. (Nothing special needs to be done for the bit-wise | |
71 | and the "exchange type" operators as the compiler already sees to | |
72 | it that values carried in registers are extended appropriately and | |
73 | everything falls into place naturally). | |
74 | ||
75 | The register constraint "b" instructs gcc to use any register | |
76 | except r0; this is sometimes required because the encoding for | |
77 | r0 is used to signify "constant zero" in a number of instructions, | |
78 | making r0 unusable in this place. For simplicity this constraint | |
79 | is used everywhere since I am to lazy to look this up on a | |
80 | per-instruction basis, and ppc has enough registers for this not | |
81 | to pose a problem. | |
82 | */ | |
83 | ||
20effc67 TL |
84 | template< bool Signed, bool Interprocess > |
85 | struct core_arch_operations< 4u, Signed, Interprocess > : | |
86 | public core_arch_operations_gcc_ppc_base | |
7c673cae | 87 | { |
f67539c2 | 88 | typedef typename storage_traits< 4u >::type storage_type; |
7c673cae | 89 | |
b32b8144 | 90 | static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 4u; |
f67539c2 | 91 | static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 4u; |
b32b8144 | 92 | static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed; |
20effc67 | 93 | static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess; |
b32b8144 | 94 | |
7c673cae FG |
95 | static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT |
96 | { | |
97 | fence_before(order); | |
98 | __asm__ __volatile__ | |
99 | ( | |
100 | "stw %1, %0\n\t" | |
101 | : "+m" (storage) | |
102 | : "r" (v) | |
103 | ); | |
104 | } | |
105 | ||
106 | static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT | |
107 | { | |
108 | storage_type v; | |
109 | if (order == memory_order_seq_cst) | |
110 | __asm__ __volatile__ ("sync" ::: "memory"); | |
11fdf7f2 | 111 | if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u) |
7c673cae FG |
112 | { |
113 | __asm__ __volatile__ | |
114 | ( | |
115 | "lwz %0, %1\n\t" | |
116 | "cmpw %0, %0\n\t" | |
1e59de90 TL |
117 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+4") |
118 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") | |
7c673cae FG |
119 | "isync\n\t" |
120 | : "=&r" (v) | |
121 | : "m" (storage) | |
122 | : "cr0", "memory" | |
123 | ); | |
124 | } | |
125 | else | |
126 | { | |
127 | __asm__ __volatile__ | |
128 | ( | |
129 | "lwz %0, %1\n\t" | |
130 | : "=&r" (v) | |
131 | : "m" (storage) | |
132 | ); | |
133 | } | |
134 | return v; | |
135 | } | |
136 | ||
137 | static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
138 | { | |
139 | storage_type original; | |
140 | fence_before(order); | |
141 | __asm__ __volatile__ | |
142 | ( | |
1e59de90 | 143 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
144 | "lwarx %0,%y1\n\t" |
145 | "stwcx. %2,%y1\n\t" | |
1e59de90 | 146 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-8") |
7c673cae FG |
147 | : "=&b" (original), "+Z" (storage) |
148 | : "b" (v) | |
149 | : "cr0" | |
150 | ); | |
151 | fence_after(order); | |
152 | return original; | |
153 | } | |
154 | ||
155 | static BOOST_FORCEINLINE bool compare_exchange_weak( | |
156 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT | |
157 | { | |
158 | int success; | |
159 | fence_before(success_order); | |
160 | __asm__ __volatile__ | |
161 | ( | |
162 | "li %1, 0\n\t" | |
163 | "lwarx %0,%y2\n\t" | |
164 | "cmpw %0, %3\n\t" | |
1e59de90 | 165 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+16") |
7c673cae | 166 | "stwcx. %4,%y2\n\t" |
1e59de90 | 167 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+8") |
7c673cae | 168 | "li %1, 1\n\t" |
1e59de90 | 169 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
170 | : "=&b" (expected), "=&b" (success), "+Z" (storage) |
171 | : "b" (expected), "b" (desired) | |
172 | : "cr0" | |
173 | ); | |
174 | if (success) | |
175 | fence_after(success_order); | |
176 | else | |
177 | fence_after(failure_order); | |
178 | return !!success; | |
179 | } | |
180 | ||
181 | static BOOST_FORCEINLINE bool compare_exchange_strong( | |
182 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT | |
183 | { | |
184 | int success; | |
185 | fence_before(success_order); | |
186 | __asm__ __volatile__ | |
187 | ( | |
188 | "li %1, 0\n\t" | |
1e59de90 TL |
189 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("0") |
190 | "lwarx %0,%y2\n\t" | |
7c673cae | 191 | "cmpw %0, %3\n\t" |
1e59de90 | 192 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+16") |
7c673cae | 193 | "stwcx. %4,%y2\n\t" |
1e59de90 | 194 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "0b", "-16") |
7c673cae | 195 | "li %1, 1\n\t" |
1e59de90 | 196 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
197 | : "=&b" (expected), "=&b" (success), "+Z" (storage) |
198 | : "b" (expected), "b" (desired) | |
199 | : "cr0" | |
200 | ); | |
201 | if (success) | |
202 | fence_after(success_order); | |
203 | else | |
204 | fence_after(failure_order); | |
205 | return !!success; | |
206 | } | |
207 | ||
208 | static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
209 | { | |
11fdf7f2 | 210 | storage_type original, result; |
7c673cae FG |
211 | fence_before(order); |
212 | __asm__ __volatile__ | |
213 | ( | |
1e59de90 | 214 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
215 | "lwarx %0,%y2\n\t" |
216 | "add %1,%0,%3\n\t" | |
217 | "stwcx. %1,%y2\n\t" | |
1e59de90 | 218 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 219 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
220 | : "b" (v) |
221 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
222 | ); | |
223 | fence_after(order); | |
224 | return original; | |
225 | } | |
226 | ||
227 | static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
228 | { | |
11fdf7f2 | 229 | storage_type original, result; |
7c673cae FG |
230 | fence_before(order); |
231 | __asm__ __volatile__ | |
232 | ( | |
1e59de90 | 233 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
234 | "lwarx %0,%y2\n\t" |
235 | "sub %1,%0,%3\n\t" | |
236 | "stwcx. %1,%y2\n\t" | |
1e59de90 | 237 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 238 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
239 | : "b" (v) |
240 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
241 | ); | |
242 | fence_after(order); | |
243 | return original; | |
244 | } | |
245 | ||
246 | static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
247 | { | |
11fdf7f2 | 248 | storage_type original, result; |
7c673cae FG |
249 | fence_before(order); |
250 | __asm__ __volatile__ | |
251 | ( | |
1e59de90 | 252 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
253 | "lwarx %0,%y2\n\t" |
254 | "and %1,%0,%3\n\t" | |
255 | "stwcx. %1,%y2\n\t" | |
1e59de90 | 256 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 257 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
258 | : "b" (v) |
259 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
260 | ); | |
261 | fence_after(order); | |
262 | return original; | |
263 | } | |
264 | ||
265 | static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
266 | { | |
11fdf7f2 | 267 | storage_type original, result; |
7c673cae FG |
268 | fence_before(order); |
269 | __asm__ __volatile__ | |
270 | ( | |
1e59de90 | 271 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
272 | "lwarx %0,%y2\n\t" |
273 | "or %1,%0,%3\n\t" | |
274 | "stwcx. %1,%y2\n\t" | |
1e59de90 | 275 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 276 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
277 | : "b" (v) |
278 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
279 | ); | |
280 | fence_after(order); | |
281 | return original; | |
282 | } | |
283 | ||
284 | static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
285 | { | |
11fdf7f2 | 286 | storage_type original, result; |
7c673cae FG |
287 | fence_before(order); |
288 | __asm__ __volatile__ | |
289 | ( | |
1e59de90 | 290 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
291 | "lwarx %0,%y2\n\t" |
292 | "xor %1,%0,%3\n\t" | |
293 | "stwcx. %1,%y2\n\t" | |
1e59de90 | 294 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 295 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
296 | : "b" (v) |
297 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
298 | ); | |
299 | fence_after(order); | |
300 | return original; | |
301 | } | |
302 | ||
303 | static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT | |
304 | { | |
305 | return !!exchange(storage, (storage_type)1, order); | |
306 | } | |
307 | ||
308 | static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT | |
309 | { | |
f67539c2 | 310 | store(storage, (storage_type)0, order); |
7c673cae | 311 | } |
b32b8144 FG |
312 | }; |
313 | ||
314 | #if defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LBARX_STBCX) | |
315 | ||
20effc67 TL |
316 | template< bool Signed, bool Interprocess > |
317 | struct core_arch_operations< 1u, Signed, Interprocess > : | |
318 | public core_arch_operations_gcc_ppc_base | |
b32b8144 | 319 | { |
f67539c2 | 320 | typedef typename storage_traits< 1u >::type storage_type; |
b32b8144 FG |
321 | |
322 | static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 1u; | |
f67539c2 | 323 | static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 1u; |
b32b8144 | 324 | static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed; |
20effc67 | 325 | static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess; |
b32b8144 FG |
326 | |
327 | static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
328 | { | |
329 | fence_before(order); | |
330 | __asm__ __volatile__ | |
331 | ( | |
332 | "stb %1, %0\n\t" | |
333 | : "+m" (storage) | |
334 | : "r" (v) | |
335 | ); | |
336 | } | |
337 | ||
338 | static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT | |
339 | { | |
340 | storage_type v; | |
341 | if (order == memory_order_seq_cst) | |
342 | __asm__ __volatile__ ("sync" ::: "memory"); | |
11fdf7f2 | 343 | if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u) |
b32b8144 FG |
344 | { |
345 | __asm__ __volatile__ | |
346 | ( | |
347 | "lbz %0, %1\n\t" | |
348 | "cmpw %0, %0\n\t" | |
1e59de90 TL |
349 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+4") |
350 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") | |
b32b8144 FG |
351 | "isync\n\t" |
352 | : "=&r" (v) | |
353 | : "m" (storage) | |
354 | : "cr0", "memory" | |
355 | ); | |
356 | } | |
357 | else | |
358 | { | |
359 | __asm__ __volatile__ | |
360 | ( | |
361 | "lbz %0, %1\n\t" | |
362 | : "=&r" (v) | |
363 | : "m" (storage) | |
364 | ); | |
365 | } | |
366 | return v; | |
367 | } | |
368 | ||
369 | static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
370 | { | |
371 | storage_type original; | |
372 | fence_before(order); | |
373 | __asm__ __volatile__ | |
374 | ( | |
1e59de90 | 375 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
b32b8144 FG |
376 | "lbarx %0,%y1\n\t" |
377 | "stbcx. %2,%y1\n\t" | |
1e59de90 | 378 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-8") |
b32b8144 FG |
379 | : "=&b" (original), "+Z" (storage) |
380 | : "b" (v) | |
381 | : "cr0" | |
382 | ); | |
383 | fence_after(order); | |
384 | return original; | |
385 | } | |
386 | ||
387 | static BOOST_FORCEINLINE bool compare_exchange_weak( | |
388 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT | |
389 | { | |
390 | int success; | |
391 | fence_before(success_order); | |
392 | __asm__ __volatile__ | |
393 | ( | |
394 | "li %1, 0\n\t" | |
395 | "lbarx %0,%y2\n\t" | |
396 | "cmpw %0, %3\n\t" | |
1e59de90 | 397 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+16") |
b32b8144 | 398 | "stbcx. %4,%y2\n\t" |
1e59de90 | 399 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+8") |
b32b8144 | 400 | "li %1, 1\n\t" |
1e59de90 | 401 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
b32b8144 | 402 | : "=&b" (expected), "=&b" (success), "+Z" (storage) |
11fdf7f2 | 403 | : "b" (expected), "b" (desired) |
b32b8144 FG |
404 | : "cr0" |
405 | ); | |
406 | if (success) | |
407 | fence_after(success_order); | |
408 | else | |
409 | fence_after(failure_order); | |
410 | return !!success; | |
411 | } | |
412 | ||
413 | static BOOST_FORCEINLINE bool compare_exchange_strong( | |
414 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT | |
415 | { | |
416 | int success; | |
417 | fence_before(success_order); | |
418 | __asm__ __volatile__ | |
419 | ( | |
420 | "li %1, 0\n\t" | |
1e59de90 TL |
421 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("0") |
422 | "lbarx %0,%y2\n\t" | |
b32b8144 | 423 | "cmpw %0, %3\n\t" |
1e59de90 | 424 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+16") |
b32b8144 | 425 | "stbcx. %4,%y2\n\t" |
1e59de90 | 426 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "0b", "-16") |
b32b8144 | 427 | "li %1, 1\n\t" |
1e59de90 | 428 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
b32b8144 | 429 | : "=&b" (expected), "=&b" (success), "+Z" (storage) |
11fdf7f2 | 430 | : "b" (expected), "b" (desired) |
b32b8144 FG |
431 | : "cr0" |
432 | ); | |
433 | if (success) | |
434 | fence_after(success_order); | |
435 | else | |
436 | fence_after(failure_order); | |
437 | return !!success; | |
438 | } | |
439 | ||
440 | static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
441 | { | |
11fdf7f2 | 442 | storage_type original, result; |
b32b8144 FG |
443 | fence_before(order); |
444 | __asm__ __volatile__ | |
445 | ( | |
1e59de90 | 446 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
b32b8144 FG |
447 | "lbarx %0,%y2\n\t" |
448 | "add %1,%0,%3\n\t" | |
449 | "stbcx. %1,%y2\n\t" | |
1e59de90 | 450 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 451 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
b32b8144 FG |
452 | : "b" (v) |
453 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
454 | ); | |
455 | fence_after(order); | |
456 | return original; | |
457 | } | |
458 | ||
459 | static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
460 | { | |
11fdf7f2 | 461 | storage_type original, result; |
b32b8144 FG |
462 | fence_before(order); |
463 | __asm__ __volatile__ | |
464 | ( | |
1e59de90 | 465 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
b32b8144 FG |
466 | "lbarx %0,%y2\n\t" |
467 | "sub %1,%0,%3\n\t" | |
468 | "stbcx. %1,%y2\n\t" | |
1e59de90 | 469 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 470 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
b32b8144 FG |
471 | : "b" (v) |
472 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
473 | ); | |
474 | fence_after(order); | |
475 | return original; | |
476 | } | |
477 | ||
478 | static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
479 | { | |
11fdf7f2 | 480 | storage_type original, result; |
b32b8144 FG |
481 | fence_before(order); |
482 | __asm__ __volatile__ | |
483 | ( | |
1e59de90 | 484 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
b32b8144 FG |
485 | "lbarx %0,%y2\n\t" |
486 | "and %1,%0,%3\n\t" | |
487 | "stbcx. %1,%y2\n\t" | |
1e59de90 | 488 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 489 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
b32b8144 FG |
490 | : "b" (v) |
491 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
492 | ); | |
493 | fence_after(order); | |
494 | return original; | |
495 | } | |
496 | ||
497 | static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
498 | { | |
11fdf7f2 | 499 | storage_type original, result; |
b32b8144 FG |
500 | fence_before(order); |
501 | __asm__ __volatile__ | |
502 | ( | |
1e59de90 | 503 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
b32b8144 FG |
504 | "lbarx %0,%y2\n\t" |
505 | "or %1,%0,%3\n\t" | |
506 | "stbcx. %1,%y2\n\t" | |
1e59de90 | 507 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 508 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
b32b8144 FG |
509 | : "b" (v) |
510 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
511 | ); | |
512 | fence_after(order); | |
513 | return original; | |
514 | } | |
515 | ||
516 | static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
517 | { | |
11fdf7f2 | 518 | storage_type original, result; |
b32b8144 FG |
519 | fence_before(order); |
520 | __asm__ __volatile__ | |
521 | ( | |
1e59de90 | 522 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
b32b8144 FG |
523 | "lbarx %0,%y2\n\t" |
524 | "xor %1,%0,%3\n\t" | |
525 | "stbcx. %1,%y2\n\t" | |
1e59de90 | 526 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 527 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
b32b8144 FG |
528 | : "b" (v) |
529 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
530 | ); | |
531 | fence_after(order); | |
532 | return original; | |
533 | } | |
534 | ||
535 | static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT | |
536 | { | |
537 | return !!exchange(storage, (storage_type)1, order); | |
538 | } | |
7c673cae | 539 | |
b32b8144 | 540 | static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT |
7c673cae | 541 | { |
f67539c2 | 542 | store(storage, (storage_type)0, order); |
7c673cae FG |
543 | } |
544 | }; | |
545 | ||
b32b8144 | 546 | #else // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LBARX_STBCX) |
7c673cae | 547 | |
20effc67 TL |
548 | template< bool Interprocess > |
549 | struct core_arch_operations< 1u, false, Interprocess > : | |
550 | public core_arch_operations< 4u, false, Interprocess > | |
7c673cae | 551 | { |
20effc67 TL |
552 | typedef core_arch_operations< 4u, false, Interprocess > base_type; |
553 | typedef typename base_type::storage_type storage_type; | |
7c673cae FG |
554 | |
555 | static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
556 | { | |
11fdf7f2 | 557 | storage_type original, result; |
20effc67 | 558 | base_type::fence_before(order); |
7c673cae FG |
559 | __asm__ __volatile__ |
560 | ( | |
1e59de90 | 561 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
562 | "lwarx %0,%y2\n\t" |
563 | "add %1,%0,%3\n\t" | |
564 | "rlwinm %1, %1, 0, 0xff\n\t" | |
565 | "stwcx. %1,%y2\n\t" | |
1e59de90 | 566 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-16") |
11fdf7f2 | 567 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
568 | : "b" (v) |
569 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
570 | ); | |
20effc67 | 571 | base_type::fence_after(order); |
7c673cae FG |
572 | return original; |
573 | } | |
574 | ||
575 | static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
576 | { | |
11fdf7f2 | 577 | storage_type original, result; |
20effc67 | 578 | base_type::fence_before(order); |
7c673cae FG |
579 | __asm__ __volatile__ |
580 | ( | |
1e59de90 | 581 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
582 | "lwarx %0,%y2\n\t" |
583 | "sub %1,%0,%3\n\t" | |
584 | "rlwinm %1, %1, 0, 0xff\n\t" | |
585 | "stwcx. %1,%y2\n\t" | |
1e59de90 | 586 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-16") |
11fdf7f2 | 587 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
588 | : "b" (v) |
589 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
590 | ); | |
20effc67 | 591 | base_type::fence_after(order); |
7c673cae FG |
592 | return original; |
593 | } | |
594 | }; | |
595 | ||
20effc67 TL |
596 | template< bool Interprocess > |
597 | struct core_arch_operations< 1u, true, Interprocess > : | |
598 | public core_arch_operations< 4u, true, Interprocess > | |
7c673cae | 599 | { |
20effc67 TL |
600 | typedef core_arch_operations< 4u, true, Interprocess > base_type; |
601 | typedef typename base_type::storage_type storage_type; | |
7c673cae FG |
602 | |
603 | static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
604 | { | |
11fdf7f2 | 605 | storage_type original, result; |
20effc67 | 606 | base_type::fence_before(order); |
7c673cae FG |
607 | __asm__ __volatile__ |
608 | ( | |
1e59de90 | 609 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
610 | "lwarx %0,%y2\n\t" |
611 | "add %1,%0,%3\n\t" | |
612 | "extsb %1, %1\n\t" | |
613 | "stwcx. %1,%y2\n\t" | |
1e59de90 | 614 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-16") |
11fdf7f2 | 615 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
616 | : "b" (v) |
617 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
618 | ); | |
20effc67 | 619 | base_type::fence_after(order); |
7c673cae FG |
620 | return original; |
621 | } | |
622 | ||
623 | static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
624 | { | |
11fdf7f2 | 625 | storage_type original, result; |
20effc67 | 626 | base_type::fence_before(order); |
7c673cae FG |
627 | __asm__ __volatile__ |
628 | ( | |
1e59de90 | 629 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
630 | "lwarx %0,%y2\n\t" |
631 | "sub %1,%0,%3\n\t" | |
632 | "extsb %1, %1\n\t" | |
633 | "stwcx. %1,%y2\n\t" | |
1e59de90 | 634 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-16") |
11fdf7f2 | 635 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
636 | : "b" (v) |
637 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
638 | ); | |
20effc67 | 639 | base_type::fence_after(order); |
7c673cae FG |
640 | return original; |
641 | } | |
642 | }; | |
643 | ||
b32b8144 FG |
644 | #endif // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LBARX_STBCX) |
645 | ||
646 | #if defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LHARX_STHCX) | |
647 | ||
20effc67 TL |
648 | template< bool Signed, bool Interprocess > |
649 | struct core_arch_operations< 2u, Signed, Interprocess > : | |
650 | public core_arch_operations_gcc_ppc_base | |
b32b8144 | 651 | { |
f67539c2 | 652 | typedef typename storage_traits< 2u >::type storage_type; |
b32b8144 FG |
653 | |
654 | static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 2u; | |
f67539c2 | 655 | static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 2u; |
b32b8144 | 656 | static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed; |
20effc67 | 657 | static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess; |
b32b8144 FG |
658 | |
659 | static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
660 | { | |
661 | fence_before(order); | |
662 | __asm__ __volatile__ | |
663 | ( | |
664 | "sth %1, %0\n\t" | |
665 | : "+m" (storage) | |
666 | : "r" (v) | |
667 | ); | |
668 | } | |
669 | ||
670 | static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT | |
671 | { | |
672 | storage_type v; | |
673 | if (order == memory_order_seq_cst) | |
674 | __asm__ __volatile__ ("sync" ::: "memory"); | |
11fdf7f2 | 675 | if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u) |
b32b8144 FG |
676 | { |
677 | __asm__ __volatile__ | |
678 | ( | |
679 | "lhz %0, %1\n\t" | |
680 | "cmpw %0, %0\n\t" | |
1e59de90 TL |
681 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+4") |
682 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") | |
b32b8144 FG |
683 | "isync\n\t" |
684 | : "=&r" (v) | |
685 | : "m" (storage) | |
686 | : "cr0", "memory" | |
687 | ); | |
688 | } | |
689 | else | |
690 | { | |
691 | __asm__ __volatile__ | |
692 | ( | |
693 | "lhz %0, %1\n\t" | |
694 | : "=&r" (v) | |
695 | : "m" (storage) | |
696 | ); | |
697 | } | |
698 | return v; | |
699 | } | |
700 | ||
701 | static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
702 | { | |
703 | storage_type original; | |
704 | fence_before(order); | |
705 | __asm__ __volatile__ | |
706 | ( | |
1e59de90 | 707 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
b32b8144 FG |
708 | "lharx %0,%y1\n\t" |
709 | "sthcx. %2,%y1\n\t" | |
1e59de90 | 710 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-8") |
b32b8144 FG |
711 | : "=&b" (original), "+Z" (storage) |
712 | : "b" (v) | |
713 | : "cr0" | |
714 | ); | |
715 | fence_after(order); | |
716 | return original; | |
717 | } | |
718 | ||
719 | static BOOST_FORCEINLINE bool compare_exchange_weak( | |
720 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT | |
721 | { | |
722 | int success; | |
723 | fence_before(success_order); | |
724 | __asm__ __volatile__ | |
725 | ( | |
726 | "li %1, 0\n\t" | |
727 | "lharx %0,%y2\n\t" | |
728 | "cmpw %0, %3\n\t" | |
1e59de90 | 729 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+16") |
b32b8144 | 730 | "sthcx. %4,%y2\n\t" |
1e59de90 | 731 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+8") |
b32b8144 | 732 | "li %1, 1\n\t" |
1e59de90 | 733 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
b32b8144 | 734 | : "=&b" (expected), "=&b" (success), "+Z" (storage) |
11fdf7f2 | 735 | : "b" (expected), "b" (desired) |
b32b8144 FG |
736 | : "cr0" |
737 | ); | |
738 | if (success) | |
739 | fence_after(success_order); | |
740 | else | |
741 | fence_after(failure_order); | |
742 | return !!success; | |
743 | } | |
744 | ||
745 | static BOOST_FORCEINLINE bool compare_exchange_strong( | |
746 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT | |
747 | { | |
748 | int success; | |
749 | fence_before(success_order); | |
750 | __asm__ __volatile__ | |
751 | ( | |
752 | "li %1, 0\n\t" | |
1e59de90 TL |
753 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("0") |
754 | "lharx %0,%y2\n\t" | |
b32b8144 | 755 | "cmpw %0, %3\n\t" |
1e59de90 | 756 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+16") |
b32b8144 | 757 | "sthcx. %4,%y2\n\t" |
1e59de90 | 758 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "0b", "-16") |
b32b8144 | 759 | "li %1, 1\n\t" |
1e59de90 | 760 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
b32b8144 | 761 | : "=&b" (expected), "=&b" (success), "+Z" (storage) |
11fdf7f2 | 762 | : "b" (expected), "b" (desired) |
b32b8144 FG |
763 | : "cr0" |
764 | ); | |
765 | if (success) | |
766 | fence_after(success_order); | |
767 | else | |
768 | fence_after(failure_order); | |
769 | return !!success; | |
770 | } | |
771 | ||
772 | static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
773 | { | |
11fdf7f2 | 774 | storage_type original, result; |
b32b8144 FG |
775 | fence_before(order); |
776 | __asm__ __volatile__ | |
777 | ( | |
1e59de90 | 778 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
b32b8144 FG |
779 | "lharx %0,%y2\n\t" |
780 | "add %1,%0,%3\n\t" | |
781 | "sthcx. %1,%y2\n\t" | |
1e59de90 | 782 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 783 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
b32b8144 FG |
784 | : "b" (v) |
785 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
786 | ); | |
787 | fence_after(order); | |
788 | return original; | |
789 | } | |
790 | ||
791 | static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
792 | { | |
11fdf7f2 | 793 | storage_type original, result; |
b32b8144 FG |
794 | fence_before(order); |
795 | __asm__ __volatile__ | |
796 | ( | |
1e59de90 | 797 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
b32b8144 FG |
798 | "lharx %0,%y2\n\t" |
799 | "sub %1,%0,%3\n\t" | |
800 | "sthcx. %1,%y2\n\t" | |
1e59de90 | 801 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 802 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
b32b8144 FG |
803 | : "b" (v) |
804 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
805 | ); | |
806 | fence_after(order); | |
807 | return original; | |
808 | } | |
809 | ||
810 | static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
811 | { | |
11fdf7f2 | 812 | storage_type original, result; |
b32b8144 FG |
813 | fence_before(order); |
814 | __asm__ __volatile__ | |
815 | ( | |
1e59de90 | 816 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
b32b8144 FG |
817 | "lharx %0,%y2\n\t" |
818 | "and %1,%0,%3\n\t" | |
819 | "sthcx. %1,%y2\n\t" | |
1e59de90 | 820 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 821 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
b32b8144 FG |
822 | : "b" (v) |
823 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
824 | ); | |
825 | fence_after(order); | |
826 | return original; | |
827 | } | |
828 | ||
829 | static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
830 | { | |
11fdf7f2 | 831 | storage_type original, result; |
b32b8144 FG |
832 | fence_before(order); |
833 | __asm__ __volatile__ | |
834 | ( | |
1e59de90 | 835 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
b32b8144 FG |
836 | "lharx %0,%y2\n\t" |
837 | "or %1,%0,%3\n\t" | |
838 | "sthcx. %1,%y2\n\t" | |
1e59de90 | 839 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 840 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
b32b8144 FG |
841 | : "b" (v) |
842 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
843 | ); | |
844 | fence_after(order); | |
845 | return original; | |
846 | } | |
847 | ||
848 | static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
849 | { | |
11fdf7f2 | 850 | storage_type original, result; |
b32b8144 FG |
851 | fence_before(order); |
852 | __asm__ __volatile__ | |
853 | ( | |
1e59de90 | 854 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
b32b8144 FG |
855 | "lharx %0,%y2\n\t" |
856 | "xor %1,%0,%3\n\t" | |
857 | "sthcx. %1,%y2\n\t" | |
1e59de90 | 858 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 859 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
b32b8144 FG |
860 | : "b" (v) |
861 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
862 | ); | |
863 | fence_after(order); | |
864 | return original; | |
865 | } | |
866 | ||
867 | static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT | |
868 | { | |
869 | return !!exchange(storage, (storage_type)1, order); | |
870 | } | |
871 | ||
872 | static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT | |
873 | { | |
f67539c2 | 874 | store(storage, (storage_type)0, order); |
b32b8144 FG |
875 | } |
876 | }; | |
877 | ||
878 | #else // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LHARX_STHCX) | |
7c673cae | 879 | |
20effc67 TL |
880 | template< bool Interprocess > |
881 | struct core_arch_operations< 2u, false, Interprocess > : | |
882 | public core_arch_operations< 4u, false, Interprocess > | |
7c673cae | 883 | { |
20effc67 TL |
884 | typedef core_arch_operations< 4u, false, Interprocess > base_type; |
885 | typedef typename base_type::storage_type storage_type; | |
7c673cae FG |
886 | |
887 | static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
888 | { | |
11fdf7f2 | 889 | storage_type original, result; |
20effc67 | 890 | base_type::fence_before(order); |
7c673cae FG |
891 | __asm__ __volatile__ |
892 | ( | |
1e59de90 | 893 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
894 | "lwarx %0,%y2\n\t" |
895 | "add %1,%0,%3\n\t" | |
896 | "rlwinm %1, %1, 0, 0xffff\n\t" | |
897 | "stwcx. %1,%y2\n\t" | |
1e59de90 | 898 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-16") |
11fdf7f2 | 899 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
900 | : "b" (v) |
901 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
902 | ); | |
20effc67 | 903 | base_type::fence_after(order); |
7c673cae FG |
904 | return original; |
905 | } | |
906 | ||
907 | static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
908 | { | |
11fdf7f2 | 909 | storage_type original, result; |
20effc67 | 910 | base_type::fence_before(order); |
7c673cae FG |
911 | __asm__ __volatile__ |
912 | ( | |
1e59de90 | 913 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
914 | "lwarx %0,%y2\n\t" |
915 | "sub %1,%0,%3\n\t" | |
916 | "rlwinm %1, %1, 0, 0xffff\n\t" | |
917 | "stwcx. %1,%y2\n\t" | |
1e59de90 | 918 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-16") |
11fdf7f2 | 919 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
920 | : "b" (v) |
921 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
922 | ); | |
20effc67 | 923 | base_type::fence_after(order); |
7c673cae FG |
924 | return original; |
925 | } | |
926 | }; | |
927 | ||
20effc67 TL |
928 | template< bool Interprocess > |
929 | struct core_arch_operations< 2u, true, Interprocess > : | |
930 | public core_arch_operations< 4u, true, Interprocess > | |
7c673cae | 931 | { |
20effc67 TL |
932 | typedef core_arch_operations< 4u, true, Interprocess > base_type; |
933 | typedef typename base_type::storage_type storage_type; | |
7c673cae FG |
934 | |
935 | static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
936 | { | |
11fdf7f2 | 937 | storage_type original, result; |
20effc67 | 938 | base_type::fence_before(order); |
7c673cae FG |
939 | __asm__ __volatile__ |
940 | ( | |
1e59de90 | 941 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
942 | "lwarx %0,%y2\n\t" |
943 | "add %1,%0,%3\n\t" | |
944 | "extsh %1, %1\n\t" | |
945 | "stwcx. %1,%y2\n\t" | |
1e59de90 | 946 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-16") |
11fdf7f2 | 947 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
948 | : "b" (v) |
949 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
950 | ); | |
20effc67 | 951 | base_type::fence_after(order); |
7c673cae FG |
952 | return original; |
953 | } | |
954 | ||
955 | static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
956 | { | |
11fdf7f2 | 957 | storage_type original, result; |
20effc67 | 958 | base_type::fence_before(order); |
7c673cae FG |
959 | __asm__ __volatile__ |
960 | ( | |
1e59de90 | 961 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
962 | "lwarx %0,%y2\n\t" |
963 | "sub %1,%0,%3\n\t" | |
964 | "extsh %1, %1\n\t" | |
965 | "stwcx. %1,%y2\n\t" | |
1e59de90 | 966 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-16") |
11fdf7f2 | 967 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
968 | : "b" (v) |
969 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
970 | ); | |
20effc67 | 971 | base_type::fence_after(order); |
7c673cae FG |
972 | return original; |
973 | } | |
974 | }; | |
975 | ||
b32b8144 | 976 | #endif // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LHARX_STHCX) |
7c673cae | 977 | |
b32b8144 | 978 | #if defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LDARX_STDCX) |
7c673cae | 979 | |
20effc67 TL |
980 | template< bool Signed, bool Interprocess > |
981 | struct core_arch_operations< 8u, Signed, Interprocess > : | |
982 | public core_arch_operations_gcc_ppc_base | |
7c673cae | 983 | { |
f67539c2 | 984 | typedef typename storage_traits< 8u >::type storage_type; |
7c673cae | 985 | |
b32b8144 | 986 | static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 8u; |
f67539c2 | 987 | static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 8u; |
b32b8144 | 988 | static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed; |
20effc67 | 989 | static BOOST_CONSTEXPR_OR_CONST bool is_interprocess = Interprocess; |
b32b8144 | 990 | |
7c673cae FG |
991 | static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT |
992 | { | |
993 | fence_before(order); | |
994 | __asm__ __volatile__ | |
995 | ( | |
996 | "std %1, %0\n\t" | |
997 | : "+m" (storage) | |
998 | : "r" (v) | |
999 | ); | |
1000 | } | |
1001 | ||
1002 | static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order order) BOOST_NOEXCEPT | |
1003 | { | |
1004 | storage_type v; | |
1005 | if (order == memory_order_seq_cst) | |
1006 | __asm__ __volatile__ ("sync" ::: "memory"); | |
11fdf7f2 | 1007 | if ((static_cast< unsigned int >(order) & (static_cast< unsigned int >(memory_order_consume) | static_cast< unsigned int >(memory_order_acquire))) != 0u) |
7c673cae FG |
1008 | { |
1009 | __asm__ __volatile__ | |
1010 | ( | |
1011 | "ld %0, %1\n\t" | |
1012 | "cmpd %0, %0\n\t" | |
1e59de90 TL |
1013 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+4") |
1014 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") | |
7c673cae FG |
1015 | "isync\n\t" |
1016 | : "=&b" (v) | |
1017 | : "m" (storage) | |
1018 | : "cr0", "memory" | |
1019 | ); | |
1020 | } | |
1021 | else | |
1022 | { | |
1023 | __asm__ __volatile__ | |
1024 | ( | |
1025 | "ld %0, %1\n\t" | |
1026 | : "=&b" (v) | |
1027 | : "m" (storage) | |
1028 | ); | |
1029 | } | |
1030 | return v; | |
1031 | } | |
1032 | ||
1033 | static BOOST_FORCEINLINE storage_type exchange(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
1034 | { | |
1035 | storage_type original; | |
1036 | fence_before(order); | |
1037 | __asm__ __volatile__ | |
1038 | ( | |
1e59de90 | 1039 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
1040 | "ldarx %0,%y1\n\t" |
1041 | "stdcx. %2,%y1\n\t" | |
1e59de90 | 1042 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-8") |
7c673cae FG |
1043 | : "=&b" (original), "+Z" (storage) |
1044 | : "b" (v) | |
1045 | : "cr0" | |
1046 | ); | |
1047 | fence_after(order); | |
1048 | return original; | |
1049 | } | |
1050 | ||
1051 | static BOOST_FORCEINLINE bool compare_exchange_weak( | |
1052 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT | |
1053 | { | |
1054 | int success; | |
1055 | fence_before(success_order); | |
1056 | __asm__ __volatile__ | |
1057 | ( | |
1058 | "li %1, 0\n\t" | |
1059 | "ldarx %0,%y2\n\t" | |
1060 | "cmpd %0, %3\n\t" | |
1e59de90 | 1061 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+16") |
7c673cae | 1062 | "stdcx. %4,%y2\n\t" |
1e59de90 | 1063 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+8") |
7c673cae | 1064 | "li %1, 1\n\t" |
1e59de90 | 1065 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
1066 | : "=&b" (expected), "=&b" (success), "+Z" (storage) |
1067 | : "b" (expected), "b" (desired) | |
1068 | : "cr0" | |
1069 | ); | |
1070 | if (success) | |
1071 | fence_after(success_order); | |
1072 | else | |
1073 | fence_after(failure_order); | |
1074 | return !!success; | |
1075 | } | |
1076 | ||
1077 | static BOOST_FORCEINLINE bool compare_exchange_strong( | |
1078 | storage_type volatile& storage, storage_type& expected, storage_type desired, memory_order success_order, memory_order failure_order) BOOST_NOEXCEPT | |
1079 | { | |
1080 | int success; | |
1081 | fence_before(success_order); | |
1082 | __asm__ __volatile__ | |
1083 | ( | |
1084 | "li %1, 0\n\t" | |
1e59de90 TL |
1085 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("0") |
1086 | "ldarx %0,%y2\n\t" | |
7c673cae | 1087 | "cmpd %0, %3\n\t" |
1e59de90 | 1088 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1f", "+16") |
7c673cae | 1089 | "stdcx. %4,%y2\n\t" |
1e59de90 | 1090 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "0b", "-16") |
7c673cae | 1091 | "li %1, 1\n\t" |
1e59de90 | 1092 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
1093 | : "=&b" (expected), "=&b" (success), "+Z" (storage) |
1094 | : "b" (expected), "b" (desired) | |
1095 | : "cr0" | |
1096 | ); | |
1097 | if (success) | |
1098 | fence_after(success_order); | |
1099 | else | |
1100 | fence_after(failure_order); | |
1101 | return !!success; | |
1102 | } | |
1103 | ||
1104 | static BOOST_FORCEINLINE storage_type fetch_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
1105 | { | |
11fdf7f2 | 1106 | storage_type original, result; |
7c673cae FG |
1107 | fence_before(order); |
1108 | __asm__ __volatile__ | |
1109 | ( | |
1e59de90 | 1110 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
1111 | "ldarx %0,%y2\n\t" |
1112 | "add %1,%0,%3\n\t" | |
1113 | "stdcx. %1,%y2\n\t" | |
1e59de90 | 1114 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 1115 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
1116 | : "b" (v) |
1117 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
1118 | ); | |
1119 | fence_after(order); | |
1120 | return original; | |
1121 | } | |
1122 | ||
1123 | static BOOST_FORCEINLINE storage_type fetch_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
1124 | { | |
11fdf7f2 | 1125 | storage_type original, result; |
7c673cae FG |
1126 | fence_before(order); |
1127 | __asm__ __volatile__ | |
1128 | ( | |
1e59de90 | 1129 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
1130 | "ldarx %0,%y2\n\t" |
1131 | "sub %1,%0,%3\n\t" | |
1132 | "stdcx. %1,%y2\n\t" | |
1e59de90 | 1133 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 1134 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
1135 | : "b" (v) |
1136 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
1137 | ); | |
1138 | fence_after(order); | |
1139 | return original; | |
1140 | } | |
1141 | ||
1142 | static BOOST_FORCEINLINE storage_type fetch_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
1143 | { | |
11fdf7f2 | 1144 | storage_type original, result; |
7c673cae FG |
1145 | fence_before(order); |
1146 | __asm__ __volatile__ | |
1147 | ( | |
1e59de90 | 1148 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
1149 | "ldarx %0,%y2\n\t" |
1150 | "and %1,%0,%3\n\t" | |
1151 | "stdcx. %1,%y2\n\t" | |
1e59de90 | 1152 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 1153 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
1154 | : "b" (v) |
1155 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
1156 | ); | |
1157 | fence_after(order); | |
1158 | return original; | |
1159 | } | |
1160 | ||
1161 | static BOOST_FORCEINLINE storage_type fetch_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
1162 | { | |
11fdf7f2 | 1163 | storage_type original, result; |
7c673cae FG |
1164 | fence_before(order); |
1165 | __asm__ __volatile__ | |
1166 | ( | |
1e59de90 | 1167 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
1168 | "ldarx %0,%y2\n\t" |
1169 | "or %1,%0,%3\n\t" | |
1170 | "stdcx. %1,%y2\n\t" | |
1e59de90 | 1171 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 1172 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
1173 | : "b" (v) |
1174 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
1175 | ); | |
1176 | fence_after(order); | |
1177 | return original; | |
1178 | } | |
1179 | ||
1180 | static BOOST_FORCEINLINE storage_type fetch_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT | |
1181 | { | |
11fdf7f2 | 1182 | storage_type original, result; |
7c673cae FG |
1183 | fence_before(order); |
1184 | __asm__ __volatile__ | |
1185 | ( | |
1e59de90 | 1186 | BOOST_ATOMIC_DETAIL_PPC_ASM_LABEL("1") |
7c673cae FG |
1187 | "ldarx %0,%y2\n\t" |
1188 | "xor %1,%0,%3\n\t" | |
1189 | "stdcx. %1,%y2\n\t" | |
1e59de90 | 1190 | BOOST_ATOMIC_DETAIL_PPC_ASM_JUMP("bne-", "1b", "-12") |
11fdf7f2 | 1191 | : "=&b" (original), "=&b" (result), "+Z" (storage) |
7c673cae FG |
1192 | : "b" (v) |
1193 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC | |
1194 | ); | |
1195 | fence_after(order); | |
1196 | return original; | |
1197 | } | |
1198 | ||
1199 | static BOOST_FORCEINLINE bool test_and_set(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT | |
1200 | { | |
1201 | return !!exchange(storage, (storage_type)1, order); | |
1202 | } | |
1203 | ||
1204 | static BOOST_FORCEINLINE void clear(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT | |
1205 | { | |
f67539c2 | 1206 | store(storage, (storage_type)0, order); |
7c673cae | 1207 | } |
7c673cae FG |
1208 | }; |
1209 | ||
b32b8144 | 1210 | #endif // defined(BOOST_ATOMIC_DETAIL_PPC_HAS_LDARX_STDCX) |
7c673cae | 1211 | |
7c673cae FG |
1212 | } // namespace detail |
1213 | } // namespace atomics | |
1214 | } // namespace boost | |
1215 | ||
20effc67 TL |
1216 | #include <boost/atomic/detail/footer.hpp> |
1217 | ||
1218 | #endif // BOOST_ATOMIC_DETAIL_CORE_ARCH_OPS_GCC_PPC_HPP_INCLUDED_ |