]>
Commit | Line | Data |
---|---|---|
b32b8144 FG |
1 | /* |
2 | * Distributed under the Boost Software License, Version 1.0. | |
3 | * (See accompanying file LICENSE_1_0.txt or copy at | |
4 | * http://www.boost.org/LICENSE_1_0.txt) | |
5 | * | |
6 | * Copyright (c) 2015 Andrey Semashev | |
7 | */ | |
8 | /*! | |
9 | * \file atomic/detail/extra_ops_gcc_x86.hpp | |
10 | * | |
11 | * This header contains implementation of the extra atomic operations for x86. | |
12 | */ | |
13 | ||
14 | #ifndef BOOST_ATOMIC_DETAIL_EXTRA_OPS_GCC_X86_HPP_INCLUDED_ | |
15 | #define BOOST_ATOMIC_DETAIL_EXTRA_OPS_GCC_X86_HPP_INCLUDED_ | |
16 | ||
17 | #include <cstddef> | |
f67539c2 | 18 | #include <boost/cstdint.hpp> |
b32b8144 FG |
19 | #include <boost/memory_order.hpp> |
20 | #include <boost/atomic/detail/config.hpp> | |
f67539c2 | 21 | #include <boost/atomic/detail/storage_traits.hpp> |
b32b8144 | 22 | #include <boost/atomic/detail/extra_operations_fwd.hpp> |
f67539c2 | 23 | #include <boost/atomic/detail/extra_ops_generic.hpp> |
b32b8144 FG |
24 | #include <boost/atomic/capabilities.hpp> |
25 | ||
26 | #ifdef BOOST_HAS_PRAGMA_ONCE | |
27 | #pragma once | |
28 | #endif | |
29 | ||
30 | namespace boost { | |
31 | namespace atomics { | |
32 | namespace detail { | |
33 | ||
b32b8144 | 34 | template< typename Base, bool Signed > |
11fdf7f2 | 35 | struct extra_operations< Base, 1u, Signed, true > : |
f67539c2 | 36 | public generic_extra_operations< Base, 1u, Signed > |
b32b8144 | 37 | { |
f67539c2 | 38 | typedef generic_extra_operations< Base, 1u, Signed > base_type; |
b32b8144 | 39 | typedef typename base_type::storage_type storage_type; |
f67539c2 | 40 | typedef typename storage_traits< 4u >::type temp_storage_type; |
b32b8144 | 41 | |
11fdf7f2 | 42 | #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, original, result)\ |
b32b8144 FG |
43 | __asm__ __volatile__\ |
44 | (\ | |
45 | ".align 16\n\t"\ | |
11fdf7f2 | 46 | "1: movzbl %[orig], %2\n\t"\ |
b32b8144 FG |
47 | op " %b2\n\t"\ |
48 | "lock; cmpxchgb %b2, %[storage]\n\t"\ | |
49 | "jne 1b"\ | |
11fdf7f2 | 50 | : [orig] "+a" (original), [storage] "+m" (storage), "=&q" (result)\ |
b32b8144 FG |
51 | : \ |
52 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\ | |
53 | ) | |
54 | ||
55 | static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
56 | { | |
11fdf7f2 TL |
57 | storage_type original = storage; |
58 | temp_storage_type result; | |
59 | BOOST_ATOMIC_DETAIL_CAS_LOOP("negb", original, result); | |
60 | return original; | |
b32b8144 FG |
61 | } |
62 | ||
63 | static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
64 | { | |
11fdf7f2 TL |
65 | storage_type original = storage; |
66 | temp_storage_type result; | |
67 | BOOST_ATOMIC_DETAIL_CAS_LOOP("notb", original, result); | |
68 | return original; | |
69 | } | |
70 | ||
71 | static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
72 | { | |
73 | storage_type original = storage; | |
74 | temp_storage_type result; | |
75 | BOOST_ATOMIC_DETAIL_CAS_LOOP("negb", original, result); | |
76 | return static_cast< storage_type >(result); | |
77 | } | |
78 | ||
79 | static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
80 | { | |
81 | storage_type original = storage; | |
82 | temp_storage_type result; | |
83 | BOOST_ATOMIC_DETAIL_CAS_LOOP("notb", original, result); | |
84 | return static_cast< storage_type >(result); | |
85 | } | |
86 | ||
87 | #undef BOOST_ATOMIC_DETAIL_CAS_LOOP | |
88 | ||
89 | #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, argument, original, result)\ | |
90 | __asm__ __volatile__\ | |
91 | (\ | |
92 | ".align 16\n\t"\ | |
93 | "1: mov %[arg], %2\n\t"\ | |
94 | op " %%al, %b2\n\t"\ | |
95 | "lock; cmpxchgb %b2, %[storage]\n\t"\ | |
96 | "jne 1b"\ | |
97 | : [orig] "+a" (original), [storage] "+m" (storage), "=&q" (result)\ | |
98 | : [arg] "ir" ((temp_storage_type)argument)\ | |
99 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\ | |
100 | ) | |
101 | ||
102 | static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
103 | { | |
104 | storage_type original = storage; | |
105 | temp_storage_type result; | |
106 | BOOST_ATOMIC_DETAIL_CAS_LOOP("andb", v, original, result); | |
107 | return static_cast< storage_type >(result); | |
108 | } | |
109 | ||
110 | static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
111 | { | |
112 | storage_type original = storage; | |
113 | temp_storage_type result; | |
114 | BOOST_ATOMIC_DETAIL_CAS_LOOP("orb", v, original, result); | |
115 | return static_cast< storage_type >(result); | |
116 | } | |
117 | ||
118 | static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
119 | { | |
120 | storage_type original = storage; | |
121 | temp_storage_type result; | |
122 | BOOST_ATOMIC_DETAIL_CAS_LOOP("xorb", v, original, result); | |
123 | return static_cast< storage_type >(result); | |
b32b8144 FG |
124 | } |
125 | ||
126 | #undef BOOST_ATOMIC_DETAIL_CAS_LOOP | |
127 | ||
11fdf7f2 TL |
128 | static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT |
129 | { | |
130 | return !!negate(storage, order); | |
131 | } | |
132 | ||
133 | static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT | |
134 | { | |
135 | return !!bitwise_complement(storage, order); | |
136 | } | |
137 | ||
b32b8144 FG |
138 | static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT |
139 | { | |
140 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
141 | { | |
142 | __asm__ __volatile__ | |
143 | ( | |
144 | "lock; incb %[storage]\n\t" | |
145 | : [storage] "+m" (storage) | |
146 | : | |
147 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
148 | ); | |
149 | } | |
150 | else | |
151 | { | |
152 | __asm__ __volatile__ | |
153 | ( | |
154 | "lock; addb %[argument], %[storage]\n\t" | |
155 | : [storage] "+m" (storage) | |
156 | : [argument] "iq" (v) | |
157 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
158 | ); | |
159 | } | |
160 | } | |
161 | ||
162 | static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
163 | { | |
164 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
165 | { | |
166 | __asm__ __volatile__ | |
167 | ( | |
168 | "lock; decb %[storage]\n\t" | |
169 | : [storage] "+m" (storage) | |
170 | : | |
171 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
172 | ); | |
173 | } | |
174 | else | |
175 | { | |
176 | __asm__ __volatile__ | |
177 | ( | |
178 | "lock; subb %[argument], %[storage]\n\t" | |
179 | : [storage] "+m" (storage) | |
180 | : [argument] "iq" (v) | |
181 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
182 | ); | |
183 | } | |
184 | } | |
185 | ||
186 | static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
187 | { | |
188 | __asm__ __volatile__ | |
189 | ( | |
190 | "lock; negb %[storage]\n\t" | |
191 | : [storage] "+m" (storage) | |
192 | : | |
193 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
194 | ); | |
195 | } | |
196 | ||
197 | static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
198 | { | |
199 | __asm__ __volatile__ | |
200 | ( | |
201 | "lock; andb %[argument], %[storage]\n\t" | |
202 | : [storage] "+m" (storage) | |
203 | : [argument] "iq" (v) | |
204 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
205 | ); | |
206 | } | |
207 | ||
208 | static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
209 | { | |
210 | __asm__ __volatile__ | |
211 | ( | |
212 | "lock; orb %[argument], %[storage]\n\t" | |
213 | : [storage] "+m" (storage) | |
214 | : [argument] "iq" (v) | |
215 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
216 | ); | |
217 | } | |
218 | ||
219 | static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
220 | { | |
221 | __asm__ __volatile__ | |
222 | ( | |
223 | "lock; xorb %[argument], %[storage]\n\t" | |
224 | : [storage] "+m" (storage) | |
225 | : [argument] "iq" (v) | |
226 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
227 | ); | |
228 | } | |
229 | ||
230 | static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
231 | { | |
232 | __asm__ __volatile__ | |
233 | ( | |
234 | "lock; notb %[storage]\n\t" | |
235 | : [storage] "+m" (storage) | |
236 | : | |
237 | : "memory" | |
238 | ); | |
239 | } | |
240 | ||
241 | static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
242 | { | |
243 | bool res; | |
244 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
245 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
246 | { | |
247 | __asm__ __volatile__ | |
248 | ( | |
249 | "lock; incb %[storage]\n\t" | |
11fdf7f2 | 250 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
251 | : |
252 | : "memory" | |
253 | ); | |
254 | } | |
255 | else | |
256 | { | |
257 | __asm__ __volatile__ | |
258 | ( | |
259 | "lock; addb %[argument], %[storage]\n\t" | |
11fdf7f2 | 260 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
261 | : [argument] "iq" (v) |
262 | : "memory" | |
263 | ); | |
264 | } | |
265 | #else | |
266 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
267 | { | |
268 | __asm__ __volatile__ | |
269 | ( | |
270 | "lock; incb %[storage]\n\t" | |
11fdf7f2 | 271 | "setnz %[result]\n\t" |
b32b8144 FG |
272 | : [storage] "+m" (storage), [result] "=q" (res) |
273 | : | |
274 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
275 | ); | |
276 | } | |
277 | else | |
278 | { | |
279 | __asm__ __volatile__ | |
280 | ( | |
281 | "lock; addb %[argument], %[storage]\n\t" | |
11fdf7f2 | 282 | "setnz %[result]\n\t" |
b32b8144 FG |
283 | : [storage] "+m" (storage), [result] "=q" (res) |
284 | : [argument] "iq" (v) | |
285 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
286 | ); | |
287 | } | |
288 | #endif | |
289 | return res; | |
290 | } | |
291 | ||
292 | static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
293 | { | |
294 | bool res; | |
295 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
296 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
297 | { | |
298 | __asm__ __volatile__ | |
299 | ( | |
300 | "lock; decb %[storage]\n\t" | |
11fdf7f2 | 301 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
302 | : |
303 | : "memory" | |
304 | ); | |
305 | } | |
306 | else | |
307 | { | |
308 | __asm__ __volatile__ | |
309 | ( | |
310 | "lock; subb %[argument], %[storage]\n\t" | |
11fdf7f2 | 311 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
312 | : [argument] "iq" (v) |
313 | : "memory" | |
314 | ); | |
315 | } | |
316 | #else | |
317 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
318 | { | |
319 | __asm__ __volatile__ | |
320 | ( | |
321 | "lock; decb %[storage]\n\t" | |
11fdf7f2 | 322 | "setnz %[result]\n\t" |
b32b8144 FG |
323 | : [storage] "+m" (storage), [result] "=q" (res) |
324 | : | |
325 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
326 | ); | |
327 | } | |
328 | else | |
329 | { | |
330 | __asm__ __volatile__ | |
331 | ( | |
332 | "lock; subb %[argument], %[storage]\n\t" | |
11fdf7f2 | 333 | "setnz %[result]\n\t" |
b32b8144 FG |
334 | : [storage] "+m" (storage), [result] "=q" (res) |
335 | : [argument] "iq" (v) | |
336 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
337 | ); | |
338 | } | |
339 | #endif | |
340 | return res; | |
341 | } | |
342 | ||
343 | static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
344 | { | |
345 | bool res; | |
346 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
347 | __asm__ __volatile__ | |
348 | ( | |
349 | "lock; andb %[argument], %[storage]\n\t" | |
11fdf7f2 | 350 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
351 | : [argument] "iq" (v) |
352 | : "memory" | |
353 | ); | |
354 | #else | |
355 | __asm__ __volatile__ | |
356 | ( | |
357 | "lock; andb %[argument], %[storage]\n\t" | |
11fdf7f2 | 358 | "setnz %[result]\n\t" |
b32b8144 FG |
359 | : [storage] "+m" (storage), [result] "=q" (res) |
360 | : [argument] "iq" (v) | |
361 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
362 | ); | |
363 | #endif | |
364 | return res; | |
365 | } | |
366 | ||
367 | static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
368 | { | |
369 | bool res; | |
370 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
371 | __asm__ __volatile__ | |
372 | ( | |
373 | "lock; orb %[argument], %[storage]\n\t" | |
11fdf7f2 | 374 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
375 | : [argument] "iq" (v) |
376 | : "memory" | |
377 | ); | |
378 | #else | |
379 | __asm__ __volatile__ | |
380 | ( | |
381 | "lock; orb %[argument], %[storage]\n\t" | |
11fdf7f2 | 382 | "setnz %[result]\n\t" |
b32b8144 FG |
383 | : [storage] "+m" (storage), [result] "=q" (res) |
384 | : [argument] "iq" (v) | |
385 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
386 | ); | |
387 | #endif | |
388 | return res; | |
389 | } | |
390 | ||
391 | static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
392 | { | |
393 | bool res; | |
394 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
395 | __asm__ __volatile__ | |
396 | ( | |
397 | "lock; xorb %[argument], %[storage]\n\t" | |
11fdf7f2 | 398 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
399 | : [argument] "iq" (v) |
400 | : "memory" | |
401 | ); | |
402 | #else | |
403 | __asm__ __volatile__ | |
404 | ( | |
405 | "lock; xorb %[argument], %[storage]\n\t" | |
11fdf7f2 | 406 | "setnz %[result]\n\t" |
b32b8144 FG |
407 | : [storage] "+m" (storage), [result] "=q" (res) |
408 | : [argument] "iq" (v) | |
409 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
410 | ); | |
411 | #endif | |
412 | return res; | |
413 | } | |
414 | }; | |
415 | ||
416 | template< typename Base, bool Signed > | |
11fdf7f2 | 417 | struct extra_operations< Base, 2u, Signed, true > : |
f67539c2 | 418 | public generic_extra_operations< Base, 2u, Signed > |
b32b8144 | 419 | { |
f67539c2 | 420 | typedef generic_extra_operations< Base, 2u, Signed > base_type; |
b32b8144 | 421 | typedef typename base_type::storage_type storage_type; |
f67539c2 | 422 | typedef typename storage_traits< 4u >::type temp_storage_type; |
b32b8144 | 423 | |
11fdf7f2 | 424 | #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, original, result)\ |
b32b8144 FG |
425 | __asm__ __volatile__\ |
426 | (\ | |
427 | ".align 16\n\t"\ | |
11fdf7f2 | 428 | "1: movzwl %[orig], %2\n\t"\ |
b32b8144 FG |
429 | op " %w2\n\t"\ |
430 | "lock; cmpxchgw %w2, %[storage]\n\t"\ | |
431 | "jne 1b"\ | |
11fdf7f2 | 432 | : [orig] "+a" (original), [storage] "+m" (storage), "=&q" (result)\ |
b32b8144 FG |
433 | : \ |
434 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\ | |
435 | ) | |
436 | ||
437 | static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
438 | { | |
11fdf7f2 TL |
439 | storage_type original = storage; |
440 | temp_storage_type result; | |
441 | BOOST_ATOMIC_DETAIL_CAS_LOOP("negw", original, result); | |
442 | return original; | |
b32b8144 FG |
443 | } |
444 | ||
445 | static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
446 | { | |
11fdf7f2 TL |
447 | storage_type original = storage; |
448 | temp_storage_type result; | |
449 | BOOST_ATOMIC_DETAIL_CAS_LOOP("notw", original, result); | |
450 | return original; | |
451 | } | |
452 | ||
453 | static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
454 | { | |
455 | storage_type original = storage; | |
456 | temp_storage_type result; | |
457 | BOOST_ATOMIC_DETAIL_CAS_LOOP("negw", original, result); | |
458 | return static_cast< storage_type >(result); | |
459 | } | |
460 | ||
461 | static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
462 | { | |
463 | storage_type original = storage; | |
464 | temp_storage_type result; | |
465 | BOOST_ATOMIC_DETAIL_CAS_LOOP("notw", original, result); | |
466 | return static_cast< storage_type >(result); | |
467 | } | |
468 | ||
469 | #undef BOOST_ATOMIC_DETAIL_CAS_LOOP | |
470 | ||
471 | #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, argument, original, result)\ | |
472 | __asm__ __volatile__\ | |
473 | (\ | |
474 | ".align 16\n\t"\ | |
475 | "1: mov %[arg], %2\n\t"\ | |
476 | op " %%ax, %w2\n\t"\ | |
477 | "lock; cmpxchgw %w2, %[storage]\n\t"\ | |
478 | "jne 1b"\ | |
479 | : [orig] "+a" (original), [storage] "+m" (storage), "=&q" (result)\ | |
480 | : [arg] "ir" ((temp_storage_type)argument)\ | |
481 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\ | |
482 | ) | |
483 | ||
484 | static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
485 | { | |
486 | storage_type original = storage; | |
487 | temp_storage_type result; | |
488 | BOOST_ATOMIC_DETAIL_CAS_LOOP("andw", v, original, result); | |
489 | return static_cast< storage_type >(result); | |
490 | } | |
491 | ||
492 | static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
493 | { | |
494 | storage_type original = storage; | |
495 | temp_storage_type result; | |
496 | BOOST_ATOMIC_DETAIL_CAS_LOOP("orw", v, original, result); | |
497 | return static_cast< storage_type >(result); | |
498 | } | |
499 | ||
500 | static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
501 | { | |
502 | storage_type original = storage; | |
503 | temp_storage_type result; | |
504 | BOOST_ATOMIC_DETAIL_CAS_LOOP("xorw", v, original, result); | |
505 | return static_cast< storage_type >(result); | |
b32b8144 FG |
506 | } |
507 | ||
508 | #undef BOOST_ATOMIC_DETAIL_CAS_LOOP | |
509 | ||
11fdf7f2 TL |
510 | static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT |
511 | { | |
512 | return !!negate(storage, order); | |
513 | } | |
514 | ||
515 | static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT | |
516 | { | |
517 | return !!bitwise_complement(storage, order); | |
518 | } | |
519 | ||
b32b8144 FG |
520 | static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT |
521 | { | |
522 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
523 | { | |
524 | __asm__ __volatile__ | |
525 | ( | |
526 | "lock; incw %[storage]\n\t" | |
527 | : [storage] "+m" (storage) | |
528 | : | |
529 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
530 | ); | |
531 | } | |
532 | else | |
533 | { | |
534 | __asm__ __volatile__ | |
535 | ( | |
536 | "lock; addw %[argument], %[storage]\n\t" | |
537 | : [storage] "+m" (storage) | |
538 | : [argument] "iq" (v) | |
539 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
540 | ); | |
541 | } | |
542 | } | |
543 | ||
544 | static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
545 | { | |
546 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
547 | { | |
548 | __asm__ __volatile__ | |
549 | ( | |
550 | "lock; decw %[storage]\n\t" | |
551 | : [storage] "+m" (storage) | |
552 | : | |
553 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
554 | ); | |
555 | } | |
556 | else | |
557 | { | |
558 | __asm__ __volatile__ | |
559 | ( | |
560 | "lock; subw %[argument], %[storage]\n\t" | |
561 | : [storage] "+m" (storage) | |
562 | : [argument] "iq" (v) | |
563 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
564 | ); | |
565 | } | |
566 | } | |
567 | ||
568 | static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
569 | { | |
570 | __asm__ __volatile__ | |
571 | ( | |
572 | "lock; negw %[storage]\n\t" | |
573 | : [storage] "+m" (storage) | |
574 | : | |
575 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
576 | ); | |
577 | } | |
578 | ||
579 | static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
580 | { | |
581 | __asm__ __volatile__ | |
582 | ( | |
583 | "lock; andw %[argument], %[storage]\n\t" | |
584 | : [storage] "+m" (storage) | |
585 | : [argument] "iq" (v) | |
586 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
587 | ); | |
588 | } | |
589 | ||
590 | static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
591 | { | |
592 | __asm__ __volatile__ | |
593 | ( | |
594 | "lock; orw %[argument], %[storage]\n\t" | |
595 | : [storage] "+m" (storage) | |
596 | : [argument] "iq" (v) | |
597 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
598 | ); | |
599 | } | |
600 | ||
601 | static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
602 | { | |
603 | __asm__ __volatile__ | |
604 | ( | |
605 | "lock; xorw %[argument], %[storage]\n\t" | |
606 | : [storage] "+m" (storage) | |
607 | : [argument] "iq" (v) | |
608 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
609 | ); | |
610 | } | |
611 | ||
612 | static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
613 | { | |
614 | __asm__ __volatile__ | |
615 | ( | |
616 | "lock; notw %[storage]\n\t" | |
617 | : [storage] "+m" (storage) | |
618 | : | |
619 | : "memory" | |
620 | ); | |
621 | } | |
622 | ||
623 | static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
624 | { | |
625 | bool res; | |
626 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
627 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
628 | { | |
629 | __asm__ __volatile__ | |
630 | ( | |
631 | "lock; incw %[storage]\n\t" | |
11fdf7f2 | 632 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
633 | : |
634 | : "memory" | |
635 | ); | |
636 | } | |
637 | else | |
638 | { | |
639 | __asm__ __volatile__ | |
640 | ( | |
641 | "lock; addw %[argument], %[storage]\n\t" | |
11fdf7f2 | 642 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
643 | : [argument] "iq" (v) |
644 | : "memory" | |
645 | ); | |
646 | } | |
647 | #else | |
648 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
649 | { | |
650 | __asm__ __volatile__ | |
651 | ( | |
652 | "lock; incw %[storage]\n\t" | |
11fdf7f2 | 653 | "setnz %[result]\n\t" |
b32b8144 FG |
654 | : [storage] "+m" (storage), [result] "=q" (res) |
655 | : | |
656 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
657 | ); | |
658 | } | |
659 | else | |
660 | { | |
661 | __asm__ __volatile__ | |
662 | ( | |
663 | "lock; addw %[argument], %[storage]\n\t" | |
11fdf7f2 | 664 | "setnz %[result]\n\t" |
b32b8144 FG |
665 | : [storage] "+m" (storage), [result] "=q" (res) |
666 | : [argument] "iq" (v) | |
667 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
668 | ); | |
669 | } | |
670 | #endif | |
671 | return res; | |
672 | } | |
673 | ||
674 | static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
675 | { | |
676 | bool res; | |
677 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
678 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
679 | { | |
680 | __asm__ __volatile__ | |
681 | ( | |
682 | "lock; decw %[storage]\n\t" | |
11fdf7f2 | 683 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
684 | : |
685 | : "memory" | |
686 | ); | |
687 | } | |
688 | else | |
689 | { | |
690 | __asm__ __volatile__ | |
691 | ( | |
692 | "lock; subw %[argument], %[storage]\n\t" | |
11fdf7f2 | 693 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
694 | : [argument] "iq" (v) |
695 | : "memory" | |
696 | ); | |
697 | } | |
698 | #else | |
699 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
700 | { | |
701 | __asm__ __volatile__ | |
702 | ( | |
703 | "lock; decw %[storage]\n\t" | |
11fdf7f2 | 704 | "setnz %[result]\n\t" |
b32b8144 FG |
705 | : [storage] "+m" (storage), [result] "=q" (res) |
706 | : | |
707 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
708 | ); | |
709 | } | |
710 | else | |
711 | { | |
712 | __asm__ __volatile__ | |
713 | ( | |
714 | "lock; subw %[argument], %[storage]\n\t" | |
11fdf7f2 | 715 | "setnz %[result]\n\t" |
b32b8144 FG |
716 | : [storage] "+m" (storage), [result] "=q" (res) |
717 | : [argument] "iq" (v) | |
718 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
719 | ); | |
720 | } | |
721 | #endif | |
722 | return res; | |
723 | } | |
724 | ||
725 | static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
726 | { | |
727 | bool res; | |
728 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
729 | __asm__ __volatile__ | |
730 | ( | |
731 | "lock; andw %[argument], %[storage]\n\t" | |
11fdf7f2 | 732 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
733 | : [argument] "iq" (v) |
734 | : "memory" | |
735 | ); | |
736 | #else | |
737 | __asm__ __volatile__ | |
738 | ( | |
739 | "lock; andw %[argument], %[storage]\n\t" | |
11fdf7f2 | 740 | "setnz %[result]\n\t" |
b32b8144 FG |
741 | : [storage] "+m" (storage), [result] "=q" (res) |
742 | : [argument] "iq" (v) | |
743 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
744 | ); | |
745 | #endif | |
746 | return res; | |
747 | } | |
748 | ||
749 | static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
750 | { | |
751 | bool res; | |
752 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
753 | __asm__ __volatile__ | |
754 | ( | |
755 | "lock; orw %[argument], %[storage]\n\t" | |
11fdf7f2 | 756 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
757 | : [argument] "iq" (v) |
758 | : "memory" | |
759 | ); | |
760 | #else | |
761 | __asm__ __volatile__ | |
762 | ( | |
763 | "lock; orw %[argument], %[storage]\n\t" | |
11fdf7f2 | 764 | "setnz %[result]\n\t" |
b32b8144 FG |
765 | : [storage] "+m" (storage), [result] "=q" (res) |
766 | : [argument] "iq" (v) | |
767 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
768 | ); | |
769 | #endif | |
770 | return res; | |
771 | } | |
772 | ||
773 | static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
774 | { | |
775 | bool res; | |
776 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
777 | __asm__ __volatile__ | |
778 | ( | |
779 | "lock; xorw %[argument], %[storage]\n\t" | |
11fdf7f2 | 780 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
781 | : [argument] "iq" (v) |
782 | : "memory" | |
783 | ); | |
784 | #else | |
785 | __asm__ __volatile__ | |
786 | ( | |
787 | "lock; xorw %[argument], %[storage]\n\t" | |
11fdf7f2 | 788 | "setnz %[result]\n\t" |
b32b8144 FG |
789 | : [storage] "+m" (storage), [result] "=q" (res) |
790 | : [argument] "iq" (v) | |
791 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
792 | ); | |
f67539c2 TL |
793 | #endif |
794 | return res; | |
795 | } | |
796 | ||
797 | static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT | |
798 | { | |
799 | bool res; | |
800 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
801 | __asm__ __volatile__ | |
802 | ( | |
803 | "lock; btsw %[bit_number], %[storage]\n\t" | |
804 | : [storage] "+m" (storage), [result] "=@ccc" (res) | |
805 | : [bit_number] "Kq" ((uint16_t)bit_number) | |
806 | : "memory" | |
807 | ); | |
808 | #else | |
809 | __asm__ __volatile__ | |
810 | ( | |
811 | "lock; btsw %[bit_number], %[storage]\n\t" | |
812 | "setc %[result]\n\t" | |
813 | : [storage] "+m" (storage), [result] "=q" (res) | |
814 | : [bit_number] "Kq" ((uint16_t)bit_number) | |
815 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
816 | ); | |
817 | #endif | |
818 | return res; | |
819 | } | |
820 | ||
821 | static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT | |
822 | { | |
823 | bool res; | |
824 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
825 | __asm__ __volatile__ | |
826 | ( | |
827 | "lock; btrw %[bit_number], %[storage]\n\t" | |
828 | : [storage] "+m" (storage), [result] "=@ccc" (res) | |
829 | : [bit_number] "Kq" ((uint16_t)bit_number) | |
830 | : "memory" | |
831 | ); | |
832 | #else | |
833 | __asm__ __volatile__ | |
834 | ( | |
835 | "lock; btrw %[bit_number], %[storage]\n\t" | |
836 | "setc %[result]\n\t" | |
837 | : [storage] "+m" (storage), [result] "=q" (res) | |
838 | : [bit_number] "Kq" ((uint16_t)bit_number) | |
839 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
840 | ); | |
841 | #endif | |
842 | return res; | |
843 | } | |
844 | ||
845 | static BOOST_FORCEINLINE bool bit_test_and_complement(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT | |
846 | { | |
847 | bool res; | |
848 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
849 | __asm__ __volatile__ | |
850 | ( | |
851 | "lock; btcw %[bit_number], %[storage]\n\t" | |
852 | : [storage] "+m" (storage), [result] "=@ccc" (res) | |
853 | : [bit_number] "Kq" ((uint16_t)bit_number) | |
854 | : "memory" | |
855 | ); | |
856 | #else | |
857 | __asm__ __volatile__ | |
858 | ( | |
859 | "lock; btcw %[bit_number], %[storage]\n\t" | |
860 | "setc %[result]\n\t" | |
861 | : [storage] "+m" (storage), [result] "=q" (res) | |
862 | : [bit_number] "Kq" ((uint16_t)bit_number) | |
863 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
864 | ); | |
b32b8144 FG |
865 | #endif |
866 | return res; | |
867 | } | |
868 | }; | |
869 | ||
870 | template< typename Base, bool Signed > | |
11fdf7f2 | 871 | struct extra_operations< Base, 4u, Signed, true > : |
f67539c2 | 872 | public generic_extra_operations< Base, 4u, Signed > |
b32b8144 | 873 | { |
f67539c2 | 874 | typedef generic_extra_operations< Base, 4u, Signed > base_type; |
b32b8144 FG |
875 | typedef typename base_type::storage_type storage_type; |
876 | ||
11fdf7f2 | 877 | #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, original, result)\ |
b32b8144 FG |
878 | __asm__ __volatile__\ |
879 | (\ | |
880 | ".align 16\n\t"\ | |
11fdf7f2 TL |
881 | "1: mov %[orig], %[res]\n\t"\ |
882 | op " %[res]\n\t"\ | |
883 | "lock; cmpxchgl %[res], %[storage]\n\t"\ | |
b32b8144 | 884 | "jne 1b"\ |
11fdf7f2 | 885 | : [orig] "+a" (original), [storage] "+m" (storage), [res] "=&r" (result)\ |
b32b8144 FG |
886 | : \ |
887 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\ | |
888 | ) | |
889 | ||
890 | static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
891 | { | |
11fdf7f2 TL |
892 | storage_type original = storage; |
893 | storage_type result; | |
894 | BOOST_ATOMIC_DETAIL_CAS_LOOP("negl", original, result); | |
895 | return original; | |
b32b8144 FG |
896 | } |
897 | ||
898 | static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
899 | { | |
11fdf7f2 TL |
900 | storage_type original = storage; |
901 | storage_type result; | |
902 | BOOST_ATOMIC_DETAIL_CAS_LOOP("notl", original, result); | |
903 | return original; | |
904 | } | |
905 | ||
906 | static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
907 | { | |
908 | storage_type original = storage; | |
909 | storage_type result; | |
910 | BOOST_ATOMIC_DETAIL_CAS_LOOP("negl", original, result); | |
911 | return result; | |
912 | } | |
913 | ||
914 | static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
915 | { | |
916 | storage_type original = storage; | |
917 | storage_type result; | |
918 | BOOST_ATOMIC_DETAIL_CAS_LOOP("notl", original, result); | |
919 | return result; | |
920 | } | |
921 | ||
922 | #undef BOOST_ATOMIC_DETAIL_CAS_LOOP | |
923 | ||
924 | #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, argument, original, result)\ | |
925 | __asm__ __volatile__\ | |
926 | (\ | |
927 | ".align 16\n\t"\ | |
928 | "1: mov %[arg], %[res]\n\t"\ | |
929 | op " %%eax, %[res]\n\t"\ | |
930 | "lock; cmpxchgl %[res], %[storage]\n\t"\ | |
931 | "jne 1b"\ | |
932 | : [orig] "+a" (original), [storage] "+m" (storage), [res] "=&r" (result)\ | |
933 | : [arg] "ir" (argument)\ | |
934 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\ | |
935 | ) | |
936 | ||
937 | static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
938 | { | |
939 | storage_type original = storage; | |
940 | storage_type result; | |
941 | BOOST_ATOMIC_DETAIL_CAS_LOOP("andl", v, original, result); | |
942 | return static_cast< storage_type >(result); | |
943 | } | |
944 | ||
945 | static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
946 | { | |
947 | storage_type original = storage; | |
948 | storage_type result; | |
949 | BOOST_ATOMIC_DETAIL_CAS_LOOP("orl", v, original, result); | |
950 | return static_cast< storage_type >(result); | |
951 | } | |
952 | ||
953 | static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
954 | { | |
955 | storage_type original = storage; | |
956 | storage_type result; | |
957 | BOOST_ATOMIC_DETAIL_CAS_LOOP("xorl", v, original, result); | |
958 | return static_cast< storage_type >(result); | |
b32b8144 FG |
959 | } |
960 | ||
961 | #undef BOOST_ATOMIC_DETAIL_CAS_LOOP | |
962 | ||
11fdf7f2 TL |
963 | static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT |
964 | { | |
965 | return !!negate(storage, order); | |
966 | } | |
967 | ||
968 | static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT | |
969 | { | |
970 | return !!bitwise_complement(storage, order); | |
971 | } | |
972 | ||
b32b8144 FG |
973 | static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT |
974 | { | |
975 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
976 | { | |
977 | __asm__ __volatile__ | |
978 | ( | |
979 | "lock; incl %[storage]\n\t" | |
980 | : [storage] "+m" (storage) | |
981 | : | |
982 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
983 | ); | |
984 | } | |
985 | else | |
986 | { | |
987 | __asm__ __volatile__ | |
988 | ( | |
989 | "lock; addl %[argument], %[storage]\n\t" | |
990 | : [storage] "+m" (storage) | |
991 | : [argument] "ir" (v) | |
992 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
993 | ); | |
994 | } | |
995 | } | |
996 | ||
997 | static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
998 | { | |
999 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
1000 | { | |
1001 | __asm__ __volatile__ | |
1002 | ( | |
1003 | "lock; decl %[storage]\n\t" | |
1004 | : [storage] "+m" (storage) | |
1005 | : | |
1006 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1007 | ); | |
1008 | } | |
1009 | else | |
1010 | { | |
1011 | __asm__ __volatile__ | |
1012 | ( | |
1013 | "lock; subl %[argument], %[storage]\n\t" | |
1014 | : [storage] "+m" (storage) | |
1015 | : [argument] "ir" (v) | |
1016 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1017 | ); | |
1018 | } | |
1019 | } | |
1020 | ||
1021 | static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
1022 | { | |
1023 | __asm__ __volatile__ | |
1024 | ( | |
1025 | "lock; negl %[storage]\n\t" | |
1026 | : [storage] "+m" (storage) | |
1027 | : | |
1028 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1029 | ); | |
1030 | } | |
1031 | ||
1032 | static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1033 | { | |
1034 | __asm__ __volatile__ | |
1035 | ( | |
1036 | "lock; andl %[argument], %[storage]\n\t" | |
1037 | : [storage] "+m" (storage) | |
1038 | : [argument] "ir" (v) | |
1039 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1040 | ); | |
1041 | } | |
1042 | ||
1043 | static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1044 | { | |
1045 | __asm__ __volatile__ | |
1046 | ( | |
1047 | "lock; orl %[argument], %[storage]\n\t" | |
1048 | : [storage] "+m" (storage) | |
1049 | : [argument] "ir" (v) | |
1050 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1051 | ); | |
1052 | } | |
1053 | ||
1054 | static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1055 | { | |
1056 | __asm__ __volatile__ | |
1057 | ( | |
1058 | "lock; xorl %[argument], %[storage]\n\t" | |
1059 | : [storage] "+m" (storage) | |
1060 | : [argument] "ir" (v) | |
1061 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1062 | ); | |
1063 | } | |
1064 | ||
1065 | static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
1066 | { | |
1067 | __asm__ __volatile__ | |
1068 | ( | |
1069 | "lock; notl %[storage]\n\t" | |
1070 | : [storage] "+m" (storage) | |
1071 | : | |
1072 | : "memory" | |
1073 | ); | |
1074 | } | |
1075 | ||
1076 | static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1077 | { | |
1078 | bool res; | |
1079 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
1080 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
1081 | { | |
1082 | __asm__ __volatile__ | |
1083 | ( | |
1084 | "lock; incl %[storage]\n\t" | |
11fdf7f2 | 1085 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
1086 | : |
1087 | : "memory" | |
1088 | ); | |
1089 | } | |
1090 | else | |
1091 | { | |
1092 | __asm__ __volatile__ | |
1093 | ( | |
1094 | "lock; addl %[argument], %[storage]\n\t" | |
11fdf7f2 | 1095 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
1096 | : [argument] "ir" (v) |
1097 | : "memory" | |
1098 | ); | |
1099 | } | |
1100 | #else | |
1101 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
1102 | { | |
1103 | __asm__ __volatile__ | |
1104 | ( | |
1105 | "lock; incl %[storage]\n\t" | |
11fdf7f2 | 1106 | "setnz %[result]\n\t" |
b32b8144 FG |
1107 | : [storage] "+m" (storage), [result] "=q" (res) |
1108 | : | |
1109 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1110 | ); | |
1111 | } | |
1112 | else | |
1113 | { | |
1114 | __asm__ __volatile__ | |
1115 | ( | |
1116 | "lock; addl %[argument], %[storage]\n\t" | |
11fdf7f2 | 1117 | "setnz %[result]\n\t" |
b32b8144 FG |
1118 | : [storage] "+m" (storage), [result] "=q" (res) |
1119 | : [argument] "ir" (v) | |
1120 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1121 | ); | |
1122 | } | |
1123 | #endif | |
1124 | return res; | |
1125 | } | |
1126 | ||
1127 | static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1128 | { | |
1129 | bool res; | |
1130 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
1131 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
1132 | { | |
1133 | __asm__ __volatile__ | |
1134 | ( | |
1135 | "lock; decl %[storage]\n\t" | |
11fdf7f2 | 1136 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
1137 | : |
1138 | : "memory" | |
1139 | ); | |
1140 | } | |
1141 | else | |
1142 | { | |
1143 | __asm__ __volatile__ | |
1144 | ( | |
1145 | "lock; subl %[argument], %[storage]\n\t" | |
11fdf7f2 | 1146 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
1147 | : [argument] "ir" (v) |
1148 | : "memory" | |
1149 | ); | |
1150 | } | |
1151 | #else | |
1152 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
1153 | { | |
1154 | __asm__ __volatile__ | |
1155 | ( | |
1156 | "lock; decl %[storage]\n\t" | |
11fdf7f2 | 1157 | "setnz %[result]\n\t" |
b32b8144 FG |
1158 | : [storage] "+m" (storage), [result] "=q" (res) |
1159 | : | |
1160 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1161 | ); | |
1162 | } | |
1163 | else | |
1164 | { | |
1165 | __asm__ __volatile__ | |
1166 | ( | |
1167 | "lock; subl %[argument], %[storage]\n\t" | |
11fdf7f2 | 1168 | "setnz %[result]\n\t" |
b32b8144 FG |
1169 | : [storage] "+m" (storage), [result] "=q" (res) |
1170 | : [argument] "ir" (v) | |
1171 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1172 | ); | |
1173 | } | |
1174 | #endif | |
1175 | return res; | |
1176 | } | |
1177 | ||
1178 | static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1179 | { | |
1180 | bool res; | |
1181 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
1182 | __asm__ __volatile__ | |
1183 | ( | |
1184 | "lock; andl %[argument], %[storage]\n\t" | |
11fdf7f2 | 1185 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
1186 | : [argument] "ir" (v) |
1187 | : "memory" | |
1188 | ); | |
1189 | #else | |
1190 | __asm__ __volatile__ | |
1191 | ( | |
1192 | "lock; andl %[argument], %[storage]\n\t" | |
11fdf7f2 | 1193 | "setnz %[result]\n\t" |
b32b8144 FG |
1194 | : [storage] "+m" (storage), [result] "=q" (res) |
1195 | : [argument] "ir" (v) | |
1196 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1197 | ); | |
1198 | #endif | |
1199 | return res; | |
1200 | } | |
1201 | ||
1202 | static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1203 | { | |
1204 | bool res; | |
1205 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
1206 | __asm__ __volatile__ | |
1207 | ( | |
1208 | "lock; orl %[argument], %[storage]\n\t" | |
11fdf7f2 | 1209 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
1210 | : [argument] "ir" (v) |
1211 | : "memory" | |
1212 | ); | |
1213 | #else | |
1214 | __asm__ __volatile__ | |
1215 | ( | |
1216 | "lock; orl %[argument], %[storage]\n\t" | |
11fdf7f2 | 1217 | "setnz %[result]\n\t" |
b32b8144 FG |
1218 | : [storage] "+m" (storage), [result] "=q" (res) |
1219 | : [argument] "ir" (v) | |
1220 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1221 | ); | |
1222 | #endif | |
1223 | return res; | |
1224 | } | |
1225 | ||
1226 | static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1227 | { | |
1228 | bool res; | |
1229 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
1230 | __asm__ __volatile__ | |
1231 | ( | |
1232 | "lock; xorl %[argument], %[storage]\n\t" | |
11fdf7f2 | 1233 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
1234 | : [argument] "ir" (v) |
1235 | : "memory" | |
1236 | ); | |
1237 | #else | |
1238 | __asm__ __volatile__ | |
1239 | ( | |
1240 | "lock; xorl %[argument], %[storage]\n\t" | |
11fdf7f2 | 1241 | "setnz %[result]\n\t" |
b32b8144 FG |
1242 | : [storage] "+m" (storage), [result] "=q" (res) |
1243 | : [argument] "ir" (v) | |
1244 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1245 | ); | |
f67539c2 TL |
1246 | #endif |
1247 | return res; | |
1248 | } | |
1249 | ||
1250 | static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT | |
1251 | { | |
1252 | bool res; | |
1253 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
1254 | __asm__ __volatile__ | |
1255 | ( | |
1256 | "lock; btsl %[bit_number], %[storage]\n\t" | |
1257 | : [storage] "+m" (storage), [result] "=@ccc" (res) | |
1258 | : [bit_number] "Kr" ((uint32_t)bit_number) | |
1259 | : "memory" | |
1260 | ); | |
1261 | #else | |
1262 | __asm__ __volatile__ | |
1263 | ( | |
1264 | "lock; btsl %[bit_number], %[storage]\n\t" | |
1265 | "setc %[result]\n\t" | |
1266 | : [storage] "+m" (storage), [result] "=q" (res) | |
1267 | : [bit_number] "Kr" ((uint32_t)bit_number) | |
1268 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1269 | ); | |
1270 | #endif | |
1271 | return res; | |
1272 | } | |
1273 | ||
1274 | static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT | |
1275 | { | |
1276 | bool res; | |
1277 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
1278 | __asm__ __volatile__ | |
1279 | ( | |
1280 | "lock; btrl %[bit_number], %[storage]\n\t" | |
1281 | : [storage] "+m" (storage), [result] "=@ccc" (res) | |
1282 | : [bit_number] "Kr" ((uint32_t)bit_number) | |
1283 | : "memory" | |
1284 | ); | |
1285 | #else | |
1286 | __asm__ __volatile__ | |
1287 | ( | |
1288 | "lock; btrl %[bit_number], %[storage]\n\t" | |
1289 | "setc %[result]\n\t" | |
1290 | : [storage] "+m" (storage), [result] "=q" (res) | |
1291 | : [bit_number] "Kr" ((uint32_t)bit_number) | |
1292 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1293 | ); | |
1294 | #endif | |
1295 | return res; | |
1296 | } | |
1297 | ||
1298 | static BOOST_FORCEINLINE bool bit_test_and_complement(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT | |
1299 | { | |
1300 | bool res; | |
1301 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
1302 | __asm__ __volatile__ | |
1303 | ( | |
1304 | "lock; btcl %[bit_number], %[storage]\n\t" | |
1305 | : [storage] "+m" (storage), [result] "=@ccc" (res) | |
1306 | : [bit_number] "Kr" ((uint32_t)bit_number) | |
1307 | : "memory" | |
1308 | ); | |
1309 | #else | |
1310 | __asm__ __volatile__ | |
1311 | ( | |
1312 | "lock; btcl %[bit_number], %[storage]\n\t" | |
1313 | "setc %[result]\n\t" | |
1314 | : [storage] "+m" (storage), [result] "=q" (res) | |
1315 | : [bit_number] "Kr" ((uint32_t)bit_number) | |
1316 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1317 | ); | |
b32b8144 FG |
1318 | #endif |
1319 | return res; | |
1320 | } | |
1321 | }; | |
1322 | ||
1323 | #if defined(__x86_64__) | |
1324 | ||
1325 | template< typename Base, bool Signed > | |
11fdf7f2 | 1326 | struct extra_operations< Base, 8u, Signed, true > : |
f67539c2 | 1327 | public generic_extra_operations< Base, 8u, Signed > |
b32b8144 | 1328 | { |
f67539c2 | 1329 | typedef generic_extra_operations< Base, 8u, Signed > base_type; |
b32b8144 FG |
1330 | typedef typename base_type::storage_type storage_type; |
1331 | ||
11fdf7f2 | 1332 | #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, original, result)\ |
b32b8144 FG |
1333 | __asm__ __volatile__\ |
1334 | (\ | |
1335 | ".align 16\n\t"\ | |
11fdf7f2 TL |
1336 | "1: mov %[orig], %[res]\n\t"\ |
1337 | op " %[res]\n\t"\ | |
1338 | "lock; cmpxchgq %[res], %[storage]\n\t"\ | |
b32b8144 | 1339 | "jne 1b"\ |
11fdf7f2 | 1340 | : [orig] "+a" (original), [storage] "+m" (storage), [res] "=&r" (result)\ |
b32b8144 FG |
1341 | : \ |
1342 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\ | |
1343 | ) | |
1344 | ||
1345 | static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
1346 | { | |
11fdf7f2 TL |
1347 | storage_type original = storage; |
1348 | storage_type result; | |
1349 | BOOST_ATOMIC_DETAIL_CAS_LOOP("negq", original, result); | |
1350 | return original; | |
b32b8144 FG |
1351 | } |
1352 | ||
1353 | static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
1354 | { | |
11fdf7f2 TL |
1355 | storage_type original = storage; |
1356 | storage_type result; | |
1357 | BOOST_ATOMIC_DETAIL_CAS_LOOP("notq", original, result); | |
1358 | return original; | |
1359 | } | |
1360 | ||
1361 | static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
1362 | { | |
1363 | storage_type original = storage; | |
1364 | storage_type result; | |
1365 | BOOST_ATOMIC_DETAIL_CAS_LOOP("negq", original, result); | |
1366 | return result; | |
1367 | } | |
1368 | ||
1369 | static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
1370 | { | |
1371 | storage_type original = storage; | |
1372 | storage_type result; | |
1373 | BOOST_ATOMIC_DETAIL_CAS_LOOP("notq", original, result); | |
1374 | return result; | |
1375 | } | |
1376 | ||
1377 | #undef BOOST_ATOMIC_DETAIL_CAS_LOOP | |
1378 | ||
1379 | #define BOOST_ATOMIC_DETAIL_CAS_LOOP(op, argument, original, result)\ | |
1380 | __asm__ __volatile__\ | |
1381 | (\ | |
1382 | ".align 16\n\t"\ | |
1383 | "1: mov %[arg], %[res]\n\t"\ | |
1384 | op " %%rax, %[res]\n\t"\ | |
1385 | "lock; cmpxchgq %[res], %[storage]\n\t"\ | |
1386 | "jne 1b"\ | |
1387 | : [orig] "+a" (original), [storage] "+m" (storage), [res] "=&r" (result)\ | |
1388 | : [arg] "r" (argument)\ | |
1389 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\ | |
1390 | ) | |
1391 | ||
1392 | static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1393 | { | |
1394 | storage_type original = storage; | |
1395 | storage_type result; | |
1396 | BOOST_ATOMIC_DETAIL_CAS_LOOP("andq", v, original, result); | |
1397 | return static_cast< storage_type >(result); | |
1398 | } | |
1399 | ||
1400 | static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1401 | { | |
1402 | storage_type original = storage; | |
1403 | storage_type result; | |
1404 | BOOST_ATOMIC_DETAIL_CAS_LOOP("orq", v, original, result); | |
1405 | return static_cast< storage_type >(result); | |
1406 | } | |
1407 | ||
1408 | static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1409 | { | |
1410 | storage_type original = storage; | |
1411 | storage_type result; | |
1412 | BOOST_ATOMIC_DETAIL_CAS_LOOP("xorq", v, original, result); | |
1413 | return static_cast< storage_type >(result); | |
b32b8144 FG |
1414 | } |
1415 | ||
1416 | #undef BOOST_ATOMIC_DETAIL_CAS_LOOP | |
1417 | ||
11fdf7f2 TL |
1418 | static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT |
1419 | { | |
1420 | return !!negate(storage, order); | |
1421 | } | |
1422 | ||
1423 | static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT | |
1424 | { | |
1425 | return !!bitwise_complement(storage, order); | |
1426 | } | |
1427 | ||
b32b8144 FG |
1428 | static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT |
1429 | { | |
1430 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
1431 | { | |
1432 | __asm__ __volatile__ | |
1433 | ( | |
1434 | "lock; incq %[storage]\n\t" | |
1435 | : [storage] "+m" (storage) | |
1436 | : | |
1437 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1438 | ); | |
1439 | } | |
1440 | else | |
1441 | { | |
1442 | __asm__ __volatile__ | |
1443 | ( | |
1444 | "lock; addq %[argument], %[storage]\n\t" | |
1445 | : [storage] "+m" (storage) | |
1446 | : [argument] "er" (v) | |
1447 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1448 | ); | |
1449 | } | |
1450 | } | |
1451 | ||
1452 | static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1453 | { | |
1454 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
1455 | { | |
1456 | __asm__ __volatile__ | |
1457 | ( | |
1458 | "lock; decq %[storage]\n\t" | |
1459 | : [storage] "+m" (storage) | |
1460 | : | |
1461 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1462 | ); | |
1463 | } | |
1464 | else | |
1465 | { | |
1466 | __asm__ __volatile__ | |
1467 | ( | |
1468 | "lock; subq %[argument], %[storage]\n\t" | |
1469 | : [storage] "+m" (storage) | |
1470 | : [argument] "er" (v) | |
1471 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1472 | ); | |
1473 | } | |
1474 | } | |
1475 | ||
1476 | static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
1477 | { | |
1478 | __asm__ __volatile__ | |
1479 | ( | |
1480 | "lock; negq %[storage]\n\t" | |
1481 | : [storage] "+m" (storage) | |
1482 | : | |
1483 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1484 | ); | |
1485 | } | |
1486 | ||
1487 | static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1488 | { | |
1489 | __asm__ __volatile__ | |
1490 | ( | |
1491 | "lock; andq %[argument], %[storage]\n\t" | |
1492 | : [storage] "+m" (storage) | |
1493 | : [argument] "er" (v) | |
1494 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1495 | ); | |
1496 | } | |
1497 | ||
1498 | static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1499 | { | |
1500 | __asm__ __volatile__ | |
1501 | ( | |
1502 | "lock; orq %[argument], %[storage]\n\t" | |
1503 | : [storage] "+m" (storage) | |
1504 | : [argument] "er" (v) | |
1505 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1506 | ); | |
1507 | } | |
1508 | ||
1509 | static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1510 | { | |
1511 | __asm__ __volatile__ | |
1512 | ( | |
1513 | "lock; xorq %[argument], %[storage]\n\t" | |
1514 | : [storage] "+m" (storage) | |
1515 | : [argument] "er" (v) | |
1516 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1517 | ); | |
1518 | } | |
1519 | ||
1520 | static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order) BOOST_NOEXCEPT | |
1521 | { | |
1522 | __asm__ __volatile__ | |
1523 | ( | |
1524 | "lock; notq %[storage]\n\t" | |
1525 | : [storage] "+m" (storage) | |
1526 | : | |
1527 | : "memory" | |
1528 | ); | |
1529 | } | |
1530 | ||
1531 | static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1532 | { | |
1533 | bool res; | |
1534 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
1535 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
1536 | { | |
1537 | __asm__ __volatile__ | |
1538 | ( | |
1539 | "lock; incq %[storage]\n\t" | |
11fdf7f2 | 1540 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
1541 | : |
1542 | : "memory" | |
1543 | ); | |
1544 | } | |
1545 | else | |
1546 | { | |
1547 | __asm__ __volatile__ | |
1548 | ( | |
1549 | "lock; addq %[argument], %[storage]\n\t" | |
11fdf7f2 | 1550 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
1551 | : [argument] "er" (v) |
1552 | : "memory" | |
1553 | ); | |
1554 | } | |
1555 | #else | |
1556 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
1557 | { | |
1558 | __asm__ __volatile__ | |
1559 | ( | |
1560 | "lock; incq %[storage]\n\t" | |
11fdf7f2 | 1561 | "setnz %[result]\n\t" |
b32b8144 FG |
1562 | : [storage] "+m" (storage), [result] "=q" (res) |
1563 | : | |
1564 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1565 | ); | |
1566 | } | |
1567 | else | |
1568 | { | |
1569 | __asm__ __volatile__ | |
1570 | ( | |
1571 | "lock; addq %[argument], %[storage]\n\t" | |
11fdf7f2 | 1572 | "setnz %[result]\n\t" |
b32b8144 FG |
1573 | : [storage] "+m" (storage), [result] "=q" (res) |
1574 | : [argument] "er" (v) | |
1575 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1576 | ); | |
1577 | } | |
1578 | #endif | |
1579 | return res; | |
1580 | } | |
1581 | ||
1582 | static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1583 | { | |
1584 | bool res; | |
1585 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
1586 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
1587 | { | |
1588 | __asm__ __volatile__ | |
1589 | ( | |
1590 | "lock; decq %[storage]\n\t" | |
11fdf7f2 | 1591 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
1592 | : |
1593 | : "memory" | |
1594 | ); | |
1595 | } | |
1596 | else | |
1597 | { | |
1598 | __asm__ __volatile__ | |
1599 | ( | |
1600 | "lock; subq %[argument], %[storage]\n\t" | |
11fdf7f2 | 1601 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
1602 | : [argument] "er" (v) |
1603 | : "memory" | |
1604 | ); | |
1605 | } | |
1606 | #else | |
1607 | if (BOOST_ATOMIC_DETAIL_IS_CONSTANT(v) && v == 1) | |
1608 | { | |
1609 | __asm__ __volatile__ | |
1610 | ( | |
1611 | "lock; decq %[storage]\n\t" | |
11fdf7f2 | 1612 | "setnz %[result]\n\t" |
b32b8144 FG |
1613 | : [storage] "+m" (storage), [result] "=q" (res) |
1614 | : | |
1615 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1616 | ); | |
1617 | } | |
1618 | else | |
1619 | { | |
1620 | __asm__ __volatile__ | |
1621 | ( | |
1622 | "lock; subq %[argument], %[storage]\n\t" | |
11fdf7f2 | 1623 | "setnz %[result]\n\t" |
b32b8144 FG |
1624 | : [storage] "+m" (storage), [result] "=q" (res) |
1625 | : [argument] "er" (v) | |
1626 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1627 | ); | |
1628 | } | |
1629 | #endif | |
1630 | return res; | |
1631 | } | |
1632 | ||
1633 | static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1634 | { | |
1635 | bool res; | |
1636 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
1637 | __asm__ __volatile__ | |
1638 | ( | |
1639 | "lock; andq %[argument], %[storage]\n\t" | |
11fdf7f2 | 1640 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
1641 | : [argument] "er" (v) |
1642 | : "memory" | |
1643 | ); | |
1644 | #else | |
1645 | __asm__ __volatile__ | |
1646 | ( | |
1647 | "lock; andq %[argument], %[storage]\n\t" | |
11fdf7f2 | 1648 | "setnz %[result]\n\t" |
b32b8144 FG |
1649 | : [storage] "+m" (storage), [result] "=q" (res) |
1650 | : [argument] "er" (v) | |
1651 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1652 | ); | |
1653 | #endif | |
1654 | return res; | |
1655 | } | |
1656 | ||
1657 | static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1658 | { | |
1659 | bool res; | |
1660 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
1661 | __asm__ __volatile__ | |
1662 | ( | |
1663 | "lock; orq %[argument], %[storage]\n\t" | |
11fdf7f2 | 1664 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
1665 | : [argument] "er" (v) |
1666 | : "memory" | |
1667 | ); | |
1668 | #else | |
1669 | __asm__ __volatile__ | |
1670 | ( | |
1671 | "lock; orq %[argument], %[storage]\n\t" | |
11fdf7f2 | 1672 | "setnz %[result]\n\t" |
b32b8144 FG |
1673 | : [storage] "+m" (storage), [result] "=q" (res) |
1674 | : [argument] "er" (v) | |
1675 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1676 | ); | |
1677 | #endif | |
1678 | return res; | |
1679 | } | |
1680 | ||
1681 | static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT | |
1682 | { | |
1683 | bool res; | |
1684 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
1685 | __asm__ __volatile__ | |
1686 | ( | |
1687 | "lock; xorq %[argument], %[storage]\n\t" | |
11fdf7f2 | 1688 | : [storage] "+m" (storage), [result] "=@ccnz" (res) |
b32b8144 FG |
1689 | : [argument] "er" (v) |
1690 | : "memory" | |
1691 | ); | |
1692 | #else | |
1693 | __asm__ __volatile__ | |
1694 | ( | |
1695 | "lock; xorq %[argument], %[storage]\n\t" | |
11fdf7f2 | 1696 | "setnz %[result]\n\t" |
b32b8144 FG |
1697 | : [storage] "+m" (storage), [result] "=q" (res) |
1698 | : [argument] "er" (v) | |
1699 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1700 | ); | |
f67539c2 TL |
1701 | #endif |
1702 | return res; | |
1703 | } | |
1704 | ||
1705 | static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT | |
1706 | { | |
1707 | bool res; | |
1708 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
1709 | __asm__ __volatile__ | |
1710 | ( | |
1711 | "lock; btsq %[bit_number], %[storage]\n\t" | |
1712 | : [storage] "+m" (storage), [result] "=@ccc" (res) | |
1713 | : [bit_number] "Kr" ((uint64_t)bit_number) | |
1714 | : "memory" | |
1715 | ); | |
1716 | #else | |
1717 | __asm__ __volatile__ | |
1718 | ( | |
1719 | "lock; btsq %[bit_number], %[storage]\n\t" | |
1720 | "setc %[result]\n\t" | |
1721 | : [storage] "+m" (storage), [result] "=q" (res) | |
1722 | : [bit_number] "Kr" ((uint64_t)bit_number) | |
1723 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1724 | ); | |
1725 | #endif | |
1726 | return res; | |
1727 | } | |
1728 | ||
1729 | static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT | |
1730 | { | |
1731 | bool res; | |
1732 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
1733 | __asm__ __volatile__ | |
1734 | ( | |
1735 | "lock; btrq %[bit_number], %[storage]\n\t" | |
1736 | : [storage] "+m" (storage), [result] "=@ccc" (res) | |
1737 | : [bit_number] "Kr" ((uint64_t)bit_number) | |
1738 | : "memory" | |
1739 | ); | |
1740 | #else | |
1741 | __asm__ __volatile__ | |
1742 | ( | |
1743 | "lock; btrq %[bit_number], %[storage]\n\t" | |
1744 | "setc %[result]\n\t" | |
1745 | : [storage] "+m" (storage), [result] "=q" (res) | |
1746 | : [bit_number] "Kr" ((uint64_t)bit_number) | |
1747 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1748 | ); | |
1749 | #endif | |
1750 | return res; | |
1751 | } | |
1752 | ||
1753 | static BOOST_FORCEINLINE bool bit_test_and_complement(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT | |
1754 | { | |
1755 | bool res; | |
1756 | #if defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) | |
1757 | __asm__ __volatile__ | |
1758 | ( | |
1759 | "lock; btcq %[bit_number], %[storage]\n\t" | |
1760 | : [storage] "+m" (storage), [result] "=@ccc" (res) | |
1761 | : [bit_number] "Kr" ((uint64_t)bit_number) | |
1762 | : "memory" | |
1763 | ); | |
1764 | #else | |
1765 | __asm__ __volatile__ | |
1766 | ( | |
1767 | "lock; btcq %[bit_number], %[storage]\n\t" | |
1768 | "setc %[result]\n\t" | |
1769 | : [storage] "+m" (storage), [result] "=q" (res) | |
1770 | : [bit_number] "Kr" ((uint64_t)bit_number) | |
1771 | : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" | |
1772 | ); | |
b32b8144 FG |
1773 | #endif |
1774 | return res; | |
1775 | } | |
1776 | }; | |
1777 | ||
1778 | #endif // defined(__x86_64__) | |
1779 | ||
1780 | } // namespace detail | |
1781 | } // namespace atomics | |
1782 | } // namespace boost | |
1783 | ||
1784 | #endif // BOOST_ATOMIC_DETAIL_EXTRA_OPS_GCC_X86_HPP_INCLUDED_ |