]> git.proxmox.com Git - ceph.git/blob - ceph/src/boost/libs/interprocess/include/boost/interprocess/detail/atomic.hpp
bump version to 12.2.2-pve1
[ceph.git] / ceph / src / boost / libs / interprocess / include / boost / interprocess / detail / atomic.hpp
1 //////////////////////////////////////////////////////////////////////////////
2 //
3 // (C) Copyright Ion Gaztanaga 2006-2012
4 // (C) Copyright Markus Schoepflin 2007
5 // (C) Copyright Bryce Lelbach 2010
6 //
7 // Distributed under the Boost Software License, Version 1.0. (See
8 // accompanying file LICENSE_1_0.txt or copy at
9 // http://www.boost.org/LICENSE_1_0.txt)
10 //
11 // See http://www.boost.org/libs/interprocess for documentation.
12 //
13 //////////////////////////////////////////////////////////////////////////////
14
15 #ifndef BOOST_INTERPROCESS_DETAIL_ATOMIC_HPP
16 #define BOOST_INTERPROCESS_DETAIL_ATOMIC_HPP
17
18 #ifndef BOOST_CONFIG_HPP
19 # include <boost/config.hpp>
20 #endif
21 #
22 #if defined(BOOST_HAS_PRAGMA_ONCE)
23 # pragma once
24 #endif
25
26 #include <boost/interprocess/detail/config_begin.hpp>
27 #include <boost/interprocess/detail/workaround.hpp>
28 #include <boost/cstdint.hpp>
29
30 namespace boost{
31 namespace interprocess{
32 namespace ipcdetail{
33
34 //! Atomically increment an boost::uint32_t by 1
35 //! "mem": pointer to the object
36 //! Returns the old value pointed to by mem
37 inline boost::uint32_t atomic_inc32(volatile boost::uint32_t *mem);
38
39 //! Atomically read an boost::uint32_t from memory
40 inline boost::uint32_t atomic_read32(volatile boost::uint32_t *mem);
41
42 //! Atomically set an boost::uint32_t in memory
43 //! "mem": pointer to the object
44 //! "param": val value that the object will assume
45 inline void atomic_write32(volatile boost::uint32_t *mem, boost::uint32_t val);
46
47 //! Compare an boost::uint32_t's value with "cmp".
48 //! If they are the same swap the value with "with"
49 //! "mem": pointer to the value
50 //! "with": what to swap it with
51 //! "cmp": the value to compare it to
52 //! Returns the old value of *mem
53 inline boost::uint32_t atomic_cas32
54 (volatile boost::uint32_t *mem, boost::uint32_t with, boost::uint32_t cmp);
55
56 } //namespace ipcdetail{
57 } //namespace interprocess{
58 } //namespace boost{
59
60 #if defined (BOOST_INTERPROCESS_WINDOWS)
61
62 #include <boost/interprocess/detail/win32_api.hpp>
63
64 #if defined( _MSC_VER )
65 extern "C" void _ReadWriteBarrier(void);
66 #pragma intrinsic(_ReadWriteBarrier)
67 #define BOOST_INTERPROCESS_READ_WRITE_BARRIER _ReadWriteBarrier()
68 #elif defined(__GNUC__)
69 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
70 #define BOOST_INTERPROCESS_READ_WRITE_BARRIER __sync_synchronize()
71 #else
72 #define BOOST_INTERPROCESS_READ_WRITE_BARRIER __asm__ __volatile__("" : : : "memory")
73 #endif
74 #endif
75
76 namespace boost{
77 namespace interprocess{
78 namespace ipcdetail{
79
80 //! Atomically decrement an boost::uint32_t by 1
81 //! "mem": pointer to the atomic value
82 //! Returns the old value pointed to by mem
83 inline boost::uint32_t atomic_dec32(volatile boost::uint32_t *mem)
84 { return winapi::interlocked_decrement(reinterpret_cast<volatile long*>(mem)) + 1; }
85
86 //! Atomically increment an apr_uint32_t by 1
87 //! "mem": pointer to the object
88 //! Returns the old value pointed to by mem
89 inline boost::uint32_t atomic_inc32(volatile boost::uint32_t *mem)
90 { return winapi::interlocked_increment(reinterpret_cast<volatile long*>(mem))-1; }
91
92 //! Atomically read an boost::uint32_t from memory
93 inline boost::uint32_t atomic_read32(volatile boost::uint32_t *mem)
94 {
95 const boost::uint32_t val = *mem;
96 BOOST_INTERPROCESS_READ_WRITE_BARRIER;
97 return val;
98 }
99
100 //! Atomically set an boost::uint32_t in memory
101 //! "mem": pointer to the object
102 //! "param": val value that the object will assume
103 inline void atomic_write32(volatile boost::uint32_t *mem, boost::uint32_t val)
104 { winapi::interlocked_exchange(reinterpret_cast<volatile long*>(mem), val); }
105
106 //! Compare an boost::uint32_t's value with "cmp".
107 //! If they are the same swap the value with "with"
108 //! "mem": pointer to the value
109 //! "with": what to swap it with
110 //! "cmp": the value to compare it to
111 //! Returns the old value of *mem
112 inline boost::uint32_t atomic_cas32
113 (volatile boost::uint32_t *mem, boost::uint32_t with, boost::uint32_t cmp)
114 { return winapi::interlocked_compare_exchange(reinterpret_cast<volatile long*>(mem), with, cmp); }
115
116 } //namespace ipcdetail{
117 } //namespace interprocess{
118 } //namespace boost{
119
120 #elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) && !defined(_CRAYC)
121
122 namespace boost {
123 namespace interprocess {
124 namespace ipcdetail{
125
126 //! Compare an boost::uint32_t's value with "cmp".
127 //! If they are the same swap the value with "with"
128 //! "mem": pointer to the value
129 //! "with" what to swap it with
130 //! "cmp": the value to compare it to
131 //! Returns the old value of *mem
132 inline boost::uint32_t atomic_cas32
133 (volatile boost::uint32_t *mem, boost::uint32_t with, boost::uint32_t cmp)
134 {
135 boost::uint32_t prev = cmp;
136 // This version by Mans Rullgard of Pathscale
137 __asm__ __volatile__ ( "lock\n\t"
138 "cmpxchg %2,%0"
139 : "+m"(*mem), "+a"(prev)
140 : "r"(with)
141 : "cc");
142
143 return prev;
144 }
145
146 //! Atomically add 'val' to an boost::uint32_t
147 //! "mem": pointer to the object
148 //! "val": amount to add
149 //! Returns the old value pointed to by mem
150 inline boost::uint32_t atomic_add32
151 (volatile boost::uint32_t *mem, boost::uint32_t val)
152 {
153 // int r = *pw;
154 // *mem += val;
155 // return r;
156 int r;
157
158 asm volatile
159 (
160 "lock\n\t"
161 "xadd %1, %0":
162 "+m"( *mem ), "=r"( r ): // outputs (%0, %1)
163 "1"( val ): // inputs (%2 == %1)
164 "memory", "cc" // clobbers
165 );
166
167 return r;
168 }
169
170 //! Atomically increment an apr_uint32_t by 1
171 //! "mem": pointer to the object
172 //! Returns the old value pointed to by mem
173 inline boost::uint32_t atomic_inc32(volatile boost::uint32_t *mem)
174 { return atomic_add32(mem, 1); }
175
176 //! Atomically decrement an boost::uint32_t by 1
177 //! "mem": pointer to the atomic value
178 //! Returns the old value pointed to by mem
179 inline boost::uint32_t atomic_dec32(volatile boost::uint32_t *mem)
180 { return atomic_add32(mem, (boost::uint32_t)-1); }
181
182 //! Atomically read an boost::uint32_t from memory
183 inline boost::uint32_t atomic_read32(volatile boost::uint32_t *mem)
184 {
185 const boost::uint32_t val = *mem;
186 __asm__ __volatile__ ( "" ::: "memory" );
187 return val;
188 }
189
190 //! Atomically set an boost::uint32_t in memory
191 //! "mem": pointer to the object
192 //! "param": val value that the object will assume
193 inline void atomic_write32(volatile boost::uint32_t *mem, boost::uint32_t val)
194 {
195 __asm__ __volatile__
196 (
197 "xchgl %0, %1"
198 : "+r" (val), "+m" (*mem)
199 :: "memory"
200 );
201 }
202
203 } //namespace ipcdetail{
204 } //namespace interprocess{
205 } //namespace boost{
206
207 #elif defined(__GNUC__) && (defined(__PPC__) || defined(__ppc__))
208
209 namespace boost {
210 namespace interprocess {
211 namespace ipcdetail{
212
213 //! Atomically add 'val' to an boost::uint32_t
214 //! "mem": pointer to the object
215 //! "val": amount to add
216 //! Returns the old value pointed to by mem
217 inline boost::uint32_t atomic_add32(volatile boost::uint32_t *mem, boost::uint32_t val)
218 {
219 boost::uint32_t prev, temp;
220
221 asm volatile ("1:\n\t"
222 "lwarx %0,0,%2\n\t"
223 "add %1,%0,%3\n\t"
224 "stwcx. %1,0,%2\n\t"
225 "bne- 1b"
226 : "=&r" (prev), "=&r" (temp)
227 : "b" (mem), "r" (val)
228 : "cc", "memory");
229 return prev;
230 }
231
232 //! Compare an boost::uint32_t's value with "cmp".
233 //! If they are the same swap the value with "with"
234 //! "mem": pointer to the value
235 //! "with" what to swap it with
236 //! "cmp": the value to compare it to
237 //! Returns the old value of *mem
238 inline boost::uint32_t atomic_cas32
239 (volatile boost::uint32_t *mem, boost::uint32_t with, boost::uint32_t cmp)
240 {
241 boost::uint32_t prev;
242
243 asm volatile ("1:\n\t"
244 "lwarx %0,0,%1\n\t"
245 "cmpw %0,%3\n\t"
246 "bne- 2f\n\t"
247 "stwcx. %2,0,%1\n\t"
248 "bne- 1b\n\t"
249 "2:"
250 : "=&r"(prev)
251 : "b" (mem), "r" (with), "r" (cmp)
252 : "cc", "memory");
253 return prev;
254 }
255
256 //! Atomically increment an apr_uint32_t by 1
257 //! "mem": pointer to the object
258 //! Returns the old value pointed to by mem
259 inline boost::uint32_t atomic_inc32(volatile boost::uint32_t *mem)
260 { return atomic_add32(mem, 1); }
261
262 //! Atomically decrement an boost::uint32_t by 1
263 //! "mem": pointer to the atomic value
264 //! Returns the old value pointed to by mem
265 inline boost::uint32_t atomic_dec32(volatile boost::uint32_t *mem)
266 { return atomic_add32(mem, boost::uint32_t(-1u)); }
267
268 //! Atomically read an boost::uint32_t from memory
269 inline boost::uint32_t atomic_read32(volatile boost::uint32_t *mem)
270 {
271 const boost::uint32_t val = *mem;
272 __asm__ __volatile__ ( "" ::: "memory" );
273 return val;
274 }
275
276 //! Atomically set an boost::uint32_t in memory
277 //! "mem": pointer to the object
278 //! "param": val value that the object will assume
279 inline void atomic_write32(volatile boost::uint32_t *mem, boost::uint32_t val)
280 { *mem = val; }
281
282 } //namespace ipcdetail{
283 } //namespace interprocess{
284 } //namespace boost{
285
286 #elif (defined(sun) || defined(__sun))
287
288 #include <atomic.h>
289
290 namespace boost{
291 namespace interprocess{
292 namespace ipcdetail{
293
294 //! Atomically add 'val' to an boost::uint32_t
295 //! "mem": pointer to the object
296 //! "val": amount to add
297 //! Returns the old value pointed to by mem
298 inline boost::uint32_t atomic_add32(volatile boost::uint32_t *mem, boost::uint32_t val)
299 { return atomic_add_32_nv(reinterpret_cast<volatile ::uint32_t*>(mem), (int32_t)val) - val; }
300
301 //! Compare an boost::uint32_t's value with "cmp".
302 //! If they are the same swap the value with "with"
303 //! "mem": pointer to the value
304 //! "with" what to swap it with
305 //! "cmp": the value to compare it to
306 //! Returns the old value of *mem
307 inline boost::uint32_t atomic_cas32
308 (volatile boost::uint32_t *mem, boost::uint32_t with, boost::uint32_t cmp)
309 { return atomic_cas_32(reinterpret_cast<volatile ::uint32_t*>(mem), cmp, with); }
310
311 //! Atomically increment an apr_uint32_t by 1
312 //! "mem": pointer to the object
313 //! Returns the old value pointed to by mem
314 inline boost::uint32_t atomic_inc32(volatile boost::uint32_t *mem)
315 { return atomic_add_32_nv(reinterpret_cast<volatile ::uint32_t*>(mem), 1) - 1; }
316
317 //! Atomically decrement an boost::uint32_t by 1
318 //! "mem": pointer to the atomic value
319 //! Returns the old value pointed to by mem
320 inline boost::uint32_t atomic_dec32(volatile boost::uint32_t *mem)
321 { return atomic_add_32_nv(reinterpret_cast<volatile ::uint32_t*>(mem), (boost::uint32_t)-1) + 1; }
322
323 //! Atomically read an boost::uint32_t from memory
324 inline boost::uint32_t atomic_read32(volatile boost::uint32_t *mem)
325 { return *mem; }
326
327 //! Atomically set an boost::uint32_t in memory
328 //! "mem": pointer to the object
329 //! "param": val value that the object will assume
330 inline void atomic_write32(volatile boost::uint32_t *mem, boost::uint32_t val)
331 { *mem = val; }
332
333 } //namespace ipcdetail{
334 } //namespace interprocess{
335 } //namespace boost{
336
337 #elif defined(__osf__) && defined(__DECCXX)
338
339 #include <machine/builtins.h>
340 #include <c_asm.h>
341
342 namespace boost{
343 namespace interprocess{
344 namespace ipcdetail{
345
346 //! Atomically decrement a uint32_t by 1
347 //! "mem": pointer to the atomic value
348 //! Returns the old value pointed to by mem
349 //! Acquire, memory barrier after decrement.
350 inline boost::uint32_t atomic_dec32(volatile boost::uint32_t *mem)
351 { boost::uint32_t old_val = __ATOMIC_DECREMENT_LONG(mem); __MB(); return old_val; }
352
353 //! Atomically increment a uint32_t by 1
354 //! "mem": pointer to the object
355 //! Returns the old value pointed to by mem
356 //! Release, memory barrier before increment.
357 inline boost::uint32_t atomic_inc32(volatile boost::uint32_t *mem)
358 { __MB(); return __ATOMIC_INCREMENT_LONG(mem); }
359
360 // Rational for the implementation of the atomic read and write functions.
361 //
362 // 1. The Alpha Architecture Handbook requires that access to a byte,
363 // an aligned word, an aligned longword, or an aligned quadword is
364 // atomic. (See 'Alpha Architecture Handbook', version 4, chapter 5.2.2.)
365 //
366 // 2. The CXX User's Guide states that volatile quantities are accessed
367 // with single assembler instructions, and that a compilation error
368 // occurs when declaring a quantity as volatile which is not properly
369 // aligned.
370
371 //! Atomically read an boost::uint32_t from memory
372 //! Acquire, memory barrier after load.
373 inline boost::uint32_t atomic_read32(volatile boost::uint32_t *mem)
374 { boost::uint32_t old_val = *mem; __MB(); return old_val; }
375
376 //! Atomically set an boost::uint32_t in memory
377 //! "mem": pointer to the object
378 //! "param": val value that the object will assume
379 //! Release, memory barrier before store.
380 inline void atomic_write32(volatile boost::uint32_t *mem, boost::uint32_t val)
381 { __MB(); *mem = val; }
382
383 //! Compare an boost::uint32_t's value with "cmp".
384 //! If they are the same swap the value with "with"
385 //! "mem": pointer to the value
386 //! "with" what to swap it with
387 //! "cmp": the value to compare it to
388 //! Returns the old value of *mem
389 //! Memory barrier between load and store.
390 inline boost::uint32_t atomic_cas32(
391 volatile boost::uint32_t *mem, boost::uint32_t with, boost::uint32_t cmp)
392 {
393 // Note:
394 //
395 // Branch prediction prefers backward branches, and the Alpha Architecture
396 // Handbook explicitely states that the loop should not be implemented like
397 // it is below. (See chapter 4.2.5.) Therefore the code should probably look
398 // like this:
399 //
400 // return asm(
401 // "10: ldl_l %v0,(%a0) ;"
402 // " cmpeq %v0,%a2,%t0 ;"
403 // " beq %t0,20f ;"
404 // " mb ;"
405 // " mov %a1,%t0 ;"
406 // " stl_c %t0,(%a0) ;"
407 // " beq %t0,30f ;"
408 // "20: ret ;"
409 // "30: br 10b;",
410 // mem, with, cmp);
411 //
412 // But as the compiler always transforms this into the form where a backward
413 // branch is taken on failure, we can as well implement it in the straight
414 // forward form, as this is what it will end up in anyway.
415
416 return asm(
417 "10: ldl_l %v0,(%a0) ;" // load prev value from mem and lock mem
418 " cmpeq %v0,%a2,%t0 ;" // compare with given value
419 " beq %t0,20f ;" // if not equal, we're done
420 " mb ;" // memory barrier
421 " mov %a1,%t0 ;" // load new value into scratch register
422 " stl_c %t0,(%a0) ;" // store new value to locked mem (overwriting scratch)
423 " beq %t0,10b ;" // store failed because lock has been stolen, retry
424 "20: ",
425 mem, with, cmp);
426 }
427
428 } //namespace ipcdetail{
429 } //namespace interprocess{
430 } //namespace boost{
431
432 #elif defined(__IBMCPP__) && (__IBMCPP__ >= 800) && defined(_AIX)
433
434 #include <builtins.h>
435
436 namespace boost {
437 namespace interprocess {
438 namespace ipcdetail{
439
440 //first define boost::uint32_t versions of __lwarx and __stwcx to avoid poluting
441 //all the functions with casts
442
443 //! From XLC documenation :
444 //! This function can be used with a subsequent stwcxu call to implement a
445 //! read-modify-write on a specified memory location. The two functions work
446 //! together to ensure that if the store is successfully performed, no other
447 //! processor or mechanism can modify the target doubleword between the time
448 //! lwarxu function is executed and the time the stwcxu functio ncompletes.
449 //! "mem" : pointer to the object
450 //! Returns the value at pointed to by mem
451 inline boost::uint32_t lwarxu(volatile boost::uint32_t *mem)
452 {
453 return static_cast<boost::uint32_t>(__lwarx(reinterpret_cast<volatile int*>(mem)));
454 }
455
456 //! "mem" : pointer to the object
457 //! "val" : the value to store
458 //! Returns true if the update of mem is successful and false if it is
459 //!unsuccessful
460 inline bool stwcxu(volatile boost::uint32_t* mem, boost::uint32_t val)
461 {
462 return (__stwcx(reinterpret_cast<volatile int*>(mem), static_cast<int>(val)) != 0);
463 }
464
465 //! "mem": pointer to the object
466 //! "val": amount to add
467 //! Returns the old value pointed to by mem
468 inline boost::uint32_t atomic_add32
469 (volatile boost::uint32_t *mem, boost::uint32_t val)
470 {
471 boost::uint32_t oldValue;
472 do
473 {
474 oldValue = lwarxu(mem);
475 }while (!stwcxu(mem, oldValue+val));
476 return oldValue;
477 }
478
479 //! Atomically increment an apr_uint32_t by 1
480 //! "mem": pointer to the object
481 //! Returns the old value pointed to by mem
482 inline boost::uint32_t atomic_inc32(volatile boost::uint32_t *mem)
483 { return atomic_add32(mem, 1); }
484
485 //! Atomically decrement an boost::uint32_t by 1
486 //! "mem": pointer to the atomic value
487 //! Returns the old value pointed to by mem
488 inline boost::uint32_t atomic_dec32(volatile boost::uint32_t *mem)
489 { return atomic_add32(mem, (boost::uint32_t)-1); }
490
491 //! Atomically read an boost::uint32_t from memory
492 inline boost::uint32_t atomic_read32(volatile boost::uint32_t *mem)
493 { return *mem; }
494
495 //! Compare an boost::uint32_t's value with "cmp".
496 //! If they are the same swap the value with "with"
497 //! "mem": pointer to the value
498 //! "with" what to swap it with
499 //! "cmp": the value to compare it to
500 //! Returns the old value of *mem
501 inline boost::uint32_t atomic_cas32
502 (volatile boost::uint32_t *mem, boost::uint32_t with, boost::uint32_t cmp)
503 {
504 boost::uint32_t oldValue;
505 boost::uint32_t valueToStore;
506 do
507 {
508 oldValue = lwarxu(mem);
509 } while (!stwcxu(mem, (oldValue == with) ? cmp : oldValue));
510
511 return oldValue;
512 }
513
514 //! Atomically set an boost::uint32_t in memory
515 //! "mem": pointer to the object
516 //! "param": val value that the object will assume
517 inline void atomic_write32(volatile boost::uint32_t *mem, boost::uint32_t val)
518 { *mem = val; }
519
520 } //namespace ipcdetail
521 } //namespace interprocess
522 } //namespace boost
523
524 #elif defined(__GNUC__) && ( __GNUC__ * 100 + __GNUC_MINOR__ >= 401 )
525
526 namespace boost {
527 namespace interprocess {
528 namespace ipcdetail{
529
530 //! Atomically add 'val' to an boost::uint32_t
531 //! "mem": pointer to the object
532 //! "val": amount to add
533 //! Returns the old value pointed to by mem
534 inline boost::uint32_t atomic_add32
535 (volatile boost::uint32_t *mem, boost::uint32_t val)
536 { return __sync_fetch_and_add(const_cast<boost::uint32_t *>(mem), val); }
537
538 //! Atomically increment an apr_uint32_t by 1
539 //! "mem": pointer to the object
540 //! Returns the old value pointed to by mem
541 inline boost::uint32_t atomic_inc32(volatile boost::uint32_t *mem)
542 { return atomic_add32(mem, 1); }
543
544 //! Atomically decrement an boost::uint32_t by 1
545 //! "mem": pointer to the atomic value
546 //! Returns the old value pointed to by mem
547 inline boost::uint32_t atomic_dec32(volatile boost::uint32_t *mem)
548 { return atomic_add32(mem, (boost::uint32_t)-1); }
549
550 //! Atomically read an boost::uint32_t from memory
551 inline boost::uint32_t atomic_read32(volatile boost::uint32_t *mem)
552 { boost::uint32_t old_val = *mem; __sync_synchronize(); return old_val; }
553
554 //! Compare an boost::uint32_t's value with "cmp".
555 //! If they are the same swap the value with "with"
556 //! "mem": pointer to the value
557 //! "with" what to swap it with
558 //! "cmp": the value to compare it to
559 //! Returns the old value of *mem
560 inline boost::uint32_t atomic_cas32
561 (volatile boost::uint32_t *mem, boost::uint32_t with, boost::uint32_t cmp)
562 { return __sync_val_compare_and_swap(const_cast<boost::uint32_t *>(mem), cmp, with); }
563
564 //! Atomically set an boost::uint32_t in memory
565 //! "mem": pointer to the object
566 //! "param": val value that the object will assume
567 inline void atomic_write32(volatile boost::uint32_t *mem, boost::uint32_t val)
568 { __sync_synchronize(); *mem = val; }
569
570 } //namespace ipcdetail{
571 } //namespace interprocess{
572 } //namespace boost{
573
574 #else
575
576 #error No atomic operations implemented for this platform, sorry!
577
578 #endif
579
580 namespace boost{
581 namespace interprocess{
582 namespace ipcdetail{
583
584 inline bool atomic_add_unless32
585 (volatile boost::uint32_t *mem, boost::uint32_t value, boost::uint32_t unless_this)
586 {
587 boost::uint32_t old, c(atomic_read32(mem));
588 while(c != unless_this && (old = atomic_cas32(mem, c + value, c)) != c){
589 c = old;
590 }
591 return c != unless_this;
592 }
593
594 } //namespace ipcdetail
595 } //namespace interprocess
596 } //namespace boost
597
598
599 #include <boost/interprocess/detail/config_end.hpp>
600
601 #endif //BOOST_INTERPROCESS_DETAIL_ATOMIC_HPP