include/linux/atomic.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /* Atomic operations usable in machine independent code */
   3 #ifndef _LINUX_ATOMIC_H
   4 #define _LINUX_ATOMIC_H
   5 #include <asm/atomic.h>
   6 #include <asm/barrier.h>
   7
   8 /*
   9  * Relaxed variants of xchg, cmpxchg and some atomic operations.
  10  *
  11  * We support four variants:
  12  *
  13  * - Fully ordered: The default implementation, no suffix required.
  14  * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
  15  * - Release: Provides RELEASE semantics, _release suffix.
  16  * - Relaxed: No ordering guarantees, _relaxed suffix.
  17  *
  18  * For compound atomics performing both a load and a store, ACQUIRE
  19  * semantics apply only to the load and RELEASE semantics only to the
  20  * store portion of the operation. Note that a failed cmpxchg_acquire
  21  * does -not- imply any memory ordering constraints.
  22  *
  23  * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
  24  */
  25
  26 #ifndef atomic_read_acquire
  27 #define  atomic_read_acquire(v)         smp_load_acquire(&(v)->counter)
  28 #endif
  29
  30 #ifndef atomic_set_release
  31 #define  atomic_set_release(v, i)       smp_store_release(&(v)->counter, (i))
  32 #endif
  33
  34 /*
  35  * The idea here is to build acquire/release variants by adding explicit
  36  * barriers on top of the relaxed variant. In the case where the relaxed
  37  * variant is already fully ordered, no additional barriers are needed.
  38  *
  39  * Besides, if an arch has a special barrier for acquire/release, it could
  40  * implement its own __atomic_op_* and use the same framework for building
  41  * variants
  42  *
  43  * If an architecture overrides __atomic_op_acquire() it will probably want
  44  * to define smp_mb__after_spinlock().
  45  */
  46 #ifndef __atomic_op_acquire
  47 #define __atomic_op_acquire(op, args...)                                \
  48 ({                                                                      \
  49         typeof(op##_relaxed(args)) __ret  = op##_relaxed(args);         \
  50         smp_mb__after_atomic();                                         \
  51         __ret;                                                          \
  52 })
  53 #endif
  54
  55 #ifndef __atomic_op_release
  56 #define __atomic_op_release(op, args...)                                \
  57 ({                                                                      \
  58         smp_mb__before_atomic();                                        \
  59         op##_relaxed(args);                                             \
  60 })
  61 #endif
  62
  63 #ifndef __atomic_op_fence
  64 #define __atomic_op_fence(op, args...)                                  \
  65 ({                                                                      \
  66         typeof(op##_relaxed(args)) __ret;                               \
  67         smp_mb__before_atomic();                                        \
  68         __ret = op##_relaxed(args);                                     \
  69         smp_mb__after_atomic();                                         \
  70         __ret;                                                          \
  71 })
  72 #endif
  73
  74 /* atomic_add_return_relaxed */
  75 #ifndef atomic_add_return_relaxed
  76 #define  atomic_add_return_relaxed      atomic_add_return
  77 #define  atomic_add_return_acquire      atomic_add_return
  78 #define  atomic_add_return_release      atomic_add_return
  79
  80 #else /* atomic_add_return_relaxed */
  81
  82 #ifndef atomic_add_return_acquire
  83 #define  atomic_add_return_acquire(...)                                 \
  84         __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
  85 #endif
  86
  87 #ifndef atomic_add_return_release
  88 #define  atomic_add_return_release(...)                                 \
  89         __atomic_op_release(atomic_add_return, __VA_ARGS__)
  90 #endif
  91
  92 #ifndef atomic_add_return
  93 #define  atomic_add_return(...)                                         \
  94         __atomic_op_fence(atomic_add_return, __VA_ARGS__)
  95 #endif
  96 #endif /* atomic_add_return_relaxed */
  97
  98 /* atomic_inc_return_relaxed */
  99 #ifndef atomic_inc_return_relaxed
 100 #define  atomic_inc_return_relaxed      atomic_inc_return
 101 #define  atomic_inc_return_acquire      atomic_inc_return
 102 #define  atomic_inc_return_release      atomic_inc_return
 103
 104 #else /* atomic_inc_return_relaxed */
 105
 106 #ifndef atomic_inc_return_acquire
 107 #define  atomic_inc_return_acquire(...)                                 \
 108         __atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
 109 #endif
 110
 111 #ifndef atomic_inc_return_release
 112 #define  atomic_inc_return_release(...)                                 \
 113         __atomic_op_release(atomic_inc_return, __VA_ARGS__)
 114 #endif
 115
 116 #ifndef atomic_inc_return
 117 #define  atomic_inc_return(...)                                         \
 118         __atomic_op_fence(atomic_inc_return, __VA_ARGS__)
 119 #endif
 120 #endif /* atomic_inc_return_relaxed */
 121
 122 /* atomic_sub_return_relaxed */
 123 #ifndef atomic_sub_return_relaxed
 124 #define  atomic_sub_return_relaxed      atomic_sub_return
 125 #define  atomic_sub_return_acquire      atomic_sub_return
 126 #define  atomic_sub_return_release      atomic_sub_return
 127
 128 #else /* atomic_sub_return_relaxed */
 129
 130 #ifndef atomic_sub_return_acquire
 131 #define  atomic_sub_return_acquire(...)                                 \
 132         __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
 133 #endif
 134
 135 #ifndef atomic_sub_return_release
 136 #define  atomic_sub_return_release(...)                                 \
 137         __atomic_op_release(atomic_sub_return, __VA_ARGS__)
 138 #endif
 139
 140 #ifndef atomic_sub_return
 141 #define  atomic_sub_return(...)                                         \
 142         __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
 143 #endif
 144 #endif /* atomic_sub_return_relaxed */
 145
 146 /* atomic_dec_return_relaxed */
 147 #ifndef atomic_dec_return_relaxed
 148 #define  atomic_dec_return_relaxed      atomic_dec_return
 149 #define  atomic_dec_return_acquire      atomic_dec_return
 150 #define  atomic_dec_return_release      atomic_dec_return
 151
 152 #else /* atomic_dec_return_relaxed */
 153
 154 #ifndef atomic_dec_return_acquire
 155 #define  atomic_dec_return_acquire(...)                                 \
 156         __atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
 157 #endif
 158
 159 #ifndef atomic_dec_return_release
 160 #define  atomic_dec_return_release(...)                                 \
 161         __atomic_op_release(atomic_dec_return, __VA_ARGS__)
 162 #endif
 163
 164 #ifndef atomic_dec_return
 165 #define  atomic_dec_return(...)                                         \
 166         __atomic_op_fence(atomic_dec_return, __VA_ARGS__)
 167 #endif
 168 #endif /* atomic_dec_return_relaxed */
 169
 170
 171 /* atomic_fetch_add_relaxed */
 172 #ifndef atomic_fetch_add_relaxed
 173 #define atomic_fetch_add_relaxed        atomic_fetch_add
 174 #define atomic_fetch_add_acquire        atomic_fetch_add
 175 #define atomic_fetch_add_release        atomic_fetch_add
 176
 177 #else /* atomic_fetch_add_relaxed */
 178
 179 #ifndef atomic_fetch_add_acquire
 180 #define atomic_fetch_add_acquire(...)                                   \
 181         __atomic_op_acquire(atomic_fetch_add, __VA_ARGS__)
 182 #endif
 183
 184 #ifndef atomic_fetch_add_release
 185 #define atomic_fetch_add_release(...)                                   \
 186         __atomic_op_release(atomic_fetch_add, __VA_ARGS__)
 187 #endif
 188
 189 #ifndef atomic_fetch_add
 190 #define atomic_fetch_add(...)                                           \
 191         __atomic_op_fence(atomic_fetch_add, __VA_ARGS__)
 192 #endif
 193 #endif /* atomic_fetch_add_relaxed */
 194
 195 /* atomic_fetch_inc_relaxed */
 196 #ifndef atomic_fetch_inc_relaxed
 197
 198 #ifndef atomic_fetch_inc
 199 #define atomic_fetch_inc(v)             atomic_fetch_add(1, (v))
 200 #define atomic_fetch_inc_relaxed(v)     atomic_fetch_add_relaxed(1, (v))
 201 #define atomic_fetch_inc_acquire(v)     atomic_fetch_add_acquire(1, (v))
 202 #define atomic_fetch_inc_release(v)     atomic_fetch_add_release(1, (v))
 203 #else /* atomic_fetch_inc */
 204 #define atomic_fetch_inc_relaxed        atomic_fetch_inc
 205 #define atomic_fetch_inc_acquire        atomic_fetch_inc
 206 #define atomic_fetch_inc_release        atomic_fetch_inc
 207 #endif /* atomic_fetch_inc */
 208
 209 #else /* atomic_fetch_inc_relaxed */
 210
 211 #ifndef atomic_fetch_inc_acquire
 212 #define atomic_fetch_inc_acquire(...)                                   \
 213         __atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__)
 214 #endif
 215
 216 #ifndef atomic_fetch_inc_release
 217 #define atomic_fetch_inc_release(...)                                   \
 218         __atomic_op_release(atomic_fetch_inc, __VA_ARGS__)
 219 #endif
 220
 221 #ifndef atomic_fetch_inc
 222 #define atomic_fetch_inc(...)                                           \
 223         __atomic_op_fence(atomic_fetch_inc, __VA_ARGS__)
 224 #endif
 225 #endif /* atomic_fetch_inc_relaxed */
 226
 227 /* atomic_fetch_sub_relaxed */
 228 #ifndef atomic_fetch_sub_relaxed
 229 #define atomic_fetch_sub_relaxed        atomic_fetch_sub
 230 #define atomic_fetch_sub_acquire        atomic_fetch_sub
 231 #define atomic_fetch_sub_release        atomic_fetch_sub
 232
 233 #else /* atomic_fetch_sub_relaxed */
 234
 235 #ifndef atomic_fetch_sub_acquire
 236 #define atomic_fetch_sub_acquire(...)                                   \
 237         __atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__)
 238 #endif
 239
 240 #ifndef atomic_fetch_sub_release
 241 #define atomic_fetch_sub_release(...)                                   \
 242         __atomic_op_release(atomic_fetch_sub, __VA_ARGS__)
 243 #endif
 244
 245 #ifndef atomic_fetch_sub
 246 #define atomic_fetch_sub(...)                                           \
 247         __atomic_op_fence(atomic_fetch_sub, __VA_ARGS__)
 248 #endif
 249 #endif /* atomic_fetch_sub_relaxed */
 250
 251 /* atomic_fetch_dec_relaxed */
 252 #ifndef atomic_fetch_dec_relaxed
 253
 254 #ifndef atomic_fetch_dec
 255 #define atomic_fetch_dec(v)             atomic_fetch_sub(1, (v))
 256 #define atomic_fetch_dec_relaxed(v)     atomic_fetch_sub_relaxed(1, (v))
 257 #define atomic_fetch_dec_acquire(v)     atomic_fetch_sub_acquire(1, (v))
 258 #define atomic_fetch_dec_release(v)     atomic_fetch_sub_release(1, (v))
 259 #else /* atomic_fetch_dec */
 260 #define atomic_fetch_dec_relaxed        atomic_fetch_dec
 261 #define atomic_fetch_dec_acquire        atomic_fetch_dec
 262 #define atomic_fetch_dec_release        atomic_fetch_dec
 263 #endif /* atomic_fetch_dec */
 264
 265 #else /* atomic_fetch_dec_relaxed */
 266
 267 #ifndef atomic_fetch_dec_acquire
 268 #define atomic_fetch_dec_acquire(...)                                   \
 269         __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
 270 #endif
 271
 272 #ifndef atomic_fetch_dec_release
 273 #define atomic_fetch_dec_release(...)                                   \
 274         __atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
 275 #endif
 276
 277 #ifndef atomic_fetch_dec
 278 #define atomic_fetch_dec(...)                                           \
 279         __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
 280 #endif
 281 #endif /* atomic_fetch_dec_relaxed */
 282
 283 /* atomic_fetch_or_relaxed */
 284 #ifndef atomic_fetch_or_relaxed
 285 #define atomic_fetch_or_relaxed atomic_fetch_or
 286 #define atomic_fetch_or_acquire atomic_fetch_or
 287 #define atomic_fetch_or_release atomic_fetch_or
 288
 289 #else /* atomic_fetch_or_relaxed */
 290
 291 #ifndef atomic_fetch_or_acquire
 292 #define atomic_fetch_or_acquire(...)                                    \
 293         __atomic_op_acquire(atomic_fetch_or, __VA_ARGS__)
 294 #endif
 295
 296 #ifndef atomic_fetch_or_release
 297 #define atomic_fetch_or_release(...)                                    \
 298         __atomic_op_release(atomic_fetch_or, __VA_ARGS__)
 299 #endif
 300
 301 #ifndef atomic_fetch_or
 302 #define atomic_fetch_or(...)                                            \
 303         __atomic_op_fence(atomic_fetch_or, __VA_ARGS__)
 304 #endif
 305 #endif /* atomic_fetch_or_relaxed */
 306
 307 /* atomic_fetch_and_relaxed */
 308 #ifndef atomic_fetch_and_relaxed
 309 #define atomic_fetch_and_relaxed        atomic_fetch_and
 310 #define atomic_fetch_and_acquire        atomic_fetch_and
 311 #define atomic_fetch_and_release        atomic_fetch_and
 312
 313 #else /* atomic_fetch_and_relaxed */
 314
 315 #ifndef atomic_fetch_and_acquire
 316 #define atomic_fetch_and_acquire(...)                                   \
 317         __atomic_op_acquire(atomic_fetch_and, __VA_ARGS__)
 318 #endif
 319
 320 #ifndef atomic_fetch_and_release
 321 #define atomic_fetch_and_release(...)                                   \
 322         __atomic_op_release(atomic_fetch_and, __VA_ARGS__)
 323 #endif
 324
 325 #ifndef atomic_fetch_and
 326 #define atomic_fetch_and(...)                                           \
 327         __atomic_op_fence(atomic_fetch_and, __VA_ARGS__)
 328 #endif
 329 #endif /* atomic_fetch_and_relaxed */
 330
 331 #ifdef atomic_andnot
 332 /* atomic_fetch_andnot_relaxed */
 333 #ifndef atomic_fetch_andnot_relaxed
 334 #define atomic_fetch_andnot_relaxed     atomic_fetch_andnot
 335 #define atomic_fetch_andnot_acquire     atomic_fetch_andnot
 336 #define atomic_fetch_andnot_release     atomic_fetch_andnot
 337
 338 #else /* atomic_fetch_andnot_relaxed */
 339
 340 #ifndef atomic_fetch_andnot_acquire
 341 #define atomic_fetch_andnot_acquire(...)                                        \
 342         __atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__)
 343 #endif
 344
 345 #ifndef atomic_fetch_andnot_release
 346 #define atomic_fetch_andnot_release(...)                                        \
 347         __atomic_op_release(atomic_fetch_andnot, __VA_ARGS__)
 348 #endif
 349
 350 #ifndef atomic_fetch_andnot
 351 #define atomic_fetch_andnot(...)                                                \
 352         __atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__)
 353 #endif
 354 #endif /* atomic_fetch_andnot_relaxed */
 355 #endif /* atomic_andnot */
 356
 357 /* atomic_fetch_xor_relaxed */
 358 #ifndef atomic_fetch_xor_relaxed
 359 #define atomic_fetch_xor_relaxed        atomic_fetch_xor
 360 #define atomic_fetch_xor_acquire        atomic_fetch_xor
 361 #define atomic_fetch_xor_release        atomic_fetch_xor
 362
 363 #else /* atomic_fetch_xor_relaxed */
 364
 365 #ifndef atomic_fetch_xor_acquire
 366 #define atomic_fetch_xor_acquire(...)                                   \
 367         __atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__)
 368 #endif
 369
 370 #ifndef atomic_fetch_xor_release
 371 #define atomic_fetch_xor_release(...)                                   \
 372         __atomic_op_release(atomic_fetch_xor, __VA_ARGS__)
 373 #endif
 374
 375 #ifndef atomic_fetch_xor
 376 #define atomic_fetch_xor(...)                                           \
 377         __atomic_op_fence(atomic_fetch_xor, __VA_ARGS__)
 378 #endif
 379 #endif /* atomic_fetch_xor_relaxed */
 380
 381
 382 /* atomic_xchg_relaxed */
 383 #ifndef atomic_xchg_relaxed
 384 #define  atomic_xchg_relaxed            atomic_xchg
 385 #define  atomic_xchg_acquire            atomic_xchg
 386 #define  atomic_xchg_release            atomic_xchg
 387
 388 #else /* atomic_xchg_relaxed */
 389
 390 #ifndef atomic_xchg_acquire
 391 #define  atomic_xchg_acquire(...)                                       \
 392         __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
 393 #endif
 394
 395 #ifndef atomic_xchg_release
 396 #define  atomic_xchg_release(...)                                       \
 397         __atomic_op_release(atomic_xchg, __VA_ARGS__)
 398 #endif
 399
 400 #ifndef atomic_xchg
 401 #define  atomic_xchg(...)                                               \
 402         __atomic_op_fence(atomic_xchg, __VA_ARGS__)
 403 #endif
 404 #endif /* atomic_xchg_relaxed */
 405
 406 /* atomic_cmpxchg_relaxed */
 407 #ifndef atomic_cmpxchg_relaxed
 408 #define  atomic_cmpxchg_relaxed         atomic_cmpxchg
 409 #define  atomic_cmpxchg_acquire         atomic_cmpxchg
 410 #define  atomic_cmpxchg_release         atomic_cmpxchg
 411
 412 #else /* atomic_cmpxchg_relaxed */
 413
 414 #ifndef atomic_cmpxchg_acquire
 415 #define  atomic_cmpxchg_acquire(...)                                    \
 416         __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
 417 #endif
 418
 419 #ifndef atomic_cmpxchg_release
 420 #define  atomic_cmpxchg_release(...)                                    \
 421         __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
 422 #endif
 423
 424 #ifndef atomic_cmpxchg
 425 #define  atomic_cmpxchg(...)                                            \
 426         __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
 427 #endif
 428 #endif /* atomic_cmpxchg_relaxed */
 429
 430 #ifndef atomic_try_cmpxchg
 431
 432 #define __atomic_try_cmpxchg(type, _p, _po, _n)                         \
 433 ({                                                                      \
 434         typeof(_po) __po = (_po);                                       \
 435         typeof(*(_po)) __r, __o = *__po;                                \
 436         __r = atomic_cmpxchg##type((_p), __o, (_n));                    \
 437         if (unlikely(__r != __o))                                       \
 438                 *__po = __r;                                            \
 439         likely(__r == __o);                                             \
 440 })
 441
 442 #define atomic_try_cmpxchg(_p, _po, _n)         __atomic_try_cmpxchg(, _p, _po, _n)
 443 #define atomic_try_cmpxchg_relaxed(_p, _po, _n) __atomic_try_cmpxchg(_relaxed, _p, _po, _n)
 444 #define atomic_try_cmpxchg_acquire(_p, _po, _n) __atomic_try_cmpxchg(_acquire, _p, _po, _n)
 445 #define atomic_try_cmpxchg_release(_p, _po, _n) __atomic_try_cmpxchg(_release, _p, _po, _n)
 446
 447 #else /* atomic_try_cmpxchg */
 448 #define atomic_try_cmpxchg_relaxed      atomic_try_cmpxchg
 449 #define atomic_try_cmpxchg_acquire      atomic_try_cmpxchg
 450 #define atomic_try_cmpxchg_release      atomic_try_cmpxchg
 451 #endif /* atomic_try_cmpxchg */
 452
 453 /* cmpxchg_relaxed */
 454 #ifndef cmpxchg_relaxed
 455 #define  cmpxchg_relaxed                cmpxchg
 456 #define  cmpxchg_acquire                cmpxchg
 457 #define  cmpxchg_release                cmpxchg
 458
 459 #else /* cmpxchg_relaxed */
 460
 461 #ifndef cmpxchg_acquire
 462 #define  cmpxchg_acquire(...)                                           \
 463         __atomic_op_acquire(cmpxchg, __VA_ARGS__)
 464 #endif
 465
 466 #ifndef cmpxchg_release
 467 #define  cmpxchg_release(...)                                           \
 468         __atomic_op_release(cmpxchg, __VA_ARGS__)
 469 #endif
 470
 471 #ifndef cmpxchg
 472 #define  cmpxchg(...)                                                   \
 473         __atomic_op_fence(cmpxchg, __VA_ARGS__)
 474 #endif
 475 #endif /* cmpxchg_relaxed */
 476
 477 /* cmpxchg64_relaxed */
 478 #ifndef cmpxchg64_relaxed
 479 #define  cmpxchg64_relaxed              cmpxchg64
 480 #define  cmpxchg64_acquire              cmpxchg64
 481 #define  cmpxchg64_release              cmpxchg64
 482
 483 #else /* cmpxchg64_relaxed */
 484
 485 #ifndef cmpxchg64_acquire
 486 #define  cmpxchg64_acquire(...)                                         \
 487         __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
 488 #endif
 489
 490 #ifndef cmpxchg64_release
 491 #define  cmpxchg64_release(...)                                         \
 492         __atomic_op_release(cmpxchg64, __VA_ARGS__)
 493 #endif
 494
 495 #ifndef cmpxchg64
 496 #define  cmpxchg64(...)                                                 \
 497         __atomic_op_fence(cmpxchg64, __VA_ARGS__)
 498 #endif
 499 #endif /* cmpxchg64_relaxed */
 500
 501 /* xchg_relaxed */
 502 #ifndef xchg_relaxed
 503 #define  xchg_relaxed                   xchg
 504 #define  xchg_acquire                   xchg
 505 #define  xchg_release                   xchg
 506
 507 #else /* xchg_relaxed */
 508
 509 #ifndef xchg_acquire
 510 #define  xchg_acquire(...)              __atomic_op_acquire(xchg, __VA_ARGS__)
 511 #endif
 512
 513 #ifndef xchg_release
 514 #define  xchg_release(...)              __atomic_op_release(xchg, __VA_ARGS__)
 515 #endif
 516
 517 #ifndef xchg
 518 #define  xchg(...)                      __atomic_op_fence(xchg, __VA_ARGS__)
 519 #endif
 520 #endif /* xchg_relaxed */
 521
 522 /**
 523  * atomic_add_unless - add unless the number is already a given value
 524  * @v: pointer of type atomic_t
 525  * @a: the amount to add to v...
 526  * @u: ...unless v is equal to u.
 527  *
 528  * Atomically adds @a to @v, so long as @v was not already @u.
 529  * Returns non-zero if @v was not @u, and zero otherwise.
 530  */
 531 static inline int atomic_add_unless(atomic_t *v, int a, int u)
 532 {
 533         return __atomic_add_unless(v, a, u) != u;
 534 }
 535
 536 /**
 537  * atomic_inc_not_zero - increment unless the number is zero
 538  * @v: pointer of type atomic_t
 539  *
 540  * Atomically increments @v by 1, so long as @v is non-zero.
 541  * Returns non-zero if @v was non-zero, and zero otherwise.
 542  */
 543 #ifndef atomic_inc_not_zero
 544 #define atomic_inc_not_zero(v)          atomic_add_unless((v), 1, 0)
 545 #endif
 546
 547 #ifndef atomic_andnot
 548 static inline void atomic_andnot(int i, atomic_t *v)
 549 {
 550         atomic_and(~i, v);
 551 }
 552
 553 static inline int atomic_fetch_andnot(int i, atomic_t *v)
 554 {
 555         return atomic_fetch_and(~i, v);
 556 }
 557
 558 static inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v)
 559 {
 560         return atomic_fetch_and_relaxed(~i, v);
 561 }
 562
 563 static inline int atomic_fetch_andnot_acquire(int i, atomic_t *v)
 564 {
 565         return atomic_fetch_and_acquire(~i, v);
 566 }
 567
 568 static inline int atomic_fetch_andnot_release(int i, atomic_t *v)
 569 {
 570         return atomic_fetch_and_release(~i, v);
 571 }
 572 #endif
 573
 574 /**
 575  * atomic_inc_not_zero_hint - increment if not null
 576  * @v: pointer of type atomic_t
 577  * @hint: probable value of the atomic before the increment
 578  *
 579  * This version of atomic_inc_not_zero() gives a hint of probable
 580  * value of the atomic. This helps processor to not read the memory
 581  * before doing the atomic read/modify/write cycle, lowering
 582  * number of bus transactions on some arches.
 583  *
 584  * Returns: 0 if increment was not done, 1 otherwise.
 585  */
 586 #ifndef atomic_inc_not_zero_hint
 587 static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
 588 {
 589         int val, c = hint;
 590
 591         /* sanity test, should be removed by compiler if hint is a constant */
 592         if (!hint)
 593                 return atomic_inc_not_zero(v);
 594
 595         do {
 596                 val = atomic_cmpxchg(v, c, c + 1);
 597                 if (val == c)
 598                         return 1;
 599                 c = val;
 600         } while (c);
 601
 602         return 0;
 603 }
 604 #endif
 605
 606 #ifndef atomic_inc_unless_negative
 607 static inline int atomic_inc_unless_negative(atomic_t *p)
 608 {
 609         int v, v1;
 610         for (v = 0; v >= 0; v = v1) {
 611                 v1 = atomic_cmpxchg(p, v, v + 1);
 612                 if (likely(v1 == v))
 613                         return 1;
 614         }
 615         return 0;
 616 }
 617 #endif
 618
 619 #ifndef atomic_dec_unless_positive
 620 static inline int atomic_dec_unless_positive(atomic_t *p)
 621 {
 622         int v, v1;
 623         for (v = 0; v <= 0; v = v1) {
 624                 v1 = atomic_cmpxchg(p, v, v - 1);
 625                 if (likely(v1 == v))
 626                         return 1;
 627         }
 628         return 0;
 629 }
 630 #endif
 631
 632 /*
 633  * atomic_dec_if_positive - decrement by 1 if old value positive
 634  * @v: pointer of type atomic_t
 635  *
 636  * The function returns the old value of *v minus 1, even if
 637  * the atomic variable, v, was not decremented.
 638  */
 639 #ifndef atomic_dec_if_positive
 640 static inline int atomic_dec_if_positive(atomic_t *v)
 641 {
 642         int c, old, dec;
 643         c = atomic_read(v);
 644         for (;;) {
 645                 dec = c - 1;
 646                 if (unlikely(dec < 0))
 647                         break;
 648                 old = atomic_cmpxchg((v), c, dec);
 649                 if (likely(old == c))
 650                         break;
 651                 c = old;
 652         }
 653         return dec;
 654 }
 655 #endif
 656
 657 #define atomic_cond_read_acquire(v, c)  smp_cond_load_acquire(&(v)->counter, (c))
 658
 659 #ifdef CONFIG_GENERIC_ATOMIC64
 660 #include <asm-generic/atomic64.h>
 661 #endif
 662
 663 #ifndef atomic64_read_acquire
 664 #define  atomic64_read_acquire(v)       smp_load_acquire(&(v)->counter)
 665 #endif
 666
 667 #ifndef atomic64_set_release
 668 #define  atomic64_set_release(v, i)     smp_store_release(&(v)->counter, (i))
 669 #endif
 670
 671 /* atomic64_add_return_relaxed */
 672 #ifndef atomic64_add_return_relaxed
 673 #define  atomic64_add_return_relaxed    atomic64_add_return
 674 #define  atomic64_add_return_acquire    atomic64_add_return
 675 #define  atomic64_add_return_release    atomic64_add_return
 676
 677 #else /* atomic64_add_return_relaxed */
 678
 679 #ifndef atomic64_add_return_acquire
 680 #define  atomic64_add_return_acquire(...)                               \
 681         __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
 682 #endif
 683
 684 #ifndef atomic64_add_return_release
 685 #define  atomic64_add_return_release(...)                               \
 686         __atomic_op_release(atomic64_add_return, __VA_ARGS__)
 687 #endif
 688
 689 #ifndef atomic64_add_return
 690 #define  atomic64_add_return(...)                                       \
 691         __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
 692 #endif
 693 #endif /* atomic64_add_return_relaxed */
 694
 695 /* atomic64_inc_return_relaxed */
 696 #ifndef atomic64_inc_return_relaxed
 697 #define  atomic64_inc_return_relaxed    atomic64_inc_return
 698 #define  atomic64_inc_return_acquire    atomic64_inc_return
 699 #define  atomic64_inc_return_release    atomic64_inc_return
 700
 701 #else /* atomic64_inc_return_relaxed */
 702
 703 #ifndef atomic64_inc_return_acquire
 704 #define  atomic64_inc_return_acquire(...)                               \
 705         __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
 706 #endif
 707
 708 #ifndef atomic64_inc_return_release
 709 #define  atomic64_inc_return_release(...)                               \
 710         __atomic_op_release(atomic64_inc_return, __VA_ARGS__)
 711 #endif
 712
 713 #ifndef atomic64_inc_return
 714 #define  atomic64_inc_return(...)                                       \
 715         __atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
 716 #endif
 717 #endif /* atomic64_inc_return_relaxed */
 718
 719
 720 /* atomic64_sub_return_relaxed */
 721 #ifndef atomic64_sub_return_relaxed
 722 #define  atomic64_sub_return_relaxed    atomic64_sub_return
 723 #define  atomic64_sub_return_acquire    atomic64_sub_return
 724 #define  atomic64_sub_return_release    atomic64_sub_return
 725
 726 #else /* atomic64_sub_return_relaxed */
 727
 728 #ifndef atomic64_sub_return_acquire
 729 #define  atomic64_sub_return_acquire(...)                               \
 730         __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
 731 #endif
 732
 733 #ifndef atomic64_sub_return_release
 734 #define  atomic64_sub_return_release(...)                               \
 735         __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
 736 #endif
 737
 738 #ifndef atomic64_sub_return
 739 #define  atomic64_sub_return(...)                                       \
 740         __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
 741 #endif
 742 #endif /* atomic64_sub_return_relaxed */
 743
 744 /* atomic64_dec_return_relaxed */
 745 #ifndef atomic64_dec_return_relaxed
 746 #define  atomic64_dec_return_relaxed    atomic64_dec_return
 747 #define  atomic64_dec_return_acquire    atomic64_dec_return
 748 #define  atomic64_dec_return_release    atomic64_dec_return
 749
 750 #else /* atomic64_dec_return_relaxed */
 751
 752 #ifndef atomic64_dec_return_acquire
 753 #define  atomic64_dec_return_acquire(...)                               \
 754         __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
 755 #endif
 756
 757 #ifndef atomic64_dec_return_release
 758 #define  atomic64_dec_return_release(...)                               \
 759         __atomic_op_release(atomic64_dec_return, __VA_ARGS__)
 760 #endif
 761
 762 #ifndef atomic64_dec_return
 763 #define  atomic64_dec_return(...)                                       \
 764         __atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
 765 #endif
 766 #endif /* atomic64_dec_return_relaxed */
 767
 768
 769 /* atomic64_fetch_add_relaxed */
 770 #ifndef atomic64_fetch_add_relaxed
 771 #define atomic64_fetch_add_relaxed      atomic64_fetch_add
 772 #define atomic64_fetch_add_acquire      atomic64_fetch_add
 773 #define atomic64_fetch_add_release      atomic64_fetch_add
 774
 775 #else /* atomic64_fetch_add_relaxed */
 776
 777 #ifndef atomic64_fetch_add_acquire
 778 #define atomic64_fetch_add_acquire(...)                                 \
 779         __atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__)
 780 #endif
 781
 782 #ifndef atomic64_fetch_add_release
 783 #define atomic64_fetch_add_release(...)                                 \
 784         __atomic_op_release(atomic64_fetch_add, __VA_ARGS__)
 785 #endif
 786
 787 #ifndef atomic64_fetch_add
 788 #define atomic64_fetch_add(...)                                         \
 789         __atomic_op_fence(atomic64_fetch_add, __VA_ARGS__)
 790 #endif
 791 #endif /* atomic64_fetch_add_relaxed */
 792
 793 /* atomic64_fetch_inc_relaxed */
 794 #ifndef atomic64_fetch_inc_relaxed
 795
 796 #ifndef atomic64_fetch_inc
 797 #define atomic64_fetch_inc(v)           atomic64_fetch_add(1, (v))
 798 #define atomic64_fetch_inc_relaxed(v)   atomic64_fetch_add_relaxed(1, (v))
 799 #define atomic64_fetch_inc_acquire(v)   atomic64_fetch_add_acquire(1, (v))
 800 #define atomic64_fetch_inc_release(v)   atomic64_fetch_add_release(1, (v))
 801 #else /* atomic64_fetch_inc */
 802 #define atomic64_fetch_inc_relaxed      atomic64_fetch_inc
 803 #define atomic64_fetch_inc_acquire      atomic64_fetch_inc
 804 #define atomic64_fetch_inc_release      atomic64_fetch_inc
 805 #endif /* atomic64_fetch_inc */
 806
 807 #else /* atomic64_fetch_inc_relaxed */
 808
 809 #ifndef atomic64_fetch_inc_acquire
 810 #define atomic64_fetch_inc_acquire(...)                                 \
 811         __atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__)
 812 #endif
 813
 814 #ifndef atomic64_fetch_inc_release
 815 #define atomic64_fetch_inc_release(...)                                 \
 816         __atomic_op_release(atomic64_fetch_inc, __VA_ARGS__)
 817 #endif
 818
 819 #ifndef atomic64_fetch_inc
 820 #define atomic64_fetch_inc(...)                                         \
 821         __atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__)
 822 #endif
 823 #endif /* atomic64_fetch_inc_relaxed */
 824
 825 /* atomic64_fetch_sub_relaxed */
 826 #ifndef atomic64_fetch_sub_relaxed
 827 #define atomic64_fetch_sub_relaxed      atomic64_fetch_sub
 828 #define atomic64_fetch_sub_acquire      atomic64_fetch_sub
 829 #define atomic64_fetch_sub_release      atomic64_fetch_sub
 830
 831 #else /* atomic64_fetch_sub_relaxed */
 832
 833 #ifndef atomic64_fetch_sub_acquire
 834 #define atomic64_fetch_sub_acquire(...)                                 \
 835         __atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__)
 836 #endif
 837
 838 #ifndef atomic64_fetch_sub_release
 839 #define atomic64_fetch_sub_release(...)                                 \
 840         __atomic_op_release(atomic64_fetch_sub, __VA_ARGS__)
 841 #endif
 842
 843 #ifndef atomic64_fetch_sub
 844 #define atomic64_fetch_sub(...)                                         \
 845         __atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__)
 846 #endif
 847 #endif /* atomic64_fetch_sub_relaxed */
 848
 849 /* atomic64_fetch_dec_relaxed */
 850 #ifndef atomic64_fetch_dec_relaxed
 851
 852 #ifndef atomic64_fetch_dec
 853 #define atomic64_fetch_dec(v)           atomic64_fetch_sub(1, (v))
 854 #define atomic64_fetch_dec_relaxed(v)   atomic64_fetch_sub_relaxed(1, (v))
 855 #define atomic64_fetch_dec_acquire(v)   atomic64_fetch_sub_acquire(1, (v))
 856 #define atomic64_fetch_dec_release(v)   atomic64_fetch_sub_release(1, (v))
 857 #else /* atomic64_fetch_dec */
 858 #define atomic64_fetch_dec_relaxed      atomic64_fetch_dec
 859 #define atomic64_fetch_dec_acquire      atomic64_fetch_dec
 860 #define atomic64_fetch_dec_release      atomic64_fetch_dec
 861 #endif /* atomic64_fetch_dec */
 862
 863 #else /* atomic64_fetch_dec_relaxed */
 864
 865 #ifndef atomic64_fetch_dec_acquire
 866 #define atomic64_fetch_dec_acquire(...)                                 \
 867         __atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__)
 868 #endif
 869
 870 #ifndef atomic64_fetch_dec_release
 871 #define atomic64_fetch_dec_release(...)                                 \
 872         __atomic_op_release(atomic64_fetch_dec, __VA_ARGS__)
 873 #endif
 874
 875 #ifndef atomic64_fetch_dec
 876 #define atomic64_fetch_dec(...)                                         \
 877         __atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__)
 878 #endif
 879 #endif /* atomic64_fetch_dec_relaxed */
 880
 881 /* atomic64_fetch_or_relaxed */
 882 #ifndef atomic64_fetch_or_relaxed
 883 #define atomic64_fetch_or_relaxed       atomic64_fetch_or
 884 #define atomic64_fetch_or_acquire       atomic64_fetch_or
 885 #define atomic64_fetch_or_release       atomic64_fetch_or
 886
 887 #else /* atomic64_fetch_or_relaxed */
 888
 889 #ifndef atomic64_fetch_or_acquire
 890 #define atomic64_fetch_or_acquire(...)                                  \
 891         __atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__)
 892 #endif
 893
 894 #ifndef atomic64_fetch_or_release
 895 #define atomic64_fetch_or_release(...)                                  \
 896         __atomic_op_release(atomic64_fetch_or, __VA_ARGS__)
 897 #endif
 898
 899 #ifndef atomic64_fetch_or
 900 #define atomic64_fetch_or(...)                                          \
 901         __atomic_op_fence(atomic64_fetch_or, __VA_ARGS__)
 902 #endif
 903 #endif /* atomic64_fetch_or_relaxed */
 904
 905 /* atomic64_fetch_and_relaxed */
 906 #ifndef atomic64_fetch_and_relaxed
 907 #define atomic64_fetch_and_relaxed      atomic64_fetch_and
 908 #define atomic64_fetch_and_acquire      atomic64_fetch_and
 909 #define atomic64_fetch_and_release      atomic64_fetch_and
 910
 911 #else /* atomic64_fetch_and_relaxed */
 912
 913 #ifndef atomic64_fetch_and_acquire
 914 #define atomic64_fetch_and_acquire(...)                                 \
 915         __atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__)
 916 #endif
 917
 918 #ifndef atomic64_fetch_and_release
 919 #define atomic64_fetch_and_release(...)                                 \
 920         __atomic_op_release(atomic64_fetch_and, __VA_ARGS__)
 921 #endif
 922
 923 #ifndef atomic64_fetch_and
 924 #define atomic64_fetch_and(...)                                         \
 925         __atomic_op_fence(atomic64_fetch_and, __VA_ARGS__)
 926 #endif
 927 #endif /* atomic64_fetch_and_relaxed */
 928
 929 #ifdef atomic64_andnot
 930 /* atomic64_fetch_andnot_relaxed */
 931 #ifndef atomic64_fetch_andnot_relaxed
 932 #define atomic64_fetch_andnot_relaxed   atomic64_fetch_andnot
 933 #define atomic64_fetch_andnot_acquire   atomic64_fetch_andnot
 934 #define atomic64_fetch_andnot_release   atomic64_fetch_andnot
 935
 936 #else /* atomic64_fetch_andnot_relaxed */
 937
 938 #ifndef atomic64_fetch_andnot_acquire
 939 #define atomic64_fetch_andnot_acquire(...)                                      \
 940         __atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__)
 941 #endif
 942
 943 #ifndef atomic64_fetch_andnot_release
 944 #define atomic64_fetch_andnot_release(...)                                      \
 945         __atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__)
 946 #endif
 947
 948 #ifndef atomic64_fetch_andnot
 949 #define atomic64_fetch_andnot(...)                                              \
 950         __atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__)
 951 #endif
 952 #endif /* atomic64_fetch_andnot_relaxed */
 953 #endif /* atomic64_andnot */
 954
 955 /* atomic64_fetch_xor_relaxed */
 956 #ifndef atomic64_fetch_xor_relaxed
 957 #define atomic64_fetch_xor_relaxed      atomic64_fetch_xor
 958 #define atomic64_fetch_xor_acquire      atomic64_fetch_xor
 959 #define atomic64_fetch_xor_release      atomic64_fetch_xor
 960
 961 #else /* atomic64_fetch_xor_relaxed */
 962
 963 #ifndef atomic64_fetch_xor_acquire
 964 #define atomic64_fetch_xor_acquire(...)                                 \
 965         __atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__)
 966 #endif
 967
 968 #ifndef atomic64_fetch_xor_release
 969 #define atomic64_fetch_xor_release(...)                                 \
 970         __atomic_op_release(atomic64_fetch_xor, __VA_ARGS__)
 971 #endif
 972
 973 #ifndef atomic64_fetch_xor
 974 #define atomic64_fetch_xor(...)                                         \
 975         __atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__)
 976 #endif
 977 #endif /* atomic64_fetch_xor_relaxed */
 978
 979
 980 /* atomic64_xchg_relaxed */
 981 #ifndef atomic64_xchg_relaxed
 982 #define  atomic64_xchg_relaxed          atomic64_xchg
 983 #define  atomic64_xchg_acquire          atomic64_xchg
 984 #define  atomic64_xchg_release          atomic64_xchg
 985
 986 #else /* atomic64_xchg_relaxed */
 987
 988 #ifndef atomic64_xchg_acquire
 989 #define  atomic64_xchg_acquire(...)                                     \
 990         __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
 991 #endif
 992
 993 #ifndef atomic64_xchg_release
 994 #define  atomic64_xchg_release(...)                                     \
 995         __atomic_op_release(atomic64_xchg, __VA_ARGS__)
 996 #endif
 997
 998 #ifndef atomic64_xchg
 999 #define  atomic64_xchg(...)                                             \
1000         __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
1001 #endif
1002 #endif /* atomic64_xchg_relaxed */
1003
1004 /* atomic64_cmpxchg_relaxed */
1005 #ifndef atomic64_cmpxchg_relaxed
1006 #define  atomic64_cmpxchg_relaxed       atomic64_cmpxchg
1007 #define  atomic64_cmpxchg_acquire       atomic64_cmpxchg
1008 #define  atomic64_cmpxchg_release       atomic64_cmpxchg
1009
1010 #else /* atomic64_cmpxchg_relaxed */
1011
1012 #ifndef atomic64_cmpxchg_acquire
1013 #define  atomic64_cmpxchg_acquire(...)                                  \
1014         __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
1015 #endif
1016
1017 #ifndef atomic64_cmpxchg_release
1018 #define  atomic64_cmpxchg_release(...)                                  \
1019         __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
1020 #endif
1021
1022 #ifndef atomic64_cmpxchg
1023 #define  atomic64_cmpxchg(...)                                          \
1024         __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
1025 #endif
1026 #endif /* atomic64_cmpxchg_relaxed */
1027
1028 #ifndef atomic64_try_cmpxchg
1029
1030 #define __atomic64_try_cmpxchg(type, _p, _po, _n)                       \
1031 ({                                                                      \
1032         typeof(_po) __po = (_po);                                       \
1033         typeof(*(_po)) __r, __o = *__po;                                \
1034         __r = atomic64_cmpxchg##type((_p), __o, (_n));                  \
1035         if (unlikely(__r != __o))                                       \
1036                 *__po = __r;                                            \
1037         likely(__r == __o);                                             \
1038 })
1039
1040 #define atomic64_try_cmpxchg(_p, _po, _n)               __atomic64_try_cmpxchg(, _p, _po, _n)
1041 #define atomic64_try_cmpxchg_relaxed(_p, _po, _n)       __atomic64_try_cmpxchg(_relaxed, _p, _po, _n)
1042 #define atomic64_try_cmpxchg_acquire(_p, _po, _n)       __atomic64_try_cmpxchg(_acquire, _p, _po, _n)
1043 #define atomic64_try_cmpxchg_release(_p, _po, _n)       __atomic64_try_cmpxchg(_release, _p, _po, _n)
1044
1045 #else /* atomic64_try_cmpxchg */
1046 #define atomic64_try_cmpxchg_relaxed    atomic64_try_cmpxchg
1047 #define atomic64_try_cmpxchg_acquire    atomic64_try_cmpxchg
1048 #define atomic64_try_cmpxchg_release    atomic64_try_cmpxchg
1049 #endif /* atomic64_try_cmpxchg */
1050
1051 #ifndef atomic64_andnot
1052 static inline void atomic64_andnot(long long i, atomic64_t *v)
1053 {
1054         atomic64_and(~i, v);
1055 }
1056
1057 static inline long long atomic64_fetch_andnot(long long i, atomic64_t *v)
1058 {
1059         return atomic64_fetch_and(~i, v);
1060 }
1061
1062 static inline long long atomic64_fetch_andnot_relaxed(long long i, atomic64_t *v)
1063 {
1064         return atomic64_fetch_and_relaxed(~i, v);
1065 }
1066
1067 static inline long long atomic64_fetch_andnot_acquire(long long i, atomic64_t *v)
1068 {
1069         return atomic64_fetch_and_acquire(~i, v);
1070 }
1071
1072 static inline long long atomic64_fetch_andnot_release(long long i, atomic64_t *v)
1073 {
1074         return atomic64_fetch_and_release(~i, v);
1075 }
1076 #endif
1077
1078 #define atomic64_cond_read_acquire(v, c)        smp_cond_load_acquire(&(v)->counter, (c))
1079
1080 #include <asm-generic/atomic-long.h>
1081
1082 #endif /* _LINUX_ATOMIC_H */