include/qemu/stats64.h

   1 /*
   2  * Atomic operations on 64-bit quantities.
   3  *
   4  * Copyright (C) 2017 Red Hat, Inc.
   5  *
   6  * Author: Paolo Bonzini <pbonzini@redhat.com>
   7  *
   8  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   9  * See the COPYING file in the top-level directory.
  10  */
  11
  12 #ifndef QEMU_STATS64_H
  13 #define QEMU_STATS64_H
  14
  15 #include "qemu/atomic.h"
  16
  17 /* This provides atomic operations on 64-bit type, using a reader-writer
  18  * spinlock on architectures that do not have 64-bit accesses.  Even on
  19  * those architectures, it tries hard not to take the lock.
  20  */
  21
  22 typedef struct Stat64 {
  23 #ifdef CONFIG_ATOMIC64
  24     aligned_uint64_t value;
  25 #else
  26     uint32_t low, high;
  27     uint32_t lock;
  28 #endif
  29 } Stat64;
  30
  31 #ifdef CONFIG_ATOMIC64
  32 static inline void stat64_init(Stat64 *s, uint64_t value)
  33 {
  34     /* This is not guaranteed to be atomic! */
  35     *s = (Stat64) { value };
  36 }
  37
  38 static inline uint64_t stat64_get(const Stat64 *s)
  39 {
  40     return qatomic_read__nocheck(&s->value);
  41 }
  42
  43 static inline void stat64_set(Stat64 *s, uint64_t value)
  44 {
  45     qatomic_set__nocheck(&s->value, value);
  46 }
  47
  48 static inline void stat64_add(Stat64 *s, uint64_t value)
  49 {
  50     qatomic_add(&s->value, value);
  51 }
  52
  53 static inline void stat64_min(Stat64 *s, uint64_t value)
  54 {
  55     uint64_t orig = qatomic_read__nocheck(&s->value);
  56     while (orig > value) {
  57         orig = qatomic_cmpxchg__nocheck(&s->value, orig, value);
  58     }
  59 }
  60
  61 static inline void stat64_max(Stat64 *s, uint64_t value)
  62 {
  63     uint64_t orig = qatomic_read__nocheck(&s->value);
  64     while (orig < value) {
  65         orig = qatomic_cmpxchg__nocheck(&s->value, orig, value);
  66     }
  67 }
  68 #else
  69 uint64_t stat64_get(const Stat64 *s);
  70 void stat64_set(Stat64 *s, uint64_t value);
  71 bool stat64_min_slow(Stat64 *s, uint64_t value);
  72 bool stat64_max_slow(Stat64 *s, uint64_t value);
  73 bool stat64_add32_carry(Stat64 *s, uint32_t low, uint32_t high);
  74
  75 static inline void stat64_init(Stat64 *s, uint64_t value)
  76 {
  77     /* This is not guaranteed to be atomic! */
  78     *s = (Stat64) { .low = value, .high = value >> 32, .lock = 0 };
  79 }
  80
  81 static inline void stat64_add(Stat64 *s, uint64_t value)
  82 {
  83     uint32_t low, high;
  84     high = value >> 32;
  85     low = (uint32_t) value;
  86     if (!low) {
  87         if (high) {
  88             qatomic_add(&s->high, high);
  89         }
  90         return;
  91     }
  92
  93     for (;;) {
  94         uint32_t orig = s->low;
  95         uint32_t result = orig + low;
  96         uint32_t old;
  97
  98         if (result < low || high) {
  99             /* If the high part is affected, take the lock.  */
 100             if (stat64_add32_carry(s, low, high)) {
 101                 return;
 102             }
 103             continue;
 104         }
 105
 106         /* No carry, try with a 32-bit cmpxchg.  The result is independent of
 107          * the high 32 bits, so it can race just fine with stat64_add32_carry
 108          * and even stat64_get!
 109          */
 110         old = qatomic_cmpxchg(&s->low, orig, result);
 111         if (orig == old) {
 112             return;
 113         }
 114     }
 115 }
 116
 117 static inline void stat64_min(Stat64 *s, uint64_t value)
 118 {
 119     uint32_t low, high;
 120     uint32_t orig_low, orig_high;
 121
 122     high = value >> 32;
 123     low = (uint32_t) value;
 124     do {
 125         orig_high = qatomic_read(&s->high);
 126         if (orig_high < high) {
 127             return;
 128         }
 129
 130         if (orig_high == high) {
 131             /* High 32 bits are equal.  Read low after high, otherwise we
 132              * can get a false positive (e.g. 0x1235,0x0000 changes to
 133              * 0x1234,0x8000 and we read it as 0x1234,0x0000). Pairs with
 134              * the write barrier in stat64_min_slow.
 135              */
 136             smp_rmb();
 137             orig_low = qatomic_read(&s->low);
 138             if (orig_low <= low) {
 139                 return;
 140             }
 141
 142             /* See if we were lucky and a writer raced against us.  The
 143              * barrier is theoretically unnecessary, but if we remove it
 144              * we may miss being lucky.
 145              */
 146             smp_rmb();
 147             orig_high = qatomic_read(&s->high);
 148             if (orig_high < high) {
 149                 return;
 150             }
 151         }
 152
 153         /* If the value changes in any way, we have to take the lock.  */
 154     } while (!stat64_min_slow(s, value));
 155 }
 156
 157 static inline void stat64_max(Stat64 *s, uint64_t value)
 158 {
 159     uint32_t low, high;
 160     uint32_t orig_low, orig_high;
 161
 162     high = value >> 32;
 163     low = (uint32_t) value;
 164     do {
 165         orig_high = qatomic_read(&s->high);
 166         if (orig_high > high) {
 167             return;
 168         }
 169
 170         if (orig_high == high) {
 171             /* High 32 bits are equal.  Read low after high, otherwise we
 172              * can get a false positive (e.g. 0x1234,0x8000 changes to
 173              * 0x1235,0x0000 and we read it as 0x1235,0x8000). Pairs with
 174              * the write barrier in stat64_max_slow.
 175              */
 176             smp_rmb();
 177             orig_low = qatomic_read(&s->low);
 178             if (orig_low >= low) {
 179                 return;
 180             }
 181
 182             /* See if we were lucky and a writer raced against us.  The
 183              * barrier is theoretically unnecessary, but if we remove it
 184              * we may miss being lucky.
 185              */
 186             smp_rmb();
 187             orig_high = qatomic_read(&s->high);
 188             if (orig_high > high) {
 189                 return;
 190             }
 191         }
 192
 193         /* If the value changes in any way, we have to take the lock.  */
 194     } while (!stat64_max_slow(s, value));
 195 }
 196
 197 #endif
 198
 199 #endif