c->usage[1] = NULL;
}
-static void fs_usage_reset(struct bch_fs_usage *fs_usage)
+/*
+ * Accumulate percpu counters onto one cpu's copy - only valid when access
+ * against any percpu counter is guarded against
+ */
+static u64 *acc_percpu_u64s(u64 __percpu *p, unsigned nr)
{
- unsigned offset = offsetof(typeof(*fs_usage), s.gc_start);
+ u64 *ret;
+ int cpu;
- memset((void *) fs_usage + offset, 0,
- sizeof(*fs_usage) - offset);
-}
+ preempt_disable();
+ ret = this_cpu_ptr(p);
+ preempt_enable();
-static void fs_usage_cpy(struct bch_fs_usage *dst,
- struct bch_fs_usage *src)
-{
- unsigned offset = offsetof(typeof(*dst), s.gc_start);
+ for_each_possible_cpu(cpu) {
+ u64 *i = per_cpu_ptr(p, cpu);
- memcpy((void *) dst + offset,
- (void *) src + offset,
- sizeof(*dst) - offset);
+ if (i != ret) {
+ acc_u64s(ret, i, nr);
+ memset(i, 0, nr * sizeof(u64));
+ }
+ }
+
+ return ret;
}
static void bch2_gc_done_nocheck(struct bch_fs *c)
{
struct bch_dev *ca;
unsigned i;
- int cpu;
{
struct genradix_iter dst_iter = genradix_iter_init(&c->stripes[0], 0);
};
for_each_member_device(ca, c, i) {
- struct bch_dev_usage *p;
-
- for_each_possible_cpu(cpu) {
- p = per_cpu_ptr(ca->usage[0], cpu);
- memset(p, 0, sizeof(*p));
- }
+ unsigned nr = sizeof(struct bch_dev_usage) / sizeof(u64);
+ struct bch_dev_usage *dst = (void *)
+ acc_percpu_u64s((void *) ca->usage[0], nr);
+ struct bch_dev_usage *src = (void *)
+ acc_percpu_u64s((void *) ca->usage[1], nr);
- preempt_disable();
- *this_cpu_ptr(ca->usage[0]) = __bch2_dev_usage_read(ca, 1);
- preempt_enable();
+ *dst = *src;
}
{
- struct bch_fs_usage src = __bch2_fs_usage_read(c, 1);
-
- for_each_possible_cpu(cpu)
- fs_usage_reset(per_cpu_ptr(c->usage[0], cpu));
-
- preempt_disable();
- fs_usage_cpy(this_cpu_ptr(c->usage[0]), &src);
- preempt_enable();
+ unsigned nr = sizeof(struct bch_fs_usage) / sizeof(u64);
+ struct bch_fs_usage *dst = (void *)
+ acc_percpu_u64s((void *) c->usage[0], nr);
+ struct bch_fs_usage *src = (void *)
+ acc_percpu_u64s((void *) c->usage[1], nr);
+ unsigned offset = offsetof(typeof(*dst), s.gc_start);
+
+ memcpy((void *) dst + offset,
+ (void *) src + offset,
+ sizeof(*dst) - offset);
}
-
}
static void bch2_gc_done(struct bch_fs *c, bool initial)
{
struct bch_dev *ca;
unsigned i;
- int cpu;
#define copy_field(_f, _msg, ...) \
- if (dst._f != src._f) { \
- bch_err(c, _msg ": got %llu, should be %llu, fixing"\
- , ##__VA_ARGS__, dst._f, src._f); \
- dst._f = src._f; \
+ if (dst->_f != src->_f) { \
+ bch_err(c, _msg ": got %llu, should be %llu, fixing" \
+ , ##__VA_ARGS__, dst->_f, src->_f); \
+ dst->_f = src->_f; \
}
#define copy_stripe_field(_f, _msg, ...) \
if (dst->_f != src->_f) { \
};
for_each_member_device(ca, c, i) {
- struct bch_dev_usage dst = __bch2_dev_usage_read(ca, 0);
- struct bch_dev_usage src = __bch2_dev_usage_read(ca, 1);
- struct bch_dev_usage *p;
+ unsigned nr = sizeof(struct bch_dev_usage) / sizeof(u64);
+ struct bch_dev_usage *dst = (void *)
+ acc_percpu_u64s((void *) ca->usage[0], nr);
+ struct bch_dev_usage *src = (void *)
+ acc_percpu_u64s((void *) ca->usage[1], nr);
unsigned b;
for (b = 0; b < BCH_DATA_NR; b++)
"sectors[%s]", bch2_data_types[b]);
copy_dev_field(sectors_fragmented,
"sectors_fragmented");
-
- for_each_possible_cpu(cpu) {
- p = per_cpu_ptr(ca->usage[0], cpu);
- memset(p, 0, sizeof(*p));
- }
-
- preempt_disable();
- p = this_cpu_ptr(ca->usage[0]);
- *p = dst;
- preempt_enable();
}
{
- struct bch_fs_usage dst = __bch2_fs_usage_read(c, 0);
- struct bch_fs_usage src = __bch2_fs_usage_read(c, 1);
+ unsigned nr = sizeof(struct bch_fs_usage) / sizeof(u64);
+ struct bch_fs_usage *dst = (void *)
+ acc_percpu_u64s((void *) c->usage[0], nr);
+ struct bch_fs_usage *src = (void *)
+ acc_percpu_u64s((void *) c->usage[1], nr);
unsigned r, b;
copy_fs_field(s.hidden, "hidden");
for (b = 0; b < BCH_DATA_NR; b++)
copy_fs_field(buckets[b],
"buckets[%s]", bch2_data_types[b]);
-
- for_each_possible_cpu(cpu)
- fs_usage_reset(per_cpu_ptr(c->usage[0], cpu));
-
- preempt_disable();
- fs_usage_cpy(this_cpu_ptr(c->usage[0]), &dst);
- preempt_enable();
}
out:
percpu_up_write(&c->mark_lock);
static inline u64 __bch2_fs_sectors_used(struct bch_fs *, struct bch_fs_usage);
-#ifdef DEBUG_BUCKETS
-
-#define lg_local_lock lg_global_lock
-#define lg_local_unlock lg_global_unlock
-
-static void bch2_fs_stats_verify(struct bch_fs *c)
-{
- struct bch_fs_usage stats =_bch2_fs_usage_read(c);
- unsigned i, j;
-
- for (i = 0; i < ARRAY_SIZE(stats.replicas); i++) {
- for (j = 0; j < ARRAY_SIZE(stats.replicas[i].data); j++)
- if ((s64) stats.replicas[i].data[j] < 0)
- panic("replicas %u %s sectors underflow: %lli\n",
- i + 1, bch_data_types[j],
- stats.replicas[i].data[j]);
-
- if ((s64) stats.replicas[i].persistent_reserved < 0)
- panic("replicas %u reserved underflow: %lli\n",
- i + 1, stats.replicas[i].persistent_reserved);
- }
-
- for (j = 0; j < ARRAY_SIZE(stats.buckets); j++)
- if ((s64) stats.replicas[i].data_buckets[j] < 0)
- panic("%s buckets underflow: %lli\n",
- bch_data_types[j],
- stats.buckets[j]);
-
- if ((s64) stats.s.online_reserved < 0)
- panic("sectors_online_reserved underflow: %lli\n",
- stats.s.online_reserved);
-}
-
-static void bch2_dev_stats_verify(struct bch_dev *ca)
-{
- struct bch_dev_usage stats =
- __bch2_dev_usage_read(ca);
- u64 n = ca->mi.nbuckets - ca->mi.first_bucket;
- unsigned i;
-
- for (i = 0; i < ARRAY_SIZE(stats.buckets); i++)
- BUG_ON(stats.buckets[i] > n);
- BUG_ON(stats.buckets_alloc > n);
- BUG_ON(stats.buckets_unavailable > n);
-}
-
-static void bch2_disk_reservations_verify(struct bch_fs *c, int flags)
-{
- if (!(flags & BCH_DISK_RESERVATION_NOFAIL)) {
- u64 used = __bch2_fs_sectors_used(c);
- u64 cached = 0;
- u64 avail = atomic64_read(&c->sectors_available);
- int cpu;
-
- for_each_possible_cpu(cpu)
- cached += per_cpu_ptr(c->usage_percpu, cpu)->available_cache;
-
- if (used + avail + cached > c->capacity)
- panic("used %llu avail %llu cached %llu capacity %llu\n",
- used, avail, cached, c->capacity);
- }
-}
-
-#else
-
-static void bch2_fs_stats_verify(struct bch_fs *c) {}
-static void bch2_dev_stats_verify(struct bch_dev *ca) {}
-static void bch2_disk_reservations_verify(struct bch_fs *c, int flags) {}
-
-#endif
-
/*
* Clear journal_seq_valid for buckets for which it's not needed, to prevent
* wraparound:
}
}
-#define bch2_usage_add(_acc, _stats) \
-do { \
- typeof(_acc) _a = (_acc), _s = (_stats); \
- unsigned i; \
- \
- for (i = 0; i < sizeof(*_a) / sizeof(u64); i++) \
- ((u64 *) (_a))[i] += ((u64 *) (_s))[i]; \
-} while (0)
-
#define bch2_usage_read_raw(_stats) \
({ \
typeof(*this_cpu_ptr(_stats)) _acc; \
- int cpu; \
\
memset(&_acc, 0, sizeof(_acc)); \
- \
- for_each_possible_cpu(cpu) \
- bch2_usage_add(&_acc, per_cpu_ptr((_stats), cpu)); \
+ acc_u64s_percpu((u64 *) &_acc, \
+ (u64 __percpu *) _stats, \
+ sizeof(_acc) / sizeof(u64)); \
\
_acc; \
})
-struct bch_dev_usage __bch2_dev_usage_read(struct bch_dev *ca, bool gc)
-{
- return bch2_usage_read_raw(ca->usage[gc]);
-}
-
struct bch_dev_usage bch2_dev_usage_read(struct bch_fs *c, struct bch_dev *ca)
{
return bch2_usage_read_raw(ca->usage[0]);
}
-struct bch_fs_usage __bch2_fs_usage_read(struct bch_fs *c, bool gc)
-{
- return bch2_usage_read_raw(c->usage[gc]);
-}
-
struct bch_fs_usage bch2_fs_usage_read(struct bch_fs *c)
{
return bch2_usage_read_raw(c->usage[0]);
}
preempt_disable();
- bch2_usage_add(this_cpu_ptr(c->usage[0]), fs_usage);
-
- if (gc_visited(c, gc_pos))
- bch2_usage_add(this_cpu_ptr(c->usage[1]), fs_usage);
-
- bch2_fs_stats_verify(c);
+ acc_u64s((u64 *) this_cpu_ptr(c->usage[0]),
+ (u64 *) fs_usage,
+ sizeof(*fs_usage) / sizeof(u64));
+
+ if (gc_visited(c, gc_pos)) {
+ BUG_ON(!c->usage[1]);
+ acc_u64s((u64 *) this_cpu_ptr(c->usage[1]),
+ (u64 *) fs_usage,
+ sizeof(*fs_usage) / sizeof(u64));
+ }
preempt_enable();
memset(fs_usage, 0, sizeof(*fs_usage));
if (!is_available_bucket(old) && is_available_bucket(new))
bch2_wake_allocator(ca);
-
- bch2_dev_stats_verify(ca);
}
void bch2_dev_usage_from_buckets(struct bch_fs *c, struct bch_dev *ca)
{
percpu_down_read(&c->mark_lock);
this_cpu_sub(c->usage[0]->s.online_reserved, res->sectors);
-
- bch2_fs_stats_verify(c);
percpu_up_read(&c->mark_lock);
res->sectors = 0;
this_cpu_add(c->usage[0]->s.online_reserved, sectors);
res->sectors += sectors;
- bch2_disk_reservations_verify(c, flags);
- bch2_fs_stats_verify(c);
preempt_enable();
percpu_up_read(&c->mark_lock);
return 0;
this_cpu_add(c->usage[0]->s.online_reserved, sectors);
res->sectors += sectors;
ret = 0;
-
- bch2_disk_reservations_verify(c, flags);
} else {
atomic64_set(&c->sectors_available, sectors_available);
ret = -ENOSPC;
}
- bch2_fs_stats_verify(c);
percpu_up_write(&c->mark_lock);
if (!(flags & BCH_DISK_RESERVATION_GC_LOCK_HELD))
/* Device usage: */
-struct bch_dev_usage __bch2_dev_usage_read(struct bch_dev *, bool);
struct bch_dev_usage bch2_dev_usage_read(struct bch_fs *, struct bch_dev *);
static inline u64 __dev_buckets_available(struct bch_dev *ca,
/* Filesystem usage: */
-struct bch_fs_usage __bch2_fs_usage_read(struct bch_fs *, bool);
struct bch_fs_usage bch2_fs_usage_read(struct bch_fs *);
u64 bch2_fs_sectors_used(struct bch_fs *, struct bch_fs_usage);
} \
} while (0)
+static inline void acc_u64s(u64 *acc, const u64 *src, unsigned nr)
+{
+ unsigned i;
+
+ for (i = 0; i < nr; i++)
+ acc[i] += src[i];
+}
+
+static inline void acc_u64s_percpu(u64 *acc, const u64 __percpu *src,
+ unsigned nr)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ acc_u64s(acc, per_cpu_ptr(src, cpu), nr);
+}
+
#endif /* _BCACHEFS_UTIL_H */
projects / mirror_ubuntu-kernels.git / commitdiff