From 23f80d2b3bf7898579c841786c49842789f32ff5 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 17 Dec 2018 08:44:56 -0500 Subject: [PATCH] bcachefs: Factor out acc_u64s() Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_gc.c | 107 +++++++++++++++++------------------- fs/bcachefs/buckets.c | 122 +++++------------------------------------ fs/bcachefs/buckets.h | 2 - fs/bcachefs/util.h | 17 ++++++ 4 files changed, 79 insertions(+), 169 deletions(-) diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c index c353fbbed975..466469a0d852 100644 --- a/fs/bcachefs/btree_gc.c +++ b/fs/bcachefs/btree_gc.c @@ -482,29 +482,35 @@ static void bch2_gc_free(struct bch_fs *c) c->usage[1] = NULL; } -static void fs_usage_reset(struct bch_fs_usage *fs_usage) +/* + * Accumulate percpu counters onto one cpu's copy - only valid when access + * against any percpu counter is guarded against + */ +static u64 *acc_percpu_u64s(u64 __percpu *p, unsigned nr) { - unsigned offset = offsetof(typeof(*fs_usage), s.gc_start); + u64 *ret; + int cpu; - memset((void *) fs_usage + offset, 0, - sizeof(*fs_usage) - offset); -} + preempt_disable(); + ret = this_cpu_ptr(p); + preempt_enable(); -static void fs_usage_cpy(struct bch_fs_usage *dst, - struct bch_fs_usage *src) -{ - unsigned offset = offsetof(typeof(*dst), s.gc_start); + for_each_possible_cpu(cpu) { + u64 *i = per_cpu_ptr(p, cpu); - memcpy((void *) dst + offset, - (void *) src + offset, - sizeof(*dst) - offset); + if (i != ret) { + acc_u64s(ret, i, nr); + memset(i, 0, nr * sizeof(u64)); + } + } + + return ret; } static void bch2_gc_done_nocheck(struct bch_fs *c) { struct bch_dev *ca; unsigned i; - int cpu; { struct genradix_iter dst_iter = genradix_iter_init(&c->stripes[0], 0); @@ -534,42 +540,39 @@ static void bch2_gc_done_nocheck(struct bch_fs *c) }; for_each_member_device(ca, c, i) { - struct bch_dev_usage *p; - - for_each_possible_cpu(cpu) { - p = per_cpu_ptr(ca->usage[0], cpu); - memset(p, 0, sizeof(*p)); - } + unsigned nr = sizeof(struct bch_dev_usage) / sizeof(u64); + struct bch_dev_usage *dst = (void *) + acc_percpu_u64s((void *) ca->usage[0], nr); + struct bch_dev_usage *src = (void *) + acc_percpu_u64s((void *) ca->usage[1], nr); - preempt_disable(); - *this_cpu_ptr(ca->usage[0]) = __bch2_dev_usage_read(ca, 1); - preempt_enable(); + *dst = *src; } { - struct bch_fs_usage src = __bch2_fs_usage_read(c, 1); - - for_each_possible_cpu(cpu) - fs_usage_reset(per_cpu_ptr(c->usage[0], cpu)); - - preempt_disable(); - fs_usage_cpy(this_cpu_ptr(c->usage[0]), &src); - preempt_enable(); + unsigned nr = sizeof(struct bch_fs_usage) / sizeof(u64); + struct bch_fs_usage *dst = (void *) + acc_percpu_u64s((void *) c->usage[0], nr); + struct bch_fs_usage *src = (void *) + acc_percpu_u64s((void *) c->usage[1], nr); + unsigned offset = offsetof(typeof(*dst), s.gc_start); + + memcpy((void *) dst + offset, + (void *) src + offset, + sizeof(*dst) - offset); } - } static void bch2_gc_done(struct bch_fs *c, bool initial) { struct bch_dev *ca; unsigned i; - int cpu; #define copy_field(_f, _msg, ...) \ - if (dst._f != src._f) { \ - bch_err(c, _msg ": got %llu, should be %llu, fixing"\ - , ##__VA_ARGS__, dst._f, src._f); \ - dst._f = src._f; \ + if (dst->_f != src->_f) { \ + bch_err(c, _msg ": got %llu, should be %llu, fixing" \ + , ##__VA_ARGS__, dst->_f, src->_f); \ + dst->_f = src->_f; \ } #define copy_stripe_field(_f, _msg, ...) \ if (dst->_f != src->_f) { \ @@ -650,9 +653,11 @@ static void bch2_gc_done(struct bch_fs *c, bool initial) }; for_each_member_device(ca, c, i) { - struct bch_dev_usage dst = __bch2_dev_usage_read(ca, 0); - struct bch_dev_usage src = __bch2_dev_usage_read(ca, 1); - struct bch_dev_usage *p; + unsigned nr = sizeof(struct bch_dev_usage) / sizeof(u64); + struct bch_dev_usage *dst = (void *) + acc_percpu_u64s((void *) ca->usage[0], nr); + struct bch_dev_usage *src = (void *) + acc_percpu_u64s((void *) ca->usage[1], nr); unsigned b; for (b = 0; b < BCH_DATA_NR; b++) @@ -666,21 +671,14 @@ static void bch2_gc_done(struct bch_fs *c, bool initial) "sectors[%s]", bch2_data_types[b]); copy_dev_field(sectors_fragmented, "sectors_fragmented"); - - for_each_possible_cpu(cpu) { - p = per_cpu_ptr(ca->usage[0], cpu); - memset(p, 0, sizeof(*p)); - } - - preempt_disable(); - p = this_cpu_ptr(ca->usage[0]); - *p = dst; - preempt_enable(); } { - struct bch_fs_usage dst = __bch2_fs_usage_read(c, 0); - struct bch_fs_usage src = __bch2_fs_usage_read(c, 1); + unsigned nr = sizeof(struct bch_fs_usage) / sizeof(u64); + struct bch_fs_usage *dst = (void *) + acc_percpu_u64s((void *) c->usage[0], nr); + struct bch_fs_usage *src = (void *) + acc_percpu_u64s((void *) c->usage[1], nr); unsigned r, b; copy_fs_field(s.hidden, "hidden"); @@ -703,13 +701,6 @@ static void bch2_gc_done(struct bch_fs *c, bool initial) for (b = 0; b < BCH_DATA_NR; b++) copy_fs_field(buckets[b], "buckets[%s]", bch2_data_types[b]); - - for_each_possible_cpu(cpu) - fs_usage_reset(per_cpu_ptr(c->usage[0], cpu)); - - preempt_disable(); - fs_usage_cpy(this_cpu_ptr(c->usage[0]), &dst); - preempt_enable(); } out: percpu_up_write(&c->mark_lock); diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index 16aafe8502a0..6501dcf12d59 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -78,77 +78,6 @@ static inline u64 __bch2_fs_sectors_used(struct bch_fs *, struct bch_fs_usage); -#ifdef DEBUG_BUCKETS - -#define lg_local_lock lg_global_lock -#define lg_local_unlock lg_global_unlock - -static void bch2_fs_stats_verify(struct bch_fs *c) -{ - struct bch_fs_usage stats =_bch2_fs_usage_read(c); - unsigned i, j; - - for (i = 0; i < ARRAY_SIZE(stats.replicas); i++) { - for (j = 0; j < ARRAY_SIZE(stats.replicas[i].data); j++) - if ((s64) stats.replicas[i].data[j] < 0) - panic("replicas %u %s sectors underflow: %lli\n", - i + 1, bch_data_types[j], - stats.replicas[i].data[j]); - - if ((s64) stats.replicas[i].persistent_reserved < 0) - panic("replicas %u reserved underflow: %lli\n", - i + 1, stats.replicas[i].persistent_reserved); - } - - for (j = 0; j < ARRAY_SIZE(stats.buckets); j++) - if ((s64) stats.replicas[i].data_buckets[j] < 0) - panic("%s buckets underflow: %lli\n", - bch_data_types[j], - stats.buckets[j]); - - if ((s64) stats.s.online_reserved < 0) - panic("sectors_online_reserved underflow: %lli\n", - stats.s.online_reserved); -} - -static void bch2_dev_stats_verify(struct bch_dev *ca) -{ - struct bch_dev_usage stats = - __bch2_dev_usage_read(ca); - u64 n = ca->mi.nbuckets - ca->mi.first_bucket; - unsigned i; - - for (i = 0; i < ARRAY_SIZE(stats.buckets); i++) - BUG_ON(stats.buckets[i] > n); - BUG_ON(stats.buckets_alloc > n); - BUG_ON(stats.buckets_unavailable > n); -} - -static void bch2_disk_reservations_verify(struct bch_fs *c, int flags) -{ - if (!(flags & BCH_DISK_RESERVATION_NOFAIL)) { - u64 used = __bch2_fs_sectors_used(c); - u64 cached = 0; - u64 avail = atomic64_read(&c->sectors_available); - int cpu; - - for_each_possible_cpu(cpu) - cached += per_cpu_ptr(c->usage_percpu, cpu)->available_cache; - - if (used + avail + cached > c->capacity) - panic("used %llu avail %llu cached %llu capacity %llu\n", - used, avail, cached, c->capacity); - } -} - -#else - -static void bch2_fs_stats_verify(struct bch_fs *c) {} -static void bch2_dev_stats_verify(struct bch_dev *ca) {} -static void bch2_disk_reservations_verify(struct bch_fs *c, int flags) {} - -#endif - /* * Clear journal_seq_valid for buckets for which it's not needed, to prevent * wraparound: @@ -186,43 +115,23 @@ void bch2_bucket_seq_cleanup(struct bch_fs *c) } } -#define bch2_usage_add(_acc, _stats) \ -do { \ - typeof(_acc) _a = (_acc), _s = (_stats); \ - unsigned i; \ - \ - for (i = 0; i < sizeof(*_a) / sizeof(u64); i++) \ - ((u64 *) (_a))[i] += ((u64 *) (_s))[i]; \ -} while (0) - #define bch2_usage_read_raw(_stats) \ ({ \ typeof(*this_cpu_ptr(_stats)) _acc; \ - int cpu; \ \ memset(&_acc, 0, sizeof(_acc)); \ - \ - for_each_possible_cpu(cpu) \ - bch2_usage_add(&_acc, per_cpu_ptr((_stats), cpu)); \ + acc_u64s_percpu((u64 *) &_acc, \ + (u64 __percpu *) _stats, \ + sizeof(_acc) / sizeof(u64)); \ \ _acc; \ }) -struct bch_dev_usage __bch2_dev_usage_read(struct bch_dev *ca, bool gc) -{ - return bch2_usage_read_raw(ca->usage[gc]); -} - struct bch_dev_usage bch2_dev_usage_read(struct bch_fs *c, struct bch_dev *ca) { return bch2_usage_read_raw(ca->usage[0]); } -struct bch_fs_usage __bch2_fs_usage_read(struct bch_fs *c, bool gc) -{ - return bch2_usage_read_raw(c->usage[gc]); -} - struct bch_fs_usage bch2_fs_usage_read(struct bch_fs *c) { return bch2_usage_read_raw(c->usage[0]); @@ -326,12 +235,16 @@ void bch2_fs_usage_apply(struct bch_fs *c, } preempt_disable(); - bch2_usage_add(this_cpu_ptr(c->usage[0]), fs_usage); - - if (gc_visited(c, gc_pos)) - bch2_usage_add(this_cpu_ptr(c->usage[1]), fs_usage); - - bch2_fs_stats_verify(c); + acc_u64s((u64 *) this_cpu_ptr(c->usage[0]), + (u64 *) fs_usage, + sizeof(*fs_usage) / sizeof(u64)); + + if (gc_visited(c, gc_pos)) { + BUG_ON(!c->usage[1]); + acc_u64s((u64 *) this_cpu_ptr(c->usage[1]), + (u64 *) fs_usage, + sizeof(*fs_usage) / sizeof(u64)); + } preempt_enable(); memset(fs_usage, 0, sizeof(*fs_usage)); @@ -392,8 +305,6 @@ static void bch2_dev_usage_update(struct bch_fs *c, struct bch_dev *ca, if (!is_available_bucket(old) && is_available_bucket(new)) bch2_wake_allocator(ca); - - bch2_dev_stats_verify(ca); } void bch2_dev_usage_from_buckets(struct bch_fs *c, struct bch_dev *ca) @@ -1011,8 +922,6 @@ void __bch2_disk_reservation_put(struct bch_fs *c, struct disk_reservation *res) { percpu_down_read(&c->mark_lock); this_cpu_sub(c->usage[0]->s.online_reserved, res->sectors); - - bch2_fs_stats_verify(c); percpu_up_read(&c->mark_lock); res->sectors = 0; @@ -1055,8 +964,6 @@ out: this_cpu_add(c->usage[0]->s.online_reserved, sectors); res->sectors += sectors; - bch2_disk_reservations_verify(c, flags); - bch2_fs_stats_verify(c); preempt_enable(); percpu_up_read(&c->mark_lock); return 0; @@ -1089,14 +996,11 @@ recalculate: this_cpu_add(c->usage[0]->s.online_reserved, sectors); res->sectors += sectors; ret = 0; - - bch2_disk_reservations_verify(c, flags); } else { atomic64_set(&c->sectors_available, sectors_available); ret = -ENOSPC; } - bch2_fs_stats_verify(c); percpu_up_write(&c->mark_lock); if (!(flags & BCH_DISK_RESERVATION_GC_LOCK_HELD)) diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h index 3db0e3b8a180..88e083325232 100644 --- a/fs/bcachefs/buckets.h +++ b/fs/bcachefs/buckets.h @@ -180,7 +180,6 @@ static inline bool bucket_needs_journal_commit(struct bucket_mark m, /* Device usage: */ -struct bch_dev_usage __bch2_dev_usage_read(struct bch_dev *, bool); struct bch_dev_usage bch2_dev_usage_read(struct bch_fs *, struct bch_dev *); static inline u64 __dev_buckets_available(struct bch_dev *ca, @@ -219,7 +218,6 @@ static inline u64 dev_buckets_free(struct bch_fs *c, struct bch_dev *ca) /* Filesystem usage: */ -struct bch_fs_usage __bch2_fs_usage_read(struct bch_fs *, bool); struct bch_fs_usage bch2_fs_usage_read(struct bch_fs *); u64 bch2_fs_sectors_used(struct bch_fs *, struct bch_fs_usage); diff --git a/fs/bcachefs/util.h b/fs/bcachefs/util.h index 7d1e6cc6afda..094c35f40209 100644 --- a/fs/bcachefs/util.h +++ b/fs/bcachefs/util.h @@ -700,4 +700,21 @@ do { \ } \ } while (0) +static inline void acc_u64s(u64 *acc, const u64 *src, unsigned nr) +{ + unsigned i; + + for (i = 0; i < nr; i++) + acc[i] += src[i]; +} + +static inline void acc_u64s_percpu(u64 *acc, const u64 __percpu *src, + unsigned nr) +{ + int cpu; + + for_each_possible_cpu(cpu) + acc_u64s(acc, per_cpu_ptr(src, cpu), nr); +} + #endif /* _BCACHEFS_UTIL_H */ -- 2.39.5