]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
cafe5635 KO |
2 | /* |
3 | * Moving/copying garbage collector | |
4 | * | |
5 | * Copyright 2012 Google, Inc. | |
6 | */ | |
7 | ||
8 | #include "bcache.h" | |
9 | #include "btree.h" | |
10 | #include "debug.h" | |
11 | #include "request.h" | |
12 | ||
c37511b8 KO |
13 | #include <trace/events/bcache.h> |
14 | ||
cafe5635 | 15 | struct moving_io { |
220bb38c | 16 | struct closure cl; |
cafe5635 | 17 | struct keybuf_key *w; |
220bb38c | 18 | struct data_insert_op op; |
cafe5635 KO |
19 | struct bbio bio; |
20 | }; | |
21 | ||
22 | static bool moving_pred(struct keybuf *buf, struct bkey *k) | |
23 | { | |
24 | struct cache_set *c = container_of(buf, struct cache_set, | |
25 | moving_gc_keys); | |
6f10f7d1 | 26 | unsigned int i; |
cafe5635 | 27 | |
10d9dcf6 KO |
28 | for (i = 0; i < KEY_PTRS(k); i++) |
29 | if (ptr_available(c, k, i) && | |
30 | GC_MOVE(PTR_BUCKET(c, k, i))) | |
cafe5635 | 31 | return true; |
cafe5635 KO |
32 | |
33 | return false; | |
34 | } | |
35 | ||
36 | /* Moving GC - IO loop */ | |
37 | ||
38 | static void moving_io_destructor(struct closure *cl) | |
39 | { | |
220bb38c | 40 | struct moving_io *io = container_of(cl, struct moving_io, cl); |
1fae7cf0 | 41 | |
cafe5635 KO |
42 | kfree(io); |
43 | } | |
44 | ||
45 | static void write_moving_finish(struct closure *cl) | |
46 | { | |
220bb38c | 47 | struct moving_io *io = container_of(cl, struct moving_io, cl); |
cafe5635 | 48 | struct bio *bio = &io->bio.bio; |
cafe5635 | 49 | |
491221f8 | 50 | bio_free_pages(bio); |
cafe5635 | 51 | |
220bb38c | 52 | if (io->op.replace_collision) |
c37511b8 | 53 | trace_bcache_gc_copy_collision(&io->w->key); |
cafe5635 | 54 | |
220bb38c | 55 | bch_keybuf_del(&io->op.c->moving_gc_keys, io->w); |
cafe5635 | 56 | |
220bb38c | 57 | up(&io->op.c->moving_in_flight); |
cafe5635 KO |
58 | |
59 | closure_return_with_destructor(cl, moving_io_destructor); | |
60 | } | |
61 | ||
4246a0b6 | 62 | static void read_moving_endio(struct bio *bio) |
cafe5635 | 63 | { |
6d3d1a9c | 64 | struct bbio *b = container_of(bio, struct bbio, bio); |
cafe5635 | 65 | struct moving_io *io = container_of(bio->bi_private, |
220bb38c | 66 | struct moving_io, cl); |
cafe5635 | 67 | |
4e4cbee9 CH |
68 | if (bio->bi_status) |
69 | io->op.status = bio->bi_status; | |
6d3d1a9c KO |
70 | else if (!KEY_DIRTY(&b->key) && |
71 | ptr_stale(io->op.c, &b->key, 0)) { | |
4e4cbee9 | 72 | io->op.status = BLK_STS_IOERR; |
6d3d1a9c | 73 | } |
cafe5635 | 74 | |
4e4cbee9 | 75 | bch_bbio_endio(io->op.c, bio, bio->bi_status, "reading data to move"); |
cafe5635 KO |
76 | } |
77 | ||
78 | static void moving_init(struct moving_io *io) | |
79 | { | |
80 | struct bio *bio = &io->bio.bio; | |
81 | ||
3a83f467 ML |
82 | bio_init(bio, bio->bi_inline_vecs, |
83 | DIV_ROUND_UP(KEY_SIZE(&io->w->key), PAGE_SECTORS)); | |
cafe5635 KO |
84 | bio_get(bio); |
85 | bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0)); | |
86 | ||
4f024f37 | 87 | bio->bi_iter.bi_size = KEY_SIZE(&io->w->key) << 9; |
220bb38c | 88 | bio->bi_private = &io->cl; |
169ef1cf | 89 | bch_bio_map(bio, NULL); |
cafe5635 KO |
90 | } |
91 | ||
92 | static void write_moving(struct closure *cl) | |
93 | { | |
220bb38c KO |
94 | struct moving_io *io = container_of(cl, struct moving_io, cl); |
95 | struct data_insert_op *op = &io->op; | |
cafe5635 | 96 | |
4e4cbee9 | 97 | if (!op->status) { |
cafe5635 KO |
98 | moving_init(io); |
99 | ||
4f024f37 | 100 | io->bio.bio.bi_iter.bi_sector = KEY_START(&io->w->key); |
220bb38c KO |
101 | op->write_prio = 1; |
102 | op->bio = &io->bio.bio; | |
cafe5635 | 103 | |
220bb38c KO |
104 | op->writeback = KEY_DIRTY(&io->w->key); |
105 | op->csum = KEY_CSUM(&io->w->key); | |
cafe5635 | 106 | |
220bb38c KO |
107 | bkey_copy(&op->replace_key, &io->w->key); |
108 | op->replace = true; | |
cafe5635 | 109 | |
220bb38c | 110 | closure_call(&op->cl, bch_data_insert, NULL, cl); |
cafe5635 KO |
111 | } |
112 | ||
da415a09 | 113 | continue_at(cl, write_moving_finish, op->wq); |
cafe5635 KO |
114 | } |
115 | ||
116 | static void read_moving_submit(struct closure *cl) | |
117 | { | |
220bb38c | 118 | struct moving_io *io = container_of(cl, struct moving_io, cl); |
cafe5635 KO |
119 | struct bio *bio = &io->bio.bio; |
120 | ||
220bb38c | 121 | bch_submit_bbio(bio, io->op.c, &io->w->key, 0); |
cafe5635 | 122 | |
da415a09 | 123 | continue_at(cl, write_moving, io->op.wq); |
cafe5635 KO |
124 | } |
125 | ||
72a44517 | 126 | static void read_moving(struct cache_set *c) |
cafe5635 | 127 | { |
cafe5635 KO |
128 | struct keybuf_key *w; |
129 | struct moving_io *io; | |
130 | struct bio *bio; | |
72a44517 KO |
131 | struct closure cl; |
132 | ||
133 | closure_init_stack(&cl); | |
cafe5635 KO |
134 | |
135 | /* XXX: if we error, background writeback could stall indefinitely */ | |
136 | ||
137 | while (!test_bit(CACHE_SET_STOPPING, &c->flags)) { | |
72c27061 KO |
138 | w = bch_keybuf_next_rescan(c, &c->moving_gc_keys, |
139 | &MAX_KEY, moving_pred); | |
cafe5635 KO |
140 | if (!w) |
141 | break; | |
142 | ||
6d3d1a9c KO |
143 | if (ptr_stale(c, &w->key, 0)) { |
144 | bch_keybuf_del(&c->moving_gc_keys, w); | |
145 | continue; | |
146 | } | |
147 | ||
cafe5635 KO |
148 | io = kzalloc(sizeof(struct moving_io) + sizeof(struct bio_vec) |
149 | * DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS), | |
150 | GFP_KERNEL); | |
151 | if (!io) | |
152 | goto err; | |
153 | ||
154 | w->private = io; | |
155 | io->w = w; | |
220bb38c KO |
156 | io->op.inode = KEY_INODE(&w->key); |
157 | io->op.c = c; | |
da415a09 | 158 | io->op.wq = c->moving_gc_wq; |
cafe5635 KO |
159 | |
160 | moving_init(io); | |
161 | bio = &io->bio.bio; | |
162 | ||
ad0d9e76 | 163 | bio_set_op_attrs(bio, REQ_OP_READ, 0); |
cafe5635 KO |
164 | bio->bi_end_io = read_moving_endio; |
165 | ||
25d8be77 | 166 | if (bch_bio_alloc_pages(bio, GFP_KERNEL)) |
cafe5635 KO |
167 | goto err; |
168 | ||
c37511b8 | 169 | trace_bcache_gc_copy(&w->key); |
cafe5635 | 170 | |
72a44517 | 171 | down(&c->moving_in_flight); |
220bb38c | 172 | closure_call(&io->cl, read_moving_submit, NULL, &cl); |
cafe5635 KO |
173 | } |
174 | ||
175 | if (0) { | |
176 | err: if (!IS_ERR_OR_NULL(w->private)) | |
177 | kfree(w->private); | |
178 | ||
179 | bch_keybuf_del(&c->moving_gc_keys, w); | |
180 | } | |
181 | ||
72a44517 | 182 | closure_sync(&cl); |
cafe5635 KO |
183 | } |
184 | ||
b1a67b0f KO |
185 | static bool bucket_cmp(struct bucket *l, struct bucket *r) |
186 | { | |
187 | return GC_SECTORS_USED(l) < GC_SECTORS_USED(r); | |
188 | } | |
189 | ||
6f10f7d1 | 190 | static unsigned int bucket_heap_top(struct cache *ca) |
b1a67b0f | 191 | { |
bee63f40 | 192 | struct bucket *b; |
1fae7cf0 | 193 | |
bee63f40 | 194 | return (b = heap_peek(&ca->heap)) ? GC_SECTORS_USED(b) : 0; |
b1a67b0f KO |
195 | } |
196 | ||
72a44517 | 197 | void bch_moving_gc(struct cache_set *c) |
cafe5635 | 198 | { |
cafe5635 KO |
199 | struct cache *ca; |
200 | struct bucket *b; | |
6f10f7d1 | 201 | unsigned int i; |
cafe5635 | 202 | |
cafe5635 | 203 | if (!c->copy_gc_enabled) |
72a44517 | 204 | return; |
cafe5635 KO |
205 | |
206 | mutex_lock(&c->bucket_lock); | |
207 | ||
208 | for_each_cache(ca, c, i) { | |
6f10f7d1 CL |
209 | unsigned int sectors_to_move = 0; |
210 | unsigned int reserve_sectors = ca->sb.bucket_size * | |
211 | fifo_used(&ca->free[RESERVE_MOVINGGC]); | |
cafe5635 KO |
212 | |
213 | ca->heap.used = 0; | |
214 | ||
215 | for_each_bucket(b, ca) { | |
3f6ef381 NS |
216 | if (GC_MARK(b) == GC_MARK_METADATA || |
217 | !GC_SECTORS_USED(b) || | |
218 | GC_SECTORS_USED(b) == ca->sb.bucket_size || | |
219 | atomic_read(&b->pin)) | |
cafe5635 KO |
220 | continue; |
221 | ||
222 | if (!heap_full(&ca->heap)) { | |
223 | sectors_to_move += GC_SECTORS_USED(b); | |
224 | heap_add(&ca->heap, b, bucket_cmp); | |
225 | } else if (bucket_cmp(b, heap_peek(&ca->heap))) { | |
b1a67b0f | 226 | sectors_to_move -= bucket_heap_top(ca); |
cafe5635 KO |
227 | sectors_to_move += GC_SECTORS_USED(b); |
228 | ||
229 | ca->heap.data[0] = b; | |
230 | heap_sift(&ca->heap, 0, bucket_cmp); | |
231 | } | |
232 | } | |
233 | ||
234 | while (sectors_to_move > reserve_sectors) { | |
235 | heap_pop(&ca->heap, b, bucket_cmp); | |
236 | sectors_to_move -= GC_SECTORS_USED(b); | |
237 | } | |
238 | ||
981aa8c0 NS |
239 | while (heap_pop(&ca->heap, b, bucket_cmp)) |
240 | SET_GC_MOVE(b, 1); | |
cafe5635 KO |
241 | } |
242 | ||
243 | mutex_unlock(&c->bucket_lock); | |
244 | ||
245 | c->moving_gc_keys.last_scanned = ZERO_KEY; | |
246 | ||
72a44517 | 247 | read_moving(c); |
cafe5635 KO |
248 | } |
249 | ||
250 | void bch_moving_init_cache_set(struct cache_set *c) | |
251 | { | |
72c27061 | 252 | bch_keybuf_init(&c->moving_gc_keys); |
72a44517 | 253 | sema_init(&c->moving_in_flight, 64); |
cafe5635 | 254 | } |