]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
cafe5635 KO |
2 | /* |
3 | * bcache journalling code, for btree insertions | |
4 | * | |
5 | * Copyright 2012 Google, Inc. | |
6 | */ | |
7 | ||
8 | #include "bcache.h" | |
9 | #include "btree.h" | |
10 | #include "debug.h" | |
9aa61a99 | 11 | #include "extents.h" |
cafe5635 | 12 | |
c37511b8 KO |
13 | #include <trace/events/bcache.h> |
14 | ||
cafe5635 KO |
15 | /* |
16 | * Journal replay/recovery: | |
17 | * | |
18 | * This code is all driven from run_cache_set(); we first read the journal | |
19 | * entries, do some other stuff, then we mark all the keys in the journal | |
20 | * entries (same as garbage collection would), then we replay them - reinserting | |
21 | * them into the cache in precisely the same order as they appear in the | |
22 | * journal. | |
23 | * | |
24 | * We only journal keys that go in leaf nodes, which simplifies things quite a | |
25 | * bit. | |
26 | */ | |
27 | ||
4246a0b6 | 28 | static void journal_read_endio(struct bio *bio) |
cafe5635 KO |
29 | { |
30 | struct closure *cl = bio->bi_private; | |
31 | closure_put(cl); | |
32 | } | |
33 | ||
34 | static int journal_read_bucket(struct cache *ca, struct list_head *list, | |
c18536a7 | 35 | unsigned bucket_index) |
cafe5635 KO |
36 | { |
37 | struct journal_device *ja = &ca->journal; | |
38 | struct bio *bio = &ja->bio; | |
39 | ||
40 | struct journal_replay *i; | |
41 | struct jset *j, *data = ca->set->journal.w[0].data; | |
c18536a7 | 42 | struct closure cl; |
cafe5635 KO |
43 | unsigned len, left, offset = 0; |
44 | int ret = 0; | |
45 | sector_t bucket = bucket_to_sector(ca->set, ca->sb.d[bucket_index]); | |
46 | ||
c18536a7 KO |
47 | closure_init_stack(&cl); |
48 | ||
b3fa7e77 | 49 | pr_debug("reading %u", bucket_index); |
cafe5635 KO |
50 | |
51 | while (offset < ca->sb.bucket_size) { | |
52 | reread: left = ca->sb.bucket_size - offset; | |
b3fa7e77 | 53 | len = min_t(unsigned, left, PAGE_SECTORS << JSET_BITS); |
cafe5635 KO |
54 | |
55 | bio_reset(bio); | |
4f024f37 | 56 | bio->bi_iter.bi_sector = bucket + offset; |
74d46992 | 57 | bio_set_dev(bio, ca->bdev); |
4f024f37 | 58 | bio->bi_iter.bi_size = len << 9; |
cafe5635 KO |
59 | |
60 | bio->bi_end_io = journal_read_endio; | |
c18536a7 | 61 | bio->bi_private = &cl; |
ad0d9e76 | 62 | bio_set_op_attrs(bio, REQ_OP_READ, 0); |
169ef1cf | 63 | bch_bio_map(bio, data); |
cafe5635 | 64 | |
bc2e6da1 | 65 | closure_bio_submit(ca->set, bio, &cl); |
c18536a7 | 66 | closure_sync(&cl); |
cafe5635 KO |
67 | |
68 | /* This function could be simpler now since we no longer write | |
69 | * journal entries that overlap bucket boundaries; this means | |
70 | * the start of a bucket will always have a valid journal entry | |
71 | * if it has any journal entries at all. | |
72 | */ | |
73 | ||
74 | j = data; | |
75 | while (len) { | |
76 | struct list_head *where; | |
77 | size_t blocks, bytes = set_bytes(j); | |
78 | ||
b3fa7e77 KO |
79 | if (j->magic != jset_magic(&ca->sb)) { |
80 | pr_debug("%u: bad magic", bucket_index); | |
cafe5635 | 81 | return ret; |
b3fa7e77 | 82 | } |
cafe5635 | 83 | |
b3fa7e77 KO |
84 | if (bytes > left << 9 || |
85 | bytes > PAGE_SIZE << JSET_BITS) { | |
86 | pr_info("%u: too big, %zu bytes, offset %u", | |
87 | bucket_index, bytes, offset); | |
cafe5635 | 88 | return ret; |
b3fa7e77 | 89 | } |
cafe5635 KO |
90 | |
91 | if (bytes > len << 9) | |
92 | goto reread; | |
93 | ||
b3fa7e77 KO |
94 | if (j->csum != csum_set(j)) { |
95 | pr_info("%u: bad csum, %zu bytes, offset %u", | |
96 | bucket_index, bytes, offset); | |
cafe5635 | 97 | return ret; |
b3fa7e77 | 98 | } |
cafe5635 | 99 | |
ee811287 | 100 | blocks = set_blocks(j, block_bytes(ca->set)); |
cafe5635 KO |
101 | |
102 | while (!list_empty(list)) { | |
103 | i = list_first_entry(list, | |
104 | struct journal_replay, list); | |
105 | if (i->j.seq >= j->last_seq) | |
106 | break; | |
107 | list_del(&i->list); | |
108 | kfree(i); | |
109 | } | |
110 | ||
111 | list_for_each_entry_reverse(i, list, list) { | |
112 | if (j->seq == i->j.seq) | |
113 | goto next_set; | |
114 | ||
115 | if (j->seq < i->j.last_seq) | |
116 | goto next_set; | |
117 | ||
118 | if (j->seq > i->j.seq) { | |
119 | where = &i->list; | |
120 | goto add; | |
121 | } | |
122 | } | |
123 | ||
124 | where = list; | |
125 | add: | |
126 | i = kmalloc(offsetof(struct journal_replay, j) + | |
127 | bytes, GFP_KERNEL); | |
128 | if (!i) | |
129 | return -ENOMEM; | |
130 | memcpy(&i->j, j, bytes); | |
131 | list_add(&i->list, where); | |
132 | ret = 1; | |
133 | ||
134 | ja->seq[bucket_index] = j->seq; | |
135 | next_set: | |
136 | offset += blocks * ca->sb.block_size; | |
137 | len -= blocks * ca->sb.block_size; | |
138 | j = ((void *) j) + blocks * block_bytes(ca); | |
139 | } | |
140 | } | |
141 | ||
142 | return ret; | |
143 | } | |
144 | ||
c18536a7 | 145 | int bch_journal_read(struct cache_set *c, struct list_head *list) |
cafe5635 KO |
146 | { |
147 | #define read_bucket(b) \ | |
148 | ({ \ | |
c18536a7 | 149 | int ret = journal_read_bucket(ca, list, b); \ |
cafe5635 KO |
150 | __set_bit(b, bitmap); \ |
151 | if (ret < 0) \ | |
152 | return ret; \ | |
153 | ret; \ | |
154 | }) | |
155 | ||
156 | struct cache *ca; | |
157 | unsigned iter; | |
158 | ||
159 | for_each_cache(ca, c, iter) { | |
160 | struct journal_device *ja = &ca->journal; | |
d1aa1ab3 | 161 | DECLARE_BITMAP(bitmap, SB_JOURNAL_BUCKETS); |
cafe5635 KO |
162 | unsigned i, l, r, m; |
163 | uint64_t seq; | |
164 | ||
165 | bitmap_zero(bitmap, SB_JOURNAL_BUCKETS); | |
166 | pr_debug("%u journal buckets", ca->sb.njournal_buckets); | |
167 | ||
c426c4fd KO |
168 | /* |
169 | * Read journal buckets ordered by golden ratio hash to quickly | |
cafe5635 KO |
170 | * find a sequence of buckets with valid journal entries |
171 | */ | |
172 | for (i = 0; i < ca->sb.njournal_buckets; i++) { | |
bb22cafd TJ |
173 | /* |
174 | * We must try the index l with ZERO first for | |
175 | * correctness due to the scenario that the journal | |
176 | * bucket is circular buffer which might have wrapped | |
177 | */ | |
cafe5635 KO |
178 | l = (i * 2654435769U) % ca->sb.njournal_buckets; |
179 | ||
180 | if (test_bit(l, bitmap)) | |
181 | break; | |
182 | ||
183 | if (read_bucket(l)) | |
184 | goto bsearch; | |
185 | } | |
186 | ||
c426c4fd KO |
187 | /* |
188 | * If that fails, check all the buckets we haven't checked | |
cafe5635 KO |
189 | * already |
190 | */ | |
191 | pr_debug("falling back to linear search"); | |
192 | ||
c426c4fd KO |
193 | for (l = find_first_zero_bit(bitmap, ca->sb.njournal_buckets); |
194 | l < ca->sb.njournal_buckets; | |
195 | l = find_next_zero_bit(bitmap, ca->sb.njournal_buckets, l + 1)) | |
cafe5635 KO |
196 | if (read_bucket(l)) |
197 | goto bsearch; | |
c426c4fd | 198 | |
6b708de6 KO |
199 | /* no journal entries on this device? */ |
200 | if (l == ca->sb.njournal_buckets) | |
c426c4fd | 201 | continue; |
cafe5635 | 202 | bsearch: |
6b708de6 KO |
203 | BUG_ON(list_empty(list)); |
204 | ||
cafe5635 | 205 | /* Binary search */ |
dbd810ab SP |
206 | m = l; |
207 | r = find_next_bit(bitmap, ca->sb.njournal_buckets, l + 1); | |
cafe5635 KO |
208 | pr_debug("starting binary search, l %u r %u", l, r); |
209 | ||
210 | while (l + 1 < r) { | |
faa56736 KO |
211 | seq = list_entry(list->prev, struct journal_replay, |
212 | list)->j.seq; | |
213 | ||
cafe5635 | 214 | m = (l + r) >> 1; |
faa56736 | 215 | read_bucket(m); |
cafe5635 | 216 | |
faa56736 KO |
217 | if (seq != list_entry(list->prev, struct journal_replay, |
218 | list)->j.seq) | |
cafe5635 KO |
219 | l = m; |
220 | else | |
221 | r = m; | |
222 | } | |
223 | ||
c426c4fd KO |
224 | /* |
225 | * Read buckets in reverse order until we stop finding more | |
cafe5635 KO |
226 | * journal entries |
227 | */ | |
c426c4fd KO |
228 | pr_debug("finishing up: m %u njournal_buckets %u", |
229 | m, ca->sb.njournal_buckets); | |
cafe5635 KO |
230 | l = m; |
231 | ||
232 | while (1) { | |
233 | if (!l--) | |
234 | l = ca->sb.njournal_buckets - 1; | |
235 | ||
236 | if (l == m) | |
237 | break; | |
238 | ||
239 | if (test_bit(l, bitmap)) | |
240 | continue; | |
241 | ||
242 | if (!read_bucket(l)) | |
243 | break; | |
244 | } | |
245 | ||
246 | seq = 0; | |
247 | ||
248 | for (i = 0; i < ca->sb.njournal_buckets; i++) | |
249 | if (ja->seq[i] > seq) { | |
250 | seq = ja->seq[i]; | |
27201cfd KO |
251 | /* |
252 | * When journal_reclaim() goes to allocate for | |
253 | * the first time, it'll use the bucket after | |
254 | * ja->cur_idx | |
255 | */ | |
256 | ja->cur_idx = i; | |
257 | ja->last_idx = ja->discard_idx = (i + 1) % | |
258 | ca->sb.njournal_buckets; | |
cafe5635 KO |
259 | |
260 | } | |
261 | } | |
262 | ||
c426c4fd KO |
263 | if (!list_empty(list)) |
264 | c->journal.seq = list_entry(list->prev, | |
265 | struct journal_replay, | |
266 | list)->j.seq; | |
cafe5635 KO |
267 | |
268 | return 0; | |
269 | #undef read_bucket | |
270 | } | |
271 | ||
272 | void bch_journal_mark(struct cache_set *c, struct list_head *list) | |
273 | { | |
274 | atomic_t p = { 0 }; | |
275 | struct bkey *k; | |
276 | struct journal_replay *i; | |
277 | struct journal *j = &c->journal; | |
278 | uint64_t last = j->seq; | |
279 | ||
280 | /* | |
281 | * journal.pin should never fill up - we never write a journal | |
282 | * entry when it would fill up. But if for some reason it does, we | |
283 | * iterate over the list in reverse order so that we can just skip that | |
284 | * refcount instead of bugging. | |
285 | */ | |
286 | ||
287 | list_for_each_entry_reverse(i, list, list) { | |
288 | BUG_ON(last < i->j.seq); | |
289 | i->pin = NULL; | |
290 | ||
291 | while (last-- != i->j.seq) | |
292 | if (fifo_free(&j->pin) > 1) { | |
293 | fifo_push_front(&j->pin, p); | |
294 | atomic_set(&fifo_front(&j->pin), 0); | |
295 | } | |
296 | ||
297 | if (fifo_free(&j->pin) > 1) { | |
298 | fifo_push_front(&j->pin, p); | |
299 | i->pin = &fifo_front(&j->pin); | |
300 | atomic_set(i->pin, 1); | |
301 | } | |
302 | ||
303 | for (k = i->j.start; | |
fafff81c | 304 | k < bset_bkey_last(&i->j); |
9aa61a99 KO |
305 | k = bkey_next(k)) |
306 | if (!__bch_extent_invalid(c, k)) { | |
307 | unsigned j; | |
cafe5635 | 308 | |
9aa61a99 KO |
309 | for (j = 0; j < KEY_PTRS(k); j++) |
310 | if (ptr_available(c, k, j)) | |
311 | atomic_inc(&PTR_BUCKET(c, k, j)->pin); | |
65ddf45a | 312 | |
9aa61a99 KO |
313 | bch_initial_mark_key(c, 0, k); |
314 | } | |
cafe5635 KO |
315 | } |
316 | } | |
317 | ||
d21f5af1 TJ |
318 | bool is_discard_enabled(struct cache_set *s) |
319 | { | |
320 | struct cache *ca; | |
321 | unsigned int i; | |
322 | ||
323 | for_each_cache(ca, s, i) | |
324 | if (ca->discard) | |
325 | return true; | |
326 | ||
327 | return false; | |
328 | } | |
329 | ||
c18536a7 | 330 | int bch_journal_replay(struct cache_set *s, struct list_head *list) |
cafe5635 KO |
331 | { |
332 | int ret = 0, keys = 0, entries = 0; | |
333 | struct bkey *k; | |
334 | struct journal_replay *i = | |
335 | list_entry(list->prev, struct journal_replay, list); | |
336 | ||
337 | uint64_t start = i->j.last_seq, end = i->j.seq, n = start; | |
0b93207a KO |
338 | struct keylist keylist; |
339 | ||
cafe5635 KO |
340 | list_for_each_entry(i, list, list) { |
341 | BUG_ON(i->pin && atomic_read(i->pin) != 1); | |
342 | ||
f9cb0a2d | 343 | if (n != i->j.seq) { |
d21f5af1 TJ |
344 | if (n == start && is_discard_enabled(s)) |
345 | pr_info("bcache: journal entries %llu-%llu may be discarded! (replaying %llu-%llu)", | |
346 | n, i->j.seq - 1, start, end); | |
347 | else { | |
348 | pr_err("bcache: journal entries %llu-%llu missing! (replaying %llu-%llu)", | |
349 | n, i->j.seq - 1, start, end); | |
350 | ret = -EIO; | |
351 | goto err; | |
352 | } | |
f9cb0a2d | 353 | } |
cafe5635 KO |
354 | |
355 | for (k = i->j.start; | |
fafff81c | 356 | k < bset_bkey_last(&i->j); |
cafe5635 | 357 | k = bkey_next(k)) { |
c37511b8 KO |
358 | trace_bcache_journal_replay_key(k); |
359 | ||
c13f3af9 | 360 | bch_keylist_init_single(&keylist, k); |
cafe5635 | 361 | |
cc7b8819 | 362 | ret = bch_btree_insert(s, &keylist, i->pin, NULL); |
cafe5635 KO |
363 | if (ret) |
364 | goto err; | |
365 | ||
0b93207a | 366 | BUG_ON(!bch_keylist_empty(&keylist)); |
cafe5635 KO |
367 | keys++; |
368 | ||
369 | cond_resched(); | |
370 | } | |
371 | ||
372 | if (i->pin) | |
373 | atomic_dec(i->pin); | |
374 | n = i->j.seq + 1; | |
375 | entries++; | |
376 | } | |
377 | ||
378 | pr_info("journal replay done, %i keys in %i entries, seq %llu", | |
379 | keys, entries, end); | |
b54d6934 | 380 | err: |
cafe5635 KO |
381 | while (!list_empty(list)) { |
382 | i = list_first_entry(list, struct journal_replay, list); | |
383 | list_del(&i->list); | |
384 | kfree(i); | |
385 | } | |
b54d6934 | 386 | |
cafe5635 KO |
387 | return ret; |
388 | } | |
389 | ||
390 | /* Journalling */ | |
531b6bda TJ |
391 | #define journal_max_cmp(l, r) \ |
392 | (fifo_idx(&c->journal.pin, btree_current_write(l)->journal) < \ | |
393 | fifo_idx(&(c)->journal.pin, btree_current_write(r)->journal)) | |
394 | #define journal_min_cmp(l, r) \ | |
395 | (fifo_idx(&c->journal.pin, btree_current_write(l)->journal) > \ | |
396 | fifo_idx(&(c)->journal.pin, btree_current_write(r)->journal)) | |
cafe5635 KO |
397 | |
398 | static void btree_flush_write(struct cache_set *c) | |
399 | { | |
400 | /* | |
401 | * Try to find the btree node with that references the oldest journal | |
402 | * entry, best is our current candidate and is locked if non NULL: | |
403 | */ | |
531b6bda TJ |
404 | struct btree *b; |
405 | int i; | |
3998e45e TJ |
406 | |
407 | atomic_long_inc(&c->flush_write); | |
531b6bda | 408 | |
a34a8bfd | 409 | retry: |
531b6bda TJ |
410 | spin_lock(&c->journal.lock); |
411 | if (heap_empty(&c->flush_btree)) { | |
412 | for_each_cached_btree(b, c, i) | |
413 | if (btree_current_write(b)->journal) { | |
414 | if (!heap_full(&c->flush_btree)) | |
415 | heap_add(&c->flush_btree, b, | |
416 | journal_max_cmp); | |
417 | else if (journal_max_cmp(b, | |
418 | heap_peek(&c->flush_btree))) { | |
419 | c->flush_btree.data[0] = b; | |
420 | heap_sift(&c->flush_btree, 0, | |
421 | journal_max_cmp); | |
422 | } | |
a34a8bfd | 423 | } |
cafe5635 | 424 | |
531b6bda TJ |
425 | for (i = c->flush_btree.used / 2 - 1; i >= 0; --i) |
426 | heap_sift(&c->flush_btree, i, journal_min_cmp); | |
427 | } | |
428 | ||
429 | b = NULL; | |
430 | heap_pop(&c->flush_btree, b, journal_min_cmp); | |
431 | spin_unlock(&c->journal.lock); | |
432 | ||
a34a8bfd | 433 | if (b) { |
2a285686 | 434 | mutex_lock(&b->write_lock); |
a34a8bfd | 435 | if (!btree_current_write(b)->journal) { |
2a285686 | 436 | mutex_unlock(&b->write_lock); |
a34a8bfd | 437 | /* We raced */ |
3998e45e | 438 | atomic_long_inc(&c->retry_flush_write); |
a34a8bfd | 439 | goto retry; |
cafe5635 KO |
440 | } |
441 | ||
2a285686 KO |
442 | __bch_btree_node_write(b, NULL); |
443 | mutex_unlock(&b->write_lock); | |
cafe5635 | 444 | } |
cafe5635 KO |
445 | } |
446 | ||
447 | #define last_seq(j) ((j)->seq - fifo_used(&(j)->pin) + 1) | |
448 | ||
4246a0b6 | 449 | static void journal_discard_endio(struct bio *bio) |
cafe5635 KO |
450 | { |
451 | struct journal_device *ja = | |
452 | container_of(bio, struct journal_device, discard_bio); | |
453 | struct cache *ca = container_of(ja, struct cache, journal); | |
454 | ||
455 | atomic_set(&ja->discard_in_flight, DISCARD_DONE); | |
456 | ||
457 | closure_wake_up(&ca->set->journal.wait); | |
458 | closure_put(&ca->set->cl); | |
459 | } | |
460 | ||
461 | static void journal_discard_work(struct work_struct *work) | |
462 | { | |
463 | struct journal_device *ja = | |
464 | container_of(work, struct journal_device, discard_work); | |
465 | ||
4e49ea4a | 466 | submit_bio(&ja->discard_bio); |
cafe5635 KO |
467 | } |
468 | ||
469 | static void do_journal_discard(struct cache *ca) | |
470 | { | |
471 | struct journal_device *ja = &ca->journal; | |
472 | struct bio *bio = &ja->discard_bio; | |
473 | ||
474 | if (!ca->discard) { | |
475 | ja->discard_idx = ja->last_idx; | |
476 | return; | |
477 | } | |
478 | ||
6d9d21e3 | 479 | switch (atomic_read(&ja->discard_in_flight)) { |
cafe5635 KO |
480 | case DISCARD_IN_FLIGHT: |
481 | return; | |
482 | ||
483 | case DISCARD_DONE: | |
484 | ja->discard_idx = (ja->discard_idx + 1) % | |
485 | ca->sb.njournal_buckets; | |
486 | ||
487 | atomic_set(&ja->discard_in_flight, DISCARD_READY); | |
488 | /* fallthrough */ | |
489 | ||
490 | case DISCARD_READY: | |
491 | if (ja->discard_idx == ja->last_idx) | |
492 | return; | |
493 | ||
494 | atomic_set(&ja->discard_in_flight, DISCARD_IN_FLIGHT); | |
495 | ||
3a83f467 | 496 | bio_init(bio, bio->bi_inline_vecs, 1); |
ad0d9e76 | 497 | bio_set_op_attrs(bio, REQ_OP_DISCARD, 0); |
4f024f37 | 498 | bio->bi_iter.bi_sector = bucket_to_sector(ca->set, |
b1a67b0f | 499 | ca->sb.d[ja->discard_idx]); |
74d46992 | 500 | bio_set_dev(bio, ca->bdev); |
4f024f37 | 501 | bio->bi_iter.bi_size = bucket_bytes(ca); |
cafe5635 KO |
502 | bio->bi_end_io = journal_discard_endio; |
503 | ||
504 | closure_get(&ca->set->cl); | |
505 | INIT_WORK(&ja->discard_work, journal_discard_work); | |
506 | schedule_work(&ja->discard_work); | |
507 | } | |
508 | } | |
509 | ||
510 | static void journal_reclaim(struct cache_set *c) | |
511 | { | |
512 | struct bkey *k = &c->journal.key; | |
513 | struct cache *ca; | |
514 | uint64_t last_seq; | |
515 | unsigned iter, n = 0; | |
0d0bf2dc | 516 | atomic_t p __maybe_unused; |
cafe5635 | 517 | |
3998e45e TJ |
518 | atomic_long_inc(&c->reclaim); |
519 | ||
cafe5635 KO |
520 | while (!atomic_read(&fifo_front(&c->journal.pin))) |
521 | fifo_pop(&c->journal.pin, p); | |
522 | ||
523 | last_seq = last_seq(&c->journal); | |
524 | ||
525 | /* Update last_idx */ | |
526 | ||
527 | for_each_cache(ca, c, iter) { | |
528 | struct journal_device *ja = &ca->journal; | |
529 | ||
530 | while (ja->last_idx != ja->cur_idx && | |
531 | ja->seq[ja->last_idx] < last_seq) | |
532 | ja->last_idx = (ja->last_idx + 1) % | |
533 | ca->sb.njournal_buckets; | |
534 | } | |
535 | ||
536 | for_each_cache(ca, c, iter) | |
537 | do_journal_discard(ca); | |
538 | ||
539 | if (c->journal.blocks_free) | |
a34a8bfd | 540 | goto out; |
cafe5635 KO |
541 | |
542 | /* | |
543 | * Allocate: | |
544 | * XXX: Sort by free journal space | |
545 | */ | |
546 | ||
547 | for_each_cache(ca, c, iter) { | |
548 | struct journal_device *ja = &ca->journal; | |
549 | unsigned next = (ja->cur_idx + 1) % ca->sb.njournal_buckets; | |
550 | ||
551 | /* No space available on this device */ | |
552 | if (next == ja->discard_idx) | |
553 | continue; | |
554 | ||
555 | ja->cur_idx = next; | |
cf33c1ee | 556 | k->ptr[n++] = MAKE_PTR(0, |
cafe5635 KO |
557 | bucket_to_sector(c, ca->sb.d[ja->cur_idx]), |
558 | ca->sb.nr_this_dev); | |
559 | } | |
560 | ||
0a46f950 CL |
561 | if (n) { |
562 | bkey_init(k); | |
563 | SET_KEY_PTRS(k, n); | |
cafe5635 | 564 | c->journal.blocks_free = c->sb.bucket_size >> c->block_bits; |
0a46f950 | 565 | } |
a34a8bfd | 566 | out: |
cafe5635 KO |
567 | if (!journal_full(&c->journal)) |
568 | __closure_wake_up(&c->journal.wait); | |
569 | } | |
570 | ||
571 | void bch_journal_next(struct journal *j) | |
572 | { | |
573 | atomic_t p = { 1 }; | |
574 | ||
575 | j->cur = (j->cur == j->w) | |
576 | ? &j->w[1] | |
577 | : &j->w[0]; | |
578 | ||
579 | /* | |
580 | * The fifo_push() needs to happen at the same time as j->seq is | |
581 | * incremented for last_seq() to be calculated correctly | |
582 | */ | |
583 | BUG_ON(!fifo_push(&j->pin, p)); | |
584 | atomic_set(&fifo_back(&j->pin), 1); | |
585 | ||
586 | j->cur->data->seq = ++j->seq; | |
dabb4433 | 587 | j->cur->dirty = false; |
cafe5635 KO |
588 | j->cur->need_write = false; |
589 | j->cur->data->keys = 0; | |
590 | ||
591 | if (fifo_full(&j->pin)) | |
592 | pr_debug("journal_pin full (%zu)", fifo_used(&j->pin)); | |
593 | } | |
594 | ||
4246a0b6 | 595 | static void journal_write_endio(struct bio *bio) |
cafe5635 KO |
596 | { |
597 | struct journal_write *w = bio->bi_private; | |
598 | ||
4e4cbee9 | 599 | cache_set_err_on(bio->bi_status, w->c, "journal io error"); |
7857d5d4 | 600 | closure_put(&w->c->journal.io); |
cafe5635 KO |
601 | } |
602 | ||
603 | static void journal_write(struct closure *); | |
604 | ||
605 | static void journal_write_done(struct closure *cl) | |
606 | { | |
7857d5d4 | 607 | struct journal *j = container_of(cl, struct journal, io); |
cafe5635 KO |
608 | struct journal_write *w = (j->cur == j->w) |
609 | ? &j->w[1] | |
610 | : &j->w[0]; | |
611 | ||
612 | __closure_wake_up(&w->wait); | |
7857d5d4 | 613 | continue_at_nobarrier(cl, journal_write, system_wq); |
cafe5635 KO |
614 | } |
615 | ||
cb7a583e | 616 | static void journal_write_unlock(struct closure *cl) |
08dec3e1 | 617 | __releases(&c->journal.lock) |
cb7a583e KO |
618 | { |
619 | struct cache_set *c = container_of(cl, struct cache_set, journal.io); | |
620 | ||
621 | c->journal.io_in_flight = 0; | |
622 | spin_unlock(&c->journal.lock); | |
623 | } | |
624 | ||
cafe5635 | 625 | static void journal_write_unlocked(struct closure *cl) |
c19ed23a | 626 | __releases(c->journal.lock) |
cafe5635 | 627 | { |
7857d5d4 | 628 | struct cache_set *c = container_of(cl, struct cache_set, journal.io); |
cafe5635 KO |
629 | struct cache *ca; |
630 | struct journal_write *w = c->journal.cur; | |
631 | struct bkey *k = &c->journal.key; | |
ee811287 KO |
632 | unsigned i, sectors = set_blocks(w->data, block_bytes(c)) * |
633 | c->sb.block_size; | |
cafe5635 KO |
634 | |
635 | struct bio *bio; | |
636 | struct bio_list list; | |
637 | bio_list_init(&list); | |
638 | ||
639 | if (!w->need_write) { | |
cb7a583e | 640 | closure_return_with_destructor(cl, journal_write_unlock); |
77b5a084 | 641 | return; |
cafe5635 KO |
642 | } else if (journal_full(&c->journal)) { |
643 | journal_reclaim(c); | |
644 | spin_unlock(&c->journal.lock); | |
645 | ||
646 | btree_flush_write(c); | |
647 | continue_at(cl, journal_write, system_wq); | |
77b5a084 | 648 | return; |
cafe5635 KO |
649 | } |
650 | ||
ee811287 | 651 | c->journal.blocks_free -= set_blocks(w->data, block_bytes(c)); |
cafe5635 KO |
652 | |
653 | w->data->btree_level = c->root->level; | |
654 | ||
655 | bkey_copy(&w->data->btree_root, &c->root->key); | |
656 | bkey_copy(&w->data->uuid_bucket, &c->uuid_bucket); | |
657 | ||
658 | for_each_cache(ca, c, i) | |
659 | w->data->prio_bucket[ca->sb.nr_this_dev] = ca->prio_buckets[0]; | |
660 | ||
81ab4190 | 661 | w->data->magic = jset_magic(&c->sb); |
cafe5635 KO |
662 | w->data->version = BCACHE_JSET_VERSION; |
663 | w->data->last_seq = last_seq(&c->journal); | |
664 | w->data->csum = csum_set(w->data); | |
665 | ||
666 | for (i = 0; i < KEY_PTRS(k); i++) { | |
667 | ca = PTR_CACHE(c, k, i); | |
668 | bio = &ca->journal.bio; | |
669 | ||
670 | atomic_long_add(sectors, &ca->meta_sectors_written); | |
671 | ||
672 | bio_reset(bio); | |
4f024f37 | 673 | bio->bi_iter.bi_sector = PTR_OFFSET(k, i); |
74d46992 | 674 | bio_set_dev(bio, ca->bdev); |
4f024f37 | 675 | bio->bi_iter.bi_size = sectors << 9; |
cafe5635 KO |
676 | |
677 | bio->bi_end_io = journal_write_endio; | |
678 | bio->bi_private = w; | |
ad0d9e76 | 679 | bio_set_op_attrs(bio, REQ_OP_WRITE, |
28a8f0d3 | 680 | REQ_SYNC|REQ_META|REQ_PREFLUSH|REQ_FUA); |
169ef1cf | 681 | bch_bio_map(bio, w->data); |
cafe5635 KO |
682 | |
683 | trace_bcache_journal_write(bio); | |
684 | bio_list_add(&list, bio); | |
685 | ||
686 | SET_PTR_OFFSET(k, i, PTR_OFFSET(k, i) + sectors); | |
687 | ||
688 | ca->journal.seq[ca->journal.cur_idx] = w->data->seq; | |
689 | } | |
690 | ||
0a46f950 CL |
691 | /* If KEY_PTRS(k) == 0, this jset gets lost in air */ |
692 | BUG_ON(i == 0); | |
693 | ||
cafe5635 KO |
694 | atomic_dec_bug(&fifo_back(&c->journal.pin)); |
695 | bch_journal_next(&c->journal); | |
696 | journal_reclaim(c); | |
697 | ||
698 | spin_unlock(&c->journal.lock); | |
699 | ||
700 | while ((bio = bio_list_pop(&list))) | |
bc2e6da1 | 701 | closure_bio_submit(c, bio, cl); |
cafe5635 KO |
702 | |
703 | continue_at(cl, journal_write_done, NULL); | |
704 | } | |
705 | ||
706 | static void journal_write(struct closure *cl) | |
707 | { | |
7857d5d4 | 708 | struct cache_set *c = container_of(cl, struct cache_set, journal.io); |
cafe5635 KO |
709 | |
710 | spin_lock(&c->journal.lock); | |
711 | journal_write_unlocked(cl); | |
712 | } | |
713 | ||
a34a8bfd | 714 | static void journal_try_write(struct cache_set *c) |
c19ed23a | 715 | __releases(c->journal.lock) |
cafe5635 | 716 | { |
7857d5d4 KO |
717 | struct closure *cl = &c->journal.io; |
718 | struct journal_write *w = c->journal.cur; | |
719 | ||
720 | w->need_write = true; | |
cafe5635 | 721 | |
cb7a583e KO |
722 | if (!c->journal.io_in_flight) { |
723 | c->journal.io_in_flight = 1; | |
724 | closure_call(cl, journal_write_unlocked, NULL, &c->cl); | |
725 | } else { | |
a34a8bfd | 726 | spin_unlock(&c->journal.lock); |
cb7a583e | 727 | } |
cafe5635 KO |
728 | } |
729 | ||
a34a8bfd KO |
730 | static struct journal_write *journal_wait_for_write(struct cache_set *c, |
731 | unsigned nkeys) | |
08dec3e1 | 732 | __acquires(&c->journal.lock) |
cafe5635 | 733 | { |
a34a8bfd KO |
734 | size_t sectors; |
735 | struct closure cl; | |
5775e213 | 736 | bool wait = false; |
cafe5635 | 737 | |
a34a8bfd KO |
738 | closure_init_stack(&cl); |
739 | ||
740 | spin_lock(&c->journal.lock); | |
741 | ||
742 | while (1) { | |
743 | struct journal_write *w = c->journal.cur; | |
744 | ||
745 | sectors = __set_blocks(w->data, w->data->keys + nkeys, | |
ee811287 | 746 | block_bytes(c)) * c->sb.block_size; |
a34a8bfd KO |
747 | |
748 | if (sectors <= min_t(size_t, | |
749 | c->journal.blocks_free * c->sb.block_size, | |
750 | PAGE_SECTORS << JSET_BITS)) | |
751 | return w; | |
752 | ||
5775e213 KO |
753 | if (wait) |
754 | closure_wait(&c->journal.wait, &cl); | |
755 | ||
a34a8bfd | 756 | if (!journal_full(&c->journal)) { |
5775e213 KO |
757 | if (wait) |
758 | trace_bcache_journal_entry_full(c); | |
a34a8bfd KO |
759 | |
760 | /* | |
761 | * XXX: If we were inserting so many keys that they | |
762 | * won't fit in an _empty_ journal write, we'll | |
763 | * deadlock. For now, handle this in | |
764 | * bch_keylist_realloc() - but something to think about. | |
765 | */ | |
766 | BUG_ON(!w->data->keys); | |
767 | ||
a34a8bfd KO |
768 | journal_try_write(c); /* unlocks */ |
769 | } else { | |
5775e213 KO |
770 | if (wait) |
771 | trace_bcache_journal_full(c); | |
a34a8bfd | 772 | |
a34a8bfd KO |
773 | journal_reclaim(c); |
774 | spin_unlock(&c->journal.lock); | |
cafe5635 | 775 | |
a34a8bfd KO |
776 | btree_flush_write(c); |
777 | } | |
cafe5635 | 778 | |
a34a8bfd KO |
779 | closure_sync(&cl); |
780 | spin_lock(&c->journal.lock); | |
5775e213 | 781 | wait = true; |
cafe5635 KO |
782 | } |
783 | } | |
784 | ||
7857d5d4 KO |
785 | static void journal_write_work(struct work_struct *work) |
786 | { | |
787 | struct cache_set *c = container_of(to_delayed_work(work), | |
788 | struct cache_set, | |
789 | journal.work); | |
790 | spin_lock(&c->journal.lock); | |
dabb4433 KO |
791 | if (c->journal.cur->dirty) |
792 | journal_try_write(c); | |
793 | else | |
794 | spin_unlock(&c->journal.lock); | |
7857d5d4 KO |
795 | } |
796 | ||
cafe5635 KO |
797 | /* |
798 | * Entry point to the journalling code - bio_insert() and btree_invalidate() | |
799 | * pass bch_journal() a list of keys to be journalled, and then | |
800 | * bch_journal() hands those same keys off to btree_insert_async() | |
801 | */ | |
802 | ||
a34a8bfd KO |
803 | atomic_t *bch_journal(struct cache_set *c, |
804 | struct keylist *keys, | |
805 | struct closure *parent) | |
cafe5635 | 806 | { |
cafe5635 | 807 | struct journal_write *w; |
a34a8bfd | 808 | atomic_t *ret; |
cafe5635 | 809 | |
1854bfa9 CL |
810 | /* No journaling if CACHE_SET_IO_DISABLE set already */ |
811 | if (unlikely(test_bit(CACHE_SET_IO_DISABLE, &c->flags))) | |
812 | return NULL; | |
813 | ||
a34a8bfd KO |
814 | if (!CACHE_SYNC(&c->sb)) |
815 | return NULL; | |
cafe5635 | 816 | |
a34a8bfd | 817 | w = journal_wait_for_write(c, bch_keylist_nkeys(keys)); |
c37511b8 | 818 | |
fafff81c | 819 | memcpy(bset_bkey_last(w->data), keys->keys, bch_keylist_bytes(keys)); |
a34a8bfd | 820 | w->data->keys += bch_keylist_nkeys(keys); |
cafe5635 | 821 | |
a34a8bfd KO |
822 | ret = &fifo_back(&c->journal.pin); |
823 | atomic_inc(ret); | |
cafe5635 | 824 | |
a34a8bfd KO |
825 | if (parent) { |
826 | closure_wait(&w->wait, parent); | |
7857d5d4 | 827 | journal_try_write(c); |
dabb4433 KO |
828 | } else if (!w->dirty) { |
829 | w->dirty = true; | |
7857d5d4 KO |
830 | schedule_delayed_work(&c->journal.work, |
831 | msecs_to_jiffies(c->journal_delay_ms)); | |
832 | spin_unlock(&c->journal.lock); | |
833 | } else { | |
834 | spin_unlock(&c->journal.lock); | |
cafe5635 | 835 | } |
a34a8bfd KO |
836 | |
837 | ||
838 | return ret; | |
839 | } | |
840 | ||
841 | void bch_journal_meta(struct cache_set *c, struct closure *cl) | |
842 | { | |
843 | struct keylist keys; | |
844 | atomic_t *ref; | |
845 | ||
846 | bch_keylist_init(&keys); | |
847 | ||
848 | ref = bch_journal(c, &keys, cl); | |
849 | if (ref) | |
850 | atomic_dec_bug(ref); | |
cafe5635 KO |
851 | } |
852 | ||
853 | void bch_journal_free(struct cache_set *c) | |
854 | { | |
855 | free_pages((unsigned long) c->journal.w[1].data, JSET_BITS); | |
856 | free_pages((unsigned long) c->journal.w[0].data, JSET_BITS); | |
857 | free_fifo(&c->journal.pin); | |
858 | } | |
859 | ||
860 | int bch_journal_alloc(struct cache_set *c) | |
861 | { | |
862 | struct journal *j = &c->journal; | |
863 | ||
cafe5635 | 864 | spin_lock_init(&j->lock); |
7857d5d4 | 865 | INIT_DELAYED_WORK(&j->work, journal_write_work); |
cafe5635 KO |
866 | |
867 | c->journal_delay_ms = 100; | |
868 | ||
869 | j->w[0].c = c; | |
870 | j->w[1].c = c; | |
871 | ||
531b6bda TJ |
872 | if (!(init_heap(&c->flush_btree, 128, GFP_KERNEL)) || |
873 | !(init_fifo(&j->pin, JOURNAL_PIN, GFP_KERNEL)) || | |
cafe5635 KO |
874 | !(j->w[0].data = (void *) __get_free_pages(GFP_KERNEL, JSET_BITS)) || |
875 | !(j->w[1].data = (void *) __get_free_pages(GFP_KERNEL, JSET_BITS))) | |
876 | return -ENOMEM; | |
877 | ||
878 | return 0; | |
879 | } |