2 * background writeback - scan btree for dirty data and write it to the backing
5 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
6 * Copyright 2012 Google, Inc.
12 #include "writeback.h"
14 #include <linux/delay.h>
15 #include <linux/kthread.h>
16 #include <linux/sched/clock.h>
17 #include <trace/events/bcache.h>
21 static void __update_writeback_rate(struct cached_dev
*dc
)
23 struct cache_set
*c
= dc
->disk
.c
;
24 uint64_t cache_sectors
= c
->nbuckets
* c
->sb
.bucket_size
-
25 bcache_flash_devs_sectors_dirty(c
);
26 uint64_t cache_dirty_target
=
27 div_u64(cache_sectors
* dc
->writeback_percent
, 100);
29 int64_t target
= div64_u64(cache_dirty_target
* bdev_sectors(dc
->bdev
),
30 c
->cached_dev_sectors
);
34 int64_t dirty
= bcache_dev_sectors_dirty(&dc
->disk
);
35 int64_t derivative
= dirty
- dc
->disk
.sectors_dirty_last
;
36 int64_t proportional
= dirty
- target
;
39 dc
->disk
.sectors_dirty_last
= dirty
;
41 /* Scale to sectors per second */
43 proportional
*= dc
->writeback_rate_update_seconds
;
44 proportional
= div_s64(proportional
, dc
->writeback_rate_p_term_inverse
);
46 derivative
= div_s64(derivative
, dc
->writeback_rate_update_seconds
);
48 derivative
= ewma_add(dc
->disk
.sectors_dirty_derivative
, derivative
,
49 (dc
->writeback_rate_d_term
/
50 dc
->writeback_rate_update_seconds
) ?: 1, 0);
52 derivative
*= dc
->writeback_rate_d_term
;
53 derivative
= div_s64(derivative
, dc
->writeback_rate_p_term_inverse
);
55 change
= proportional
+ derivative
;
57 /* Don't increase writeback rate if the device isn't keeping up */
59 time_after64(local_clock(),
60 dc
->writeback_rate
.next
+ NSEC_PER_MSEC
))
63 dc
->writeback_rate
.rate
=
64 clamp_t(int64_t, (int64_t) dc
->writeback_rate
.rate
+ change
,
67 dc
->writeback_rate_proportional
= proportional
;
68 dc
->writeback_rate_derivative
= derivative
;
69 dc
->writeback_rate_change
= change
;
70 dc
->writeback_rate_target
= target
;
73 static void update_writeback_rate(struct work_struct
*work
)
75 struct cached_dev
*dc
= container_of(to_delayed_work(work
),
77 writeback_rate_update
);
79 down_read(&dc
->writeback_lock
);
81 if (atomic_read(&dc
->has_dirty
) &&
82 dc
->writeback_percent
)
83 __update_writeback_rate(dc
);
85 up_read(&dc
->writeback_lock
);
87 schedule_delayed_work(&dc
->writeback_rate_update
,
88 dc
->writeback_rate_update_seconds
* HZ
);
91 static unsigned writeback_delay(struct cached_dev
*dc
, unsigned sectors
)
93 if (test_bit(BCACHE_DEV_DETACHING
, &dc
->disk
.flags
) ||
94 !dc
->writeback_percent
)
97 return bch_next_delay(&dc
->writeback_rate
, sectors
);
102 struct cached_dev
*dc
;
106 static void dirty_init(struct keybuf_key
*w
)
108 struct dirty_io
*io
= w
->private;
109 struct bio
*bio
= &io
->bio
;
111 bio_init(bio
, bio
->bi_inline_vecs
,
112 DIV_ROUND_UP(KEY_SIZE(&w
->key
), PAGE_SECTORS
));
113 if (!io
->dc
->writeback_percent
)
114 bio_set_prio(bio
, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE
, 0));
116 bio
->bi_iter
.bi_size
= KEY_SIZE(&w
->key
) << 9;
118 bch_bio_map(bio
, NULL
);
121 static void dirty_io_destructor(struct closure
*cl
)
123 struct dirty_io
*io
= container_of(cl
, struct dirty_io
, cl
);
127 static void write_dirty_finish(struct closure
*cl
)
129 struct dirty_io
*io
= container_of(cl
, struct dirty_io
, cl
);
130 struct keybuf_key
*w
= io
->bio
.bi_private
;
131 struct cached_dev
*dc
= io
->dc
;
133 bio_free_pages(&io
->bio
);
135 /* This is kind of a dumb way of signalling errors. */
136 if (KEY_DIRTY(&w
->key
)) {
141 bch_keylist_init(&keys
);
143 bkey_copy(keys
.top
, &w
->key
);
144 SET_KEY_DIRTY(keys
.top
, false);
145 bch_keylist_push(&keys
);
147 for (i
= 0; i
< KEY_PTRS(&w
->key
); i
++)
148 atomic_inc(&PTR_BUCKET(dc
->disk
.c
, &w
->key
, i
)->pin
);
150 ret
= bch_btree_insert(dc
->disk
.c
, &keys
, NULL
, &w
->key
);
153 trace_bcache_writeback_collision(&w
->key
);
156 ? &dc
->disk
.c
->writeback_keys_failed
157 : &dc
->disk
.c
->writeback_keys_done
);
160 bch_keybuf_del(&dc
->writeback_keys
, w
);
163 closure_return_with_destructor(cl
, dirty_io_destructor
);
166 static void dirty_endio(struct bio
*bio
)
168 struct keybuf_key
*w
= bio
->bi_private
;
169 struct dirty_io
*io
= w
->private;
172 SET_KEY_DIRTY(&w
->key
, false);
174 closure_put(&io
->cl
);
177 static void write_dirty(struct closure
*cl
)
179 struct dirty_io
*io
= container_of(cl
, struct dirty_io
, cl
);
180 struct keybuf_key
*w
= io
->bio
.bi_private
;
183 bio_set_op_attrs(&io
->bio
, REQ_OP_WRITE
, 0);
184 io
->bio
.bi_iter
.bi_sector
= KEY_START(&w
->key
);
185 bio_set_dev(&io
->bio
, io
->dc
->bdev
);
186 io
->bio
.bi_end_io
= dirty_endio
;
188 closure_bio_submit(&io
->bio
, cl
);
190 continue_at(cl
, write_dirty_finish
, io
->dc
->writeback_write_wq
);
193 static void read_dirty_endio(struct bio
*bio
)
195 struct keybuf_key
*w
= bio
->bi_private
;
196 struct dirty_io
*io
= w
->private;
198 bch_count_io_errors(PTR_CACHE(io
->dc
->disk
.c
, &w
->key
, 0),
199 bio
->bi_status
, "reading dirty data from cache");
204 static void read_dirty_submit(struct closure
*cl
)
206 struct dirty_io
*io
= container_of(cl
, struct dirty_io
, cl
);
208 closure_bio_submit(&io
->bio
, cl
);
210 continue_at(cl
, write_dirty
, io
->dc
->writeback_write_wq
);
213 static void read_dirty(struct cached_dev
*dc
)
216 struct keybuf_key
*w
;
220 closure_init_stack(&cl
);
223 * XXX: if we error, background writeback just spins. Should use some
227 while (!kthread_should_stop()) {
229 w
= bch_keybuf_next(&dc
->writeback_keys
);
233 BUG_ON(ptr_stale(dc
->disk
.c
, &w
->key
, 0));
235 if (KEY_START(&w
->key
) != dc
->last_read
||
236 jiffies_to_msecs(delay
) > 50)
237 while (!kthread_should_stop() && delay
)
238 delay
= schedule_timeout_interruptible(delay
);
240 dc
->last_read
= KEY_OFFSET(&w
->key
);
242 io
= kzalloc(sizeof(struct dirty_io
) + sizeof(struct bio_vec
)
243 * DIV_ROUND_UP(KEY_SIZE(&w
->key
), PAGE_SECTORS
),
252 bio_set_op_attrs(&io
->bio
, REQ_OP_READ
, 0);
253 io
->bio
.bi_iter
.bi_sector
= PTR_OFFSET(&w
->key
, 0);
254 bio_set_dev(&io
->bio
, PTR_CACHE(dc
->disk
.c
, &w
->key
, 0)->bdev
);
255 io
->bio
.bi_end_io
= read_dirty_endio
;
257 if (bio_alloc_pages(&io
->bio
, GFP_KERNEL
))
260 trace_bcache_writeback(&w
->key
);
262 down(&dc
->in_flight
);
263 closure_call(&io
->cl
, read_dirty_submit
, NULL
, &cl
);
265 delay
= writeback_delay(dc
, KEY_SIZE(&w
->key
));
272 bch_keybuf_del(&dc
->writeback_keys
, w
);
276 * Wait for outstanding writeback IOs to finish (and keybuf slots to be
277 * freed) before refilling again
282 /* Scan for dirty data */
284 void bcache_dev_sectors_dirty_add(struct cache_set
*c
, unsigned inode
,
285 uint64_t offset
, int nr_sectors
)
287 struct bcache_device
*d
= c
->devices
[inode
];
288 unsigned stripe_offset
, stripe
, sectors_dirty
;
293 stripe
= offset_to_stripe(d
, offset
);
294 stripe_offset
= offset
& (d
->stripe_size
- 1);
297 int s
= min_t(unsigned, abs(nr_sectors
),
298 d
->stripe_size
- stripe_offset
);
303 if (stripe
>= d
->nr_stripes
)
306 sectors_dirty
= atomic_add_return(s
,
307 d
->stripe_sectors_dirty
+ stripe
);
308 if (sectors_dirty
== d
->stripe_size
)
309 set_bit(stripe
, d
->full_dirty_stripes
);
311 clear_bit(stripe
, d
->full_dirty_stripes
);
319 static bool dirty_pred(struct keybuf
*buf
, struct bkey
*k
)
321 struct cached_dev
*dc
= container_of(buf
, struct cached_dev
, writeback_keys
);
323 BUG_ON(KEY_INODE(k
) != dc
->disk
.id
);
328 static void refill_full_stripes(struct cached_dev
*dc
)
330 struct keybuf
*buf
= &dc
->writeback_keys
;
331 unsigned start_stripe
, stripe
, next_stripe
;
332 bool wrapped
= false;
334 stripe
= offset_to_stripe(&dc
->disk
, KEY_OFFSET(&buf
->last_scanned
));
336 if (stripe
>= dc
->disk
.nr_stripes
)
339 start_stripe
= stripe
;
342 stripe
= find_next_bit(dc
->disk
.full_dirty_stripes
,
343 dc
->disk
.nr_stripes
, stripe
);
345 if (stripe
== dc
->disk
.nr_stripes
)
348 next_stripe
= find_next_zero_bit(dc
->disk
.full_dirty_stripes
,
349 dc
->disk
.nr_stripes
, stripe
);
351 buf
->last_scanned
= KEY(dc
->disk
.id
,
352 stripe
* dc
->disk
.stripe_size
, 0);
354 bch_refill_keybuf(dc
->disk
.c
, buf
,
356 next_stripe
* dc
->disk
.stripe_size
, 0),
359 if (array_freelist_empty(&buf
->freelist
))
362 stripe
= next_stripe
;
364 if (wrapped
&& stripe
> start_stripe
)
367 if (stripe
== dc
->disk
.nr_stripes
) {
375 * Returns true if we scanned the entire disk
377 static bool refill_dirty(struct cached_dev
*dc
)
379 struct keybuf
*buf
= &dc
->writeback_keys
;
380 struct bkey start
= KEY(dc
->disk
.id
, 0, 0);
381 struct bkey end
= KEY(dc
->disk
.id
, MAX_KEY_OFFSET
, 0);
382 struct bkey start_pos
;
385 * make sure keybuf pos is inside the range for this disk - at bringup
386 * we might not be attached yet so this disk's inode nr isn't
389 if (bkey_cmp(&buf
->last_scanned
, &start
) < 0 ||
390 bkey_cmp(&buf
->last_scanned
, &end
) > 0)
391 buf
->last_scanned
= start
;
393 if (dc
->partial_stripes_expensive
) {
394 refill_full_stripes(dc
);
395 if (array_freelist_empty(&buf
->freelist
))
399 start_pos
= buf
->last_scanned
;
400 bch_refill_keybuf(dc
->disk
.c
, buf
, &end
, dirty_pred
);
402 if (bkey_cmp(&buf
->last_scanned
, &end
) < 0)
406 * If we get to the end start scanning again from the beginning, and
407 * only scan up to where we initially started scanning from:
409 buf
->last_scanned
= start
;
410 bch_refill_keybuf(dc
->disk
.c
, buf
, &start_pos
, dirty_pred
);
412 return bkey_cmp(&buf
->last_scanned
, &start_pos
) >= 0;
415 static int bch_writeback_thread(void *arg
)
417 struct cached_dev
*dc
= arg
;
418 bool searched_full_index
;
420 while (!kthread_should_stop()) {
421 down_write(&dc
->writeback_lock
);
422 if (!atomic_read(&dc
->has_dirty
) ||
423 (!test_bit(BCACHE_DEV_DETACHING
, &dc
->disk
.flags
) &&
424 !dc
->writeback_running
)) {
425 up_write(&dc
->writeback_lock
);
426 set_current_state(TASK_INTERRUPTIBLE
);
428 if (kthread_should_stop())
435 searched_full_index
= refill_dirty(dc
);
437 if (searched_full_index
&&
438 RB_EMPTY_ROOT(&dc
->writeback_keys
.keys
)) {
439 atomic_set(&dc
->has_dirty
, 0);
441 SET_BDEV_STATE(&dc
->sb
, BDEV_STATE_CLEAN
);
442 bch_write_bdev_super(dc
, NULL
);
445 up_write(&dc
->writeback_lock
);
447 bch_ratelimit_reset(&dc
->writeback_rate
);
450 if (searched_full_index
) {
451 unsigned delay
= dc
->writeback_delay
* HZ
;
454 !kthread_should_stop() &&
455 !test_bit(BCACHE_DEV_DETACHING
, &dc
->disk
.flags
))
456 delay
= schedule_timeout_interruptible(delay
);
465 struct sectors_dirty_init
{
470 static int sectors_dirty_init_fn(struct btree_op
*_op
, struct btree
*b
,
473 struct sectors_dirty_init
*op
= container_of(_op
,
474 struct sectors_dirty_init
, op
);
475 if (KEY_INODE(k
) > op
->inode
)
479 bcache_dev_sectors_dirty_add(b
->c
, KEY_INODE(k
),
480 KEY_START(k
), KEY_SIZE(k
));
485 void bch_sectors_dirty_init(struct bcache_device
*d
)
487 struct sectors_dirty_init op
;
489 bch_btree_op_init(&op
.op
, -1);
492 bch_btree_map_keys(&op
.op
, d
->c
, &KEY(op
.inode
, 0, 0),
493 sectors_dirty_init_fn
, 0);
495 d
->sectors_dirty_last
= bcache_dev_sectors_dirty(d
);
498 void bch_cached_dev_writeback_init(struct cached_dev
*dc
)
500 sema_init(&dc
->in_flight
, 64);
501 init_rwsem(&dc
->writeback_lock
);
502 bch_keybuf_init(&dc
->writeback_keys
);
504 dc
->writeback_metadata
= true;
505 dc
->writeback_running
= true;
506 dc
->writeback_percent
= 10;
507 dc
->writeback_delay
= 30;
508 dc
->writeback_rate
.rate
= 1024;
510 dc
->writeback_rate_update_seconds
= 5;
511 dc
->writeback_rate_d_term
= 30;
512 dc
->writeback_rate_p_term_inverse
= 6000;
514 INIT_DELAYED_WORK(&dc
->writeback_rate_update
, update_writeback_rate
);
517 int bch_cached_dev_writeback_start(struct cached_dev
*dc
)
519 dc
->writeback_write_wq
= alloc_workqueue("bcache_writeback_wq",
521 if (!dc
->writeback_write_wq
)
524 dc
->writeback_thread
= kthread_create(bch_writeback_thread
, dc
,
526 if (IS_ERR(dc
->writeback_thread
))
527 return PTR_ERR(dc
->writeback_thread
);
529 schedule_delayed_work(&dc
->writeback_rate_update
,
530 dc
->writeback_rate_update_seconds
* HZ
);
532 bch_writeback_queue(dc
);