[mirror_ubuntu-artful-kernel.git] / drivers / md / bcache / writeback.c

/*
 * background writeback - scan btree for dirty data and write it to the backing
 * device
 *
 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
 * Copyright 2012 Google, Inc.
 */

#include "bcache.h"
#include "btree.h"
#include "debug.h"
#include "writeback.h"

#include <linux/delay.h>
#include <linux/kthread.h>
#include <linux/sched/clock.h>
#include <trace/events/bcache.h>

/* Rate limiting */

static void __update_writeback_rate(struct cached_dev *dc)
{
	struct cache_set *c = dc->disk.c;
	uint64_t cache_sectors = c->nbuckets * c->sb.bucket_size;
	uint64_t cache_dirty_target =
		div_u64(cache_sectors * dc->writeback_percent, 100);

	int64_t target = div64_u64(cache_dirty_target * bdev_sectors(dc->bdev),
				   c->cached_dev_sectors);

	/* PD controller */

	int64_t dirty = bcache_dev_sectors_dirty(&dc->disk);
	int64_t derivative = dirty - dc->disk.sectors_dirty_last;
	int64_t proportional = dirty - target;
	int64_t change;

	dc->disk.sectors_dirty_last = dirty;

	/* Scale to sectors per second */

	proportional *= dc->writeback_rate_update_seconds;
	proportional = div_s64(proportional, dc->writeback_rate_p_term_inverse);

	derivative = div_s64(derivative, dc->writeback_rate_update_seconds);

	derivative = ewma_add(dc->disk.sectors_dirty_derivative, derivative,
			      (dc->writeback_rate_d_term /
			       dc->writeback_rate_update_seconds) ?: 1, 0);

	derivative *= dc->writeback_rate_d_term;
	derivative = div_s64(derivative, dc->writeback_rate_p_term_inverse);

	change = proportional + derivative;

	/* Don't increase writeback rate if the device isn't keeping up */
	if (change > 0 &&
	    time_after64(local_clock(),
			 dc->writeback_rate.next + NSEC_PER_MSEC))
		change = 0;

	dc->writeback_rate.rate =
		clamp_t(int64_t, (int64_t) dc->writeback_rate.rate + change,
			1, NSEC_PER_MSEC);

	dc->writeback_rate_proportional = proportional;
	dc->writeback_rate_derivative = derivative;
	dc->writeback_rate_change = change;
	dc->writeback_rate_target = target;
}

static void update_writeback_rate(struct work_struct *work)
{
	struct cached_dev *dc = container_of(to_delayed_work(work),
					     struct cached_dev,
					     writeback_rate_update);

	down_read(&dc->writeback_lock);

	if (atomic_read(&dc->has_dirty) &&
	    dc->writeback_percent)
		__update_writeback_rate(dc);

	up_read(&dc->writeback_lock);

	schedule_delayed_work(&dc->writeback_rate_update,
			      dc->writeback_rate_update_seconds * HZ);
}

static unsigned writeback_delay(struct cached_dev *dc, unsigned sectors)
{
	if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) ||
	    !dc->writeback_percent)
		return 0;

	return bch_next_delay(&dc->writeback_rate, sectors);
}

struct dirty_io {
	struct closure		cl;
	struct cached_dev	*dc;
	struct bio		bio;
};

static void dirty_init(struct keybuf_key *w)
{
	struct dirty_io *io = w->private;
	struct bio *bio = &io->bio;

	bio_init(bio, bio->bi_inline_vecs,
		 DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS));
	if (!io->dc->writeback_percent)
		bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));

	bio->bi_iter.bi_size	= KEY_SIZE(&w->key) << 9;
	bio->bi_private		= w;
	bch_bio_map(bio, NULL);
}

static void dirty_io_destructor(struct closure *cl)
{
	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
	kfree(io);
}

static void write_dirty_finish(struct closure *cl)
{
	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
	struct keybuf_key *w = io->bio.bi_private;
	struct cached_dev *dc = io->dc;

	bio_free_pages(&io->bio);

	/* This is kind of a dumb way of signalling errors. */
	if (KEY_DIRTY(&w->key)) {
		int ret;
		unsigned i;
		struct keylist keys;

		bch_keylist_init(&keys);

		bkey_copy(keys.top, &w->key);
		SET_KEY_DIRTY(keys.top, false);
		bch_keylist_push(&keys);

		for (i = 0; i < KEY_PTRS(&w->key); i++)
			atomic_inc(&PTR_BUCKET(dc->disk.c, &w->key, i)->pin);

		ret = bch_btree_insert(dc->disk.c, &keys, NULL, &w->key);

		if (ret)
			trace_bcache_writeback_collision(&w->key);

		atomic_long_inc(ret
				? &dc->disk.c->writeback_keys_failed
				: &dc->disk.c->writeback_keys_done);
	}

	bch_keybuf_del(&dc->writeback_keys, w);
	up(&dc->in_flight);

	closure_return_with_destructor(cl, dirty_io_destructor);
}

static void dirty_endio(struct bio *bio)
{
	struct keybuf_key *w = bio->bi_private;
	struct dirty_io *io = w->private;

	if (bio->bi_error)
		SET_KEY_DIRTY(&w->key, false);

	closure_put(&io->cl);
}

static void write_dirty(struct closure *cl)
{
	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
	struct keybuf_key *w = io->bio.bi_private;

	dirty_init(w);
	bio_set_op_attrs(&io->bio, REQ_OP_WRITE, 0);
	io->bio.bi_iter.bi_sector = KEY_START(&w->key);
	io->bio.bi_bdev		= io->dc->bdev;
	io->bio.bi_end_io	= dirty_endio;

	closure_bio_submit(&io->bio, cl);

	continue_at(cl, write_dirty_finish, system_wq);
}

static void read_dirty_endio(struct bio *bio)
{
	struct keybuf_key *w = bio->bi_private;
	struct dirty_io *io = w->private;

	bch_count_io_errors(PTR_CACHE(io->dc->disk.c, &w->key, 0),
			    bio->bi_error, "reading dirty data from cache");

	dirty_endio(bio);
}

static void read_dirty_submit(struct closure *cl)
{
	struct dirty_io *io = container_of(cl, struct dirty_io, cl);

	closure_bio_submit(&io->bio, cl);

	continue_at(cl, write_dirty, system_wq);
}

static void read_dirty(struct cached_dev *dc)
{
	unsigned delay = 0;
	struct keybuf_key *w;
	struct dirty_io *io;
	struct closure cl;

	closure_init_stack(&cl);

	/*
	 * XXX: if we error, background writeback just spins. Should use some
	 * mempools.
	 */

	while (!kthread_should_stop()) {

		w = bch_keybuf_next(&dc->writeback_keys);
		if (!w)
			break;

		BUG_ON(ptr_stale(dc->disk.c, &w->key, 0));

		if (KEY_START(&w->key) != dc->last_read ||
		    jiffies_to_msecs(delay) > 50)
			while (!kthread_should_stop() && delay)
				delay = schedule_timeout_interruptible(delay);

		dc->last_read	= KEY_OFFSET(&w->key);

		io = kzalloc(sizeof(struct dirty_io) + sizeof(struct bio_vec)
			     * DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS),
			     GFP_KERNEL);
		if (!io)
			goto err;

		w->private	= io;
		io->dc		= dc;

		dirty_init(w);
		bio_set_op_attrs(&io->bio, REQ_OP_READ, 0);
		io->bio.bi_iter.bi_sector = PTR_OFFSET(&w->key, 0);
		io->bio.bi_bdev		= PTR_CACHE(dc->disk.c,
						    &w->key, 0)->bdev;
		io->bio.bi_end_io	= read_dirty_endio;

		if (bio_alloc_pages(&io->bio, GFP_KERNEL))
			goto err_free;

		trace_bcache_writeback(&w->key);

		down(&dc->in_flight);
		closure_call(&io->cl, read_dirty_submit, NULL, &cl);

		delay = writeback_delay(dc, KEY_SIZE(&w->key));
	}

	if (0) {
err_free:
		kfree(w->private);
err:
		bch_keybuf_del(&dc->writeback_keys, w);
	}

	/*
	 * Wait for outstanding writeback IOs to finish (and keybuf slots to be
	 * freed) before refilling again
	 */
	closure_sync(&cl);
}

/* Scan for dirty data */

void bcache_dev_sectors_dirty_add(struct cache_set *c, unsigned inode,
				  uint64_t offset, int nr_sectors)
{
	struct bcache_device *d = c->devices[inode];
	unsigned stripe_offset, stripe, sectors_dirty;

	if (!d)
		return;

	stripe = offset_to_stripe(d, offset);
	stripe_offset = offset & (d->stripe_size - 1);

	while (nr_sectors) {
		int s = min_t(unsigned, abs(nr_sectors),
			      d->stripe_size - stripe_offset);

		if (nr_sectors < 0)
			s = -s;

		if (stripe >= d->nr_stripes)
			return;

		sectors_dirty = atomic_add_return(s,
					d->stripe_sectors_dirty + stripe);
		if (sectors_dirty == d->stripe_size)
			set_bit(stripe, d->full_dirty_stripes);
		else
			clear_bit(stripe, d->full_dirty_stripes);

		nr_sectors -= s;
		stripe_offset = 0;
		stripe++;
	}
}

static bool dirty_pred(struct keybuf *buf, struct bkey *k)
{
	struct cached_dev *dc = container_of(buf, struct cached_dev, writeback_keys);

	BUG_ON(KEY_INODE(k) != dc->disk.id);

	return KEY_DIRTY(k);
}

static void refill_full_stripes(struct cached_dev *dc)
{
	struct keybuf *buf = &dc->writeback_keys;
	unsigned start_stripe, stripe, next_stripe;
	bool wrapped = false;

	stripe = offset_to_stripe(&dc->disk, KEY_OFFSET(&buf->last_scanned));

	if (stripe >= dc->disk.nr_stripes)
		stripe = 0;

	start_stripe = stripe;

	while (1) {
		stripe = find_next_bit(dc->disk.full_dirty_stripes,
				       dc->disk.nr_stripes, stripe);

		if (stripe == dc->disk.nr_stripes)
			goto next;

		next_stripe = find_next_zero_bit(dc->disk.full_dirty_stripes,
						 dc->disk.nr_stripes, stripe);

		buf->last_scanned = KEY(dc->disk.id,
					stripe * dc->disk.stripe_size, 0);

		bch_refill_keybuf(dc->disk.c, buf,
				  &KEY(dc->disk.id,
				       next_stripe * dc->disk.stripe_size, 0),
				  dirty_pred);

		if (array_freelist_empty(&buf->freelist))
			return;

		stripe = next_stripe;
next:
		if (wrapped && stripe > start_stripe)
			return;

		if (stripe == dc->disk.nr_stripes) {
			stripe = 0;
			wrapped = true;
		}
	}
}

/*
 * Returns true if we scanned the entire disk
 */
static bool refill_dirty(struct cached_dev *dc)
{
	struct keybuf *buf = &dc->writeback_keys;
	struct bkey start = KEY(dc->disk.id, 0, 0);
	struct bkey end = KEY(dc->disk.id, MAX_KEY_OFFSET, 0);
	struct bkey start_pos;

	/*
	 * make sure keybuf pos is inside the range for this disk - at bringup
	 * we might not be attached yet so this disk's inode nr isn't
	 * initialized then
	 */
	if (bkey_cmp(&buf->last_scanned, &start) < 0 ||
	    bkey_cmp(&buf->last_scanned, &end) > 0)
		buf->last_scanned = start;

	if (dc->partial_stripes_expensive) {
		refill_full_stripes(dc);
		if (array_freelist_empty(&buf->freelist))
			return false;
	}

	start_pos = buf->last_scanned;
	bch_refill_keybuf(dc->disk.c, buf, &end, dirty_pred);

	if (bkey_cmp(&buf->last_scanned, &end) < 0)
		return false;

	/*
	 * If we get to the end start scanning again from the beginning, and
	 * only scan up to where we initially started scanning from:
	 */
	buf->last_scanned = start;
	bch_refill_keybuf(dc->disk.c, buf, &start_pos, dirty_pred);

	return bkey_cmp(&buf->last_scanned, &start_pos) >= 0;
}

static int bch_writeback_thread(void *arg)
{
	struct cached_dev *dc = arg;
	bool searched_full_index;

	while (!kthread_should_stop()) {
		down_write(&dc->writeback_lock);
		if (!atomic_read(&dc->has_dirty) ||
		    (!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) &&
		     !dc->writeback_running)) {
			up_write(&dc->writeback_lock);
			set_current_state(TASK_INTERRUPTIBLE);

			if (kthread_should_stop())
				return 0;

			schedule();
			continue;
		}

		searched_full_index = refill_dirty(dc);

		if (searched_full_index &&
		    RB_EMPTY_ROOT(&dc->writeback_keys.keys)) {
			atomic_set(&dc->has_dirty, 0);
			cached_dev_put(dc);
			SET_BDEV_STATE(&dc->sb, BDEV_STATE_CLEAN);
			bch_write_bdev_super(dc, NULL);
		}

		up_write(&dc->writeback_lock);

		bch_ratelimit_reset(&dc->writeback_rate);
		read_dirty(dc);

		if (searched_full_index) {
			unsigned delay = dc->writeback_delay * HZ;

			while (delay &&
			       !kthread_should_stop() &&
			       !test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags))
				delay = schedule_timeout_interruptible(delay);
		}
	}

	return 0;
}

/* Init */

struct sectors_dirty_init {
	struct btree_op	op;
	unsigned	inode;
};

static int sectors_dirty_init_fn(struct btree_op *_op, struct btree *b,
				 struct bkey *k)
{
	struct sectors_dirty_init *op = container_of(_op,
						struct sectors_dirty_init, op);
	if (KEY_INODE(k) > op->inode)
		return MAP_DONE;

	if (KEY_DIRTY(k))
		bcache_dev_sectors_dirty_add(b->c, KEY_INODE(k),
					     KEY_START(k), KEY_SIZE(k));

	return MAP_CONTINUE;
}

void bch_sectors_dirty_init(struct cached_dev *dc)
{
	struct sectors_dirty_init op;

	bch_btree_op_init(&op.op, -1);
	op.inode = dc->disk.id;

	bch_btree_map_keys(&op.op, dc->disk.c, &KEY(op.inode, 0, 0),
			   sectors_dirty_init_fn, 0);

	dc->disk.sectors_dirty_last = bcache_dev_sectors_dirty(&dc->disk);
}

void bch_cached_dev_writeback_init(struct cached_dev *dc)
{
	sema_init(&dc->in_flight, 64);
	init_rwsem(&dc->writeback_lock);
	bch_keybuf_init(&dc->writeback_keys);

	dc->writeback_metadata		= true;
	dc->writeback_running		= true;
	dc->writeback_percent		= 10;
	dc->writeback_delay		= 30;
	dc->writeback_rate.rate		= 1024;

	dc->writeback_rate_update_seconds = 5;
	dc->writeback_rate_d_term	= 30;
	dc->writeback_rate_p_term_inverse = 6000;

	INIT_DELAYED_WORK(&dc->writeback_rate_update, update_writeback_rate);
}

int bch_cached_dev_writeback_start(struct cached_dev *dc)
{
	dc->writeback_thread = kthread_create(bch_writeback_thread, dc,
					      "bcache_writeback");
	if (IS_ERR(dc->writeback_thread))
		return PTR_ERR(dc->writeback_thread);

	schedule_delayed_work(&dc->writeback_rate_update,
			      dc->writeback_rate_update_seconds * HZ);

	bch_writeback_queue(dc);

	return 0;
}
Commit	Line	Data
cafe5635 KO	1	/*
	2	* background writeback - scan btree for dirty data and write it to the backing
	3	* device
	4	*
	5	* Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
	6	* Copyright 2012 Google, Inc.
	7	*/
	8
	9	#include "bcache.h"
	10	#include "btree.h"
	11	#include "debug.h"
279afbad	12	#include "writeback.h"
cafe5635	13
5e6926da	14	#include <linux/delay.h>
5e6926da	15	#include <linux/kthread.h>
e6017571	16	#include <linux/sched/clock.h>
c37511b8 KO	17	#include <trace/events/bcache.h>
c37511b8 KO	18
cafe5635 KO	19	/* Rate limiting */
	20
	21	static void __update_writeback_rate(struct cached_dev *dc)
	22	{
	23	struct cache_set *c = dc->disk.c;
	24	uint64_t cache_sectors = c->nbuckets * c->sb.bucket_size;
	25	uint64_t cache_dirty_target =
	26	div_u64(cache_sectors * dc->writeback_percent, 100);
	27
	28	int64_t target = div64_u64(cache_dirty_target * bdev_sectors(dc->bdev),
	29	c->cached_dev_sectors);
	30
	31	/* PD controller */
	32
279afbad	33	int64_t dirty = bcache_dev_sectors_dirty(&dc->disk);
cafe5635	34	int64_t derivative = dirty - dc->disk.sectors_dirty_last;
16749c23 KO	35	int64_t proportional = dirty - target;
16749c23 KO	36	int64_t change;
cafe5635 KO	37
	38	dc->disk.sectors_dirty_last = dirty;
	39
16749c23	40	/* Scale to sectors per second */
cafe5635	41
16749c23 KO	42	proportional *= dc->writeback_rate_update_seconds;
16749c23 KO	43	proportional = div_s64(proportional, dc->writeback_rate_p_term_inverse);
cafe5635	44
16749c23	45	derivative = div_s64(derivative, dc->writeback_rate_update_seconds);
cafe5635	46
16749c23 KO	47	derivative = ewma_add(dc->disk.sectors_dirty_derivative, derivative,
	48	(dc->writeback_rate_d_term /
	49	dc->writeback_rate_update_seconds) ?: 1, 0);
	50
	51	derivative *= dc->writeback_rate_d_term;
	52	derivative = div_s64(derivative, dc->writeback_rate_p_term_inverse);
cafe5635	53
16749c23	54	change = proportional + derivative;
cafe5635 KO	55
	56	/* Don't increase writeback rate if the device isn't keeping up */
	57	if (change > 0 &&
	58	time_after64(local_clock(),
16749c23	59	dc->writeback_rate.next + NSEC_PER_MSEC))
cafe5635 KO	60	change = 0;
	61
	62	dc->writeback_rate.rate =
16749c23	63	clamp_t(int64_t, (int64_t) dc->writeback_rate.rate + change,
cafe5635	64	1, NSEC_PER_MSEC);
16749c23 KO	65
16749c23 KO	66	dc->writeback_rate_proportional = proportional;
cafe5635 KO	67	dc->writeback_rate_derivative = derivative;
	68	dc->writeback_rate_change = change;
	69	dc->writeback_rate_target = target;
cafe5635 KO	70	}
	71
	72	static void update_writeback_rate(struct work_struct *work)
	73	{
	74	struct cached_dev *dc = container_of(to_delayed_work(work),
	75	struct cached_dev,
	76	writeback_rate_update);
	77
	78	down_read(&dc->writeback_lock);
	79
	80	if (atomic_read(&dc->has_dirty) &&
	81	dc->writeback_percent)
	82	__update_writeback_rate(dc);
	83
	84	up_read(&dc->writeback_lock);
5e6926da KO	85
	86	schedule_delayed_work(&dc->writeback_rate_update,
	87	dc->writeback_rate_update_seconds * HZ);
cafe5635 KO	88	}
	89
	90	static unsigned writeback_delay(struct cached_dev *dc, unsigned sectors)
	91	{
c4d951dd	92	if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) \|\|
cafe5635 KO	93	!dc->writeback_percent)
	94	return 0;
	95
16749c23	96	return bch_next_delay(&dc->writeback_rate, sectors);
cafe5635 KO	97	}
cafe5635 KO	98
5e6926da KO	99	struct dirty_io {
	100	struct closure cl;
	101	struct cached_dev *dc;
	102	struct bio bio;
	103	};
72c27061	104
cafe5635 KO	105	static void dirty_init(struct keybuf_key *w)
	106	{
	107	struct dirty_io *io = w->private;
	108	struct bio *bio = &io->bio;
	109
3a83f467 ML	110	bio_init(bio, bio->bi_inline_vecs,
3a83f467 ML	111	DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS));
cafe5635 KO	112	if (!io->dc->writeback_percent)
	113	bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
	114
4f024f37	115	bio->bi_iter.bi_size = KEY_SIZE(&w->key) << 9;
cafe5635	116	bio->bi_private = w;
169ef1cf	117	bch_bio_map(bio, NULL);
cafe5635 KO	118	}
cafe5635 KO	119
cafe5635 KO	120	static void dirty_io_destructor(struct closure *cl)
	121	{
	122	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
	123	kfree(io);
	124	}
	125
	126	static void write_dirty_finish(struct closure *cl)
	127	{
	128	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
	129	struct keybuf_key *w = io->bio.bi_private;
	130	struct cached_dev *dc = io->dc;
cafe5635	131
491221f8	132	bio_free_pages(&io->bio);
cafe5635 KO	133
	134	/* This is kind of a dumb way of signalling errors. */
	135	if (KEY_DIRTY(&w->key)) {
cc7b8819	136	int ret;
cafe5635	137	unsigned i;
0b93207a KO	138	struct keylist keys;
0b93207a KO	139
0b93207a	140	bch_keylist_init(&keys);
cafe5635	141
1b207d80 KO	142	bkey_copy(keys.top, &w->key);
	143	SET_KEY_DIRTY(keys.top, false);
	144	bch_keylist_push(&keys);
cafe5635 KO	145
	146	for (i = 0; i < KEY_PTRS(&w->key); i++)
	147	atomic_inc(&PTR_BUCKET(dc->disk.c, &w->key, i)->pin);
	148
cc7b8819	149	ret = bch_btree_insert(dc->disk.c, &keys, NULL, &w->key);
cafe5635	150
6054c6d4	151	if (ret)
c37511b8 KO	152	trace_bcache_writeback_collision(&w->key);
c37511b8 KO	153
6054c6d4	154	atomic_long_inc(ret
cafe5635 KO	155	? &dc->disk.c->writeback_keys_failed
	156	: &dc->disk.c->writeback_keys_done);
	157	}
	158
	159	bch_keybuf_del(&dc->writeback_keys, w);
c2a4f318	160	up(&dc->in_flight);
cafe5635 KO	161
	162	closure_return_with_destructor(cl, dirty_io_destructor);
	163	}
	164
4246a0b6	165	static void dirty_endio(struct bio *bio)
cafe5635 KO	166	{
	167	struct keybuf_key *w = bio->bi_private;
	168	struct dirty_io *io = w->private;
	169
4246a0b6	170	if (bio->bi_error)
cafe5635 KO	171	SET_KEY_DIRTY(&w->key, false);
	172
	173	closure_put(&io->cl);
	174	}
	175
	176	static void write_dirty(struct closure *cl)
	177	{
	178	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
	179	struct keybuf_key *w = io->bio.bi_private;
	180
	181	dirty_init(w);
ad0d9e76	182	bio_set_op_attrs(&io->bio, REQ_OP_WRITE, 0);
4f024f37	183	io->bio.bi_iter.bi_sector = KEY_START(&w->key);
cafe5635 KO	184	io->bio.bi_bdev = io->dc->bdev;
	185	io->bio.bi_end_io = dirty_endio;
	186
749b61da	187	closure_bio_submit(&io->bio, cl);
cafe5635	188
c2a4f318	189	continue_at(cl, write_dirty_finish, system_wq);
cafe5635 KO	190	}
cafe5635 KO	191
4246a0b6	192	static void read_dirty_endio(struct bio *bio)
cafe5635 KO	193	{
	194	struct keybuf_key *w = bio->bi_private;
	195	struct dirty_io *io = w->private;
	196
	197	bch_count_io_errors(PTR_CACHE(io->dc->disk.c, &w->key, 0),
4246a0b6	198	bio->bi_error, "reading dirty data from cache");
cafe5635	199
4246a0b6	200	dirty_endio(bio);
cafe5635 KO	201	}
	202
	203	static void read_dirty_submit(struct closure *cl)
	204	{
	205	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
	206
749b61da	207	closure_bio_submit(&io->bio, cl);
cafe5635	208
c2a4f318	209	continue_at(cl, write_dirty, system_wq);
cafe5635 KO	210	}
cafe5635 KO	211
5e6926da	212	static void read_dirty(struct cached_dev *dc)
cafe5635	213	{
5e6926da	214	unsigned delay = 0;
cafe5635 KO	215	struct keybuf_key *w;
cafe5635 KO	216	struct dirty_io *io;
5e6926da KO	217	struct closure cl;
	218
	219	closure_init_stack(&cl);
cafe5635 KO	220
	221	/*
	222	* XXX: if we error, background writeback just spins. Should use some
	223	* mempools.
	224	*/
	225
5e6926da	226	while (!kthread_should_stop()) {
5e6926da	227
cafe5635 KO	228	w = bch_keybuf_next(&dc->writeback_keys);
	229	if (!w)
	230	break;
	231
	232	BUG_ON(ptr_stale(dc->disk.c, &w->key, 0));
	233
5e6926da KO	234	if (KEY_START(&w->key) != dc->last_read \|\|
	235	jiffies_to_msecs(delay) > 50)
	236	while (!kthread_should_stop() && delay)
9e5c3535	237	delay = schedule_timeout_interruptible(delay);
cafe5635 KO	238
	239	dc->last_read = KEY_OFFSET(&w->key);
	240
	241	io = kzalloc(sizeof(struct dirty_io) + sizeof(struct bio_vec)
	242	* DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS),
	243	GFP_KERNEL);
	244	if (!io)
	245	goto err;
	246
	247	w->private = io;
	248	io->dc = dc;
	249
	250	dirty_init(w);
ad0d9e76	251	bio_set_op_attrs(&io->bio, REQ_OP_READ, 0);
4f024f37	252	io->bio.bi_iter.bi_sector = PTR_OFFSET(&w->key, 0);
cafe5635 KO	253	io->bio.bi_bdev = PTR_CACHE(dc->disk.c,
cafe5635 KO	254	&w->key, 0)->bdev;
cafe5635 KO	255	io->bio.bi_end_io = read_dirty_endio;
cafe5635 KO	256
8e51e414	257	if (bio_alloc_pages(&io->bio, GFP_KERNEL))
cafe5635 KO	258	goto err_free;
cafe5635 KO	259
c37511b8	260	trace_bcache_writeback(&w->key);
cafe5635	261
c2a4f318	262	down(&dc->in_flight);
5e6926da	263	closure_call(&io->cl, read_dirty_submit, NULL, &cl);
cafe5635 KO	264
cafe5635 KO	265	delay = writeback_delay(dc, KEY_SIZE(&w->key));
cafe5635 KO	266	}
	267
	268	if (0) {
	269	err_free:
	270	kfree(w->private);
	271	err:
	272	bch_keybuf_del(&dc->writeback_keys, w);
	273	}
	274
c2a4f318 KO	275	/*
	276	* Wait for outstanding writeback IOs to finish (and keybuf slots to be
	277	* freed) before refilling again
	278	*/
5e6926da KO	279	closure_sync(&cl);
	280	}
	281
	282	/* Scan for dirty data */
	283
	284	void bcache_dev_sectors_dirty_add(struct cache_set *c, unsigned inode,
	285	uint64_t offset, int nr_sectors)
	286	{
	287	struct bcache_device *d = c->devices[inode];
48a915a8	288	unsigned stripe_offset, stripe, sectors_dirty;
5e6926da KO	289
	290	if (!d)
	291	return;
	292
48a915a8	293	stripe = offset_to_stripe(d, offset);
5e6926da KO	294	stripe_offset = offset & (d->stripe_size - 1);
	295
	296	while (nr_sectors) {
	297	int s = min_t(unsigned, abs(nr_sectors),
	298	d->stripe_size - stripe_offset);
	299
	300	if (nr_sectors < 0)
	301	s = -s;
	302
48a915a8 KO	303	if (stripe >= d->nr_stripes)
	304	return;
	305
	306	sectors_dirty = atomic_add_return(s,
	307	d->stripe_sectors_dirty + stripe);
	308	if (sectors_dirty == d->stripe_size)
	309	set_bit(stripe, d->full_dirty_stripes);
	310	else
	311	clear_bit(stripe, d->full_dirty_stripes);
	312
5e6926da KO	313	nr_sectors -= s;
	314	stripe_offset = 0;
	315	stripe++;
	316	}
	317	}
	318
	319	static bool dirty_pred(struct keybuf buf, struct bkey k)
	320	{
627ccd20 KO	321	struct cached_dev *dc = container_of(buf, struct cached_dev, writeback_keys);
	322
	323	BUG_ON(KEY_INODE(k) != dc->disk.id);
	324
5e6926da KO	325	return KEY_DIRTY(k);
	326	}
	327
48a915a8	328	static void refill_full_stripes(struct cached_dev *dc)
5e6926da	329	{
48a915a8 KO	330	struct keybuf *buf = &dc->writeback_keys;
	331	unsigned start_stripe, stripe, next_stripe;
	332	bool wrapped = false;
	333
	334	stripe = offset_to_stripe(&dc->disk, KEY_OFFSET(&buf->last_scanned));
5e6926da	335
48a915a8 KO	336	if (stripe >= dc->disk.nr_stripes)
48a915a8 KO	337	stripe = 0;
5e6926da	338
48a915a8	339	start_stripe = stripe;
5e6926da KO	340
5e6926da KO	341	while (1) {
48a915a8 KO	342	stripe = find_next_bit(dc->disk.full_dirty_stripes,
48a915a8 KO	343	dc->disk.nr_stripes, stripe);
5e6926da	344
48a915a8 KO	345	if (stripe == dc->disk.nr_stripes)
48a915a8 KO	346	goto next;
5e6926da	347
48a915a8 KO	348	next_stripe = find_next_zero_bit(dc->disk.full_dirty_stripes,
	349	dc->disk.nr_stripes, stripe);
	350
	351	buf->last_scanned = KEY(dc->disk.id,
	352	stripe * dc->disk.stripe_size, 0);
	353
	354	bch_refill_keybuf(dc->disk.c, buf,
	355	&KEY(dc->disk.id,
	356	next_stripe * dc->disk.stripe_size, 0),
	357	dirty_pred);
	358
	359	if (array_freelist_empty(&buf->freelist))
	360	return;
	361
	362	stripe = next_stripe;
	363	next:
	364	if (wrapped && stripe > start_stripe)
	365	return;
	366
	367	if (stripe == dc->disk.nr_stripes) {
	368	stripe = 0;
	369	wrapped = true;
	370	}
5e6926da KO	371	}
	372	}
	373
627ccd20 KO	374	/*
	375	* Returns true if we scanned the entire disk
	376	*/
5e6926da KO	377	static bool refill_dirty(struct cached_dev *dc)
	378	{
	379	struct keybuf *buf = &dc->writeback_keys;
627ccd20	380	struct bkey start = KEY(dc->disk.id, 0, 0);
5e6926da	381	struct bkey end = KEY(dc->disk.id, MAX_KEY_OFFSET, 0);
627ccd20 KO	382	struct bkey start_pos;
	383
	384	/*
	385	* make sure keybuf pos is inside the range for this disk - at bringup
	386	* we might not be attached yet so this disk's inode nr isn't
	387	* initialized then
	388	*/
	389	if (bkey_cmp(&buf->last_scanned, &start) < 0 \|\|
	390	bkey_cmp(&buf->last_scanned, &end) > 0)
	391	buf->last_scanned = start;
48a915a8 KO	392
	393	if (dc->partial_stripes_expensive) {
	394	refill_full_stripes(dc);
	395	if (array_freelist_empty(&buf->freelist))
	396	return false;
	397	}
5e6926da	398
627ccd20	399	start_pos = buf->last_scanned;
48a915a8	400	bch_refill_keybuf(dc->disk.c, buf, &end, dirty_pred);
5e6926da	401
627ccd20 KO	402	if (bkey_cmp(&buf->last_scanned, &end) < 0)
	403	return false;
	404
	405	/*
	406	* If we get to the end start scanning again from the beginning, and
	407	* only scan up to where we initially started scanning from:
	408	*/
	409	buf->last_scanned = start;
	410	bch_refill_keybuf(dc->disk.c, buf, &start_pos, dirty_pred);
	411
	412	return bkey_cmp(&buf->last_scanned, &start_pos) >= 0;
5e6926da KO	413	}
	414
	415	static int bch_writeback_thread(void *arg)
	416	{
	417	struct cached_dev *dc = arg;
	418	bool searched_full_index;
	419
	420	while (!kthread_should_stop()) {
	421	down_write(&dc->writeback_lock);
	422	if (!atomic_read(&dc->has_dirty) \|\|
c4d951dd	423	(!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) &&
5e6926da KO	424	!dc->writeback_running)) {
	425	up_write(&dc->writeback_lock);
	426	set_current_state(TASK_INTERRUPTIBLE);
	427
	428	if (kthread_should_stop())
	429	return 0;
	430
5e6926da KO	431	schedule();
	432	continue;
	433	}
	434
	435	searched_full_index = refill_dirty(dc);
	436
	437	if (searched_full_index &&
	438	RB_EMPTY_ROOT(&dc->writeback_keys.keys)) {
	439	atomic_set(&dc->has_dirty, 0);
	440	cached_dev_put(dc);
	441	SET_BDEV_STATE(&dc->sb, BDEV_STATE_CLEAN);
	442	bch_write_bdev_super(dc, NULL);
	443	}
	444
	445	up_write(&dc->writeback_lock);
	446
	447	bch_ratelimit_reset(&dc->writeback_rate);
	448	read_dirty(dc);
	449
	450	if (searched_full_index) {
	451	unsigned delay = dc->writeback_delay * HZ;
	452
	453	while (delay &&
	454	!kthread_should_stop() &&
c4d951dd	455	!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags))
9e5c3535	456	delay = schedule_timeout_interruptible(delay);
5e6926da KO	457	}
	458	}
	459
	460	return 0;
cafe5635 KO	461	}
cafe5635 KO	462
444fc0b6 KO	463	/* Init */
444fc0b6 KO	464
c18536a7 KO	465	struct sectors_dirty_init {
	466	struct btree_op op;
	467	unsigned inode;
	468	};
	469
	470	static int sectors_dirty_init_fn(struct btree_op _op, struct btree b,
48dad8ba	471	struct bkey *k)
444fc0b6	472	{
c18536a7 KO	473	struct sectors_dirty_init *op = container_of(_op,
c18536a7 KO	474	struct sectors_dirty_init, op);
48dad8ba KO	475	if (KEY_INODE(k) > op->inode)
48dad8ba KO	476	return MAP_DONE;
444fc0b6	477
48dad8ba KO	478	if (KEY_DIRTY(k))
	479	bcache_dev_sectors_dirty_add(b->c, KEY_INODE(k),
	480	KEY_START(k), KEY_SIZE(k));
	481
	482	return MAP_CONTINUE;
444fc0b6 KO	483	}
	484
	485	void bch_sectors_dirty_init(struct cached_dev *dc)
	486	{
c18536a7	487	struct sectors_dirty_init op;
444fc0b6	488
b54d6934	489	bch_btree_op_init(&op.op, -1);
48dad8ba KO	490	op.inode = dc->disk.id;
48dad8ba KO	491
c18536a7	492	bch_btree_map_keys(&op.op, dc->disk.c, &KEY(op.inode, 0, 0),
48dad8ba	493	sectors_dirty_init_fn, 0);
16749c23 KO	494
16749c23 KO	495	dc->disk.sectors_dirty_last = bcache_dev_sectors_dirty(&dc->disk);
444fc0b6 KO	496	}
444fc0b6 KO	497
9e5c3535	498	void bch_cached_dev_writeback_init(struct cached_dev *dc)
cafe5635	499	{
c2a4f318	500	sema_init(&dc->in_flight, 64);
cafe5635	501	init_rwsem(&dc->writeback_lock);
72c27061	502	bch_keybuf_init(&dc->writeback_keys);
cafe5635 KO	503
	504	dc->writeback_metadata = true;
	505	dc->writeback_running = true;
	506	dc->writeback_percent = 10;
	507	dc->writeback_delay = 30;
	508	dc->writeback_rate.rate = 1024;
	509
16749c23 KO	510	dc->writeback_rate_update_seconds = 5;
	511	dc->writeback_rate_d_term = 30;
	512	dc->writeback_rate_p_term_inverse = 6000;
cafe5635	513
9e5c3535 SP	514	INIT_DELAYED_WORK(&dc->writeback_rate_update, update_writeback_rate);
	515	}
	516
	517	int bch_cached_dev_writeback_start(struct cached_dev *dc)
	518	{
5e6926da KO	519	dc->writeback_thread = kthread_create(bch_writeback_thread, dc,
	520	"bcache_writeback");
	521	if (IS_ERR(dc->writeback_thread))
	522	return PTR_ERR(dc->writeback_thread);
	523
cafe5635 KO	524	schedule_delayed_work(&dc->writeback_rate_update,
cafe5635 KO	525	dc->writeback_rate_update_seconds * HZ);
cafe5635	526
9e5c3535 SP	527	bch_writeback_queue(dc);
9e5c3535 SP	528
cafe5635 KO	529	return 0;
cafe5635 KO	530	}