[mirror_ubuntu-artful-kernel.git] / include / linux / backing-dev.h

/*
 * include/linux/backing-dev.h
 *
 * low-level device information and state which is propagated up through
 * to high-level code.
 */

#ifndef _LINUX_BACKING_DEV_H
#define _LINUX_BACKING_DEV_H

#include <linux/kernel.h>
#include <linux/fs.h>
#include <linux/sched.h>
#include <linux/blkdev.h>
#include <linux/writeback.h>
#include <linux/blk-cgroup.h>
#include <linux/backing-dev-defs.h>
#include <linux/slab.h>

int __must_check bdi_init(struct backing_dev_info *bdi);

static inline struct backing_dev_info *bdi_get(struct backing_dev_info *bdi)
{
	kref_get(&bdi->refcnt);
	return bdi;
}

void bdi_put(struct backing_dev_info *bdi);

__printf(3, 4)
int bdi_register(struct backing_dev_info *bdi, struct device *parent,
		const char *fmt, ...);
int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev);
int bdi_register_owner(struct backing_dev_info *bdi, struct device *owner);
void bdi_unregister(struct backing_dev_info *bdi);

int __must_check bdi_setup_and_register(struct backing_dev_info *, char *);
void bdi_destroy(struct backing_dev_info *bdi);
struct backing_dev_info *bdi_alloc_node(gfp_t gfp_mask, int node_id);

void wb_start_writeback(struct bdi_writeback *wb, long nr_pages,
			bool range_cyclic, enum wb_reason reason);
void wb_start_background_writeback(struct bdi_writeback *wb);
void wb_workfn(struct work_struct *work);
void wb_wakeup_delayed(struct bdi_writeback *wb);

extern spinlock_t bdi_lock;
extern struct list_head bdi_list;

extern struct workqueue_struct *bdi_wq;

static inline bool wb_has_dirty_io(struct bdi_writeback *wb)
{
	return test_bit(WB_has_dirty_io, &wb->state);
}

static inline bool bdi_has_dirty_io(struct backing_dev_info *bdi)
{
	/*
	 * @bdi->tot_write_bandwidth is guaranteed to be > 0 if there are
	 * any dirty wbs.  See wb_update_write_bandwidth().
	 */
	return atomic_long_read(&bdi->tot_write_bandwidth);
}

static inline void __add_wb_stat(struct bdi_writeback *wb,
				 enum wb_stat_item item, s64 amount)
{
	__percpu_counter_add(&wb->stat[item], amount, WB_STAT_BATCH);
}

static inline void __inc_wb_stat(struct bdi_writeback *wb,
				 enum wb_stat_item item)
{
	__add_wb_stat(wb, item, 1);
}

static inline void inc_wb_stat(struct bdi_writeback *wb, enum wb_stat_item item)
{
	unsigned long flags;

	local_irq_save(flags);
	__inc_wb_stat(wb, item);
	local_irq_restore(flags);
}

static inline void __dec_wb_stat(struct bdi_writeback *wb,
				 enum wb_stat_item item)
{
	__add_wb_stat(wb, item, -1);
}

static inline void dec_wb_stat(struct bdi_writeback *wb, enum wb_stat_item item)
{
	unsigned long flags;

	local_irq_save(flags);
	__dec_wb_stat(wb, item);
	local_irq_restore(flags);
}

static inline s64 wb_stat(struct bdi_writeback *wb, enum wb_stat_item item)
{
	return percpu_counter_read_positive(&wb->stat[item]);
}

static inline s64 __wb_stat_sum(struct bdi_writeback *wb,
				enum wb_stat_item item)
{
	return percpu_counter_sum_positive(&wb->stat[item]);
}

static inline s64 wb_stat_sum(struct bdi_writeback *wb, enum wb_stat_item item)
{
	s64 sum;
	unsigned long flags;

	local_irq_save(flags);
	sum = __wb_stat_sum(wb, item);
	local_irq_restore(flags);

	return sum;
}

extern void wb_writeout_inc(struct bdi_writeback *wb);

/*
 * maximal error of a stat counter.
 */
static inline unsigned long wb_stat_error(struct bdi_writeback *wb)
{
#ifdef CONFIG_SMP
	return nr_cpu_ids * WB_STAT_BATCH;
#else
	return 1;
#endif
}

int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio);
int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio);

/*
 * Flags in backing_dev_info::capability
 *
 * The first three flags control whether dirty pages will contribute to the
 * VM's accounting and whether writepages() should be called for dirty pages
 * (something that would not, for example, be appropriate for ramfs)
 *
 * WARNING: these flags are closely related and should not normally be
 * used separately.  The BDI_CAP_NO_ACCT_AND_WRITEBACK combines these
 * three flags into a single convenience macro.
 *
 * BDI_CAP_NO_ACCT_DIRTY:  Dirty pages shouldn't contribute to accounting
 * BDI_CAP_NO_WRITEBACK:   Don't write pages back
 * BDI_CAP_NO_ACCT_WB:     Don't automatically account writeback pages
 * BDI_CAP_STRICTLIMIT:    Keep number of dirty pages below bdi threshold.
 *
 * BDI_CAP_CGROUP_WRITEBACK: Supports cgroup-aware writeback.
 */
#define BDI_CAP_NO_ACCT_DIRTY	0x00000001
#define BDI_CAP_NO_WRITEBACK	0x00000002
#define BDI_CAP_NO_ACCT_WB	0x00000004
#define BDI_CAP_STABLE_WRITES	0x00000008
#define BDI_CAP_STRICTLIMIT	0x00000010
#define BDI_CAP_CGROUP_WRITEBACK 0x00000020

#define BDI_CAP_NO_ACCT_AND_WRITEBACK \
	(BDI_CAP_NO_WRITEBACK | BDI_CAP_NO_ACCT_DIRTY | BDI_CAP_NO_ACCT_WB)

extern struct backing_dev_info noop_backing_dev_info;

/**
 * writeback_in_progress - determine whether there is writeback in progress
 * @wb: bdi_writeback of interest
 *
 * Determine whether there is writeback waiting to be handled against a
 * bdi_writeback.
 */
static inline bool writeback_in_progress(struct bdi_writeback *wb)
{
	return test_bit(WB_writeback_running, &wb->state);
}

static inline struct backing_dev_info *inode_to_bdi(struct inode *inode)
{
	struct super_block *sb;

	if (!inode)
		return &noop_backing_dev_info;

	sb = inode->i_sb;
#ifdef CONFIG_BLOCK
	if (sb_is_blkdev_sb(sb))
		return I_BDEV(inode)->bd_bdi;
#endif
	return sb->s_bdi;
}

static inline int wb_congested(struct bdi_writeback *wb, int cong_bits)
{
	struct backing_dev_info *bdi = wb->bdi;

	if (bdi->congested_fn)
		return bdi->congested_fn(bdi->congested_data, cong_bits);
	return wb->congested->state & cong_bits;
}

long congestion_wait(int sync, long timeout);
long wait_iff_congested(struct pglist_data *pgdat, int sync, long timeout);
int pdflush_proc_obsolete(struct ctl_table *table, int write,
		void __user *buffer, size_t *lenp, loff_t *ppos);

static inline bool bdi_cap_stable_pages_required(struct backing_dev_info *bdi)
{
	return bdi->capabilities & BDI_CAP_STABLE_WRITES;
}

static inline bool bdi_cap_writeback_dirty(struct backing_dev_info *bdi)
{
	return !(bdi->capabilities & BDI_CAP_NO_WRITEBACK);
}

static inline bool bdi_cap_account_dirty(struct backing_dev_info *bdi)
{
	return !(bdi->capabilities & BDI_CAP_NO_ACCT_DIRTY);
}

static inline bool bdi_cap_account_writeback(struct backing_dev_info *bdi)
{
	/* Paranoia: BDI_CAP_NO_WRITEBACK implies BDI_CAP_NO_ACCT_WB */
	return !(bdi->capabilities & (BDI_CAP_NO_ACCT_WB |
				      BDI_CAP_NO_WRITEBACK));
}

static inline bool mapping_cap_writeback_dirty(struct address_space *mapping)
{
	return bdi_cap_writeback_dirty(inode_to_bdi(mapping->host));
}

static inline bool mapping_cap_account_dirty(struct address_space *mapping)
{
	return bdi_cap_account_dirty(inode_to_bdi(mapping->host));
}

static inline int bdi_sched_wait(void *word)
{
	schedule();
	return 0;
}

#ifdef CONFIG_CGROUP_WRITEBACK

struct bdi_writeback_congested *
wb_congested_get_create(struct backing_dev_info *bdi, int blkcg_id, gfp_t gfp);
void wb_congested_put(struct bdi_writeback_congested *congested);
struct bdi_writeback *wb_get_create(struct backing_dev_info *bdi,
				    struct cgroup_subsys_state *memcg_css,
				    gfp_t gfp);
void wb_memcg_offline(struct mem_cgroup *memcg);
void wb_blkcg_offline(struct blkcg *blkcg);
int inode_congested(struct inode *inode, int cong_bits);

/**
 * inode_cgwb_enabled - test whether cgroup writeback is enabled on an inode
 * @inode: inode of interest
 *
 * cgroup writeback requires support from both the bdi and filesystem.
 * Also, both memcg and iocg have to be on the default hierarchy.  Test
 * whether all conditions are met.
 *
 * Note that the test result may change dynamically on the same inode
 * depending on how memcg and iocg are configured.
 */
static inline bool inode_cgwb_enabled(struct inode *inode)
{
	struct backing_dev_info *bdi = inode_to_bdi(inode);

	return cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
		cgroup_subsys_on_dfl(io_cgrp_subsys) &&
		bdi_cap_account_dirty(bdi) &&
		(bdi->capabilities & BDI_CAP_CGROUP_WRITEBACK) &&
		(inode->i_sb->s_iflags & SB_I_CGROUPWB);
}

/**
 * wb_find_current - find wb for %current on a bdi
 * @bdi: bdi of interest
 *
 * Find the wb of @bdi which matches both the memcg and blkcg of %current.
 * Must be called under rcu_read_lock() which protects the returend wb.
 * NULL if not found.
 */
static inline struct bdi_writeback *wb_find_current(struct backing_dev_info *bdi)
{
	struct cgroup_subsys_state *memcg_css;
	struct bdi_writeback *wb;

	memcg_css = task_css(current, memory_cgrp_id);
	if (!memcg_css->parent)
		return &bdi->wb;

	wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);

	/*
	 * %current's blkcg equals the effective blkcg of its memcg.  No
	 * need to use the relatively expensive cgroup_get_e_css().
	 */
	if (likely(wb && wb->blkcg_css == task_css(current, io_cgrp_id)))
		return wb;
	return NULL;
}

/**
 * wb_get_create_current - get or create wb for %current on a bdi
 * @bdi: bdi of interest
 * @gfp: allocation mask
 *
 * Equivalent to wb_get_create() on %current's memcg.  This function is
 * called from a relatively hot path and optimizes the common cases using
 * wb_find_current().
 */
static inline struct bdi_writeback *
wb_get_create_current(struct backing_dev_info *bdi, gfp_t gfp)
{
	struct bdi_writeback *wb;

	rcu_read_lock();
	wb = wb_find_current(bdi);
	if (wb && unlikely(!wb_tryget(wb)))
		wb = NULL;
	rcu_read_unlock();

	if (unlikely(!wb)) {
		struct cgroup_subsys_state *memcg_css;

		memcg_css = task_get_css(current, memory_cgrp_id);
		wb = wb_get_create(bdi, memcg_css, gfp);
		css_put(memcg_css);
	}
	return wb;
}

/**
 * inode_to_wb_is_valid - test whether an inode has a wb associated
 * @inode: inode of interest
 *
 * Returns %true if @inode has a wb associated.  May be called without any
 * locking.
 */
static inline bool inode_to_wb_is_valid(struct inode *inode)
{
	return inode->i_wb;
}

/**
 * inode_to_wb - determine the wb of an inode
 * @inode: inode of interest
 *
 * Returns the wb @inode is currently associated with.  The caller must be
 * holding either @inode->i_lock, @inode->i_mapping->tree_lock, or the
 * associated wb's list_lock.
 */
static inline struct bdi_writeback *inode_to_wb(struct inode *inode)
{
#ifdef CONFIG_LOCKDEP
	WARN_ON_ONCE(debug_locks &&
		     (!lockdep_is_held(&inode->i_lock) &&
		      !lockdep_is_held(&inode->i_mapping->tree_lock) &&
		      !lockdep_is_held(&inode->i_wb->list_lock)));
#endif
	return inode->i_wb;
}

/**
 * unlocked_inode_to_wb_begin - begin unlocked inode wb access transaction
 * @inode: target inode
 * @lockedp: temp bool output param, to be passed to the end function
 *
 * The caller wants to access the wb associated with @inode but isn't
 * holding inode->i_lock, mapping->tree_lock or wb->list_lock.  This
 * function determines the wb associated with @inode and ensures that the
 * association doesn't change until the transaction is finished with
 * unlocked_inode_to_wb_end().
 *
 * The caller must call unlocked_inode_to_wb_end() with *@lockdep
 * afterwards and can't sleep during transaction.  IRQ may or may not be
 * disabled on return.
 */
static inline struct bdi_writeback *
unlocked_inode_to_wb_begin(struct inode *inode, bool *lockedp)
{
	rcu_read_lock();

	/*
	 * Paired with store_release in inode_switch_wb_work_fn() and
	 * ensures that we see the new wb if we see cleared I_WB_SWITCH.
	 */
	*lockedp = smp_load_acquire(&inode->i_state) & I_WB_SWITCH;

	if (unlikely(*lockedp))
		spin_lock_irq(&inode->i_mapping->tree_lock);

	/*
	 * Protected by either !I_WB_SWITCH + rcu_read_lock() or tree_lock.
	 * inode_to_wb() will bark.  Deref directly.
	 */
	return inode->i_wb;
}

/**
 * unlocked_inode_to_wb_end - end inode wb access transaction
 * @inode: target inode
 * @locked: *@lockedp from unlocked_inode_to_wb_begin()
 */
static inline void unlocked_inode_to_wb_end(struct inode *inode, bool locked)
{
	if (unlikely(locked))
		spin_unlock_irq(&inode->i_mapping->tree_lock);

	rcu_read_unlock();
}

#else	/* CONFIG_CGROUP_WRITEBACK */

static inline bool inode_cgwb_enabled(struct inode *inode)
{
	return false;
}

static inline struct bdi_writeback_congested *
wb_congested_get_create(struct backing_dev_info *bdi, int blkcg_id, gfp_t gfp)
{
	atomic_inc(&bdi->wb_congested->refcnt);
	return bdi->wb_congested;
}

static inline void wb_congested_put(struct bdi_writeback_congested *congested)
{
	if (atomic_dec_and_test(&congested->refcnt))
		kfree(congested);
}

static inline struct bdi_writeback *wb_find_current(struct backing_dev_info *bdi)
{
	return &bdi->wb;
}

static inline struct bdi_writeback *
wb_get_create_current(struct backing_dev_info *bdi, gfp_t gfp)
{
	return &bdi->wb;
}

static inline bool inode_to_wb_is_valid(struct inode *inode)
{
	return true;
}

static inline struct bdi_writeback *inode_to_wb(struct inode *inode)
{
	return &inode_to_bdi(inode)->wb;
}

static inline struct bdi_writeback *
unlocked_inode_to_wb_begin(struct inode *inode, bool *lockedp)
{
	return inode_to_wb(inode);
}

static inline void unlocked_inode_to_wb_end(struct inode *inode, bool locked)
{
}

static inline void wb_memcg_offline(struct mem_cgroup *memcg)
{
}

static inline void wb_blkcg_offline(struct blkcg *blkcg)
{
}

static inline int inode_congested(struct inode *inode, int cong_bits)
{
	return wb_congested(&inode_to_bdi(inode)->wb, cong_bits);
}

#endif	/* CONFIG_CGROUP_WRITEBACK */

static inline int inode_read_congested(struct inode *inode)
{
	return inode_congested(inode, 1 << WB_sync_congested);
}

static inline int inode_write_congested(struct inode *inode)
{
	return inode_congested(inode, 1 << WB_async_congested);
}

static inline int inode_rw_congested(struct inode *inode)
{
	return inode_congested(inode, (1 << WB_sync_congested) |
				      (1 << WB_async_congested));
}

static inline int bdi_congested(struct backing_dev_info *bdi, int cong_bits)
{
	return wb_congested(&bdi->wb, cong_bits);
}

static inline int bdi_read_congested(struct backing_dev_info *bdi)
{
	return bdi_congested(bdi, 1 << WB_sync_congested);
}

static inline int bdi_write_congested(struct backing_dev_info *bdi)
{
	return bdi_congested(bdi, 1 << WB_async_congested);
}

static inline int bdi_rw_congested(struct backing_dev_info *bdi)
{
	return bdi_congested(bdi, (1 << WB_sync_congested) |
				  (1 << WB_async_congested));
}

#endif	/* _LINUX_BACKING_DEV_H */
Commit	Line	Data
1da177e4 LT	1	/*
	2	* include/linux/backing-dev.h
	3	*
	4	* low-level device information and state which is propagated up through
	5	* to high-level code.
	6	*/
	7
	8	#ifndef _LINUX_BACKING_DEV_H
	9	#define _LINUX_BACKING_DEV_H
	10
cf0ca9fe	11	#include <linux/kernel.h>
e4ad08fe	12	#include <linux/fs.h>
03ba3782	13	#include <linux/sched.h>
a212b105	14	#include <linux/blkdev.h>
03ba3782	15	#include <linux/writeback.h>
52ebea74	16	#include <linux/blk-cgroup.h>
66114cad	17	#include <linux/backing-dev-defs.h>
a13f35e8	18	#include <linux/slab.h>
de1414a6	19
8077c0d9	20	int __must_check bdi_init(struct backing_dev_info *bdi);
d03f6cdc JK	21
	22	static inline struct backing_dev_info bdi_get(struct backing_dev_info bdi)
	23	{
	24	kref_get(&bdi->refcnt);
	25	return bdi;
	26	}
	27
	28	void bdi_put(struct backing_dev_info *bdi);
b2e8fb6e	29
d2cc4dde	30	__printf(3, 4)
cf0ca9fe PZ	31	int bdi_register(struct backing_dev_info bdi, struct device parent,
	32	const char *fmt, ...);
	33	int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev);
df08c32c	34	int bdi_register_owner(struct backing_dev_info bdi, struct device owner);
b02176f3 TH	35	void bdi_unregister(struct backing_dev_info *bdi);
b02176f3 TH	36
b4caecd4	37	int __must_check bdi_setup_and_register(struct backing_dev_info , char );
b02176f3	38	void bdi_destroy(struct backing_dev_info *bdi);
d03f6cdc	39	struct backing_dev_info *bdi_alloc_node(gfp_t gfp_mask, int node_id);
b02176f3	40
c00ddad3 TH	41	void wb_start_writeback(struct bdi_writeback *wb, long nr_pages,
c00ddad3 TH	42	bool range_cyclic, enum wb_reason reason);
9ecf4866	43	void wb_start_background_writeback(struct bdi_writeback *wb);
f0054bb1	44	void wb_workfn(struct work_struct *work);
f0054bb1	45	void wb_wakeup_delayed(struct bdi_writeback *wb);
cf0ca9fe	46
03ba3782	47	extern spinlock_t bdi_lock;
66f3b8e2 JA	48	extern struct list_head bdi_list;
66f3b8e2 JA	49
839a8e86 TH	50	extern struct workqueue_struct *bdi_wq;
839a8e86 TH	51
d6c10f1f	52	static inline bool wb_has_dirty_io(struct bdi_writeback *wb)
03ba3782	53	{
d6c10f1f	54	return test_bit(WB_has_dirty_io, &wb->state);
03ba3782 JA	55	}
03ba3782 JA	56
95a46c65 TH	57	static inline bool bdi_has_dirty_io(struct backing_dev_info *bdi)
	58	{
	59	/*
	60	* @bdi->tot_write_bandwidth is guaranteed to be > 0 if there are
	61	* any dirty wbs. See wb_update_write_bandwidth().
	62	*/
	63	return atomic_long_read(&bdi->tot_write_bandwidth);
03ba3782 JA	64	}
03ba3782 JA	65
93f78d88 TH	66	static inline void __add_wb_stat(struct bdi_writeback *wb,
93f78d88 TH	67	enum wb_stat_item item, s64 amount)
b2e8fb6e	68	{
93f78d88	69	__percpu_counter_add(&wb->stat[item], amount, WB_STAT_BATCH);
b2e8fb6e PZ	70	}
b2e8fb6e PZ	71
93f78d88 TH	72	static inline void __inc_wb_stat(struct bdi_writeback *wb,
93f78d88 TH	73	enum wb_stat_item item)
b2e8fb6e	74	{
93f78d88	75	__add_wb_stat(wb, item, 1);
b2e8fb6e PZ	76	}
b2e8fb6e PZ	77
93f78d88	78	static inline void inc_wb_stat(struct bdi_writeback *wb, enum wb_stat_item item)
b2e8fb6e PZ	79	{
	80	unsigned long flags;
	81
	82	local_irq_save(flags);
93f78d88	83	__inc_wb_stat(wb, item);
b2e8fb6e PZ	84	local_irq_restore(flags);
	85	}
	86
93f78d88 TH	87	static inline void __dec_wb_stat(struct bdi_writeback *wb,
93f78d88 TH	88	enum wb_stat_item item)
b2e8fb6e	89	{
93f78d88	90	__add_wb_stat(wb, item, -1);
b2e8fb6e PZ	91	}
b2e8fb6e PZ	92
93f78d88	93	static inline void dec_wb_stat(struct bdi_writeback *wb, enum wb_stat_item item)
b2e8fb6e PZ	94	{
	95	unsigned long flags;
	96
	97	local_irq_save(flags);
93f78d88	98	__dec_wb_stat(wb, item);
b2e8fb6e PZ	99	local_irq_restore(flags);
	100	}
	101
93f78d88	102	static inline s64 wb_stat(struct bdi_writeback *wb, enum wb_stat_item item)
b2e8fb6e	103	{
93f78d88	104	return percpu_counter_read_positive(&wb->stat[item]);
b2e8fb6e PZ	105	}
b2e8fb6e PZ	106
93f78d88 TH	107	static inline s64 __wb_stat_sum(struct bdi_writeback *wb,
93f78d88 TH	108	enum wb_stat_item item)
b2e8fb6e	109	{
93f78d88	110	return percpu_counter_sum_positive(&wb->stat[item]);
b2e8fb6e PZ	111	}
b2e8fb6e PZ	112
93f78d88	113	static inline s64 wb_stat_sum(struct bdi_writeback *wb, enum wb_stat_item item)
e0bf68dd	114	{
b2e8fb6e PZ	115	s64 sum;
	116	unsigned long flags;
	117
	118	local_irq_save(flags);
93f78d88	119	sum = __wb_stat_sum(wb, item);
b2e8fb6e PZ	120	local_irq_restore(flags);
	121
	122	return sum;
e0bf68dd PZ	123	}
e0bf68dd PZ	124
93f78d88	125	extern void wb_writeout_inc(struct bdi_writeback *wb);
dd5656e5	126
b2e8fb6e PZ	127	/*
	128	* maximal error of a stat counter.
	129	*/
93f78d88	130	static inline unsigned long wb_stat_error(struct bdi_writeback *wb)
e0bf68dd	131	{
b2e8fb6e	132	#ifdef CONFIG_SMP
93f78d88	133	return nr_cpu_ids * WB_STAT_BATCH;
b2e8fb6e PZ	134	#else
	135	return 1;
	136	#endif
e0bf68dd	137	}
1da177e4	138
189d3c4a	139	int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio);
a42dde04	140	int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio);
189d3c4a	141
1da177e4 LT	142	/*
1da177e4 LT	143	* Flags in backing_dev_info::capability
e4ad08fe MS	144	*
	145	* The first three flags control whether dirty pages will contribute to the
	146	* VM's accounting and whether writepages() should be called for dirty pages
	147	* (something that would not, for example, be appropriate for ramfs)
	148	*
	149	* WARNING: these flags are closely related and should not normally be
	150	* used separately. The BDI_CAP_NO_ACCT_AND_WRITEBACK combines these
	151	* three flags into a single convenience macro.
	152	*
	153	* BDI_CAP_NO_ACCT_DIRTY: Dirty pages shouldn't contribute to accounting
	154	* BDI_CAP_NO_WRITEBACK: Don't write pages back
	155	* BDI_CAP_NO_ACCT_WB: Don't automatically account writeback pages
5a537485	156	* BDI_CAP_STRICTLIMIT: Keep number of dirty pages below bdi threshold.
89e9b9e0 TH	157	*
89e9b9e0 TH	158	* BDI_CAP_CGROUP_WRITEBACK: Supports cgroup-aware writeback.
1da177e4	159	*/
e4ad08fe MS	160	#define BDI_CAP_NO_ACCT_DIRTY 0x00000001
e4ad08fe MS	161	#define BDI_CAP_NO_WRITEBACK 0x00000002
b4caecd4 CH	162	#define BDI_CAP_NO_ACCT_WB 0x00000004
	163	#define BDI_CAP_STABLE_WRITES 0x00000008
	164	#define BDI_CAP_STRICTLIMIT 0x00000010
89e9b9e0	165	#define BDI_CAP_CGROUP_WRITEBACK 0x00000020
1da177e4	166
e4ad08fe MS	167	#define BDI_CAP_NO_ACCT_AND_WRITEBACK \
	168	(BDI_CAP_NO_WRITEBACK \| BDI_CAP_NO_ACCT_DIRTY \| BDI_CAP_NO_ACCT_WB)
	169
5129a469	170	extern struct backing_dev_info noop_backing_dev_info;
1da177e4	171
bc05873d TH	172	/**
	173	* writeback_in_progress - determine whether there is writeback in progress
	174	* @wb: bdi_writeback of interest
	175	*
	176	* Determine whether there is writeback waiting to be handled against a
	177	* bdi_writeback.
	178	*/
	179	static inline bool writeback_in_progress(struct bdi_writeback *wb)
1da177e4	180	{
bc05873d	181	return test_bit(WB_writeback_running, &wb->state);
1da177e4 LT	182	}
1da177e4 LT	183
a212b105	184	static inline struct backing_dev_info inode_to_bdi(struct inode inode)
1da177e4	185	{
a212b105	186	struct super_block *sb;
1da177e4	187
a212b105 TH	188	if (!inode)
	189	return &noop_backing_dev_info;
	190
	191	sb = inode->i_sb;
	192	#ifdef CONFIG_BLOCK
	193	if (sb_is_blkdev_sb(sb))
efa7c9f9	194	return I_BDEV(inode)->bd_bdi;
a212b105 TH	195	#endif
a212b105 TH	196	return sb->s_bdi;
1da177e4 LT	197	}
1da177e4 LT	198
ec8a6f26	199	static inline int wb_congested(struct bdi_writeback *wb, int cong_bits)
1da177e4	200	{
ec8a6f26	201	struct backing_dev_info *bdi = wb->bdi;
1da177e4	202
ec8a6f26 TH	203	if (bdi->congested_fn)
	204	return bdi->congested_fn(bdi->congested_data, cong_bits);
	205	return wb->congested->state & cong_bits;
1da177e4	206	}
373c0a7e	207
8aa7e847	208	long congestion_wait(int sync, long timeout);
599d0c95	209	long wait_iff_congested(struct pglist_data *pgdat, int sync, long timeout);
3965c9ae WL	210	int pdflush_proc_obsolete(struct ctl_table *table, int write,
3965c9ae WL	211	void __user buffer, size_t lenp, loff_t *ppos);
1da177e4	212
7d311cda DW	213	static inline bool bdi_cap_stable_pages_required(struct backing_dev_info *bdi)
	214	{
	215	return bdi->capabilities & BDI_CAP_STABLE_WRITES;
	216	}
	217
e4ad08fe MS	218	static inline bool bdi_cap_writeback_dirty(struct backing_dev_info *bdi)
	219	{
	220	return !(bdi->capabilities & BDI_CAP_NO_WRITEBACK);
	221	}
	222
	223	static inline bool bdi_cap_account_dirty(struct backing_dev_info *bdi)
	224	{
	225	return !(bdi->capabilities & BDI_CAP_NO_ACCT_DIRTY);
	226	}
1da177e4	227
e4ad08fe MS	228	static inline bool bdi_cap_account_writeback(struct backing_dev_info *bdi)
	229	{
	230	/* Paranoia: BDI_CAP_NO_WRITEBACK implies BDI_CAP_NO_ACCT_WB */
	231	return !(bdi->capabilities & (BDI_CAP_NO_ACCT_WB \|
	232	BDI_CAP_NO_WRITEBACK));
	233	}
1da177e4	234
e4ad08fe MS	235	static inline bool mapping_cap_writeback_dirty(struct address_space *mapping)
e4ad08fe MS	236	{
de1414a6	237	return bdi_cap_writeback_dirty(inode_to_bdi(mapping->host));
e4ad08fe	238	}
1da177e4	239
e4ad08fe MS	240	static inline bool mapping_cap_account_dirty(struct address_space *mapping)
e4ad08fe MS	241	{
de1414a6	242	return bdi_cap_account_dirty(inode_to_bdi(mapping->host));
e4ad08fe	243	}
1da177e4	244
03ba3782 JA	245	static inline int bdi_sched_wait(void *word)
	246	{
	247	schedule();
	248	return 0;
	249	}
	250
89e9b9e0 TH	251	#ifdef CONFIG_CGROUP_WRITEBACK
89e9b9e0 TH	252
52ebea74 TH	253	struct bdi_writeback_congested *
	254	wb_congested_get_create(struct backing_dev_info *bdi, int blkcg_id, gfp_t gfp);
	255	void wb_congested_put(struct bdi_writeback_congested *congested);
	256	struct bdi_writeback wb_get_create(struct backing_dev_info bdi,
	257	struct cgroup_subsys_state *memcg_css,
	258	gfp_t gfp);
52ebea74 TH	259	void wb_memcg_offline(struct mem_cgroup *memcg);
52ebea74 TH	260	void wb_blkcg_offline(struct blkcg *blkcg);
703c2708	261	int inode_congested(struct inode *inode, int cong_bits);
52ebea74	262
89e9b9e0 TH	263	/**
	264	* inode_cgwb_enabled - test whether cgroup writeback is enabled on an inode
	265	* @inode: inode of interest
	266	*
	267	* cgroup writeback requires support from both the bdi and filesystem.
9badce00 TH	268	* Also, both memcg and iocg have to be on the default hierarchy. Test
	269	* whether all conditions are met.
	270	*
	271	* Note that the test result may change dynamically on the same inode
	272	* depending on how memcg and iocg are configured.
89e9b9e0 TH	273	*/
	274	static inline bool inode_cgwb_enabled(struct inode *inode)
	275	{
	276	struct backing_dev_info *bdi = inode_to_bdi(inode);
	277
c0522908 TH	278	return cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
c0522908 TH	279	cgroup_subsys_on_dfl(io_cgrp_subsys) &&
9badce00	280	bdi_cap_account_dirty(bdi) &&
89e9b9e0	281	(bdi->capabilities & BDI_CAP_CGROUP_WRITEBACK) &&
46b15caa	282	(inode->i_sb->s_iflags & SB_I_CGROUPWB);
89e9b9e0 TH	283	}
89e9b9e0 TH	284
52ebea74 TH	285	/**
	286	* wb_find_current - find wb for %current on a bdi
	287	* @bdi: bdi of interest
	288	*
	289	* Find the wb of @bdi which matches both the memcg and blkcg of %current.
	290	* Must be called under rcu_read_lock() which protects the returend wb.
	291	* NULL if not found.
	292	*/
	293	static inline struct bdi_writeback wb_find_current(struct backing_dev_info bdi)
	294	{
	295	struct cgroup_subsys_state *memcg_css;
	296	struct bdi_writeback *wb;
	297
	298	memcg_css = task_css(current, memory_cgrp_id);
	299	if (!memcg_css->parent)
	300	return &bdi->wb;
	301
	302	wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);
	303
	304	/*
	305	* %current's blkcg equals the effective blkcg of its memcg. No
	306	* need to use the relatively expensive cgroup_get_e_css().
	307	*/
c165b3e3	308	if (likely(wb && wb->blkcg_css == task_css(current, io_cgrp_id)))
52ebea74 TH	309	return wb;
	310	return NULL;
	311	}
	312
	313	/**
	314	* wb_get_create_current - get or create wb for %current on a bdi
	315	* @bdi: bdi of interest
	316	* @gfp: allocation mask
	317	*
	318	* Equivalent to wb_get_create() on %current's memcg. This function is
	319	* called from a relatively hot path and optimizes the common cases using
	320	* wb_find_current().
	321	*/
	322	static inline struct bdi_writeback *
	323	wb_get_create_current(struct backing_dev_info *bdi, gfp_t gfp)
	324	{
	325	struct bdi_writeback *wb;
	326
	327	rcu_read_lock();
	328	wb = wb_find_current(bdi);
	329	if (wb && unlikely(!wb_tryget(wb)))
	330	wb = NULL;
	331	rcu_read_unlock();
	332
	333	if (unlikely(!wb)) {
	334	struct cgroup_subsys_state *memcg_css;
	335
	336	memcg_css = task_get_css(current, memory_cgrp_id);
	337	wb = wb_get_create(bdi, memcg_css, gfp);
	338	css_put(memcg_css);
	339	}
	340	return wb;
	341	}
	342
aaa2cacf TH	343	/**
	344	* inode_to_wb_is_valid - test whether an inode has a wb associated
	345	* @inode: inode of interest
	346	*
	347	* Returns %true if @inode has a wb associated. May be called without any
	348	* locking.
	349	*/
	350	static inline bool inode_to_wb_is_valid(struct inode *inode)
	351	{
	352	return inode->i_wb;
	353	}
	354
52ebea74 TH	355	/**
	356	* inode_to_wb - determine the wb of an inode
	357	* @inode: inode of interest
	358	*
aaa2cacf TH	359	* Returns the wb @inode is currently associated with. The caller must be
	360	* holding either @inode->i_lock, @inode->i_mapping->tree_lock, or the
	361	* associated wb's list_lock.
52ebea74 TH	362	*/
	363	static inline struct bdi_writeback inode_to_wb(struct inode inode)
	364	{
aaa2cacf TH	365	#ifdef CONFIG_LOCKDEP
	366	WARN_ON_ONCE(debug_locks &&
	367	(!lockdep_is_held(&inode->i_lock) &&
	368	!lockdep_is_held(&inode->i_mapping->tree_lock) &&
	369	!lockdep_is_held(&inode->i_wb->list_lock)));
	370	#endif
52ebea74 TH	371	return inode->i_wb;
	372	}
	373
682aa8e1 TH	374	/**
	375	* unlocked_inode_to_wb_begin - begin unlocked inode wb access transaction
	376	* @inode: target inode
	377	* @lockedp: temp bool output param, to be passed to the end function
	378	*
	379	* The caller wants to access the wb associated with @inode but isn't
	380	* holding inode->i_lock, mapping->tree_lock or wb->list_lock. This
	381	* function determines the wb associated with @inode and ensures that the
	382	* association doesn't change until the transaction is finished with
	383	* unlocked_inode_to_wb_end().
	384	*
	385	* The caller must call unlocked_inode_to_wb_end() with *@lockdep
	386	* afterwards and can't sleep during transaction. IRQ may or may not be
	387	* disabled on return.
	388	*/
	389	static inline struct bdi_writeback *
	390	unlocked_inode_to_wb_begin(struct inode inode, bool lockedp)
	391	{
	392	rcu_read_lock();
	393
	394	/*
	395	* Paired with store_release in inode_switch_wb_work_fn() and
	396	* ensures that we see the new wb if we see cleared I_WB_SWITCH.
	397	*/
	398	*lockedp = smp_load_acquire(&inode->i_state) & I_WB_SWITCH;
	399
	400	if (unlikely(*lockedp))
	401	spin_lock_irq(&inode->i_mapping->tree_lock);
aaa2cacf TH	402
	403	/*
	404	* Protected by either !I_WB_SWITCH + rcu_read_lock() or tree_lock.
	405	* inode_to_wb() will bark. Deref directly.
	406	*/
	407	return inode->i_wb;
682aa8e1 TH	408	}
	409
	410	/**
	411	* unlocked_inode_to_wb_end - end inode wb access transaction
	412	* @inode: target inode
	413	* @locked: *@lockedp from unlocked_inode_to_wb_begin()
	414	*/
	415	static inline void unlocked_inode_to_wb_end(struct inode *inode, bool locked)
	416	{
	417	if (unlikely(locked))
	418	spin_unlock_irq(&inode->i_mapping->tree_lock);
	419
	420	rcu_read_unlock();
	421	}
	422
89e9b9e0 TH	423	#else /* CONFIG_CGROUP_WRITEBACK */
	424
	425	static inline bool inode_cgwb_enabled(struct inode *inode)
	426	{
	427	return false;
	428	}
	429
52ebea74 TH	430	static inline struct bdi_writeback_congested *
	431	wb_congested_get_create(struct backing_dev_info *bdi, int blkcg_id, gfp_t gfp)
	432	{
a13f35e8 TH	433	atomic_inc(&bdi->wb_congested->refcnt);
a13f35e8 TH	434	return bdi->wb_congested;
52ebea74 TH	435	}
	436
	437	static inline void wb_congested_put(struct bdi_writeback_congested *congested)
	438	{
a13f35e8 TH	439	if (atomic_dec_and_test(&congested->refcnt))
a13f35e8 TH	440	kfree(congested);
52ebea74 TH	441	}
52ebea74 TH	442
52ebea74 TH	443	static inline struct bdi_writeback wb_find_current(struct backing_dev_info bdi)
	444	{
	445	return &bdi->wb;
	446	}
	447
	448	static inline struct bdi_writeback *
	449	wb_get_create_current(struct backing_dev_info *bdi, gfp_t gfp)
	450	{
	451	return &bdi->wb;
	452	}
	453
aaa2cacf TH	454	static inline bool inode_to_wb_is_valid(struct inode *inode)
	455	{
	456	return true;
	457	}
	458
52ebea74 TH	459	static inline struct bdi_writeback inode_to_wb(struct inode inode)
	460	{
	461	return &inode_to_bdi(inode)->wb;
	462	}
	463
682aa8e1 TH	464	static inline struct bdi_writeback *
	465	unlocked_inode_to_wb_begin(struct inode inode, bool lockedp)
	466	{
	467	return inode_to_wb(inode);
	468	}
	469
	470	static inline void unlocked_inode_to_wb_end(struct inode *inode, bool locked)
	471	{
	472	}
	473
52ebea74 TH	474	static inline void wb_memcg_offline(struct mem_cgroup *memcg)
	475	{
	476	}
	477
	478	static inline void wb_blkcg_offline(struct blkcg *blkcg)
	479	{
	480	}
	481
703c2708 TH	482	static inline int inode_congested(struct inode *inode, int cong_bits)
	483	{
	484	return wb_congested(&inode_to_bdi(inode)->wb, cong_bits);
	485	}
	486
89e9b9e0 TH	487	#endif /* CONFIG_CGROUP_WRITEBACK */
89e9b9e0 TH	488
703c2708 TH	489	static inline int inode_read_congested(struct inode *inode)
	490	{
	491	return inode_congested(inode, 1 << WB_sync_congested);
	492	}
	493
	494	static inline int inode_write_congested(struct inode *inode)
	495	{
	496	return inode_congested(inode, 1 << WB_async_congested);
	497	}
	498
	499	static inline int inode_rw_congested(struct inode *inode)
	500	{
	501	return inode_congested(inode, (1 << WB_sync_congested) \|
	502	(1 << WB_async_congested));
	503	}
	504
ec8a6f26 TH	505	static inline int bdi_congested(struct backing_dev_info *bdi, int cong_bits)
	506	{
	507	return wb_congested(&bdi->wb, cong_bits);
	508	}
	509
	510	static inline int bdi_read_congested(struct backing_dev_info *bdi)
	511	{
	512	return bdi_congested(bdi, 1 << WB_sync_congested);
	513	}
	514
	515	static inline int bdi_write_congested(struct backing_dev_info *bdi)
	516	{
	517	return bdi_congested(bdi, 1 << WB_async_congested);
	518	}
	519
	520	static inline int bdi_rw_congested(struct backing_dev_info *bdi)
	521	{
	522	return bdi_congested(bdi, (1 << WB_sync_congested) \|
	523	(1 << WB_async_congested));
	524	}
	525
89e9b9e0	526	#endif /* _LINUX_BACKING_DEV_H */