]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * include/linux/backing-dev.h | |
3 | * | |
4 | * low-level device information and state which is propagated up through | |
5 | * to high-level code. | |
6 | */ | |
7 | ||
8 | #ifndef _LINUX_BACKING_DEV_H | |
9 | #define _LINUX_BACKING_DEV_H | |
10 | ||
cf0ca9fe | 11 | #include <linux/kernel.h> |
e4ad08fe | 12 | #include <linux/fs.h> |
03ba3782 | 13 | #include <linux/sched.h> |
a212b105 | 14 | #include <linux/blkdev.h> |
03ba3782 | 15 | #include <linux/writeback.h> |
52ebea74 | 16 | #include <linux/blk-cgroup.h> |
66114cad | 17 | #include <linux/backing-dev-defs.h> |
1da177e4 | 18 | |
8077c0d9 | 19 | int __must_check bdi_init(struct backing_dev_info *bdi); |
b2e8fb6e PZ |
20 | void bdi_destroy(struct backing_dev_info *bdi); |
21 | ||
d2cc4dde | 22 | __printf(3, 4) |
cf0ca9fe PZ |
23 | int bdi_register(struct backing_dev_info *bdi, struct device *parent, |
24 | const char *fmt, ...); | |
25 | int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev); | |
26 | void bdi_unregister(struct backing_dev_info *bdi); | |
b4caecd4 | 27 | int __must_check bdi_setup_and_register(struct backing_dev_info *, char *); |
0e175a18 CW |
28 | void bdi_start_writeback(struct backing_dev_info *bdi, long nr_pages, |
29 | enum wb_reason reason); | |
c5444198 | 30 | void bdi_start_background_writeback(struct backing_dev_info *bdi); |
f0054bb1 | 31 | void wb_workfn(struct work_struct *work); |
03ba3782 | 32 | int bdi_has_dirty_io(struct backing_dev_info *bdi); |
f0054bb1 | 33 | void wb_wakeup_delayed(struct bdi_writeback *wb); |
cf0ca9fe | 34 | |
03ba3782 | 35 | extern spinlock_t bdi_lock; |
66f3b8e2 JA |
36 | extern struct list_head bdi_list; |
37 | ||
839a8e86 TH |
38 | extern struct workqueue_struct *bdi_wq; |
39 | ||
03ba3782 JA |
40 | static inline int wb_has_dirty_io(struct bdi_writeback *wb) |
41 | { | |
42 | return !list_empty(&wb->b_dirty) || | |
43 | !list_empty(&wb->b_io) || | |
44 | !list_empty(&wb->b_more_io); | |
45 | } | |
46 | ||
93f78d88 TH |
47 | static inline void __add_wb_stat(struct bdi_writeback *wb, |
48 | enum wb_stat_item item, s64 amount) | |
b2e8fb6e | 49 | { |
93f78d88 | 50 | __percpu_counter_add(&wb->stat[item], amount, WB_STAT_BATCH); |
b2e8fb6e PZ |
51 | } |
52 | ||
93f78d88 TH |
53 | static inline void __inc_wb_stat(struct bdi_writeback *wb, |
54 | enum wb_stat_item item) | |
b2e8fb6e | 55 | { |
93f78d88 | 56 | __add_wb_stat(wb, item, 1); |
b2e8fb6e PZ |
57 | } |
58 | ||
93f78d88 | 59 | static inline void inc_wb_stat(struct bdi_writeback *wb, enum wb_stat_item item) |
b2e8fb6e PZ |
60 | { |
61 | unsigned long flags; | |
62 | ||
63 | local_irq_save(flags); | |
93f78d88 | 64 | __inc_wb_stat(wb, item); |
b2e8fb6e PZ |
65 | local_irq_restore(flags); |
66 | } | |
67 | ||
93f78d88 TH |
68 | static inline void __dec_wb_stat(struct bdi_writeback *wb, |
69 | enum wb_stat_item item) | |
b2e8fb6e | 70 | { |
93f78d88 | 71 | __add_wb_stat(wb, item, -1); |
b2e8fb6e PZ |
72 | } |
73 | ||
93f78d88 | 74 | static inline void dec_wb_stat(struct bdi_writeback *wb, enum wb_stat_item item) |
b2e8fb6e PZ |
75 | { |
76 | unsigned long flags; | |
77 | ||
78 | local_irq_save(flags); | |
93f78d88 | 79 | __dec_wb_stat(wb, item); |
b2e8fb6e PZ |
80 | local_irq_restore(flags); |
81 | } | |
82 | ||
93f78d88 | 83 | static inline s64 wb_stat(struct bdi_writeback *wb, enum wb_stat_item item) |
b2e8fb6e | 84 | { |
93f78d88 | 85 | return percpu_counter_read_positive(&wb->stat[item]); |
b2e8fb6e PZ |
86 | } |
87 | ||
93f78d88 TH |
88 | static inline s64 __wb_stat_sum(struct bdi_writeback *wb, |
89 | enum wb_stat_item item) | |
b2e8fb6e | 90 | { |
93f78d88 | 91 | return percpu_counter_sum_positive(&wb->stat[item]); |
b2e8fb6e PZ |
92 | } |
93 | ||
93f78d88 | 94 | static inline s64 wb_stat_sum(struct bdi_writeback *wb, enum wb_stat_item item) |
e0bf68dd | 95 | { |
b2e8fb6e PZ |
96 | s64 sum; |
97 | unsigned long flags; | |
98 | ||
99 | local_irq_save(flags); | |
93f78d88 | 100 | sum = __wb_stat_sum(wb, item); |
b2e8fb6e PZ |
101 | local_irq_restore(flags); |
102 | ||
103 | return sum; | |
e0bf68dd PZ |
104 | } |
105 | ||
93f78d88 | 106 | extern void wb_writeout_inc(struct bdi_writeback *wb); |
dd5656e5 | 107 | |
b2e8fb6e PZ |
108 | /* |
109 | * maximal error of a stat counter. | |
110 | */ | |
93f78d88 | 111 | static inline unsigned long wb_stat_error(struct bdi_writeback *wb) |
e0bf68dd | 112 | { |
b2e8fb6e | 113 | #ifdef CONFIG_SMP |
93f78d88 | 114 | return nr_cpu_ids * WB_STAT_BATCH; |
b2e8fb6e PZ |
115 | #else |
116 | return 1; | |
117 | #endif | |
e0bf68dd | 118 | } |
1da177e4 | 119 | |
189d3c4a | 120 | int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio); |
a42dde04 | 121 | int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio); |
189d3c4a | 122 | |
1da177e4 LT |
123 | /* |
124 | * Flags in backing_dev_info::capability | |
e4ad08fe MS |
125 | * |
126 | * The first three flags control whether dirty pages will contribute to the | |
127 | * VM's accounting and whether writepages() should be called for dirty pages | |
128 | * (something that would not, for example, be appropriate for ramfs) | |
129 | * | |
130 | * WARNING: these flags are closely related and should not normally be | |
131 | * used separately. The BDI_CAP_NO_ACCT_AND_WRITEBACK combines these | |
132 | * three flags into a single convenience macro. | |
133 | * | |
134 | * BDI_CAP_NO_ACCT_DIRTY: Dirty pages shouldn't contribute to accounting | |
135 | * BDI_CAP_NO_WRITEBACK: Don't write pages back | |
136 | * BDI_CAP_NO_ACCT_WB: Don't automatically account writeback pages | |
5a537485 | 137 | * BDI_CAP_STRICTLIMIT: Keep number of dirty pages below bdi threshold. |
89e9b9e0 TH |
138 | * |
139 | * BDI_CAP_CGROUP_WRITEBACK: Supports cgroup-aware writeback. | |
1da177e4 | 140 | */ |
e4ad08fe MS |
141 | #define BDI_CAP_NO_ACCT_DIRTY 0x00000001 |
142 | #define BDI_CAP_NO_WRITEBACK 0x00000002 | |
b4caecd4 CH |
143 | #define BDI_CAP_NO_ACCT_WB 0x00000004 |
144 | #define BDI_CAP_STABLE_WRITES 0x00000008 | |
145 | #define BDI_CAP_STRICTLIMIT 0x00000010 | |
89e9b9e0 | 146 | #define BDI_CAP_CGROUP_WRITEBACK 0x00000020 |
1da177e4 | 147 | |
e4ad08fe MS |
148 | #define BDI_CAP_NO_ACCT_AND_WRITEBACK \ |
149 | (BDI_CAP_NO_WRITEBACK | BDI_CAP_NO_ACCT_DIRTY | BDI_CAP_NO_ACCT_WB) | |
150 | ||
5129a469 | 151 | extern struct backing_dev_info noop_backing_dev_info; |
1da177e4 | 152 | |
1da177e4 | 153 | int writeback_in_progress(struct backing_dev_info *bdi); |
1da177e4 | 154 | |
a212b105 TH |
155 | static inline struct backing_dev_info *inode_to_bdi(struct inode *inode) |
156 | { | |
157 | struct super_block *sb; | |
158 | ||
159 | if (!inode) | |
160 | return &noop_backing_dev_info; | |
161 | ||
162 | sb = inode->i_sb; | |
163 | #ifdef CONFIG_BLOCK | |
164 | if (sb_is_blkdev_sb(sb)) | |
165 | return blk_get_backing_dev_info(I_BDEV(inode)); | |
166 | #endif | |
167 | return sb->s_bdi; | |
168 | } | |
169 | ||
ec8a6f26 | 170 | static inline int wb_congested(struct bdi_writeback *wb, int cong_bits) |
1da177e4 | 171 | { |
ec8a6f26 | 172 | struct backing_dev_info *bdi = wb->bdi; |
1da177e4 | 173 | |
ec8a6f26 TH |
174 | if (bdi->congested_fn) |
175 | return bdi->congested_fn(bdi->congested_data, cong_bits); | |
176 | return wb->congested->state & cong_bits; | |
1da177e4 LT |
177 | } |
178 | ||
8aa7e847 | 179 | long congestion_wait(int sync, long timeout); |
0e093d99 | 180 | long wait_iff_congested(struct zone *zone, int sync, long timeout); |
3965c9ae WL |
181 | int pdflush_proc_obsolete(struct ctl_table *table, int write, |
182 | void __user *buffer, size_t *lenp, loff_t *ppos); | |
1da177e4 | 183 | |
7d311cda DW |
184 | static inline bool bdi_cap_stable_pages_required(struct backing_dev_info *bdi) |
185 | { | |
186 | return bdi->capabilities & BDI_CAP_STABLE_WRITES; | |
187 | } | |
188 | ||
e4ad08fe MS |
189 | static inline bool bdi_cap_writeback_dirty(struct backing_dev_info *bdi) |
190 | { | |
191 | return !(bdi->capabilities & BDI_CAP_NO_WRITEBACK); | |
192 | } | |
193 | ||
194 | static inline bool bdi_cap_account_dirty(struct backing_dev_info *bdi) | |
195 | { | |
196 | return !(bdi->capabilities & BDI_CAP_NO_ACCT_DIRTY); | |
197 | } | |
1da177e4 | 198 | |
e4ad08fe MS |
199 | static inline bool bdi_cap_account_writeback(struct backing_dev_info *bdi) |
200 | { | |
201 | /* Paranoia: BDI_CAP_NO_WRITEBACK implies BDI_CAP_NO_ACCT_WB */ | |
202 | return !(bdi->capabilities & (BDI_CAP_NO_ACCT_WB | | |
203 | BDI_CAP_NO_WRITEBACK)); | |
204 | } | |
1da177e4 | 205 | |
e4ad08fe MS |
206 | static inline bool mapping_cap_writeback_dirty(struct address_space *mapping) |
207 | { | |
de1414a6 | 208 | return bdi_cap_writeback_dirty(inode_to_bdi(mapping->host)); |
e4ad08fe | 209 | } |
1da177e4 | 210 | |
e4ad08fe MS |
211 | static inline bool mapping_cap_account_dirty(struct address_space *mapping) |
212 | { | |
de1414a6 | 213 | return bdi_cap_account_dirty(inode_to_bdi(mapping->host)); |
e4ad08fe | 214 | } |
1da177e4 | 215 | |
03ba3782 JA |
216 | static inline int bdi_sched_wait(void *word) |
217 | { | |
218 | schedule(); | |
219 | return 0; | |
220 | } | |
221 | ||
89e9b9e0 TH |
222 | #ifdef CONFIG_CGROUP_WRITEBACK |
223 | ||
52ebea74 TH |
224 | struct bdi_writeback_congested * |
225 | wb_congested_get_create(struct backing_dev_info *bdi, int blkcg_id, gfp_t gfp); | |
226 | void wb_congested_put(struct bdi_writeback_congested *congested); | |
227 | struct bdi_writeback *wb_get_create(struct backing_dev_info *bdi, | |
228 | struct cgroup_subsys_state *memcg_css, | |
229 | gfp_t gfp); | |
230 | void __inode_attach_wb(struct inode *inode, struct page *page); | |
231 | void wb_memcg_offline(struct mem_cgroup *memcg); | |
232 | void wb_blkcg_offline(struct blkcg *blkcg); | |
233 | ||
89e9b9e0 TH |
234 | /** |
235 | * inode_cgwb_enabled - test whether cgroup writeback is enabled on an inode | |
236 | * @inode: inode of interest | |
237 | * | |
238 | * cgroup writeback requires support from both the bdi and filesystem. | |
239 | * Test whether @inode has both. | |
240 | */ | |
241 | static inline bool inode_cgwb_enabled(struct inode *inode) | |
242 | { | |
243 | struct backing_dev_info *bdi = inode_to_bdi(inode); | |
244 | ||
245 | return bdi_cap_account_dirty(bdi) && | |
246 | (bdi->capabilities & BDI_CAP_CGROUP_WRITEBACK) && | |
247 | (inode->i_sb->s_type->fs_flags & FS_CGROUP_WRITEBACK); | |
248 | } | |
249 | ||
52ebea74 TH |
250 | /** |
251 | * wb_tryget - try to increment a wb's refcount | |
252 | * @wb: bdi_writeback to get | |
253 | */ | |
254 | static inline bool wb_tryget(struct bdi_writeback *wb) | |
255 | { | |
256 | if (wb != &wb->bdi->wb) | |
257 | return percpu_ref_tryget(&wb->refcnt); | |
258 | return true; | |
259 | } | |
260 | ||
261 | /** | |
262 | * wb_get - increment a wb's refcount | |
263 | * @wb: bdi_writeback to get | |
264 | */ | |
265 | static inline void wb_get(struct bdi_writeback *wb) | |
266 | { | |
267 | if (wb != &wb->bdi->wb) | |
268 | percpu_ref_get(&wb->refcnt); | |
269 | } | |
270 | ||
271 | /** | |
272 | * wb_put - decrement a wb's refcount | |
273 | * @wb: bdi_writeback to put | |
274 | */ | |
275 | static inline void wb_put(struct bdi_writeback *wb) | |
276 | { | |
277 | if (wb != &wb->bdi->wb) | |
278 | percpu_ref_put(&wb->refcnt); | |
279 | } | |
280 | ||
281 | /** | |
282 | * wb_find_current - find wb for %current on a bdi | |
283 | * @bdi: bdi of interest | |
284 | * | |
285 | * Find the wb of @bdi which matches both the memcg and blkcg of %current. | |
286 | * Must be called under rcu_read_lock() which protects the returend wb. | |
287 | * NULL if not found. | |
288 | */ | |
289 | static inline struct bdi_writeback *wb_find_current(struct backing_dev_info *bdi) | |
290 | { | |
291 | struct cgroup_subsys_state *memcg_css; | |
292 | struct bdi_writeback *wb; | |
293 | ||
294 | memcg_css = task_css(current, memory_cgrp_id); | |
295 | if (!memcg_css->parent) | |
296 | return &bdi->wb; | |
297 | ||
298 | wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id); | |
299 | ||
300 | /* | |
301 | * %current's blkcg equals the effective blkcg of its memcg. No | |
302 | * need to use the relatively expensive cgroup_get_e_css(). | |
303 | */ | |
304 | if (likely(wb && wb->blkcg_css == task_css(current, blkio_cgrp_id))) | |
305 | return wb; | |
306 | return NULL; | |
307 | } | |
308 | ||
309 | /** | |
310 | * wb_get_create_current - get or create wb for %current on a bdi | |
311 | * @bdi: bdi of interest | |
312 | * @gfp: allocation mask | |
313 | * | |
314 | * Equivalent to wb_get_create() on %current's memcg. This function is | |
315 | * called from a relatively hot path and optimizes the common cases using | |
316 | * wb_find_current(). | |
317 | */ | |
318 | static inline struct bdi_writeback * | |
319 | wb_get_create_current(struct backing_dev_info *bdi, gfp_t gfp) | |
320 | { | |
321 | struct bdi_writeback *wb; | |
322 | ||
323 | rcu_read_lock(); | |
324 | wb = wb_find_current(bdi); | |
325 | if (wb && unlikely(!wb_tryget(wb))) | |
326 | wb = NULL; | |
327 | rcu_read_unlock(); | |
328 | ||
329 | if (unlikely(!wb)) { | |
330 | struct cgroup_subsys_state *memcg_css; | |
331 | ||
332 | memcg_css = task_get_css(current, memory_cgrp_id); | |
333 | wb = wb_get_create(bdi, memcg_css, gfp); | |
334 | css_put(memcg_css); | |
335 | } | |
336 | return wb; | |
337 | } | |
338 | ||
339 | /** | |
340 | * inode_attach_wb - associate an inode with its wb | |
341 | * @inode: inode of interest | |
342 | * @page: page being dirtied (may be NULL) | |
343 | * | |
344 | * If @inode doesn't have its wb, associate it with the wb matching the | |
345 | * memcg of @page or, if @page is NULL, %current. May be called w/ or w/o | |
346 | * @inode->i_lock. | |
347 | */ | |
348 | static inline void inode_attach_wb(struct inode *inode, struct page *page) | |
349 | { | |
350 | if (!inode->i_wb) | |
351 | __inode_attach_wb(inode, page); | |
352 | } | |
353 | ||
354 | /** | |
355 | * inode_detach_wb - disassociate an inode from its wb | |
356 | * @inode: inode of interest | |
357 | * | |
358 | * @inode is being freed. Detach from its wb. | |
359 | */ | |
360 | static inline void inode_detach_wb(struct inode *inode) | |
361 | { | |
362 | if (inode->i_wb) { | |
363 | wb_put(inode->i_wb); | |
364 | inode->i_wb = NULL; | |
365 | } | |
366 | } | |
367 | ||
368 | /** | |
369 | * inode_to_wb - determine the wb of an inode | |
370 | * @inode: inode of interest | |
371 | * | |
372 | * Returns the wb @inode is currently associated with. | |
373 | */ | |
374 | static inline struct bdi_writeback *inode_to_wb(struct inode *inode) | |
375 | { | |
376 | return inode->i_wb; | |
377 | } | |
378 | ||
89e9b9e0 TH |
379 | #else /* CONFIG_CGROUP_WRITEBACK */ |
380 | ||
381 | static inline bool inode_cgwb_enabled(struct inode *inode) | |
382 | { | |
383 | return false; | |
384 | } | |
385 | ||
52ebea74 TH |
386 | static inline struct bdi_writeback_congested * |
387 | wb_congested_get_create(struct backing_dev_info *bdi, int blkcg_id, gfp_t gfp) | |
388 | { | |
389 | return bdi->wb.congested; | |
390 | } | |
391 | ||
392 | static inline void wb_congested_put(struct bdi_writeback_congested *congested) | |
393 | { | |
394 | } | |
395 | ||
396 | static inline bool wb_tryget(struct bdi_writeback *wb) | |
397 | { | |
398 | return true; | |
399 | } | |
400 | ||
401 | static inline void wb_get(struct bdi_writeback *wb) | |
402 | { | |
403 | } | |
404 | ||
405 | static inline void wb_put(struct bdi_writeback *wb) | |
406 | { | |
407 | } | |
408 | ||
409 | static inline struct bdi_writeback *wb_find_current(struct backing_dev_info *bdi) | |
410 | { | |
411 | return &bdi->wb; | |
412 | } | |
413 | ||
414 | static inline struct bdi_writeback * | |
415 | wb_get_create_current(struct backing_dev_info *bdi, gfp_t gfp) | |
416 | { | |
417 | return &bdi->wb; | |
418 | } | |
419 | ||
420 | static inline void inode_attach_wb(struct inode *inode, struct page *page) | |
421 | { | |
422 | } | |
423 | ||
424 | static inline void inode_detach_wb(struct inode *inode) | |
425 | { | |
426 | } | |
427 | ||
428 | static inline struct bdi_writeback *inode_to_wb(struct inode *inode) | |
429 | { | |
430 | return &inode_to_bdi(inode)->wb; | |
431 | } | |
432 | ||
433 | static inline void wb_memcg_offline(struct mem_cgroup *memcg) | |
434 | { | |
435 | } | |
436 | ||
437 | static inline void wb_blkcg_offline(struct blkcg *blkcg) | |
438 | { | |
439 | } | |
440 | ||
89e9b9e0 TH |
441 | #endif /* CONFIG_CGROUP_WRITEBACK */ |
442 | ||
ec8a6f26 TH |
443 | static inline int bdi_congested(struct backing_dev_info *bdi, int cong_bits) |
444 | { | |
445 | return wb_congested(&bdi->wb, cong_bits); | |
446 | } | |
447 | ||
448 | static inline int bdi_read_congested(struct backing_dev_info *bdi) | |
449 | { | |
450 | return bdi_congested(bdi, 1 << WB_sync_congested); | |
451 | } | |
452 | ||
453 | static inline int bdi_write_congested(struct backing_dev_info *bdi) | |
454 | { | |
455 | return bdi_congested(bdi, 1 << WB_async_congested); | |
456 | } | |
457 | ||
458 | static inline int bdi_rw_congested(struct backing_dev_info *bdi) | |
459 | { | |
460 | return bdi_congested(bdi, (1 << WB_sync_congested) | | |
461 | (1 << WB_async_congested)); | |
462 | } | |
463 | ||
89e9b9e0 | 464 | #endif /* _LINUX_BACKING_DEV_H */ |