2 #include <linux/wait.h>
3 #include <linux/backing-dev.h>
4 #include <linux/kthread.h>
5 #include <linux/freezer.h>
7 #include <linux/pagemap.h>
9 #include <linux/sched.h>
10 #include <linux/module.h>
11 #include <linux/writeback.h>
12 #include <linux/device.h>
13 #include <trace/events/writeback.h>
15 static atomic_long_t bdi_seq
= ATOMIC_LONG_INIT(0);
17 struct backing_dev_info noop_backing_dev_info
= {
19 .capabilities
= BDI_CAP_NO_ACCT_AND_WRITEBACK
,
21 EXPORT_SYMBOL_GPL(noop_backing_dev_info
);
23 static struct class *bdi_class
;
26 * bdi_lock protects updates to bdi_list. bdi_list has RCU reader side
29 DEFINE_SPINLOCK(bdi_lock
);
32 /* bdi_wq serves all asynchronous writeback tasks */
33 struct workqueue_struct
*bdi_wq
;
35 #ifdef CONFIG_DEBUG_FS
36 #include <linux/debugfs.h>
37 #include <linux/seq_file.h>
39 static struct dentry
*bdi_debug_root
;
41 static void bdi_debug_init(void)
43 bdi_debug_root
= debugfs_create_dir("bdi", NULL
);
46 static int bdi_debug_stats_show(struct seq_file
*m
, void *v
)
48 struct backing_dev_info
*bdi
= m
->private;
49 struct bdi_writeback
*wb
= &bdi
->wb
;
50 unsigned long background_thresh
;
51 unsigned long dirty_thresh
;
52 unsigned long bdi_thresh
;
53 unsigned long nr_dirty
, nr_io
, nr_more_io
, nr_dirty_time
;
56 nr_dirty
= nr_io
= nr_more_io
= nr_dirty_time
= 0;
57 spin_lock(&wb
->list_lock
);
58 list_for_each_entry(inode
, &wb
->b_dirty
, i_wb_list
)
60 list_for_each_entry(inode
, &wb
->b_io
, i_wb_list
)
62 list_for_each_entry(inode
, &wb
->b_more_io
, i_wb_list
)
64 list_for_each_entry(inode
, &wb
->b_dirty_time
, i_wb_list
)
65 if (inode
->i_state
& I_DIRTY_TIME
)
67 spin_unlock(&wb
->list_lock
);
69 global_dirty_limits(&background_thresh
, &dirty_thresh
);
70 bdi_thresh
= wb_dirty_limit(wb
, dirty_thresh
);
72 #define K(x) ((x) << (PAGE_SHIFT - 10))
74 "BdiWriteback: %10lu kB\n"
75 "BdiReclaimable: %10lu kB\n"
76 "BdiDirtyThresh: %10lu kB\n"
77 "DirtyThresh: %10lu kB\n"
78 "BackgroundThresh: %10lu kB\n"
79 "BdiDirtied: %10lu kB\n"
80 "BdiWritten: %10lu kB\n"
81 "BdiWriteBandwidth: %10lu kBps\n"
85 "b_dirty_time: %10lu\n"
88 (unsigned long) K(wb_stat(wb
, WB_WRITEBACK
)),
89 (unsigned long) K(wb_stat(wb
, WB_RECLAIMABLE
)),
93 (unsigned long) K(wb_stat(wb
, WB_DIRTIED
)),
94 (unsigned long) K(wb_stat(wb
, WB_WRITTEN
)),
95 (unsigned long) K(wb
->write_bandwidth
),
100 !list_empty(&bdi
->bdi_list
), bdi
->wb
.state
);
106 static int bdi_debug_stats_open(struct inode
*inode
, struct file
*file
)
108 return single_open(file
, bdi_debug_stats_show
, inode
->i_private
);
111 static const struct file_operations bdi_debug_stats_fops
= {
112 .open
= bdi_debug_stats_open
,
115 .release
= single_release
,
118 static void bdi_debug_register(struct backing_dev_info
*bdi
, const char *name
)
120 bdi
->debug_dir
= debugfs_create_dir(name
, bdi_debug_root
);
121 bdi
->debug_stats
= debugfs_create_file("stats", 0444, bdi
->debug_dir
,
122 bdi
, &bdi_debug_stats_fops
);
125 static void bdi_debug_unregister(struct backing_dev_info
*bdi
)
127 debugfs_remove(bdi
->debug_stats
);
128 debugfs_remove(bdi
->debug_dir
);
131 static inline void bdi_debug_init(void)
134 static inline void bdi_debug_register(struct backing_dev_info
*bdi
,
138 static inline void bdi_debug_unregister(struct backing_dev_info
*bdi
)
143 static ssize_t
read_ahead_kb_store(struct device
*dev
,
144 struct device_attribute
*attr
,
145 const char *buf
, size_t count
)
147 struct backing_dev_info
*bdi
= dev_get_drvdata(dev
);
148 unsigned long read_ahead_kb
;
151 ret
= kstrtoul(buf
, 10, &read_ahead_kb
);
155 bdi
->ra_pages
= read_ahead_kb
>> (PAGE_SHIFT
- 10);
160 #define K(pages) ((pages) << (PAGE_SHIFT - 10))
162 #define BDI_SHOW(name, expr) \
163 static ssize_t name##_show(struct device *dev, \
164 struct device_attribute *attr, char *page) \
166 struct backing_dev_info *bdi = dev_get_drvdata(dev); \
168 return snprintf(page, PAGE_SIZE-1, "%lld\n", (long long)expr); \
170 static DEVICE_ATTR_RW(name);
172 BDI_SHOW(read_ahead_kb
, K(bdi
->ra_pages
))
174 static ssize_t
min_ratio_store(struct device
*dev
,
175 struct device_attribute
*attr
, const char *buf
, size_t count
)
177 struct backing_dev_info
*bdi
= dev_get_drvdata(dev
);
181 ret
= kstrtouint(buf
, 10, &ratio
);
185 ret
= bdi_set_min_ratio(bdi
, ratio
);
191 BDI_SHOW(min_ratio
, bdi
->min_ratio
)
193 static ssize_t
max_ratio_store(struct device
*dev
,
194 struct device_attribute
*attr
, const char *buf
, size_t count
)
196 struct backing_dev_info
*bdi
= dev_get_drvdata(dev
);
200 ret
= kstrtouint(buf
, 10, &ratio
);
204 ret
= bdi_set_max_ratio(bdi
, ratio
);
210 BDI_SHOW(max_ratio
, bdi
->max_ratio
)
212 static ssize_t
stable_pages_required_show(struct device
*dev
,
213 struct device_attribute
*attr
,
216 struct backing_dev_info
*bdi
= dev_get_drvdata(dev
);
218 return snprintf(page
, PAGE_SIZE
-1, "%d\n",
219 bdi_cap_stable_pages_required(bdi
) ? 1 : 0);
221 static DEVICE_ATTR_RO(stable_pages_required
);
223 static struct attribute
*bdi_dev_attrs
[] = {
224 &dev_attr_read_ahead_kb
.attr
,
225 &dev_attr_min_ratio
.attr
,
226 &dev_attr_max_ratio
.attr
,
227 &dev_attr_stable_pages_required
.attr
,
230 ATTRIBUTE_GROUPS(bdi_dev
);
232 static __init
int bdi_class_init(void)
234 bdi_class
= class_create(THIS_MODULE
, "bdi");
235 if (IS_ERR(bdi_class
))
236 return PTR_ERR(bdi_class
);
238 bdi_class
->dev_groups
= bdi_dev_groups
;
242 postcore_initcall(bdi_class_init
);
244 static int __init
default_bdi_init(void)
248 bdi_wq
= alloc_workqueue("writeback", WQ_MEM_RECLAIM
| WQ_FREEZABLE
|
249 WQ_UNBOUND
| WQ_SYSFS
, 0);
253 err
= bdi_init(&noop_backing_dev_info
);
257 subsys_initcall(default_bdi_init
);
260 * This function is used when the first inode for this wb is marked dirty. It
261 * wakes-up the corresponding bdi thread which should then take care of the
262 * periodic background write-out of dirty inodes. Since the write-out would
263 * starts only 'dirty_writeback_interval' centisecs from now anyway, we just
264 * set up a timer which wakes the bdi thread up later.
266 * Note, we wouldn't bother setting up the timer, but this function is on the
267 * fast-path (used by '__mark_inode_dirty()'), so we save few context switches
268 * by delaying the wake-up.
270 * We have to be careful not to postpone flush work if it is scheduled for
271 * earlier. Thus we use queue_delayed_work().
273 void wb_wakeup_delayed(struct bdi_writeback
*wb
)
275 unsigned long timeout
;
277 timeout
= msecs_to_jiffies(dirty_writeback_interval
* 10);
278 spin_lock_bh(&wb
->work_lock
);
279 if (test_bit(WB_registered
, &wb
->state
))
280 queue_delayed_work(bdi_wq
, &wb
->dwork
, timeout
);
281 spin_unlock_bh(&wb
->work_lock
);
285 * Initial write bandwidth: 100 MB/s
287 #define INIT_BW (100 << (20 - PAGE_SHIFT))
289 static int wb_init(struct bdi_writeback
*wb
, struct backing_dev_info
*bdi
,
294 memset(wb
, 0, sizeof(*wb
));
297 wb
->last_old_flush
= jiffies
;
298 INIT_LIST_HEAD(&wb
->b_dirty
);
299 INIT_LIST_HEAD(&wb
->b_io
);
300 INIT_LIST_HEAD(&wb
->b_more_io
);
301 INIT_LIST_HEAD(&wb
->b_dirty_time
);
302 spin_lock_init(&wb
->list_lock
);
304 wb
->bw_time_stamp
= jiffies
;
305 wb
->balanced_dirty_ratelimit
= INIT_BW
;
306 wb
->dirty_ratelimit
= INIT_BW
;
307 wb
->write_bandwidth
= INIT_BW
;
308 wb
->avg_write_bandwidth
= INIT_BW
;
310 spin_lock_init(&wb
->work_lock
);
311 INIT_LIST_HEAD(&wb
->work_list
);
312 INIT_DELAYED_WORK(&wb
->dwork
, wb_workfn
);
314 err
= fprop_local_init_percpu(&wb
->completions
, gfp
);
318 for (i
= 0; i
< NR_WB_STAT_ITEMS
; i
++) {
319 err
= percpu_counter_init(&wb
->stat
[i
], 0, gfp
);
322 percpu_counter_destroy(&wb
->stat
[i
]);
323 fprop_local_destroy_percpu(&wb
->completions
);
332 * Remove bdi from the global list and shutdown any threads we have running
334 static void wb_shutdown(struct bdi_writeback
*wb
)
336 /* Make sure nobody queues further work */
337 spin_lock_bh(&wb
->work_lock
);
338 if (!test_and_clear_bit(WB_registered
, &wb
->state
)) {
339 spin_unlock_bh(&wb
->work_lock
);
342 spin_unlock_bh(&wb
->work_lock
);
345 * Drain work list and shutdown the delayed_work. !WB_registered
346 * tells wb_workfn() that @wb is dying and its work_list needs to
347 * be drained no matter what.
349 mod_delayed_work(bdi_wq
, &wb
->dwork
, 0);
350 flush_delayed_work(&wb
->dwork
);
351 WARN_ON(!list_empty(&wb
->work_list
));
354 static void wb_exit(struct bdi_writeback
*wb
)
358 WARN_ON(delayed_work_pending(&wb
->dwork
));
360 for (i
= 0; i
< NR_WB_STAT_ITEMS
; i
++)
361 percpu_counter_destroy(&wb
->stat
[i
]);
363 fprop_local_destroy_percpu(&wb
->completions
);
366 #ifdef CONFIG_CGROUP_WRITEBACK
368 #include <linux/memcontrol.h>
371 * cgwb_lock protects bdi->cgwb_tree, bdi->cgwb_congested_tree,
372 * blkcg->cgwb_list, and memcg->cgwb_list. bdi->cgwb_tree is also RCU
373 * protected. cgwb_release_wait is used to wait for the completion of cgwb
374 * releases from bdi destruction path.
376 static DEFINE_SPINLOCK(cgwb_lock
);
377 static DECLARE_WAIT_QUEUE_HEAD(cgwb_release_wait
);
380 * wb_congested_get_create - get or create a wb_congested
381 * @bdi: associated bdi
382 * @blkcg_id: ID of the associated blkcg
383 * @gfp: allocation mask
385 * Look up the wb_congested for @blkcg_id on @bdi. If missing, create one.
386 * The returned wb_congested has its reference count incremented. Returns
389 struct bdi_writeback_congested
*
390 wb_congested_get_create(struct backing_dev_info
*bdi
, int blkcg_id
, gfp_t gfp
)
392 struct bdi_writeback_congested
*new_congested
= NULL
, *congested
;
393 struct rb_node
**node
, *parent
;
397 return &bdi
->wb_congested
;
399 spin_lock_irqsave(&cgwb_lock
, flags
);
401 node
= &bdi
->cgwb_congested_tree
.rb_node
;
404 while (*node
!= NULL
) {
406 congested
= container_of(parent
, struct bdi_writeback_congested
,
408 if (congested
->blkcg_id
< blkcg_id
)
409 node
= &parent
->rb_left
;
410 else if (congested
->blkcg_id
> blkcg_id
)
411 node
= &parent
->rb_right
;
417 /* !found and storage for new one already allocated, insert */
418 congested
= new_congested
;
419 new_congested
= NULL
;
420 rb_link_node(&congested
->rb_node
, parent
, node
);
421 rb_insert_color(&congested
->rb_node
, &bdi
->cgwb_congested_tree
);
422 atomic_inc(&bdi
->usage_cnt
);
426 spin_unlock_irqrestore(&cgwb_lock
, flags
);
428 /* allocate storage for new one and retry */
429 new_congested
= kzalloc(sizeof(*new_congested
), gfp
);
433 atomic_set(&new_congested
->refcnt
, 0);
434 new_congested
->bdi
= bdi
;
435 new_congested
->blkcg_id
= blkcg_id
;
439 atomic_inc(&congested
->refcnt
);
440 spin_unlock_irqrestore(&cgwb_lock
, flags
);
441 kfree(new_congested
);
446 * wb_congested_put - put a wb_congested
447 * @congested: wb_congested to put
449 * Put @congested and destroy it if the refcnt reaches zero.
451 void wb_congested_put(struct bdi_writeback_congested
*congested
)
453 struct backing_dev_info
*bdi
= congested
->bdi
;
456 if (congested
->blkcg_id
== 1)
459 local_irq_save(flags
);
460 if (!atomic_dec_and_lock(&congested
->refcnt
, &cgwb_lock
)) {
461 local_irq_restore(flags
);
465 rb_erase(&congested
->rb_node
, &congested
->bdi
->cgwb_congested_tree
);
466 spin_unlock_irqrestore(&cgwb_lock
, flags
);
469 if (atomic_dec_and_test(&bdi
->usage_cnt
))
470 wake_up_all(&cgwb_release_wait
);
473 static void cgwb_release_workfn(struct work_struct
*work
)
475 struct bdi_writeback
*wb
= container_of(work
, struct bdi_writeback
,
477 struct backing_dev_info
*bdi
= wb
->bdi
;
481 css_put(wb
->memcg_css
);
482 css_put(wb
->blkcg_css
);
483 wb_congested_put(wb
->congested
);
485 percpu_ref_exit(&wb
->refcnt
);
489 if (atomic_dec_and_test(&bdi
->usage_cnt
))
490 wake_up_all(&cgwb_release_wait
);
493 static void cgwb_release(struct percpu_ref
*refcnt
)
495 struct bdi_writeback
*wb
= container_of(refcnt
, struct bdi_writeback
,
497 schedule_work(&wb
->release_work
);
500 static void cgwb_kill(struct bdi_writeback
*wb
)
502 lockdep_assert_held(&cgwb_lock
);
504 WARN_ON(!radix_tree_delete(&wb
->bdi
->cgwb_tree
, wb
->memcg_css
->id
));
505 list_del(&wb
->memcg_node
);
506 list_del(&wb
->blkcg_node
);
507 percpu_ref_kill(&wb
->refcnt
);
510 static int cgwb_create(struct backing_dev_info
*bdi
,
511 struct cgroup_subsys_state
*memcg_css
, gfp_t gfp
)
513 struct mem_cgroup
*memcg
;
514 struct cgroup_subsys_state
*blkcg_css
;
516 struct list_head
*memcg_cgwb_list
, *blkcg_cgwb_list
;
517 struct bdi_writeback
*wb
;
521 memcg
= mem_cgroup_from_css(memcg_css
);
522 blkcg_css
= cgroup_get_e_css(memcg_css
->cgroup
, &blkio_cgrp_subsys
);
523 blkcg
= css_to_blkcg(blkcg_css
);
524 memcg_cgwb_list
= mem_cgroup_cgwb_list(memcg
);
525 blkcg_cgwb_list
= &blkcg
->cgwb_list
;
527 /* look up again under lock and discard on blkcg mismatch */
528 spin_lock_irqsave(&cgwb_lock
, flags
);
529 wb
= radix_tree_lookup(&bdi
->cgwb_tree
, memcg_css
->id
);
530 if (wb
&& wb
->blkcg_css
!= blkcg_css
) {
534 spin_unlock_irqrestore(&cgwb_lock
, flags
);
538 /* need to create a new one */
539 wb
= kmalloc(sizeof(*wb
), gfp
);
543 ret
= wb_init(wb
, bdi
, gfp
);
547 ret
= percpu_ref_init(&wb
->refcnt
, cgwb_release
, 0, gfp
);
551 wb
->congested
= wb_congested_get_create(bdi
, blkcg_css
->id
, gfp
);
555 wb
->memcg_css
= memcg_css
;
556 wb
->blkcg_css
= blkcg_css
;
557 INIT_WORK(&wb
->release_work
, cgwb_release_workfn
);
558 set_bit(WB_registered
, &wb
->state
);
561 * The root wb determines the registered state of the whole bdi and
562 * memcg_cgwb_list and blkcg_cgwb_list's next pointers indicate
563 * whether they're still online. Don't link @wb if any is dead.
564 * See wb_memcg_offline() and wb_blkcg_offline().
567 spin_lock_irqsave(&cgwb_lock
, flags
);
568 if (test_bit(WB_registered
, &bdi
->wb
.state
) &&
569 blkcg_cgwb_list
->next
&& memcg_cgwb_list
->next
) {
570 /* we might have raced another instance of this function */
571 ret
= radix_tree_insert(&bdi
->cgwb_tree
, memcg_css
->id
, wb
);
573 atomic_inc(&bdi
->usage_cnt
);
574 list_add(&wb
->memcg_node
, memcg_cgwb_list
);
575 list_add(&wb
->blkcg_node
, blkcg_cgwb_list
);
580 spin_unlock_irqrestore(&cgwb_lock
, flags
);
584 goto err_put_congested
;
589 wb_congested_put(wb
->congested
);
591 percpu_ref_exit(&wb
->refcnt
);
602 * wb_get_create - get wb for a given memcg, create if necessary
604 * @memcg_css: cgroup_subsys_state of the target memcg (must have positive ref)
605 * @gfp: allocation mask to use
607 * Try to get the wb for @memcg_css on @bdi. If it doesn't exist, try to
608 * create one. The returned wb has its refcount incremented.
610 * This function uses css_get() on @memcg_css and thus expects its refcnt
611 * to be positive on invocation. IOW, rcu_read_lock() protection on
612 * @memcg_css isn't enough. try_get it before calling this function.
614 * A wb is keyed by its associated memcg. As blkcg implicitly enables
615 * memcg on the default hierarchy, memcg association is guaranteed to be
616 * more specific (equal or descendant to the associated blkcg) and thus can
617 * identify both the memcg and blkcg associations.
619 * Because the blkcg associated with a memcg may change as blkcg is enabled
620 * and disabled closer to root in the hierarchy, each wb keeps track of
621 * both the memcg and blkcg associated with it and verifies the blkcg on
622 * each lookup. On mismatch, the existing wb is discarded and a new one is
625 struct bdi_writeback
*wb_get_create(struct backing_dev_info
*bdi
,
626 struct cgroup_subsys_state
*memcg_css
,
629 struct bdi_writeback
*wb
;
631 might_sleep_if(gfp
& __GFP_WAIT
);
633 if (!memcg_css
->parent
)
638 wb
= radix_tree_lookup(&bdi
->cgwb_tree
, memcg_css
->id
);
640 struct cgroup_subsys_state
*blkcg_css
;
642 /* see whether the blkcg association has changed */
643 blkcg_css
= cgroup_get_e_css(memcg_css
->cgroup
,
645 if (unlikely(wb
->blkcg_css
!= blkcg_css
||
651 } while (!wb
&& !cgwb_create(bdi
, memcg_css
, gfp
));
656 void __inode_attach_wb(struct inode
*inode
, struct page
*page
)
658 struct backing_dev_info
*bdi
= inode_to_bdi(inode
);
659 struct bdi_writeback
*wb
= NULL
;
661 if (inode_cgwb_enabled(inode
)) {
662 struct cgroup_subsys_state
*memcg_css
;
665 memcg_css
= mem_cgroup_css_from_page(page
);
666 wb
= wb_get_create(bdi
, memcg_css
, GFP_ATOMIC
);
668 /* must pin memcg_css, see wb_get_create() */
669 memcg_css
= task_get_css(current
, memory_cgrp_id
);
670 wb
= wb_get_create(bdi
, memcg_css
, GFP_ATOMIC
);
679 * There may be multiple instances of this function racing to
680 * update the same inode. Use cmpxchg() to tell the winner.
682 if (unlikely(cmpxchg(&inode
->i_wb
, NULL
, wb
)))
686 static void cgwb_bdi_init(struct backing_dev_info
*bdi
)
688 bdi
->wb
.memcg_css
= mem_cgroup_root_css
;
689 bdi
->wb
.blkcg_css
= blkcg_root_css
;
690 bdi
->wb_congested
.blkcg_id
= 1;
691 INIT_RADIX_TREE(&bdi
->cgwb_tree
, GFP_ATOMIC
);
692 bdi
->cgwb_congested_tree
= RB_ROOT
;
693 atomic_set(&bdi
->usage_cnt
, 1);
696 static void cgwb_bdi_destroy(struct backing_dev_info
*bdi
)
698 struct radix_tree_iter iter
;
701 WARN_ON(test_bit(WB_registered
, &bdi
->wb
.state
));
703 spin_lock_irq(&cgwb_lock
);
704 radix_tree_for_each_slot(slot
, &bdi
->cgwb_tree
, &iter
, 0)
706 spin_unlock_irq(&cgwb_lock
);
709 * All cgwb's and their congested states must be shutdown and
710 * released before returning. Drain the usage counter to wait for
711 * all cgwb's and cgwb_congested's ever created on @bdi.
713 atomic_dec(&bdi
->usage_cnt
);
714 wait_event(cgwb_release_wait
, !atomic_read(&bdi
->usage_cnt
));
718 * wb_memcg_offline - kill all wb's associated with a memcg being offlined
719 * @memcg: memcg being offlined
721 * Also prevents creation of any new wb's associated with @memcg.
723 void wb_memcg_offline(struct mem_cgroup
*memcg
)
725 LIST_HEAD(to_destroy
);
726 struct list_head
*memcg_cgwb_list
= mem_cgroup_cgwb_list(memcg
);
727 struct bdi_writeback
*wb
, *next
;
729 spin_lock_irq(&cgwb_lock
);
730 list_for_each_entry_safe(wb
, next
, memcg_cgwb_list
, memcg_node
)
732 memcg_cgwb_list
->next
= NULL
; /* prevent new wb's */
733 spin_unlock_irq(&cgwb_lock
);
737 * wb_blkcg_offline - kill all wb's associated with a blkcg being offlined
738 * @blkcg: blkcg being offlined
740 * Also prevents creation of any new wb's associated with @blkcg.
742 void wb_blkcg_offline(struct blkcg
*blkcg
)
744 LIST_HEAD(to_destroy
);
745 struct bdi_writeback
*wb
, *next
;
747 spin_lock_irq(&cgwb_lock
);
748 list_for_each_entry_safe(wb
, next
, &blkcg
->cgwb_list
, blkcg_node
)
750 blkcg
->cgwb_list
.next
= NULL
; /* prevent new wb's */
751 spin_unlock_irq(&cgwb_lock
);
754 #else /* CONFIG_CGROUP_WRITEBACK */
756 static void cgwb_bdi_init(struct backing_dev_info
*bdi
) { }
757 static void cgwb_bdi_destroy(struct backing_dev_info
*bdi
) { }
759 #endif /* CONFIG_CGROUP_WRITEBACK */
761 int bdi_init(struct backing_dev_info
*bdi
)
768 bdi
->max_ratio
= 100;
769 bdi
->max_prop_frac
= FPROP_FRAC_BASE
;
770 INIT_LIST_HEAD(&bdi
->bdi_list
);
771 init_waitqueue_head(&bdi
->wb_waitq
);
773 err
= wb_init(&bdi
->wb
, bdi
, GFP_KERNEL
);
777 bdi
->wb_congested
.state
= 0;
778 bdi
->wb
.congested
= &bdi
->wb_congested
;
783 EXPORT_SYMBOL(bdi_init
);
785 int bdi_register(struct backing_dev_info
*bdi
, struct device
*parent
,
786 const char *fmt
, ...)
791 if (bdi
->dev
) /* The driver needs to use separate queues per device */
795 dev
= device_create_vargs(bdi_class
, parent
, MKDEV(0, 0), bdi
, fmt
, args
);
802 bdi_debug_register(bdi
, dev_name(dev
));
803 set_bit(WB_registered
, &bdi
->wb
.state
);
805 spin_lock_bh(&bdi_lock
);
806 list_add_tail_rcu(&bdi
->bdi_list
, &bdi_list
);
807 spin_unlock_bh(&bdi_lock
);
809 trace_writeback_bdi_register(bdi
);
812 EXPORT_SYMBOL(bdi_register
);
814 int bdi_register_dev(struct backing_dev_info
*bdi
, dev_t dev
)
816 return bdi_register(bdi
, NULL
, "%u:%u", MAJOR(dev
), MINOR(dev
));
818 EXPORT_SYMBOL(bdi_register_dev
);
821 * Remove bdi from bdi_list, and ensure that it is no longer visible
823 static void bdi_remove_from_list(struct backing_dev_info
*bdi
)
825 spin_lock_bh(&bdi_lock
);
826 list_del_rcu(&bdi
->bdi_list
);
827 spin_unlock_bh(&bdi_lock
);
829 synchronize_rcu_expedited();
833 * Called when the device behind @bdi has been removed or ejected.
835 * We can't really do much here except for reducing the dirty ratio at
836 * the moment. In the future we should be able to set a flag so that
837 * the filesystem can handle errors at mark_inode_dirty time instead
838 * of only at writeback time.
840 void bdi_unregister(struct backing_dev_info
*bdi
)
842 if (WARN_ON_ONCE(!bdi
->dev
))
845 bdi_set_min_ratio(bdi
, 0);
847 EXPORT_SYMBOL(bdi_unregister
);
849 void bdi_destroy(struct backing_dev_info
*bdi
)
851 /* make sure nobody finds us on the bdi_list anymore */
852 bdi_remove_from_list(bdi
);
853 wb_shutdown(&bdi
->wb
);
854 cgwb_bdi_destroy(bdi
);
857 bdi_debug_unregister(bdi
);
858 device_unregister(bdi
->dev
);
864 EXPORT_SYMBOL(bdi_destroy
);
867 * For use from filesystems to quickly init and register a bdi associated
868 * with dirty writeback
870 int bdi_setup_and_register(struct backing_dev_info
*bdi
, char *name
)
875 bdi
->capabilities
= 0;
880 err
= bdi_register(bdi
, NULL
, "%.28s-%ld", name
,
881 atomic_long_inc_return(&bdi_seq
));
889 EXPORT_SYMBOL(bdi_setup_and_register
);
891 static wait_queue_head_t congestion_wqh
[2] = {
892 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh
[0]),
893 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh
[1])
895 static atomic_t nr_wb_congested
[2];
897 void clear_wb_congested(struct bdi_writeback_congested
*congested
, int sync
)
899 wait_queue_head_t
*wqh
= &congestion_wqh
[sync
];
902 bit
= sync
? WB_sync_congested
: WB_async_congested
;
903 if (test_and_clear_bit(bit
, &congested
->state
))
904 atomic_dec(&nr_wb_congested
[sync
]);
905 smp_mb__after_atomic();
906 if (waitqueue_active(wqh
))
909 EXPORT_SYMBOL(clear_wb_congested
);
911 void set_wb_congested(struct bdi_writeback_congested
*congested
, int sync
)
915 bit
= sync
? WB_sync_congested
: WB_async_congested
;
916 if (!test_and_set_bit(bit
, &congested
->state
))
917 atomic_inc(&nr_wb_congested
[sync
]);
919 EXPORT_SYMBOL(set_wb_congested
);
922 * congestion_wait - wait for a backing_dev to become uncongested
923 * @sync: SYNC or ASYNC IO
924 * @timeout: timeout in jiffies
926 * Waits for up to @timeout jiffies for a backing_dev (any backing_dev) to exit
927 * write congestion. If no backing_devs are congested then just wait for the
928 * next write to be completed.
930 long congestion_wait(int sync
, long timeout
)
933 unsigned long start
= jiffies
;
935 wait_queue_head_t
*wqh
= &congestion_wqh
[sync
];
937 prepare_to_wait(wqh
, &wait
, TASK_UNINTERRUPTIBLE
);
938 ret
= io_schedule_timeout(timeout
);
939 finish_wait(wqh
, &wait
);
941 trace_writeback_congestion_wait(jiffies_to_usecs(timeout
),
942 jiffies_to_usecs(jiffies
- start
));
946 EXPORT_SYMBOL(congestion_wait
);
949 * wait_iff_congested - Conditionally wait for a backing_dev to become uncongested or a zone to complete writes
950 * @zone: A zone to check if it is heavily congested
951 * @sync: SYNC or ASYNC IO
952 * @timeout: timeout in jiffies
954 * In the event of a congested backing_dev (any backing_dev) and the given
955 * @zone has experienced recent congestion, this waits for up to @timeout
956 * jiffies for either a BDI to exit congestion of the given @sync queue
957 * or a write to complete.
959 * In the absence of zone congestion, cond_resched() is called to yield
960 * the processor if necessary but otherwise does not sleep.
962 * The return value is 0 if the sleep is for the full timeout. Otherwise,
963 * it is the number of jiffies that were still remaining when the function
964 * returned. return_value == timeout implies the function did not sleep.
966 long wait_iff_congested(struct zone
*zone
, int sync
, long timeout
)
969 unsigned long start
= jiffies
;
971 wait_queue_head_t
*wqh
= &congestion_wqh
[sync
];
974 * If there is no congestion, or heavy congestion is not being
975 * encountered in the current zone, yield if necessary instead
976 * of sleeping on the congestion queue
978 if (atomic_read(&nr_wb_congested
[sync
]) == 0 ||
979 !test_bit(ZONE_CONGESTED
, &zone
->flags
)) {
982 /* In case we scheduled, work out time remaining */
983 ret
= timeout
- (jiffies
- start
);
990 /* Sleep until uncongested or a write happens */
991 prepare_to_wait(wqh
, &wait
, TASK_UNINTERRUPTIBLE
);
992 ret
= io_schedule_timeout(timeout
);
993 finish_wait(wqh
, &wait
);
996 trace_writeback_wait_iff_congested(jiffies_to_usecs(timeout
),
997 jiffies_to_usecs(jiffies
- start
));
1001 EXPORT_SYMBOL(wait_iff_congested
);
1003 int pdflush_proc_obsolete(struct ctl_table
*table
, int write
,
1004 void __user
*buffer
, size_t *lenp
, loff_t
*ppos
)
1006 char kbuf
[] = "0\n";
1008 if (*ppos
|| *lenp
< sizeof(kbuf
)) {
1013 if (copy_to_user(buffer
, kbuf
, sizeof(kbuf
)))
1015 printk_once(KERN_WARNING
"%s exported in /proc is scheduled for removal\n",