1 // SPDX-License-Identifier: GPL-2.0
3 * Common Block IO controller cgroup interface
5 * Based on ideas and code from CFQ, CFS and BFQ:
6 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
8 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
9 * Paolo Valente <paolo.valente@unimore.it>
11 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
12 * Nauman Rafique <nauman@google.com>
14 * For policy-specific per-blkcg data:
15 * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
16 * Arianna Avanzini <avanzini.arianna@gmail.com>
18 #include <linux/ioprio.h>
19 #include <linux/kdev_t.h>
20 #include <linux/module.h>
21 #include <linux/sched/signal.h>
22 #include <linux/err.h>
23 #include <linux/blkdev.h>
24 #include <linux/backing-dev.h>
25 #include <linux/slab.h>
26 #include <linux/genhd.h>
27 #include <linux/delay.h>
28 #include <linux/atomic.h>
29 #include <linux/ctype.h>
30 #include <linux/blk-cgroup.h>
31 #include <linux/tracehook.h>
32 #include <linux/psi.h>
36 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
37 * blkcg_pol_register_mutex nests outside of it and synchronizes entire
38 * policy [un]register operations including cgroup file additions /
39 * removals. Putting cgroup file registration outside blkcg_pol_mutex
40 * allows grabbing it from cgroup callbacks.
42 static DEFINE_MUTEX(blkcg_pol_register_mutex
);
43 static DEFINE_MUTEX(blkcg_pol_mutex
);
45 struct blkcg blkcg_root
;
46 EXPORT_SYMBOL_GPL(blkcg_root
);
48 struct cgroup_subsys_state
* const blkcg_root_css
= &blkcg_root
.css
;
49 EXPORT_SYMBOL_GPL(blkcg_root_css
);
51 static struct blkcg_policy
*blkcg_policy
[BLKCG_MAX_POLS
];
53 static LIST_HEAD(all_blkcgs
); /* protected by blkcg_pol_mutex */
55 bool blkcg_debug_stats
= false;
56 static struct workqueue_struct
*blkcg_punt_bio_wq
;
58 static bool blkcg_policy_enabled(struct request_queue
*q
,
59 const struct blkcg_policy
*pol
)
61 return pol
&& test_bit(pol
->plid
, q
->blkcg_pols
);
65 * blkg_free - free a blkg
68 * Free @blkg which may be partially allocated.
70 static void blkg_free(struct blkcg_gq
*blkg
)
77 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++)
79 blkcg_policy
[i
]->pd_free_fn(blkg
->pd
[i
]);
81 free_percpu(blkg
->iostat_cpu
);
82 percpu_ref_exit(&blkg
->refcnt
);
86 static void __blkg_release(struct rcu_head
*rcu
)
88 struct blkcg_gq
*blkg
= container_of(rcu
, struct blkcg_gq
, rcu_head
);
90 WARN_ON(!bio_list_empty(&blkg
->async_bios
));
92 /* release the blkcg and parent blkg refs this blkg has been holding */
93 css_put(&blkg
->blkcg
->css
);
95 blkg_put(blkg
->parent
);
100 * A group is RCU protected, but having an rcu lock does not mean that one
101 * can access all the fields of blkg and assume these are valid. For
102 * example, don't try to follow throtl_data and request queue links.
104 * Having a reference to blkg under an rcu allows accesses to only values
105 * local to groups like group stats and group rate limits.
107 static void blkg_release(struct percpu_ref
*ref
)
109 struct blkcg_gq
*blkg
= container_of(ref
, struct blkcg_gq
, refcnt
);
111 call_rcu(&blkg
->rcu_head
, __blkg_release
);
114 static void blkg_async_bio_workfn(struct work_struct
*work
)
116 struct blkcg_gq
*blkg
= container_of(work
, struct blkcg_gq
,
118 struct bio_list bios
= BIO_EMPTY_LIST
;
120 struct blk_plug plug
;
121 bool need_plug
= false;
123 /* as long as there are pending bios, @blkg can't go away */
124 spin_lock_bh(&blkg
->async_bio_lock
);
125 bio_list_merge(&bios
, &blkg
->async_bios
);
126 bio_list_init(&blkg
->async_bios
);
127 spin_unlock_bh(&blkg
->async_bio_lock
);
129 /* start plug only when bio_list contains at least 2 bios */
130 if (bios
.head
&& bios
.head
->bi_next
) {
132 blk_start_plug(&plug
);
134 while ((bio
= bio_list_pop(&bios
)))
137 blk_finish_plug(&plug
);
141 * blkg_alloc - allocate a blkg
142 * @blkcg: block cgroup the new blkg is associated with
143 * @q: request_queue the new blkg is associated with
144 * @gfp_mask: allocation mask to use
146 * Allocate a new blkg assocating @blkcg and @q.
148 static struct blkcg_gq
*blkg_alloc(struct blkcg
*blkcg
, struct request_queue
*q
,
151 struct blkcg_gq
*blkg
;
154 /* alloc and init base part */
155 blkg
= kzalloc_node(sizeof(*blkg
), gfp_mask
, q
->node
);
159 if (percpu_ref_init(&blkg
->refcnt
, blkg_release
, 0, gfp_mask
))
162 blkg
->iostat_cpu
= alloc_percpu_gfp(struct blkg_iostat_set
, gfp_mask
);
163 if (!blkg
->iostat_cpu
)
167 INIT_LIST_HEAD(&blkg
->q_node
);
168 spin_lock_init(&blkg
->async_bio_lock
);
169 bio_list_init(&blkg
->async_bios
);
170 INIT_WORK(&blkg
->async_bio_work
, blkg_async_bio_workfn
);
173 u64_stats_init(&blkg
->iostat
.sync
);
174 for_each_possible_cpu(cpu
)
175 u64_stats_init(&per_cpu_ptr(blkg
->iostat_cpu
, cpu
)->sync
);
177 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++) {
178 struct blkcg_policy
*pol
= blkcg_policy
[i
];
179 struct blkg_policy_data
*pd
;
181 if (!blkcg_policy_enabled(q
, pol
))
184 /* alloc per-policy data and attach it to blkg */
185 pd
= pol
->pd_alloc_fn(gfp_mask
, q
, blkcg
);
201 struct blkcg_gq
*blkg_lookup_slowpath(struct blkcg
*blkcg
,
202 struct request_queue
*q
, bool update_hint
)
204 struct blkcg_gq
*blkg
;
207 * Hint didn't match. Look up from the radix tree. Note that the
208 * hint can only be updated under queue_lock as otherwise @blkg
209 * could have already been removed from blkg_tree. The caller is
210 * responsible for grabbing queue_lock if @update_hint.
212 blkg
= radix_tree_lookup(&blkcg
->blkg_tree
, q
->id
);
213 if (blkg
&& blkg
->q
== q
) {
215 lockdep_assert_held(&q
->queue_lock
);
216 rcu_assign_pointer(blkcg
->blkg_hint
, blkg
);
223 EXPORT_SYMBOL_GPL(blkg_lookup_slowpath
);
226 * If @new_blkg is %NULL, this function tries to allocate a new one as
227 * necessary using %GFP_NOWAIT. @new_blkg is always consumed on return.
229 static struct blkcg_gq
*blkg_create(struct blkcg
*blkcg
,
230 struct request_queue
*q
,
231 struct blkcg_gq
*new_blkg
)
233 struct blkcg_gq
*blkg
;
236 WARN_ON_ONCE(!rcu_read_lock_held());
237 lockdep_assert_held(&q
->queue_lock
);
239 /* request_queue is dying, do not create/recreate a blkg */
240 if (blk_queue_dying(q
)) {
245 /* blkg holds a reference to blkcg */
246 if (!css_tryget_online(&blkcg
->css
)) {
253 new_blkg
= blkg_alloc(blkcg
, q
, GFP_NOWAIT
| __GFP_NOWARN
);
254 if (unlikely(!new_blkg
)) {
262 if (blkcg_parent(blkcg
)) {
263 blkg
->parent
= __blkg_lookup(blkcg_parent(blkcg
), q
, false);
264 if (WARN_ON_ONCE(!blkg
->parent
)) {
268 blkg_get(blkg
->parent
);
271 /* invoke per-policy init */
272 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++) {
273 struct blkcg_policy
*pol
= blkcg_policy
[i
];
275 if (blkg
->pd
[i
] && pol
->pd_init_fn
)
276 pol
->pd_init_fn(blkg
->pd
[i
]);
280 spin_lock(&blkcg
->lock
);
281 ret
= radix_tree_insert(&blkcg
->blkg_tree
, q
->id
, blkg
);
283 hlist_add_head_rcu(&blkg
->blkcg_node
, &blkcg
->blkg_list
);
284 list_add(&blkg
->q_node
, &q
->blkg_list
);
286 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++) {
287 struct blkcg_policy
*pol
= blkcg_policy
[i
];
289 if (blkg
->pd
[i
] && pol
->pd_online_fn
)
290 pol
->pd_online_fn(blkg
->pd
[i
]);
294 spin_unlock(&blkcg
->lock
);
299 /* @blkg failed fully initialized, use the usual release path */
304 css_put(&blkcg
->css
);
311 * blkg_lookup_create - lookup blkg, try to create one if not there
312 * @blkcg: blkcg of interest
313 * @q: request_queue of interest
315 * Lookup blkg for the @blkcg - @q pair. If it doesn't exist, try to
316 * create one. blkg creation is performed recursively from blkcg_root such
317 * that all non-root blkg's have access to the parent blkg. This function
318 * should be called under RCU read lock and takes @q->queue_lock.
320 * Returns the blkg or the closest blkg if blkg_create() fails as it walks
323 static struct blkcg_gq
*blkg_lookup_create(struct blkcg
*blkcg
,
324 struct request_queue
*q
)
326 struct blkcg_gq
*blkg
;
329 WARN_ON_ONCE(!rcu_read_lock_held());
331 blkg
= blkg_lookup(blkcg
, q
);
335 spin_lock_irqsave(&q
->queue_lock
, flags
);
336 blkg
= __blkg_lookup(blkcg
, q
, true);
341 * Create blkgs walking down from blkcg_root to @blkcg, so that all
342 * non-root blkgs have access to their parents. Returns the closest
343 * blkg to the intended blkg should blkg_create() fail.
346 struct blkcg
*pos
= blkcg
;
347 struct blkcg
*parent
= blkcg_parent(blkcg
);
348 struct blkcg_gq
*ret_blkg
= q
->root_blkg
;
351 blkg
= __blkg_lookup(parent
, q
, false);
353 /* remember closest blkg */
358 parent
= blkcg_parent(parent
);
361 blkg
= blkg_create(pos
, q
, NULL
);
371 spin_unlock_irqrestore(&q
->queue_lock
, flags
);
375 static void blkg_destroy(struct blkcg_gq
*blkg
)
377 struct blkcg
*blkcg
= blkg
->blkcg
;
380 lockdep_assert_held(&blkg
->q
->queue_lock
);
381 lockdep_assert_held(&blkcg
->lock
);
383 /* Something wrong if we are trying to remove same group twice */
384 WARN_ON_ONCE(list_empty(&blkg
->q_node
));
385 WARN_ON_ONCE(hlist_unhashed(&blkg
->blkcg_node
));
387 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++) {
388 struct blkcg_policy
*pol
= blkcg_policy
[i
];
390 if (blkg
->pd
[i
] && pol
->pd_offline_fn
)
391 pol
->pd_offline_fn(blkg
->pd
[i
]);
394 blkg
->online
= false;
396 radix_tree_delete(&blkcg
->blkg_tree
, blkg
->q
->id
);
397 list_del_init(&blkg
->q_node
);
398 hlist_del_init_rcu(&blkg
->blkcg_node
);
401 * Both setting lookup hint to and clearing it from @blkg are done
402 * under queue_lock. If it's not pointing to @blkg now, it never
403 * will. Hint assignment itself can race safely.
405 if (rcu_access_pointer(blkcg
->blkg_hint
) == blkg
)
406 rcu_assign_pointer(blkcg
->blkg_hint
, NULL
);
409 * Put the reference taken at the time of creation so that when all
410 * queues are gone, group can be destroyed.
412 percpu_ref_kill(&blkg
->refcnt
);
416 * blkg_destroy_all - destroy all blkgs associated with a request_queue
417 * @q: request_queue of interest
419 * Destroy all blkgs associated with @q.
421 static void blkg_destroy_all(struct request_queue
*q
)
423 struct blkcg_gq
*blkg
, *n
;
425 spin_lock_irq(&q
->queue_lock
);
426 list_for_each_entry_safe(blkg
, n
, &q
->blkg_list
, q_node
) {
427 struct blkcg
*blkcg
= blkg
->blkcg
;
429 spin_lock(&blkcg
->lock
);
431 spin_unlock(&blkcg
->lock
);
435 spin_unlock_irq(&q
->queue_lock
);
438 static int blkcg_reset_stats(struct cgroup_subsys_state
*css
,
439 struct cftype
*cftype
, u64 val
)
441 struct blkcg
*blkcg
= css_to_blkcg(css
);
442 struct blkcg_gq
*blkg
;
445 mutex_lock(&blkcg_pol_mutex
);
446 spin_lock_irq(&blkcg
->lock
);
449 * Note that stat reset is racy - it doesn't synchronize against
450 * stat updates. This is a debug feature which shouldn't exist
451 * anyway. If you get hit by a race, retry.
453 hlist_for_each_entry(blkg
, &blkcg
->blkg_list
, blkcg_node
) {
454 for_each_possible_cpu(cpu
) {
455 struct blkg_iostat_set
*bis
=
456 per_cpu_ptr(blkg
->iostat_cpu
, cpu
);
457 memset(bis
, 0, sizeof(*bis
));
459 memset(&blkg
->iostat
, 0, sizeof(blkg
->iostat
));
461 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++) {
462 struct blkcg_policy
*pol
= blkcg_policy
[i
];
464 if (blkg
->pd
[i
] && pol
->pd_reset_stats_fn
)
465 pol
->pd_reset_stats_fn(blkg
->pd
[i
]);
469 spin_unlock_irq(&blkcg
->lock
);
470 mutex_unlock(&blkcg_pol_mutex
);
474 const char *blkg_dev_name(struct blkcg_gq
*blkg
)
476 /* some drivers (floppy) instantiate a queue w/o disk registered */
477 if (blkg
->q
->backing_dev_info
->dev
)
478 return bdi_dev_name(blkg
->q
->backing_dev_info
);
483 * blkcg_print_blkgs - helper for printing per-blkg data
484 * @sf: seq_file to print to
485 * @blkcg: blkcg of interest
486 * @prfill: fill function to print out a blkg
487 * @pol: policy in question
488 * @data: data to be passed to @prfill
489 * @show_total: to print out sum of prfill return values or not
491 * This function invokes @prfill on each blkg of @blkcg if pd for the
492 * policy specified by @pol exists. @prfill is invoked with @sf, the
493 * policy data and @data and the matching queue lock held. If @show_total
494 * is %true, the sum of the return values from @prfill is printed with
495 * "Total" label at the end.
497 * This is to be used to construct print functions for
498 * cftype->read_seq_string method.
500 void blkcg_print_blkgs(struct seq_file
*sf
, struct blkcg
*blkcg
,
501 u64 (*prfill
)(struct seq_file
*,
502 struct blkg_policy_data
*, int),
503 const struct blkcg_policy
*pol
, int data
,
506 struct blkcg_gq
*blkg
;
510 hlist_for_each_entry_rcu(blkg
, &blkcg
->blkg_list
, blkcg_node
) {
511 spin_lock_irq(&blkg
->q
->queue_lock
);
512 if (blkcg_policy_enabled(blkg
->q
, pol
))
513 total
+= prfill(sf
, blkg
->pd
[pol
->plid
], data
);
514 spin_unlock_irq(&blkg
->q
->queue_lock
);
519 seq_printf(sf
, "Total %llu\n", (unsigned long long)total
);
521 EXPORT_SYMBOL_GPL(blkcg_print_blkgs
);
524 * __blkg_prfill_u64 - prfill helper for a single u64 value
525 * @sf: seq_file to print to
526 * @pd: policy private data of interest
529 * Print @v to @sf for the device assocaited with @pd.
531 u64
__blkg_prfill_u64(struct seq_file
*sf
, struct blkg_policy_data
*pd
, u64 v
)
533 const char *dname
= blkg_dev_name(pd
->blkg
);
538 seq_printf(sf
, "%s %llu\n", dname
, (unsigned long long)v
);
541 EXPORT_SYMBOL_GPL(__blkg_prfill_u64
);
543 /* Performs queue bypass and policy enabled checks then looks up blkg. */
544 static struct blkcg_gq
*blkg_lookup_check(struct blkcg
*blkcg
,
545 const struct blkcg_policy
*pol
,
546 struct request_queue
*q
)
548 WARN_ON_ONCE(!rcu_read_lock_held());
549 lockdep_assert_held(&q
->queue_lock
);
551 if (!blkcg_policy_enabled(q
, pol
))
552 return ERR_PTR(-EOPNOTSUPP
);
553 return __blkg_lookup(blkcg
, q
, true /* update_hint */);
557 * blkcg_conf_open_bdev - parse and open bdev for per-blkg config update
558 * @inputp: input string pointer
560 * Parse the device node prefix part, MAJ:MIN, of per-blkg config update
561 * from @input and get and return the matching bdev. *@inputp is
562 * updated to point past the device node prefix. Returns an ERR_PTR()
565 * Use this function iff blkg_conf_prep() can't be used for some reason.
567 struct block_device
*blkcg_conf_open_bdev(char **inputp
)
569 char *input
= *inputp
;
570 unsigned int major
, minor
;
571 struct block_device
*bdev
;
574 if (sscanf(input
, "%u:%u%n", &major
, &minor
, &key_len
) != 2)
575 return ERR_PTR(-EINVAL
);
578 if (!isspace(*input
))
579 return ERR_PTR(-EINVAL
);
580 input
= skip_spaces(input
);
582 bdev
= blkdev_get_no_open(MKDEV(major
, minor
));
584 return ERR_PTR(-ENODEV
);
585 if (bdev_is_partition(bdev
)) {
586 blkdev_put_no_open(bdev
);
587 return ERR_PTR(-ENODEV
);
595 * blkg_conf_prep - parse and prepare for per-blkg config update
596 * @blkcg: target block cgroup
597 * @pol: target policy
598 * @input: input string
599 * @ctx: blkg_conf_ctx to be filled
601 * Parse per-blkg config update from @input and initialize @ctx with the
602 * result. @ctx->blkg points to the blkg to be updated and @ctx->body the
603 * part of @input following MAJ:MIN. This function returns with RCU read
604 * lock and queue lock held and must be paired with blkg_conf_finish().
606 int blkg_conf_prep(struct blkcg
*blkcg
, const struct blkcg_policy
*pol
,
607 char *input
, struct blkg_conf_ctx
*ctx
)
608 __acquires(rcu
) __acquires(&bdev
->bd_disk
->queue
->queue_lock
)
610 struct block_device
*bdev
;
611 struct request_queue
*q
;
612 struct blkcg_gq
*blkg
;
615 bdev
= blkcg_conf_open_bdev(&input
);
617 return PTR_ERR(bdev
);
619 q
= bdev
->bd_disk
->queue
;
622 spin_lock_irq(&q
->queue_lock
);
624 blkg
= blkg_lookup_check(blkcg
, pol
, q
);
634 * Create blkgs walking down from blkcg_root to @blkcg, so that all
635 * non-root blkgs have access to their parents.
638 struct blkcg
*pos
= blkcg
;
639 struct blkcg
*parent
;
640 struct blkcg_gq
*new_blkg
;
642 parent
= blkcg_parent(blkcg
);
643 while (parent
&& !__blkg_lookup(parent
, q
, false)) {
645 parent
= blkcg_parent(parent
);
648 /* Drop locks to do new blkg allocation with GFP_KERNEL. */
649 spin_unlock_irq(&q
->queue_lock
);
652 new_blkg
= blkg_alloc(pos
, q
, GFP_KERNEL
);
653 if (unlikely(!new_blkg
)) {
658 if (radix_tree_preload(GFP_KERNEL
)) {
665 spin_lock_irq(&q
->queue_lock
);
667 blkg
= blkg_lookup_check(pos
, pol
, q
);
677 blkg
= blkg_create(pos
, q
, new_blkg
);
684 radix_tree_preload_end();
696 radix_tree_preload_end();
698 spin_unlock_irq(&q
->queue_lock
);
701 blkdev_put_no_open(bdev
);
703 * If queue was bypassing, we should retry. Do so after a
704 * short msleep(). It isn't strictly necessary but queue
705 * can be bypassing for some time and it's always nice to
706 * avoid busy looping.
710 ret
= restart_syscall();
714 EXPORT_SYMBOL_GPL(blkg_conf_prep
);
717 * blkg_conf_finish - finish up per-blkg config update
718 * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep()
720 * Finish up after per-blkg config update. This function must be paired
721 * with blkg_conf_prep().
723 void blkg_conf_finish(struct blkg_conf_ctx
*ctx
)
724 __releases(&ctx
->bdev
->bd_disk
->queue
->queue_lock
) __releases(rcu
)
726 spin_unlock_irq(&ctx
->bdev
->bd_disk
->queue
->queue_lock
);
728 blkdev_put_no_open(ctx
->bdev
);
730 EXPORT_SYMBOL_GPL(blkg_conf_finish
);
732 static void blkg_iostat_set(struct blkg_iostat
*dst
, struct blkg_iostat
*src
)
736 for (i
= 0; i
< BLKG_IOSTAT_NR
; i
++) {
737 dst
->bytes
[i
] = src
->bytes
[i
];
738 dst
->ios
[i
] = src
->ios
[i
];
742 static void blkg_iostat_add(struct blkg_iostat
*dst
, struct blkg_iostat
*src
)
746 for (i
= 0; i
< BLKG_IOSTAT_NR
; i
++) {
747 dst
->bytes
[i
] += src
->bytes
[i
];
748 dst
->ios
[i
] += src
->ios
[i
];
752 static void blkg_iostat_sub(struct blkg_iostat
*dst
, struct blkg_iostat
*src
)
756 for (i
= 0; i
< BLKG_IOSTAT_NR
; i
++) {
757 dst
->bytes
[i
] -= src
->bytes
[i
];
758 dst
->ios
[i
] -= src
->ios
[i
];
762 static void blkcg_rstat_flush(struct cgroup_subsys_state
*css
, int cpu
)
764 struct blkcg
*blkcg
= css_to_blkcg(css
);
765 struct blkcg_gq
*blkg
;
769 hlist_for_each_entry_rcu(blkg
, &blkcg
->blkg_list
, blkcg_node
) {
770 struct blkcg_gq
*parent
= blkg
->parent
;
771 struct blkg_iostat_set
*bisc
= per_cpu_ptr(blkg
->iostat_cpu
, cpu
);
772 struct blkg_iostat cur
, delta
;
775 /* fetch the current per-cpu values */
777 seq
= u64_stats_fetch_begin(&bisc
->sync
);
778 blkg_iostat_set(&cur
, &bisc
->cur
);
779 } while (u64_stats_fetch_retry(&bisc
->sync
, seq
));
781 /* propagate percpu delta to global */
782 u64_stats_update_begin(&blkg
->iostat
.sync
);
783 blkg_iostat_set(&delta
, &cur
);
784 blkg_iostat_sub(&delta
, &bisc
->last
);
785 blkg_iostat_add(&blkg
->iostat
.cur
, &delta
);
786 blkg_iostat_add(&bisc
->last
, &delta
);
787 u64_stats_update_end(&blkg
->iostat
.sync
);
789 /* propagate global delta to parent */
791 u64_stats_update_begin(&parent
->iostat
.sync
);
792 blkg_iostat_set(&delta
, &blkg
->iostat
.cur
);
793 blkg_iostat_sub(&delta
, &blkg
->iostat
.last
);
794 blkg_iostat_add(&parent
->iostat
.cur
, &delta
);
795 blkg_iostat_add(&blkg
->iostat
.last
, &delta
);
796 u64_stats_update_end(&parent
->iostat
.sync
);
804 * The rstat algorithms intentionally don't handle the root cgroup to avoid
805 * incurring overhead when no cgroups are defined. For that reason,
806 * cgroup_rstat_flush in blkcg_print_stat does not actually fill out the
807 * iostat in the root cgroup's blkcg_gq.
809 * However, we would like to re-use the printing code between the root and
810 * non-root cgroups to the extent possible. For that reason, we simulate
811 * flushing the root cgroup's stats by explicitly filling in the iostat
812 * with disk level statistics.
814 static void blkcg_fill_root_iostats(void)
816 struct class_dev_iter iter
;
819 class_dev_iter_init(&iter
, &block_class
, NULL
, &disk_type
);
820 while ((dev
= class_dev_iter_next(&iter
))) {
821 struct block_device
*bdev
= dev_to_bdev(dev
);
822 struct blkcg_gq
*blkg
=
823 blk_queue_root_blkg(bdev
->bd_disk
->queue
);
824 struct blkg_iostat tmp
;
827 memset(&tmp
, 0, sizeof(tmp
));
828 for_each_possible_cpu(cpu
) {
829 struct disk_stats
*cpu_dkstats
;
831 cpu_dkstats
= per_cpu_ptr(bdev
->bd_stats
, cpu
);
832 tmp
.ios
[BLKG_IOSTAT_READ
] +=
833 cpu_dkstats
->ios
[STAT_READ
];
834 tmp
.ios
[BLKG_IOSTAT_WRITE
] +=
835 cpu_dkstats
->ios
[STAT_WRITE
];
836 tmp
.ios
[BLKG_IOSTAT_DISCARD
] +=
837 cpu_dkstats
->ios
[STAT_DISCARD
];
838 // convert sectors to bytes
839 tmp
.bytes
[BLKG_IOSTAT_READ
] +=
840 cpu_dkstats
->sectors
[STAT_READ
] << 9;
841 tmp
.bytes
[BLKG_IOSTAT_WRITE
] +=
842 cpu_dkstats
->sectors
[STAT_WRITE
] << 9;
843 tmp
.bytes
[BLKG_IOSTAT_DISCARD
] +=
844 cpu_dkstats
->sectors
[STAT_DISCARD
] << 9;
846 u64_stats_update_begin(&blkg
->iostat
.sync
);
847 blkg_iostat_set(&blkg
->iostat
.cur
, &tmp
);
848 u64_stats_update_end(&blkg
->iostat
.sync
);
853 static int blkcg_print_stat(struct seq_file
*sf
, void *v
)
855 struct blkcg
*blkcg
= css_to_blkcg(seq_css(sf
));
856 struct blkcg_gq
*blkg
;
858 if (!seq_css(sf
)->parent
)
859 blkcg_fill_root_iostats();
861 cgroup_rstat_flush(blkcg
->css
.cgroup
);
865 hlist_for_each_entry_rcu(blkg
, &blkcg
->blkg_list
, blkcg_node
) {
866 struct blkg_iostat_set
*bis
= &blkg
->iostat
;
869 u64 rbytes
, wbytes
, rios
, wios
, dbytes
, dios
;
870 size_t size
= seq_get_buf(sf
, &buf
), off
= 0;
872 bool has_stats
= false;
875 spin_lock_irq(&blkg
->q
->queue_lock
);
880 dname
= blkg_dev_name(blkg
);
885 * Hooray string manipulation, count is the size written NOT
886 * INCLUDING THE \0, so size is now count+1 less than what we
887 * had before, but we want to start writing the next bit from
888 * the \0 so we only add count to buf.
890 off
+= scnprintf(buf
+off
, size
-off
, "%s ", dname
);
893 seq
= u64_stats_fetch_begin(&bis
->sync
);
895 rbytes
= bis
->cur
.bytes
[BLKG_IOSTAT_READ
];
896 wbytes
= bis
->cur
.bytes
[BLKG_IOSTAT_WRITE
];
897 dbytes
= bis
->cur
.bytes
[BLKG_IOSTAT_DISCARD
];
898 rios
= bis
->cur
.ios
[BLKG_IOSTAT_READ
];
899 wios
= bis
->cur
.ios
[BLKG_IOSTAT_WRITE
];
900 dios
= bis
->cur
.ios
[BLKG_IOSTAT_DISCARD
];
901 } while (u64_stats_fetch_retry(&bis
->sync
, seq
));
903 if (rbytes
|| wbytes
|| rios
|| wios
) {
905 off
+= scnprintf(buf
+off
, size
-off
,
906 "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",
907 rbytes
, wbytes
, rios
, wios
,
911 if (blkcg_debug_stats
&& atomic_read(&blkg
->use_delay
)) {
913 off
+= scnprintf(buf
+off
, size
-off
,
914 " use_delay=%d delay_nsec=%llu",
915 atomic_read(&blkg
->use_delay
),
916 (unsigned long long)atomic64_read(&blkg
->delay_nsec
));
919 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++) {
920 struct blkcg_policy
*pol
= blkcg_policy
[i
];
923 if (!blkg
->pd
[i
] || !pol
->pd_stat_fn
)
926 written
= pol
->pd_stat_fn(blkg
->pd
[i
], buf
+off
, size
-off
);
933 if (off
< size
- 1) {
934 off
+= scnprintf(buf
+off
, size
-off
, "\n");
941 spin_unlock_irq(&blkg
->q
->queue_lock
);
948 static struct cftype blkcg_files
[] = {
951 .seq_show
= blkcg_print_stat
,
956 static struct cftype blkcg_legacy_files
[] = {
958 .name
= "reset_stats",
959 .write_u64
= blkcg_reset_stats
,
965 * blkcg destruction is a three-stage process.
967 * 1. Destruction starts. The blkcg_css_offline() callback is invoked
968 * which offlines writeback. Here we tie the next stage of blkg destruction
969 * to the completion of writeback associated with the blkcg. This lets us
970 * avoid punting potentially large amounts of outstanding writeback to root
971 * while maintaining any ongoing policies. The next stage is triggered when
972 * the nr_cgwbs count goes to zero.
974 * 2. When the nr_cgwbs count goes to zero, blkcg_destroy_blkgs() is called
975 * and handles the destruction of blkgs. Here the css reference held by
976 * the blkg is put back eventually allowing blkcg_css_free() to be called.
977 * This work may occur in cgwb_release_workfn() on the cgwb_release
978 * workqueue. Any submitted ios that fail to get the blkg ref will be
979 * punted to the root_blkg.
981 * 3. Once the blkcg ref count goes to zero, blkcg_css_free() is called.
982 * This finally frees the blkcg.
986 * blkcg_css_offline - cgroup css_offline callback
987 * @css: css of interest
989 * This function is called when @css is about to go away. Here the cgwbs are
990 * offlined first and only once writeback associated with the blkcg has
991 * finished do we start step 2 (see above).
993 static void blkcg_css_offline(struct cgroup_subsys_state
*css
)
995 struct blkcg
*blkcg
= css_to_blkcg(css
);
997 /* this prevents anyone from attaching or migrating to this blkcg */
998 wb_blkcg_offline(blkcg
);
1000 /* put the base online pin allowing step 2 to be triggered */
1001 blkcg_unpin_online(blkcg
);
1005 * blkcg_destroy_blkgs - responsible for shooting down blkgs
1006 * @blkcg: blkcg of interest
1008 * blkgs should be removed while holding both q and blkcg locks. As blkcg lock
1009 * is nested inside q lock, this function performs reverse double lock dancing.
1010 * Destroying the blkgs releases the reference held on the blkcg's css allowing
1011 * blkcg_css_free to eventually be called.
1013 * This is the blkcg counterpart of ioc_release_fn().
1015 void blkcg_destroy_blkgs(struct blkcg
*blkcg
)
1019 spin_lock_irq(&blkcg
->lock
);
1021 while (!hlist_empty(&blkcg
->blkg_list
)) {
1022 struct blkcg_gq
*blkg
= hlist_entry(blkcg
->blkg_list
.first
,
1023 struct blkcg_gq
, blkcg_node
);
1024 struct request_queue
*q
= blkg
->q
;
1026 if (need_resched() || !spin_trylock(&q
->queue_lock
)) {
1028 * Given that the system can accumulate a huge number
1029 * of blkgs in pathological cases, check to see if we
1030 * need to rescheduling to avoid softlockup.
1032 spin_unlock_irq(&blkcg
->lock
);
1034 spin_lock_irq(&blkcg
->lock
);
1039 spin_unlock(&q
->queue_lock
);
1042 spin_unlock_irq(&blkcg
->lock
);
1045 static void blkcg_css_free(struct cgroup_subsys_state
*css
)
1047 struct blkcg
*blkcg
= css_to_blkcg(css
);
1050 mutex_lock(&blkcg_pol_mutex
);
1052 list_del(&blkcg
->all_blkcgs_node
);
1054 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++)
1056 blkcg_policy
[i
]->cpd_free_fn(blkcg
->cpd
[i
]);
1058 mutex_unlock(&blkcg_pol_mutex
);
1063 static struct cgroup_subsys_state
*
1064 blkcg_css_alloc(struct cgroup_subsys_state
*parent_css
)
1066 struct blkcg
*blkcg
;
1067 struct cgroup_subsys_state
*ret
;
1070 mutex_lock(&blkcg_pol_mutex
);
1073 blkcg
= &blkcg_root
;
1075 blkcg
= kzalloc(sizeof(*blkcg
), GFP_KERNEL
);
1077 ret
= ERR_PTR(-ENOMEM
);
1082 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++) {
1083 struct blkcg_policy
*pol
= blkcg_policy
[i
];
1084 struct blkcg_policy_data
*cpd
;
1087 * If the policy hasn't been attached yet, wait for it
1088 * to be attached before doing anything else. Otherwise,
1089 * check if the policy requires any specific per-cgroup
1090 * data: if it does, allocate and initialize it.
1092 if (!pol
|| !pol
->cpd_alloc_fn
)
1095 cpd
= pol
->cpd_alloc_fn(GFP_KERNEL
);
1097 ret
= ERR_PTR(-ENOMEM
);
1100 blkcg
->cpd
[i
] = cpd
;
1103 if (pol
->cpd_init_fn
)
1104 pol
->cpd_init_fn(cpd
);
1107 spin_lock_init(&blkcg
->lock
);
1108 refcount_set(&blkcg
->online_pin
, 1);
1109 INIT_RADIX_TREE(&blkcg
->blkg_tree
, GFP_NOWAIT
| __GFP_NOWARN
);
1110 INIT_HLIST_HEAD(&blkcg
->blkg_list
);
1111 #ifdef CONFIG_CGROUP_WRITEBACK
1112 INIT_LIST_HEAD(&blkcg
->cgwb_list
);
1114 list_add_tail(&blkcg
->all_blkcgs_node
, &all_blkcgs
);
1116 mutex_unlock(&blkcg_pol_mutex
);
1120 for (i
--; i
>= 0; i
--)
1122 blkcg_policy
[i
]->cpd_free_fn(blkcg
->cpd
[i
]);
1124 if (blkcg
!= &blkcg_root
)
1127 mutex_unlock(&blkcg_pol_mutex
);
1131 static int blkcg_css_online(struct cgroup_subsys_state
*css
)
1133 struct blkcg
*blkcg
= css_to_blkcg(css
);
1134 struct blkcg
*parent
= blkcg_parent(blkcg
);
1137 * blkcg_pin_online() is used to delay blkcg offline so that blkgs
1138 * don't go offline while cgwbs are still active on them. Pin the
1139 * parent so that offline always happens towards the root.
1142 blkcg_pin_online(parent
);
1147 * blkcg_init_queue - initialize blkcg part of request queue
1148 * @q: request_queue to initialize
1150 * Called from blk_alloc_queue(). Responsible for initializing blkcg
1151 * part of new request_queue @q.
1154 * 0 on success, -errno on failure.
1156 int blkcg_init_queue(struct request_queue
*q
)
1158 struct blkcg_gq
*new_blkg
, *blkg
;
1162 new_blkg
= blkg_alloc(&blkcg_root
, q
, GFP_KERNEL
);
1166 preloaded
= !radix_tree_preload(GFP_KERNEL
);
1168 /* Make sure the root blkg exists. */
1170 spin_lock_irq(&q
->queue_lock
);
1171 blkg
= blkg_create(&blkcg_root
, q
, new_blkg
);
1174 q
->root_blkg
= blkg
;
1175 spin_unlock_irq(&q
->queue_lock
);
1179 radix_tree_preload_end();
1181 ret
= blk_throtl_init(q
);
1183 goto err_destroy_all
;
1185 ret
= blk_iolatency_init(q
);
1188 goto err_destroy_all
;
1193 blkg_destroy_all(q
);
1196 spin_unlock_irq(&q
->queue_lock
);
1199 radix_tree_preload_end();
1200 return PTR_ERR(blkg
);
1204 * blkcg_exit_queue - exit and release blkcg part of request_queue
1205 * @q: request_queue being released
1207 * Called from blk_exit_queue(). Responsible for exiting blkcg part.
1209 void blkcg_exit_queue(struct request_queue
*q
)
1211 blkg_destroy_all(q
);
1216 * We cannot support shared io contexts, as we have no mean to support
1217 * two tasks with the same ioc in two different groups without major rework
1218 * of the main cic data structures. For now we allow a task to change
1219 * its cgroup only if it's the only owner of its ioc.
1221 static int blkcg_can_attach(struct cgroup_taskset
*tset
)
1223 struct task_struct
*task
;
1224 struct cgroup_subsys_state
*dst_css
;
1225 struct io_context
*ioc
;
1228 /* task_lock() is needed to avoid races with exit_io_context() */
1229 cgroup_taskset_for_each(task
, dst_css
, tset
) {
1231 ioc
= task
->io_context
;
1232 if (ioc
&& atomic_read(&ioc
->nr_tasks
) > 1)
1241 static void blkcg_bind(struct cgroup_subsys_state
*root_css
)
1245 mutex_lock(&blkcg_pol_mutex
);
1247 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++) {
1248 struct blkcg_policy
*pol
= blkcg_policy
[i
];
1249 struct blkcg
*blkcg
;
1251 if (!pol
|| !pol
->cpd_bind_fn
)
1254 list_for_each_entry(blkcg
, &all_blkcgs
, all_blkcgs_node
)
1255 if (blkcg
->cpd
[pol
->plid
])
1256 pol
->cpd_bind_fn(blkcg
->cpd
[pol
->plid
]);
1258 mutex_unlock(&blkcg_pol_mutex
);
1261 static void blkcg_exit(struct task_struct
*tsk
)
1263 if (tsk
->throttle_queue
)
1264 blk_put_queue(tsk
->throttle_queue
);
1265 tsk
->throttle_queue
= NULL
;
1268 struct cgroup_subsys io_cgrp_subsys
= {
1269 .css_alloc
= blkcg_css_alloc
,
1270 .css_online
= blkcg_css_online
,
1271 .css_offline
= blkcg_css_offline
,
1272 .css_free
= blkcg_css_free
,
1273 .can_attach
= blkcg_can_attach
,
1274 .css_rstat_flush
= blkcg_rstat_flush
,
1276 .dfl_cftypes
= blkcg_files
,
1277 .legacy_cftypes
= blkcg_legacy_files
,
1278 .legacy_name
= "blkio",
1282 * This ensures that, if available, memcg is automatically enabled
1283 * together on the default hierarchy so that the owner cgroup can
1284 * be retrieved from writeback pages.
1286 .depends_on
= 1 << memory_cgrp_id
,
1289 EXPORT_SYMBOL_GPL(io_cgrp_subsys
);
1292 * blkcg_activate_policy - activate a blkcg policy on a request_queue
1293 * @q: request_queue of interest
1294 * @pol: blkcg policy to activate
1296 * Activate @pol on @q. Requires %GFP_KERNEL context. @q goes through
1297 * bypass mode to populate its blkgs with policy_data for @pol.
1299 * Activation happens with @q bypassed, so nobody would be accessing blkgs
1300 * from IO path. Update of each blkg is protected by both queue and blkcg
1301 * locks so that holding either lock and testing blkcg_policy_enabled() is
1302 * always enough for dereferencing policy data.
1304 * The caller is responsible for synchronizing [de]activations and policy
1305 * [un]registerations. Returns 0 on success, -errno on failure.
1307 int blkcg_activate_policy(struct request_queue
*q
,
1308 const struct blkcg_policy
*pol
)
1310 struct blkg_policy_data
*pd_prealloc
= NULL
;
1311 struct blkcg_gq
*blkg
, *pinned_blkg
= NULL
;
1314 if (blkcg_policy_enabled(q
, pol
))
1318 blk_mq_freeze_queue(q
);
1320 spin_lock_irq(&q
->queue_lock
);
1322 /* blkg_list is pushed at the head, reverse walk to allocate parents first */
1323 list_for_each_entry_reverse(blkg
, &q
->blkg_list
, q_node
) {
1324 struct blkg_policy_data
*pd
;
1326 if (blkg
->pd
[pol
->plid
])
1329 /* If prealloc matches, use it; otherwise try GFP_NOWAIT */
1330 if (blkg
== pinned_blkg
) {
1334 pd
= pol
->pd_alloc_fn(GFP_NOWAIT
| __GFP_NOWARN
, q
,
1340 * GFP_NOWAIT failed. Free the existing one and
1341 * prealloc for @blkg w/ GFP_KERNEL.
1344 blkg_put(pinned_blkg
);
1348 spin_unlock_irq(&q
->queue_lock
);
1351 pol
->pd_free_fn(pd_prealloc
);
1352 pd_prealloc
= pol
->pd_alloc_fn(GFP_KERNEL
, q
,
1360 blkg
->pd
[pol
->plid
] = pd
;
1362 pd
->plid
= pol
->plid
;
1365 /* all allocated, init in the same order */
1366 if (pol
->pd_init_fn
)
1367 list_for_each_entry_reverse(blkg
, &q
->blkg_list
, q_node
)
1368 pol
->pd_init_fn(blkg
->pd
[pol
->plid
]);
1370 __set_bit(pol
->plid
, q
->blkcg_pols
);
1373 spin_unlock_irq(&q
->queue_lock
);
1376 blk_mq_unfreeze_queue(q
);
1378 blkg_put(pinned_blkg
);
1380 pol
->pd_free_fn(pd_prealloc
);
1384 /* alloc failed, nothing's initialized yet, free everything */
1385 spin_lock_irq(&q
->queue_lock
);
1386 list_for_each_entry(blkg
, &q
->blkg_list
, q_node
) {
1387 if (blkg
->pd
[pol
->plid
]) {
1388 pol
->pd_free_fn(blkg
->pd
[pol
->plid
]);
1389 blkg
->pd
[pol
->plid
] = NULL
;
1392 spin_unlock_irq(&q
->queue_lock
);
1396 EXPORT_SYMBOL_GPL(blkcg_activate_policy
);
1399 * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue
1400 * @q: request_queue of interest
1401 * @pol: blkcg policy to deactivate
1403 * Deactivate @pol on @q. Follows the same synchronization rules as
1404 * blkcg_activate_policy().
1406 void blkcg_deactivate_policy(struct request_queue
*q
,
1407 const struct blkcg_policy
*pol
)
1409 struct blkcg_gq
*blkg
;
1411 if (!blkcg_policy_enabled(q
, pol
))
1415 blk_mq_freeze_queue(q
);
1417 spin_lock_irq(&q
->queue_lock
);
1419 __clear_bit(pol
->plid
, q
->blkcg_pols
);
1421 list_for_each_entry(blkg
, &q
->blkg_list
, q_node
) {
1422 if (blkg
->pd
[pol
->plid
]) {
1423 if (pol
->pd_offline_fn
)
1424 pol
->pd_offline_fn(blkg
->pd
[pol
->plid
]);
1425 pol
->pd_free_fn(blkg
->pd
[pol
->plid
]);
1426 blkg
->pd
[pol
->plid
] = NULL
;
1430 spin_unlock_irq(&q
->queue_lock
);
1433 blk_mq_unfreeze_queue(q
);
1435 EXPORT_SYMBOL_GPL(blkcg_deactivate_policy
);
1438 * blkcg_policy_register - register a blkcg policy
1439 * @pol: blkcg policy to register
1441 * Register @pol with blkcg core. Might sleep and @pol may be modified on
1442 * successful registration. Returns 0 on success and -errno on failure.
1444 int blkcg_policy_register(struct blkcg_policy
*pol
)
1446 struct blkcg
*blkcg
;
1449 mutex_lock(&blkcg_pol_register_mutex
);
1450 mutex_lock(&blkcg_pol_mutex
);
1452 /* find an empty slot */
1454 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++)
1455 if (!blkcg_policy
[i
])
1457 if (i
>= BLKCG_MAX_POLS
) {
1458 pr_warn("blkcg_policy_register: BLKCG_MAX_POLS too small\n");
1462 /* Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs */
1463 if ((!pol
->cpd_alloc_fn
^ !pol
->cpd_free_fn
) ||
1464 (!pol
->pd_alloc_fn
^ !pol
->pd_free_fn
))
1469 blkcg_policy
[pol
->plid
] = pol
;
1471 /* allocate and install cpd's */
1472 if (pol
->cpd_alloc_fn
) {
1473 list_for_each_entry(blkcg
, &all_blkcgs
, all_blkcgs_node
) {
1474 struct blkcg_policy_data
*cpd
;
1476 cpd
= pol
->cpd_alloc_fn(GFP_KERNEL
);
1480 blkcg
->cpd
[pol
->plid
] = cpd
;
1482 cpd
->plid
= pol
->plid
;
1483 if (pol
->cpd_init_fn
)
1484 pol
->cpd_init_fn(cpd
);
1488 mutex_unlock(&blkcg_pol_mutex
);
1490 /* everything is in place, add intf files for the new policy */
1491 if (pol
->dfl_cftypes
)
1492 WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys
,
1494 if (pol
->legacy_cftypes
)
1495 WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys
,
1496 pol
->legacy_cftypes
));
1497 mutex_unlock(&blkcg_pol_register_mutex
);
1501 if (pol
->cpd_free_fn
) {
1502 list_for_each_entry(blkcg
, &all_blkcgs
, all_blkcgs_node
) {
1503 if (blkcg
->cpd
[pol
->plid
]) {
1504 pol
->cpd_free_fn(blkcg
->cpd
[pol
->plid
]);
1505 blkcg
->cpd
[pol
->plid
] = NULL
;
1509 blkcg_policy
[pol
->plid
] = NULL
;
1511 mutex_unlock(&blkcg_pol_mutex
);
1512 mutex_unlock(&blkcg_pol_register_mutex
);
1515 EXPORT_SYMBOL_GPL(blkcg_policy_register
);
1518 * blkcg_policy_unregister - unregister a blkcg policy
1519 * @pol: blkcg policy to unregister
1521 * Undo blkcg_policy_register(@pol). Might sleep.
1523 void blkcg_policy_unregister(struct blkcg_policy
*pol
)
1525 struct blkcg
*blkcg
;
1527 mutex_lock(&blkcg_pol_register_mutex
);
1529 if (WARN_ON(blkcg_policy
[pol
->plid
] != pol
))
1532 /* kill the intf files first */
1533 if (pol
->dfl_cftypes
)
1534 cgroup_rm_cftypes(pol
->dfl_cftypes
);
1535 if (pol
->legacy_cftypes
)
1536 cgroup_rm_cftypes(pol
->legacy_cftypes
);
1538 /* remove cpds and unregister */
1539 mutex_lock(&blkcg_pol_mutex
);
1541 if (pol
->cpd_free_fn
) {
1542 list_for_each_entry(blkcg
, &all_blkcgs
, all_blkcgs_node
) {
1543 if (blkcg
->cpd
[pol
->plid
]) {
1544 pol
->cpd_free_fn(blkcg
->cpd
[pol
->plid
]);
1545 blkcg
->cpd
[pol
->plid
] = NULL
;
1549 blkcg_policy
[pol
->plid
] = NULL
;
1551 mutex_unlock(&blkcg_pol_mutex
);
1553 mutex_unlock(&blkcg_pol_register_mutex
);
1555 EXPORT_SYMBOL_GPL(blkcg_policy_unregister
);
1557 bool __blkcg_punt_bio_submit(struct bio
*bio
)
1559 struct blkcg_gq
*blkg
= bio
->bi_blkg
;
1561 /* consume the flag first */
1562 bio
->bi_opf
&= ~REQ_CGROUP_PUNT
;
1564 /* never bounce for the root cgroup */
1568 spin_lock_bh(&blkg
->async_bio_lock
);
1569 bio_list_add(&blkg
->async_bios
, bio
);
1570 spin_unlock_bh(&blkg
->async_bio_lock
);
1572 queue_work(blkcg_punt_bio_wq
, &blkg
->async_bio_work
);
1577 * Scale the accumulated delay based on how long it has been since we updated
1578 * the delay. We only call this when we are adding delay, in case it's been a
1579 * while since we added delay, and when we are checking to see if we need to
1580 * delay a task, to account for any delays that may have occurred.
1582 static void blkcg_scale_delay(struct blkcg_gq
*blkg
, u64 now
)
1584 u64 old
= atomic64_read(&blkg
->delay_start
);
1586 /* negative use_delay means no scaling, see blkcg_set_delay() */
1587 if (atomic_read(&blkg
->use_delay
) < 0)
1591 * We only want to scale down every second. The idea here is that we
1592 * want to delay people for min(delay_nsec, NSEC_PER_SEC) in a certain
1593 * time window. We only want to throttle tasks for recent delay that
1594 * has occurred, in 1 second time windows since that's the maximum
1595 * things can be throttled. We save the current delay window in
1596 * blkg->last_delay so we know what amount is still left to be charged
1597 * to the blkg from this point onward. blkg->last_use keeps track of
1598 * the use_delay counter. The idea is if we're unthrottling the blkg we
1599 * are ok with whatever is happening now, and we can take away more of
1600 * the accumulated delay as we've already throttled enough that
1601 * everybody is happy with their IO latencies.
1603 if (time_before64(old
+ NSEC_PER_SEC
, now
) &&
1604 atomic64_cmpxchg(&blkg
->delay_start
, old
, now
) == old
) {
1605 u64 cur
= atomic64_read(&blkg
->delay_nsec
);
1606 u64 sub
= min_t(u64
, blkg
->last_delay
, now
- old
);
1607 int cur_use
= atomic_read(&blkg
->use_delay
);
1610 * We've been unthrottled, subtract a larger chunk of our
1611 * accumulated delay.
1613 if (cur_use
< blkg
->last_use
)
1614 sub
= max_t(u64
, sub
, blkg
->last_delay
>> 1);
1617 * This shouldn't happen, but handle it anyway. Our delay_nsec
1618 * should only ever be growing except here where we subtract out
1619 * min(last_delay, 1 second), but lord knows bugs happen and I'd
1620 * rather not end up with negative numbers.
1622 if (unlikely(cur
< sub
)) {
1623 atomic64_set(&blkg
->delay_nsec
, 0);
1624 blkg
->last_delay
= 0;
1626 atomic64_sub(sub
, &blkg
->delay_nsec
);
1627 blkg
->last_delay
= cur
- sub
;
1629 blkg
->last_use
= cur_use
;
1634 * This is called when we want to actually walk up the hierarchy and check to
1635 * see if we need to throttle, and then actually throttle if there is some
1636 * accumulated delay. This should only be called upon return to user space so
1637 * we're not holding some lock that would induce a priority inversion.
1639 static void blkcg_maybe_throttle_blkg(struct blkcg_gq
*blkg
, bool use_memdelay
)
1641 unsigned long pflags
;
1643 u64 now
= ktime_to_ns(ktime_get());
1648 while (blkg
->parent
) {
1649 int use_delay
= atomic_read(&blkg
->use_delay
);
1654 blkcg_scale_delay(blkg
, now
);
1655 this_delay
= atomic64_read(&blkg
->delay_nsec
);
1656 if (this_delay
> delay_nsec
) {
1657 delay_nsec
= this_delay
;
1658 clamp
= use_delay
> 0;
1661 blkg
= blkg
->parent
;
1668 * Let's not sleep for all eternity if we've amassed a huge delay.
1669 * Swapping or metadata IO can accumulate 10's of seconds worth of
1670 * delay, and we want userspace to be able to do _something_ so cap the
1671 * delays at 0.25s. If there's 10's of seconds worth of delay then the
1672 * tasks will be delayed for 0.25 second for every syscall. If
1673 * blkcg_set_delay() was used as indicated by negative use_delay, the
1674 * caller is responsible for regulating the range.
1677 delay_nsec
= min_t(u64
, delay_nsec
, 250 * NSEC_PER_MSEC
);
1680 psi_memstall_enter(&pflags
);
1682 exp
= ktime_add_ns(now
, delay_nsec
);
1683 tok
= io_schedule_prepare();
1685 __set_current_state(TASK_KILLABLE
);
1686 if (!schedule_hrtimeout(&exp
, HRTIMER_MODE_ABS
))
1688 } while (!fatal_signal_pending(current
));
1689 io_schedule_finish(tok
);
1692 psi_memstall_leave(&pflags
);
1696 * blkcg_maybe_throttle_current - throttle the current task if it has been marked
1698 * This is only called if we've been marked with set_notify_resume(). Obviously
1699 * we can be set_notify_resume() for reasons other than blkcg throttling, so we
1700 * check to see if current->throttle_queue is set and if not this doesn't do
1701 * anything. This should only ever be called by the resume code, it's not meant
1702 * to be called by people willy-nilly as it will actually do the work to
1703 * throttle the task if it is setup for throttling.
1705 void blkcg_maybe_throttle_current(void)
1707 struct request_queue
*q
= current
->throttle_queue
;
1708 struct cgroup_subsys_state
*css
;
1709 struct blkcg
*blkcg
;
1710 struct blkcg_gq
*blkg
;
1711 bool use_memdelay
= current
->use_memdelay
;
1716 current
->throttle_queue
= NULL
;
1717 current
->use_memdelay
= false;
1720 css
= kthread_blkcg();
1722 blkcg
= css_to_blkcg(css
);
1724 blkcg
= css_to_blkcg(task_css(current
, io_cgrp_id
));
1728 blkg
= blkg_lookup(blkcg
, q
);
1731 if (!blkg_tryget(blkg
))
1735 blkcg_maybe_throttle_blkg(blkg
, use_memdelay
);
1745 * blkcg_schedule_throttle - this task needs to check for throttling
1746 * @q: the request queue IO was submitted on
1747 * @use_memdelay: do we charge this to memory delay for PSI
1749 * This is called by the IO controller when we know there's delay accumulated
1750 * for the blkg for this task. We do not pass the blkg because there are places
1751 * we call this that may not have that information, the swapping code for
1752 * instance will only have a request_queue at that point. This set's the
1753 * notify_resume for the task to check and see if it requires throttling before
1754 * returning to user space.
1756 * We will only schedule once per syscall. You can call this over and over
1757 * again and it will only do the check once upon return to user space, and only
1758 * throttle once. If the task needs to be throttled again it'll need to be
1759 * re-set at the next time we see the task.
1761 void blkcg_schedule_throttle(struct request_queue
*q
, bool use_memdelay
)
1763 if (unlikely(current
->flags
& PF_KTHREAD
))
1766 if (current
->throttle_queue
!= q
) {
1767 if (!blk_get_queue(q
))
1770 if (current
->throttle_queue
)
1771 blk_put_queue(current
->throttle_queue
);
1772 current
->throttle_queue
= q
;
1776 current
->use_memdelay
= use_memdelay
;
1777 set_notify_resume(current
);
1781 * blkcg_add_delay - add delay to this blkg
1782 * @blkg: blkg of interest
1783 * @now: the current time in nanoseconds
1784 * @delta: how many nanoseconds of delay to add
1786 * Charge @delta to the blkg's current delay accumulation. This is used to
1787 * throttle tasks if an IO controller thinks we need more throttling.
1789 void blkcg_add_delay(struct blkcg_gq
*blkg
, u64 now
, u64 delta
)
1791 if (WARN_ON_ONCE(atomic_read(&blkg
->use_delay
) < 0))
1793 blkcg_scale_delay(blkg
, now
);
1794 atomic64_add(delta
, &blkg
->delay_nsec
);
1798 * blkg_tryget_closest - try and get a blkg ref on the closet blkg
1802 * As the failure mode here is to walk up the blkg tree, this ensure that the
1803 * blkg->parent pointers are always valid. This returns the blkg that it ended
1804 * up taking a reference on or %NULL if no reference was taken.
1806 static inline struct blkcg_gq
*blkg_tryget_closest(struct bio
*bio
,
1807 struct cgroup_subsys_state
*css
)
1809 struct blkcg_gq
*blkg
, *ret_blkg
= NULL
;
1812 blkg
= blkg_lookup_create(css_to_blkcg(css
),
1813 bio
->bi_bdev
->bd_disk
->queue
);
1815 if (blkg_tryget(blkg
)) {
1819 blkg
= blkg
->parent
;
1827 * bio_associate_blkg_from_css - associate a bio with a specified css
1831 * Associate @bio with the blkg found by combining the css's blkg and the
1832 * request_queue of the @bio. An association failure is handled by walking up
1833 * the blkg tree. Therefore, the blkg associated can be anything between @blkg
1834 * and q->root_blkg. This situation only happens when a cgroup is dying and
1835 * then the remaining bios will spill to the closest alive blkg.
1837 * A reference will be taken on the blkg and will be released when @bio is
1840 void bio_associate_blkg_from_css(struct bio
*bio
,
1841 struct cgroup_subsys_state
*css
)
1844 blkg_put(bio
->bi_blkg
);
1846 if (css
&& css
->parent
) {
1847 bio
->bi_blkg
= blkg_tryget_closest(bio
, css
);
1849 blkg_get(bio
->bi_bdev
->bd_disk
->queue
->root_blkg
);
1850 bio
->bi_blkg
= bio
->bi_bdev
->bd_disk
->queue
->root_blkg
;
1853 EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css
);
1856 * bio_associate_blkg - associate a bio with a blkg
1859 * Associate @bio with the blkg found from the bio's css and request_queue.
1860 * If one is not found, bio_lookup_blkg() creates the blkg. If a blkg is
1861 * already associated, the css is reused and association redone as the
1862 * request_queue may have changed.
1864 void bio_associate_blkg(struct bio
*bio
)
1866 struct cgroup_subsys_state
*css
;
1871 css
= &bio_blkcg(bio
)->css
;
1875 bio_associate_blkg_from_css(bio
, css
);
1879 EXPORT_SYMBOL_GPL(bio_associate_blkg
);
1882 * bio_clone_blkg_association - clone blkg association from src to dst bio
1883 * @dst: destination bio
1886 void bio_clone_blkg_association(struct bio
*dst
, struct bio
*src
)
1890 blkg_put(dst
->bi_blkg
);
1891 blkg_get(src
->bi_blkg
);
1892 dst
->bi_blkg
= src
->bi_blkg
;
1895 EXPORT_SYMBOL_GPL(bio_clone_blkg_association
);
1897 static int blk_cgroup_io_type(struct bio
*bio
)
1899 if (op_is_discard(bio
->bi_opf
))
1900 return BLKG_IOSTAT_DISCARD
;
1901 if (op_is_write(bio
->bi_opf
))
1902 return BLKG_IOSTAT_WRITE
;
1903 return BLKG_IOSTAT_READ
;
1906 void blk_cgroup_bio_start(struct bio
*bio
)
1908 int rwd
= blk_cgroup_io_type(bio
), cpu
;
1909 struct blkg_iostat_set
*bis
;
1912 bis
= per_cpu_ptr(bio
->bi_blkg
->iostat_cpu
, cpu
);
1913 u64_stats_update_begin(&bis
->sync
);
1916 * If the bio is flagged with BIO_CGROUP_ACCT it means this is a split
1917 * bio and we would have already accounted for the size of the bio.
1919 if (!bio_flagged(bio
, BIO_CGROUP_ACCT
)) {
1920 bio_set_flag(bio
, BIO_CGROUP_ACCT
);
1921 bis
->cur
.bytes
[rwd
] += bio
->bi_iter
.bi_size
;
1923 bis
->cur
.ios
[rwd
]++;
1925 u64_stats_update_end(&bis
->sync
);
1926 if (cgroup_subsys_on_dfl(io_cgrp_subsys
))
1927 cgroup_rstat_updated(bio
->bi_blkg
->blkcg
->css
.cgroup
, cpu
);
1931 static int __init
blkcg_init(void)
1933 blkcg_punt_bio_wq
= alloc_workqueue("blkcg_punt_bio",
1934 WQ_MEM_RECLAIM
| WQ_FREEZABLE
|
1935 WQ_UNBOUND
| WQ_SYSFS
, 0);
1936 if (!blkcg_punt_bio_wq
)
1940 subsys_initcall(blkcg_init
);
1942 module_param(blkcg_debug_stats
, bool, 0644);
1943 MODULE_PARM_DESC(blkcg_debug_stats
, "True if you want debug stats, false if not");