1 // SPDX-License-Identifier: GPL-2.0
3 * Common Block IO controller cgroup interface
5 * Based on ideas and code from CFQ, CFS and BFQ:
6 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
8 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
9 * Paolo Valente <paolo.valente@unimore.it>
11 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
12 * Nauman Rafique <nauman@google.com>
14 * For policy-specific per-blkcg data:
15 * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
16 * Arianna Avanzini <avanzini.arianna@gmail.com>
18 #include <linux/ioprio.h>
19 #include <linux/kdev_t.h>
20 #include <linux/module.h>
21 #include <linux/sched/signal.h>
22 #include <linux/err.h>
23 #include <linux/blkdev.h>
24 #include <linux/backing-dev.h>
25 #include <linux/slab.h>
26 #include <linux/genhd.h>
27 #include <linux/delay.h>
28 #include <linux/atomic.h>
29 #include <linux/ctype.h>
30 #include <linux/blk-cgroup.h>
31 #include <linux/tracehook.h>
32 #include <linux/psi.h>
35 #define MAX_KEY_LEN 100
38 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
39 * blkcg_pol_register_mutex nests outside of it and synchronizes entire
40 * policy [un]register operations including cgroup file additions /
41 * removals. Putting cgroup file registration outside blkcg_pol_mutex
42 * allows grabbing it from cgroup callbacks.
44 static DEFINE_MUTEX(blkcg_pol_register_mutex
);
45 static DEFINE_MUTEX(blkcg_pol_mutex
);
47 struct blkcg blkcg_root
;
48 EXPORT_SYMBOL_GPL(blkcg_root
);
50 struct cgroup_subsys_state
* const blkcg_root_css
= &blkcg_root
.css
;
51 EXPORT_SYMBOL_GPL(blkcg_root_css
);
53 static struct blkcg_policy
*blkcg_policy
[BLKCG_MAX_POLS
];
55 static LIST_HEAD(all_blkcgs
); /* protected by blkcg_pol_mutex */
57 bool blkcg_debug_stats
= false;
58 static struct workqueue_struct
*blkcg_punt_bio_wq
;
60 static bool blkcg_policy_enabled(struct request_queue
*q
,
61 const struct blkcg_policy
*pol
)
63 return pol
&& test_bit(pol
->plid
, q
->blkcg_pols
);
67 * blkg_free - free a blkg
70 * Free @blkg which may be partially allocated.
72 static void blkg_free(struct blkcg_gq
*blkg
)
79 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++)
81 blkcg_policy
[i
]->pd_free_fn(blkg
->pd
[i
]);
83 free_percpu(blkg
->iostat_cpu
);
84 percpu_ref_exit(&blkg
->refcnt
);
88 static void __blkg_release(struct rcu_head
*rcu
)
90 struct blkcg_gq
*blkg
= container_of(rcu
, struct blkcg_gq
, rcu_head
);
92 WARN_ON(!bio_list_empty(&blkg
->async_bios
));
94 /* release the blkcg and parent blkg refs this blkg has been holding */
95 css_put(&blkg
->blkcg
->css
);
97 blkg_put(blkg
->parent
);
102 * A group is RCU protected, but having an rcu lock does not mean that one
103 * can access all the fields of blkg and assume these are valid. For
104 * example, don't try to follow throtl_data and request queue links.
106 * Having a reference to blkg under an rcu allows accesses to only values
107 * local to groups like group stats and group rate limits.
109 static void blkg_release(struct percpu_ref
*ref
)
111 struct blkcg_gq
*blkg
= container_of(ref
, struct blkcg_gq
, refcnt
);
113 call_rcu(&blkg
->rcu_head
, __blkg_release
);
116 static void blkg_async_bio_workfn(struct work_struct
*work
)
118 struct blkcg_gq
*blkg
= container_of(work
, struct blkcg_gq
,
120 struct bio_list bios
= BIO_EMPTY_LIST
;
122 struct blk_plug plug
;
123 bool need_plug
= false;
125 /* as long as there are pending bios, @blkg can't go away */
126 spin_lock_bh(&blkg
->async_bio_lock
);
127 bio_list_merge(&bios
, &blkg
->async_bios
);
128 bio_list_init(&blkg
->async_bios
);
129 spin_unlock_bh(&blkg
->async_bio_lock
);
131 /* start plug only when bio_list contains at least 2 bios */
132 if (bios
.head
&& bios
.head
->bi_next
) {
134 blk_start_plug(&plug
);
136 while ((bio
= bio_list_pop(&bios
)))
139 blk_finish_plug(&plug
);
143 * blkg_alloc - allocate a blkg
144 * @blkcg: block cgroup the new blkg is associated with
145 * @q: request_queue the new blkg is associated with
146 * @gfp_mask: allocation mask to use
148 * Allocate a new blkg assocating @blkcg and @q.
150 static struct blkcg_gq
*blkg_alloc(struct blkcg
*blkcg
, struct request_queue
*q
,
153 struct blkcg_gq
*blkg
;
156 /* alloc and init base part */
157 blkg
= kzalloc_node(sizeof(*blkg
), gfp_mask
, q
->node
);
161 if (percpu_ref_init(&blkg
->refcnt
, blkg_release
, 0, gfp_mask
))
164 blkg
->iostat_cpu
= alloc_percpu_gfp(struct blkg_iostat_set
, gfp_mask
);
165 if (!blkg
->iostat_cpu
)
169 INIT_LIST_HEAD(&blkg
->q_node
);
170 spin_lock_init(&blkg
->async_bio_lock
);
171 bio_list_init(&blkg
->async_bios
);
172 INIT_WORK(&blkg
->async_bio_work
, blkg_async_bio_workfn
);
175 u64_stats_init(&blkg
->iostat
.sync
);
176 for_each_possible_cpu(cpu
)
177 u64_stats_init(&per_cpu_ptr(blkg
->iostat_cpu
, cpu
)->sync
);
179 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++) {
180 struct blkcg_policy
*pol
= blkcg_policy
[i
];
181 struct blkg_policy_data
*pd
;
183 if (!blkcg_policy_enabled(q
, pol
))
186 /* alloc per-policy data and attach it to blkg */
187 pd
= pol
->pd_alloc_fn(gfp_mask
, q
, blkcg
);
203 struct blkcg_gq
*blkg_lookup_slowpath(struct blkcg
*blkcg
,
204 struct request_queue
*q
, bool update_hint
)
206 struct blkcg_gq
*blkg
;
209 * Hint didn't match. Look up from the radix tree. Note that the
210 * hint can only be updated under queue_lock as otherwise @blkg
211 * could have already been removed from blkg_tree. The caller is
212 * responsible for grabbing queue_lock if @update_hint.
214 blkg
= radix_tree_lookup(&blkcg
->blkg_tree
, q
->id
);
215 if (blkg
&& blkg
->q
== q
) {
217 lockdep_assert_held(&q
->queue_lock
);
218 rcu_assign_pointer(blkcg
->blkg_hint
, blkg
);
225 EXPORT_SYMBOL_GPL(blkg_lookup_slowpath
);
228 * If @new_blkg is %NULL, this function tries to allocate a new one as
229 * necessary using %GFP_NOWAIT. @new_blkg is always consumed on return.
231 static struct blkcg_gq
*blkg_create(struct blkcg
*blkcg
,
232 struct request_queue
*q
,
233 struct blkcg_gq
*new_blkg
)
235 struct blkcg_gq
*blkg
;
238 WARN_ON_ONCE(!rcu_read_lock_held());
239 lockdep_assert_held(&q
->queue_lock
);
241 /* request_queue is dying, do not create/recreate a blkg */
242 if (blk_queue_dying(q
)) {
247 /* blkg holds a reference to blkcg */
248 if (!css_tryget_online(&blkcg
->css
)) {
255 new_blkg
= blkg_alloc(blkcg
, q
, GFP_NOWAIT
| __GFP_NOWARN
);
256 if (unlikely(!new_blkg
)) {
264 if (blkcg_parent(blkcg
)) {
265 blkg
->parent
= __blkg_lookup(blkcg_parent(blkcg
), q
, false);
266 if (WARN_ON_ONCE(!blkg
->parent
)) {
270 blkg_get(blkg
->parent
);
273 /* invoke per-policy init */
274 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++) {
275 struct blkcg_policy
*pol
= blkcg_policy
[i
];
277 if (blkg
->pd
[i
] && pol
->pd_init_fn
)
278 pol
->pd_init_fn(blkg
->pd
[i
]);
282 spin_lock(&blkcg
->lock
);
283 ret
= radix_tree_insert(&blkcg
->blkg_tree
, q
->id
, blkg
);
285 hlist_add_head_rcu(&blkg
->blkcg_node
, &blkcg
->blkg_list
);
286 list_add(&blkg
->q_node
, &q
->blkg_list
);
288 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++) {
289 struct blkcg_policy
*pol
= blkcg_policy
[i
];
291 if (blkg
->pd
[i
] && pol
->pd_online_fn
)
292 pol
->pd_online_fn(blkg
->pd
[i
]);
296 spin_unlock(&blkcg
->lock
);
301 /* @blkg failed fully initialized, use the usual release path */
306 css_put(&blkcg
->css
);
313 * blkg_lookup_create - lookup blkg, try to create one if not there
314 * @blkcg: blkcg of interest
315 * @q: request_queue of interest
317 * Lookup blkg for the @blkcg - @q pair. If it doesn't exist, try to
318 * create one. blkg creation is performed recursively from blkcg_root such
319 * that all non-root blkg's have access to the parent blkg. This function
320 * should be called under RCU read lock and takes @q->queue_lock.
322 * Returns the blkg or the closest blkg if blkg_create() fails as it walks
325 static struct blkcg_gq
*blkg_lookup_create(struct blkcg
*blkcg
,
326 struct request_queue
*q
)
328 struct blkcg_gq
*blkg
;
331 WARN_ON_ONCE(!rcu_read_lock_held());
333 blkg
= blkg_lookup(blkcg
, q
);
337 spin_lock_irqsave(&q
->queue_lock
, flags
);
338 blkg
= __blkg_lookup(blkcg
, q
, true);
343 * Create blkgs walking down from blkcg_root to @blkcg, so that all
344 * non-root blkgs have access to their parents. Returns the closest
345 * blkg to the intended blkg should blkg_create() fail.
348 struct blkcg
*pos
= blkcg
;
349 struct blkcg
*parent
= blkcg_parent(blkcg
);
350 struct blkcg_gq
*ret_blkg
= q
->root_blkg
;
353 blkg
= __blkg_lookup(parent
, q
, false);
355 /* remember closest blkg */
360 parent
= blkcg_parent(parent
);
363 blkg
= blkg_create(pos
, q
, NULL
);
373 spin_unlock_irqrestore(&q
->queue_lock
, flags
);
377 static void blkg_destroy(struct blkcg_gq
*blkg
)
379 struct blkcg
*blkcg
= blkg
->blkcg
;
382 lockdep_assert_held(&blkg
->q
->queue_lock
);
383 lockdep_assert_held(&blkcg
->lock
);
385 /* Something wrong if we are trying to remove same group twice */
386 WARN_ON_ONCE(list_empty(&blkg
->q_node
));
387 WARN_ON_ONCE(hlist_unhashed(&blkg
->blkcg_node
));
389 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++) {
390 struct blkcg_policy
*pol
= blkcg_policy
[i
];
392 if (blkg
->pd
[i
] && pol
->pd_offline_fn
)
393 pol
->pd_offline_fn(blkg
->pd
[i
]);
396 blkg
->online
= false;
398 radix_tree_delete(&blkcg
->blkg_tree
, blkg
->q
->id
);
399 list_del_init(&blkg
->q_node
);
400 hlist_del_init_rcu(&blkg
->blkcg_node
);
403 * Both setting lookup hint to and clearing it from @blkg are done
404 * under queue_lock. If it's not pointing to @blkg now, it never
405 * will. Hint assignment itself can race safely.
407 if (rcu_access_pointer(blkcg
->blkg_hint
) == blkg
)
408 rcu_assign_pointer(blkcg
->blkg_hint
, NULL
);
411 * Put the reference taken at the time of creation so that when all
412 * queues are gone, group can be destroyed.
414 percpu_ref_kill(&blkg
->refcnt
);
418 * blkg_destroy_all - destroy all blkgs associated with a request_queue
419 * @q: request_queue of interest
421 * Destroy all blkgs associated with @q.
423 static void blkg_destroy_all(struct request_queue
*q
)
425 struct blkcg_gq
*blkg
, *n
;
427 spin_lock_irq(&q
->queue_lock
);
428 list_for_each_entry_safe(blkg
, n
, &q
->blkg_list
, q_node
) {
429 struct blkcg
*blkcg
= blkg
->blkcg
;
431 spin_lock(&blkcg
->lock
);
433 spin_unlock(&blkcg
->lock
);
437 spin_unlock_irq(&q
->queue_lock
);
440 static int blkcg_reset_stats(struct cgroup_subsys_state
*css
,
441 struct cftype
*cftype
, u64 val
)
443 struct blkcg
*blkcg
= css_to_blkcg(css
);
444 struct blkcg_gq
*blkg
;
447 mutex_lock(&blkcg_pol_mutex
);
448 spin_lock_irq(&blkcg
->lock
);
451 * Note that stat reset is racy - it doesn't synchronize against
452 * stat updates. This is a debug feature which shouldn't exist
453 * anyway. If you get hit by a race, retry.
455 hlist_for_each_entry(blkg
, &blkcg
->blkg_list
, blkcg_node
) {
456 for_each_possible_cpu(cpu
) {
457 struct blkg_iostat_set
*bis
=
458 per_cpu_ptr(blkg
->iostat_cpu
, cpu
);
459 memset(bis
, 0, sizeof(*bis
));
461 memset(&blkg
->iostat
, 0, sizeof(blkg
->iostat
));
463 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++) {
464 struct blkcg_policy
*pol
= blkcg_policy
[i
];
466 if (blkg
->pd
[i
] && pol
->pd_reset_stats_fn
)
467 pol
->pd_reset_stats_fn(blkg
->pd
[i
]);
471 spin_unlock_irq(&blkcg
->lock
);
472 mutex_unlock(&blkcg_pol_mutex
);
476 const char *blkg_dev_name(struct blkcg_gq
*blkg
)
478 /* some drivers (floppy) instantiate a queue w/o disk registered */
479 if (blkg
->q
->backing_dev_info
->dev
)
480 return bdi_dev_name(blkg
->q
->backing_dev_info
);
485 * blkcg_print_blkgs - helper for printing per-blkg data
486 * @sf: seq_file to print to
487 * @blkcg: blkcg of interest
488 * @prfill: fill function to print out a blkg
489 * @pol: policy in question
490 * @data: data to be passed to @prfill
491 * @show_total: to print out sum of prfill return values or not
493 * This function invokes @prfill on each blkg of @blkcg if pd for the
494 * policy specified by @pol exists. @prfill is invoked with @sf, the
495 * policy data and @data and the matching queue lock held. If @show_total
496 * is %true, the sum of the return values from @prfill is printed with
497 * "Total" label at the end.
499 * This is to be used to construct print functions for
500 * cftype->read_seq_string method.
502 void blkcg_print_blkgs(struct seq_file
*sf
, struct blkcg
*blkcg
,
503 u64 (*prfill
)(struct seq_file
*,
504 struct blkg_policy_data
*, int),
505 const struct blkcg_policy
*pol
, int data
,
508 struct blkcg_gq
*blkg
;
512 hlist_for_each_entry_rcu(blkg
, &blkcg
->blkg_list
, blkcg_node
) {
513 spin_lock_irq(&blkg
->q
->queue_lock
);
514 if (blkcg_policy_enabled(blkg
->q
, pol
))
515 total
+= prfill(sf
, blkg
->pd
[pol
->plid
], data
);
516 spin_unlock_irq(&blkg
->q
->queue_lock
);
521 seq_printf(sf
, "Total %llu\n", (unsigned long long)total
);
523 EXPORT_SYMBOL_GPL(blkcg_print_blkgs
);
526 * __blkg_prfill_u64 - prfill helper for a single u64 value
527 * @sf: seq_file to print to
528 * @pd: policy private data of interest
531 * Print @v to @sf for the device assocaited with @pd.
533 u64
__blkg_prfill_u64(struct seq_file
*sf
, struct blkg_policy_data
*pd
, u64 v
)
535 const char *dname
= blkg_dev_name(pd
->blkg
);
540 seq_printf(sf
, "%s %llu\n", dname
, (unsigned long long)v
);
543 EXPORT_SYMBOL_GPL(__blkg_prfill_u64
);
545 /* Performs queue bypass and policy enabled checks then looks up blkg. */
546 static struct blkcg_gq
*blkg_lookup_check(struct blkcg
*blkcg
,
547 const struct blkcg_policy
*pol
,
548 struct request_queue
*q
)
550 WARN_ON_ONCE(!rcu_read_lock_held());
551 lockdep_assert_held(&q
->queue_lock
);
553 if (!blkcg_policy_enabled(q
, pol
))
554 return ERR_PTR(-EOPNOTSUPP
);
555 return __blkg_lookup(blkcg
, q
, true /* update_hint */);
559 * blkg_conf_prep - parse and prepare for per-blkg config update
560 * @inputp: input string pointer
562 * Parse the device node prefix part, MAJ:MIN, of per-blkg config update
563 * from @input and get and return the matching gendisk. *@inputp is
564 * updated to point past the device node prefix. Returns an ERR_PTR()
567 * Use this function iff blkg_conf_prep() can't be used for some reason.
569 struct gendisk
*blkcg_conf_get_disk(char **inputp
)
571 char *input
= *inputp
;
572 unsigned int major
, minor
;
573 struct gendisk
*disk
;
576 if (sscanf(input
, "%u:%u%n", &major
, &minor
, &key_len
) != 2)
577 return ERR_PTR(-EINVAL
);
580 if (!isspace(*input
))
581 return ERR_PTR(-EINVAL
);
582 input
= skip_spaces(input
);
584 disk
= get_gendisk(MKDEV(major
, minor
), &part
);
586 return ERR_PTR(-ENODEV
);
588 put_disk_and_module(disk
);
589 return ERR_PTR(-ENODEV
);
597 * blkg_conf_prep - parse and prepare for per-blkg config update
598 * @blkcg: target block cgroup
599 * @pol: target policy
600 * @input: input string
601 * @ctx: blkg_conf_ctx to be filled
603 * Parse per-blkg config update from @input and initialize @ctx with the
604 * result. @ctx->blkg points to the blkg to be updated and @ctx->body the
605 * part of @input following MAJ:MIN. This function returns with RCU read
606 * lock and queue lock held and must be paired with blkg_conf_finish().
608 int blkg_conf_prep(struct blkcg
*blkcg
, const struct blkcg_policy
*pol
,
609 char *input
, struct blkg_conf_ctx
*ctx
)
610 __acquires(rcu
) __acquires(&disk
->queue
->queue_lock
)
612 struct gendisk
*disk
;
613 struct request_queue
*q
;
614 struct blkcg_gq
*blkg
;
617 disk
= blkcg_conf_get_disk(&input
);
619 return PTR_ERR(disk
);
624 spin_lock_irq(&q
->queue_lock
);
626 blkg
= blkg_lookup_check(blkcg
, pol
, q
);
636 * Create blkgs walking down from blkcg_root to @blkcg, so that all
637 * non-root blkgs have access to their parents.
640 struct blkcg
*pos
= blkcg
;
641 struct blkcg
*parent
;
642 struct blkcg_gq
*new_blkg
;
644 parent
= blkcg_parent(blkcg
);
645 while (parent
&& !__blkg_lookup(parent
, q
, false)) {
647 parent
= blkcg_parent(parent
);
650 /* Drop locks to do new blkg allocation with GFP_KERNEL. */
651 spin_unlock_irq(&q
->queue_lock
);
654 new_blkg
= blkg_alloc(pos
, q
, GFP_KERNEL
);
655 if (unlikely(!new_blkg
)) {
660 if (radix_tree_preload(GFP_KERNEL
)) {
667 spin_lock_irq(&q
->queue_lock
);
669 blkg
= blkg_lookup_check(pos
, pol
, q
);
679 blkg
= blkg_create(pos
, q
, new_blkg
);
686 radix_tree_preload_end();
698 radix_tree_preload_end();
700 spin_unlock_irq(&q
->queue_lock
);
703 put_disk_and_module(disk
);
705 * If queue was bypassing, we should retry. Do so after a
706 * short msleep(). It isn't strictly necessary but queue
707 * can be bypassing for some time and it's always nice to
708 * avoid busy looping.
712 ret
= restart_syscall();
716 EXPORT_SYMBOL_GPL(blkg_conf_prep
);
719 * blkg_conf_finish - finish up per-blkg config update
720 * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep()
722 * Finish up after per-blkg config update. This function must be paired
723 * with blkg_conf_prep().
725 void blkg_conf_finish(struct blkg_conf_ctx
*ctx
)
726 __releases(&ctx
->disk
->queue
->queue_lock
) __releases(rcu
)
728 spin_unlock_irq(&ctx
->disk
->queue
->queue_lock
);
730 put_disk_and_module(ctx
->disk
);
732 EXPORT_SYMBOL_GPL(blkg_conf_finish
);
734 static void blkg_iostat_set(struct blkg_iostat
*dst
, struct blkg_iostat
*src
)
738 for (i
= 0; i
< BLKG_IOSTAT_NR
; i
++) {
739 dst
->bytes
[i
] = src
->bytes
[i
];
740 dst
->ios
[i
] = src
->ios
[i
];
744 static void blkg_iostat_add(struct blkg_iostat
*dst
, struct blkg_iostat
*src
)
748 for (i
= 0; i
< BLKG_IOSTAT_NR
; i
++) {
749 dst
->bytes
[i
] += src
->bytes
[i
];
750 dst
->ios
[i
] += src
->ios
[i
];
754 static void blkg_iostat_sub(struct blkg_iostat
*dst
, struct blkg_iostat
*src
)
758 for (i
= 0; i
< BLKG_IOSTAT_NR
; i
++) {
759 dst
->bytes
[i
] -= src
->bytes
[i
];
760 dst
->ios
[i
] -= src
->ios
[i
];
764 static void blkcg_rstat_flush(struct cgroup_subsys_state
*css
, int cpu
)
766 struct blkcg
*blkcg
= css_to_blkcg(css
);
767 struct blkcg_gq
*blkg
;
771 hlist_for_each_entry_rcu(blkg
, &blkcg
->blkg_list
, blkcg_node
) {
772 struct blkcg_gq
*parent
= blkg
->parent
;
773 struct blkg_iostat_set
*bisc
= per_cpu_ptr(blkg
->iostat_cpu
, cpu
);
774 struct blkg_iostat cur
, delta
;
777 /* fetch the current per-cpu values */
779 seq
= u64_stats_fetch_begin(&bisc
->sync
);
780 blkg_iostat_set(&cur
, &bisc
->cur
);
781 } while (u64_stats_fetch_retry(&bisc
->sync
, seq
));
783 /* propagate percpu delta to global */
784 u64_stats_update_begin(&blkg
->iostat
.sync
);
785 blkg_iostat_set(&delta
, &cur
);
786 blkg_iostat_sub(&delta
, &bisc
->last
);
787 blkg_iostat_add(&blkg
->iostat
.cur
, &delta
);
788 blkg_iostat_add(&bisc
->last
, &delta
);
789 u64_stats_update_end(&blkg
->iostat
.sync
);
791 /* propagate global delta to parent */
793 u64_stats_update_begin(&parent
->iostat
.sync
);
794 blkg_iostat_set(&delta
, &blkg
->iostat
.cur
);
795 blkg_iostat_sub(&delta
, &blkg
->iostat
.last
);
796 blkg_iostat_add(&parent
->iostat
.cur
, &delta
);
797 blkg_iostat_add(&blkg
->iostat
.last
, &delta
);
798 u64_stats_update_end(&parent
->iostat
.sync
);
806 * The rstat algorithms intentionally don't handle the root cgroup to avoid
807 * incurring overhead when no cgroups are defined. For that reason,
808 * cgroup_rstat_flush in blkcg_print_stat does not actually fill out the
809 * iostat in the root cgroup's blkcg_gq.
811 * However, we would like to re-use the printing code between the root and
812 * non-root cgroups to the extent possible. For that reason, we simulate
813 * flushing the root cgroup's stats by explicitly filling in the iostat
814 * with disk level statistics.
816 static void blkcg_fill_root_iostats(void)
818 struct class_dev_iter iter
;
821 class_dev_iter_init(&iter
, &block_class
, NULL
, &disk_type
);
822 while ((dev
= class_dev_iter_next(&iter
))) {
823 struct gendisk
*disk
= dev_to_disk(dev
);
824 struct hd_struct
*part
= disk_get_part(disk
, 0);
825 struct blkcg_gq
*blkg
= blk_queue_root_blkg(disk
->queue
);
826 struct blkg_iostat tmp
;
829 memset(&tmp
, 0, sizeof(tmp
));
830 for_each_possible_cpu(cpu
) {
831 struct disk_stats
*cpu_dkstats
;
833 cpu_dkstats
= per_cpu_ptr(part
->dkstats
, cpu
);
834 tmp
.ios
[BLKG_IOSTAT_READ
] +=
835 cpu_dkstats
->ios
[STAT_READ
];
836 tmp
.ios
[BLKG_IOSTAT_WRITE
] +=
837 cpu_dkstats
->ios
[STAT_WRITE
];
838 tmp
.ios
[BLKG_IOSTAT_DISCARD
] +=
839 cpu_dkstats
->ios
[STAT_DISCARD
];
840 // convert sectors to bytes
841 tmp
.bytes
[BLKG_IOSTAT_READ
] +=
842 cpu_dkstats
->sectors
[STAT_READ
] << 9;
843 tmp
.bytes
[BLKG_IOSTAT_WRITE
] +=
844 cpu_dkstats
->sectors
[STAT_WRITE
] << 9;
845 tmp
.bytes
[BLKG_IOSTAT_DISCARD
] +=
846 cpu_dkstats
->sectors
[STAT_DISCARD
] << 9;
848 u64_stats_update_begin(&blkg
->iostat
.sync
);
849 blkg_iostat_set(&blkg
->iostat
.cur
, &tmp
);
850 u64_stats_update_end(&blkg
->iostat
.sync
);
856 static int blkcg_print_stat(struct seq_file
*sf
, void *v
)
858 struct blkcg
*blkcg
= css_to_blkcg(seq_css(sf
));
859 struct blkcg_gq
*blkg
;
861 if (!seq_css(sf
)->parent
)
862 blkcg_fill_root_iostats();
864 cgroup_rstat_flush(blkcg
->css
.cgroup
);
868 hlist_for_each_entry_rcu(blkg
, &blkcg
->blkg_list
, blkcg_node
) {
869 struct blkg_iostat_set
*bis
= &blkg
->iostat
;
872 u64 rbytes
, wbytes
, rios
, wios
, dbytes
, dios
;
873 size_t size
= seq_get_buf(sf
, &buf
), off
= 0;
875 bool has_stats
= false;
878 spin_lock_irq(&blkg
->q
->queue_lock
);
883 dname
= blkg_dev_name(blkg
);
888 * Hooray string manipulation, count is the size written NOT
889 * INCLUDING THE \0, so size is now count+1 less than what we
890 * had before, but we want to start writing the next bit from
891 * the \0 so we only add count to buf.
893 off
+= scnprintf(buf
+off
, size
-off
, "%s ", dname
);
896 seq
= u64_stats_fetch_begin(&bis
->sync
);
898 rbytes
= bis
->cur
.bytes
[BLKG_IOSTAT_READ
];
899 wbytes
= bis
->cur
.bytes
[BLKG_IOSTAT_WRITE
];
900 dbytes
= bis
->cur
.bytes
[BLKG_IOSTAT_DISCARD
];
901 rios
= bis
->cur
.ios
[BLKG_IOSTAT_READ
];
902 wios
= bis
->cur
.ios
[BLKG_IOSTAT_WRITE
];
903 dios
= bis
->cur
.ios
[BLKG_IOSTAT_DISCARD
];
904 } while (u64_stats_fetch_retry(&bis
->sync
, seq
));
906 if (rbytes
|| wbytes
|| rios
|| wios
) {
908 off
+= scnprintf(buf
+off
, size
-off
,
909 "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",
910 rbytes
, wbytes
, rios
, wios
,
914 if (blkcg_debug_stats
&& atomic_read(&blkg
->use_delay
)) {
916 off
+= scnprintf(buf
+off
, size
-off
,
917 " use_delay=%d delay_nsec=%llu",
918 atomic_read(&blkg
->use_delay
),
919 (unsigned long long)atomic64_read(&blkg
->delay_nsec
));
922 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++) {
923 struct blkcg_policy
*pol
= blkcg_policy
[i
];
926 if (!blkg
->pd
[i
] || !pol
->pd_stat_fn
)
929 written
= pol
->pd_stat_fn(blkg
->pd
[i
], buf
+off
, size
-off
);
936 if (off
< size
- 1) {
937 off
+= scnprintf(buf
+off
, size
-off
, "\n");
944 spin_unlock_irq(&blkg
->q
->queue_lock
);
951 static struct cftype blkcg_files
[] = {
954 .seq_show
= blkcg_print_stat
,
959 static struct cftype blkcg_legacy_files
[] = {
961 .name
= "reset_stats",
962 .write_u64
= blkcg_reset_stats
,
968 * blkcg destruction is a three-stage process.
970 * 1. Destruction starts. The blkcg_css_offline() callback is invoked
971 * which offlines writeback. Here we tie the next stage of blkg destruction
972 * to the completion of writeback associated with the blkcg. This lets us
973 * avoid punting potentially large amounts of outstanding writeback to root
974 * while maintaining any ongoing policies. The next stage is triggered when
975 * the nr_cgwbs count goes to zero.
977 * 2. When the nr_cgwbs count goes to zero, blkcg_destroy_blkgs() is called
978 * and handles the destruction of blkgs. Here the css reference held by
979 * the blkg is put back eventually allowing blkcg_css_free() to be called.
980 * This work may occur in cgwb_release_workfn() on the cgwb_release
981 * workqueue. Any submitted ios that fail to get the blkg ref will be
982 * punted to the root_blkg.
984 * 3. Once the blkcg ref count goes to zero, blkcg_css_free() is called.
985 * This finally frees the blkcg.
989 * blkcg_css_offline - cgroup css_offline callback
990 * @css: css of interest
992 * This function is called when @css is about to go away. Here the cgwbs are
993 * offlined first and only once writeback associated with the blkcg has
994 * finished do we start step 2 (see above).
996 static void blkcg_css_offline(struct cgroup_subsys_state
*css
)
998 struct blkcg
*blkcg
= css_to_blkcg(css
);
1000 /* this prevents anyone from attaching or migrating to this blkcg */
1001 wb_blkcg_offline(blkcg
);
1003 /* put the base online pin allowing step 2 to be triggered */
1004 blkcg_unpin_online(blkcg
);
1008 * blkcg_destroy_blkgs - responsible for shooting down blkgs
1009 * @blkcg: blkcg of interest
1011 * blkgs should be removed while holding both q and blkcg locks. As blkcg lock
1012 * is nested inside q lock, this function performs reverse double lock dancing.
1013 * Destroying the blkgs releases the reference held on the blkcg's css allowing
1014 * blkcg_css_free to eventually be called.
1016 * This is the blkcg counterpart of ioc_release_fn().
1018 void blkcg_destroy_blkgs(struct blkcg
*blkcg
)
1020 spin_lock_irq(&blkcg
->lock
);
1022 while (!hlist_empty(&blkcg
->blkg_list
)) {
1023 struct blkcg_gq
*blkg
= hlist_entry(blkcg
->blkg_list
.first
,
1024 struct blkcg_gq
, blkcg_node
);
1025 struct request_queue
*q
= blkg
->q
;
1027 if (spin_trylock(&q
->queue_lock
)) {
1029 spin_unlock(&q
->queue_lock
);
1031 spin_unlock_irq(&blkcg
->lock
);
1033 spin_lock_irq(&blkcg
->lock
);
1037 spin_unlock_irq(&blkcg
->lock
);
1040 static void blkcg_css_free(struct cgroup_subsys_state
*css
)
1042 struct blkcg
*blkcg
= css_to_blkcg(css
);
1045 mutex_lock(&blkcg_pol_mutex
);
1047 list_del(&blkcg
->all_blkcgs_node
);
1049 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++)
1051 blkcg_policy
[i
]->cpd_free_fn(blkcg
->cpd
[i
]);
1053 mutex_unlock(&blkcg_pol_mutex
);
1058 static struct cgroup_subsys_state
*
1059 blkcg_css_alloc(struct cgroup_subsys_state
*parent_css
)
1061 struct blkcg
*blkcg
;
1062 struct cgroup_subsys_state
*ret
;
1065 mutex_lock(&blkcg_pol_mutex
);
1068 blkcg
= &blkcg_root
;
1070 blkcg
= kzalloc(sizeof(*blkcg
), GFP_KERNEL
);
1072 ret
= ERR_PTR(-ENOMEM
);
1077 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++) {
1078 struct blkcg_policy
*pol
= blkcg_policy
[i
];
1079 struct blkcg_policy_data
*cpd
;
1082 * If the policy hasn't been attached yet, wait for it
1083 * to be attached before doing anything else. Otherwise,
1084 * check if the policy requires any specific per-cgroup
1085 * data: if it does, allocate and initialize it.
1087 if (!pol
|| !pol
->cpd_alloc_fn
)
1090 cpd
= pol
->cpd_alloc_fn(GFP_KERNEL
);
1092 ret
= ERR_PTR(-ENOMEM
);
1095 blkcg
->cpd
[i
] = cpd
;
1098 if (pol
->cpd_init_fn
)
1099 pol
->cpd_init_fn(cpd
);
1102 spin_lock_init(&blkcg
->lock
);
1103 refcount_set(&blkcg
->online_pin
, 1);
1104 INIT_RADIX_TREE(&blkcg
->blkg_tree
, GFP_NOWAIT
| __GFP_NOWARN
);
1105 INIT_HLIST_HEAD(&blkcg
->blkg_list
);
1106 #ifdef CONFIG_CGROUP_WRITEBACK
1107 INIT_LIST_HEAD(&blkcg
->cgwb_list
);
1109 list_add_tail(&blkcg
->all_blkcgs_node
, &all_blkcgs
);
1111 mutex_unlock(&blkcg_pol_mutex
);
1115 for (i
--; i
>= 0; i
--)
1117 blkcg_policy
[i
]->cpd_free_fn(blkcg
->cpd
[i
]);
1119 if (blkcg
!= &blkcg_root
)
1122 mutex_unlock(&blkcg_pol_mutex
);
1126 static int blkcg_css_online(struct cgroup_subsys_state
*css
)
1128 struct blkcg
*blkcg
= css_to_blkcg(css
);
1129 struct blkcg
*parent
= blkcg_parent(blkcg
);
1132 * blkcg_pin_online() is used to delay blkcg offline so that blkgs
1133 * don't go offline while cgwbs are still active on them. Pin the
1134 * parent so that offline always happens towards the root.
1137 blkcg_pin_online(parent
);
1142 * blkcg_init_queue - initialize blkcg part of request queue
1143 * @q: request_queue to initialize
1145 * Called from blk_alloc_queue(). Responsible for initializing blkcg
1146 * part of new request_queue @q.
1149 * 0 on success, -errno on failure.
1151 int blkcg_init_queue(struct request_queue
*q
)
1153 struct blkcg_gq
*new_blkg
, *blkg
;
1157 new_blkg
= blkg_alloc(&blkcg_root
, q
, GFP_KERNEL
);
1161 preloaded
= !radix_tree_preload(GFP_KERNEL
);
1163 /* Make sure the root blkg exists. */
1165 spin_lock_irq(&q
->queue_lock
);
1166 blkg
= blkg_create(&blkcg_root
, q
, new_blkg
);
1169 q
->root_blkg
= blkg
;
1170 spin_unlock_irq(&q
->queue_lock
);
1174 radix_tree_preload_end();
1176 ret
= blk_throtl_init(q
);
1178 goto err_destroy_all
;
1180 ret
= blk_iolatency_init(q
);
1183 goto err_destroy_all
;
1188 blkg_destroy_all(q
);
1191 spin_unlock_irq(&q
->queue_lock
);
1194 radix_tree_preload_end();
1195 return PTR_ERR(blkg
);
1199 * blkcg_exit_queue - exit and release blkcg part of request_queue
1200 * @q: request_queue being released
1202 * Called from blk_exit_queue(). Responsible for exiting blkcg part.
1204 void blkcg_exit_queue(struct request_queue
*q
)
1206 blkg_destroy_all(q
);
1211 * We cannot support shared io contexts, as we have no mean to support
1212 * two tasks with the same ioc in two different groups without major rework
1213 * of the main cic data structures. For now we allow a task to change
1214 * its cgroup only if it's the only owner of its ioc.
1216 static int blkcg_can_attach(struct cgroup_taskset
*tset
)
1218 struct task_struct
*task
;
1219 struct cgroup_subsys_state
*dst_css
;
1220 struct io_context
*ioc
;
1223 /* task_lock() is needed to avoid races with exit_io_context() */
1224 cgroup_taskset_for_each(task
, dst_css
, tset
) {
1226 ioc
= task
->io_context
;
1227 if (ioc
&& atomic_read(&ioc
->nr_tasks
) > 1)
1236 static void blkcg_bind(struct cgroup_subsys_state
*root_css
)
1240 mutex_lock(&blkcg_pol_mutex
);
1242 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++) {
1243 struct blkcg_policy
*pol
= blkcg_policy
[i
];
1244 struct blkcg
*blkcg
;
1246 if (!pol
|| !pol
->cpd_bind_fn
)
1249 list_for_each_entry(blkcg
, &all_blkcgs
, all_blkcgs_node
)
1250 if (blkcg
->cpd
[pol
->plid
])
1251 pol
->cpd_bind_fn(blkcg
->cpd
[pol
->plid
]);
1253 mutex_unlock(&blkcg_pol_mutex
);
1256 static void blkcg_exit(struct task_struct
*tsk
)
1258 if (tsk
->throttle_queue
)
1259 blk_put_queue(tsk
->throttle_queue
);
1260 tsk
->throttle_queue
= NULL
;
1263 struct cgroup_subsys io_cgrp_subsys
= {
1264 .css_alloc
= blkcg_css_alloc
,
1265 .css_online
= blkcg_css_online
,
1266 .css_offline
= blkcg_css_offline
,
1267 .css_free
= blkcg_css_free
,
1268 .can_attach
= blkcg_can_attach
,
1269 .css_rstat_flush
= blkcg_rstat_flush
,
1271 .dfl_cftypes
= blkcg_files
,
1272 .legacy_cftypes
= blkcg_legacy_files
,
1273 .legacy_name
= "blkio",
1277 * This ensures that, if available, memcg is automatically enabled
1278 * together on the default hierarchy so that the owner cgroup can
1279 * be retrieved from writeback pages.
1281 .depends_on
= 1 << memory_cgrp_id
,
1284 EXPORT_SYMBOL_GPL(io_cgrp_subsys
);
1287 * blkcg_activate_policy - activate a blkcg policy on a request_queue
1288 * @q: request_queue of interest
1289 * @pol: blkcg policy to activate
1291 * Activate @pol on @q. Requires %GFP_KERNEL context. @q goes through
1292 * bypass mode to populate its blkgs with policy_data for @pol.
1294 * Activation happens with @q bypassed, so nobody would be accessing blkgs
1295 * from IO path. Update of each blkg is protected by both queue and blkcg
1296 * locks so that holding either lock and testing blkcg_policy_enabled() is
1297 * always enough for dereferencing policy data.
1299 * The caller is responsible for synchronizing [de]activations and policy
1300 * [un]registerations. Returns 0 on success, -errno on failure.
1302 int blkcg_activate_policy(struct request_queue
*q
,
1303 const struct blkcg_policy
*pol
)
1305 struct blkg_policy_data
*pd_prealloc
= NULL
;
1306 struct blkcg_gq
*blkg
, *pinned_blkg
= NULL
;
1309 if (blkcg_policy_enabled(q
, pol
))
1313 blk_mq_freeze_queue(q
);
1315 spin_lock_irq(&q
->queue_lock
);
1317 /* blkg_list is pushed at the head, reverse walk to allocate parents first */
1318 list_for_each_entry_reverse(blkg
, &q
->blkg_list
, q_node
) {
1319 struct blkg_policy_data
*pd
;
1321 if (blkg
->pd
[pol
->plid
])
1324 /* If prealloc matches, use it; otherwise try GFP_NOWAIT */
1325 if (blkg
== pinned_blkg
) {
1329 pd
= pol
->pd_alloc_fn(GFP_NOWAIT
| __GFP_NOWARN
, q
,
1335 * GFP_NOWAIT failed. Free the existing one and
1336 * prealloc for @blkg w/ GFP_KERNEL.
1339 blkg_put(pinned_blkg
);
1343 spin_unlock_irq(&q
->queue_lock
);
1346 pol
->pd_free_fn(pd_prealloc
);
1347 pd_prealloc
= pol
->pd_alloc_fn(GFP_KERNEL
, q
,
1355 blkg
->pd
[pol
->plid
] = pd
;
1357 pd
->plid
= pol
->plid
;
1360 /* all allocated, init in the same order */
1361 if (pol
->pd_init_fn
)
1362 list_for_each_entry_reverse(blkg
, &q
->blkg_list
, q_node
)
1363 pol
->pd_init_fn(blkg
->pd
[pol
->plid
]);
1365 __set_bit(pol
->plid
, q
->blkcg_pols
);
1368 spin_unlock_irq(&q
->queue_lock
);
1371 blk_mq_unfreeze_queue(q
);
1373 blkg_put(pinned_blkg
);
1375 pol
->pd_free_fn(pd_prealloc
);
1379 /* alloc failed, nothing's initialized yet, free everything */
1380 spin_lock_irq(&q
->queue_lock
);
1381 list_for_each_entry(blkg
, &q
->blkg_list
, q_node
) {
1382 if (blkg
->pd
[pol
->plid
]) {
1383 pol
->pd_free_fn(blkg
->pd
[pol
->plid
]);
1384 blkg
->pd
[pol
->plid
] = NULL
;
1387 spin_unlock_irq(&q
->queue_lock
);
1391 EXPORT_SYMBOL_GPL(blkcg_activate_policy
);
1394 * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue
1395 * @q: request_queue of interest
1396 * @pol: blkcg policy to deactivate
1398 * Deactivate @pol on @q. Follows the same synchronization rules as
1399 * blkcg_activate_policy().
1401 void blkcg_deactivate_policy(struct request_queue
*q
,
1402 const struct blkcg_policy
*pol
)
1404 struct blkcg_gq
*blkg
;
1406 if (!blkcg_policy_enabled(q
, pol
))
1410 blk_mq_freeze_queue(q
);
1412 spin_lock_irq(&q
->queue_lock
);
1414 __clear_bit(pol
->plid
, q
->blkcg_pols
);
1416 list_for_each_entry(blkg
, &q
->blkg_list
, q_node
) {
1417 if (blkg
->pd
[pol
->plid
]) {
1418 if (pol
->pd_offline_fn
)
1419 pol
->pd_offline_fn(blkg
->pd
[pol
->plid
]);
1420 pol
->pd_free_fn(blkg
->pd
[pol
->plid
]);
1421 blkg
->pd
[pol
->plid
] = NULL
;
1425 spin_unlock_irq(&q
->queue_lock
);
1428 blk_mq_unfreeze_queue(q
);
1430 EXPORT_SYMBOL_GPL(blkcg_deactivate_policy
);
1433 * blkcg_policy_register - register a blkcg policy
1434 * @pol: blkcg policy to register
1436 * Register @pol with blkcg core. Might sleep and @pol may be modified on
1437 * successful registration. Returns 0 on success and -errno on failure.
1439 int blkcg_policy_register(struct blkcg_policy
*pol
)
1441 struct blkcg
*blkcg
;
1444 mutex_lock(&blkcg_pol_register_mutex
);
1445 mutex_lock(&blkcg_pol_mutex
);
1447 /* find an empty slot */
1449 for (i
= 0; i
< BLKCG_MAX_POLS
; i
++)
1450 if (!blkcg_policy
[i
])
1452 if (i
>= BLKCG_MAX_POLS
) {
1453 pr_warn("blkcg_policy_register: BLKCG_MAX_POLS too small\n");
1457 /* Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs */
1458 if ((!pol
->cpd_alloc_fn
^ !pol
->cpd_free_fn
) ||
1459 (!pol
->pd_alloc_fn
^ !pol
->pd_free_fn
))
1464 blkcg_policy
[pol
->plid
] = pol
;
1466 /* allocate and install cpd's */
1467 if (pol
->cpd_alloc_fn
) {
1468 list_for_each_entry(blkcg
, &all_blkcgs
, all_blkcgs_node
) {
1469 struct blkcg_policy_data
*cpd
;
1471 cpd
= pol
->cpd_alloc_fn(GFP_KERNEL
);
1475 blkcg
->cpd
[pol
->plid
] = cpd
;
1477 cpd
->plid
= pol
->plid
;
1478 if (pol
->cpd_init_fn
)
1479 pol
->cpd_init_fn(cpd
);
1483 mutex_unlock(&blkcg_pol_mutex
);
1485 /* everything is in place, add intf files for the new policy */
1486 if (pol
->dfl_cftypes
)
1487 WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys
,
1489 if (pol
->legacy_cftypes
)
1490 WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys
,
1491 pol
->legacy_cftypes
));
1492 mutex_unlock(&blkcg_pol_register_mutex
);
1496 if (pol
->cpd_free_fn
) {
1497 list_for_each_entry(blkcg
, &all_blkcgs
, all_blkcgs_node
) {
1498 if (blkcg
->cpd
[pol
->plid
]) {
1499 pol
->cpd_free_fn(blkcg
->cpd
[pol
->plid
]);
1500 blkcg
->cpd
[pol
->plid
] = NULL
;
1504 blkcg_policy
[pol
->plid
] = NULL
;
1506 mutex_unlock(&blkcg_pol_mutex
);
1507 mutex_unlock(&blkcg_pol_register_mutex
);
1510 EXPORT_SYMBOL_GPL(blkcg_policy_register
);
1513 * blkcg_policy_unregister - unregister a blkcg policy
1514 * @pol: blkcg policy to unregister
1516 * Undo blkcg_policy_register(@pol). Might sleep.
1518 void blkcg_policy_unregister(struct blkcg_policy
*pol
)
1520 struct blkcg
*blkcg
;
1522 mutex_lock(&blkcg_pol_register_mutex
);
1524 if (WARN_ON(blkcg_policy
[pol
->plid
] != pol
))
1527 /* kill the intf files first */
1528 if (pol
->dfl_cftypes
)
1529 cgroup_rm_cftypes(pol
->dfl_cftypes
);
1530 if (pol
->legacy_cftypes
)
1531 cgroup_rm_cftypes(pol
->legacy_cftypes
);
1533 /* remove cpds and unregister */
1534 mutex_lock(&blkcg_pol_mutex
);
1536 if (pol
->cpd_free_fn
) {
1537 list_for_each_entry(blkcg
, &all_blkcgs
, all_blkcgs_node
) {
1538 if (blkcg
->cpd
[pol
->plid
]) {
1539 pol
->cpd_free_fn(blkcg
->cpd
[pol
->plid
]);
1540 blkcg
->cpd
[pol
->plid
] = NULL
;
1544 blkcg_policy
[pol
->plid
] = NULL
;
1546 mutex_unlock(&blkcg_pol_mutex
);
1548 mutex_unlock(&blkcg_pol_register_mutex
);
1550 EXPORT_SYMBOL_GPL(blkcg_policy_unregister
);
1552 bool __blkcg_punt_bio_submit(struct bio
*bio
)
1554 struct blkcg_gq
*blkg
= bio
->bi_blkg
;
1556 /* consume the flag first */
1557 bio
->bi_opf
&= ~REQ_CGROUP_PUNT
;
1559 /* never bounce for the root cgroup */
1563 spin_lock_bh(&blkg
->async_bio_lock
);
1564 bio_list_add(&blkg
->async_bios
, bio
);
1565 spin_unlock_bh(&blkg
->async_bio_lock
);
1567 queue_work(blkcg_punt_bio_wq
, &blkg
->async_bio_work
);
1572 * Scale the accumulated delay based on how long it has been since we updated
1573 * the delay. We only call this when we are adding delay, in case it's been a
1574 * while since we added delay, and when we are checking to see if we need to
1575 * delay a task, to account for any delays that may have occurred.
1577 static void blkcg_scale_delay(struct blkcg_gq
*blkg
, u64 now
)
1579 u64 old
= atomic64_read(&blkg
->delay_start
);
1581 /* negative use_delay means no scaling, see blkcg_set_delay() */
1582 if (atomic_read(&blkg
->use_delay
) < 0)
1586 * We only want to scale down every second. The idea here is that we
1587 * want to delay people for min(delay_nsec, NSEC_PER_SEC) in a certain
1588 * time window. We only want to throttle tasks for recent delay that
1589 * has occurred, in 1 second time windows since that's the maximum
1590 * things can be throttled. We save the current delay window in
1591 * blkg->last_delay so we know what amount is still left to be charged
1592 * to the blkg from this point onward. blkg->last_use keeps track of
1593 * the use_delay counter. The idea is if we're unthrottling the blkg we
1594 * are ok with whatever is happening now, and we can take away more of
1595 * the accumulated delay as we've already throttled enough that
1596 * everybody is happy with their IO latencies.
1598 if (time_before64(old
+ NSEC_PER_SEC
, now
) &&
1599 atomic64_cmpxchg(&blkg
->delay_start
, old
, now
) == old
) {
1600 u64 cur
= atomic64_read(&blkg
->delay_nsec
);
1601 u64 sub
= min_t(u64
, blkg
->last_delay
, now
- old
);
1602 int cur_use
= atomic_read(&blkg
->use_delay
);
1605 * We've been unthrottled, subtract a larger chunk of our
1606 * accumulated delay.
1608 if (cur_use
< blkg
->last_use
)
1609 sub
= max_t(u64
, sub
, blkg
->last_delay
>> 1);
1612 * This shouldn't happen, but handle it anyway. Our delay_nsec
1613 * should only ever be growing except here where we subtract out
1614 * min(last_delay, 1 second), but lord knows bugs happen and I'd
1615 * rather not end up with negative numbers.
1617 if (unlikely(cur
< sub
)) {
1618 atomic64_set(&blkg
->delay_nsec
, 0);
1619 blkg
->last_delay
= 0;
1621 atomic64_sub(sub
, &blkg
->delay_nsec
);
1622 blkg
->last_delay
= cur
- sub
;
1624 blkg
->last_use
= cur_use
;
1629 * This is called when we want to actually walk up the hierarchy and check to
1630 * see if we need to throttle, and then actually throttle if there is some
1631 * accumulated delay. This should only be called upon return to user space so
1632 * we're not holding some lock that would induce a priority inversion.
1634 static void blkcg_maybe_throttle_blkg(struct blkcg_gq
*blkg
, bool use_memdelay
)
1636 unsigned long pflags
;
1638 u64 now
= ktime_to_ns(ktime_get());
1643 while (blkg
->parent
) {
1644 int use_delay
= atomic_read(&blkg
->use_delay
);
1649 blkcg_scale_delay(blkg
, now
);
1650 this_delay
= atomic64_read(&blkg
->delay_nsec
);
1651 if (this_delay
> delay_nsec
) {
1652 delay_nsec
= this_delay
;
1653 clamp
= use_delay
> 0;
1656 blkg
= blkg
->parent
;
1663 * Let's not sleep for all eternity if we've amassed a huge delay.
1664 * Swapping or metadata IO can accumulate 10's of seconds worth of
1665 * delay, and we want userspace to be able to do _something_ so cap the
1666 * delays at 0.25s. If there's 10's of seconds worth of delay then the
1667 * tasks will be delayed for 0.25 second for every syscall. If
1668 * blkcg_set_delay() was used as indicated by negative use_delay, the
1669 * caller is responsible for regulating the range.
1672 delay_nsec
= min_t(u64
, delay_nsec
, 250 * NSEC_PER_MSEC
);
1675 psi_memstall_enter(&pflags
);
1677 exp
= ktime_add_ns(now
, delay_nsec
);
1678 tok
= io_schedule_prepare();
1680 __set_current_state(TASK_KILLABLE
);
1681 if (!schedule_hrtimeout(&exp
, HRTIMER_MODE_ABS
))
1683 } while (!fatal_signal_pending(current
));
1684 io_schedule_finish(tok
);
1687 psi_memstall_leave(&pflags
);
1691 * blkcg_maybe_throttle_current - throttle the current task if it has been marked
1693 * This is only called if we've been marked with set_notify_resume(). Obviously
1694 * we can be set_notify_resume() for reasons other than blkcg throttling, so we
1695 * check to see if current->throttle_queue is set and if not this doesn't do
1696 * anything. This should only ever be called by the resume code, it's not meant
1697 * to be called by people willy-nilly as it will actually do the work to
1698 * throttle the task if it is setup for throttling.
1700 void blkcg_maybe_throttle_current(void)
1702 struct request_queue
*q
= current
->throttle_queue
;
1703 struct cgroup_subsys_state
*css
;
1704 struct blkcg
*blkcg
;
1705 struct blkcg_gq
*blkg
;
1706 bool use_memdelay
= current
->use_memdelay
;
1711 current
->throttle_queue
= NULL
;
1712 current
->use_memdelay
= false;
1715 css
= kthread_blkcg();
1717 blkcg
= css_to_blkcg(css
);
1719 blkcg
= css_to_blkcg(task_css(current
, io_cgrp_id
));
1723 blkg
= blkg_lookup(blkcg
, q
);
1726 if (!blkg_tryget(blkg
))
1730 blkcg_maybe_throttle_blkg(blkg
, use_memdelay
);
1740 * blkcg_schedule_throttle - this task needs to check for throttling
1741 * @q: the request queue IO was submitted on
1742 * @use_memdelay: do we charge this to memory delay for PSI
1744 * This is called by the IO controller when we know there's delay accumulated
1745 * for the blkg for this task. We do not pass the blkg because there are places
1746 * we call this that may not have that information, the swapping code for
1747 * instance will only have a request_queue at that point. This set's the
1748 * notify_resume for the task to check and see if it requires throttling before
1749 * returning to user space.
1751 * We will only schedule once per syscall. You can call this over and over
1752 * again and it will only do the check once upon return to user space, and only
1753 * throttle once. If the task needs to be throttled again it'll need to be
1754 * re-set at the next time we see the task.
1756 void blkcg_schedule_throttle(struct request_queue
*q
, bool use_memdelay
)
1758 if (unlikely(current
->flags
& PF_KTHREAD
))
1761 if (!blk_get_queue(q
))
1764 if (current
->throttle_queue
)
1765 blk_put_queue(current
->throttle_queue
);
1766 current
->throttle_queue
= q
;
1768 current
->use_memdelay
= use_memdelay
;
1769 set_notify_resume(current
);
1773 * blkcg_add_delay - add delay to this blkg
1774 * @blkg: blkg of interest
1775 * @now: the current time in nanoseconds
1776 * @delta: how many nanoseconds of delay to add
1778 * Charge @delta to the blkg's current delay accumulation. This is used to
1779 * throttle tasks if an IO controller thinks we need more throttling.
1781 void blkcg_add_delay(struct blkcg_gq
*blkg
, u64 now
, u64 delta
)
1783 if (WARN_ON_ONCE(atomic_read(&blkg
->use_delay
) < 0))
1785 blkcg_scale_delay(blkg
, now
);
1786 atomic64_add(delta
, &blkg
->delay_nsec
);
1790 * blkg_tryget_closest - try and get a blkg ref on the closet blkg
1794 * As the failure mode here is to walk up the blkg tree, this ensure that the
1795 * blkg->parent pointers are always valid. This returns the blkg that it ended
1796 * up taking a reference on or %NULL if no reference was taken.
1798 static inline struct blkcg_gq
*blkg_tryget_closest(struct bio
*bio
,
1799 struct cgroup_subsys_state
*css
)
1801 struct blkcg_gq
*blkg
, *ret_blkg
= NULL
;
1804 blkg
= blkg_lookup_create(css_to_blkcg(css
), bio
->bi_disk
->queue
);
1806 if (blkg_tryget(blkg
)) {
1810 blkg
= blkg
->parent
;
1818 * bio_associate_blkg_from_css - associate a bio with a specified css
1822 * Associate @bio with the blkg found by combining the css's blkg and the
1823 * request_queue of the @bio. An association failure is handled by walking up
1824 * the blkg tree. Therefore, the blkg associated can be anything between @blkg
1825 * and q->root_blkg. This situation only happens when a cgroup is dying and
1826 * then the remaining bios will spill to the closest alive blkg.
1828 * A reference will be taken on the blkg and will be released when @bio is
1831 void bio_associate_blkg_from_css(struct bio
*bio
,
1832 struct cgroup_subsys_state
*css
)
1835 blkg_put(bio
->bi_blkg
);
1837 if (css
&& css
->parent
) {
1838 bio
->bi_blkg
= blkg_tryget_closest(bio
, css
);
1840 blkg_get(bio
->bi_disk
->queue
->root_blkg
);
1841 bio
->bi_blkg
= bio
->bi_disk
->queue
->root_blkg
;
1844 EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css
);
1847 * bio_associate_blkg - associate a bio with a blkg
1850 * Associate @bio with the blkg found from the bio's css and request_queue.
1851 * If one is not found, bio_lookup_blkg() creates the blkg. If a blkg is
1852 * already associated, the css is reused and association redone as the
1853 * request_queue may have changed.
1855 void bio_associate_blkg(struct bio
*bio
)
1857 struct cgroup_subsys_state
*css
;
1862 css
= &bio_blkcg(bio
)->css
;
1866 bio_associate_blkg_from_css(bio
, css
);
1870 EXPORT_SYMBOL_GPL(bio_associate_blkg
);
1873 * bio_clone_blkg_association - clone blkg association from src to dst bio
1874 * @dst: destination bio
1877 void bio_clone_blkg_association(struct bio
*dst
, struct bio
*src
)
1881 blkg_put(dst
->bi_blkg
);
1882 blkg_get(src
->bi_blkg
);
1883 dst
->bi_blkg
= src
->bi_blkg
;
1886 EXPORT_SYMBOL_GPL(bio_clone_blkg_association
);
1888 static int blk_cgroup_io_type(struct bio
*bio
)
1890 if (op_is_discard(bio
->bi_opf
))
1891 return BLKG_IOSTAT_DISCARD
;
1892 if (op_is_write(bio
->bi_opf
))
1893 return BLKG_IOSTAT_WRITE
;
1894 return BLKG_IOSTAT_READ
;
1897 void blk_cgroup_bio_start(struct bio
*bio
)
1899 int rwd
= blk_cgroup_io_type(bio
), cpu
;
1900 struct blkg_iostat_set
*bis
;
1903 bis
= per_cpu_ptr(bio
->bi_blkg
->iostat_cpu
, cpu
);
1904 u64_stats_update_begin(&bis
->sync
);
1907 * If the bio is flagged with BIO_CGROUP_ACCT it means this is a split
1908 * bio and we would have already accounted for the size of the bio.
1910 if (!bio_flagged(bio
, BIO_CGROUP_ACCT
)) {
1911 bio_set_flag(bio
, BIO_CGROUP_ACCT
);
1912 bis
->cur
.bytes
[rwd
] += bio
->bi_iter
.bi_size
;
1914 bis
->cur
.ios
[rwd
]++;
1916 u64_stats_update_end(&bis
->sync
);
1917 if (cgroup_subsys_on_dfl(io_cgrp_subsys
))
1918 cgroup_rstat_updated(bio
->bi_blkg
->blkcg
->css
.cgroup
, cpu
);
1922 static int __init
blkcg_init(void)
1924 blkcg_punt_bio_wq
= alloc_workqueue("blkcg_punt_bio",
1925 WQ_MEM_RECLAIM
| WQ_FREEZABLE
|
1926 WQ_UNBOUND
| WQ_SYSFS
, 0);
1927 if (!blkcg_punt_bio_wq
)
1931 subsys_initcall(blkcg_init
);
1933 module_param(blkcg_debug_stats
, bool, 0644);
1934 MODULE_PARM_DESC(blkcg_debug_stats
, "True if you want debug stats, false if not");