1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * cgroups support for the BFQ I/O scheduler.
5 #include <linux/module.h>
6 #include <linux/slab.h>
7 #include <linux/blkdev.h>
8 #include <linux/cgroup.h>
9 #include <linux/elevator.h>
10 #include <linux/ktime.h>
11 #include <linux/rbtree.h>
12 #include <linux/ioprio.h>
13 #include <linux/sbitmap.h>
14 #include <linux/delay.h>
16 #include "bfq-iosched.h"
18 #ifdef CONFIG_BFQ_CGROUP_DEBUG
19 static int bfq_stat_init(struct bfq_stat
*stat
, gfp_t gfp
)
23 ret
= percpu_counter_init(&stat
->cpu_cnt
, 0, gfp
);
27 atomic64_set(&stat
->aux_cnt
, 0);
31 static void bfq_stat_exit(struct bfq_stat
*stat
)
33 percpu_counter_destroy(&stat
->cpu_cnt
);
37 * bfq_stat_add - add a value to a bfq_stat
38 * @stat: target bfq_stat
41 * Add @val to @stat. The caller must ensure that IRQ on the same CPU
42 * don't re-enter this function for the same counter.
44 static inline void bfq_stat_add(struct bfq_stat
*stat
, uint64_t val
)
46 percpu_counter_add_batch(&stat
->cpu_cnt
, val
, BLKG_STAT_CPU_BATCH
);
50 * bfq_stat_read - read the current value of a bfq_stat
51 * @stat: bfq_stat to read
53 static inline uint64_t bfq_stat_read(struct bfq_stat
*stat
)
55 return percpu_counter_sum_positive(&stat
->cpu_cnt
);
59 * bfq_stat_reset - reset a bfq_stat
60 * @stat: bfq_stat to reset
62 static inline void bfq_stat_reset(struct bfq_stat
*stat
)
64 percpu_counter_set(&stat
->cpu_cnt
, 0);
65 atomic64_set(&stat
->aux_cnt
, 0);
69 * bfq_stat_add_aux - add a bfq_stat into another's aux count
70 * @to: the destination bfq_stat
73 * Add @from's count including the aux one to @to's aux count.
75 static inline void bfq_stat_add_aux(struct bfq_stat
*to
,
76 struct bfq_stat
*from
)
78 atomic64_add(bfq_stat_read(from
) + atomic64_read(&from
->aux_cnt
),
83 * blkg_prfill_stat - prfill callback for bfq_stat
84 * @sf: seq_file to print to
85 * @pd: policy private data of interest
86 * @off: offset to the bfq_stat in @pd
88 * prfill callback for printing a bfq_stat.
90 static u64
blkg_prfill_stat(struct seq_file
*sf
, struct blkg_policy_data
*pd
,
93 return __blkg_prfill_u64(sf
, pd
, bfq_stat_read((void *)pd
+ off
));
96 /* bfqg stats flags */
97 enum bfqg_stats_flags
{
98 BFQG_stats_waiting
= 0,
103 #define BFQG_FLAG_FNS(name) \
104 static void bfqg_stats_mark_##name(struct bfqg_stats *stats) \
106 stats->flags |= (1 << BFQG_stats_##name); \
108 static void bfqg_stats_clear_##name(struct bfqg_stats *stats) \
110 stats->flags &= ~(1 << BFQG_stats_##name); \
112 static int bfqg_stats_##name(struct bfqg_stats *stats) \
114 return (stats->flags & (1 << BFQG_stats_##name)) != 0; \
117 BFQG_FLAG_FNS(waiting)
118 BFQG_FLAG_FNS(idling
)
122 /* This should be called with the scheduler lock held. */
123 static void bfqg_stats_update_group_wait_time(struct bfqg_stats
*stats
)
127 if (!bfqg_stats_waiting(stats
))
130 now
= ktime_get_ns();
131 if (now
> stats
->start_group_wait_time
)
132 bfq_stat_add(&stats
->group_wait_time
,
133 now
- stats
->start_group_wait_time
);
134 bfqg_stats_clear_waiting(stats
);
137 /* This should be called with the scheduler lock held. */
138 static void bfqg_stats_set_start_group_wait_time(struct bfq_group
*bfqg
,
139 struct bfq_group
*curr_bfqg
)
141 struct bfqg_stats
*stats
= &bfqg
->stats
;
143 if (bfqg_stats_waiting(stats
))
145 if (bfqg
== curr_bfqg
)
147 stats
->start_group_wait_time
= ktime_get_ns();
148 bfqg_stats_mark_waiting(stats
);
151 /* This should be called with the scheduler lock held. */
152 static void bfqg_stats_end_empty_time(struct bfqg_stats
*stats
)
156 if (!bfqg_stats_empty(stats
))
159 now
= ktime_get_ns();
160 if (now
> stats
->start_empty_time
)
161 bfq_stat_add(&stats
->empty_time
,
162 now
- stats
->start_empty_time
);
163 bfqg_stats_clear_empty(stats
);
166 void bfqg_stats_update_dequeue(struct bfq_group
*bfqg
)
168 bfq_stat_add(&bfqg
->stats
.dequeue
, 1);
171 void bfqg_stats_set_start_empty_time(struct bfq_group
*bfqg
)
173 struct bfqg_stats
*stats
= &bfqg
->stats
;
175 if (blkg_rwstat_total(&stats
->queued
))
179 * group is already marked empty. This can happen if bfqq got new
180 * request in parent group and moved to this group while being added
181 * to service tree. Just ignore the event and move on.
183 if (bfqg_stats_empty(stats
))
186 stats
->start_empty_time
= ktime_get_ns();
187 bfqg_stats_mark_empty(stats
);
190 void bfqg_stats_update_idle_time(struct bfq_group
*bfqg
)
192 struct bfqg_stats
*stats
= &bfqg
->stats
;
194 if (bfqg_stats_idling(stats
)) {
195 u64 now
= ktime_get_ns();
197 if (now
> stats
->start_idle_time
)
198 bfq_stat_add(&stats
->idle_time
,
199 now
- stats
->start_idle_time
);
200 bfqg_stats_clear_idling(stats
);
204 void bfqg_stats_set_start_idle_time(struct bfq_group
*bfqg
)
206 struct bfqg_stats
*stats
= &bfqg
->stats
;
208 stats
->start_idle_time
= ktime_get_ns();
209 bfqg_stats_mark_idling(stats
);
212 void bfqg_stats_update_avg_queue_size(struct bfq_group
*bfqg
)
214 struct bfqg_stats
*stats
= &bfqg
->stats
;
216 bfq_stat_add(&stats
->avg_queue_size_sum
,
217 blkg_rwstat_total(&stats
->queued
));
218 bfq_stat_add(&stats
->avg_queue_size_samples
, 1);
219 bfqg_stats_update_group_wait_time(stats
);
222 void bfqg_stats_update_io_add(struct bfq_group
*bfqg
, struct bfq_queue
*bfqq
,
225 blkg_rwstat_add(&bfqg
->stats
.queued
, op
, 1);
226 bfqg_stats_end_empty_time(&bfqg
->stats
);
227 if (!(bfqq
== ((struct bfq_data
*)bfqg
->bfqd
)->in_service_queue
))
228 bfqg_stats_set_start_group_wait_time(bfqg
, bfqq_group(bfqq
));
231 void bfqg_stats_update_io_remove(struct bfq_group
*bfqg
, unsigned int op
)
233 blkg_rwstat_add(&bfqg
->stats
.queued
, op
, -1);
236 void bfqg_stats_update_io_merged(struct bfq_group
*bfqg
, unsigned int op
)
238 blkg_rwstat_add(&bfqg
->stats
.merged
, op
, 1);
241 void bfqg_stats_update_completion(struct bfq_group
*bfqg
, u64 start_time_ns
,
242 u64 io_start_time_ns
, unsigned int op
)
244 struct bfqg_stats
*stats
= &bfqg
->stats
;
245 u64 now
= ktime_get_ns();
247 if (now
> io_start_time_ns
)
248 blkg_rwstat_add(&stats
->service_time
, op
,
249 now
- io_start_time_ns
);
250 if (io_start_time_ns
> start_time_ns
)
251 blkg_rwstat_add(&stats
->wait_time
, op
,
252 io_start_time_ns
- start_time_ns
);
255 #else /* CONFIG_BFQ_CGROUP_DEBUG */
257 void bfqg_stats_update_io_add(struct bfq_group
*bfqg
, struct bfq_queue
*bfqq
,
259 void bfqg_stats_update_io_remove(struct bfq_group
*bfqg
, unsigned int op
) { }
260 void bfqg_stats_update_io_merged(struct bfq_group
*bfqg
, unsigned int op
) { }
261 void bfqg_stats_update_completion(struct bfq_group
*bfqg
, u64 start_time_ns
,
262 u64 io_start_time_ns
, unsigned int op
) { }
263 void bfqg_stats_update_dequeue(struct bfq_group
*bfqg
) { }
264 void bfqg_stats_set_start_empty_time(struct bfq_group
*bfqg
) { }
265 void bfqg_stats_update_idle_time(struct bfq_group
*bfqg
) { }
266 void bfqg_stats_set_start_idle_time(struct bfq_group
*bfqg
) { }
267 void bfqg_stats_update_avg_queue_size(struct bfq_group
*bfqg
) { }
269 #endif /* CONFIG_BFQ_CGROUP_DEBUG */
271 #ifdef CONFIG_BFQ_GROUP_IOSCHED
274 * blk-cgroup policy-related handlers
275 * The following functions help in converting between blk-cgroup
276 * internal structures and BFQ-specific structures.
279 static struct bfq_group
*pd_to_bfqg(struct blkg_policy_data
*pd
)
281 return pd
? container_of(pd
, struct bfq_group
, pd
) : NULL
;
284 struct blkcg_gq
*bfqg_to_blkg(struct bfq_group
*bfqg
)
286 return pd_to_blkg(&bfqg
->pd
);
289 static struct bfq_group
*blkg_to_bfqg(struct blkcg_gq
*blkg
)
291 return pd_to_bfqg(blkg_to_pd(blkg
, &blkcg_policy_bfq
));
296 * The following functions help in navigating the bfq_group hierarchy
297 * by allowing to find the parent of a bfq_group or the bfq_group
298 * associated to a bfq_queue.
301 static struct bfq_group
*bfqg_parent(struct bfq_group
*bfqg
)
303 struct blkcg_gq
*pblkg
= bfqg_to_blkg(bfqg
)->parent
;
305 return pblkg
? blkg_to_bfqg(pblkg
) : NULL
;
308 struct bfq_group
*bfqq_group(struct bfq_queue
*bfqq
)
310 struct bfq_entity
*group_entity
= bfqq
->entity
.parent
;
312 return group_entity
? container_of(group_entity
, struct bfq_group
,
314 bfqq
->bfqd
->root_group
;
318 * The following two functions handle get and put of a bfq_group by
319 * wrapping the related blk-cgroup hooks.
322 static void bfqg_get(struct bfq_group
*bfqg
)
327 static void bfqg_put(struct bfq_group
*bfqg
)
335 static void bfqg_and_blkg_get(struct bfq_group
*bfqg
)
337 /* see comments in bfq_bic_update_cgroup for why refcounting bfqg */
340 blkg_get(bfqg_to_blkg(bfqg
));
343 void bfqg_and_blkg_put(struct bfq_group
*bfqg
)
345 blkg_put(bfqg_to_blkg(bfqg
));
351 static void bfqg_stats_reset(struct bfqg_stats
*stats
)
353 #ifdef CONFIG_BFQ_CGROUP_DEBUG
354 /* queued stats shouldn't be cleared */
355 blkg_rwstat_reset(&stats
->merged
);
356 blkg_rwstat_reset(&stats
->service_time
);
357 blkg_rwstat_reset(&stats
->wait_time
);
358 bfq_stat_reset(&stats
->time
);
359 bfq_stat_reset(&stats
->avg_queue_size_sum
);
360 bfq_stat_reset(&stats
->avg_queue_size_samples
);
361 bfq_stat_reset(&stats
->dequeue
);
362 bfq_stat_reset(&stats
->group_wait_time
);
363 bfq_stat_reset(&stats
->idle_time
);
364 bfq_stat_reset(&stats
->empty_time
);
369 static void bfqg_stats_add_aux(struct bfqg_stats
*to
, struct bfqg_stats
*from
)
374 #ifdef CONFIG_BFQ_CGROUP_DEBUG
375 /* queued stats shouldn't be cleared */
376 blkg_rwstat_add_aux(&to
->merged
, &from
->merged
);
377 blkg_rwstat_add_aux(&to
->service_time
, &from
->service_time
);
378 blkg_rwstat_add_aux(&to
->wait_time
, &from
->wait_time
);
379 bfq_stat_add_aux(&from
->time
, &from
->time
);
380 bfq_stat_add_aux(&to
->avg_queue_size_sum
, &from
->avg_queue_size_sum
);
381 bfq_stat_add_aux(&to
->avg_queue_size_samples
,
382 &from
->avg_queue_size_samples
);
383 bfq_stat_add_aux(&to
->dequeue
, &from
->dequeue
);
384 bfq_stat_add_aux(&to
->group_wait_time
, &from
->group_wait_time
);
385 bfq_stat_add_aux(&to
->idle_time
, &from
->idle_time
);
386 bfq_stat_add_aux(&to
->empty_time
, &from
->empty_time
);
391 * Transfer @bfqg's stats to its parent's aux counts so that the ancestors'
392 * recursive stats can still account for the amount used by this bfqg after
395 static void bfqg_stats_xfer_dead(struct bfq_group
*bfqg
)
397 struct bfq_group
*parent
;
399 if (!bfqg
) /* root_group */
402 parent
= bfqg_parent(bfqg
);
404 lockdep_assert_held(&bfqg_to_blkg(bfqg
)->q
->queue_lock
);
406 if (unlikely(!parent
))
409 bfqg_stats_add_aux(&parent
->stats
, &bfqg
->stats
);
410 bfqg_stats_reset(&bfqg
->stats
);
413 void bfq_init_entity(struct bfq_entity
*entity
, struct bfq_group
*bfqg
)
415 struct bfq_queue
*bfqq
= bfq_entity_to_bfqq(entity
);
417 entity
->weight
= entity
->new_weight
;
418 entity
->orig_weight
= entity
->new_weight
;
420 bfqq
->ioprio
= bfqq
->new_ioprio
;
421 bfqq
->ioprio_class
= bfqq
->new_ioprio_class
;
423 * Make sure that bfqg and its associated blkg do not
424 * disappear before entity.
426 bfqg_and_blkg_get(bfqg
);
428 entity
->parent
= bfqg
->my_entity
; /* NULL for root group */
429 entity
->sched_data
= &bfqg
->sched_data
;
432 static void bfqg_stats_exit(struct bfqg_stats
*stats
)
434 #ifdef CONFIG_BFQ_CGROUP_DEBUG
435 blkg_rwstat_exit(&stats
->merged
);
436 blkg_rwstat_exit(&stats
->service_time
);
437 blkg_rwstat_exit(&stats
->wait_time
);
438 blkg_rwstat_exit(&stats
->queued
);
439 bfq_stat_exit(&stats
->time
);
440 bfq_stat_exit(&stats
->avg_queue_size_sum
);
441 bfq_stat_exit(&stats
->avg_queue_size_samples
);
442 bfq_stat_exit(&stats
->dequeue
);
443 bfq_stat_exit(&stats
->group_wait_time
);
444 bfq_stat_exit(&stats
->idle_time
);
445 bfq_stat_exit(&stats
->empty_time
);
449 static int bfqg_stats_init(struct bfqg_stats
*stats
, gfp_t gfp
)
451 #ifdef CONFIG_BFQ_CGROUP_DEBUG
452 if (blkg_rwstat_init(&stats
->merged
, gfp
) ||
453 blkg_rwstat_init(&stats
->service_time
, gfp
) ||
454 blkg_rwstat_init(&stats
->wait_time
, gfp
) ||
455 blkg_rwstat_init(&stats
->queued
, gfp
) ||
456 bfq_stat_init(&stats
->time
, gfp
) ||
457 bfq_stat_init(&stats
->avg_queue_size_sum
, gfp
) ||
458 bfq_stat_init(&stats
->avg_queue_size_samples
, gfp
) ||
459 bfq_stat_init(&stats
->dequeue
, gfp
) ||
460 bfq_stat_init(&stats
->group_wait_time
, gfp
) ||
461 bfq_stat_init(&stats
->idle_time
, gfp
) ||
462 bfq_stat_init(&stats
->empty_time
, gfp
)) {
463 bfqg_stats_exit(stats
);
471 static struct bfq_group_data
*cpd_to_bfqgd(struct blkcg_policy_data
*cpd
)
473 return cpd
? container_of(cpd
, struct bfq_group_data
, pd
) : NULL
;
476 static struct bfq_group_data
*blkcg_to_bfqgd(struct blkcg
*blkcg
)
478 return cpd_to_bfqgd(blkcg_to_cpd(blkcg
, &blkcg_policy_bfq
));
481 static struct blkcg_policy_data
*bfq_cpd_alloc(gfp_t gfp
)
483 struct bfq_group_data
*bgd
;
485 bgd
= kzalloc(sizeof(*bgd
), gfp
);
491 static void bfq_cpd_init(struct blkcg_policy_data
*cpd
)
493 struct bfq_group_data
*d
= cpd_to_bfqgd(cpd
);
495 d
->weight
= cgroup_subsys_on_dfl(io_cgrp_subsys
) ?
496 CGROUP_WEIGHT_DFL
: BFQ_WEIGHT_LEGACY_DFL
;
499 static void bfq_cpd_free(struct blkcg_policy_data
*cpd
)
501 kfree(cpd_to_bfqgd(cpd
));
504 static struct blkg_policy_data
*bfq_pd_alloc(gfp_t gfp
, struct request_queue
*q
,
507 struct bfq_group
*bfqg
;
509 bfqg
= kzalloc_node(sizeof(*bfqg
), gfp
, q
->node
);
513 if (bfqg_stats_init(&bfqg
->stats
, gfp
)) {
518 /* see comments in bfq_bic_update_cgroup for why refcounting */
523 static void bfq_pd_init(struct blkg_policy_data
*pd
)
525 struct blkcg_gq
*blkg
= pd_to_blkg(pd
);
526 struct bfq_group
*bfqg
= blkg_to_bfqg(blkg
);
527 struct bfq_data
*bfqd
= blkg
->q
->elevator
->elevator_data
;
528 struct bfq_entity
*entity
= &bfqg
->entity
;
529 struct bfq_group_data
*d
= blkcg_to_bfqgd(blkg
->blkcg
);
531 entity
->orig_weight
= entity
->weight
= entity
->new_weight
= d
->weight
;
532 entity
->my_sched_data
= &bfqg
->sched_data
;
533 bfqg
->my_entity
= entity
; /*
534 * the root_group's will be set to NULL
535 * in bfq_init_queue()
538 bfqg
->active_entities
= 0;
539 bfqg
->rq_pos_tree
= RB_ROOT
;
542 static void bfq_pd_free(struct blkg_policy_data
*pd
)
544 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
546 bfqg_stats_exit(&bfqg
->stats
);
550 static void bfq_pd_reset_stats(struct blkg_policy_data
*pd
)
552 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
554 bfqg_stats_reset(&bfqg
->stats
);
557 static void bfq_group_set_parent(struct bfq_group
*bfqg
,
558 struct bfq_group
*parent
)
560 struct bfq_entity
*entity
;
562 entity
= &bfqg
->entity
;
563 entity
->parent
= parent
->my_entity
;
564 entity
->sched_data
= &parent
->sched_data
;
567 static struct bfq_group
*bfq_lookup_bfqg(struct bfq_data
*bfqd
,
570 struct blkcg_gq
*blkg
;
572 blkg
= blkg_lookup(blkcg
, bfqd
->queue
);
574 return blkg_to_bfqg(blkg
);
578 struct bfq_group
*bfq_find_set_group(struct bfq_data
*bfqd
,
581 struct bfq_group
*bfqg
, *parent
;
582 struct bfq_entity
*entity
;
584 bfqg
= bfq_lookup_bfqg(bfqd
, blkcg
);
590 * Update chain of bfq_groups as we might be handling a leaf group
591 * which, along with some of its relatives, has not been hooked yet
592 * to the private hierarchy of BFQ.
594 entity
= &bfqg
->entity
;
595 for_each_entity(entity
) {
596 struct bfq_group
*curr_bfqg
= container_of(entity
,
597 struct bfq_group
, entity
);
598 if (curr_bfqg
!= bfqd
->root_group
) {
599 parent
= bfqg_parent(curr_bfqg
);
601 parent
= bfqd
->root_group
;
602 bfq_group_set_parent(curr_bfqg
, parent
);
610 * bfq_bfqq_move - migrate @bfqq to @bfqg.
611 * @bfqd: queue descriptor.
612 * @bfqq: the queue to move.
613 * @bfqg: the group to move to.
615 * Move @bfqq to @bfqg, deactivating it from its old group and reactivating
616 * it on the new one. Avoid putting the entity on the old group idle tree.
618 * Must be called under the scheduler lock, to make sure that the blkg
619 * owning @bfqg does not disappear (see comments in
620 * bfq_bic_update_cgroup on guaranteeing the consistency of blkg
623 void bfq_bfqq_move(struct bfq_data
*bfqd
, struct bfq_queue
*bfqq
,
624 struct bfq_group
*bfqg
)
626 struct bfq_entity
*entity
= &bfqq
->entity
;
629 * oom_bfqq is not allowed to move, oom_bfqq will hold ref to root_group
630 * until elevator exit.
632 if (bfqq
== &bfqd
->oom_bfqq
)
635 * Get extra reference to prevent bfqq from being freed in
636 * next possible expire or deactivate.
640 /* If bfqq is empty, then bfq_bfqq_expire also invokes
641 * bfq_del_bfqq_busy, thereby removing bfqq and its entity
642 * from data structures related to current group. Otherwise we
643 * need to remove bfqq explicitly with bfq_deactivate_bfqq, as
646 if (bfqq
== bfqd
->in_service_queue
)
647 bfq_bfqq_expire(bfqd
, bfqd
->in_service_queue
,
648 false, BFQQE_PREEMPTED
);
650 if (bfq_bfqq_busy(bfqq
))
651 bfq_deactivate_bfqq(bfqd
, bfqq
, false, false);
652 else if (entity
->on_st
)
653 bfq_put_idle_entity(bfq_entity_service_tree(entity
), entity
);
654 bfqg_and_blkg_put(bfqq_group(bfqq
));
656 entity
->parent
= bfqg
->my_entity
;
657 entity
->sched_data
= &bfqg
->sched_data
;
658 /* pin down bfqg and its associated blkg */
659 bfqg_and_blkg_get(bfqg
);
661 if (bfq_bfqq_busy(bfqq
)) {
662 if (unlikely(!bfqd
->nonrot_with_queueing
))
663 bfq_pos_tree_add_move(bfqd
, bfqq
);
664 bfq_activate_bfqq(bfqd
, bfqq
);
667 if (!bfqd
->in_service_queue
&& !bfqd
->rq_in_driver
)
668 bfq_schedule_dispatch(bfqd
);
669 /* release extra ref taken above, bfqq may happen to be freed now */
674 * __bfq_bic_change_cgroup - move @bic to @cgroup.
675 * @bfqd: the queue descriptor.
676 * @bic: the bic to move.
677 * @blkcg: the blk-cgroup to move to.
679 * Move bic to blkcg, assuming that bfqd->lock is held; which makes
680 * sure that the reference to cgroup is valid across the call (see
681 * comments in bfq_bic_update_cgroup on this issue)
683 * NOTE: an alternative approach might have been to store the current
684 * cgroup in bfqq and getting a reference to it, reducing the lookup
685 * time here, at the price of slightly more complex code.
687 static struct bfq_group
*__bfq_bic_change_cgroup(struct bfq_data
*bfqd
,
688 struct bfq_io_cq
*bic
,
691 struct bfq_queue
*async_bfqq
= bic_to_bfqq(bic
, 0);
692 struct bfq_queue
*sync_bfqq
= bic_to_bfqq(bic
, 1);
693 struct bfq_group
*bfqg
;
694 struct bfq_entity
*entity
;
696 bfqg
= bfq_find_set_group(bfqd
, blkcg
);
699 bfqg
= bfqd
->root_group
;
702 entity
= &async_bfqq
->entity
;
704 if (entity
->sched_data
!= &bfqg
->sched_data
) {
705 bic_set_bfqq(bic
, NULL
, 0);
706 bfq_release_process_ref(bfqd
, async_bfqq
);
711 entity
= &sync_bfqq
->entity
;
712 if (entity
->sched_data
!= &bfqg
->sched_data
)
713 bfq_bfqq_move(bfqd
, sync_bfqq
, bfqg
);
719 void bfq_bic_update_cgroup(struct bfq_io_cq
*bic
, struct bio
*bio
)
721 struct bfq_data
*bfqd
= bic_to_bfqd(bic
);
722 struct bfq_group
*bfqg
= NULL
;
726 serial_nr
= __bio_blkcg(bio
)->css
.serial_nr
;
729 * Check whether blkcg has changed. The condition may trigger
730 * spuriously on a newly created cic but there's no harm.
732 if (unlikely(!bfqd
) || likely(bic
->blkcg_serial_nr
== serial_nr
))
735 bfqg
= __bfq_bic_change_cgroup(bfqd
, bic
, __bio_blkcg(bio
));
737 * Update blkg_path for bfq_log_* functions. We cache this
738 * path, and update it here, for the following
739 * reasons. Operations on blkg objects in blk-cgroup are
740 * protected with the request_queue lock, and not with the
741 * lock that protects the instances of this scheduler
742 * (bfqd->lock). This exposes BFQ to the following sort of
745 * The blkg_lookup performed in bfq_get_queue, protected
746 * through rcu, may happen to return the address of a copy of
747 * the original blkg. If this is the case, then the
748 * bfqg_and_blkg_get performed in bfq_get_queue, to pin down
749 * the blkg, is useless: it does not prevent blk-cgroup code
750 * from destroying both the original blkg and all objects
751 * directly or indirectly referred by the copy of the
754 * On the bright side, destroy operations on a blkg invoke, as
755 * a first step, hooks of the scheduler associated with the
756 * blkg. And these hooks are executed with bfqd->lock held for
757 * BFQ. As a consequence, for any blkg associated with the
758 * request queue this instance of the scheduler is attached
759 * to, we are guaranteed that such a blkg is not destroyed, and
760 * that all the pointers it contains are consistent, while we
761 * are holding bfqd->lock. A blkg_lookup performed with
762 * bfqd->lock held then returns a fully consistent blkg, which
763 * remains consistent until this lock is held.
765 * Thanks to the last fact, and to the fact that: (1) bfqg has
766 * been obtained through a blkg_lookup in the above
767 * assignment, and (2) bfqd->lock is being held, here we can
768 * safely use the policy data for the involved blkg (i.e., the
769 * field bfqg->pd) to get to the blkg associated with bfqg,
770 * and then we can safely use any field of blkg. After we
771 * release bfqd->lock, even just getting blkg through this
772 * bfqg may cause dangling references to be traversed, as
773 * bfqg->pd may not exist any more.
775 * In view of the above facts, here we cache, in the bfqg, any
776 * blkg data we may need for this bic, and for its associated
777 * bfq_queue. As of now, we need to cache only the path of the
778 * blkg, which is used in the bfq_log_* functions.
780 * Finally, note that bfqg itself needs to be protected from
781 * destruction on the blkg_free of the original blkg (which
782 * invokes bfq_pd_free). We use an additional private
783 * refcounter for bfqg, to let it disappear only after no
784 * bfq_queue refers to it any longer.
786 blkg_path(bfqg_to_blkg(bfqg
), bfqg
->blkg_path
, sizeof(bfqg
->blkg_path
));
787 bic
->blkcg_serial_nr
= serial_nr
;
793 * bfq_flush_idle_tree - deactivate any entity on the idle tree of @st.
794 * @st: the service tree being flushed.
796 static void bfq_flush_idle_tree(struct bfq_service_tree
*st
)
798 struct bfq_entity
*entity
= st
->first_idle
;
800 for (; entity
; entity
= st
->first_idle
)
801 __bfq_deactivate_entity(entity
, false);
805 * bfq_reparent_leaf_entity - move leaf entity to the root_group.
806 * @bfqd: the device data structure with the root group.
807 * @entity: the entity to move, if entity is a leaf; or the parent entity
808 * of an active leaf entity to move, if entity is not a leaf.
810 static void bfq_reparent_leaf_entity(struct bfq_data
*bfqd
,
811 struct bfq_entity
*entity
,
814 struct bfq_queue
*bfqq
;
815 struct bfq_entity
*child_entity
= entity
;
817 while (child_entity
->my_sched_data
) { /* leaf not reached yet */
818 struct bfq_sched_data
*child_sd
= child_entity
->my_sched_data
;
819 struct bfq_service_tree
*child_st
= child_sd
->service_tree
+
821 struct rb_root
*child_active
= &child_st
->active
;
823 child_entity
= bfq_entity_of(rb_first(child_active
));
826 child_entity
= child_sd
->in_service_entity
;
829 bfqq
= bfq_entity_to_bfqq(child_entity
);
830 bfq_bfqq_move(bfqd
, bfqq
, bfqd
->root_group
);
834 * bfq_reparent_active_queues - move to the root group all active queues.
835 * @bfqd: the device data structure with the root group.
836 * @bfqg: the group to move from.
837 * @st: the service tree to start the search from.
839 static void bfq_reparent_active_queues(struct bfq_data
*bfqd
,
840 struct bfq_group
*bfqg
,
841 struct bfq_service_tree
*st
,
844 struct rb_root
*active
= &st
->active
;
845 struct bfq_entity
*entity
;
847 while ((entity
= bfq_entity_of(rb_first(active
))))
848 bfq_reparent_leaf_entity(bfqd
, entity
, ioprio_class
);
850 if (bfqg
->sched_data
.in_service_entity
)
851 bfq_reparent_leaf_entity(bfqd
,
852 bfqg
->sched_data
.in_service_entity
,
857 * bfq_pd_offline - deactivate the entity associated with @pd,
858 * and reparent its children entities.
859 * @pd: descriptor of the policy going offline.
861 * blkio already grabs the queue_lock for us, so no need to use
864 static void bfq_pd_offline(struct blkg_policy_data
*pd
)
866 struct bfq_service_tree
*st
;
867 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
868 struct bfq_data
*bfqd
= bfqg
->bfqd
;
869 struct bfq_entity
*entity
= bfqg
->my_entity
;
873 spin_lock_irqsave(&bfqd
->lock
, flags
);
875 if (!entity
) /* root group */
876 goto put_async_queues
;
879 * Empty all service_trees belonging to this group before
880 * deactivating the group itself.
882 for (i
= 0; i
< BFQ_IOPRIO_CLASSES
; i
++) {
883 st
= bfqg
->sched_data
.service_tree
+ i
;
886 * It may happen that some queues are still active
887 * (busy) upon group destruction (if the corresponding
888 * processes have been forced to terminate). We move
889 * all the leaf entities corresponding to these queues
891 * Also, it may happen that the group has an entity
892 * in service, which is disconnected from the active
893 * tree: it must be moved, too.
894 * There is no need to put the sync queues, as the
895 * scheduler has taken no reference.
897 bfq_reparent_active_queues(bfqd
, bfqg
, st
, i
);
900 * The idle tree may still contain bfq_queues
901 * belonging to exited task because they never
902 * migrated to a different cgroup from the one being
903 * destroyed now. In addition, even
904 * bfq_reparent_active_queues() may happen to add some
905 * entities to the idle tree. It happens if, in some
906 * of the calls to bfq_bfqq_move() performed by
907 * bfq_reparent_active_queues(), the queue to move is
908 * empty and gets expired.
910 bfq_flush_idle_tree(st
);
913 __bfq_deactivate_entity(entity
, false);
916 bfq_put_async_queues(bfqd
, bfqg
);
918 spin_unlock_irqrestore(&bfqd
->lock
, flags
);
920 * @blkg is going offline and will be ignored by
921 * blkg_[rw]stat_recursive_sum(). Transfer stats to the parent so
922 * that they don't get lost. If IOs complete after this point, the
923 * stats for them will be lost. Oh well...
925 bfqg_stats_xfer_dead(bfqg
);
928 void bfq_end_wr_async(struct bfq_data
*bfqd
)
930 struct blkcg_gq
*blkg
;
932 list_for_each_entry(blkg
, &bfqd
->queue
->blkg_list
, q_node
) {
933 struct bfq_group
*bfqg
= blkg_to_bfqg(blkg
);
935 bfq_end_wr_async_queues(bfqd
, bfqg
);
937 bfq_end_wr_async_queues(bfqd
, bfqd
->root_group
);
940 static int bfq_io_show_weight_legacy(struct seq_file
*sf
, void *v
)
942 struct blkcg
*blkcg
= css_to_blkcg(seq_css(sf
));
943 struct bfq_group_data
*bfqgd
= blkcg_to_bfqgd(blkcg
);
944 unsigned int val
= 0;
949 seq_printf(sf
, "%u\n", val
);
954 static u64
bfqg_prfill_weight_device(struct seq_file
*sf
,
955 struct blkg_policy_data
*pd
, int off
)
957 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
959 if (!bfqg
->entity
.dev_weight
)
961 return __blkg_prfill_u64(sf
, pd
, bfqg
->entity
.dev_weight
);
964 static int bfq_io_show_weight(struct seq_file
*sf
, void *v
)
966 struct blkcg
*blkcg
= css_to_blkcg(seq_css(sf
));
967 struct bfq_group_data
*bfqgd
= blkcg_to_bfqgd(blkcg
);
969 seq_printf(sf
, "default %u\n", bfqgd
->weight
);
970 blkcg_print_blkgs(sf
, blkcg
, bfqg_prfill_weight_device
,
971 &blkcg_policy_bfq
, 0, false);
975 static void bfq_group_set_weight(struct bfq_group
*bfqg
, u64 weight
, u64 dev_weight
)
977 weight
= dev_weight
?: weight
;
979 bfqg
->entity
.dev_weight
= dev_weight
;
981 * Setting the prio_changed flag of the entity
982 * to 1 with new_weight == weight would re-set
983 * the value of the weight to its ioprio mapping.
984 * Set the flag only if necessary.
986 if ((unsigned short)weight
!= bfqg
->entity
.new_weight
) {
987 bfqg
->entity
.new_weight
= (unsigned short)weight
;
989 * Make sure that the above new value has been
990 * stored in bfqg->entity.new_weight before
991 * setting the prio_changed flag. In fact,
992 * this flag may be read asynchronously (in
993 * critical sections protected by a different
994 * lock than that held here), and finding this
995 * flag set may cause the execution of the code
996 * for updating parameters whose value may
997 * depend also on bfqg->entity.new_weight (in
998 * __bfq_entity_update_weight_prio).
999 * This barrier makes sure that the new value
1000 * of bfqg->entity.new_weight is correctly
1001 * seen in that code.
1004 bfqg
->entity
.prio_changed
= 1;
1008 static int bfq_io_set_weight_legacy(struct cgroup_subsys_state
*css
,
1009 struct cftype
*cftype
,
1012 struct blkcg
*blkcg
= css_to_blkcg(css
);
1013 struct bfq_group_data
*bfqgd
= blkcg_to_bfqgd(blkcg
);
1014 struct blkcg_gq
*blkg
;
1017 if (val
< BFQ_MIN_WEIGHT
|| val
> BFQ_MAX_WEIGHT
)
1021 spin_lock_irq(&blkcg
->lock
);
1022 bfqgd
->weight
= (unsigned short)val
;
1023 hlist_for_each_entry(blkg
, &blkcg
->blkg_list
, blkcg_node
) {
1024 struct bfq_group
*bfqg
= blkg_to_bfqg(blkg
);
1027 bfq_group_set_weight(bfqg
, val
, 0);
1029 spin_unlock_irq(&blkcg
->lock
);
1034 static ssize_t
bfq_io_set_device_weight(struct kernfs_open_file
*of
,
1035 char *buf
, size_t nbytes
,
1039 struct blkg_conf_ctx ctx
;
1040 struct blkcg
*blkcg
= css_to_blkcg(of_css(of
));
1041 struct bfq_group
*bfqg
;
1044 ret
= blkg_conf_prep(blkcg
, &blkcg_policy_bfq
, buf
, &ctx
);
1048 if (sscanf(ctx
.body
, "%llu", &v
) == 1) {
1049 /* require "default" on dfl */
1053 } else if (!strcmp(strim(ctx
.body
), "default")) {
1060 bfqg
= blkg_to_bfqg(ctx
.blkg
);
1063 if (!v
|| (v
>= BFQ_MIN_WEIGHT
&& v
<= BFQ_MAX_WEIGHT
)) {
1064 bfq_group_set_weight(bfqg
, bfqg
->entity
.weight
, v
);
1068 blkg_conf_finish(&ctx
);
1069 return ret
?: nbytes
;
1072 static ssize_t
bfq_io_set_weight(struct kernfs_open_file
*of
,
1073 char *buf
, size_t nbytes
,
1082 /* "WEIGHT" or "default WEIGHT" sets the default weight */
1083 v
= simple_strtoull(buf
, &endp
, 0);
1084 if (*endp
== '\0' || sscanf(buf
, "default %llu", &v
) == 1) {
1085 ret
= bfq_io_set_weight_legacy(of_css(of
), NULL
, v
);
1086 return ret
?: nbytes
;
1089 return bfq_io_set_device_weight(of
, buf
, nbytes
, off
);
1092 #ifdef CONFIG_BFQ_CGROUP_DEBUG
1093 static int bfqg_print_stat(struct seq_file
*sf
, void *v
)
1095 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)), blkg_prfill_stat
,
1096 &blkcg_policy_bfq
, seq_cft(sf
)->private, false);
1100 static int bfqg_print_rwstat(struct seq_file
*sf
, void *v
)
1102 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)), blkg_prfill_rwstat
,
1103 &blkcg_policy_bfq
, seq_cft(sf
)->private, true);
1107 static u64
bfqg_prfill_stat_recursive(struct seq_file
*sf
,
1108 struct blkg_policy_data
*pd
, int off
)
1110 struct blkcg_gq
*blkg
= pd_to_blkg(pd
);
1111 struct blkcg_gq
*pos_blkg
;
1112 struct cgroup_subsys_state
*pos_css
;
1115 lockdep_assert_held(&blkg
->q
->queue_lock
);
1118 blkg_for_each_descendant_pre(pos_blkg
, pos_css
, blkg
) {
1119 struct bfq_stat
*stat
;
1121 if (!pos_blkg
->online
)
1124 stat
= (void *)blkg_to_pd(pos_blkg
, &blkcg_policy_bfq
) + off
;
1125 sum
+= bfq_stat_read(stat
) + atomic64_read(&stat
->aux_cnt
);
1129 return __blkg_prfill_u64(sf
, pd
, sum
);
1132 static u64
bfqg_prfill_rwstat_recursive(struct seq_file
*sf
,
1133 struct blkg_policy_data
*pd
, int off
)
1135 struct blkg_rwstat_sample sum
;
1137 blkg_rwstat_recursive_sum(pd_to_blkg(pd
), &blkcg_policy_bfq
, off
, &sum
);
1138 return __blkg_prfill_rwstat(sf
, pd
, &sum
);
1141 static int bfqg_print_stat_recursive(struct seq_file
*sf
, void *v
)
1143 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1144 bfqg_prfill_stat_recursive
, &blkcg_policy_bfq
,
1145 seq_cft(sf
)->private, false);
1149 static int bfqg_print_rwstat_recursive(struct seq_file
*sf
, void *v
)
1151 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1152 bfqg_prfill_rwstat_recursive
, &blkcg_policy_bfq
,
1153 seq_cft(sf
)->private, true);
1157 static u64
bfqg_prfill_sectors(struct seq_file
*sf
, struct blkg_policy_data
*pd
,
1160 u64 sum
= blkg_rwstat_total(&pd
->blkg
->stat_bytes
);
1162 return __blkg_prfill_u64(sf
, pd
, sum
>> 9);
1165 static int bfqg_print_stat_sectors(struct seq_file
*sf
, void *v
)
1167 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1168 bfqg_prfill_sectors
, &blkcg_policy_bfq
, 0, false);
1172 static u64
bfqg_prfill_sectors_recursive(struct seq_file
*sf
,
1173 struct blkg_policy_data
*pd
, int off
)
1175 struct blkg_rwstat_sample tmp
;
1177 blkg_rwstat_recursive_sum(pd
->blkg
, NULL
,
1178 offsetof(struct blkcg_gq
, stat_bytes
), &tmp
);
1180 return __blkg_prfill_u64(sf
, pd
,
1181 (tmp
.cnt
[BLKG_RWSTAT_READ
] + tmp
.cnt
[BLKG_RWSTAT_WRITE
]) >> 9);
1184 static int bfqg_print_stat_sectors_recursive(struct seq_file
*sf
, void *v
)
1186 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1187 bfqg_prfill_sectors_recursive
, &blkcg_policy_bfq
, 0,
1192 static u64
bfqg_prfill_avg_queue_size(struct seq_file
*sf
,
1193 struct blkg_policy_data
*pd
, int off
)
1195 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
1196 u64 samples
= bfq_stat_read(&bfqg
->stats
.avg_queue_size_samples
);
1200 v
= bfq_stat_read(&bfqg
->stats
.avg_queue_size_sum
);
1201 v
= div64_u64(v
, samples
);
1203 __blkg_prfill_u64(sf
, pd
, v
);
1207 /* print avg_queue_size */
1208 static int bfqg_print_avg_queue_size(struct seq_file
*sf
, void *v
)
1210 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1211 bfqg_prfill_avg_queue_size
, &blkcg_policy_bfq
,
1215 #endif /* CONFIG_BFQ_CGROUP_DEBUG */
1217 struct bfq_group
*bfq_create_group_hierarchy(struct bfq_data
*bfqd
, int node
)
1221 ret
= blkcg_activate_policy(bfqd
->queue
, &blkcg_policy_bfq
);
1225 return blkg_to_bfqg(bfqd
->queue
->root_blkg
);
1228 struct blkcg_policy blkcg_policy_bfq
= {
1229 .dfl_cftypes
= bfq_blkg_files
,
1230 .legacy_cftypes
= bfq_blkcg_legacy_files
,
1232 .cpd_alloc_fn
= bfq_cpd_alloc
,
1233 .cpd_init_fn
= bfq_cpd_init
,
1234 .cpd_bind_fn
= bfq_cpd_init
,
1235 .cpd_free_fn
= bfq_cpd_free
,
1237 .pd_alloc_fn
= bfq_pd_alloc
,
1238 .pd_init_fn
= bfq_pd_init
,
1239 .pd_offline_fn
= bfq_pd_offline
,
1240 .pd_free_fn
= bfq_pd_free
,
1241 .pd_reset_stats_fn
= bfq_pd_reset_stats
,
1244 struct cftype bfq_blkcg_legacy_files
[] = {
1246 .name
= "bfq.weight",
1247 .flags
= CFTYPE_NOT_ON_ROOT
,
1248 .seq_show
= bfq_io_show_weight_legacy
,
1249 .write_u64
= bfq_io_set_weight_legacy
,
1252 .name
= "bfq.weight_device",
1253 .flags
= CFTYPE_NOT_ON_ROOT
,
1254 .seq_show
= bfq_io_show_weight
,
1255 .write
= bfq_io_set_weight
,
1258 /* statistics, covers only the tasks in the bfqg */
1260 .name
= "bfq.io_service_bytes",
1261 .private = (unsigned long)&blkcg_policy_bfq
,
1262 .seq_show
= blkg_print_stat_bytes
,
1265 .name
= "bfq.io_serviced",
1266 .private = (unsigned long)&blkcg_policy_bfq
,
1267 .seq_show
= blkg_print_stat_ios
,
1269 #ifdef CONFIG_BFQ_CGROUP_DEBUG
1272 .private = offsetof(struct bfq_group
, stats
.time
),
1273 .seq_show
= bfqg_print_stat
,
1276 .name
= "bfq.sectors",
1277 .seq_show
= bfqg_print_stat_sectors
,
1280 .name
= "bfq.io_service_time",
1281 .private = offsetof(struct bfq_group
, stats
.service_time
),
1282 .seq_show
= bfqg_print_rwstat
,
1285 .name
= "bfq.io_wait_time",
1286 .private = offsetof(struct bfq_group
, stats
.wait_time
),
1287 .seq_show
= bfqg_print_rwstat
,
1290 .name
= "bfq.io_merged",
1291 .private = offsetof(struct bfq_group
, stats
.merged
),
1292 .seq_show
= bfqg_print_rwstat
,
1295 .name
= "bfq.io_queued",
1296 .private = offsetof(struct bfq_group
, stats
.queued
),
1297 .seq_show
= bfqg_print_rwstat
,
1299 #endif /* CONFIG_BFQ_CGROUP_DEBUG */
1301 /* the same statistics which cover the bfqg and its descendants */
1303 .name
= "bfq.io_service_bytes_recursive",
1304 .private = (unsigned long)&blkcg_policy_bfq
,
1305 .seq_show
= blkg_print_stat_bytes_recursive
,
1308 .name
= "bfq.io_serviced_recursive",
1309 .private = (unsigned long)&blkcg_policy_bfq
,
1310 .seq_show
= blkg_print_stat_ios_recursive
,
1312 #ifdef CONFIG_BFQ_CGROUP_DEBUG
1314 .name
= "bfq.time_recursive",
1315 .private = offsetof(struct bfq_group
, stats
.time
),
1316 .seq_show
= bfqg_print_stat_recursive
,
1319 .name
= "bfq.sectors_recursive",
1320 .seq_show
= bfqg_print_stat_sectors_recursive
,
1323 .name
= "bfq.io_service_time_recursive",
1324 .private = offsetof(struct bfq_group
, stats
.service_time
),
1325 .seq_show
= bfqg_print_rwstat_recursive
,
1328 .name
= "bfq.io_wait_time_recursive",
1329 .private = offsetof(struct bfq_group
, stats
.wait_time
),
1330 .seq_show
= bfqg_print_rwstat_recursive
,
1333 .name
= "bfq.io_merged_recursive",
1334 .private = offsetof(struct bfq_group
, stats
.merged
),
1335 .seq_show
= bfqg_print_rwstat_recursive
,
1338 .name
= "bfq.io_queued_recursive",
1339 .private = offsetof(struct bfq_group
, stats
.queued
),
1340 .seq_show
= bfqg_print_rwstat_recursive
,
1343 .name
= "bfq.avg_queue_size",
1344 .seq_show
= bfqg_print_avg_queue_size
,
1347 .name
= "bfq.group_wait_time",
1348 .private = offsetof(struct bfq_group
, stats
.group_wait_time
),
1349 .seq_show
= bfqg_print_stat
,
1352 .name
= "bfq.idle_time",
1353 .private = offsetof(struct bfq_group
, stats
.idle_time
),
1354 .seq_show
= bfqg_print_stat
,
1357 .name
= "bfq.empty_time",
1358 .private = offsetof(struct bfq_group
, stats
.empty_time
),
1359 .seq_show
= bfqg_print_stat
,
1362 .name
= "bfq.dequeue",
1363 .private = offsetof(struct bfq_group
, stats
.dequeue
),
1364 .seq_show
= bfqg_print_stat
,
1366 #endif /* CONFIG_BFQ_CGROUP_DEBUG */
1370 struct cftype bfq_blkg_files
[] = {
1372 .name
= "bfq.weight",
1373 .flags
= CFTYPE_NOT_ON_ROOT
,
1374 .seq_show
= bfq_io_show_weight
,
1375 .write
= bfq_io_set_weight
,
1380 #else /* CONFIG_BFQ_GROUP_IOSCHED */
1382 void bfq_bfqq_move(struct bfq_data
*bfqd
, struct bfq_queue
*bfqq
,
1383 struct bfq_group
*bfqg
) {}
1385 void bfq_init_entity(struct bfq_entity
*entity
, struct bfq_group
*bfqg
)
1387 struct bfq_queue
*bfqq
= bfq_entity_to_bfqq(entity
);
1389 entity
->weight
= entity
->new_weight
;
1390 entity
->orig_weight
= entity
->new_weight
;
1392 bfqq
->ioprio
= bfqq
->new_ioprio
;
1393 bfqq
->ioprio_class
= bfqq
->new_ioprio_class
;
1395 entity
->sched_data
= &bfqg
->sched_data
;
1398 void bfq_bic_update_cgroup(struct bfq_io_cq
*bic
, struct bio
*bio
) {}
1400 void bfq_end_wr_async(struct bfq_data
*bfqd
)
1402 bfq_end_wr_async_queues(bfqd
, bfqd
->root_group
);
1405 struct bfq_group
*bfq_find_set_group(struct bfq_data
*bfqd
, struct blkcg
*blkcg
)
1407 return bfqd
->root_group
;
1410 struct bfq_group
*bfqq_group(struct bfq_queue
*bfqq
)
1412 return bfqq
->bfqd
->root_group
;
1415 void bfqg_and_blkg_get(struct bfq_group
*bfqg
) {}
1417 void bfqg_and_blkg_put(struct bfq_group
*bfqg
) {}
1419 struct bfq_group
*bfq_create_group_hierarchy(struct bfq_data
*bfqd
, int node
)
1421 struct bfq_group
*bfqg
;
1424 bfqg
= kmalloc_node(sizeof(*bfqg
), GFP_KERNEL
| __GFP_ZERO
, node
);
1428 for (i
= 0; i
< BFQ_IOPRIO_CLASSES
; i
++)
1429 bfqg
->sched_data
.service_tree
[i
] = BFQ_SERVICE_TREE_INIT
;
1433 #endif /* CONFIG_BFQ_GROUP_IOSCHED */