1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * cgroups support for the BFQ I/O scheduler.
5 #include <linux/module.h>
6 #include <linux/slab.h>
7 #include <linux/blkdev.h>
8 #include <linux/cgroup.h>
9 #include <linux/elevator.h>
10 #include <linux/ktime.h>
11 #include <linux/rbtree.h>
12 #include <linux/ioprio.h>
13 #include <linux/sbitmap.h>
14 #include <linux/delay.h>
16 #include "bfq-iosched.h"
18 #ifdef CONFIG_BFQ_CGROUP_DEBUG
19 static int bfq_stat_init(struct bfq_stat
*stat
, gfp_t gfp
)
23 ret
= percpu_counter_init(&stat
->cpu_cnt
, 0, gfp
);
27 atomic64_set(&stat
->aux_cnt
, 0);
31 static void bfq_stat_exit(struct bfq_stat
*stat
)
33 percpu_counter_destroy(&stat
->cpu_cnt
);
37 * bfq_stat_add - add a value to a bfq_stat
38 * @stat: target bfq_stat
41 * Add @val to @stat. The caller must ensure that IRQ on the same CPU
42 * don't re-enter this function for the same counter.
44 static inline void bfq_stat_add(struct bfq_stat
*stat
, uint64_t val
)
46 percpu_counter_add_batch(&stat
->cpu_cnt
, val
, BLKG_STAT_CPU_BATCH
);
50 * bfq_stat_read - read the current value of a bfq_stat
51 * @stat: bfq_stat to read
53 static inline uint64_t bfq_stat_read(struct bfq_stat
*stat
)
55 return percpu_counter_sum_positive(&stat
->cpu_cnt
);
59 * bfq_stat_reset - reset a bfq_stat
60 * @stat: bfq_stat to reset
62 static inline void bfq_stat_reset(struct bfq_stat
*stat
)
64 percpu_counter_set(&stat
->cpu_cnt
, 0);
65 atomic64_set(&stat
->aux_cnt
, 0);
69 * bfq_stat_add_aux - add a bfq_stat into another's aux count
70 * @to: the destination bfq_stat
73 * Add @from's count including the aux one to @to's aux count.
75 static inline void bfq_stat_add_aux(struct bfq_stat
*to
,
76 struct bfq_stat
*from
)
78 atomic64_add(bfq_stat_read(from
) + atomic64_read(&from
->aux_cnt
),
83 * blkg_prfill_stat - prfill callback for bfq_stat
84 * @sf: seq_file to print to
85 * @pd: policy private data of interest
86 * @off: offset to the bfq_stat in @pd
88 * prfill callback for printing a bfq_stat.
90 static u64
blkg_prfill_stat(struct seq_file
*sf
, struct blkg_policy_data
*pd
,
93 return __blkg_prfill_u64(sf
, pd
, bfq_stat_read((void *)pd
+ off
));
96 /* bfqg stats flags */
97 enum bfqg_stats_flags
{
98 BFQG_stats_waiting
= 0,
103 #define BFQG_FLAG_FNS(name) \
104 static void bfqg_stats_mark_##name(struct bfqg_stats *stats) \
106 stats->flags |= (1 << BFQG_stats_##name); \
108 static void bfqg_stats_clear_##name(struct bfqg_stats *stats) \
110 stats->flags &= ~(1 << BFQG_stats_##name); \
112 static int bfqg_stats_##name(struct bfqg_stats *stats) \
114 return (stats->flags & (1 << BFQG_stats_##name)) != 0; \
117 BFQG_FLAG_FNS(waiting)
118 BFQG_FLAG_FNS(idling
)
122 /* This should be called with the scheduler lock held. */
123 static void bfqg_stats_update_group_wait_time(struct bfqg_stats
*stats
)
127 if (!bfqg_stats_waiting(stats
))
130 now
= ktime_get_ns();
131 if (now
> stats
->start_group_wait_time
)
132 bfq_stat_add(&stats
->group_wait_time
,
133 now
- stats
->start_group_wait_time
);
134 bfqg_stats_clear_waiting(stats
);
137 /* This should be called with the scheduler lock held. */
138 static void bfqg_stats_set_start_group_wait_time(struct bfq_group
*bfqg
,
139 struct bfq_group
*curr_bfqg
)
141 struct bfqg_stats
*stats
= &bfqg
->stats
;
143 if (bfqg_stats_waiting(stats
))
145 if (bfqg
== curr_bfqg
)
147 stats
->start_group_wait_time
= ktime_get_ns();
148 bfqg_stats_mark_waiting(stats
);
151 /* This should be called with the scheduler lock held. */
152 static void bfqg_stats_end_empty_time(struct bfqg_stats
*stats
)
156 if (!bfqg_stats_empty(stats
))
159 now
= ktime_get_ns();
160 if (now
> stats
->start_empty_time
)
161 bfq_stat_add(&stats
->empty_time
,
162 now
- stats
->start_empty_time
);
163 bfqg_stats_clear_empty(stats
);
166 void bfqg_stats_update_dequeue(struct bfq_group
*bfqg
)
168 bfq_stat_add(&bfqg
->stats
.dequeue
, 1);
171 void bfqg_stats_set_start_empty_time(struct bfq_group
*bfqg
)
173 struct bfqg_stats
*stats
= &bfqg
->stats
;
175 if (blkg_rwstat_total(&stats
->queued
))
179 * group is already marked empty. This can happen if bfqq got new
180 * request in parent group and moved to this group while being added
181 * to service tree. Just ignore the event and move on.
183 if (bfqg_stats_empty(stats
))
186 stats
->start_empty_time
= ktime_get_ns();
187 bfqg_stats_mark_empty(stats
);
190 void bfqg_stats_update_idle_time(struct bfq_group
*bfqg
)
192 struct bfqg_stats
*stats
= &bfqg
->stats
;
194 if (bfqg_stats_idling(stats
)) {
195 u64 now
= ktime_get_ns();
197 if (now
> stats
->start_idle_time
)
198 bfq_stat_add(&stats
->idle_time
,
199 now
- stats
->start_idle_time
);
200 bfqg_stats_clear_idling(stats
);
204 void bfqg_stats_set_start_idle_time(struct bfq_group
*bfqg
)
206 struct bfqg_stats
*stats
= &bfqg
->stats
;
208 stats
->start_idle_time
= ktime_get_ns();
209 bfqg_stats_mark_idling(stats
);
212 void bfqg_stats_update_avg_queue_size(struct bfq_group
*bfqg
)
214 struct bfqg_stats
*stats
= &bfqg
->stats
;
216 bfq_stat_add(&stats
->avg_queue_size_sum
,
217 blkg_rwstat_total(&stats
->queued
));
218 bfq_stat_add(&stats
->avg_queue_size_samples
, 1);
219 bfqg_stats_update_group_wait_time(stats
);
222 void bfqg_stats_update_io_add(struct bfq_group
*bfqg
, struct bfq_queue
*bfqq
,
225 blkg_rwstat_add(&bfqg
->stats
.queued
, op
, 1);
226 bfqg_stats_end_empty_time(&bfqg
->stats
);
227 if (!(bfqq
== ((struct bfq_data
*)bfqg
->bfqd
)->in_service_queue
))
228 bfqg_stats_set_start_group_wait_time(bfqg
, bfqq_group(bfqq
));
231 void bfqg_stats_update_io_remove(struct bfq_group
*bfqg
, unsigned int op
)
233 blkg_rwstat_add(&bfqg
->stats
.queued
, op
, -1);
236 void bfqg_stats_update_io_merged(struct bfq_group
*bfqg
, unsigned int op
)
238 blkg_rwstat_add(&bfqg
->stats
.merged
, op
, 1);
241 void bfqg_stats_update_completion(struct bfq_group
*bfqg
, u64 start_time_ns
,
242 u64 io_start_time_ns
, unsigned int op
)
244 struct bfqg_stats
*stats
= &bfqg
->stats
;
245 u64 now
= ktime_get_ns();
247 if (now
> io_start_time_ns
)
248 blkg_rwstat_add(&stats
->service_time
, op
,
249 now
- io_start_time_ns
);
250 if (io_start_time_ns
> start_time_ns
)
251 blkg_rwstat_add(&stats
->wait_time
, op
,
252 io_start_time_ns
- start_time_ns
);
255 #else /* CONFIG_BFQ_CGROUP_DEBUG */
257 void bfqg_stats_update_io_add(struct bfq_group
*bfqg
, struct bfq_queue
*bfqq
,
259 void bfqg_stats_update_io_remove(struct bfq_group
*bfqg
, unsigned int op
) { }
260 void bfqg_stats_update_io_merged(struct bfq_group
*bfqg
, unsigned int op
) { }
261 void bfqg_stats_update_completion(struct bfq_group
*bfqg
, u64 start_time_ns
,
262 u64 io_start_time_ns
, unsigned int op
) { }
263 void bfqg_stats_update_dequeue(struct bfq_group
*bfqg
) { }
264 void bfqg_stats_set_start_empty_time(struct bfq_group
*bfqg
) { }
265 void bfqg_stats_update_idle_time(struct bfq_group
*bfqg
) { }
266 void bfqg_stats_set_start_idle_time(struct bfq_group
*bfqg
) { }
267 void bfqg_stats_update_avg_queue_size(struct bfq_group
*bfqg
) { }
269 #endif /* CONFIG_BFQ_CGROUP_DEBUG */
271 #ifdef CONFIG_BFQ_GROUP_IOSCHED
274 * blk-cgroup policy-related handlers
275 * The following functions help in converting between blk-cgroup
276 * internal structures and BFQ-specific structures.
279 static struct bfq_group
*pd_to_bfqg(struct blkg_policy_data
*pd
)
281 return pd
? container_of(pd
, struct bfq_group
, pd
) : NULL
;
284 struct blkcg_gq
*bfqg_to_blkg(struct bfq_group
*bfqg
)
286 return pd_to_blkg(&bfqg
->pd
);
289 static struct bfq_group
*blkg_to_bfqg(struct blkcg_gq
*blkg
)
291 return pd_to_bfqg(blkg_to_pd(blkg
, &blkcg_policy_bfq
));
296 * The following functions help in navigating the bfq_group hierarchy
297 * by allowing to find the parent of a bfq_group or the bfq_group
298 * associated to a bfq_queue.
301 static struct bfq_group
*bfqg_parent(struct bfq_group
*bfqg
)
303 struct blkcg_gq
*pblkg
= bfqg_to_blkg(bfqg
)->parent
;
305 return pblkg
? blkg_to_bfqg(pblkg
) : NULL
;
308 struct bfq_group
*bfqq_group(struct bfq_queue
*bfqq
)
310 struct bfq_entity
*group_entity
= bfqq
->entity
.parent
;
312 return group_entity
? container_of(group_entity
, struct bfq_group
,
314 bfqq
->bfqd
->root_group
;
318 * The following two functions handle get and put of a bfq_group by
319 * wrapping the related blk-cgroup hooks.
322 static void bfqg_get(struct bfq_group
*bfqg
)
327 static void bfqg_put(struct bfq_group
*bfqg
)
335 static void bfqg_and_blkg_get(struct bfq_group
*bfqg
)
337 /* see comments in bfq_bic_update_cgroup for why refcounting bfqg */
340 blkg_get(bfqg_to_blkg(bfqg
));
343 void bfqg_and_blkg_put(struct bfq_group
*bfqg
)
345 blkg_put(bfqg_to_blkg(bfqg
));
350 void bfqg_stats_update_legacy_io(struct request_queue
*q
, struct request
*rq
)
352 struct bfq_group
*bfqg
= blkg_to_bfqg(rq
->bio
->bi_blkg
);
357 blkg_rwstat_add(&bfqg
->stats
.bytes
, rq
->cmd_flags
, blk_rq_bytes(rq
));
358 blkg_rwstat_add(&bfqg
->stats
.ios
, rq
->cmd_flags
, 1);
362 static void bfqg_stats_reset(struct bfqg_stats
*stats
)
364 #ifdef CONFIG_BFQ_CGROUP_DEBUG
365 /* queued stats shouldn't be cleared */
366 blkg_rwstat_reset(&stats
->merged
);
367 blkg_rwstat_reset(&stats
->service_time
);
368 blkg_rwstat_reset(&stats
->wait_time
);
369 bfq_stat_reset(&stats
->time
);
370 bfq_stat_reset(&stats
->avg_queue_size_sum
);
371 bfq_stat_reset(&stats
->avg_queue_size_samples
);
372 bfq_stat_reset(&stats
->dequeue
);
373 bfq_stat_reset(&stats
->group_wait_time
);
374 bfq_stat_reset(&stats
->idle_time
);
375 bfq_stat_reset(&stats
->empty_time
);
380 static void bfqg_stats_add_aux(struct bfqg_stats
*to
, struct bfqg_stats
*from
)
385 #ifdef CONFIG_BFQ_CGROUP_DEBUG
386 /* queued stats shouldn't be cleared */
387 blkg_rwstat_add_aux(&to
->merged
, &from
->merged
);
388 blkg_rwstat_add_aux(&to
->service_time
, &from
->service_time
);
389 blkg_rwstat_add_aux(&to
->wait_time
, &from
->wait_time
);
390 bfq_stat_add_aux(&from
->time
, &from
->time
);
391 bfq_stat_add_aux(&to
->avg_queue_size_sum
, &from
->avg_queue_size_sum
);
392 bfq_stat_add_aux(&to
->avg_queue_size_samples
,
393 &from
->avg_queue_size_samples
);
394 bfq_stat_add_aux(&to
->dequeue
, &from
->dequeue
);
395 bfq_stat_add_aux(&to
->group_wait_time
, &from
->group_wait_time
);
396 bfq_stat_add_aux(&to
->idle_time
, &from
->idle_time
);
397 bfq_stat_add_aux(&to
->empty_time
, &from
->empty_time
);
402 * Transfer @bfqg's stats to its parent's aux counts so that the ancestors'
403 * recursive stats can still account for the amount used by this bfqg after
406 static void bfqg_stats_xfer_dead(struct bfq_group
*bfqg
)
408 struct bfq_group
*parent
;
410 if (!bfqg
) /* root_group */
413 parent
= bfqg_parent(bfqg
);
415 lockdep_assert_held(&bfqg_to_blkg(bfqg
)->q
->queue_lock
);
417 if (unlikely(!parent
))
420 bfqg_stats_add_aux(&parent
->stats
, &bfqg
->stats
);
421 bfqg_stats_reset(&bfqg
->stats
);
424 void bfq_init_entity(struct bfq_entity
*entity
, struct bfq_group
*bfqg
)
426 struct bfq_queue
*bfqq
= bfq_entity_to_bfqq(entity
);
428 entity
->weight
= entity
->new_weight
;
429 entity
->orig_weight
= entity
->new_weight
;
431 bfqq
->ioprio
= bfqq
->new_ioprio
;
432 bfqq
->ioprio_class
= bfqq
->new_ioprio_class
;
434 * Make sure that bfqg and its associated blkg do not
435 * disappear before entity.
437 bfqg_and_blkg_get(bfqg
);
439 entity
->parent
= bfqg
->my_entity
; /* NULL for root group */
440 entity
->sched_data
= &bfqg
->sched_data
;
443 static void bfqg_stats_exit(struct bfqg_stats
*stats
)
445 blkg_rwstat_exit(&stats
->bytes
);
446 blkg_rwstat_exit(&stats
->ios
);
447 #ifdef CONFIG_BFQ_CGROUP_DEBUG
448 blkg_rwstat_exit(&stats
->merged
);
449 blkg_rwstat_exit(&stats
->service_time
);
450 blkg_rwstat_exit(&stats
->wait_time
);
451 blkg_rwstat_exit(&stats
->queued
);
452 bfq_stat_exit(&stats
->time
);
453 bfq_stat_exit(&stats
->avg_queue_size_sum
);
454 bfq_stat_exit(&stats
->avg_queue_size_samples
);
455 bfq_stat_exit(&stats
->dequeue
);
456 bfq_stat_exit(&stats
->group_wait_time
);
457 bfq_stat_exit(&stats
->idle_time
);
458 bfq_stat_exit(&stats
->empty_time
);
462 static int bfqg_stats_init(struct bfqg_stats
*stats
, gfp_t gfp
)
464 if (blkg_rwstat_init(&stats
->bytes
, gfp
) ||
465 blkg_rwstat_init(&stats
->ios
, gfp
))
468 #ifdef CONFIG_BFQ_CGROUP_DEBUG
469 if (blkg_rwstat_init(&stats
->merged
, gfp
) ||
470 blkg_rwstat_init(&stats
->service_time
, gfp
) ||
471 blkg_rwstat_init(&stats
->wait_time
, gfp
) ||
472 blkg_rwstat_init(&stats
->queued
, gfp
) ||
473 bfq_stat_init(&stats
->time
, gfp
) ||
474 bfq_stat_init(&stats
->avg_queue_size_sum
, gfp
) ||
475 bfq_stat_init(&stats
->avg_queue_size_samples
, gfp
) ||
476 bfq_stat_init(&stats
->dequeue
, gfp
) ||
477 bfq_stat_init(&stats
->group_wait_time
, gfp
) ||
478 bfq_stat_init(&stats
->idle_time
, gfp
) ||
479 bfq_stat_init(&stats
->empty_time
, gfp
)) {
480 bfqg_stats_exit(stats
);
488 static struct bfq_group_data
*cpd_to_bfqgd(struct blkcg_policy_data
*cpd
)
490 return cpd
? container_of(cpd
, struct bfq_group_data
, pd
) : NULL
;
493 static struct bfq_group_data
*blkcg_to_bfqgd(struct blkcg
*blkcg
)
495 return cpd_to_bfqgd(blkcg_to_cpd(blkcg
, &blkcg_policy_bfq
));
498 static struct blkcg_policy_data
*bfq_cpd_alloc(gfp_t gfp
)
500 struct bfq_group_data
*bgd
;
502 bgd
= kzalloc(sizeof(*bgd
), gfp
);
508 static void bfq_cpd_init(struct blkcg_policy_data
*cpd
)
510 struct bfq_group_data
*d
= cpd_to_bfqgd(cpd
);
512 d
->weight
= cgroup_subsys_on_dfl(io_cgrp_subsys
) ?
513 CGROUP_WEIGHT_DFL
: BFQ_WEIGHT_LEGACY_DFL
;
516 static void bfq_cpd_free(struct blkcg_policy_data
*cpd
)
518 kfree(cpd_to_bfqgd(cpd
));
521 static struct blkg_policy_data
*bfq_pd_alloc(gfp_t gfp
, struct request_queue
*q
,
524 struct bfq_group
*bfqg
;
526 bfqg
= kzalloc_node(sizeof(*bfqg
), gfp
, q
->node
);
530 if (bfqg_stats_init(&bfqg
->stats
, gfp
)) {
535 /* see comments in bfq_bic_update_cgroup for why refcounting */
540 static void bfq_pd_init(struct blkg_policy_data
*pd
)
542 struct blkcg_gq
*blkg
= pd_to_blkg(pd
);
543 struct bfq_group
*bfqg
= blkg_to_bfqg(blkg
);
544 struct bfq_data
*bfqd
= blkg
->q
->elevator
->elevator_data
;
545 struct bfq_entity
*entity
= &bfqg
->entity
;
546 struct bfq_group_data
*d
= blkcg_to_bfqgd(blkg
->blkcg
);
548 entity
->orig_weight
= entity
->weight
= entity
->new_weight
= d
->weight
;
549 entity
->my_sched_data
= &bfqg
->sched_data
;
550 entity
->last_bfqq_created
= NULL
;
552 bfqg
->my_entity
= entity
; /*
553 * the root_group's will be set to NULL
554 * in bfq_init_queue()
557 bfqg
->active_entities
= 0;
558 bfqg
->rq_pos_tree
= RB_ROOT
;
561 static void bfq_pd_free(struct blkg_policy_data
*pd
)
563 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
565 bfqg_stats_exit(&bfqg
->stats
);
569 static void bfq_pd_reset_stats(struct blkg_policy_data
*pd
)
571 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
573 bfqg_stats_reset(&bfqg
->stats
);
576 static void bfq_group_set_parent(struct bfq_group
*bfqg
,
577 struct bfq_group
*parent
)
579 struct bfq_entity
*entity
;
581 entity
= &bfqg
->entity
;
582 entity
->parent
= parent
->my_entity
;
583 entity
->sched_data
= &parent
->sched_data
;
586 static struct bfq_group
*bfq_lookup_bfqg(struct bfq_data
*bfqd
,
589 struct blkcg_gq
*blkg
;
591 blkg
= blkg_lookup(blkcg
, bfqd
->queue
);
593 return blkg_to_bfqg(blkg
);
597 struct bfq_group
*bfq_find_set_group(struct bfq_data
*bfqd
,
600 struct bfq_group
*bfqg
, *parent
;
601 struct bfq_entity
*entity
;
603 bfqg
= bfq_lookup_bfqg(bfqd
, blkcg
);
609 * Update chain of bfq_groups as we might be handling a leaf group
610 * which, along with some of its relatives, has not been hooked yet
611 * to the private hierarchy of BFQ.
613 entity
= &bfqg
->entity
;
614 for_each_entity(entity
) {
615 struct bfq_group
*curr_bfqg
= container_of(entity
,
616 struct bfq_group
, entity
);
617 if (curr_bfqg
!= bfqd
->root_group
) {
618 parent
= bfqg_parent(curr_bfqg
);
620 parent
= bfqd
->root_group
;
621 bfq_group_set_parent(curr_bfqg
, parent
);
629 * bfq_bfqq_move - migrate @bfqq to @bfqg.
630 * @bfqd: queue descriptor.
631 * @bfqq: the queue to move.
632 * @bfqg: the group to move to.
634 * Move @bfqq to @bfqg, deactivating it from its old group and reactivating
635 * it on the new one. Avoid putting the entity on the old group idle tree.
637 * Must be called under the scheduler lock, to make sure that the blkg
638 * owning @bfqg does not disappear (see comments in
639 * bfq_bic_update_cgroup on guaranteeing the consistency of blkg
642 void bfq_bfqq_move(struct bfq_data
*bfqd
, struct bfq_queue
*bfqq
,
643 struct bfq_group
*bfqg
)
645 struct bfq_entity
*entity
= &bfqq
->entity
;
648 * Get extra reference to prevent bfqq from being freed in
649 * next possible expire or deactivate.
653 /* If bfqq is empty, then bfq_bfqq_expire also invokes
654 * bfq_del_bfqq_busy, thereby removing bfqq and its entity
655 * from data structures related to current group. Otherwise we
656 * need to remove bfqq explicitly with bfq_deactivate_bfqq, as
659 if (bfqq
== bfqd
->in_service_queue
)
660 bfq_bfqq_expire(bfqd
, bfqd
->in_service_queue
,
661 false, BFQQE_PREEMPTED
);
663 if (bfq_bfqq_busy(bfqq
))
664 bfq_deactivate_bfqq(bfqd
, bfqq
, false, false);
665 else if (entity
->on_st_or_in_serv
)
666 bfq_put_idle_entity(bfq_entity_service_tree(entity
), entity
);
667 bfqg_and_blkg_put(bfqq_group(bfqq
));
669 entity
->parent
= bfqg
->my_entity
;
670 entity
->sched_data
= &bfqg
->sched_data
;
671 /* pin down bfqg and its associated blkg */
672 bfqg_and_blkg_get(bfqg
);
674 if (bfq_bfqq_busy(bfqq
)) {
675 if (unlikely(!bfqd
->nonrot_with_queueing
))
676 bfq_pos_tree_add_move(bfqd
, bfqq
);
677 bfq_activate_bfqq(bfqd
, bfqq
);
680 if (!bfqd
->in_service_queue
&& !bfqd
->rq_in_driver
)
681 bfq_schedule_dispatch(bfqd
);
682 /* release extra ref taken above, bfqq may happen to be freed now */
687 * __bfq_bic_change_cgroup - move @bic to @cgroup.
688 * @bfqd: the queue descriptor.
689 * @bic: the bic to move.
690 * @blkcg: the blk-cgroup to move to.
692 * Move bic to blkcg, assuming that bfqd->lock is held; which makes
693 * sure that the reference to cgroup is valid across the call (see
694 * comments in bfq_bic_update_cgroup on this issue)
696 * NOTE: an alternative approach might have been to store the current
697 * cgroup in bfqq and getting a reference to it, reducing the lookup
698 * time here, at the price of slightly more complex code.
700 static struct bfq_group
*__bfq_bic_change_cgroup(struct bfq_data
*bfqd
,
701 struct bfq_io_cq
*bic
,
704 struct bfq_queue
*async_bfqq
= bic_to_bfqq(bic
, 0);
705 struct bfq_queue
*sync_bfqq
= bic_to_bfqq(bic
, 1);
706 struct bfq_group
*bfqg
;
707 struct bfq_entity
*entity
;
709 bfqg
= bfq_find_set_group(bfqd
, blkcg
);
712 bfqg
= bfqd
->root_group
;
715 entity
= &async_bfqq
->entity
;
717 if (entity
->sched_data
!= &bfqg
->sched_data
) {
718 bic_set_bfqq(bic
, NULL
, 0);
719 bfq_release_process_ref(bfqd
, async_bfqq
);
724 entity
= &sync_bfqq
->entity
;
725 if (entity
->sched_data
!= &bfqg
->sched_data
)
726 bfq_bfqq_move(bfqd
, sync_bfqq
, bfqg
);
732 void bfq_bic_update_cgroup(struct bfq_io_cq
*bic
, struct bio
*bio
)
734 struct bfq_data
*bfqd
= bic_to_bfqd(bic
);
735 struct bfq_group
*bfqg
= NULL
;
739 serial_nr
= __bio_blkcg(bio
)->css
.serial_nr
;
742 * Check whether blkcg has changed. The condition may trigger
743 * spuriously on a newly created cic but there's no harm.
745 if (unlikely(!bfqd
) || likely(bic
->blkcg_serial_nr
== serial_nr
))
748 bfqg
= __bfq_bic_change_cgroup(bfqd
, bic
, __bio_blkcg(bio
));
750 * Update blkg_path for bfq_log_* functions. We cache this
751 * path, and update it here, for the following
752 * reasons. Operations on blkg objects in blk-cgroup are
753 * protected with the request_queue lock, and not with the
754 * lock that protects the instances of this scheduler
755 * (bfqd->lock). This exposes BFQ to the following sort of
758 * The blkg_lookup performed in bfq_get_queue, protected
759 * through rcu, may happen to return the address of a copy of
760 * the original blkg. If this is the case, then the
761 * bfqg_and_blkg_get performed in bfq_get_queue, to pin down
762 * the blkg, is useless: it does not prevent blk-cgroup code
763 * from destroying both the original blkg and all objects
764 * directly or indirectly referred by the copy of the
767 * On the bright side, destroy operations on a blkg invoke, as
768 * a first step, hooks of the scheduler associated with the
769 * blkg. And these hooks are executed with bfqd->lock held for
770 * BFQ. As a consequence, for any blkg associated with the
771 * request queue this instance of the scheduler is attached
772 * to, we are guaranteed that such a blkg is not destroyed, and
773 * that all the pointers it contains are consistent, while we
774 * are holding bfqd->lock. A blkg_lookup performed with
775 * bfqd->lock held then returns a fully consistent blkg, which
776 * remains consistent until this lock is held.
778 * Thanks to the last fact, and to the fact that: (1) bfqg has
779 * been obtained through a blkg_lookup in the above
780 * assignment, and (2) bfqd->lock is being held, here we can
781 * safely use the policy data for the involved blkg (i.e., the
782 * field bfqg->pd) to get to the blkg associated with bfqg,
783 * and then we can safely use any field of blkg. After we
784 * release bfqd->lock, even just getting blkg through this
785 * bfqg may cause dangling references to be traversed, as
786 * bfqg->pd may not exist any more.
788 * In view of the above facts, here we cache, in the bfqg, any
789 * blkg data we may need for this bic, and for its associated
790 * bfq_queue. As of now, we need to cache only the path of the
791 * blkg, which is used in the bfq_log_* functions.
793 * Finally, note that bfqg itself needs to be protected from
794 * destruction on the blkg_free of the original blkg (which
795 * invokes bfq_pd_free). We use an additional private
796 * refcounter for bfqg, to let it disappear only after no
797 * bfq_queue refers to it any longer.
799 blkg_path(bfqg_to_blkg(bfqg
), bfqg
->blkg_path
, sizeof(bfqg
->blkg_path
));
800 bic
->blkcg_serial_nr
= serial_nr
;
806 * bfq_flush_idle_tree - deactivate any entity on the idle tree of @st.
807 * @st: the service tree being flushed.
809 static void bfq_flush_idle_tree(struct bfq_service_tree
*st
)
811 struct bfq_entity
*entity
= st
->first_idle
;
813 for (; entity
; entity
= st
->first_idle
)
814 __bfq_deactivate_entity(entity
, false);
818 * bfq_reparent_leaf_entity - move leaf entity to the root_group.
819 * @bfqd: the device data structure with the root group.
820 * @entity: the entity to move, if entity is a leaf; or the parent entity
821 * of an active leaf entity to move, if entity is not a leaf.
823 static void bfq_reparent_leaf_entity(struct bfq_data
*bfqd
,
824 struct bfq_entity
*entity
,
827 struct bfq_queue
*bfqq
;
828 struct bfq_entity
*child_entity
= entity
;
830 while (child_entity
->my_sched_data
) { /* leaf not reached yet */
831 struct bfq_sched_data
*child_sd
= child_entity
->my_sched_data
;
832 struct bfq_service_tree
*child_st
= child_sd
->service_tree
+
834 struct rb_root
*child_active
= &child_st
->active
;
836 child_entity
= bfq_entity_of(rb_first(child_active
));
839 child_entity
= child_sd
->in_service_entity
;
842 bfqq
= bfq_entity_to_bfqq(child_entity
);
843 bfq_bfqq_move(bfqd
, bfqq
, bfqd
->root_group
);
847 * bfq_reparent_active_queues - move to the root group all active queues.
848 * @bfqd: the device data structure with the root group.
849 * @bfqg: the group to move from.
850 * @st: the service tree to start the search from.
852 static void bfq_reparent_active_queues(struct bfq_data
*bfqd
,
853 struct bfq_group
*bfqg
,
854 struct bfq_service_tree
*st
,
857 struct rb_root
*active
= &st
->active
;
858 struct bfq_entity
*entity
;
860 while ((entity
= bfq_entity_of(rb_first(active
))))
861 bfq_reparent_leaf_entity(bfqd
, entity
, ioprio_class
);
863 if (bfqg
->sched_data
.in_service_entity
)
864 bfq_reparent_leaf_entity(bfqd
,
865 bfqg
->sched_data
.in_service_entity
,
870 * bfq_pd_offline - deactivate the entity associated with @pd,
871 * and reparent its children entities.
872 * @pd: descriptor of the policy going offline.
874 * blkio already grabs the queue_lock for us, so no need to use
877 static void bfq_pd_offline(struct blkg_policy_data
*pd
)
879 struct bfq_service_tree
*st
;
880 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
881 struct bfq_data
*bfqd
= bfqg
->bfqd
;
882 struct bfq_entity
*entity
= bfqg
->my_entity
;
886 spin_lock_irqsave(&bfqd
->lock
, flags
);
888 if (!entity
) /* root group */
889 goto put_async_queues
;
892 * Empty all service_trees belonging to this group before
893 * deactivating the group itself.
895 for (i
= 0; i
< BFQ_IOPRIO_CLASSES
; i
++) {
896 st
= bfqg
->sched_data
.service_tree
+ i
;
899 * It may happen that some queues are still active
900 * (busy) upon group destruction (if the corresponding
901 * processes have been forced to terminate). We move
902 * all the leaf entities corresponding to these queues
904 * Also, it may happen that the group has an entity
905 * in service, which is disconnected from the active
906 * tree: it must be moved, too.
907 * There is no need to put the sync queues, as the
908 * scheduler has taken no reference.
910 bfq_reparent_active_queues(bfqd
, bfqg
, st
, i
);
913 * The idle tree may still contain bfq_queues
914 * belonging to exited task because they never
915 * migrated to a different cgroup from the one being
916 * destroyed now. In addition, even
917 * bfq_reparent_active_queues() may happen to add some
918 * entities to the idle tree. It happens if, in some
919 * of the calls to bfq_bfqq_move() performed by
920 * bfq_reparent_active_queues(), the queue to move is
921 * empty and gets expired.
923 bfq_flush_idle_tree(st
);
926 __bfq_deactivate_entity(entity
, false);
929 bfq_put_async_queues(bfqd
, bfqg
);
931 spin_unlock_irqrestore(&bfqd
->lock
, flags
);
933 * @blkg is going offline and will be ignored by
934 * blkg_[rw]stat_recursive_sum(). Transfer stats to the parent so
935 * that they don't get lost. If IOs complete after this point, the
936 * stats for them will be lost. Oh well...
938 bfqg_stats_xfer_dead(bfqg
);
941 void bfq_end_wr_async(struct bfq_data
*bfqd
)
943 struct blkcg_gq
*blkg
;
945 list_for_each_entry(blkg
, &bfqd
->queue
->blkg_list
, q_node
) {
946 struct bfq_group
*bfqg
= blkg_to_bfqg(blkg
);
948 bfq_end_wr_async_queues(bfqd
, bfqg
);
950 bfq_end_wr_async_queues(bfqd
, bfqd
->root_group
);
953 static int bfq_io_show_weight_legacy(struct seq_file
*sf
, void *v
)
955 struct blkcg
*blkcg
= css_to_blkcg(seq_css(sf
));
956 struct bfq_group_data
*bfqgd
= blkcg_to_bfqgd(blkcg
);
957 unsigned int val
= 0;
962 seq_printf(sf
, "%u\n", val
);
967 static u64
bfqg_prfill_weight_device(struct seq_file
*sf
,
968 struct blkg_policy_data
*pd
, int off
)
970 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
972 if (!bfqg
->entity
.dev_weight
)
974 return __blkg_prfill_u64(sf
, pd
, bfqg
->entity
.dev_weight
);
977 static int bfq_io_show_weight(struct seq_file
*sf
, void *v
)
979 struct blkcg
*blkcg
= css_to_blkcg(seq_css(sf
));
980 struct bfq_group_data
*bfqgd
= blkcg_to_bfqgd(blkcg
);
982 seq_printf(sf
, "default %u\n", bfqgd
->weight
);
983 blkcg_print_blkgs(sf
, blkcg
, bfqg_prfill_weight_device
,
984 &blkcg_policy_bfq
, 0, false);
988 static void bfq_group_set_weight(struct bfq_group
*bfqg
, u64 weight
, u64 dev_weight
)
990 weight
= dev_weight
?: weight
;
992 bfqg
->entity
.dev_weight
= dev_weight
;
994 * Setting the prio_changed flag of the entity
995 * to 1 with new_weight == weight would re-set
996 * the value of the weight to its ioprio mapping.
997 * Set the flag only if necessary.
999 if ((unsigned short)weight
!= bfqg
->entity
.new_weight
) {
1000 bfqg
->entity
.new_weight
= (unsigned short)weight
;
1002 * Make sure that the above new value has been
1003 * stored in bfqg->entity.new_weight before
1004 * setting the prio_changed flag. In fact,
1005 * this flag may be read asynchronously (in
1006 * critical sections protected by a different
1007 * lock than that held here), and finding this
1008 * flag set may cause the execution of the code
1009 * for updating parameters whose value may
1010 * depend also on bfqg->entity.new_weight (in
1011 * __bfq_entity_update_weight_prio).
1012 * This barrier makes sure that the new value
1013 * of bfqg->entity.new_weight is correctly
1014 * seen in that code.
1017 bfqg
->entity
.prio_changed
= 1;
1021 static int bfq_io_set_weight_legacy(struct cgroup_subsys_state
*css
,
1022 struct cftype
*cftype
,
1025 struct blkcg
*blkcg
= css_to_blkcg(css
);
1026 struct bfq_group_data
*bfqgd
= blkcg_to_bfqgd(blkcg
);
1027 struct blkcg_gq
*blkg
;
1030 if (val
< BFQ_MIN_WEIGHT
|| val
> BFQ_MAX_WEIGHT
)
1034 spin_lock_irq(&blkcg
->lock
);
1035 bfqgd
->weight
= (unsigned short)val
;
1036 hlist_for_each_entry(blkg
, &blkcg
->blkg_list
, blkcg_node
) {
1037 struct bfq_group
*bfqg
= blkg_to_bfqg(blkg
);
1040 bfq_group_set_weight(bfqg
, val
, 0);
1042 spin_unlock_irq(&blkcg
->lock
);
1047 static ssize_t
bfq_io_set_device_weight(struct kernfs_open_file
*of
,
1048 char *buf
, size_t nbytes
,
1052 struct blkg_conf_ctx ctx
;
1053 struct blkcg
*blkcg
= css_to_blkcg(of_css(of
));
1054 struct bfq_group
*bfqg
;
1057 ret
= blkg_conf_prep(blkcg
, &blkcg_policy_bfq
, buf
, &ctx
);
1061 if (sscanf(ctx
.body
, "%llu", &v
) == 1) {
1062 /* require "default" on dfl */
1066 } else if (!strcmp(strim(ctx
.body
), "default")) {
1073 bfqg
= blkg_to_bfqg(ctx
.blkg
);
1076 if (!v
|| (v
>= BFQ_MIN_WEIGHT
&& v
<= BFQ_MAX_WEIGHT
)) {
1077 bfq_group_set_weight(bfqg
, bfqg
->entity
.weight
, v
);
1081 blkg_conf_finish(&ctx
);
1082 return ret
?: nbytes
;
1085 static ssize_t
bfq_io_set_weight(struct kernfs_open_file
*of
,
1086 char *buf
, size_t nbytes
,
1095 /* "WEIGHT" or "default WEIGHT" sets the default weight */
1096 v
= simple_strtoull(buf
, &endp
, 0);
1097 if (*endp
== '\0' || sscanf(buf
, "default %llu", &v
) == 1) {
1098 ret
= bfq_io_set_weight_legacy(of_css(of
), NULL
, v
);
1099 return ret
?: nbytes
;
1102 return bfq_io_set_device_weight(of
, buf
, nbytes
, off
);
1105 static int bfqg_print_rwstat(struct seq_file
*sf
, void *v
)
1107 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)), blkg_prfill_rwstat
,
1108 &blkcg_policy_bfq
, seq_cft(sf
)->private, true);
1112 static u64
bfqg_prfill_rwstat_recursive(struct seq_file
*sf
,
1113 struct blkg_policy_data
*pd
, int off
)
1115 struct blkg_rwstat_sample sum
;
1117 blkg_rwstat_recursive_sum(pd_to_blkg(pd
), &blkcg_policy_bfq
, off
, &sum
);
1118 return __blkg_prfill_rwstat(sf
, pd
, &sum
);
1121 static int bfqg_print_rwstat_recursive(struct seq_file
*sf
, void *v
)
1123 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1124 bfqg_prfill_rwstat_recursive
, &blkcg_policy_bfq
,
1125 seq_cft(sf
)->private, true);
1129 #ifdef CONFIG_BFQ_CGROUP_DEBUG
1130 static int bfqg_print_stat(struct seq_file
*sf
, void *v
)
1132 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)), blkg_prfill_stat
,
1133 &blkcg_policy_bfq
, seq_cft(sf
)->private, false);
1137 static u64
bfqg_prfill_stat_recursive(struct seq_file
*sf
,
1138 struct blkg_policy_data
*pd
, int off
)
1140 struct blkcg_gq
*blkg
= pd_to_blkg(pd
);
1141 struct blkcg_gq
*pos_blkg
;
1142 struct cgroup_subsys_state
*pos_css
;
1145 lockdep_assert_held(&blkg
->q
->queue_lock
);
1148 blkg_for_each_descendant_pre(pos_blkg
, pos_css
, blkg
) {
1149 struct bfq_stat
*stat
;
1151 if (!pos_blkg
->online
)
1154 stat
= (void *)blkg_to_pd(pos_blkg
, &blkcg_policy_bfq
) + off
;
1155 sum
+= bfq_stat_read(stat
) + atomic64_read(&stat
->aux_cnt
);
1159 return __blkg_prfill_u64(sf
, pd
, sum
);
1162 static int bfqg_print_stat_recursive(struct seq_file
*sf
, void *v
)
1164 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1165 bfqg_prfill_stat_recursive
, &blkcg_policy_bfq
,
1166 seq_cft(sf
)->private, false);
1170 static u64
bfqg_prfill_sectors(struct seq_file
*sf
, struct blkg_policy_data
*pd
,
1173 struct bfq_group
*bfqg
= blkg_to_bfqg(pd
->blkg
);
1174 u64 sum
= blkg_rwstat_total(&bfqg
->stats
.bytes
);
1176 return __blkg_prfill_u64(sf
, pd
, sum
>> 9);
1179 static int bfqg_print_stat_sectors(struct seq_file
*sf
, void *v
)
1181 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1182 bfqg_prfill_sectors
, &blkcg_policy_bfq
, 0, false);
1186 static u64
bfqg_prfill_sectors_recursive(struct seq_file
*sf
,
1187 struct blkg_policy_data
*pd
, int off
)
1189 struct blkg_rwstat_sample tmp
;
1191 blkg_rwstat_recursive_sum(pd
->blkg
, &blkcg_policy_bfq
,
1192 offsetof(struct bfq_group
, stats
.bytes
), &tmp
);
1194 return __blkg_prfill_u64(sf
, pd
,
1195 (tmp
.cnt
[BLKG_RWSTAT_READ
] + tmp
.cnt
[BLKG_RWSTAT_WRITE
]) >> 9);
1198 static int bfqg_print_stat_sectors_recursive(struct seq_file
*sf
, void *v
)
1200 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1201 bfqg_prfill_sectors_recursive
, &blkcg_policy_bfq
, 0,
1206 static u64
bfqg_prfill_avg_queue_size(struct seq_file
*sf
,
1207 struct blkg_policy_data
*pd
, int off
)
1209 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
1210 u64 samples
= bfq_stat_read(&bfqg
->stats
.avg_queue_size_samples
);
1214 v
= bfq_stat_read(&bfqg
->stats
.avg_queue_size_sum
);
1215 v
= div64_u64(v
, samples
);
1217 __blkg_prfill_u64(sf
, pd
, v
);
1221 /* print avg_queue_size */
1222 static int bfqg_print_avg_queue_size(struct seq_file
*sf
, void *v
)
1224 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1225 bfqg_prfill_avg_queue_size
, &blkcg_policy_bfq
,
1229 #endif /* CONFIG_BFQ_CGROUP_DEBUG */
1231 struct bfq_group
*bfq_create_group_hierarchy(struct bfq_data
*bfqd
, int node
)
1235 ret
= blkcg_activate_policy(bfqd
->queue
, &blkcg_policy_bfq
);
1239 return blkg_to_bfqg(bfqd
->queue
->root_blkg
);
1242 struct blkcg_policy blkcg_policy_bfq
= {
1243 .dfl_cftypes
= bfq_blkg_files
,
1244 .legacy_cftypes
= bfq_blkcg_legacy_files
,
1246 .cpd_alloc_fn
= bfq_cpd_alloc
,
1247 .cpd_init_fn
= bfq_cpd_init
,
1248 .cpd_bind_fn
= bfq_cpd_init
,
1249 .cpd_free_fn
= bfq_cpd_free
,
1251 .pd_alloc_fn
= bfq_pd_alloc
,
1252 .pd_init_fn
= bfq_pd_init
,
1253 .pd_offline_fn
= bfq_pd_offline
,
1254 .pd_free_fn
= bfq_pd_free
,
1255 .pd_reset_stats_fn
= bfq_pd_reset_stats
,
1258 struct cftype bfq_blkcg_legacy_files
[] = {
1260 .name
= "bfq.weight",
1261 .flags
= CFTYPE_NOT_ON_ROOT
,
1262 .seq_show
= bfq_io_show_weight_legacy
,
1263 .write_u64
= bfq_io_set_weight_legacy
,
1266 .name
= "bfq.weight_device",
1267 .flags
= CFTYPE_NOT_ON_ROOT
,
1268 .seq_show
= bfq_io_show_weight
,
1269 .write
= bfq_io_set_weight
,
1272 /* statistics, covers only the tasks in the bfqg */
1274 .name
= "bfq.io_service_bytes",
1275 .private = offsetof(struct bfq_group
, stats
.bytes
),
1276 .seq_show
= bfqg_print_rwstat
,
1279 .name
= "bfq.io_serviced",
1280 .private = offsetof(struct bfq_group
, stats
.ios
),
1281 .seq_show
= bfqg_print_rwstat
,
1283 #ifdef CONFIG_BFQ_CGROUP_DEBUG
1286 .private = offsetof(struct bfq_group
, stats
.time
),
1287 .seq_show
= bfqg_print_stat
,
1290 .name
= "bfq.sectors",
1291 .seq_show
= bfqg_print_stat_sectors
,
1294 .name
= "bfq.io_service_time",
1295 .private = offsetof(struct bfq_group
, stats
.service_time
),
1296 .seq_show
= bfqg_print_rwstat
,
1299 .name
= "bfq.io_wait_time",
1300 .private = offsetof(struct bfq_group
, stats
.wait_time
),
1301 .seq_show
= bfqg_print_rwstat
,
1304 .name
= "bfq.io_merged",
1305 .private = offsetof(struct bfq_group
, stats
.merged
),
1306 .seq_show
= bfqg_print_rwstat
,
1309 .name
= "bfq.io_queued",
1310 .private = offsetof(struct bfq_group
, stats
.queued
),
1311 .seq_show
= bfqg_print_rwstat
,
1313 #endif /* CONFIG_BFQ_CGROUP_DEBUG */
1315 /* the same statistics which cover the bfqg and its descendants */
1317 .name
= "bfq.io_service_bytes_recursive",
1318 .private = offsetof(struct bfq_group
, stats
.bytes
),
1319 .seq_show
= bfqg_print_rwstat_recursive
,
1322 .name
= "bfq.io_serviced_recursive",
1323 .private = offsetof(struct bfq_group
, stats
.ios
),
1324 .seq_show
= bfqg_print_rwstat_recursive
,
1326 #ifdef CONFIG_BFQ_CGROUP_DEBUG
1328 .name
= "bfq.time_recursive",
1329 .private = offsetof(struct bfq_group
, stats
.time
),
1330 .seq_show
= bfqg_print_stat_recursive
,
1333 .name
= "bfq.sectors_recursive",
1334 .seq_show
= bfqg_print_stat_sectors_recursive
,
1337 .name
= "bfq.io_service_time_recursive",
1338 .private = offsetof(struct bfq_group
, stats
.service_time
),
1339 .seq_show
= bfqg_print_rwstat_recursive
,
1342 .name
= "bfq.io_wait_time_recursive",
1343 .private = offsetof(struct bfq_group
, stats
.wait_time
),
1344 .seq_show
= bfqg_print_rwstat_recursive
,
1347 .name
= "bfq.io_merged_recursive",
1348 .private = offsetof(struct bfq_group
, stats
.merged
),
1349 .seq_show
= bfqg_print_rwstat_recursive
,
1352 .name
= "bfq.io_queued_recursive",
1353 .private = offsetof(struct bfq_group
, stats
.queued
),
1354 .seq_show
= bfqg_print_rwstat_recursive
,
1357 .name
= "bfq.avg_queue_size",
1358 .seq_show
= bfqg_print_avg_queue_size
,
1361 .name
= "bfq.group_wait_time",
1362 .private = offsetof(struct bfq_group
, stats
.group_wait_time
),
1363 .seq_show
= bfqg_print_stat
,
1366 .name
= "bfq.idle_time",
1367 .private = offsetof(struct bfq_group
, stats
.idle_time
),
1368 .seq_show
= bfqg_print_stat
,
1371 .name
= "bfq.empty_time",
1372 .private = offsetof(struct bfq_group
, stats
.empty_time
),
1373 .seq_show
= bfqg_print_stat
,
1376 .name
= "bfq.dequeue",
1377 .private = offsetof(struct bfq_group
, stats
.dequeue
),
1378 .seq_show
= bfqg_print_stat
,
1380 #endif /* CONFIG_BFQ_CGROUP_DEBUG */
1384 struct cftype bfq_blkg_files
[] = {
1386 .name
= "bfq.weight",
1387 .flags
= CFTYPE_NOT_ON_ROOT
,
1388 .seq_show
= bfq_io_show_weight
,
1389 .write
= bfq_io_set_weight
,
1394 #else /* CONFIG_BFQ_GROUP_IOSCHED */
1396 void bfq_bfqq_move(struct bfq_data
*bfqd
, struct bfq_queue
*bfqq
,
1397 struct bfq_group
*bfqg
) {}
1399 void bfq_init_entity(struct bfq_entity
*entity
, struct bfq_group
*bfqg
)
1401 struct bfq_queue
*bfqq
= bfq_entity_to_bfqq(entity
);
1403 entity
->weight
= entity
->new_weight
;
1404 entity
->orig_weight
= entity
->new_weight
;
1406 bfqq
->ioprio
= bfqq
->new_ioprio
;
1407 bfqq
->ioprio_class
= bfqq
->new_ioprio_class
;
1409 entity
->sched_data
= &bfqg
->sched_data
;
1412 void bfq_bic_update_cgroup(struct bfq_io_cq
*bic
, struct bio
*bio
) {}
1414 void bfq_end_wr_async(struct bfq_data
*bfqd
)
1416 bfq_end_wr_async_queues(bfqd
, bfqd
->root_group
);
1419 struct bfq_group
*bfq_find_set_group(struct bfq_data
*bfqd
, struct blkcg
*blkcg
)
1421 return bfqd
->root_group
;
1424 struct bfq_group
*bfqq_group(struct bfq_queue
*bfqq
)
1426 return bfqq
->bfqd
->root_group
;
1429 void bfqg_and_blkg_get(struct bfq_group
*bfqg
) {}
1431 void bfqg_and_blkg_put(struct bfq_group
*bfqg
) {}
1433 struct bfq_group
*bfq_create_group_hierarchy(struct bfq_data
*bfqd
, int node
)
1435 struct bfq_group
*bfqg
;
1438 bfqg
= kmalloc_node(sizeof(*bfqg
), GFP_KERNEL
| __GFP_ZERO
, node
);
1442 for (i
= 0; i
< BFQ_IOPRIO_CLASSES
; i
++)
1443 bfqg
->sched_data
.service_tree
[i
] = BFQ_SERVICE_TREE_INIT
;
1447 #endif /* CONFIG_BFQ_GROUP_IOSCHED */