1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * cgroups support for the BFQ I/O scheduler.
5 #include <linux/module.h>
6 #include <linux/slab.h>
7 #include <linux/blkdev.h>
8 #include <linux/cgroup.h>
9 #include <linux/elevator.h>
10 #include <linux/ktime.h>
11 #include <linux/rbtree.h>
12 #include <linux/ioprio.h>
13 #include <linux/sbitmap.h>
14 #include <linux/delay.h>
16 #include "bfq-iosched.h"
18 #ifdef CONFIG_BFQ_CGROUP_DEBUG
19 static int bfq_stat_init(struct bfq_stat
*stat
, gfp_t gfp
)
23 ret
= percpu_counter_init(&stat
->cpu_cnt
, 0, gfp
);
27 atomic64_set(&stat
->aux_cnt
, 0);
31 static void bfq_stat_exit(struct bfq_stat
*stat
)
33 percpu_counter_destroy(&stat
->cpu_cnt
);
37 * bfq_stat_add - add a value to a bfq_stat
38 * @stat: target bfq_stat
41 * Add @val to @stat. The caller must ensure that IRQ on the same CPU
42 * don't re-enter this function for the same counter.
44 static inline void bfq_stat_add(struct bfq_stat
*stat
, uint64_t val
)
46 percpu_counter_add_batch(&stat
->cpu_cnt
, val
, BLKG_STAT_CPU_BATCH
);
50 * bfq_stat_read - read the current value of a bfq_stat
51 * @stat: bfq_stat to read
53 static inline uint64_t bfq_stat_read(struct bfq_stat
*stat
)
55 return percpu_counter_sum_positive(&stat
->cpu_cnt
);
59 * bfq_stat_reset - reset a bfq_stat
60 * @stat: bfq_stat to reset
62 static inline void bfq_stat_reset(struct bfq_stat
*stat
)
64 percpu_counter_set(&stat
->cpu_cnt
, 0);
65 atomic64_set(&stat
->aux_cnt
, 0);
69 * bfq_stat_add_aux - add a bfq_stat into another's aux count
70 * @to: the destination bfq_stat
73 * Add @from's count including the aux one to @to's aux count.
75 static inline void bfq_stat_add_aux(struct bfq_stat
*to
,
76 struct bfq_stat
*from
)
78 atomic64_add(bfq_stat_read(from
) + atomic64_read(&from
->aux_cnt
),
83 * blkg_prfill_stat - prfill callback for bfq_stat
84 * @sf: seq_file to print to
85 * @pd: policy private data of interest
86 * @off: offset to the bfq_stat in @pd
88 * prfill callback for printing a bfq_stat.
90 static u64
blkg_prfill_stat(struct seq_file
*sf
, struct blkg_policy_data
*pd
,
93 return __blkg_prfill_u64(sf
, pd
, bfq_stat_read((void *)pd
+ off
));
96 /* bfqg stats flags */
97 enum bfqg_stats_flags
{
98 BFQG_stats_waiting
= 0,
103 #define BFQG_FLAG_FNS(name) \
104 static void bfqg_stats_mark_##name(struct bfqg_stats *stats) \
106 stats->flags |= (1 << BFQG_stats_##name); \
108 static void bfqg_stats_clear_##name(struct bfqg_stats *stats) \
110 stats->flags &= ~(1 << BFQG_stats_##name); \
112 static int bfqg_stats_##name(struct bfqg_stats *stats) \
114 return (stats->flags & (1 << BFQG_stats_##name)) != 0; \
117 BFQG_FLAG_FNS(waiting)
118 BFQG_FLAG_FNS(idling
)
122 /* This should be called with the scheduler lock held. */
123 static void bfqg_stats_update_group_wait_time(struct bfqg_stats
*stats
)
127 if (!bfqg_stats_waiting(stats
))
130 now
= ktime_get_ns();
131 if (now
> stats
->start_group_wait_time
)
132 bfq_stat_add(&stats
->group_wait_time
,
133 now
- stats
->start_group_wait_time
);
134 bfqg_stats_clear_waiting(stats
);
137 /* This should be called with the scheduler lock held. */
138 static void bfqg_stats_set_start_group_wait_time(struct bfq_group
*bfqg
,
139 struct bfq_group
*curr_bfqg
)
141 struct bfqg_stats
*stats
= &bfqg
->stats
;
143 if (bfqg_stats_waiting(stats
))
145 if (bfqg
== curr_bfqg
)
147 stats
->start_group_wait_time
= ktime_get_ns();
148 bfqg_stats_mark_waiting(stats
);
151 /* This should be called with the scheduler lock held. */
152 static void bfqg_stats_end_empty_time(struct bfqg_stats
*stats
)
156 if (!bfqg_stats_empty(stats
))
159 now
= ktime_get_ns();
160 if (now
> stats
->start_empty_time
)
161 bfq_stat_add(&stats
->empty_time
,
162 now
- stats
->start_empty_time
);
163 bfqg_stats_clear_empty(stats
);
166 void bfqg_stats_update_dequeue(struct bfq_group
*bfqg
)
168 bfq_stat_add(&bfqg
->stats
.dequeue
, 1);
171 void bfqg_stats_set_start_empty_time(struct bfq_group
*bfqg
)
173 struct bfqg_stats
*stats
= &bfqg
->stats
;
175 if (blkg_rwstat_total(&stats
->queued
))
179 * group is already marked empty. This can happen if bfqq got new
180 * request in parent group and moved to this group while being added
181 * to service tree. Just ignore the event and move on.
183 if (bfqg_stats_empty(stats
))
186 stats
->start_empty_time
= ktime_get_ns();
187 bfqg_stats_mark_empty(stats
);
190 void bfqg_stats_update_idle_time(struct bfq_group
*bfqg
)
192 struct bfqg_stats
*stats
= &bfqg
->stats
;
194 if (bfqg_stats_idling(stats
)) {
195 u64 now
= ktime_get_ns();
197 if (now
> stats
->start_idle_time
)
198 bfq_stat_add(&stats
->idle_time
,
199 now
- stats
->start_idle_time
);
200 bfqg_stats_clear_idling(stats
);
204 void bfqg_stats_set_start_idle_time(struct bfq_group
*bfqg
)
206 struct bfqg_stats
*stats
= &bfqg
->stats
;
208 stats
->start_idle_time
= ktime_get_ns();
209 bfqg_stats_mark_idling(stats
);
212 void bfqg_stats_update_avg_queue_size(struct bfq_group
*bfqg
)
214 struct bfqg_stats
*stats
= &bfqg
->stats
;
216 bfq_stat_add(&stats
->avg_queue_size_sum
,
217 blkg_rwstat_total(&stats
->queued
));
218 bfq_stat_add(&stats
->avg_queue_size_samples
, 1);
219 bfqg_stats_update_group_wait_time(stats
);
222 void bfqg_stats_update_io_add(struct bfq_group
*bfqg
, struct bfq_queue
*bfqq
,
225 blkg_rwstat_add(&bfqg
->stats
.queued
, op
, 1);
226 bfqg_stats_end_empty_time(&bfqg
->stats
);
227 if (!(bfqq
== ((struct bfq_data
*)bfqg
->bfqd
)->in_service_queue
))
228 bfqg_stats_set_start_group_wait_time(bfqg
, bfqq_group(bfqq
));
231 void bfqg_stats_update_io_remove(struct bfq_group
*bfqg
, unsigned int op
)
233 blkg_rwstat_add(&bfqg
->stats
.queued
, op
, -1);
236 void bfqg_stats_update_io_merged(struct bfq_group
*bfqg
, unsigned int op
)
238 blkg_rwstat_add(&bfqg
->stats
.merged
, op
, 1);
241 void bfqg_stats_update_completion(struct bfq_group
*bfqg
, u64 start_time_ns
,
242 u64 io_start_time_ns
, unsigned int op
)
244 struct bfqg_stats
*stats
= &bfqg
->stats
;
245 u64 now
= ktime_get_ns();
247 if (now
> io_start_time_ns
)
248 blkg_rwstat_add(&stats
->service_time
, op
,
249 now
- io_start_time_ns
);
250 if (io_start_time_ns
> start_time_ns
)
251 blkg_rwstat_add(&stats
->wait_time
, op
,
252 io_start_time_ns
- start_time_ns
);
255 #else /* CONFIG_BFQ_CGROUP_DEBUG */
257 void bfqg_stats_update_io_add(struct bfq_group
*bfqg
, struct bfq_queue
*bfqq
,
259 void bfqg_stats_update_io_remove(struct bfq_group
*bfqg
, unsigned int op
) { }
260 void bfqg_stats_update_io_merged(struct bfq_group
*bfqg
, unsigned int op
) { }
261 void bfqg_stats_update_completion(struct bfq_group
*bfqg
, u64 start_time_ns
,
262 u64 io_start_time_ns
, unsigned int op
) { }
263 void bfqg_stats_update_dequeue(struct bfq_group
*bfqg
) { }
264 void bfqg_stats_set_start_empty_time(struct bfq_group
*bfqg
) { }
265 void bfqg_stats_update_idle_time(struct bfq_group
*bfqg
) { }
266 void bfqg_stats_set_start_idle_time(struct bfq_group
*bfqg
) { }
267 void bfqg_stats_update_avg_queue_size(struct bfq_group
*bfqg
) { }
269 #endif /* CONFIG_BFQ_CGROUP_DEBUG */
271 #ifdef CONFIG_BFQ_GROUP_IOSCHED
274 * blk-cgroup policy-related handlers
275 * The following functions help in converting between blk-cgroup
276 * internal structures and BFQ-specific structures.
279 static struct bfq_group
*pd_to_bfqg(struct blkg_policy_data
*pd
)
281 return pd
? container_of(pd
, struct bfq_group
, pd
) : NULL
;
284 struct blkcg_gq
*bfqg_to_blkg(struct bfq_group
*bfqg
)
286 return pd_to_blkg(&bfqg
->pd
);
289 static struct bfq_group
*blkg_to_bfqg(struct blkcg_gq
*blkg
)
291 return pd_to_bfqg(blkg_to_pd(blkg
, &blkcg_policy_bfq
));
296 * The following functions help in navigating the bfq_group hierarchy
297 * by allowing to find the parent of a bfq_group or the bfq_group
298 * associated to a bfq_queue.
301 static struct bfq_group
*bfqg_parent(struct bfq_group
*bfqg
)
303 struct blkcg_gq
*pblkg
= bfqg_to_blkg(bfqg
)->parent
;
305 return pblkg
? blkg_to_bfqg(pblkg
) : NULL
;
308 struct bfq_group
*bfqq_group(struct bfq_queue
*bfqq
)
310 struct bfq_entity
*group_entity
= bfqq
->entity
.parent
;
312 return group_entity
? container_of(group_entity
, struct bfq_group
,
314 bfqq
->bfqd
->root_group
;
318 * The following two functions handle get and put of a bfq_group by
319 * wrapping the related blk-cgroup hooks.
322 static void bfqg_get(struct bfq_group
*bfqg
)
327 static void bfqg_put(struct bfq_group
*bfqg
)
335 static void bfqg_and_blkg_get(struct bfq_group
*bfqg
)
337 /* see comments in bfq_bic_update_cgroup for why refcounting bfqg */
340 blkg_get(bfqg_to_blkg(bfqg
));
343 void bfqg_and_blkg_put(struct bfq_group
*bfqg
)
345 blkg_put(bfqg_to_blkg(bfqg
));
350 void bfqg_stats_update_legacy_io(struct request_queue
*q
, struct request
*rq
)
352 struct bfq_group
*bfqg
= blkg_to_bfqg(rq
->bio
->bi_blkg
);
357 blkg_rwstat_add(&bfqg
->stats
.bytes
, rq
->cmd_flags
, blk_rq_bytes(rq
));
358 blkg_rwstat_add(&bfqg
->stats
.ios
, rq
->cmd_flags
, 1);
362 static void bfqg_stats_reset(struct bfqg_stats
*stats
)
364 #ifdef CONFIG_BFQ_CGROUP_DEBUG
365 /* queued stats shouldn't be cleared */
366 blkg_rwstat_reset(&stats
->merged
);
367 blkg_rwstat_reset(&stats
->service_time
);
368 blkg_rwstat_reset(&stats
->wait_time
);
369 bfq_stat_reset(&stats
->time
);
370 bfq_stat_reset(&stats
->avg_queue_size_sum
);
371 bfq_stat_reset(&stats
->avg_queue_size_samples
);
372 bfq_stat_reset(&stats
->dequeue
);
373 bfq_stat_reset(&stats
->group_wait_time
);
374 bfq_stat_reset(&stats
->idle_time
);
375 bfq_stat_reset(&stats
->empty_time
);
380 static void bfqg_stats_add_aux(struct bfqg_stats
*to
, struct bfqg_stats
*from
)
385 #ifdef CONFIG_BFQ_CGROUP_DEBUG
386 /* queued stats shouldn't be cleared */
387 blkg_rwstat_add_aux(&to
->merged
, &from
->merged
);
388 blkg_rwstat_add_aux(&to
->service_time
, &from
->service_time
);
389 blkg_rwstat_add_aux(&to
->wait_time
, &from
->wait_time
);
390 bfq_stat_add_aux(&from
->time
, &from
->time
);
391 bfq_stat_add_aux(&to
->avg_queue_size_sum
, &from
->avg_queue_size_sum
);
392 bfq_stat_add_aux(&to
->avg_queue_size_samples
,
393 &from
->avg_queue_size_samples
);
394 bfq_stat_add_aux(&to
->dequeue
, &from
->dequeue
);
395 bfq_stat_add_aux(&to
->group_wait_time
, &from
->group_wait_time
);
396 bfq_stat_add_aux(&to
->idle_time
, &from
->idle_time
);
397 bfq_stat_add_aux(&to
->empty_time
, &from
->empty_time
);
402 * Transfer @bfqg's stats to its parent's aux counts so that the ancestors'
403 * recursive stats can still account for the amount used by this bfqg after
406 static void bfqg_stats_xfer_dead(struct bfq_group
*bfqg
)
408 struct bfq_group
*parent
;
410 if (!bfqg
) /* root_group */
413 parent
= bfqg_parent(bfqg
);
415 lockdep_assert_held(&bfqg_to_blkg(bfqg
)->q
->queue_lock
);
417 if (unlikely(!parent
))
420 bfqg_stats_add_aux(&parent
->stats
, &bfqg
->stats
);
421 bfqg_stats_reset(&bfqg
->stats
);
424 void bfq_init_entity(struct bfq_entity
*entity
, struct bfq_group
*bfqg
)
426 struct bfq_queue
*bfqq
= bfq_entity_to_bfqq(entity
);
428 entity
->weight
= entity
->new_weight
;
429 entity
->orig_weight
= entity
->new_weight
;
431 bfqq
->ioprio
= bfqq
->new_ioprio
;
432 bfqq
->ioprio_class
= bfqq
->new_ioprio_class
;
434 * Make sure that bfqg and its associated blkg do not
435 * disappear before entity.
437 bfqg_and_blkg_get(bfqg
);
439 entity
->parent
= bfqg
->my_entity
; /* NULL for root group */
440 entity
->sched_data
= &bfqg
->sched_data
;
443 static void bfqg_stats_exit(struct bfqg_stats
*stats
)
445 blkg_rwstat_exit(&stats
->bytes
);
446 blkg_rwstat_exit(&stats
->ios
);
447 #ifdef CONFIG_BFQ_CGROUP_DEBUG
448 blkg_rwstat_exit(&stats
->merged
);
449 blkg_rwstat_exit(&stats
->service_time
);
450 blkg_rwstat_exit(&stats
->wait_time
);
451 blkg_rwstat_exit(&stats
->queued
);
452 bfq_stat_exit(&stats
->time
);
453 bfq_stat_exit(&stats
->avg_queue_size_sum
);
454 bfq_stat_exit(&stats
->avg_queue_size_samples
);
455 bfq_stat_exit(&stats
->dequeue
);
456 bfq_stat_exit(&stats
->group_wait_time
);
457 bfq_stat_exit(&stats
->idle_time
);
458 bfq_stat_exit(&stats
->empty_time
);
462 static int bfqg_stats_init(struct bfqg_stats
*stats
, gfp_t gfp
)
464 if (blkg_rwstat_init(&stats
->bytes
, gfp
) ||
465 blkg_rwstat_init(&stats
->ios
, gfp
))
468 #ifdef CONFIG_BFQ_CGROUP_DEBUG
469 if (blkg_rwstat_init(&stats
->merged
, gfp
) ||
470 blkg_rwstat_init(&stats
->service_time
, gfp
) ||
471 blkg_rwstat_init(&stats
->wait_time
, gfp
) ||
472 blkg_rwstat_init(&stats
->queued
, gfp
) ||
473 bfq_stat_init(&stats
->time
, gfp
) ||
474 bfq_stat_init(&stats
->avg_queue_size_sum
, gfp
) ||
475 bfq_stat_init(&stats
->avg_queue_size_samples
, gfp
) ||
476 bfq_stat_init(&stats
->dequeue
, gfp
) ||
477 bfq_stat_init(&stats
->group_wait_time
, gfp
) ||
478 bfq_stat_init(&stats
->idle_time
, gfp
) ||
479 bfq_stat_init(&stats
->empty_time
, gfp
)) {
480 bfqg_stats_exit(stats
);
488 static struct bfq_group_data
*cpd_to_bfqgd(struct blkcg_policy_data
*cpd
)
490 return cpd
? container_of(cpd
, struct bfq_group_data
, pd
) : NULL
;
493 static struct bfq_group_data
*blkcg_to_bfqgd(struct blkcg
*blkcg
)
495 return cpd_to_bfqgd(blkcg_to_cpd(blkcg
, &blkcg_policy_bfq
));
498 static struct blkcg_policy_data
*bfq_cpd_alloc(gfp_t gfp
)
500 struct bfq_group_data
*bgd
;
502 bgd
= kzalloc(sizeof(*bgd
), gfp
);
508 static void bfq_cpd_init(struct blkcg_policy_data
*cpd
)
510 struct bfq_group_data
*d
= cpd_to_bfqgd(cpd
);
512 d
->weight
= cgroup_subsys_on_dfl(io_cgrp_subsys
) ?
513 CGROUP_WEIGHT_DFL
: BFQ_WEIGHT_LEGACY_DFL
;
516 static void bfq_cpd_free(struct blkcg_policy_data
*cpd
)
518 kfree(cpd_to_bfqgd(cpd
));
521 static struct blkg_policy_data
*bfq_pd_alloc(gfp_t gfp
, struct request_queue
*q
,
524 struct bfq_group
*bfqg
;
526 bfqg
= kzalloc_node(sizeof(*bfqg
), gfp
, q
->node
);
530 if (bfqg_stats_init(&bfqg
->stats
, gfp
)) {
535 /* see comments in bfq_bic_update_cgroup for why refcounting */
540 static void bfq_pd_init(struct blkg_policy_data
*pd
)
542 struct blkcg_gq
*blkg
= pd_to_blkg(pd
);
543 struct bfq_group
*bfqg
= blkg_to_bfqg(blkg
);
544 struct bfq_data
*bfqd
= blkg
->q
->elevator
->elevator_data
;
545 struct bfq_entity
*entity
= &bfqg
->entity
;
546 struct bfq_group_data
*d
= blkcg_to_bfqgd(blkg
->blkcg
);
548 entity
->orig_weight
= entity
->weight
= entity
->new_weight
= d
->weight
;
549 entity
->my_sched_data
= &bfqg
->sched_data
;
550 entity
->last_bfqq_created
= NULL
;
552 bfqg
->my_entity
= entity
; /*
553 * the root_group's will be set to NULL
554 * in bfq_init_queue()
557 bfqg
->active_entities
= 0;
558 bfqg
->rq_pos_tree
= RB_ROOT
;
561 static void bfq_pd_free(struct blkg_policy_data
*pd
)
563 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
565 bfqg_stats_exit(&bfqg
->stats
);
569 static void bfq_pd_reset_stats(struct blkg_policy_data
*pd
)
571 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
573 bfqg_stats_reset(&bfqg
->stats
);
576 static void bfq_group_set_parent(struct bfq_group
*bfqg
,
577 struct bfq_group
*parent
)
579 struct bfq_entity
*entity
;
581 entity
= &bfqg
->entity
;
582 entity
->parent
= parent
->my_entity
;
583 entity
->sched_data
= &parent
->sched_data
;
586 static struct bfq_group
*bfq_lookup_bfqg(struct bfq_data
*bfqd
,
589 struct blkcg_gq
*blkg
;
591 blkg
= blkg_lookup(blkcg
, bfqd
->queue
);
593 return blkg_to_bfqg(blkg
);
597 struct bfq_group
*bfq_find_set_group(struct bfq_data
*bfqd
,
600 struct bfq_group
*bfqg
, *parent
;
601 struct bfq_entity
*entity
;
603 bfqg
= bfq_lookup_bfqg(bfqd
, blkcg
);
609 * Update chain of bfq_groups as we might be handling a leaf group
610 * which, along with some of its relatives, has not been hooked yet
611 * to the private hierarchy of BFQ.
613 entity
= &bfqg
->entity
;
614 for_each_entity(entity
) {
615 struct bfq_group
*curr_bfqg
= container_of(entity
,
616 struct bfq_group
, entity
);
617 if (curr_bfqg
!= bfqd
->root_group
) {
618 parent
= bfqg_parent(curr_bfqg
);
620 parent
= bfqd
->root_group
;
621 bfq_group_set_parent(curr_bfqg
, parent
);
629 * bfq_bfqq_move - migrate @bfqq to @bfqg.
630 * @bfqd: queue descriptor.
631 * @bfqq: the queue to move.
632 * @bfqg: the group to move to.
634 * Move @bfqq to @bfqg, deactivating it from its old group and reactivating
635 * it on the new one. Avoid putting the entity on the old group idle tree.
637 * Must be called under the scheduler lock, to make sure that the blkg
638 * owning @bfqg does not disappear (see comments in
639 * bfq_bic_update_cgroup on guaranteeing the consistency of blkg
642 void bfq_bfqq_move(struct bfq_data
*bfqd
, struct bfq_queue
*bfqq
,
643 struct bfq_group
*bfqg
)
645 struct bfq_entity
*entity
= &bfqq
->entity
;
648 * oom_bfqq is not allowed to move, oom_bfqq will hold ref to root_group
649 * until elevator exit.
651 if (bfqq
== &bfqd
->oom_bfqq
)
654 * Get extra reference to prevent bfqq from being freed in
655 * next possible expire or deactivate.
659 /* If bfqq is empty, then bfq_bfqq_expire also invokes
660 * bfq_del_bfqq_busy, thereby removing bfqq and its entity
661 * from data structures related to current group. Otherwise we
662 * need to remove bfqq explicitly with bfq_deactivate_bfqq, as
665 if (bfqq
== bfqd
->in_service_queue
)
666 bfq_bfqq_expire(bfqd
, bfqd
->in_service_queue
,
667 false, BFQQE_PREEMPTED
);
669 if (bfq_bfqq_busy(bfqq
))
670 bfq_deactivate_bfqq(bfqd
, bfqq
, false, false);
671 else if (entity
->on_st_or_in_serv
)
672 bfq_put_idle_entity(bfq_entity_service_tree(entity
), entity
);
673 bfqg_and_blkg_put(bfqq_group(bfqq
));
675 if (entity
->parent
&&
676 entity
->parent
->last_bfqq_created
== bfqq
)
677 entity
->parent
->last_bfqq_created
= NULL
;
678 else if (bfqd
->last_bfqq_created
== bfqq
)
679 bfqd
->last_bfqq_created
= NULL
;
681 entity
->parent
= bfqg
->my_entity
;
682 entity
->sched_data
= &bfqg
->sched_data
;
683 /* pin down bfqg and its associated blkg */
684 bfqg_and_blkg_get(bfqg
);
686 if (bfq_bfqq_busy(bfqq
)) {
687 if (unlikely(!bfqd
->nonrot_with_queueing
))
688 bfq_pos_tree_add_move(bfqd
, bfqq
);
689 bfq_activate_bfqq(bfqd
, bfqq
);
692 if (!bfqd
->in_service_queue
&& !bfqd
->rq_in_driver
)
693 bfq_schedule_dispatch(bfqd
);
694 /* release extra ref taken above, bfqq may happen to be freed now */
699 * __bfq_bic_change_cgroup - move @bic to @cgroup.
700 * @bfqd: the queue descriptor.
701 * @bic: the bic to move.
702 * @blkcg: the blk-cgroup to move to.
704 * Move bic to blkcg, assuming that bfqd->lock is held; which makes
705 * sure that the reference to cgroup is valid across the call (see
706 * comments in bfq_bic_update_cgroup on this issue)
708 * NOTE: an alternative approach might have been to store the current
709 * cgroup in bfqq and getting a reference to it, reducing the lookup
710 * time here, at the price of slightly more complex code.
712 static struct bfq_group
*__bfq_bic_change_cgroup(struct bfq_data
*bfqd
,
713 struct bfq_io_cq
*bic
,
716 struct bfq_queue
*async_bfqq
= bic_to_bfqq(bic
, 0);
717 struct bfq_queue
*sync_bfqq
= bic_to_bfqq(bic
, 1);
718 struct bfq_group
*bfqg
;
719 struct bfq_entity
*entity
;
721 bfqg
= bfq_find_set_group(bfqd
, blkcg
);
724 bfqg
= bfqd
->root_group
;
727 entity
= &async_bfqq
->entity
;
729 if (entity
->sched_data
!= &bfqg
->sched_data
) {
730 bic_set_bfqq(bic
, NULL
, 0);
731 bfq_release_process_ref(bfqd
, async_bfqq
);
736 entity
= &sync_bfqq
->entity
;
737 if (entity
->sched_data
!= &bfqg
->sched_data
)
738 bfq_bfqq_move(bfqd
, sync_bfqq
, bfqg
);
744 void bfq_bic_update_cgroup(struct bfq_io_cq
*bic
, struct bio
*bio
)
746 struct bfq_data
*bfqd
= bic_to_bfqd(bic
);
747 struct bfq_group
*bfqg
= NULL
;
751 serial_nr
= __bio_blkcg(bio
)->css
.serial_nr
;
754 * Check whether blkcg has changed. The condition may trigger
755 * spuriously on a newly created cic but there's no harm.
757 if (unlikely(!bfqd
) || likely(bic
->blkcg_serial_nr
== serial_nr
))
760 bfqg
= __bfq_bic_change_cgroup(bfqd
, bic
, __bio_blkcg(bio
));
762 * Update blkg_path for bfq_log_* functions. We cache this
763 * path, and update it here, for the following
764 * reasons. Operations on blkg objects in blk-cgroup are
765 * protected with the request_queue lock, and not with the
766 * lock that protects the instances of this scheduler
767 * (bfqd->lock). This exposes BFQ to the following sort of
770 * The blkg_lookup performed in bfq_get_queue, protected
771 * through rcu, may happen to return the address of a copy of
772 * the original blkg. If this is the case, then the
773 * bfqg_and_blkg_get performed in bfq_get_queue, to pin down
774 * the blkg, is useless: it does not prevent blk-cgroup code
775 * from destroying both the original blkg and all objects
776 * directly or indirectly referred by the copy of the
779 * On the bright side, destroy operations on a blkg invoke, as
780 * a first step, hooks of the scheduler associated with the
781 * blkg. And these hooks are executed with bfqd->lock held for
782 * BFQ. As a consequence, for any blkg associated with the
783 * request queue this instance of the scheduler is attached
784 * to, we are guaranteed that such a blkg is not destroyed, and
785 * that all the pointers it contains are consistent, while we
786 * are holding bfqd->lock. A blkg_lookup performed with
787 * bfqd->lock held then returns a fully consistent blkg, which
788 * remains consistent until this lock is held.
790 * Thanks to the last fact, and to the fact that: (1) bfqg has
791 * been obtained through a blkg_lookup in the above
792 * assignment, and (2) bfqd->lock is being held, here we can
793 * safely use the policy data for the involved blkg (i.e., the
794 * field bfqg->pd) to get to the blkg associated with bfqg,
795 * and then we can safely use any field of blkg. After we
796 * release bfqd->lock, even just getting blkg through this
797 * bfqg may cause dangling references to be traversed, as
798 * bfqg->pd may not exist any more.
800 * In view of the above facts, here we cache, in the bfqg, any
801 * blkg data we may need for this bic, and for its associated
802 * bfq_queue. As of now, we need to cache only the path of the
803 * blkg, which is used in the bfq_log_* functions.
805 * Finally, note that bfqg itself needs to be protected from
806 * destruction on the blkg_free of the original blkg (which
807 * invokes bfq_pd_free). We use an additional private
808 * refcounter for bfqg, to let it disappear only after no
809 * bfq_queue refers to it any longer.
811 blkg_path(bfqg_to_blkg(bfqg
), bfqg
->blkg_path
, sizeof(bfqg
->blkg_path
));
812 bic
->blkcg_serial_nr
= serial_nr
;
818 * bfq_flush_idle_tree - deactivate any entity on the idle tree of @st.
819 * @st: the service tree being flushed.
821 static void bfq_flush_idle_tree(struct bfq_service_tree
*st
)
823 struct bfq_entity
*entity
= st
->first_idle
;
825 for (; entity
; entity
= st
->first_idle
)
826 __bfq_deactivate_entity(entity
, false);
830 * bfq_reparent_leaf_entity - move leaf entity to the root_group.
831 * @bfqd: the device data structure with the root group.
832 * @entity: the entity to move, if entity is a leaf; or the parent entity
833 * of an active leaf entity to move, if entity is not a leaf.
835 static void bfq_reparent_leaf_entity(struct bfq_data
*bfqd
,
836 struct bfq_entity
*entity
,
839 struct bfq_queue
*bfqq
;
840 struct bfq_entity
*child_entity
= entity
;
842 while (child_entity
->my_sched_data
) { /* leaf not reached yet */
843 struct bfq_sched_data
*child_sd
= child_entity
->my_sched_data
;
844 struct bfq_service_tree
*child_st
= child_sd
->service_tree
+
846 struct rb_root
*child_active
= &child_st
->active
;
848 child_entity
= bfq_entity_of(rb_first(child_active
));
851 child_entity
= child_sd
->in_service_entity
;
854 bfqq
= bfq_entity_to_bfqq(child_entity
);
855 bfq_bfqq_move(bfqd
, bfqq
, bfqd
->root_group
);
859 * bfq_reparent_active_queues - move to the root group all active queues.
860 * @bfqd: the device data structure with the root group.
861 * @bfqg: the group to move from.
862 * @st: the service tree to start the search from.
864 static void bfq_reparent_active_queues(struct bfq_data
*bfqd
,
865 struct bfq_group
*bfqg
,
866 struct bfq_service_tree
*st
,
869 struct rb_root
*active
= &st
->active
;
870 struct bfq_entity
*entity
;
872 while ((entity
= bfq_entity_of(rb_first(active
))))
873 bfq_reparent_leaf_entity(bfqd
, entity
, ioprio_class
);
875 if (bfqg
->sched_data
.in_service_entity
)
876 bfq_reparent_leaf_entity(bfqd
,
877 bfqg
->sched_data
.in_service_entity
,
882 * bfq_pd_offline - deactivate the entity associated with @pd,
883 * and reparent its children entities.
884 * @pd: descriptor of the policy going offline.
886 * blkio already grabs the queue_lock for us, so no need to use
889 static void bfq_pd_offline(struct blkg_policy_data
*pd
)
891 struct bfq_service_tree
*st
;
892 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
893 struct bfq_data
*bfqd
= bfqg
->bfqd
;
894 struct bfq_entity
*entity
= bfqg
->my_entity
;
898 spin_lock_irqsave(&bfqd
->lock
, flags
);
900 if (!entity
) /* root group */
901 goto put_async_queues
;
904 * Empty all service_trees belonging to this group before
905 * deactivating the group itself.
907 for (i
= 0; i
< BFQ_IOPRIO_CLASSES
; i
++) {
908 st
= bfqg
->sched_data
.service_tree
+ i
;
911 * It may happen that some queues are still active
912 * (busy) upon group destruction (if the corresponding
913 * processes have been forced to terminate). We move
914 * all the leaf entities corresponding to these queues
916 * Also, it may happen that the group has an entity
917 * in service, which is disconnected from the active
918 * tree: it must be moved, too.
919 * There is no need to put the sync queues, as the
920 * scheduler has taken no reference.
922 bfq_reparent_active_queues(bfqd
, bfqg
, st
, i
);
925 * The idle tree may still contain bfq_queues
926 * belonging to exited task because they never
927 * migrated to a different cgroup from the one being
928 * destroyed now. In addition, even
929 * bfq_reparent_active_queues() may happen to add some
930 * entities to the idle tree. It happens if, in some
931 * of the calls to bfq_bfqq_move() performed by
932 * bfq_reparent_active_queues(), the queue to move is
933 * empty and gets expired.
935 bfq_flush_idle_tree(st
);
938 __bfq_deactivate_entity(entity
, false);
941 bfq_put_async_queues(bfqd
, bfqg
);
943 spin_unlock_irqrestore(&bfqd
->lock
, flags
);
945 * @blkg is going offline and will be ignored by
946 * blkg_[rw]stat_recursive_sum(). Transfer stats to the parent so
947 * that they don't get lost. If IOs complete after this point, the
948 * stats for them will be lost. Oh well...
950 bfqg_stats_xfer_dead(bfqg
);
953 void bfq_end_wr_async(struct bfq_data
*bfqd
)
955 struct blkcg_gq
*blkg
;
957 list_for_each_entry(blkg
, &bfqd
->queue
->blkg_list
, q_node
) {
958 struct bfq_group
*bfqg
= blkg_to_bfqg(blkg
);
960 bfq_end_wr_async_queues(bfqd
, bfqg
);
962 bfq_end_wr_async_queues(bfqd
, bfqd
->root_group
);
965 static int bfq_io_show_weight_legacy(struct seq_file
*sf
, void *v
)
967 struct blkcg
*blkcg
= css_to_blkcg(seq_css(sf
));
968 struct bfq_group_data
*bfqgd
= blkcg_to_bfqgd(blkcg
);
969 unsigned int val
= 0;
974 seq_printf(sf
, "%u\n", val
);
979 static u64
bfqg_prfill_weight_device(struct seq_file
*sf
,
980 struct blkg_policy_data
*pd
, int off
)
982 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
984 if (!bfqg
->entity
.dev_weight
)
986 return __blkg_prfill_u64(sf
, pd
, bfqg
->entity
.dev_weight
);
989 static int bfq_io_show_weight(struct seq_file
*sf
, void *v
)
991 struct blkcg
*blkcg
= css_to_blkcg(seq_css(sf
));
992 struct bfq_group_data
*bfqgd
= blkcg_to_bfqgd(blkcg
);
994 seq_printf(sf
, "default %u\n", bfqgd
->weight
);
995 blkcg_print_blkgs(sf
, blkcg
, bfqg_prfill_weight_device
,
996 &blkcg_policy_bfq
, 0, false);
1000 static void bfq_group_set_weight(struct bfq_group
*bfqg
, u64 weight
, u64 dev_weight
)
1002 weight
= dev_weight
?: weight
;
1004 bfqg
->entity
.dev_weight
= dev_weight
;
1006 * Setting the prio_changed flag of the entity
1007 * to 1 with new_weight == weight would re-set
1008 * the value of the weight to its ioprio mapping.
1009 * Set the flag only if necessary.
1011 if ((unsigned short)weight
!= bfqg
->entity
.new_weight
) {
1012 bfqg
->entity
.new_weight
= (unsigned short)weight
;
1014 * Make sure that the above new value has been
1015 * stored in bfqg->entity.new_weight before
1016 * setting the prio_changed flag. In fact,
1017 * this flag may be read asynchronously (in
1018 * critical sections protected by a different
1019 * lock than that held here), and finding this
1020 * flag set may cause the execution of the code
1021 * for updating parameters whose value may
1022 * depend also on bfqg->entity.new_weight (in
1023 * __bfq_entity_update_weight_prio).
1024 * This barrier makes sure that the new value
1025 * of bfqg->entity.new_weight is correctly
1026 * seen in that code.
1029 bfqg
->entity
.prio_changed
= 1;
1033 static int bfq_io_set_weight_legacy(struct cgroup_subsys_state
*css
,
1034 struct cftype
*cftype
,
1037 struct blkcg
*blkcg
= css_to_blkcg(css
);
1038 struct bfq_group_data
*bfqgd
= blkcg_to_bfqgd(blkcg
);
1039 struct blkcg_gq
*blkg
;
1042 if (val
< BFQ_MIN_WEIGHT
|| val
> BFQ_MAX_WEIGHT
)
1046 spin_lock_irq(&blkcg
->lock
);
1047 bfqgd
->weight
= (unsigned short)val
;
1048 hlist_for_each_entry(blkg
, &blkcg
->blkg_list
, blkcg_node
) {
1049 struct bfq_group
*bfqg
= blkg_to_bfqg(blkg
);
1052 bfq_group_set_weight(bfqg
, val
, 0);
1054 spin_unlock_irq(&blkcg
->lock
);
1059 static ssize_t
bfq_io_set_device_weight(struct kernfs_open_file
*of
,
1060 char *buf
, size_t nbytes
,
1064 struct blkg_conf_ctx ctx
;
1065 struct blkcg
*blkcg
= css_to_blkcg(of_css(of
));
1066 struct bfq_group
*bfqg
;
1069 ret
= blkg_conf_prep(blkcg
, &blkcg_policy_bfq
, buf
, &ctx
);
1073 if (sscanf(ctx
.body
, "%llu", &v
) == 1) {
1074 /* require "default" on dfl */
1078 } else if (!strcmp(strim(ctx
.body
), "default")) {
1085 bfqg
= blkg_to_bfqg(ctx
.blkg
);
1088 if (!v
|| (v
>= BFQ_MIN_WEIGHT
&& v
<= BFQ_MAX_WEIGHT
)) {
1089 bfq_group_set_weight(bfqg
, bfqg
->entity
.weight
, v
);
1093 blkg_conf_finish(&ctx
);
1094 return ret
?: nbytes
;
1097 static ssize_t
bfq_io_set_weight(struct kernfs_open_file
*of
,
1098 char *buf
, size_t nbytes
,
1107 /* "WEIGHT" or "default WEIGHT" sets the default weight */
1108 v
= simple_strtoull(buf
, &endp
, 0);
1109 if (*endp
== '\0' || sscanf(buf
, "default %llu", &v
) == 1) {
1110 ret
= bfq_io_set_weight_legacy(of_css(of
), NULL
, v
);
1111 return ret
?: nbytes
;
1114 return bfq_io_set_device_weight(of
, buf
, nbytes
, off
);
1117 static int bfqg_print_rwstat(struct seq_file
*sf
, void *v
)
1119 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)), blkg_prfill_rwstat
,
1120 &blkcg_policy_bfq
, seq_cft(sf
)->private, true);
1124 static u64
bfqg_prfill_rwstat_recursive(struct seq_file
*sf
,
1125 struct blkg_policy_data
*pd
, int off
)
1127 struct blkg_rwstat_sample sum
;
1129 blkg_rwstat_recursive_sum(pd_to_blkg(pd
), &blkcg_policy_bfq
, off
, &sum
);
1130 return __blkg_prfill_rwstat(sf
, pd
, &sum
);
1133 static int bfqg_print_rwstat_recursive(struct seq_file
*sf
, void *v
)
1135 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1136 bfqg_prfill_rwstat_recursive
, &blkcg_policy_bfq
,
1137 seq_cft(sf
)->private, true);
1141 #ifdef CONFIG_BFQ_CGROUP_DEBUG
1142 static int bfqg_print_stat(struct seq_file
*sf
, void *v
)
1144 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)), blkg_prfill_stat
,
1145 &blkcg_policy_bfq
, seq_cft(sf
)->private, false);
1149 static u64
bfqg_prfill_stat_recursive(struct seq_file
*sf
,
1150 struct blkg_policy_data
*pd
, int off
)
1152 struct blkcg_gq
*blkg
= pd_to_blkg(pd
);
1153 struct blkcg_gq
*pos_blkg
;
1154 struct cgroup_subsys_state
*pos_css
;
1157 lockdep_assert_held(&blkg
->q
->queue_lock
);
1160 blkg_for_each_descendant_pre(pos_blkg
, pos_css
, blkg
) {
1161 struct bfq_stat
*stat
;
1163 if (!pos_blkg
->online
)
1166 stat
= (void *)blkg_to_pd(pos_blkg
, &blkcg_policy_bfq
) + off
;
1167 sum
+= bfq_stat_read(stat
) + atomic64_read(&stat
->aux_cnt
);
1171 return __blkg_prfill_u64(sf
, pd
, sum
);
1174 static int bfqg_print_stat_recursive(struct seq_file
*sf
, void *v
)
1176 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1177 bfqg_prfill_stat_recursive
, &blkcg_policy_bfq
,
1178 seq_cft(sf
)->private, false);
1182 static u64
bfqg_prfill_sectors(struct seq_file
*sf
, struct blkg_policy_data
*pd
,
1185 struct bfq_group
*bfqg
= blkg_to_bfqg(pd
->blkg
);
1186 u64 sum
= blkg_rwstat_total(&bfqg
->stats
.bytes
);
1188 return __blkg_prfill_u64(sf
, pd
, sum
>> 9);
1191 static int bfqg_print_stat_sectors(struct seq_file
*sf
, void *v
)
1193 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1194 bfqg_prfill_sectors
, &blkcg_policy_bfq
, 0, false);
1198 static u64
bfqg_prfill_sectors_recursive(struct seq_file
*sf
,
1199 struct blkg_policy_data
*pd
, int off
)
1201 struct blkg_rwstat_sample tmp
;
1203 blkg_rwstat_recursive_sum(pd
->blkg
, &blkcg_policy_bfq
,
1204 offsetof(struct bfq_group
, stats
.bytes
), &tmp
);
1206 return __blkg_prfill_u64(sf
, pd
,
1207 (tmp
.cnt
[BLKG_RWSTAT_READ
] + tmp
.cnt
[BLKG_RWSTAT_WRITE
]) >> 9);
1210 static int bfqg_print_stat_sectors_recursive(struct seq_file
*sf
, void *v
)
1212 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1213 bfqg_prfill_sectors_recursive
, &blkcg_policy_bfq
, 0,
1218 static u64
bfqg_prfill_avg_queue_size(struct seq_file
*sf
,
1219 struct blkg_policy_data
*pd
, int off
)
1221 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
1222 u64 samples
= bfq_stat_read(&bfqg
->stats
.avg_queue_size_samples
);
1226 v
= bfq_stat_read(&bfqg
->stats
.avg_queue_size_sum
);
1227 v
= div64_u64(v
, samples
);
1229 __blkg_prfill_u64(sf
, pd
, v
);
1233 /* print avg_queue_size */
1234 static int bfqg_print_avg_queue_size(struct seq_file
*sf
, void *v
)
1236 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1237 bfqg_prfill_avg_queue_size
, &blkcg_policy_bfq
,
1241 #endif /* CONFIG_BFQ_CGROUP_DEBUG */
1243 struct bfq_group
*bfq_create_group_hierarchy(struct bfq_data
*bfqd
, int node
)
1247 ret
= blkcg_activate_policy(bfqd
->queue
, &blkcg_policy_bfq
);
1251 return blkg_to_bfqg(bfqd
->queue
->root_blkg
);
1254 struct blkcg_policy blkcg_policy_bfq
= {
1255 .dfl_cftypes
= bfq_blkg_files
,
1256 .legacy_cftypes
= bfq_blkcg_legacy_files
,
1258 .cpd_alloc_fn
= bfq_cpd_alloc
,
1259 .cpd_init_fn
= bfq_cpd_init
,
1260 .cpd_bind_fn
= bfq_cpd_init
,
1261 .cpd_free_fn
= bfq_cpd_free
,
1263 .pd_alloc_fn
= bfq_pd_alloc
,
1264 .pd_init_fn
= bfq_pd_init
,
1265 .pd_offline_fn
= bfq_pd_offline
,
1266 .pd_free_fn
= bfq_pd_free
,
1267 .pd_reset_stats_fn
= bfq_pd_reset_stats
,
1270 struct cftype bfq_blkcg_legacy_files
[] = {
1272 .name
= "bfq.weight",
1273 .flags
= CFTYPE_NOT_ON_ROOT
,
1274 .seq_show
= bfq_io_show_weight_legacy
,
1275 .write_u64
= bfq_io_set_weight_legacy
,
1278 .name
= "bfq.weight_device",
1279 .flags
= CFTYPE_NOT_ON_ROOT
,
1280 .seq_show
= bfq_io_show_weight
,
1281 .write
= bfq_io_set_weight
,
1284 /* statistics, covers only the tasks in the bfqg */
1286 .name
= "bfq.io_service_bytes",
1287 .private = offsetof(struct bfq_group
, stats
.bytes
),
1288 .seq_show
= bfqg_print_rwstat
,
1291 .name
= "bfq.io_serviced",
1292 .private = offsetof(struct bfq_group
, stats
.ios
),
1293 .seq_show
= bfqg_print_rwstat
,
1295 #ifdef CONFIG_BFQ_CGROUP_DEBUG
1298 .private = offsetof(struct bfq_group
, stats
.time
),
1299 .seq_show
= bfqg_print_stat
,
1302 .name
= "bfq.sectors",
1303 .seq_show
= bfqg_print_stat_sectors
,
1306 .name
= "bfq.io_service_time",
1307 .private = offsetof(struct bfq_group
, stats
.service_time
),
1308 .seq_show
= bfqg_print_rwstat
,
1311 .name
= "bfq.io_wait_time",
1312 .private = offsetof(struct bfq_group
, stats
.wait_time
),
1313 .seq_show
= bfqg_print_rwstat
,
1316 .name
= "bfq.io_merged",
1317 .private = offsetof(struct bfq_group
, stats
.merged
),
1318 .seq_show
= bfqg_print_rwstat
,
1321 .name
= "bfq.io_queued",
1322 .private = offsetof(struct bfq_group
, stats
.queued
),
1323 .seq_show
= bfqg_print_rwstat
,
1325 #endif /* CONFIG_BFQ_CGROUP_DEBUG */
1327 /* the same statistics which cover the bfqg and its descendants */
1329 .name
= "bfq.io_service_bytes_recursive",
1330 .private = offsetof(struct bfq_group
, stats
.bytes
),
1331 .seq_show
= bfqg_print_rwstat_recursive
,
1334 .name
= "bfq.io_serviced_recursive",
1335 .private = offsetof(struct bfq_group
, stats
.ios
),
1336 .seq_show
= bfqg_print_rwstat_recursive
,
1338 #ifdef CONFIG_BFQ_CGROUP_DEBUG
1340 .name
= "bfq.time_recursive",
1341 .private = offsetof(struct bfq_group
, stats
.time
),
1342 .seq_show
= bfqg_print_stat_recursive
,
1345 .name
= "bfq.sectors_recursive",
1346 .seq_show
= bfqg_print_stat_sectors_recursive
,
1349 .name
= "bfq.io_service_time_recursive",
1350 .private = offsetof(struct bfq_group
, stats
.service_time
),
1351 .seq_show
= bfqg_print_rwstat_recursive
,
1354 .name
= "bfq.io_wait_time_recursive",
1355 .private = offsetof(struct bfq_group
, stats
.wait_time
),
1356 .seq_show
= bfqg_print_rwstat_recursive
,
1359 .name
= "bfq.io_merged_recursive",
1360 .private = offsetof(struct bfq_group
, stats
.merged
),
1361 .seq_show
= bfqg_print_rwstat_recursive
,
1364 .name
= "bfq.io_queued_recursive",
1365 .private = offsetof(struct bfq_group
, stats
.queued
),
1366 .seq_show
= bfqg_print_rwstat_recursive
,
1369 .name
= "bfq.avg_queue_size",
1370 .seq_show
= bfqg_print_avg_queue_size
,
1373 .name
= "bfq.group_wait_time",
1374 .private = offsetof(struct bfq_group
, stats
.group_wait_time
),
1375 .seq_show
= bfqg_print_stat
,
1378 .name
= "bfq.idle_time",
1379 .private = offsetof(struct bfq_group
, stats
.idle_time
),
1380 .seq_show
= bfqg_print_stat
,
1383 .name
= "bfq.empty_time",
1384 .private = offsetof(struct bfq_group
, stats
.empty_time
),
1385 .seq_show
= bfqg_print_stat
,
1388 .name
= "bfq.dequeue",
1389 .private = offsetof(struct bfq_group
, stats
.dequeue
),
1390 .seq_show
= bfqg_print_stat
,
1392 #endif /* CONFIG_BFQ_CGROUP_DEBUG */
1396 struct cftype bfq_blkg_files
[] = {
1398 .name
= "bfq.weight",
1399 .flags
= CFTYPE_NOT_ON_ROOT
,
1400 .seq_show
= bfq_io_show_weight
,
1401 .write
= bfq_io_set_weight
,
1406 #else /* CONFIG_BFQ_GROUP_IOSCHED */
1408 void bfq_bfqq_move(struct bfq_data
*bfqd
, struct bfq_queue
*bfqq
,
1409 struct bfq_group
*bfqg
) {}
1411 void bfq_init_entity(struct bfq_entity
*entity
, struct bfq_group
*bfqg
)
1413 struct bfq_queue
*bfqq
= bfq_entity_to_bfqq(entity
);
1415 entity
->weight
= entity
->new_weight
;
1416 entity
->orig_weight
= entity
->new_weight
;
1418 bfqq
->ioprio
= bfqq
->new_ioprio
;
1419 bfqq
->ioprio_class
= bfqq
->new_ioprio_class
;
1421 entity
->sched_data
= &bfqg
->sched_data
;
1424 void bfq_bic_update_cgroup(struct bfq_io_cq
*bic
, struct bio
*bio
) {}
1426 void bfq_end_wr_async(struct bfq_data
*bfqd
)
1428 bfq_end_wr_async_queues(bfqd
, bfqd
->root_group
);
1431 struct bfq_group
*bfq_find_set_group(struct bfq_data
*bfqd
, struct blkcg
*blkcg
)
1433 return bfqd
->root_group
;
1436 struct bfq_group
*bfqq_group(struct bfq_queue
*bfqq
)
1438 return bfqq
->bfqd
->root_group
;
1441 void bfqg_and_blkg_get(struct bfq_group
*bfqg
) {}
1443 void bfqg_and_blkg_put(struct bfq_group
*bfqg
) {}
1445 struct bfq_group
*bfq_create_group_hierarchy(struct bfq_data
*bfqd
, int node
)
1447 struct bfq_group
*bfqg
;
1450 bfqg
= kmalloc_node(sizeof(*bfqg
), GFP_KERNEL
| __GFP_ZERO
, node
);
1454 for (i
= 0; i
< BFQ_IOPRIO_CLASSES
; i
++)
1455 bfqg
->sched_data
.service_tree
[i
] = BFQ_SERVICE_TREE_INIT
;
1459 #endif /* CONFIG_BFQ_GROUP_IOSCHED */