1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * cgroups support for the BFQ I/O scheduler.
5 #include <linux/module.h>
6 #include <linux/slab.h>
7 #include <linux/blkdev.h>
8 #include <linux/cgroup.h>
9 #include <linux/ktime.h>
10 #include <linux/rbtree.h>
11 #include <linux/ioprio.h>
12 #include <linux/sbitmap.h>
13 #include <linux/delay.h>
16 #include "bfq-iosched.h"
18 #ifdef CONFIG_BFQ_CGROUP_DEBUG
19 static int bfq_stat_init(struct bfq_stat
*stat
, gfp_t gfp
)
23 ret
= percpu_counter_init(&stat
->cpu_cnt
, 0, gfp
);
27 atomic64_set(&stat
->aux_cnt
, 0);
31 static void bfq_stat_exit(struct bfq_stat
*stat
)
33 percpu_counter_destroy(&stat
->cpu_cnt
);
37 * bfq_stat_add - add a value to a bfq_stat
38 * @stat: target bfq_stat
41 * Add @val to @stat. The caller must ensure that IRQ on the same CPU
42 * don't re-enter this function for the same counter.
44 static inline void bfq_stat_add(struct bfq_stat
*stat
, uint64_t val
)
46 percpu_counter_add_batch(&stat
->cpu_cnt
, val
, BLKG_STAT_CPU_BATCH
);
50 * bfq_stat_read - read the current value of a bfq_stat
51 * @stat: bfq_stat to read
53 static inline uint64_t bfq_stat_read(struct bfq_stat
*stat
)
55 return percpu_counter_sum_positive(&stat
->cpu_cnt
);
59 * bfq_stat_reset - reset a bfq_stat
60 * @stat: bfq_stat to reset
62 static inline void bfq_stat_reset(struct bfq_stat
*stat
)
64 percpu_counter_set(&stat
->cpu_cnt
, 0);
65 atomic64_set(&stat
->aux_cnt
, 0);
69 * bfq_stat_add_aux - add a bfq_stat into another's aux count
70 * @to: the destination bfq_stat
73 * Add @from's count including the aux one to @to's aux count.
75 static inline void bfq_stat_add_aux(struct bfq_stat
*to
,
76 struct bfq_stat
*from
)
78 atomic64_add(bfq_stat_read(from
) + atomic64_read(&from
->aux_cnt
),
83 * blkg_prfill_stat - prfill callback for bfq_stat
84 * @sf: seq_file to print to
85 * @pd: policy private data of interest
86 * @off: offset to the bfq_stat in @pd
88 * prfill callback for printing a bfq_stat.
90 static u64
blkg_prfill_stat(struct seq_file
*sf
, struct blkg_policy_data
*pd
,
93 return __blkg_prfill_u64(sf
, pd
, bfq_stat_read((void *)pd
+ off
));
96 /* bfqg stats flags */
97 enum bfqg_stats_flags
{
98 BFQG_stats_waiting
= 0,
103 #define BFQG_FLAG_FNS(name) \
104 static void bfqg_stats_mark_##name(struct bfqg_stats *stats) \
106 stats->flags |= (1 << BFQG_stats_##name); \
108 static void bfqg_stats_clear_##name(struct bfqg_stats *stats) \
110 stats->flags &= ~(1 << BFQG_stats_##name); \
112 static int bfqg_stats_##name(struct bfqg_stats *stats) \
114 return (stats->flags & (1 << BFQG_stats_##name)) != 0; \
117 BFQG_FLAG_FNS(waiting)
118 BFQG_FLAG_FNS(idling
)
122 /* This should be called with the scheduler lock held. */
123 static void bfqg_stats_update_group_wait_time(struct bfqg_stats
*stats
)
127 if (!bfqg_stats_waiting(stats
))
130 now
= ktime_get_ns();
131 if (now
> stats
->start_group_wait_time
)
132 bfq_stat_add(&stats
->group_wait_time
,
133 now
- stats
->start_group_wait_time
);
134 bfqg_stats_clear_waiting(stats
);
137 /* This should be called with the scheduler lock held. */
138 static void bfqg_stats_set_start_group_wait_time(struct bfq_group
*bfqg
,
139 struct bfq_group
*curr_bfqg
)
141 struct bfqg_stats
*stats
= &bfqg
->stats
;
143 if (bfqg_stats_waiting(stats
))
145 if (bfqg
== curr_bfqg
)
147 stats
->start_group_wait_time
= ktime_get_ns();
148 bfqg_stats_mark_waiting(stats
);
151 /* This should be called with the scheduler lock held. */
152 static void bfqg_stats_end_empty_time(struct bfqg_stats
*stats
)
156 if (!bfqg_stats_empty(stats
))
159 now
= ktime_get_ns();
160 if (now
> stats
->start_empty_time
)
161 bfq_stat_add(&stats
->empty_time
,
162 now
- stats
->start_empty_time
);
163 bfqg_stats_clear_empty(stats
);
166 void bfqg_stats_update_dequeue(struct bfq_group
*bfqg
)
168 bfq_stat_add(&bfqg
->stats
.dequeue
, 1);
171 void bfqg_stats_set_start_empty_time(struct bfq_group
*bfqg
)
173 struct bfqg_stats
*stats
= &bfqg
->stats
;
175 if (blkg_rwstat_total(&stats
->queued
))
179 * group is already marked empty. This can happen if bfqq got new
180 * request in parent group and moved to this group while being added
181 * to service tree. Just ignore the event and move on.
183 if (bfqg_stats_empty(stats
))
186 stats
->start_empty_time
= ktime_get_ns();
187 bfqg_stats_mark_empty(stats
);
190 void bfqg_stats_update_idle_time(struct bfq_group
*bfqg
)
192 struct bfqg_stats
*stats
= &bfqg
->stats
;
194 if (bfqg_stats_idling(stats
)) {
195 u64 now
= ktime_get_ns();
197 if (now
> stats
->start_idle_time
)
198 bfq_stat_add(&stats
->idle_time
,
199 now
- stats
->start_idle_time
);
200 bfqg_stats_clear_idling(stats
);
204 void bfqg_stats_set_start_idle_time(struct bfq_group
*bfqg
)
206 struct bfqg_stats
*stats
= &bfqg
->stats
;
208 stats
->start_idle_time
= ktime_get_ns();
209 bfqg_stats_mark_idling(stats
);
212 void bfqg_stats_update_avg_queue_size(struct bfq_group
*bfqg
)
214 struct bfqg_stats
*stats
= &bfqg
->stats
;
216 bfq_stat_add(&stats
->avg_queue_size_sum
,
217 blkg_rwstat_total(&stats
->queued
));
218 bfq_stat_add(&stats
->avg_queue_size_samples
, 1);
219 bfqg_stats_update_group_wait_time(stats
);
222 void bfqg_stats_update_io_add(struct bfq_group
*bfqg
, struct bfq_queue
*bfqq
,
225 blkg_rwstat_add(&bfqg
->stats
.queued
, op
, 1);
226 bfqg_stats_end_empty_time(&bfqg
->stats
);
227 if (!(bfqq
== ((struct bfq_data
*)bfqg
->bfqd
)->in_service_queue
))
228 bfqg_stats_set_start_group_wait_time(bfqg
, bfqq_group(bfqq
));
231 void bfqg_stats_update_io_remove(struct bfq_group
*bfqg
, unsigned int op
)
233 blkg_rwstat_add(&bfqg
->stats
.queued
, op
, -1);
236 void bfqg_stats_update_io_merged(struct bfq_group
*bfqg
, unsigned int op
)
238 blkg_rwstat_add(&bfqg
->stats
.merged
, op
, 1);
241 void bfqg_stats_update_completion(struct bfq_group
*bfqg
, u64 start_time_ns
,
242 u64 io_start_time_ns
, unsigned int op
)
244 struct bfqg_stats
*stats
= &bfqg
->stats
;
245 u64 now
= ktime_get_ns();
247 if (now
> io_start_time_ns
)
248 blkg_rwstat_add(&stats
->service_time
, op
,
249 now
- io_start_time_ns
);
250 if (io_start_time_ns
> start_time_ns
)
251 blkg_rwstat_add(&stats
->wait_time
, op
,
252 io_start_time_ns
- start_time_ns
);
255 #else /* CONFIG_BFQ_CGROUP_DEBUG */
257 void bfqg_stats_update_io_add(struct bfq_group
*bfqg
, struct bfq_queue
*bfqq
,
259 void bfqg_stats_update_io_remove(struct bfq_group
*bfqg
, unsigned int op
) { }
260 void bfqg_stats_update_io_merged(struct bfq_group
*bfqg
, unsigned int op
) { }
261 void bfqg_stats_update_completion(struct bfq_group
*bfqg
, u64 start_time_ns
,
262 u64 io_start_time_ns
, unsigned int op
) { }
263 void bfqg_stats_update_dequeue(struct bfq_group
*bfqg
) { }
264 void bfqg_stats_set_start_empty_time(struct bfq_group
*bfqg
) { }
265 void bfqg_stats_update_idle_time(struct bfq_group
*bfqg
) { }
266 void bfqg_stats_set_start_idle_time(struct bfq_group
*bfqg
) { }
267 void bfqg_stats_update_avg_queue_size(struct bfq_group
*bfqg
) { }
269 #endif /* CONFIG_BFQ_CGROUP_DEBUG */
271 #ifdef CONFIG_BFQ_GROUP_IOSCHED
274 * blk-cgroup policy-related handlers
275 * The following functions help in converting between blk-cgroup
276 * internal structures and BFQ-specific structures.
279 static struct bfq_group
*pd_to_bfqg(struct blkg_policy_data
*pd
)
281 return pd
? container_of(pd
, struct bfq_group
, pd
) : NULL
;
284 struct blkcg_gq
*bfqg_to_blkg(struct bfq_group
*bfqg
)
286 return pd_to_blkg(&bfqg
->pd
);
289 static struct bfq_group
*blkg_to_bfqg(struct blkcg_gq
*blkg
)
291 return pd_to_bfqg(blkg_to_pd(blkg
, &blkcg_policy_bfq
));
296 * The following functions help in navigating the bfq_group hierarchy
297 * by allowing to find the parent of a bfq_group or the bfq_group
298 * associated to a bfq_queue.
301 static struct bfq_group
*bfqg_parent(struct bfq_group
*bfqg
)
303 struct blkcg_gq
*pblkg
= bfqg_to_blkg(bfqg
)->parent
;
305 return pblkg
? blkg_to_bfqg(pblkg
) : NULL
;
308 struct bfq_group
*bfqq_group(struct bfq_queue
*bfqq
)
310 struct bfq_entity
*group_entity
= bfqq
->entity
.parent
;
312 return group_entity
? container_of(group_entity
, struct bfq_group
,
314 bfqq
->bfqd
->root_group
;
318 * The following two functions handle get and put of a bfq_group by
319 * wrapping the related blk-cgroup hooks.
322 static void bfqg_get(struct bfq_group
*bfqg
)
327 static void bfqg_put(struct bfq_group
*bfqg
)
335 static void bfqg_and_blkg_get(struct bfq_group
*bfqg
)
337 /* see comments in bfq_bic_update_cgroup for why refcounting bfqg */
340 blkg_get(bfqg_to_blkg(bfqg
));
343 void bfqg_and_blkg_put(struct bfq_group
*bfqg
)
345 blkg_put(bfqg_to_blkg(bfqg
));
350 void bfqg_stats_update_legacy_io(struct request_queue
*q
, struct request
*rq
)
352 struct bfq_group
*bfqg
= blkg_to_bfqg(rq
->bio
->bi_blkg
);
357 blkg_rwstat_add(&bfqg
->stats
.bytes
, rq
->cmd_flags
, blk_rq_bytes(rq
));
358 blkg_rwstat_add(&bfqg
->stats
.ios
, rq
->cmd_flags
, 1);
362 static void bfqg_stats_reset(struct bfqg_stats
*stats
)
364 #ifdef CONFIG_BFQ_CGROUP_DEBUG
365 /* queued stats shouldn't be cleared */
366 blkg_rwstat_reset(&stats
->merged
);
367 blkg_rwstat_reset(&stats
->service_time
);
368 blkg_rwstat_reset(&stats
->wait_time
);
369 bfq_stat_reset(&stats
->time
);
370 bfq_stat_reset(&stats
->avg_queue_size_sum
);
371 bfq_stat_reset(&stats
->avg_queue_size_samples
);
372 bfq_stat_reset(&stats
->dequeue
);
373 bfq_stat_reset(&stats
->group_wait_time
);
374 bfq_stat_reset(&stats
->idle_time
);
375 bfq_stat_reset(&stats
->empty_time
);
380 static void bfqg_stats_add_aux(struct bfqg_stats
*to
, struct bfqg_stats
*from
)
385 #ifdef CONFIG_BFQ_CGROUP_DEBUG
386 /* queued stats shouldn't be cleared */
387 blkg_rwstat_add_aux(&to
->merged
, &from
->merged
);
388 blkg_rwstat_add_aux(&to
->service_time
, &from
->service_time
);
389 blkg_rwstat_add_aux(&to
->wait_time
, &from
->wait_time
);
390 bfq_stat_add_aux(&from
->time
, &from
->time
);
391 bfq_stat_add_aux(&to
->avg_queue_size_sum
, &from
->avg_queue_size_sum
);
392 bfq_stat_add_aux(&to
->avg_queue_size_samples
,
393 &from
->avg_queue_size_samples
);
394 bfq_stat_add_aux(&to
->dequeue
, &from
->dequeue
);
395 bfq_stat_add_aux(&to
->group_wait_time
, &from
->group_wait_time
);
396 bfq_stat_add_aux(&to
->idle_time
, &from
->idle_time
);
397 bfq_stat_add_aux(&to
->empty_time
, &from
->empty_time
);
402 * Transfer @bfqg's stats to its parent's aux counts so that the ancestors'
403 * recursive stats can still account for the amount used by this bfqg after
406 static void bfqg_stats_xfer_dead(struct bfq_group
*bfqg
)
408 struct bfq_group
*parent
;
410 if (!bfqg
) /* root_group */
413 parent
= bfqg_parent(bfqg
);
415 lockdep_assert_held(&bfqg_to_blkg(bfqg
)->q
->queue_lock
);
417 if (unlikely(!parent
))
420 bfqg_stats_add_aux(&parent
->stats
, &bfqg
->stats
);
421 bfqg_stats_reset(&bfqg
->stats
);
424 void bfq_init_entity(struct bfq_entity
*entity
, struct bfq_group
*bfqg
)
426 struct bfq_queue
*bfqq
= bfq_entity_to_bfqq(entity
);
428 entity
->weight
= entity
->new_weight
;
429 entity
->orig_weight
= entity
->new_weight
;
431 bfqq
->ioprio
= bfqq
->new_ioprio
;
432 bfqq
->ioprio_class
= bfqq
->new_ioprio_class
;
434 * Make sure that bfqg and its associated blkg do not
435 * disappear before entity.
437 bfqg_and_blkg_get(bfqg
);
439 entity
->parent
= bfqg
->my_entity
; /* NULL for root group */
440 entity
->sched_data
= &bfqg
->sched_data
;
443 static void bfqg_stats_exit(struct bfqg_stats
*stats
)
445 blkg_rwstat_exit(&stats
->bytes
);
446 blkg_rwstat_exit(&stats
->ios
);
447 #ifdef CONFIG_BFQ_CGROUP_DEBUG
448 blkg_rwstat_exit(&stats
->merged
);
449 blkg_rwstat_exit(&stats
->service_time
);
450 blkg_rwstat_exit(&stats
->wait_time
);
451 blkg_rwstat_exit(&stats
->queued
);
452 bfq_stat_exit(&stats
->time
);
453 bfq_stat_exit(&stats
->avg_queue_size_sum
);
454 bfq_stat_exit(&stats
->avg_queue_size_samples
);
455 bfq_stat_exit(&stats
->dequeue
);
456 bfq_stat_exit(&stats
->group_wait_time
);
457 bfq_stat_exit(&stats
->idle_time
);
458 bfq_stat_exit(&stats
->empty_time
);
462 static int bfqg_stats_init(struct bfqg_stats
*stats
, gfp_t gfp
)
464 if (blkg_rwstat_init(&stats
->bytes
, gfp
) ||
465 blkg_rwstat_init(&stats
->ios
, gfp
))
468 #ifdef CONFIG_BFQ_CGROUP_DEBUG
469 if (blkg_rwstat_init(&stats
->merged
, gfp
) ||
470 blkg_rwstat_init(&stats
->service_time
, gfp
) ||
471 blkg_rwstat_init(&stats
->wait_time
, gfp
) ||
472 blkg_rwstat_init(&stats
->queued
, gfp
) ||
473 bfq_stat_init(&stats
->time
, gfp
) ||
474 bfq_stat_init(&stats
->avg_queue_size_sum
, gfp
) ||
475 bfq_stat_init(&stats
->avg_queue_size_samples
, gfp
) ||
476 bfq_stat_init(&stats
->dequeue
, gfp
) ||
477 bfq_stat_init(&stats
->group_wait_time
, gfp
) ||
478 bfq_stat_init(&stats
->idle_time
, gfp
) ||
479 bfq_stat_init(&stats
->empty_time
, gfp
))
486 bfqg_stats_exit(stats
);
490 static struct bfq_group_data
*cpd_to_bfqgd(struct blkcg_policy_data
*cpd
)
492 return cpd
? container_of(cpd
, struct bfq_group_data
, pd
) : NULL
;
495 static struct bfq_group_data
*blkcg_to_bfqgd(struct blkcg
*blkcg
)
497 return cpd_to_bfqgd(blkcg_to_cpd(blkcg
, &blkcg_policy_bfq
));
500 static struct blkcg_policy_data
*bfq_cpd_alloc(gfp_t gfp
)
502 struct bfq_group_data
*bgd
;
504 bgd
= kzalloc(sizeof(*bgd
), gfp
);
510 static void bfq_cpd_init(struct blkcg_policy_data
*cpd
)
512 struct bfq_group_data
*d
= cpd_to_bfqgd(cpd
);
514 d
->weight
= cgroup_subsys_on_dfl(io_cgrp_subsys
) ?
515 CGROUP_WEIGHT_DFL
: BFQ_WEIGHT_LEGACY_DFL
;
518 static void bfq_cpd_free(struct blkcg_policy_data
*cpd
)
520 kfree(cpd_to_bfqgd(cpd
));
523 static struct blkg_policy_data
*bfq_pd_alloc(gfp_t gfp
, struct request_queue
*q
,
526 struct bfq_group
*bfqg
;
528 bfqg
= kzalloc_node(sizeof(*bfqg
), gfp
, q
->node
);
532 if (bfqg_stats_init(&bfqg
->stats
, gfp
)) {
537 /* see comments in bfq_bic_update_cgroup for why refcounting */
542 static void bfq_pd_init(struct blkg_policy_data
*pd
)
544 struct blkcg_gq
*blkg
= pd_to_blkg(pd
);
545 struct bfq_group
*bfqg
= blkg_to_bfqg(blkg
);
546 struct bfq_data
*bfqd
= blkg
->q
->elevator
->elevator_data
;
547 struct bfq_entity
*entity
= &bfqg
->entity
;
548 struct bfq_group_data
*d
= blkcg_to_bfqgd(blkg
->blkcg
);
550 entity
->orig_weight
= entity
->weight
= entity
->new_weight
= d
->weight
;
551 entity
->my_sched_data
= &bfqg
->sched_data
;
552 entity
->last_bfqq_created
= NULL
;
554 bfqg
->my_entity
= entity
; /*
555 * the root_group's will be set to NULL
556 * in bfq_init_queue()
559 bfqg
->active_entities
= 0;
560 bfqg
->rq_pos_tree
= RB_ROOT
;
563 static void bfq_pd_free(struct blkg_policy_data
*pd
)
565 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
567 bfqg_stats_exit(&bfqg
->stats
);
571 static void bfq_pd_reset_stats(struct blkg_policy_data
*pd
)
573 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
575 bfqg_stats_reset(&bfqg
->stats
);
578 static void bfq_group_set_parent(struct bfq_group
*bfqg
,
579 struct bfq_group
*parent
)
581 struct bfq_entity
*entity
;
583 entity
= &bfqg
->entity
;
584 entity
->parent
= parent
->my_entity
;
585 entity
->sched_data
= &parent
->sched_data
;
588 static struct bfq_group
*bfq_lookup_bfqg(struct bfq_data
*bfqd
,
591 struct blkcg_gq
*blkg
;
593 blkg
= blkg_lookup(blkcg
, bfqd
->queue
);
595 return blkg_to_bfqg(blkg
);
599 struct bfq_group
*bfq_find_set_group(struct bfq_data
*bfqd
,
602 struct bfq_group
*bfqg
, *parent
;
603 struct bfq_entity
*entity
;
605 bfqg
= bfq_lookup_bfqg(bfqd
, blkcg
);
611 * Update chain of bfq_groups as we might be handling a leaf group
612 * which, along with some of its relatives, has not been hooked yet
613 * to the private hierarchy of BFQ.
615 entity
= &bfqg
->entity
;
616 for_each_entity(entity
) {
617 struct bfq_group
*curr_bfqg
= container_of(entity
,
618 struct bfq_group
, entity
);
619 if (curr_bfqg
!= bfqd
->root_group
) {
620 parent
= bfqg_parent(curr_bfqg
);
622 parent
= bfqd
->root_group
;
623 bfq_group_set_parent(curr_bfqg
, parent
);
631 * bfq_bfqq_move - migrate @bfqq to @bfqg.
632 * @bfqd: queue descriptor.
633 * @bfqq: the queue to move.
634 * @bfqg: the group to move to.
636 * Move @bfqq to @bfqg, deactivating it from its old group and reactivating
637 * it on the new one. Avoid putting the entity on the old group idle tree.
639 * Must be called under the scheduler lock, to make sure that the blkg
640 * owning @bfqg does not disappear (see comments in
641 * bfq_bic_update_cgroup on guaranteeing the consistency of blkg
644 void bfq_bfqq_move(struct bfq_data
*bfqd
, struct bfq_queue
*bfqq
,
645 struct bfq_group
*bfqg
)
647 struct bfq_entity
*entity
= &bfqq
->entity
;
650 * Get extra reference to prevent bfqq from being freed in
651 * next possible expire or deactivate.
655 /* If bfqq is empty, then bfq_bfqq_expire also invokes
656 * bfq_del_bfqq_busy, thereby removing bfqq and its entity
657 * from data structures related to current group. Otherwise we
658 * need to remove bfqq explicitly with bfq_deactivate_bfqq, as
661 if (bfqq
== bfqd
->in_service_queue
)
662 bfq_bfqq_expire(bfqd
, bfqd
->in_service_queue
,
663 false, BFQQE_PREEMPTED
);
665 if (bfq_bfqq_busy(bfqq
))
666 bfq_deactivate_bfqq(bfqd
, bfqq
, false, false);
667 else if (entity
->on_st_or_in_serv
)
668 bfq_put_idle_entity(bfq_entity_service_tree(entity
), entity
);
669 bfqg_and_blkg_put(bfqq_group(bfqq
));
671 if (entity
->parent
&&
672 entity
->parent
->last_bfqq_created
== bfqq
)
673 entity
->parent
->last_bfqq_created
= NULL
;
674 else if (bfqd
->last_bfqq_created
== bfqq
)
675 bfqd
->last_bfqq_created
= NULL
;
677 entity
->parent
= bfqg
->my_entity
;
678 entity
->sched_data
= &bfqg
->sched_data
;
679 /* pin down bfqg and its associated blkg */
680 bfqg_and_blkg_get(bfqg
);
682 if (bfq_bfqq_busy(bfqq
)) {
683 if (unlikely(!bfqd
->nonrot_with_queueing
))
684 bfq_pos_tree_add_move(bfqd
, bfqq
);
685 bfq_activate_bfqq(bfqd
, bfqq
);
688 if (!bfqd
->in_service_queue
&& !bfqd
->rq_in_driver
)
689 bfq_schedule_dispatch(bfqd
);
690 /* release extra ref taken above, bfqq may happen to be freed now */
695 * __bfq_bic_change_cgroup - move @bic to @cgroup.
696 * @bfqd: the queue descriptor.
697 * @bic: the bic to move.
698 * @blkcg: the blk-cgroup to move to.
700 * Move bic to blkcg, assuming that bfqd->lock is held; which makes
701 * sure that the reference to cgroup is valid across the call (see
702 * comments in bfq_bic_update_cgroup on this issue)
704 * NOTE: an alternative approach might have been to store the current
705 * cgroup in bfqq and getting a reference to it, reducing the lookup
706 * time here, at the price of slightly more complex code.
708 static struct bfq_group
*__bfq_bic_change_cgroup(struct bfq_data
*bfqd
,
709 struct bfq_io_cq
*bic
,
712 struct bfq_queue
*async_bfqq
= bic_to_bfqq(bic
, 0);
713 struct bfq_queue
*sync_bfqq
= bic_to_bfqq(bic
, 1);
714 struct bfq_group
*bfqg
;
715 struct bfq_entity
*entity
;
717 bfqg
= bfq_find_set_group(bfqd
, blkcg
);
720 bfqg
= bfqd
->root_group
;
723 entity
= &async_bfqq
->entity
;
725 if (entity
->sched_data
!= &bfqg
->sched_data
) {
726 bic_set_bfqq(bic
, NULL
, 0);
727 bfq_release_process_ref(bfqd
, async_bfqq
);
732 entity
= &sync_bfqq
->entity
;
733 if (entity
->sched_data
!= &bfqg
->sched_data
)
734 bfq_bfqq_move(bfqd
, sync_bfqq
, bfqg
);
740 void bfq_bic_update_cgroup(struct bfq_io_cq
*bic
, struct bio
*bio
)
742 struct bfq_data
*bfqd
= bic_to_bfqd(bic
);
743 struct bfq_group
*bfqg
= NULL
;
747 serial_nr
= __bio_blkcg(bio
)->css
.serial_nr
;
750 * Check whether blkcg has changed. The condition may trigger
751 * spuriously on a newly created cic but there's no harm.
753 if (unlikely(!bfqd
) || likely(bic
->blkcg_serial_nr
== serial_nr
))
756 bfqg
= __bfq_bic_change_cgroup(bfqd
, bic
, __bio_blkcg(bio
));
758 * Update blkg_path for bfq_log_* functions. We cache this
759 * path, and update it here, for the following
760 * reasons. Operations on blkg objects in blk-cgroup are
761 * protected with the request_queue lock, and not with the
762 * lock that protects the instances of this scheduler
763 * (bfqd->lock). This exposes BFQ to the following sort of
766 * The blkg_lookup performed in bfq_get_queue, protected
767 * through rcu, may happen to return the address of a copy of
768 * the original blkg. If this is the case, then the
769 * bfqg_and_blkg_get performed in bfq_get_queue, to pin down
770 * the blkg, is useless: it does not prevent blk-cgroup code
771 * from destroying both the original blkg and all objects
772 * directly or indirectly referred by the copy of the
775 * On the bright side, destroy operations on a blkg invoke, as
776 * a first step, hooks of the scheduler associated with the
777 * blkg. And these hooks are executed with bfqd->lock held for
778 * BFQ. As a consequence, for any blkg associated with the
779 * request queue this instance of the scheduler is attached
780 * to, we are guaranteed that such a blkg is not destroyed, and
781 * that all the pointers it contains are consistent, while we
782 * are holding bfqd->lock. A blkg_lookup performed with
783 * bfqd->lock held then returns a fully consistent blkg, which
784 * remains consistent until this lock is held.
786 * Thanks to the last fact, and to the fact that: (1) bfqg has
787 * been obtained through a blkg_lookup in the above
788 * assignment, and (2) bfqd->lock is being held, here we can
789 * safely use the policy data for the involved blkg (i.e., the
790 * field bfqg->pd) to get to the blkg associated with bfqg,
791 * and then we can safely use any field of blkg. After we
792 * release bfqd->lock, even just getting blkg through this
793 * bfqg may cause dangling references to be traversed, as
794 * bfqg->pd may not exist any more.
796 * In view of the above facts, here we cache, in the bfqg, any
797 * blkg data we may need for this bic, and for its associated
798 * bfq_queue. As of now, we need to cache only the path of the
799 * blkg, which is used in the bfq_log_* functions.
801 * Finally, note that bfqg itself needs to be protected from
802 * destruction on the blkg_free of the original blkg (which
803 * invokes bfq_pd_free). We use an additional private
804 * refcounter for bfqg, to let it disappear only after no
805 * bfq_queue refers to it any longer.
807 blkg_path(bfqg_to_blkg(bfqg
), bfqg
->blkg_path
, sizeof(bfqg
->blkg_path
));
808 bic
->blkcg_serial_nr
= serial_nr
;
814 * bfq_flush_idle_tree - deactivate any entity on the idle tree of @st.
815 * @st: the service tree being flushed.
817 static void bfq_flush_idle_tree(struct bfq_service_tree
*st
)
819 struct bfq_entity
*entity
= st
->first_idle
;
821 for (; entity
; entity
= st
->first_idle
)
822 __bfq_deactivate_entity(entity
, false);
826 * bfq_reparent_leaf_entity - move leaf entity to the root_group.
827 * @bfqd: the device data structure with the root group.
828 * @entity: the entity to move, if entity is a leaf; or the parent entity
829 * of an active leaf entity to move, if entity is not a leaf.
831 static void bfq_reparent_leaf_entity(struct bfq_data
*bfqd
,
832 struct bfq_entity
*entity
,
835 struct bfq_queue
*bfqq
;
836 struct bfq_entity
*child_entity
= entity
;
838 while (child_entity
->my_sched_data
) { /* leaf not reached yet */
839 struct bfq_sched_data
*child_sd
= child_entity
->my_sched_data
;
840 struct bfq_service_tree
*child_st
= child_sd
->service_tree
+
842 struct rb_root
*child_active
= &child_st
->active
;
844 child_entity
= bfq_entity_of(rb_first(child_active
));
847 child_entity
= child_sd
->in_service_entity
;
850 bfqq
= bfq_entity_to_bfqq(child_entity
);
851 bfq_bfqq_move(bfqd
, bfqq
, bfqd
->root_group
);
855 * bfq_reparent_active_queues - move to the root group all active queues.
856 * @bfqd: the device data structure with the root group.
857 * @bfqg: the group to move from.
858 * @st: the service tree to start the search from.
860 static void bfq_reparent_active_queues(struct bfq_data
*bfqd
,
861 struct bfq_group
*bfqg
,
862 struct bfq_service_tree
*st
,
865 struct rb_root
*active
= &st
->active
;
866 struct bfq_entity
*entity
;
868 while ((entity
= bfq_entity_of(rb_first(active
))))
869 bfq_reparent_leaf_entity(bfqd
, entity
, ioprio_class
);
871 if (bfqg
->sched_data
.in_service_entity
)
872 bfq_reparent_leaf_entity(bfqd
,
873 bfqg
->sched_data
.in_service_entity
,
878 * bfq_pd_offline - deactivate the entity associated with @pd,
879 * and reparent its children entities.
880 * @pd: descriptor of the policy going offline.
882 * blkio already grabs the queue_lock for us, so no need to use
885 static void bfq_pd_offline(struct blkg_policy_data
*pd
)
887 struct bfq_service_tree
*st
;
888 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
889 struct bfq_data
*bfqd
= bfqg
->bfqd
;
890 struct bfq_entity
*entity
= bfqg
->my_entity
;
894 spin_lock_irqsave(&bfqd
->lock
, flags
);
896 if (!entity
) /* root group */
897 goto put_async_queues
;
900 * Empty all service_trees belonging to this group before
901 * deactivating the group itself.
903 for (i
= 0; i
< BFQ_IOPRIO_CLASSES
; i
++) {
904 st
= bfqg
->sched_data
.service_tree
+ i
;
907 * It may happen that some queues are still active
908 * (busy) upon group destruction (if the corresponding
909 * processes have been forced to terminate). We move
910 * all the leaf entities corresponding to these queues
912 * Also, it may happen that the group has an entity
913 * in service, which is disconnected from the active
914 * tree: it must be moved, too.
915 * There is no need to put the sync queues, as the
916 * scheduler has taken no reference.
918 bfq_reparent_active_queues(bfqd
, bfqg
, st
, i
);
921 * The idle tree may still contain bfq_queues
922 * belonging to exited task because they never
923 * migrated to a different cgroup from the one being
924 * destroyed now. In addition, even
925 * bfq_reparent_active_queues() may happen to add some
926 * entities to the idle tree. It happens if, in some
927 * of the calls to bfq_bfqq_move() performed by
928 * bfq_reparent_active_queues(), the queue to move is
929 * empty and gets expired.
931 bfq_flush_idle_tree(st
);
934 __bfq_deactivate_entity(entity
, false);
937 bfq_put_async_queues(bfqd
, bfqg
);
939 spin_unlock_irqrestore(&bfqd
->lock
, flags
);
941 * @blkg is going offline and will be ignored by
942 * blkg_[rw]stat_recursive_sum(). Transfer stats to the parent so
943 * that they don't get lost. If IOs complete after this point, the
944 * stats for them will be lost. Oh well...
946 bfqg_stats_xfer_dead(bfqg
);
949 void bfq_end_wr_async(struct bfq_data
*bfqd
)
951 struct blkcg_gq
*blkg
;
953 list_for_each_entry(blkg
, &bfqd
->queue
->blkg_list
, q_node
) {
954 struct bfq_group
*bfqg
= blkg_to_bfqg(blkg
);
956 bfq_end_wr_async_queues(bfqd
, bfqg
);
958 bfq_end_wr_async_queues(bfqd
, bfqd
->root_group
);
961 static int bfq_io_show_weight_legacy(struct seq_file
*sf
, void *v
)
963 struct blkcg
*blkcg
= css_to_blkcg(seq_css(sf
));
964 struct bfq_group_data
*bfqgd
= blkcg_to_bfqgd(blkcg
);
965 unsigned int val
= 0;
970 seq_printf(sf
, "%u\n", val
);
975 static u64
bfqg_prfill_weight_device(struct seq_file
*sf
,
976 struct blkg_policy_data
*pd
, int off
)
978 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
980 if (!bfqg
->entity
.dev_weight
)
982 return __blkg_prfill_u64(sf
, pd
, bfqg
->entity
.dev_weight
);
985 static int bfq_io_show_weight(struct seq_file
*sf
, void *v
)
987 struct blkcg
*blkcg
= css_to_blkcg(seq_css(sf
));
988 struct bfq_group_data
*bfqgd
= blkcg_to_bfqgd(blkcg
);
990 seq_printf(sf
, "default %u\n", bfqgd
->weight
);
991 blkcg_print_blkgs(sf
, blkcg
, bfqg_prfill_weight_device
,
992 &blkcg_policy_bfq
, 0, false);
996 static void bfq_group_set_weight(struct bfq_group
*bfqg
, u64 weight
, u64 dev_weight
)
998 weight
= dev_weight
?: weight
;
1000 bfqg
->entity
.dev_weight
= dev_weight
;
1002 * Setting the prio_changed flag of the entity
1003 * to 1 with new_weight == weight would re-set
1004 * the value of the weight to its ioprio mapping.
1005 * Set the flag only if necessary.
1007 if ((unsigned short)weight
!= bfqg
->entity
.new_weight
) {
1008 bfqg
->entity
.new_weight
= (unsigned short)weight
;
1010 * Make sure that the above new value has been
1011 * stored in bfqg->entity.new_weight before
1012 * setting the prio_changed flag. In fact,
1013 * this flag may be read asynchronously (in
1014 * critical sections protected by a different
1015 * lock than that held here), and finding this
1016 * flag set may cause the execution of the code
1017 * for updating parameters whose value may
1018 * depend also on bfqg->entity.new_weight (in
1019 * __bfq_entity_update_weight_prio).
1020 * This barrier makes sure that the new value
1021 * of bfqg->entity.new_weight is correctly
1022 * seen in that code.
1025 bfqg
->entity
.prio_changed
= 1;
1029 static int bfq_io_set_weight_legacy(struct cgroup_subsys_state
*css
,
1030 struct cftype
*cftype
,
1033 struct blkcg
*blkcg
= css_to_blkcg(css
);
1034 struct bfq_group_data
*bfqgd
= blkcg_to_bfqgd(blkcg
);
1035 struct blkcg_gq
*blkg
;
1038 if (val
< BFQ_MIN_WEIGHT
|| val
> BFQ_MAX_WEIGHT
)
1042 spin_lock_irq(&blkcg
->lock
);
1043 bfqgd
->weight
= (unsigned short)val
;
1044 hlist_for_each_entry(blkg
, &blkcg
->blkg_list
, blkcg_node
) {
1045 struct bfq_group
*bfqg
= blkg_to_bfqg(blkg
);
1048 bfq_group_set_weight(bfqg
, val
, 0);
1050 spin_unlock_irq(&blkcg
->lock
);
1055 static ssize_t
bfq_io_set_device_weight(struct kernfs_open_file
*of
,
1056 char *buf
, size_t nbytes
,
1060 struct blkg_conf_ctx ctx
;
1061 struct blkcg
*blkcg
= css_to_blkcg(of_css(of
));
1062 struct bfq_group
*bfqg
;
1065 ret
= blkg_conf_prep(blkcg
, &blkcg_policy_bfq
, buf
, &ctx
);
1069 if (sscanf(ctx
.body
, "%llu", &v
) == 1) {
1070 /* require "default" on dfl */
1074 } else if (!strcmp(strim(ctx
.body
), "default")) {
1081 bfqg
= blkg_to_bfqg(ctx
.blkg
);
1084 if (!v
|| (v
>= BFQ_MIN_WEIGHT
&& v
<= BFQ_MAX_WEIGHT
)) {
1085 bfq_group_set_weight(bfqg
, bfqg
->entity
.weight
, v
);
1089 blkg_conf_finish(&ctx
);
1090 return ret
?: nbytes
;
1093 static ssize_t
bfq_io_set_weight(struct kernfs_open_file
*of
,
1094 char *buf
, size_t nbytes
,
1103 /* "WEIGHT" or "default WEIGHT" sets the default weight */
1104 v
= simple_strtoull(buf
, &endp
, 0);
1105 if (*endp
== '\0' || sscanf(buf
, "default %llu", &v
) == 1) {
1106 ret
= bfq_io_set_weight_legacy(of_css(of
), NULL
, v
);
1107 return ret
?: nbytes
;
1110 return bfq_io_set_device_weight(of
, buf
, nbytes
, off
);
1113 static int bfqg_print_rwstat(struct seq_file
*sf
, void *v
)
1115 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)), blkg_prfill_rwstat
,
1116 &blkcg_policy_bfq
, seq_cft(sf
)->private, true);
1120 static u64
bfqg_prfill_rwstat_recursive(struct seq_file
*sf
,
1121 struct blkg_policy_data
*pd
, int off
)
1123 struct blkg_rwstat_sample sum
;
1125 blkg_rwstat_recursive_sum(pd_to_blkg(pd
), &blkcg_policy_bfq
, off
, &sum
);
1126 return __blkg_prfill_rwstat(sf
, pd
, &sum
);
1129 static int bfqg_print_rwstat_recursive(struct seq_file
*sf
, void *v
)
1131 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1132 bfqg_prfill_rwstat_recursive
, &blkcg_policy_bfq
,
1133 seq_cft(sf
)->private, true);
1137 #ifdef CONFIG_BFQ_CGROUP_DEBUG
1138 static int bfqg_print_stat(struct seq_file
*sf
, void *v
)
1140 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)), blkg_prfill_stat
,
1141 &blkcg_policy_bfq
, seq_cft(sf
)->private, false);
1145 static u64
bfqg_prfill_stat_recursive(struct seq_file
*sf
,
1146 struct blkg_policy_data
*pd
, int off
)
1148 struct blkcg_gq
*blkg
= pd_to_blkg(pd
);
1149 struct blkcg_gq
*pos_blkg
;
1150 struct cgroup_subsys_state
*pos_css
;
1153 lockdep_assert_held(&blkg
->q
->queue_lock
);
1156 blkg_for_each_descendant_pre(pos_blkg
, pos_css
, blkg
) {
1157 struct bfq_stat
*stat
;
1159 if (!pos_blkg
->online
)
1162 stat
= (void *)blkg_to_pd(pos_blkg
, &blkcg_policy_bfq
) + off
;
1163 sum
+= bfq_stat_read(stat
) + atomic64_read(&stat
->aux_cnt
);
1167 return __blkg_prfill_u64(sf
, pd
, sum
);
1170 static int bfqg_print_stat_recursive(struct seq_file
*sf
, void *v
)
1172 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1173 bfqg_prfill_stat_recursive
, &blkcg_policy_bfq
,
1174 seq_cft(sf
)->private, false);
1178 static u64
bfqg_prfill_sectors(struct seq_file
*sf
, struct blkg_policy_data
*pd
,
1181 struct bfq_group
*bfqg
= blkg_to_bfqg(pd
->blkg
);
1182 u64 sum
= blkg_rwstat_total(&bfqg
->stats
.bytes
);
1184 return __blkg_prfill_u64(sf
, pd
, sum
>> 9);
1187 static int bfqg_print_stat_sectors(struct seq_file
*sf
, void *v
)
1189 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1190 bfqg_prfill_sectors
, &blkcg_policy_bfq
, 0, false);
1194 static u64
bfqg_prfill_sectors_recursive(struct seq_file
*sf
,
1195 struct blkg_policy_data
*pd
, int off
)
1197 struct blkg_rwstat_sample tmp
;
1199 blkg_rwstat_recursive_sum(pd
->blkg
, &blkcg_policy_bfq
,
1200 offsetof(struct bfq_group
, stats
.bytes
), &tmp
);
1202 return __blkg_prfill_u64(sf
, pd
,
1203 (tmp
.cnt
[BLKG_RWSTAT_READ
] + tmp
.cnt
[BLKG_RWSTAT_WRITE
]) >> 9);
1206 static int bfqg_print_stat_sectors_recursive(struct seq_file
*sf
, void *v
)
1208 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1209 bfqg_prfill_sectors_recursive
, &blkcg_policy_bfq
, 0,
1214 static u64
bfqg_prfill_avg_queue_size(struct seq_file
*sf
,
1215 struct blkg_policy_data
*pd
, int off
)
1217 struct bfq_group
*bfqg
= pd_to_bfqg(pd
);
1218 u64 samples
= bfq_stat_read(&bfqg
->stats
.avg_queue_size_samples
);
1222 v
= bfq_stat_read(&bfqg
->stats
.avg_queue_size_sum
);
1223 v
= div64_u64(v
, samples
);
1225 __blkg_prfill_u64(sf
, pd
, v
);
1229 /* print avg_queue_size */
1230 static int bfqg_print_avg_queue_size(struct seq_file
*sf
, void *v
)
1232 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
1233 bfqg_prfill_avg_queue_size
, &blkcg_policy_bfq
,
1237 #endif /* CONFIG_BFQ_CGROUP_DEBUG */
1239 struct bfq_group
*bfq_create_group_hierarchy(struct bfq_data
*bfqd
, int node
)
1243 ret
= blkcg_activate_policy(bfqd
->queue
, &blkcg_policy_bfq
);
1247 return blkg_to_bfqg(bfqd
->queue
->root_blkg
);
1250 struct blkcg_policy blkcg_policy_bfq
= {
1251 .dfl_cftypes
= bfq_blkg_files
,
1252 .legacy_cftypes
= bfq_blkcg_legacy_files
,
1254 .cpd_alloc_fn
= bfq_cpd_alloc
,
1255 .cpd_init_fn
= bfq_cpd_init
,
1256 .cpd_bind_fn
= bfq_cpd_init
,
1257 .cpd_free_fn
= bfq_cpd_free
,
1259 .pd_alloc_fn
= bfq_pd_alloc
,
1260 .pd_init_fn
= bfq_pd_init
,
1261 .pd_offline_fn
= bfq_pd_offline
,
1262 .pd_free_fn
= bfq_pd_free
,
1263 .pd_reset_stats_fn
= bfq_pd_reset_stats
,
1266 struct cftype bfq_blkcg_legacy_files
[] = {
1268 .name
= "bfq.weight",
1269 .flags
= CFTYPE_NOT_ON_ROOT
,
1270 .seq_show
= bfq_io_show_weight_legacy
,
1271 .write_u64
= bfq_io_set_weight_legacy
,
1274 .name
= "bfq.weight_device",
1275 .flags
= CFTYPE_NOT_ON_ROOT
,
1276 .seq_show
= bfq_io_show_weight
,
1277 .write
= bfq_io_set_weight
,
1280 /* statistics, covers only the tasks in the bfqg */
1282 .name
= "bfq.io_service_bytes",
1283 .private = offsetof(struct bfq_group
, stats
.bytes
),
1284 .seq_show
= bfqg_print_rwstat
,
1287 .name
= "bfq.io_serviced",
1288 .private = offsetof(struct bfq_group
, stats
.ios
),
1289 .seq_show
= bfqg_print_rwstat
,
1291 #ifdef CONFIG_BFQ_CGROUP_DEBUG
1294 .private = offsetof(struct bfq_group
, stats
.time
),
1295 .seq_show
= bfqg_print_stat
,
1298 .name
= "bfq.sectors",
1299 .seq_show
= bfqg_print_stat_sectors
,
1302 .name
= "bfq.io_service_time",
1303 .private = offsetof(struct bfq_group
, stats
.service_time
),
1304 .seq_show
= bfqg_print_rwstat
,
1307 .name
= "bfq.io_wait_time",
1308 .private = offsetof(struct bfq_group
, stats
.wait_time
),
1309 .seq_show
= bfqg_print_rwstat
,
1312 .name
= "bfq.io_merged",
1313 .private = offsetof(struct bfq_group
, stats
.merged
),
1314 .seq_show
= bfqg_print_rwstat
,
1317 .name
= "bfq.io_queued",
1318 .private = offsetof(struct bfq_group
, stats
.queued
),
1319 .seq_show
= bfqg_print_rwstat
,
1321 #endif /* CONFIG_BFQ_CGROUP_DEBUG */
1323 /* the same statistics which cover the bfqg and its descendants */
1325 .name
= "bfq.io_service_bytes_recursive",
1326 .private = offsetof(struct bfq_group
, stats
.bytes
),
1327 .seq_show
= bfqg_print_rwstat_recursive
,
1330 .name
= "bfq.io_serviced_recursive",
1331 .private = offsetof(struct bfq_group
, stats
.ios
),
1332 .seq_show
= bfqg_print_rwstat_recursive
,
1334 #ifdef CONFIG_BFQ_CGROUP_DEBUG
1336 .name
= "bfq.time_recursive",
1337 .private = offsetof(struct bfq_group
, stats
.time
),
1338 .seq_show
= bfqg_print_stat_recursive
,
1341 .name
= "bfq.sectors_recursive",
1342 .seq_show
= bfqg_print_stat_sectors_recursive
,
1345 .name
= "bfq.io_service_time_recursive",
1346 .private = offsetof(struct bfq_group
, stats
.service_time
),
1347 .seq_show
= bfqg_print_rwstat_recursive
,
1350 .name
= "bfq.io_wait_time_recursive",
1351 .private = offsetof(struct bfq_group
, stats
.wait_time
),
1352 .seq_show
= bfqg_print_rwstat_recursive
,
1355 .name
= "bfq.io_merged_recursive",
1356 .private = offsetof(struct bfq_group
, stats
.merged
),
1357 .seq_show
= bfqg_print_rwstat_recursive
,
1360 .name
= "bfq.io_queued_recursive",
1361 .private = offsetof(struct bfq_group
, stats
.queued
),
1362 .seq_show
= bfqg_print_rwstat_recursive
,
1365 .name
= "bfq.avg_queue_size",
1366 .seq_show
= bfqg_print_avg_queue_size
,
1369 .name
= "bfq.group_wait_time",
1370 .private = offsetof(struct bfq_group
, stats
.group_wait_time
),
1371 .seq_show
= bfqg_print_stat
,
1374 .name
= "bfq.idle_time",
1375 .private = offsetof(struct bfq_group
, stats
.idle_time
),
1376 .seq_show
= bfqg_print_stat
,
1379 .name
= "bfq.empty_time",
1380 .private = offsetof(struct bfq_group
, stats
.empty_time
),
1381 .seq_show
= bfqg_print_stat
,
1384 .name
= "bfq.dequeue",
1385 .private = offsetof(struct bfq_group
, stats
.dequeue
),
1386 .seq_show
= bfqg_print_stat
,
1388 #endif /* CONFIG_BFQ_CGROUP_DEBUG */
1392 struct cftype bfq_blkg_files
[] = {
1394 .name
= "bfq.weight",
1395 .flags
= CFTYPE_NOT_ON_ROOT
,
1396 .seq_show
= bfq_io_show_weight
,
1397 .write
= bfq_io_set_weight
,
1402 #else /* CONFIG_BFQ_GROUP_IOSCHED */
1404 void bfq_bfqq_move(struct bfq_data
*bfqd
, struct bfq_queue
*bfqq
,
1405 struct bfq_group
*bfqg
) {}
1407 void bfq_init_entity(struct bfq_entity
*entity
, struct bfq_group
*bfqg
)
1409 struct bfq_queue
*bfqq
= bfq_entity_to_bfqq(entity
);
1411 entity
->weight
= entity
->new_weight
;
1412 entity
->orig_weight
= entity
->new_weight
;
1414 bfqq
->ioprio
= bfqq
->new_ioprio
;
1415 bfqq
->ioprio_class
= bfqq
->new_ioprio_class
;
1417 entity
->sched_data
= &bfqg
->sched_data
;
1420 void bfq_bic_update_cgroup(struct bfq_io_cq
*bic
, struct bio
*bio
) {}
1422 void bfq_end_wr_async(struct bfq_data
*bfqd
)
1424 bfq_end_wr_async_queues(bfqd
, bfqd
->root_group
);
1427 struct bfq_group
*bfq_find_set_group(struct bfq_data
*bfqd
, struct blkcg
*blkcg
)
1429 return bfqd
->root_group
;
1432 struct bfq_group
*bfqq_group(struct bfq_queue
*bfqq
)
1434 return bfqq
->bfqd
->root_group
;
1437 void bfqg_and_blkg_get(struct bfq_group
*bfqg
) {}
1439 void bfqg_and_blkg_put(struct bfq_group
*bfqg
) {}
1441 struct bfq_group
*bfq_create_group_hierarchy(struct bfq_data
*bfqd
, int node
)
1443 struct bfq_group
*bfqg
;
1446 bfqg
= kmalloc_node(sizeof(*bfqg
), GFP_KERNEL
| __GFP_ZERO
, node
);
1450 for (i
= 0; i
< BFQ_IOPRIO_CLASSES
; i
++)
1451 bfqg
->sched_data
.service_tree
[i
] = BFQ_SERVICE_TREE_INIT
;
1455 #endif /* CONFIG_BFQ_GROUP_IOSCHED */