2 * Interface for controlling IO bandwidth on a request queue
4 * Copyright (C) 2010 Vivek Goyal <vgoyal@redhat.com>
7 #include <linux/module.h>
8 #include <linux/slab.h>
9 #include <linux/blkdev.h>
10 #include <linux/bio.h>
11 #include <linux/blktrace_api.h>
12 #include <linux/blk-cgroup.h>
15 /* Max dispatch from a group in 1 round */
16 static int throtl_grp_quantum
= 8;
18 /* Total max dispatch from all groups in one round */
19 static int throtl_quantum
= 32;
21 /* Throttling is performed over 100ms slice and after that slice is renewed */
22 static unsigned long throtl_slice
= HZ
/10; /* 100 ms */
24 static struct blkcg_policy blkcg_policy_throtl
;
26 /* A workqueue to queue throttle related work */
27 static struct workqueue_struct
*kthrotld_workqueue
;
30 * To implement hierarchical throttling, throtl_grps form a tree and bios
31 * are dispatched upwards level by level until they reach the top and get
32 * issued. When dispatching bios from the children and local group at each
33 * level, if the bios are dispatched into a single bio_list, there's a risk
34 * of a local or child group which can queue many bios at once filling up
35 * the list starving others.
37 * To avoid such starvation, dispatched bios are queued separately
38 * according to where they came from. When they are again dispatched to
39 * the parent, they're popped in round-robin order so that no single source
40 * hogs the dispatch window.
42 * throtl_qnode is used to keep the queued bios separated by their sources.
43 * Bios are queued to throtl_qnode which in turn is queued to
44 * throtl_service_queue and then dispatched in round-robin order.
46 * It's also used to track the reference counts on blkg's. A qnode always
47 * belongs to a throtl_grp and gets queued on itself or the parent, so
48 * incrementing the reference of the associated throtl_grp when a qnode is
49 * queued and decrementing when dequeued is enough to keep the whole blkg
50 * tree pinned while bios are in flight.
53 struct list_head node
; /* service_queue->queued[] */
54 struct bio_list bios
; /* queued bios */
55 struct throtl_grp
*tg
; /* tg this qnode belongs to */
58 struct throtl_service_queue
{
59 struct throtl_service_queue
*parent_sq
; /* the parent service_queue */
62 * Bios queued directly to this service_queue or dispatched from
63 * children throtl_grp's.
65 struct list_head queued
[2]; /* throtl_qnode [READ/WRITE] */
66 unsigned int nr_queued
[2]; /* number of queued bios */
69 * RB tree of active children throtl_grp's, which are sorted by
72 struct rb_root pending_tree
; /* RB tree of active tgs */
73 struct rb_node
*first_pending
; /* first node in the tree */
74 unsigned int nr_pending
; /* # queued in the tree */
75 unsigned long first_pending_disptime
; /* disptime of the first tg */
76 struct timer_list pending_timer
; /* fires on first_pending_disptime */
80 THROTL_TG_PENDING
= 1 << 0, /* on parent's pending tree */
81 THROTL_TG_WAS_EMPTY
= 1 << 1, /* bio_lists[] became non-empty */
84 #define rb_entry_tg(node) rb_entry((node), struct throtl_grp, rb_node)
93 /* must be the first member */
94 struct blkg_policy_data pd
;
96 /* active throtl group service_queue member */
97 struct rb_node rb_node
;
99 /* throtl_data this group belongs to */
100 struct throtl_data
*td
;
102 /* this group's service queue */
103 struct throtl_service_queue service_queue
;
106 * qnode_on_self is used when bios are directly queued to this
107 * throtl_grp so that local bios compete fairly with bios
108 * dispatched from children. qnode_on_parent is used when bios are
109 * dispatched from this throtl_grp into its parent and will compete
110 * with the sibling qnode_on_parents and the parent's
113 struct throtl_qnode qnode_on_self
[2];
114 struct throtl_qnode qnode_on_parent
[2];
117 * Dispatch time in jiffies. This is the estimated time when group
118 * will unthrottle and is ready to dispatch more bio. It is used as
119 * key to sort active groups in service tree.
121 unsigned long disptime
;
125 /* are there any throtl rules between this group and td? */
128 /* internally used bytes per second rate limits */
129 uint64_t bps
[2][LIMIT_CNT
];
130 /* user configured bps limits */
131 uint64_t bps_conf
[2][LIMIT_CNT
];
133 /* internally used IOPS limits */
134 unsigned int iops
[2][LIMIT_CNT
];
135 /* user configured IOPS limits */
136 unsigned int iops_conf
[2][LIMIT_CNT
];
138 /* Number of bytes disptached in current slice */
139 uint64_t bytes_disp
[2];
140 /* Number of bio's dispatched in current slice */
141 unsigned int io_disp
[2];
143 unsigned long last_low_overflow_time
[2];
145 uint64_t last_bytes_disp
[2];
146 unsigned int last_io_disp
[2];
148 unsigned long last_check_time
;
150 /* When did we start a new slice */
151 unsigned long slice_start
[2];
152 unsigned long slice_end
[2];
157 /* service tree for active throtl groups */
158 struct throtl_service_queue service_queue
;
160 struct request_queue
*queue
;
162 /* Total Number of queued bios on READ and WRITE lists */
163 unsigned int nr_queued
[2];
165 /* Work for dispatching throttled bios */
166 struct work_struct dispatch_work
;
167 unsigned int limit_index
;
168 bool limit_valid
[LIMIT_CNT
];
170 unsigned long low_upgrade_time
;
171 unsigned long low_downgrade_time
;
174 static void throtl_pending_timer_fn(unsigned long arg
);
176 static inline struct throtl_grp
*pd_to_tg(struct blkg_policy_data
*pd
)
178 return pd
? container_of(pd
, struct throtl_grp
, pd
) : NULL
;
181 static inline struct throtl_grp
*blkg_to_tg(struct blkcg_gq
*blkg
)
183 return pd_to_tg(blkg_to_pd(blkg
, &blkcg_policy_throtl
));
186 static inline struct blkcg_gq
*tg_to_blkg(struct throtl_grp
*tg
)
188 return pd_to_blkg(&tg
->pd
);
192 * sq_to_tg - return the throl_grp the specified service queue belongs to
193 * @sq: the throtl_service_queue of interest
195 * Return the throtl_grp @sq belongs to. If @sq is the top-level one
196 * embedded in throtl_data, %NULL is returned.
198 static struct throtl_grp
*sq_to_tg(struct throtl_service_queue
*sq
)
200 if (sq
&& sq
->parent_sq
)
201 return container_of(sq
, struct throtl_grp
, service_queue
);
207 * sq_to_td - return throtl_data the specified service queue belongs to
208 * @sq: the throtl_service_queue of interest
210 * A service_queue can be embedded in either a throtl_grp or throtl_data.
211 * Determine the associated throtl_data accordingly and return it.
213 static struct throtl_data
*sq_to_td(struct throtl_service_queue
*sq
)
215 struct throtl_grp
*tg
= sq_to_tg(sq
);
220 return container_of(sq
, struct throtl_data
, service_queue
);
223 static uint64_t tg_bps_limit(struct throtl_grp
*tg
, int rw
)
225 struct blkcg_gq
*blkg
= tg_to_blkg(tg
);
228 if (cgroup_subsys_on_dfl(io_cgrp_subsys
) && !blkg
->parent
)
230 ret
= tg
->bps
[rw
][tg
->td
->limit_index
];
231 if (ret
== 0 && tg
->td
->limit_index
== LIMIT_LOW
)
232 return tg
->bps
[rw
][LIMIT_MAX
];
236 static unsigned int tg_iops_limit(struct throtl_grp
*tg
, int rw
)
238 struct blkcg_gq
*blkg
= tg_to_blkg(tg
);
241 if (cgroup_subsys_on_dfl(io_cgrp_subsys
) && !blkg
->parent
)
243 ret
= tg
->iops
[rw
][tg
->td
->limit_index
];
244 if (ret
== 0 && tg
->td
->limit_index
== LIMIT_LOW
)
245 return tg
->iops
[rw
][LIMIT_MAX
];
250 * throtl_log - log debug message via blktrace
251 * @sq: the service_queue being reported
252 * @fmt: printf format string
255 * The messages are prefixed with "throtl BLKG_NAME" if @sq belongs to a
256 * throtl_grp; otherwise, just "throtl".
258 #define throtl_log(sq, fmt, args...) do { \
259 struct throtl_grp *__tg = sq_to_tg((sq)); \
260 struct throtl_data *__td = sq_to_td((sq)); \
263 if (likely(!blk_trace_note_message_enabled(__td->queue))) \
268 blkg_path(tg_to_blkg(__tg), __pbuf, sizeof(__pbuf)); \
269 blk_add_trace_msg(__td->queue, "throtl %s " fmt, __pbuf, ##args); \
271 blk_add_trace_msg(__td->queue, "throtl " fmt, ##args); \
275 static void throtl_qnode_init(struct throtl_qnode
*qn
, struct throtl_grp
*tg
)
277 INIT_LIST_HEAD(&qn
->node
);
278 bio_list_init(&qn
->bios
);
283 * throtl_qnode_add_bio - add a bio to a throtl_qnode and activate it
284 * @bio: bio being added
285 * @qn: qnode to add bio to
286 * @queued: the service_queue->queued[] list @qn belongs to
288 * Add @bio to @qn and put @qn on @queued if it's not already on.
289 * @qn->tg's reference count is bumped when @qn is activated. See the
290 * comment on top of throtl_qnode definition for details.
292 static void throtl_qnode_add_bio(struct bio
*bio
, struct throtl_qnode
*qn
,
293 struct list_head
*queued
)
295 bio_list_add(&qn
->bios
, bio
);
296 if (list_empty(&qn
->node
)) {
297 list_add_tail(&qn
->node
, queued
);
298 blkg_get(tg_to_blkg(qn
->tg
));
303 * throtl_peek_queued - peek the first bio on a qnode list
304 * @queued: the qnode list to peek
306 static struct bio
*throtl_peek_queued(struct list_head
*queued
)
308 struct throtl_qnode
*qn
= list_first_entry(queued
, struct throtl_qnode
, node
);
311 if (list_empty(queued
))
314 bio
= bio_list_peek(&qn
->bios
);
320 * throtl_pop_queued - pop the first bio form a qnode list
321 * @queued: the qnode list to pop a bio from
322 * @tg_to_put: optional out argument for throtl_grp to put
324 * Pop the first bio from the qnode list @queued. After popping, the first
325 * qnode is removed from @queued if empty or moved to the end of @queued so
326 * that the popping order is round-robin.
328 * When the first qnode is removed, its associated throtl_grp should be put
329 * too. If @tg_to_put is NULL, this function automatically puts it;
330 * otherwise, *@tg_to_put is set to the throtl_grp to put and the caller is
331 * responsible for putting it.
333 static struct bio
*throtl_pop_queued(struct list_head
*queued
,
334 struct throtl_grp
**tg_to_put
)
336 struct throtl_qnode
*qn
= list_first_entry(queued
, struct throtl_qnode
, node
);
339 if (list_empty(queued
))
342 bio
= bio_list_pop(&qn
->bios
);
345 if (bio_list_empty(&qn
->bios
)) {
346 list_del_init(&qn
->node
);
350 blkg_put(tg_to_blkg(qn
->tg
));
352 list_move_tail(&qn
->node
, queued
);
358 /* init a service_queue, assumes the caller zeroed it */
359 static void throtl_service_queue_init(struct throtl_service_queue
*sq
)
361 INIT_LIST_HEAD(&sq
->queued
[0]);
362 INIT_LIST_HEAD(&sq
->queued
[1]);
363 sq
->pending_tree
= RB_ROOT
;
364 setup_timer(&sq
->pending_timer
, throtl_pending_timer_fn
,
368 static struct blkg_policy_data
*throtl_pd_alloc(gfp_t gfp
, int node
)
370 struct throtl_grp
*tg
;
373 tg
= kzalloc_node(sizeof(*tg
), gfp
, node
);
377 throtl_service_queue_init(&tg
->service_queue
);
379 for (rw
= READ
; rw
<= WRITE
; rw
++) {
380 throtl_qnode_init(&tg
->qnode_on_self
[rw
], tg
);
381 throtl_qnode_init(&tg
->qnode_on_parent
[rw
], tg
);
384 RB_CLEAR_NODE(&tg
->rb_node
);
385 tg
->bps
[READ
][LIMIT_MAX
] = U64_MAX
;
386 tg
->bps
[WRITE
][LIMIT_MAX
] = U64_MAX
;
387 tg
->iops
[READ
][LIMIT_MAX
] = UINT_MAX
;
388 tg
->iops
[WRITE
][LIMIT_MAX
] = UINT_MAX
;
389 tg
->bps_conf
[READ
][LIMIT_MAX
] = U64_MAX
;
390 tg
->bps_conf
[WRITE
][LIMIT_MAX
] = U64_MAX
;
391 tg
->iops_conf
[READ
][LIMIT_MAX
] = UINT_MAX
;
392 tg
->iops_conf
[WRITE
][LIMIT_MAX
] = UINT_MAX
;
393 /* LIMIT_LOW will have default value 0 */
398 static void throtl_pd_init(struct blkg_policy_data
*pd
)
400 struct throtl_grp
*tg
= pd_to_tg(pd
);
401 struct blkcg_gq
*blkg
= tg_to_blkg(tg
);
402 struct throtl_data
*td
= blkg
->q
->td
;
403 struct throtl_service_queue
*sq
= &tg
->service_queue
;
406 * If on the default hierarchy, we switch to properly hierarchical
407 * behavior where limits on a given throtl_grp are applied to the
408 * whole subtree rather than just the group itself. e.g. If 16M
409 * read_bps limit is set on the root group, the whole system can't
410 * exceed 16M for the device.
412 * If not on the default hierarchy, the broken flat hierarchy
413 * behavior is retained where all throtl_grps are treated as if
414 * they're all separate root groups right below throtl_data.
415 * Limits of a group don't interact with limits of other groups
416 * regardless of the position of the group in the hierarchy.
418 sq
->parent_sq
= &td
->service_queue
;
419 if (cgroup_subsys_on_dfl(io_cgrp_subsys
) && blkg
->parent
)
420 sq
->parent_sq
= &blkg_to_tg(blkg
->parent
)->service_queue
;
425 * Set has_rules[] if @tg or any of its parents have limits configured.
426 * This doesn't require walking up to the top of the hierarchy as the
427 * parent's has_rules[] is guaranteed to be correct.
429 static void tg_update_has_rules(struct throtl_grp
*tg
)
431 struct throtl_grp
*parent_tg
= sq_to_tg(tg
->service_queue
.parent_sq
);
432 struct throtl_data
*td
= tg
->td
;
435 for (rw
= READ
; rw
<= WRITE
; rw
++)
436 tg
->has_rules
[rw
] = (parent_tg
&& parent_tg
->has_rules
[rw
]) ||
437 (td
->limit_valid
[td
->limit_index
] &&
438 (tg_bps_limit(tg
, rw
) != U64_MAX
||
439 tg_iops_limit(tg
, rw
) != UINT_MAX
));
442 static void throtl_pd_online(struct blkg_policy_data
*pd
)
445 * We don't want new groups to escape the limits of its ancestors.
446 * Update has_rules[] after a new group is brought online.
448 tg_update_has_rules(pd_to_tg(pd
));
451 static void blk_throtl_update_limit_valid(struct throtl_data
*td
)
453 struct cgroup_subsys_state
*pos_css
;
454 struct blkcg_gq
*blkg
;
455 bool low_valid
= false;
458 blkg_for_each_descendant_post(blkg
, pos_css
, td
->queue
->root_blkg
) {
459 struct throtl_grp
*tg
= blkg_to_tg(blkg
);
461 if (tg
->bps
[READ
][LIMIT_LOW
] || tg
->bps
[WRITE
][LIMIT_LOW
] ||
462 tg
->iops
[READ
][LIMIT_LOW
] || tg
->iops
[WRITE
][LIMIT_LOW
])
467 td
->limit_valid
[LIMIT_LOW
] = low_valid
;
470 static void throtl_upgrade_state(struct throtl_data
*td
);
471 static void throtl_pd_offline(struct blkg_policy_data
*pd
)
473 struct throtl_grp
*tg
= pd_to_tg(pd
);
475 tg
->bps
[READ
][LIMIT_LOW
] = 0;
476 tg
->bps
[WRITE
][LIMIT_LOW
] = 0;
477 tg
->iops
[READ
][LIMIT_LOW
] = 0;
478 tg
->iops
[WRITE
][LIMIT_LOW
] = 0;
480 blk_throtl_update_limit_valid(tg
->td
);
482 if (!tg
->td
->limit_valid
[tg
->td
->limit_index
])
483 throtl_upgrade_state(tg
->td
);
486 static void throtl_pd_free(struct blkg_policy_data
*pd
)
488 struct throtl_grp
*tg
= pd_to_tg(pd
);
490 del_timer_sync(&tg
->service_queue
.pending_timer
);
494 static struct throtl_grp
*
495 throtl_rb_first(struct throtl_service_queue
*parent_sq
)
497 /* Service tree is empty */
498 if (!parent_sq
->nr_pending
)
501 if (!parent_sq
->first_pending
)
502 parent_sq
->first_pending
= rb_first(&parent_sq
->pending_tree
);
504 if (parent_sq
->first_pending
)
505 return rb_entry_tg(parent_sq
->first_pending
);
510 static void rb_erase_init(struct rb_node
*n
, struct rb_root
*root
)
516 static void throtl_rb_erase(struct rb_node
*n
,
517 struct throtl_service_queue
*parent_sq
)
519 if (parent_sq
->first_pending
== n
)
520 parent_sq
->first_pending
= NULL
;
521 rb_erase_init(n
, &parent_sq
->pending_tree
);
522 --parent_sq
->nr_pending
;
525 static void update_min_dispatch_time(struct throtl_service_queue
*parent_sq
)
527 struct throtl_grp
*tg
;
529 tg
= throtl_rb_first(parent_sq
);
533 parent_sq
->first_pending_disptime
= tg
->disptime
;
536 static void tg_service_queue_add(struct throtl_grp
*tg
)
538 struct throtl_service_queue
*parent_sq
= tg
->service_queue
.parent_sq
;
539 struct rb_node
**node
= &parent_sq
->pending_tree
.rb_node
;
540 struct rb_node
*parent
= NULL
;
541 struct throtl_grp
*__tg
;
542 unsigned long key
= tg
->disptime
;
545 while (*node
!= NULL
) {
547 __tg
= rb_entry_tg(parent
);
549 if (time_before(key
, __tg
->disptime
))
550 node
= &parent
->rb_left
;
552 node
= &parent
->rb_right
;
558 parent_sq
->first_pending
= &tg
->rb_node
;
560 rb_link_node(&tg
->rb_node
, parent
, node
);
561 rb_insert_color(&tg
->rb_node
, &parent_sq
->pending_tree
);
564 static void __throtl_enqueue_tg(struct throtl_grp
*tg
)
566 tg_service_queue_add(tg
);
567 tg
->flags
|= THROTL_TG_PENDING
;
568 tg
->service_queue
.parent_sq
->nr_pending
++;
571 static void throtl_enqueue_tg(struct throtl_grp
*tg
)
573 if (!(tg
->flags
& THROTL_TG_PENDING
))
574 __throtl_enqueue_tg(tg
);
577 static void __throtl_dequeue_tg(struct throtl_grp
*tg
)
579 throtl_rb_erase(&tg
->rb_node
, tg
->service_queue
.parent_sq
);
580 tg
->flags
&= ~THROTL_TG_PENDING
;
583 static void throtl_dequeue_tg(struct throtl_grp
*tg
)
585 if (tg
->flags
& THROTL_TG_PENDING
)
586 __throtl_dequeue_tg(tg
);
589 /* Call with queue lock held */
590 static void throtl_schedule_pending_timer(struct throtl_service_queue
*sq
,
591 unsigned long expires
)
593 mod_timer(&sq
->pending_timer
, expires
);
594 throtl_log(sq
, "schedule timer. delay=%lu jiffies=%lu",
595 expires
- jiffies
, jiffies
);
599 * throtl_schedule_next_dispatch - schedule the next dispatch cycle
600 * @sq: the service_queue to schedule dispatch for
601 * @force: force scheduling
603 * Arm @sq->pending_timer so that the next dispatch cycle starts on the
604 * dispatch time of the first pending child. Returns %true if either timer
605 * is armed or there's no pending child left. %false if the current
606 * dispatch window is still open and the caller should continue
609 * If @force is %true, the dispatch timer is always scheduled and this
610 * function is guaranteed to return %true. This is to be used when the
611 * caller can't dispatch itself and needs to invoke pending_timer
612 * unconditionally. Note that forced scheduling is likely to induce short
613 * delay before dispatch starts even if @sq->first_pending_disptime is not
614 * in the future and thus shouldn't be used in hot paths.
616 static bool throtl_schedule_next_dispatch(struct throtl_service_queue
*sq
,
619 /* any pending children left? */
623 update_min_dispatch_time(sq
);
625 /* is the next dispatch time in the future? */
626 if (force
|| time_after(sq
->first_pending_disptime
, jiffies
)) {
627 throtl_schedule_pending_timer(sq
, sq
->first_pending_disptime
);
631 /* tell the caller to continue dispatching */
635 static inline void throtl_start_new_slice_with_credit(struct throtl_grp
*tg
,
636 bool rw
, unsigned long start
)
638 tg
->bytes_disp
[rw
] = 0;
642 * Previous slice has expired. We must have trimmed it after last
643 * bio dispatch. That means since start of last slice, we never used
644 * that bandwidth. Do try to make use of that bandwidth while giving
647 if (time_after_eq(start
, tg
->slice_start
[rw
]))
648 tg
->slice_start
[rw
] = start
;
650 tg
->slice_end
[rw
] = jiffies
+ throtl_slice
;
651 throtl_log(&tg
->service_queue
,
652 "[%c] new slice with credit start=%lu end=%lu jiffies=%lu",
653 rw
== READ
? 'R' : 'W', tg
->slice_start
[rw
],
654 tg
->slice_end
[rw
], jiffies
);
657 static inline void throtl_start_new_slice(struct throtl_grp
*tg
, bool rw
)
659 tg
->bytes_disp
[rw
] = 0;
661 tg
->slice_start
[rw
] = jiffies
;
662 tg
->slice_end
[rw
] = jiffies
+ throtl_slice
;
663 throtl_log(&tg
->service_queue
,
664 "[%c] new slice start=%lu end=%lu jiffies=%lu",
665 rw
== READ
? 'R' : 'W', tg
->slice_start
[rw
],
666 tg
->slice_end
[rw
], jiffies
);
669 static inline void throtl_set_slice_end(struct throtl_grp
*tg
, bool rw
,
670 unsigned long jiffy_end
)
672 tg
->slice_end
[rw
] = roundup(jiffy_end
, throtl_slice
);
675 static inline void throtl_extend_slice(struct throtl_grp
*tg
, bool rw
,
676 unsigned long jiffy_end
)
678 tg
->slice_end
[rw
] = roundup(jiffy_end
, throtl_slice
);
679 throtl_log(&tg
->service_queue
,
680 "[%c] extend slice start=%lu end=%lu jiffies=%lu",
681 rw
== READ
? 'R' : 'W', tg
->slice_start
[rw
],
682 tg
->slice_end
[rw
], jiffies
);
685 /* Determine if previously allocated or extended slice is complete or not */
686 static bool throtl_slice_used(struct throtl_grp
*tg
, bool rw
)
688 if (time_in_range(jiffies
, tg
->slice_start
[rw
], tg
->slice_end
[rw
]))
694 /* Trim the used slices and adjust slice start accordingly */
695 static inline void throtl_trim_slice(struct throtl_grp
*tg
, bool rw
)
697 unsigned long nr_slices
, time_elapsed
, io_trim
;
700 BUG_ON(time_before(tg
->slice_end
[rw
], tg
->slice_start
[rw
]));
703 * If bps are unlimited (-1), then time slice don't get
704 * renewed. Don't try to trim the slice if slice is used. A new
705 * slice will start when appropriate.
707 if (throtl_slice_used(tg
, rw
))
711 * A bio has been dispatched. Also adjust slice_end. It might happen
712 * that initially cgroup limit was very low resulting in high
713 * slice_end, but later limit was bumped up and bio was dispached
714 * sooner, then we need to reduce slice_end. A high bogus slice_end
715 * is bad because it does not allow new slice to start.
718 throtl_set_slice_end(tg
, rw
, jiffies
+ throtl_slice
);
720 time_elapsed
= jiffies
- tg
->slice_start
[rw
];
722 nr_slices
= time_elapsed
/ throtl_slice
;
726 tmp
= tg_bps_limit(tg
, rw
) * throtl_slice
* nr_slices
;
730 io_trim
= (tg_iops_limit(tg
, rw
) * throtl_slice
* nr_slices
) / HZ
;
732 if (!bytes_trim
&& !io_trim
)
735 if (tg
->bytes_disp
[rw
] >= bytes_trim
)
736 tg
->bytes_disp
[rw
] -= bytes_trim
;
738 tg
->bytes_disp
[rw
] = 0;
740 if (tg
->io_disp
[rw
] >= io_trim
)
741 tg
->io_disp
[rw
] -= io_trim
;
745 tg
->slice_start
[rw
] += nr_slices
* throtl_slice
;
747 throtl_log(&tg
->service_queue
,
748 "[%c] trim slice nr=%lu bytes=%llu io=%lu start=%lu end=%lu jiffies=%lu",
749 rw
== READ
? 'R' : 'W', nr_slices
, bytes_trim
, io_trim
,
750 tg
->slice_start
[rw
], tg
->slice_end
[rw
], jiffies
);
753 static bool tg_with_in_iops_limit(struct throtl_grp
*tg
, struct bio
*bio
,
756 bool rw
= bio_data_dir(bio
);
757 unsigned int io_allowed
;
758 unsigned long jiffy_elapsed
, jiffy_wait
, jiffy_elapsed_rnd
;
761 jiffy_elapsed
= jiffy_elapsed_rnd
= jiffies
- tg
->slice_start
[rw
];
763 /* Slice has just started. Consider one slice interval */
765 jiffy_elapsed_rnd
= throtl_slice
;
767 jiffy_elapsed_rnd
= roundup(jiffy_elapsed_rnd
, throtl_slice
);
770 * jiffy_elapsed_rnd should not be a big value as minimum iops can be
771 * 1 then at max jiffy elapsed should be equivalent of 1 second as we
772 * will allow dispatch after 1 second and after that slice should
776 tmp
= (u64
)tg_iops_limit(tg
, rw
) * jiffy_elapsed_rnd
;
780 io_allowed
= UINT_MAX
;
784 if (tg
->io_disp
[rw
] + 1 <= io_allowed
) {
790 /* Calc approx time to dispatch */
791 jiffy_wait
= ((tg
->io_disp
[rw
] + 1) * HZ
) / tg_iops_limit(tg
, rw
) + 1;
793 if (jiffy_wait
> jiffy_elapsed
)
794 jiffy_wait
= jiffy_wait
- jiffy_elapsed
;
803 static bool tg_with_in_bps_limit(struct throtl_grp
*tg
, struct bio
*bio
,
806 bool rw
= bio_data_dir(bio
);
807 u64 bytes_allowed
, extra_bytes
, tmp
;
808 unsigned long jiffy_elapsed
, jiffy_wait
, jiffy_elapsed_rnd
;
810 jiffy_elapsed
= jiffy_elapsed_rnd
= jiffies
- tg
->slice_start
[rw
];
812 /* Slice has just started. Consider one slice interval */
814 jiffy_elapsed_rnd
= throtl_slice
;
816 jiffy_elapsed_rnd
= roundup(jiffy_elapsed_rnd
, throtl_slice
);
818 tmp
= tg_bps_limit(tg
, rw
) * jiffy_elapsed_rnd
;
822 if (tg
->bytes_disp
[rw
] + bio
->bi_iter
.bi_size
<= bytes_allowed
) {
828 /* Calc approx time to dispatch */
829 extra_bytes
= tg
->bytes_disp
[rw
] + bio
->bi_iter
.bi_size
- bytes_allowed
;
830 jiffy_wait
= div64_u64(extra_bytes
* HZ
, tg_bps_limit(tg
, rw
));
836 * This wait time is without taking into consideration the rounding
837 * up we did. Add that time also.
839 jiffy_wait
= jiffy_wait
+ (jiffy_elapsed_rnd
- jiffy_elapsed
);
846 * Returns whether one can dispatch a bio or not. Also returns approx number
847 * of jiffies to wait before this bio is with-in IO rate and can be dispatched
849 static bool tg_may_dispatch(struct throtl_grp
*tg
, struct bio
*bio
,
852 bool rw
= bio_data_dir(bio
);
853 unsigned long bps_wait
= 0, iops_wait
= 0, max_wait
= 0;
856 * Currently whole state machine of group depends on first bio
857 * queued in the group bio list. So one should not be calling
858 * this function with a different bio if there are other bios
861 BUG_ON(tg
->service_queue
.nr_queued
[rw
] &&
862 bio
!= throtl_peek_queued(&tg
->service_queue
.queued
[rw
]));
864 /* If tg->bps = -1, then BW is unlimited */
865 if (tg_bps_limit(tg
, rw
) == U64_MAX
&&
866 tg_iops_limit(tg
, rw
) == UINT_MAX
) {
873 * If previous slice expired, start a new one otherwise renew/extend
874 * existing slice to make sure it is at least throtl_slice interval
875 * long since now. New slice is started only for empty throttle group.
876 * If there is queued bio, that means there should be an active
877 * slice and it should be extended instead.
879 if (throtl_slice_used(tg
, rw
) && !(tg
->service_queue
.nr_queued
[rw
]))
880 throtl_start_new_slice(tg
, rw
);
882 if (time_before(tg
->slice_end
[rw
], jiffies
+ throtl_slice
))
883 throtl_extend_slice(tg
, rw
, jiffies
+ throtl_slice
);
886 if (tg_with_in_bps_limit(tg
, bio
, &bps_wait
) &&
887 tg_with_in_iops_limit(tg
, bio
, &iops_wait
)) {
893 max_wait
= max(bps_wait
, iops_wait
);
898 if (time_before(tg
->slice_end
[rw
], jiffies
+ max_wait
))
899 throtl_extend_slice(tg
, rw
, jiffies
+ max_wait
);
904 static void throtl_charge_bio(struct throtl_grp
*tg
, struct bio
*bio
)
906 bool rw
= bio_data_dir(bio
);
908 /* Charge the bio to the group */
909 tg
->bytes_disp
[rw
] += bio
->bi_iter
.bi_size
;
911 tg
->last_bytes_disp
[rw
] += bio
->bi_iter
.bi_size
;
912 tg
->last_io_disp
[rw
]++;
915 * BIO_THROTTLED is used to prevent the same bio to be throttled
916 * more than once as a throttled bio will go through blk-throtl the
917 * second time when it eventually gets issued. Set it when a bio
918 * is being charged to a tg.
920 if (!bio_flagged(bio
, BIO_THROTTLED
))
921 bio_set_flag(bio
, BIO_THROTTLED
);
925 * throtl_add_bio_tg - add a bio to the specified throtl_grp
928 * @tg: the target throtl_grp
930 * Add @bio to @tg's service_queue using @qn. If @qn is not specified,
931 * tg->qnode_on_self[] is used.
933 static void throtl_add_bio_tg(struct bio
*bio
, struct throtl_qnode
*qn
,
934 struct throtl_grp
*tg
)
936 struct throtl_service_queue
*sq
= &tg
->service_queue
;
937 bool rw
= bio_data_dir(bio
);
940 qn
= &tg
->qnode_on_self
[rw
];
943 * If @tg doesn't currently have any bios queued in the same
944 * direction, queueing @bio can change when @tg should be
945 * dispatched. Mark that @tg was empty. This is automatically
946 * cleaered on the next tg_update_disptime().
948 if (!sq
->nr_queued
[rw
])
949 tg
->flags
|= THROTL_TG_WAS_EMPTY
;
951 throtl_qnode_add_bio(bio
, qn
, &sq
->queued
[rw
]);
954 throtl_enqueue_tg(tg
);
957 static void tg_update_disptime(struct throtl_grp
*tg
)
959 struct throtl_service_queue
*sq
= &tg
->service_queue
;
960 unsigned long read_wait
= -1, write_wait
= -1, min_wait
= -1, disptime
;
963 bio
= throtl_peek_queued(&sq
->queued
[READ
]);
965 tg_may_dispatch(tg
, bio
, &read_wait
);
967 bio
= throtl_peek_queued(&sq
->queued
[WRITE
]);
969 tg_may_dispatch(tg
, bio
, &write_wait
);
971 min_wait
= min(read_wait
, write_wait
);
972 disptime
= jiffies
+ min_wait
;
974 /* Update dispatch time */
975 throtl_dequeue_tg(tg
);
976 tg
->disptime
= disptime
;
977 throtl_enqueue_tg(tg
);
979 /* see throtl_add_bio_tg() */
980 tg
->flags
&= ~THROTL_TG_WAS_EMPTY
;
983 static void start_parent_slice_with_credit(struct throtl_grp
*child_tg
,
984 struct throtl_grp
*parent_tg
, bool rw
)
986 if (throtl_slice_used(parent_tg
, rw
)) {
987 throtl_start_new_slice_with_credit(parent_tg
, rw
,
988 child_tg
->slice_start
[rw
]);
993 static void tg_dispatch_one_bio(struct throtl_grp
*tg
, bool rw
)
995 struct throtl_service_queue
*sq
= &tg
->service_queue
;
996 struct throtl_service_queue
*parent_sq
= sq
->parent_sq
;
997 struct throtl_grp
*parent_tg
= sq_to_tg(parent_sq
);
998 struct throtl_grp
*tg_to_put
= NULL
;
1002 * @bio is being transferred from @tg to @parent_sq. Popping a bio
1003 * from @tg may put its reference and @parent_sq might end up
1004 * getting released prematurely. Remember the tg to put and put it
1005 * after @bio is transferred to @parent_sq.
1007 bio
= throtl_pop_queued(&sq
->queued
[rw
], &tg_to_put
);
1008 sq
->nr_queued
[rw
]--;
1010 throtl_charge_bio(tg
, bio
);
1013 * If our parent is another tg, we just need to transfer @bio to
1014 * the parent using throtl_add_bio_tg(). If our parent is
1015 * @td->service_queue, @bio is ready to be issued. Put it on its
1016 * bio_lists[] and decrease total number queued. The caller is
1017 * responsible for issuing these bios.
1020 throtl_add_bio_tg(bio
, &tg
->qnode_on_parent
[rw
], parent_tg
);
1021 start_parent_slice_with_credit(tg
, parent_tg
, rw
);
1023 throtl_qnode_add_bio(bio
, &tg
->qnode_on_parent
[rw
],
1024 &parent_sq
->queued
[rw
]);
1025 BUG_ON(tg
->td
->nr_queued
[rw
] <= 0);
1026 tg
->td
->nr_queued
[rw
]--;
1029 throtl_trim_slice(tg
, rw
);
1032 blkg_put(tg_to_blkg(tg_to_put
));
1035 static int throtl_dispatch_tg(struct throtl_grp
*tg
)
1037 struct throtl_service_queue
*sq
= &tg
->service_queue
;
1038 unsigned int nr_reads
= 0, nr_writes
= 0;
1039 unsigned int max_nr_reads
= throtl_grp_quantum
*3/4;
1040 unsigned int max_nr_writes
= throtl_grp_quantum
- max_nr_reads
;
1043 /* Try to dispatch 75% READS and 25% WRITES */
1045 while ((bio
= throtl_peek_queued(&sq
->queued
[READ
])) &&
1046 tg_may_dispatch(tg
, bio
, NULL
)) {
1048 tg_dispatch_one_bio(tg
, bio_data_dir(bio
));
1051 if (nr_reads
>= max_nr_reads
)
1055 while ((bio
= throtl_peek_queued(&sq
->queued
[WRITE
])) &&
1056 tg_may_dispatch(tg
, bio
, NULL
)) {
1058 tg_dispatch_one_bio(tg
, bio_data_dir(bio
));
1061 if (nr_writes
>= max_nr_writes
)
1065 return nr_reads
+ nr_writes
;
1068 static int throtl_select_dispatch(struct throtl_service_queue
*parent_sq
)
1070 unsigned int nr_disp
= 0;
1073 struct throtl_grp
*tg
= throtl_rb_first(parent_sq
);
1074 struct throtl_service_queue
*sq
= &tg
->service_queue
;
1079 if (time_before(jiffies
, tg
->disptime
))
1082 throtl_dequeue_tg(tg
);
1084 nr_disp
+= throtl_dispatch_tg(tg
);
1086 if (sq
->nr_queued
[0] || sq
->nr_queued
[1])
1087 tg_update_disptime(tg
);
1089 if (nr_disp
>= throtl_quantum
)
1096 static bool throtl_can_upgrade(struct throtl_data
*td
,
1097 struct throtl_grp
*this_tg
);
1099 * throtl_pending_timer_fn - timer function for service_queue->pending_timer
1100 * @arg: the throtl_service_queue being serviced
1102 * This timer is armed when a child throtl_grp with active bio's become
1103 * pending and queued on the service_queue's pending_tree and expires when
1104 * the first child throtl_grp should be dispatched. This function
1105 * dispatches bio's from the children throtl_grps to the parent
1108 * If the parent's parent is another throtl_grp, dispatching is propagated
1109 * by either arming its pending_timer or repeating dispatch directly. If
1110 * the top-level service_tree is reached, throtl_data->dispatch_work is
1111 * kicked so that the ready bio's are issued.
1113 static void throtl_pending_timer_fn(unsigned long arg
)
1115 struct throtl_service_queue
*sq
= (void *)arg
;
1116 struct throtl_grp
*tg
= sq_to_tg(sq
);
1117 struct throtl_data
*td
= sq_to_td(sq
);
1118 struct request_queue
*q
= td
->queue
;
1119 struct throtl_service_queue
*parent_sq
;
1123 spin_lock_irq(q
->queue_lock
);
1124 if (throtl_can_upgrade(td
, NULL
))
1125 throtl_upgrade_state(td
);
1128 parent_sq
= sq
->parent_sq
;
1132 throtl_log(sq
, "dispatch nr_queued=%u read=%u write=%u",
1133 sq
->nr_queued
[READ
] + sq
->nr_queued
[WRITE
],
1134 sq
->nr_queued
[READ
], sq
->nr_queued
[WRITE
]);
1136 ret
= throtl_select_dispatch(sq
);
1138 throtl_log(sq
, "bios disp=%u", ret
);
1142 if (throtl_schedule_next_dispatch(sq
, false))
1145 /* this dispatch windows is still open, relax and repeat */
1146 spin_unlock_irq(q
->queue_lock
);
1148 spin_lock_irq(q
->queue_lock
);
1155 /* @parent_sq is another throl_grp, propagate dispatch */
1156 if (tg
->flags
& THROTL_TG_WAS_EMPTY
) {
1157 tg_update_disptime(tg
);
1158 if (!throtl_schedule_next_dispatch(parent_sq
, false)) {
1159 /* window is already open, repeat dispatching */
1166 /* reached the top-level, queue issueing */
1167 queue_work(kthrotld_workqueue
, &td
->dispatch_work
);
1170 spin_unlock_irq(q
->queue_lock
);
1174 * blk_throtl_dispatch_work_fn - work function for throtl_data->dispatch_work
1175 * @work: work item being executed
1177 * This function is queued for execution when bio's reach the bio_lists[]
1178 * of throtl_data->service_queue. Those bio's are ready and issued by this
1181 static void blk_throtl_dispatch_work_fn(struct work_struct
*work
)
1183 struct throtl_data
*td
= container_of(work
, struct throtl_data
,
1185 struct throtl_service_queue
*td_sq
= &td
->service_queue
;
1186 struct request_queue
*q
= td
->queue
;
1187 struct bio_list bio_list_on_stack
;
1189 struct blk_plug plug
;
1192 bio_list_init(&bio_list_on_stack
);
1194 spin_lock_irq(q
->queue_lock
);
1195 for (rw
= READ
; rw
<= WRITE
; rw
++)
1196 while ((bio
= throtl_pop_queued(&td_sq
->queued
[rw
], NULL
)))
1197 bio_list_add(&bio_list_on_stack
, bio
);
1198 spin_unlock_irq(q
->queue_lock
);
1200 if (!bio_list_empty(&bio_list_on_stack
)) {
1201 blk_start_plug(&plug
);
1202 while((bio
= bio_list_pop(&bio_list_on_stack
)))
1203 generic_make_request(bio
);
1204 blk_finish_plug(&plug
);
1208 static u64
tg_prfill_conf_u64(struct seq_file
*sf
, struct blkg_policy_data
*pd
,
1211 struct throtl_grp
*tg
= pd_to_tg(pd
);
1212 u64 v
= *(u64
*)((void *)tg
+ off
);
1216 return __blkg_prfill_u64(sf
, pd
, v
);
1219 static u64
tg_prfill_conf_uint(struct seq_file
*sf
, struct blkg_policy_data
*pd
,
1222 struct throtl_grp
*tg
= pd_to_tg(pd
);
1223 unsigned int v
= *(unsigned int *)((void *)tg
+ off
);
1227 return __blkg_prfill_u64(sf
, pd
, v
);
1230 static int tg_print_conf_u64(struct seq_file
*sf
, void *v
)
1232 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)), tg_prfill_conf_u64
,
1233 &blkcg_policy_throtl
, seq_cft(sf
)->private, false);
1237 static int tg_print_conf_uint(struct seq_file
*sf
, void *v
)
1239 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)), tg_prfill_conf_uint
,
1240 &blkcg_policy_throtl
, seq_cft(sf
)->private, false);
1244 static void tg_conf_updated(struct throtl_grp
*tg
)
1246 struct throtl_service_queue
*sq
= &tg
->service_queue
;
1247 struct cgroup_subsys_state
*pos_css
;
1248 struct blkcg_gq
*blkg
;
1250 throtl_log(&tg
->service_queue
,
1251 "limit change rbps=%llu wbps=%llu riops=%u wiops=%u",
1252 tg_bps_limit(tg
, READ
), tg_bps_limit(tg
, WRITE
),
1253 tg_iops_limit(tg
, READ
), tg_iops_limit(tg
, WRITE
));
1256 * Update has_rules[] flags for the updated tg's subtree. A tg is
1257 * considered to have rules if either the tg itself or any of its
1258 * ancestors has rules. This identifies groups without any
1259 * restrictions in the whole hierarchy and allows them to bypass
1262 blkg_for_each_descendant_pre(blkg
, pos_css
, tg_to_blkg(tg
))
1263 tg_update_has_rules(blkg_to_tg(blkg
));
1266 * We're already holding queue_lock and know @tg is valid. Let's
1267 * apply the new config directly.
1269 * Restart the slices for both READ and WRITES. It might happen
1270 * that a group's limit are dropped suddenly and we don't want to
1271 * account recently dispatched IO with new low rate.
1273 throtl_start_new_slice(tg
, 0);
1274 throtl_start_new_slice(tg
, 1);
1276 if (tg
->flags
& THROTL_TG_PENDING
) {
1277 tg_update_disptime(tg
);
1278 throtl_schedule_next_dispatch(sq
->parent_sq
, true);
1282 static ssize_t
tg_set_conf(struct kernfs_open_file
*of
,
1283 char *buf
, size_t nbytes
, loff_t off
, bool is_u64
)
1285 struct blkcg
*blkcg
= css_to_blkcg(of_css(of
));
1286 struct blkg_conf_ctx ctx
;
1287 struct throtl_grp
*tg
;
1291 ret
= blkg_conf_prep(blkcg
, &blkcg_policy_throtl
, buf
, &ctx
);
1296 if (sscanf(ctx
.body
, "%llu", &v
) != 1)
1301 tg
= blkg_to_tg(ctx
.blkg
);
1304 *(u64
*)((void *)tg
+ of_cft(of
)->private) = v
;
1306 *(unsigned int *)((void *)tg
+ of_cft(of
)->private) = v
;
1308 tg_conf_updated(tg
);
1311 blkg_conf_finish(&ctx
);
1312 return ret
?: nbytes
;
1315 static ssize_t
tg_set_conf_u64(struct kernfs_open_file
*of
,
1316 char *buf
, size_t nbytes
, loff_t off
)
1318 return tg_set_conf(of
, buf
, nbytes
, off
, true);
1321 static ssize_t
tg_set_conf_uint(struct kernfs_open_file
*of
,
1322 char *buf
, size_t nbytes
, loff_t off
)
1324 return tg_set_conf(of
, buf
, nbytes
, off
, false);
1327 static struct cftype throtl_legacy_files
[] = {
1329 .name
= "throttle.read_bps_device",
1330 .private = offsetof(struct throtl_grp
, bps
[READ
][LIMIT_MAX
]),
1331 .seq_show
= tg_print_conf_u64
,
1332 .write
= tg_set_conf_u64
,
1335 .name
= "throttle.write_bps_device",
1336 .private = offsetof(struct throtl_grp
, bps
[WRITE
][LIMIT_MAX
]),
1337 .seq_show
= tg_print_conf_u64
,
1338 .write
= tg_set_conf_u64
,
1341 .name
= "throttle.read_iops_device",
1342 .private = offsetof(struct throtl_grp
, iops
[READ
][LIMIT_MAX
]),
1343 .seq_show
= tg_print_conf_uint
,
1344 .write
= tg_set_conf_uint
,
1347 .name
= "throttle.write_iops_device",
1348 .private = offsetof(struct throtl_grp
, iops
[WRITE
][LIMIT_MAX
]),
1349 .seq_show
= tg_print_conf_uint
,
1350 .write
= tg_set_conf_uint
,
1353 .name
= "throttle.io_service_bytes",
1354 .private = (unsigned long)&blkcg_policy_throtl
,
1355 .seq_show
= blkg_print_stat_bytes
,
1358 .name
= "throttle.io_serviced",
1359 .private = (unsigned long)&blkcg_policy_throtl
,
1360 .seq_show
= blkg_print_stat_ios
,
1365 static u64
tg_prfill_limit(struct seq_file
*sf
, struct blkg_policy_data
*pd
,
1368 struct throtl_grp
*tg
= pd_to_tg(pd
);
1369 const char *dname
= blkg_dev_name(pd
->blkg
);
1370 char bufs
[4][21] = { "max", "max", "max", "max" };
1372 unsigned int iops_dft
;
1377 if (off
== LIMIT_LOW
) {
1382 iops_dft
= UINT_MAX
;
1385 if (tg
->bps_conf
[READ
][off
] == bps_dft
&&
1386 tg
->bps_conf
[WRITE
][off
] == bps_dft
&&
1387 tg
->iops_conf
[READ
][off
] == iops_dft
&&
1388 tg
->iops_conf
[WRITE
][off
] == iops_dft
)
1391 if (tg
->bps_conf
[READ
][off
] != bps_dft
)
1392 snprintf(bufs
[0], sizeof(bufs
[0]), "%llu",
1393 tg
->bps_conf
[READ
][off
]);
1394 if (tg
->bps_conf
[WRITE
][off
] != bps_dft
)
1395 snprintf(bufs
[1], sizeof(bufs
[1]), "%llu",
1396 tg
->bps_conf
[WRITE
][off
]);
1397 if (tg
->iops_conf
[READ
][off
] != iops_dft
)
1398 snprintf(bufs
[2], sizeof(bufs
[2]), "%u",
1399 tg
->iops_conf
[READ
][off
]);
1400 if (tg
->iops_conf
[WRITE
][off
] != iops_dft
)
1401 snprintf(bufs
[3], sizeof(bufs
[3]), "%u",
1402 tg
->iops_conf
[WRITE
][off
]);
1404 seq_printf(sf
, "%s rbps=%s wbps=%s riops=%s wiops=%s\n",
1405 dname
, bufs
[0], bufs
[1], bufs
[2], bufs
[3]);
1409 static int tg_print_limit(struct seq_file
*sf
, void *v
)
1411 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)), tg_prfill_limit
,
1412 &blkcg_policy_throtl
, seq_cft(sf
)->private, false);
1416 static ssize_t
tg_set_limit(struct kernfs_open_file
*of
,
1417 char *buf
, size_t nbytes
, loff_t off
)
1419 struct blkcg
*blkcg
= css_to_blkcg(of_css(of
));
1420 struct blkg_conf_ctx ctx
;
1421 struct throtl_grp
*tg
;
1424 int index
= of_cft(of
)->private;
1426 ret
= blkg_conf_prep(blkcg
, &blkcg_policy_throtl
, buf
, &ctx
);
1430 tg
= blkg_to_tg(ctx
.blkg
);
1432 v
[0] = tg
->bps_conf
[READ
][index
];
1433 v
[1] = tg
->bps_conf
[WRITE
][index
];
1434 v
[2] = tg
->iops_conf
[READ
][index
];
1435 v
[3] = tg
->iops_conf
[WRITE
][index
];
1438 char tok
[27]; /* wiops=18446744073709551616 */
1443 if (sscanf(ctx
.body
, "%26s%n", tok
, &len
) != 1)
1452 if (!p
|| (sscanf(p
, "%llu", &val
) != 1 && strcmp(p
, "max")))
1460 if (!strcmp(tok
, "rbps"))
1462 else if (!strcmp(tok
, "wbps"))
1464 else if (!strcmp(tok
, "riops"))
1465 v
[2] = min_t(u64
, val
, UINT_MAX
);
1466 else if (!strcmp(tok
, "wiops"))
1467 v
[3] = min_t(u64
, val
, UINT_MAX
);
1472 tg
->bps_conf
[READ
][index
] = v
[0];
1473 tg
->bps_conf
[WRITE
][index
] = v
[1];
1474 tg
->iops_conf
[READ
][index
] = v
[2];
1475 tg
->iops_conf
[WRITE
][index
] = v
[3];
1477 if (index
== LIMIT_MAX
) {
1478 tg
->bps
[READ
][index
] = v
[0];
1479 tg
->bps
[WRITE
][index
] = v
[1];
1480 tg
->iops
[READ
][index
] = v
[2];
1481 tg
->iops
[WRITE
][index
] = v
[3];
1483 tg
->bps
[READ
][LIMIT_LOW
] = min(tg
->bps_conf
[READ
][LIMIT_LOW
],
1484 tg
->bps_conf
[READ
][LIMIT_MAX
]);
1485 tg
->bps
[WRITE
][LIMIT_LOW
] = min(tg
->bps_conf
[WRITE
][LIMIT_LOW
],
1486 tg
->bps_conf
[WRITE
][LIMIT_MAX
]);
1487 tg
->iops
[READ
][LIMIT_LOW
] = min(tg
->iops_conf
[READ
][LIMIT_LOW
],
1488 tg
->iops_conf
[READ
][LIMIT_MAX
]);
1489 tg
->iops
[WRITE
][LIMIT_LOW
] = min(tg
->iops_conf
[WRITE
][LIMIT_LOW
],
1490 tg
->iops_conf
[WRITE
][LIMIT_MAX
]);
1492 if (index
== LIMIT_LOW
) {
1493 blk_throtl_update_limit_valid(tg
->td
);
1494 if (tg
->td
->limit_valid
[LIMIT_LOW
])
1495 tg
->td
->limit_index
= LIMIT_LOW
;
1497 tg_conf_updated(tg
);
1500 blkg_conf_finish(&ctx
);
1501 return ret
?: nbytes
;
1504 static struct cftype throtl_files
[] = {
1505 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
1508 .flags
= CFTYPE_NOT_ON_ROOT
,
1509 .seq_show
= tg_print_limit
,
1510 .write
= tg_set_limit
,
1511 .private = LIMIT_LOW
,
1516 .flags
= CFTYPE_NOT_ON_ROOT
,
1517 .seq_show
= tg_print_limit
,
1518 .write
= tg_set_limit
,
1519 .private = LIMIT_MAX
,
1524 static void throtl_shutdown_wq(struct request_queue
*q
)
1526 struct throtl_data
*td
= q
->td
;
1528 cancel_work_sync(&td
->dispatch_work
);
1531 static struct blkcg_policy blkcg_policy_throtl
= {
1532 .dfl_cftypes
= throtl_files
,
1533 .legacy_cftypes
= throtl_legacy_files
,
1535 .pd_alloc_fn
= throtl_pd_alloc
,
1536 .pd_init_fn
= throtl_pd_init
,
1537 .pd_online_fn
= throtl_pd_online
,
1538 .pd_offline_fn
= throtl_pd_offline
,
1539 .pd_free_fn
= throtl_pd_free
,
1542 static unsigned long __tg_last_low_overflow_time(struct throtl_grp
*tg
)
1544 unsigned long rtime
= jiffies
, wtime
= jiffies
;
1546 if (tg
->bps
[READ
][LIMIT_LOW
] || tg
->iops
[READ
][LIMIT_LOW
])
1547 rtime
= tg
->last_low_overflow_time
[READ
];
1548 if (tg
->bps
[WRITE
][LIMIT_LOW
] || tg
->iops
[WRITE
][LIMIT_LOW
])
1549 wtime
= tg
->last_low_overflow_time
[WRITE
];
1550 return min(rtime
, wtime
);
1553 /* tg should not be an intermediate node */
1554 static unsigned long tg_last_low_overflow_time(struct throtl_grp
*tg
)
1556 struct throtl_service_queue
*parent_sq
;
1557 struct throtl_grp
*parent
= tg
;
1558 unsigned long ret
= __tg_last_low_overflow_time(tg
);
1561 parent_sq
= parent
->service_queue
.parent_sq
;
1562 parent
= sq_to_tg(parent_sq
);
1567 * The parent doesn't have low limit, it always reaches low
1568 * limit. Its overflow time is useless for children
1570 if (!parent
->bps
[READ
][LIMIT_LOW
] &&
1571 !parent
->iops
[READ
][LIMIT_LOW
] &&
1572 !parent
->bps
[WRITE
][LIMIT_LOW
] &&
1573 !parent
->iops
[WRITE
][LIMIT_LOW
])
1575 if (time_after(__tg_last_low_overflow_time(parent
), ret
))
1576 ret
= __tg_last_low_overflow_time(parent
);
1581 static bool throtl_tg_can_upgrade(struct throtl_grp
*tg
)
1583 struct throtl_service_queue
*sq
= &tg
->service_queue
;
1584 bool read_limit
, write_limit
;
1587 * if cgroup reaches low limit (if low limit is 0, the cgroup always
1588 * reaches), it's ok to upgrade to next limit
1590 read_limit
= tg
->bps
[READ
][LIMIT_LOW
] || tg
->iops
[READ
][LIMIT_LOW
];
1591 write_limit
= tg
->bps
[WRITE
][LIMIT_LOW
] || tg
->iops
[WRITE
][LIMIT_LOW
];
1592 if (!read_limit
&& !write_limit
)
1594 if (read_limit
&& sq
->nr_queued
[READ
] &&
1595 (!write_limit
|| sq
->nr_queued
[WRITE
]))
1597 if (write_limit
&& sq
->nr_queued
[WRITE
] &&
1598 (!read_limit
|| sq
->nr_queued
[READ
]))
1603 static bool throtl_hierarchy_can_upgrade(struct throtl_grp
*tg
)
1606 if (throtl_tg_can_upgrade(tg
))
1608 tg
= sq_to_tg(tg
->service_queue
.parent_sq
);
1609 if (!tg
|| !tg_to_blkg(tg
)->parent
)
1615 static bool throtl_can_upgrade(struct throtl_data
*td
,
1616 struct throtl_grp
*this_tg
)
1618 struct cgroup_subsys_state
*pos_css
;
1619 struct blkcg_gq
*blkg
;
1621 if (td
->limit_index
!= LIMIT_LOW
)
1624 if (time_before(jiffies
, td
->low_downgrade_time
+ throtl_slice
))
1628 blkg_for_each_descendant_post(blkg
, pos_css
, td
->queue
->root_blkg
) {
1629 struct throtl_grp
*tg
= blkg_to_tg(blkg
);
1633 if (!list_empty(&tg_to_blkg(tg
)->blkcg
->css
.children
))
1635 if (!throtl_hierarchy_can_upgrade(tg
)) {
1644 static void throtl_upgrade_state(struct throtl_data
*td
)
1646 struct cgroup_subsys_state
*pos_css
;
1647 struct blkcg_gq
*blkg
;
1649 td
->limit_index
= LIMIT_MAX
;
1650 td
->low_upgrade_time
= jiffies
;
1652 blkg_for_each_descendant_post(blkg
, pos_css
, td
->queue
->root_blkg
) {
1653 struct throtl_grp
*tg
= blkg_to_tg(blkg
);
1654 struct throtl_service_queue
*sq
= &tg
->service_queue
;
1656 tg
->disptime
= jiffies
- 1;
1657 throtl_select_dispatch(sq
);
1658 throtl_schedule_next_dispatch(sq
, false);
1661 throtl_select_dispatch(&td
->service_queue
);
1662 throtl_schedule_next_dispatch(&td
->service_queue
, false);
1663 queue_work(kthrotld_workqueue
, &td
->dispatch_work
);
1666 static void throtl_downgrade_state(struct throtl_data
*td
, int new)
1668 td
->limit_index
= new;
1669 td
->low_downgrade_time
= jiffies
;
1672 static bool throtl_tg_can_downgrade(struct throtl_grp
*tg
)
1674 struct throtl_data
*td
= tg
->td
;
1675 unsigned long now
= jiffies
;
1678 * If cgroup is below low limit, consider downgrade and throttle other
1681 if (time_after_eq(now
, td
->low_upgrade_time
+ throtl_slice
) &&
1682 time_after_eq(now
, tg_last_low_overflow_time(tg
) + throtl_slice
))
1687 static bool throtl_hierarchy_can_downgrade(struct throtl_grp
*tg
)
1690 if (!throtl_tg_can_downgrade(tg
))
1692 tg
= sq_to_tg(tg
->service_queue
.parent_sq
);
1693 if (!tg
|| !tg_to_blkg(tg
)->parent
)
1699 static void throtl_downgrade_check(struct throtl_grp
*tg
)
1703 unsigned long elapsed_time
;
1704 unsigned long now
= jiffies
;
1706 if (tg
->td
->limit_index
!= LIMIT_MAX
||
1707 !tg
->td
->limit_valid
[LIMIT_LOW
])
1709 if (!list_empty(&tg_to_blkg(tg
)->blkcg
->css
.children
))
1711 if (time_after(tg
->last_check_time
+ throtl_slice
, now
))
1714 elapsed_time
= now
- tg
->last_check_time
;
1715 tg
->last_check_time
= now
;
1717 if (time_before(now
, tg_last_low_overflow_time(tg
) + throtl_slice
))
1720 if (tg
->bps
[READ
][LIMIT_LOW
]) {
1721 bps
= tg
->last_bytes_disp
[READ
] * HZ
;
1722 do_div(bps
, elapsed_time
);
1723 if (bps
>= tg
->bps
[READ
][LIMIT_LOW
])
1724 tg
->last_low_overflow_time
[READ
] = now
;
1727 if (tg
->bps
[WRITE
][LIMIT_LOW
]) {
1728 bps
= tg
->last_bytes_disp
[WRITE
] * HZ
;
1729 do_div(bps
, elapsed_time
);
1730 if (bps
>= tg
->bps
[WRITE
][LIMIT_LOW
])
1731 tg
->last_low_overflow_time
[WRITE
] = now
;
1734 if (tg
->iops
[READ
][LIMIT_LOW
]) {
1735 iops
= tg
->last_io_disp
[READ
] * HZ
/ elapsed_time
;
1736 if (iops
>= tg
->iops
[READ
][LIMIT_LOW
])
1737 tg
->last_low_overflow_time
[READ
] = now
;
1740 if (tg
->iops
[WRITE
][LIMIT_LOW
]) {
1741 iops
= tg
->last_io_disp
[WRITE
] * HZ
/ elapsed_time
;
1742 if (iops
>= tg
->iops
[WRITE
][LIMIT_LOW
])
1743 tg
->last_low_overflow_time
[WRITE
] = now
;
1747 * If cgroup is below low limit, consider downgrade and throttle other
1750 if (throtl_hierarchy_can_downgrade(tg
))
1751 throtl_downgrade_state(tg
->td
, LIMIT_LOW
);
1753 tg
->last_bytes_disp
[READ
] = 0;
1754 tg
->last_bytes_disp
[WRITE
] = 0;
1755 tg
->last_io_disp
[READ
] = 0;
1756 tg
->last_io_disp
[WRITE
] = 0;
1759 bool blk_throtl_bio(struct request_queue
*q
, struct blkcg_gq
*blkg
,
1762 struct throtl_qnode
*qn
= NULL
;
1763 struct throtl_grp
*tg
= blkg_to_tg(blkg
?: q
->root_blkg
);
1764 struct throtl_service_queue
*sq
;
1765 bool rw
= bio_data_dir(bio
);
1766 bool throttled
= false;
1768 WARN_ON_ONCE(!rcu_read_lock_held());
1770 /* see throtl_charge_bio() */
1771 if (bio_flagged(bio
, BIO_THROTTLED
) || !tg
->has_rules
[rw
])
1774 spin_lock_irq(q
->queue_lock
);
1776 if (unlikely(blk_queue_bypass(q
)))
1779 sq
= &tg
->service_queue
;
1783 if (tg
->last_low_overflow_time
[rw
] == 0)
1784 tg
->last_low_overflow_time
[rw
] = jiffies
;
1785 throtl_downgrade_check(tg
);
1786 /* throtl is FIFO - if bios are already queued, should queue */
1787 if (sq
->nr_queued
[rw
])
1790 /* if above limits, break to queue */
1791 if (!tg_may_dispatch(tg
, bio
, NULL
)) {
1792 tg
->last_low_overflow_time
[rw
] = jiffies
;
1793 if (throtl_can_upgrade(tg
->td
, tg
)) {
1794 throtl_upgrade_state(tg
->td
);
1800 /* within limits, let's charge and dispatch directly */
1801 throtl_charge_bio(tg
, bio
);
1804 * We need to trim slice even when bios are not being queued
1805 * otherwise it might happen that a bio is not queued for
1806 * a long time and slice keeps on extending and trim is not
1807 * called for a long time. Now if limits are reduced suddenly
1808 * we take into account all the IO dispatched so far at new
1809 * low rate and * newly queued IO gets a really long dispatch
1812 * So keep on trimming slice even if bio is not queued.
1814 throtl_trim_slice(tg
, rw
);
1817 * @bio passed through this layer without being throttled.
1818 * Climb up the ladder. If we''re already at the top, it
1819 * can be executed directly.
1821 qn
= &tg
->qnode_on_parent
[rw
];
1828 /* out-of-limit, queue to @tg */
1829 throtl_log(sq
, "[%c] bio. bdisp=%llu sz=%u bps=%llu iodisp=%u iops=%u queued=%d/%d",
1830 rw
== READ
? 'R' : 'W',
1831 tg
->bytes_disp
[rw
], bio
->bi_iter
.bi_size
,
1832 tg_bps_limit(tg
, rw
),
1833 tg
->io_disp
[rw
], tg_iops_limit(tg
, rw
),
1834 sq
->nr_queued
[READ
], sq
->nr_queued
[WRITE
]);
1836 tg
->last_low_overflow_time
[rw
] = jiffies
;
1838 bio_associate_current(bio
);
1839 tg
->td
->nr_queued
[rw
]++;
1840 throtl_add_bio_tg(bio
, qn
, tg
);
1844 * Update @tg's dispatch time and force schedule dispatch if @tg
1845 * was empty before @bio. The forced scheduling isn't likely to
1846 * cause undue delay as @bio is likely to be dispatched directly if
1847 * its @tg's disptime is not in the future.
1849 if (tg
->flags
& THROTL_TG_WAS_EMPTY
) {
1850 tg_update_disptime(tg
);
1851 throtl_schedule_next_dispatch(tg
->service_queue
.parent_sq
, true);
1855 spin_unlock_irq(q
->queue_lock
);
1858 * As multiple blk-throtls may stack in the same issue path, we
1859 * don't want bios to leave with the flag set. Clear the flag if
1863 bio_clear_flag(bio
, BIO_THROTTLED
);
1868 * Dispatch all bios from all children tg's queued on @parent_sq. On
1869 * return, @parent_sq is guaranteed to not have any active children tg's
1870 * and all bios from previously active tg's are on @parent_sq->bio_lists[].
1872 static void tg_drain_bios(struct throtl_service_queue
*parent_sq
)
1874 struct throtl_grp
*tg
;
1876 while ((tg
= throtl_rb_first(parent_sq
))) {
1877 struct throtl_service_queue
*sq
= &tg
->service_queue
;
1880 throtl_dequeue_tg(tg
);
1882 while ((bio
= throtl_peek_queued(&sq
->queued
[READ
])))
1883 tg_dispatch_one_bio(tg
, bio_data_dir(bio
));
1884 while ((bio
= throtl_peek_queued(&sq
->queued
[WRITE
])))
1885 tg_dispatch_one_bio(tg
, bio_data_dir(bio
));
1890 * blk_throtl_drain - drain throttled bios
1891 * @q: request_queue to drain throttled bios for
1893 * Dispatch all currently throttled bios on @q through ->make_request_fn().
1895 void blk_throtl_drain(struct request_queue
*q
)
1896 __releases(q
->queue_lock
) __acquires(q
->queue_lock
)
1898 struct throtl_data
*td
= q
->td
;
1899 struct blkcg_gq
*blkg
;
1900 struct cgroup_subsys_state
*pos_css
;
1904 queue_lockdep_assert_held(q
);
1908 * Drain each tg while doing post-order walk on the blkg tree, so
1909 * that all bios are propagated to td->service_queue. It'd be
1910 * better to walk service_queue tree directly but blkg walk is
1913 blkg_for_each_descendant_post(blkg
, pos_css
, td
->queue
->root_blkg
)
1914 tg_drain_bios(&blkg_to_tg(blkg
)->service_queue
);
1916 /* finally, transfer bios from top-level tg's into the td */
1917 tg_drain_bios(&td
->service_queue
);
1920 spin_unlock_irq(q
->queue_lock
);
1922 /* all bios now should be in td->service_queue, issue them */
1923 for (rw
= READ
; rw
<= WRITE
; rw
++)
1924 while ((bio
= throtl_pop_queued(&td
->service_queue
.queued
[rw
],
1926 generic_make_request(bio
);
1928 spin_lock_irq(q
->queue_lock
);
1931 int blk_throtl_init(struct request_queue
*q
)
1933 struct throtl_data
*td
;
1936 td
= kzalloc_node(sizeof(*td
), GFP_KERNEL
, q
->node
);
1940 INIT_WORK(&td
->dispatch_work
, blk_throtl_dispatch_work_fn
);
1941 throtl_service_queue_init(&td
->service_queue
);
1946 td
->limit_valid
[LIMIT_MAX
] = true;
1947 td
->limit_index
= LIMIT_MAX
;
1948 td
->low_upgrade_time
= jiffies
;
1949 td
->low_downgrade_time
= jiffies
;
1950 /* activate policy */
1951 ret
= blkcg_activate_policy(q
, &blkcg_policy_throtl
);
1957 void blk_throtl_exit(struct request_queue
*q
)
1960 throtl_shutdown_wq(q
);
1961 blkcg_deactivate_policy(q
, &blkcg_policy_throtl
);
1965 static int __init
throtl_init(void)
1967 kthrotld_workqueue
= alloc_workqueue("kthrotld", WQ_MEM_RECLAIM
, 0);
1968 if (!kthrotld_workqueue
)
1969 panic("Failed to create kthrotld\n");
1971 return blkcg_policy_register(&blkcg_policy_throtl
);
1974 module_init(throtl_init
);