1 /* SPDX-License-Identifier: GPL-2.0
3 * IO cost model based controller.
5 * Copyright (C) 2019 Tejun Heo <tj@kernel.org>
6 * Copyright (C) 2019 Andy Newell <newella@fb.com>
7 * Copyright (C) 2019 Facebook
9 * One challenge of controlling IO resources is the lack of trivially
10 * observable cost metric. This is distinguished from CPU and memory where
11 * wallclock time and the number of bytes can serve as accurate enough
14 * Bandwidth and iops are the most commonly used metrics for IO devices but
15 * depending on the type and specifics of the device, different IO patterns
16 * easily lead to multiple orders of magnitude variations rendering them
17 * useless for the purpose of IO capacity distribution. While on-device
18 * time, with a lot of clutches, could serve as a useful approximation for
19 * non-queued rotational devices, this is no longer viable with modern
20 * devices, even the rotational ones.
22 * While there is no cost metric we can trivially observe, it isn't a
23 * complete mystery. For example, on a rotational device, seek cost
24 * dominates while a contiguous transfer contributes a smaller amount
25 * proportional to the size. If we can characterize at least the relative
26 * costs of these different types of IOs, it should be possible to
27 * implement a reasonable work-conserving proportional IO resource
32 * IO cost model estimates the cost of an IO given its basic parameters and
33 * history (e.g. the end sector of the last IO). The cost is measured in
34 * device time. If a given IO is estimated to cost 10ms, the device should
35 * be able to process ~100 of those IOs in a second.
37 * Currently, there's only one builtin cost model - linear. Each IO is
38 * classified as sequential or random and given a base cost accordingly.
39 * On top of that, a size cost proportional to the length of the IO is
40 * added. While simple, this model captures the operational
41 * characteristics of a wide varienty of devices well enough. Default
42 * paramters for several different classes of devices are provided and the
43 * parameters can be configured from userspace via
44 * /sys/fs/cgroup/io.cost.model.
46 * If needed, tools/cgroup/iocost_coef_gen.py can be used to generate
47 * device-specific coefficients.
51 * The device virtual time (vtime) is used as the primary control metric.
52 * The control strategy is composed of the following three parts.
54 * 2-1. Vtime Distribution
56 * When a cgroup becomes active in terms of IOs, its hierarchical share is
57 * calculated. Please consider the following hierarchy where the numbers
58 * inside parentheses denote the configured weights.
64 * A0 (w:100) A1 (w:100)
66 * If B is idle and only A0 and A1 are actively issuing IOs, as the two are
67 * of equal weight, each gets 50% share. If then B starts issuing IOs, B
68 * gets 300/(100+300) or 75% share, and A0 and A1 equally splits the rest,
69 * 12.5% each. The distribution mechanism only cares about these flattened
70 * shares. They're called hweights (hierarchical weights) and always add
71 * upto 1 (HWEIGHT_WHOLE).
73 * A given cgroup's vtime runs slower in inverse proportion to its hweight.
74 * For example, with 12.5% weight, A0's time runs 8 times slower (100/12.5)
75 * against the device vtime - an IO which takes 10ms on the underlying
76 * device is considered to take 80ms on A0.
78 * This constitutes the basis of IO capacity distribution. Each cgroup's
79 * vtime is running at a rate determined by its hweight. A cgroup tracks
80 * the vtime consumed by past IOs and can issue a new IO iff doing so
81 * wouldn't outrun the current device vtime. Otherwise, the IO is
82 * suspended until the vtime has progressed enough to cover it.
84 * 2-2. Vrate Adjustment
86 * It's unrealistic to expect the cost model to be perfect. There are too
87 * many devices and even on the same device the overall performance
88 * fluctuates depending on numerous factors such as IO mixture and device
89 * internal garbage collection. The controller needs to adapt dynamically.
91 * This is achieved by adjusting the overall IO rate according to how busy
92 * the device is. If the device becomes overloaded, we're sending down too
93 * many IOs and should generally slow down. If there are waiting issuers
94 * but the device isn't saturated, we're issuing too few and should
97 * To slow down, we lower the vrate - the rate at which the device vtime
98 * passes compared to the wall clock. For example, if the vtime is running
99 * at the vrate of 75%, all cgroups added up would only be able to issue
100 * 750ms worth of IOs per second, and vice-versa for speeding up.
102 * Device business is determined using two criteria - rq wait and
103 * completion latencies.
105 * When a device gets saturated, the on-device and then the request queues
106 * fill up and a bio which is ready to be issued has to wait for a request
107 * to become available. When this delay becomes noticeable, it's a clear
108 * indication that the device is saturated and we lower the vrate. This
109 * saturation signal is fairly conservative as it only triggers when both
110 * hardware and software queues are filled up, and is used as the default
113 * As devices can have deep queues and be unfair in how the queued commands
114 * are executed, soley depending on rq wait may not result in satisfactory
115 * control quality. For a better control quality, completion latency QoS
116 * parameters can be configured so that the device is considered saturated
117 * if N'th percentile completion latency rises above the set point.
119 * The completion latency requirements are a function of both the
120 * underlying device characteristics and the desired IO latency quality of
121 * service. There is an inherent trade-off - the tighter the latency QoS,
122 * the higher the bandwidth lossage. Latency QoS is disabled by default
123 * and can be set through /sys/fs/cgroup/io.cost.qos.
125 * 2-3. Work Conservation
127 * Imagine two cgroups A and B with equal weights. A is issuing a small IO
128 * periodically while B is sending out enough parallel IOs to saturate the
129 * device on its own. Let's say A's usage amounts to 100ms worth of IO
130 * cost per second, i.e., 10% of the device capacity. The naive
131 * distribution of half and half would lead to 60% utilization of the
132 * device, a significant reduction in the total amount of work done
133 * compared to free-for-all competition. This is too high a cost to pay
136 * To conserve the total amount of work done, we keep track of how much
137 * each active cgroup is actually using and yield part of its weight if
138 * there are other cgroups which can make use of it. In the above case,
139 * A's weight will be lowered so that it hovers above the actual usage and
140 * B would be able to use the rest.
142 * As we don't want to penalize a cgroup for donating its weight, the
143 * surplus weight adjustment factors in a margin and has an immediate
144 * snapback mechanism in case the cgroup needs more IO vtime for itself.
146 * Note that adjusting down surplus weights has the same effects as
147 * accelerating vtime for other cgroups and work conservation can also be
148 * implemented by adjusting vrate dynamically. However, squaring who can
149 * donate and should take back how much requires hweight propagations
150 * anyway making it easier to implement and understand as a separate
155 * Instead of debugfs or other clumsy monitoring mechanisms, this
156 * controller uses a drgn based monitoring script -
157 * tools/cgroup/iocost_monitor.py. For details on drgn, please see
158 * https://github.com/osandov/drgn. The ouput looks like the following.
160 * sdb RUN per=300ms cur_per=234.218:v203.695 busy= +1 vrate= 62.12%
161 * active weight hweight% inflt% dbt delay usages%
162 * test/a * 50/ 50 33.33/ 33.33 27.65 2 0*041 033:033:033
163 * test/b * 100/ 100 66.67/ 66.67 17.56 0 0*000 066:079:077
165 * - per : Timer period
166 * - cur_per : Internal wall and device vtime clock
167 * - vrate : Device virtual time rate against wall clock
168 * - weight : Surplus-adjusted and configured weights
169 * - hweight : Surplus-adjusted and configured hierarchical weights
170 * - inflt : The percentage of in-flight IO cost at the end of last period
171 * - del_ms : Deferred issuer delay induction level and duration
172 * - usages : Usage history
175 #include <linux/kernel.h>
176 #include <linux/module.h>
177 #include <linux/timer.h>
178 #include <linux/time64.h>
179 #include <linux/parser.h>
180 #include <linux/sched/signal.h>
181 #include <linux/blk-cgroup.h>
182 #include <asm/local.h>
183 #include <asm/local64.h>
184 #include "blk-rq-qos.h"
185 #include "blk-stat.h"
188 #ifdef CONFIG_TRACEPOINTS
190 /* copied from TRACE_CGROUP_PATH, see cgroup-internal.h */
191 #define TRACE_IOCG_PATH_LEN 1024
192 static DEFINE_SPINLOCK(trace_iocg_path_lock
);
193 static char trace_iocg_path
[TRACE_IOCG_PATH_LEN
];
195 #define TRACE_IOCG_PATH(type, iocg, ...) \
197 unsigned long flags; \
198 if (trace_iocost_##type##_enabled()) { \
199 spin_lock_irqsave(&trace_iocg_path_lock, flags); \
200 cgroup_path(iocg_to_blkg(iocg)->blkcg->css.cgroup, \
201 trace_iocg_path, TRACE_IOCG_PATH_LEN); \
202 trace_iocost_##type(iocg, trace_iocg_path, \
204 spin_unlock_irqrestore(&trace_iocg_path_lock, flags); \
208 #else /* CONFIG_TRACE_POINTS */
209 #define TRACE_IOCG_PATH(type, iocg, ...) do { } while (0)
210 #endif /* CONFIG_TRACE_POINTS */
215 /* timer period is calculated from latency requirements, bound it */
216 MIN_PERIOD
= USEC_PER_MSEC
,
217 MAX_PERIOD
= USEC_PER_SEC
,
220 * A cgroup's vtime can run 50% behind the device vtime, which
221 * serves as its IO credit buffer. Surplus weight adjustment is
222 * immediately canceled if the vtime margin runs below 10%.
225 INUSE_MARGIN_PCT
= 10,
227 /* Have some play in waitq timer operations */
228 WAITQ_TIMER_MARGIN_PCT
= 5,
231 * vtime can wrap well within a reasonable uptime when vrate is
232 * consistently raised. Don't trust recorded cgroup vtime if the
233 * period counter indicates that it's older than 5mins.
235 VTIME_VALID_DUR
= 300 * USEC_PER_SEC
,
238 * Remember the past three non-zero usages and use the max for
239 * surplus calculation. Three slots guarantee that we remember one
240 * full period usage from the last active stretch even after
241 * partial deactivation and re-activation periods. Don't start
242 * giving away weight before collecting two data points to prevent
243 * hweight adjustments based on one partial activation period.
246 MIN_VALID_USAGES
= 2,
248 /* 1/64k is granular enough and can easily be handled w/ u32 */
249 HWEIGHT_WHOLE
= 1 << 16,
252 * As vtime is used to calculate the cost of each IO, it needs to
253 * be fairly high precision. For example, it should be able to
254 * represent the cost of a single page worth of discard with
255 * suffificient accuracy. At the same time, it should be able to
256 * represent reasonably long enough durations to be useful and
257 * convenient during operation.
259 * 1s worth of vtime is 2^37. This gives us both sub-nanosecond
260 * granularity and days of wrap-around time even at extreme vrates.
262 VTIME_PER_SEC_SHIFT
= 37,
263 VTIME_PER_SEC
= 1LLU << VTIME_PER_SEC_SHIFT
,
264 VTIME_PER_USEC
= VTIME_PER_SEC
/ USEC_PER_SEC
,
265 VTIME_PER_NSEC
= VTIME_PER_SEC
/ NSEC_PER_SEC
,
267 /* bound vrate adjustments within two orders of magnitude */
268 VRATE_MIN_PPM
= 10000, /* 1% */
269 VRATE_MAX_PPM
= 100000000, /* 10000% */
271 VRATE_MIN
= VTIME_PER_USEC
* VRATE_MIN_PPM
/ MILLION
,
272 VRATE_CLAMP_ADJ_PCT
= 4,
274 /* if IOs end up waiting for requests, issue less */
275 RQ_WAIT_BUSY_PCT
= 5,
277 /* unbusy hysterisis */
280 /* don't let cmds which take a very long time pin lagging for too long */
281 MAX_LAGGING_PERIODS
= 10,
284 * If usage% * 1.25 + 2% is lower than hweight% by more than 3%,
285 * donate the surplus.
287 SURPLUS_SCALE_PCT
= 125, /* * 125% */
288 SURPLUS_SCALE_ABS
= HWEIGHT_WHOLE
/ 50, /* + 2% */
289 SURPLUS_MIN_ADJ_DELTA
= HWEIGHT_WHOLE
/ 33, /* 3% */
291 /* switch iff the conditions are met for longer than this */
292 AUTOP_CYCLE_NSEC
= 10LLU * NSEC_PER_SEC
,
295 * Count IO size in 4k pages. The 12bit shift helps keeping
296 * size-proportional components of cost calculation in closer
297 * numbers of digits to per-IO cost components.
300 IOC_PAGE_SIZE
= 1 << IOC_PAGE_SHIFT
,
301 IOC_SECT_TO_PAGE_SHIFT
= IOC_PAGE_SHIFT
- SECTOR_SHIFT
,
303 /* if apart further than 16M, consider randio for linear model */
304 LCOEF_RANDIO_PAGES
= 4096,
313 /* io.cost.qos controls including per-dev enable of the whole controller */
320 /* io.cost.qos params */
331 /* io.cost.model controls */
338 /* builtin linear cost model coefficients */
370 u32 qos
[NR_QOS_PARAMS
];
371 u64 i_lcoefs
[NR_I_LCOEFS
];
372 u64 lcoefs
[NR_LCOEFS
];
373 u32 too_fast_vrate_pct
;
374 u32 too_slow_vrate_pct
;
384 struct ioc_pcpu_stat
{
385 struct ioc_missed missed
[2];
387 local64_t rq_wait_ns
;
397 struct ioc_params params
;
404 struct timer_list timer
;
405 struct list_head active_iocgs
; /* active cgroups */
406 struct ioc_pcpu_stat __percpu
*pcpu_stat
;
408 enum ioc_running running
;
409 atomic64_t vtime_rate
;
411 seqcount_spinlock_t period_seqcount
;
412 u32 period_at
; /* wallclock starttime */
413 u64 period_at_vtime
; /* vtime starttime */
415 atomic64_t cur_period
; /* inc'd each period */
416 int busy_level
; /* saturation history */
418 u64 inuse_margin_vtime
;
419 bool weights_updated
;
420 atomic_t hweight_gen
; /* for lazy hweights */
422 u64 autop_too_fast_at
;
423 u64 autop_too_slow_at
;
425 bool user_qos_params
:1;
426 bool user_cost_model
:1;
429 /* per device-cgroup pair */
431 struct blkg_policy_data pd
;
435 * A iocg can get its weight from two sources - an explicit
436 * per-device-cgroup configuration or the default weight of the
437 * cgroup. `cfg_weight` is the explicit per-device-cgroup
438 * configuration. `weight` is the effective considering both
441 * When an idle cgroup becomes active its `active` goes from 0 to
442 * `weight`. `inuse` is the surplus adjusted active weight.
443 * `active` and `inuse` are used to calculate `hweight_active` and
446 * `last_inuse` remembers `inuse` while an iocg is idle to persist
447 * surplus adjustments.
455 sector_t cursor
; /* to detect randio */
458 * `vtime` is this iocg's vtime cursor which progresses as IOs are
459 * issued. If lagging behind device vtime, the delta represents
460 * the currently available IO budget. If runnning ahead, the
463 * `vtime_done` is the same but progressed on completion rather
464 * than issue. The delta behind `vtime` represents the cost of
465 * currently in-flight IOs.
467 * `last_vtime` is used to remember `vtime` at the end of the last
468 * period to calculate utilization.
471 atomic64_t done_vtime
;
476 * The period this iocg was last active in. Used for deactivation
477 * and invalidating `vtime`.
479 atomic64_t active_period
;
480 struct list_head active_list
;
482 /* see __propagate_active_weight() and current_hweight() for details */
483 u64 child_active_sum
;
490 struct wait_queue_head waitq
;
491 struct hrtimer waitq_timer
;
492 struct hrtimer delay_timer
;
494 /* usage is recorded as fractions of HWEIGHT_WHOLE */
496 u32 usages
[NR_USAGE_SLOTS
];
498 /* this iocg's depth in the hierarchy and ancestors including self */
500 struct ioc_gq
*ancestors
[];
505 struct blkcg_policy_data cpd
;
506 unsigned int dfl_weight
;
517 struct wait_queue_entry wait
;
523 struct iocg_wake_ctx
{
529 static const struct ioc_params autop
[] = {
532 [QOS_RLAT
] = 250000, /* 250ms */
534 [QOS_MIN
] = VRATE_MIN_PPM
,
535 [QOS_MAX
] = VRATE_MAX_PPM
,
538 [I_LCOEF_RBPS
] = 174019176,
539 [I_LCOEF_RSEQIOPS
] = 41708,
540 [I_LCOEF_RRANDIOPS
] = 370,
541 [I_LCOEF_WBPS
] = 178075866,
542 [I_LCOEF_WSEQIOPS
] = 42705,
543 [I_LCOEF_WRANDIOPS
] = 378,
548 [QOS_RLAT
] = 25000, /* 25ms */
550 [QOS_MIN
] = VRATE_MIN_PPM
,
551 [QOS_MAX
] = VRATE_MAX_PPM
,
554 [I_LCOEF_RBPS
] = 245855193,
555 [I_LCOEF_RSEQIOPS
] = 61575,
556 [I_LCOEF_RRANDIOPS
] = 6946,
557 [I_LCOEF_WBPS
] = 141365009,
558 [I_LCOEF_WSEQIOPS
] = 33716,
559 [I_LCOEF_WRANDIOPS
] = 26796,
564 [QOS_RLAT
] = 25000, /* 25ms */
566 [QOS_MIN
] = VRATE_MIN_PPM
,
567 [QOS_MAX
] = VRATE_MAX_PPM
,
570 [I_LCOEF_RBPS
] = 488636629,
571 [I_LCOEF_RSEQIOPS
] = 8932,
572 [I_LCOEF_RRANDIOPS
] = 8518,
573 [I_LCOEF_WBPS
] = 427891549,
574 [I_LCOEF_WSEQIOPS
] = 28755,
575 [I_LCOEF_WRANDIOPS
] = 21940,
577 .too_fast_vrate_pct
= 500,
581 [QOS_RLAT
] = 5000, /* 5ms */
583 [QOS_MIN
] = VRATE_MIN_PPM
,
584 [QOS_MAX
] = VRATE_MAX_PPM
,
587 [I_LCOEF_RBPS
] = 3102524156LLU,
588 [I_LCOEF_RSEQIOPS
] = 724816,
589 [I_LCOEF_RRANDIOPS
] = 778122,
590 [I_LCOEF_WBPS
] = 1742780862LLU,
591 [I_LCOEF_WSEQIOPS
] = 425702,
592 [I_LCOEF_WRANDIOPS
] = 443193,
594 .too_slow_vrate_pct
= 10,
599 * vrate adjust percentages indexed by ioc->busy_level. We adjust up on
600 * vtime credit shortage and down on device saturation.
602 static u32 vrate_adj_pct
[] =
604 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
605 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
606 4, 4, 4, 4, 4, 4, 4, 4, 8, 8, 8, 8, 8, 8, 8, 8, 16 };
608 static struct blkcg_policy blkcg_policy_iocost
;
610 /* accessors and helpers */
611 static struct ioc
*rqos_to_ioc(struct rq_qos
*rqos
)
613 return container_of(rqos
, struct ioc
, rqos
);
616 static struct ioc
*q_to_ioc(struct request_queue
*q
)
618 return rqos_to_ioc(rq_qos_id(q
, RQ_QOS_COST
));
621 static const char *q_name(struct request_queue
*q
)
623 if (test_bit(QUEUE_FLAG_REGISTERED
, &q
->queue_flags
))
624 return kobject_name(q
->kobj
.parent
);
629 static const char __maybe_unused
*ioc_name(struct ioc
*ioc
)
631 return q_name(ioc
->rqos
.q
);
634 static struct ioc_gq
*pd_to_iocg(struct blkg_policy_data
*pd
)
636 return pd
? container_of(pd
, struct ioc_gq
, pd
) : NULL
;
639 static struct ioc_gq
*blkg_to_iocg(struct blkcg_gq
*blkg
)
641 return pd_to_iocg(blkg_to_pd(blkg
, &blkcg_policy_iocost
));
644 static struct blkcg_gq
*iocg_to_blkg(struct ioc_gq
*iocg
)
646 return pd_to_blkg(&iocg
->pd
);
649 static struct ioc_cgrp
*blkcg_to_iocc(struct blkcg
*blkcg
)
651 return container_of(blkcg_to_cpd(blkcg
, &blkcg_policy_iocost
),
652 struct ioc_cgrp
, cpd
);
656 * Scale @abs_cost to the inverse of @hw_inuse. The lower the hierarchical
657 * weight, the more expensive each IO. Must round up.
659 static u64
abs_cost_to_cost(u64 abs_cost
, u32 hw_inuse
)
661 return DIV64_U64_ROUND_UP(abs_cost
* HWEIGHT_WHOLE
, hw_inuse
);
665 * The inverse of abs_cost_to_cost(). Must round up.
667 static u64
cost_to_abs_cost(u64 cost
, u32 hw_inuse
)
669 return DIV64_U64_ROUND_UP(cost
* hw_inuse
, HWEIGHT_WHOLE
);
672 static void iocg_commit_bio(struct ioc_gq
*iocg
, struct bio
*bio
, u64 cost
)
674 bio
->bi_iocost_cost
= cost
;
675 atomic64_add(cost
, &iocg
->vtime
);
678 #define CREATE_TRACE_POINTS
679 #include <trace/events/iocost.h>
681 /* latency Qos params changed, update period_us and all the dependent params */
682 static void ioc_refresh_period_us(struct ioc
*ioc
)
684 u32 ppm
, lat
, multi
, period_us
;
686 lockdep_assert_held(&ioc
->lock
);
688 /* pick the higher latency target */
689 if (ioc
->params
.qos
[QOS_RLAT
] >= ioc
->params
.qos
[QOS_WLAT
]) {
690 ppm
= ioc
->params
.qos
[QOS_RPPM
];
691 lat
= ioc
->params
.qos
[QOS_RLAT
];
693 ppm
= ioc
->params
.qos
[QOS_WPPM
];
694 lat
= ioc
->params
.qos
[QOS_WLAT
];
698 * We want the period to be long enough to contain a healthy number
699 * of IOs while short enough for granular control. Define it as a
700 * multiple of the latency target. Ideally, the multiplier should
701 * be scaled according to the percentile so that it would nominally
702 * contain a certain number of requests. Let's be simpler and
703 * scale it linearly so that it's 2x >= pct(90) and 10x at pct(50).
706 multi
= max_t(u32
, (MILLION
- ppm
) / 50000, 2);
709 period_us
= multi
* lat
;
710 period_us
= clamp_t(u32
, period_us
, MIN_PERIOD
, MAX_PERIOD
);
712 /* calculate dependent params */
713 ioc
->period_us
= period_us
;
714 ioc
->margin_us
= period_us
* MARGIN_PCT
/ 100;
715 ioc
->inuse_margin_vtime
= DIV64_U64_ROUND_UP(
716 period_us
* VTIME_PER_USEC
* INUSE_MARGIN_PCT
, 100);
719 static int ioc_autop_idx(struct ioc
*ioc
)
721 int idx
= ioc
->autop_idx
;
722 const struct ioc_params
*p
= &autop
[idx
];
727 if (!blk_queue_nonrot(ioc
->rqos
.q
))
730 /* handle SATA SSDs w/ broken NCQ */
731 if (blk_queue_depth(ioc
->rqos
.q
) == 1)
732 return AUTOP_SSD_QD1
;
734 /* use one of the normal ssd sets */
735 if (idx
< AUTOP_SSD_DFL
)
736 return AUTOP_SSD_DFL
;
738 /* if user is overriding anything, maintain what was there */
739 if (ioc
->user_qos_params
|| ioc
->user_cost_model
)
742 /* step up/down based on the vrate */
743 vrate_pct
= div64_u64(atomic64_read(&ioc
->vtime_rate
) * 100,
745 now_ns
= ktime_get_ns();
747 if (p
->too_fast_vrate_pct
&& p
->too_fast_vrate_pct
<= vrate_pct
) {
748 if (!ioc
->autop_too_fast_at
)
749 ioc
->autop_too_fast_at
= now_ns
;
750 if (now_ns
- ioc
->autop_too_fast_at
>= AUTOP_CYCLE_NSEC
)
753 ioc
->autop_too_fast_at
= 0;
756 if (p
->too_slow_vrate_pct
&& p
->too_slow_vrate_pct
>= vrate_pct
) {
757 if (!ioc
->autop_too_slow_at
)
758 ioc
->autop_too_slow_at
= now_ns
;
759 if (now_ns
- ioc
->autop_too_slow_at
>= AUTOP_CYCLE_NSEC
)
762 ioc
->autop_too_slow_at
= 0;
769 * Take the followings as input
771 * @bps maximum sequential throughput
772 * @seqiops maximum sequential 4k iops
773 * @randiops maximum random 4k iops
775 * and calculate the linear model cost coefficients.
777 * *@page per-page cost 1s / (@bps / 4096)
778 * *@seqio base cost of a seq IO max((1s / @seqiops) - *@page, 0)
779 * @randiops base cost of a rand IO max((1s / @randiops) - *@page, 0)
781 static void calc_lcoefs(u64 bps
, u64 seqiops
, u64 randiops
,
782 u64
*page
, u64
*seqio
, u64
*randio
)
786 *page
= *seqio
= *randio
= 0;
789 *page
= DIV64_U64_ROUND_UP(VTIME_PER_SEC
,
790 DIV_ROUND_UP_ULL(bps
, IOC_PAGE_SIZE
));
793 v
= DIV64_U64_ROUND_UP(VTIME_PER_SEC
, seqiops
);
799 v
= DIV64_U64_ROUND_UP(VTIME_PER_SEC
, randiops
);
805 static void ioc_refresh_lcoefs(struct ioc
*ioc
)
807 u64
*u
= ioc
->params
.i_lcoefs
;
808 u64
*c
= ioc
->params
.lcoefs
;
810 calc_lcoefs(u
[I_LCOEF_RBPS
], u
[I_LCOEF_RSEQIOPS
], u
[I_LCOEF_RRANDIOPS
],
811 &c
[LCOEF_RPAGE
], &c
[LCOEF_RSEQIO
], &c
[LCOEF_RRANDIO
]);
812 calc_lcoefs(u
[I_LCOEF_WBPS
], u
[I_LCOEF_WSEQIOPS
], u
[I_LCOEF_WRANDIOPS
],
813 &c
[LCOEF_WPAGE
], &c
[LCOEF_WSEQIO
], &c
[LCOEF_WRANDIO
]);
816 static bool ioc_refresh_params(struct ioc
*ioc
, bool force
)
818 const struct ioc_params
*p
;
821 lockdep_assert_held(&ioc
->lock
);
823 idx
= ioc_autop_idx(ioc
);
826 if (idx
== ioc
->autop_idx
&& !force
)
829 if (idx
!= ioc
->autop_idx
)
830 atomic64_set(&ioc
->vtime_rate
, VTIME_PER_USEC
);
832 ioc
->autop_idx
= idx
;
833 ioc
->autop_too_fast_at
= 0;
834 ioc
->autop_too_slow_at
= 0;
836 if (!ioc
->user_qos_params
)
837 memcpy(ioc
->params
.qos
, p
->qos
, sizeof(p
->qos
));
838 if (!ioc
->user_cost_model
)
839 memcpy(ioc
->params
.i_lcoefs
, p
->i_lcoefs
, sizeof(p
->i_lcoefs
));
841 ioc_refresh_period_us(ioc
);
842 ioc_refresh_lcoefs(ioc
);
844 ioc
->vrate_min
= DIV64_U64_ROUND_UP((u64
)ioc
->params
.qos
[QOS_MIN
] *
845 VTIME_PER_USEC
, MILLION
);
846 ioc
->vrate_max
= div64_u64((u64
)ioc
->params
.qos
[QOS_MAX
] *
847 VTIME_PER_USEC
, MILLION
);
852 /* take a snapshot of the current [v]time and vrate */
853 static void ioc_now(struct ioc
*ioc
, struct ioc_now
*now
)
857 now
->now_ns
= ktime_get();
858 now
->now
= ktime_to_us(now
->now_ns
);
859 now
->vrate
= atomic64_read(&ioc
->vtime_rate
);
862 * The current vtime is
864 * vtime at period start + (wallclock time since the start) * vrate
866 * As a consistent snapshot of `period_at_vtime` and `period_at` is
867 * needed, they're seqcount protected.
870 seq
= read_seqcount_begin(&ioc
->period_seqcount
);
871 now
->vnow
= ioc
->period_at_vtime
+
872 (now
->now
- ioc
->period_at
) * now
->vrate
;
873 } while (read_seqcount_retry(&ioc
->period_seqcount
, seq
));
876 static void ioc_start_period(struct ioc
*ioc
, struct ioc_now
*now
)
878 WARN_ON_ONCE(ioc
->running
!= IOC_RUNNING
);
880 write_seqcount_begin(&ioc
->period_seqcount
);
881 ioc
->period_at
= now
->now
;
882 ioc
->period_at_vtime
= now
->vnow
;
883 write_seqcount_end(&ioc
->period_seqcount
);
885 ioc
->timer
.expires
= jiffies
+ usecs_to_jiffies(ioc
->period_us
);
886 add_timer(&ioc
->timer
);
890 * Update @iocg's `active` and `inuse` to @active and @inuse, update level
891 * weight sums and propagate upwards accordingly.
893 static void __propagate_active_weight(struct ioc_gq
*iocg
, u32 active
, u32 inuse
)
895 struct ioc
*ioc
= iocg
->ioc
;
898 lockdep_assert_held(&ioc
->lock
);
900 inuse
= min(active
, inuse
);
902 for (lvl
= iocg
->level
- 1; lvl
>= 0; lvl
--) {
903 struct ioc_gq
*parent
= iocg
->ancestors
[lvl
];
904 struct ioc_gq
*child
= iocg
->ancestors
[lvl
+ 1];
905 u32 parent_active
= 0, parent_inuse
= 0;
907 /* update the level sums */
908 parent
->child_active_sum
+= (s32
)(active
- child
->active
);
909 parent
->child_inuse_sum
+= (s32
)(inuse
- child
->inuse
);
910 /* apply the udpates */
911 child
->active
= active
;
912 child
->inuse
= inuse
;
915 * The delta between inuse and active sums indicates that
916 * that much of weight is being given away. Parent's inuse
917 * and active should reflect the ratio.
919 if (parent
->child_active_sum
) {
920 parent_active
= parent
->weight
;
921 parent_inuse
= DIV64_U64_ROUND_UP(
922 parent_active
* parent
->child_inuse_sum
,
923 parent
->child_active_sum
);
926 /* do we need to keep walking up? */
927 if (parent_active
== parent
->active
&&
928 parent_inuse
== parent
->inuse
)
931 active
= parent_active
;
932 inuse
= parent_inuse
;
935 ioc
->weights_updated
= true;
938 static void commit_active_weights(struct ioc
*ioc
)
940 lockdep_assert_held(&ioc
->lock
);
942 if (ioc
->weights_updated
) {
943 /* paired with rmb in current_hweight(), see there */
945 atomic_inc(&ioc
->hweight_gen
);
946 ioc
->weights_updated
= false;
950 static void propagate_active_weight(struct ioc_gq
*iocg
, u32 active
, u32 inuse
)
952 __propagate_active_weight(iocg
, active
, inuse
);
953 commit_active_weights(iocg
->ioc
);
956 static void current_hweight(struct ioc_gq
*iocg
, u32
*hw_activep
, u32
*hw_inusep
)
958 struct ioc
*ioc
= iocg
->ioc
;
963 /* hot path - if uptodate, use cached */
964 ioc_gen
= atomic_read(&ioc
->hweight_gen
);
965 if (ioc_gen
== iocg
->hweight_gen
)
969 * Paired with wmb in commit_active_weights(). If we saw the
970 * updated hweight_gen, all the weight updates from
971 * __propagate_active_weight() are visible too.
973 * We can race with weight updates during calculation and get it
974 * wrong. However, hweight_gen would have changed and a future
975 * reader will recalculate and we're guaranteed to discard the
980 hwa
= hwi
= HWEIGHT_WHOLE
;
981 for (lvl
= 0; lvl
<= iocg
->level
- 1; lvl
++) {
982 struct ioc_gq
*parent
= iocg
->ancestors
[lvl
];
983 struct ioc_gq
*child
= iocg
->ancestors
[lvl
+ 1];
984 u32 active_sum
= READ_ONCE(parent
->child_active_sum
);
985 u32 inuse_sum
= READ_ONCE(parent
->child_inuse_sum
);
986 u32 active
= READ_ONCE(child
->active
);
987 u32 inuse
= READ_ONCE(child
->inuse
);
989 /* we can race with deactivations and either may read as zero */
990 if (!active_sum
|| !inuse_sum
)
993 active_sum
= max(active
, active_sum
);
994 hwa
= hwa
* active
/ active_sum
; /* max 16bits * 10000 */
996 inuse_sum
= max(inuse
, inuse_sum
);
997 hwi
= hwi
* inuse
/ inuse_sum
; /* max 16bits * 10000 */
1000 iocg
->hweight_active
= max_t(u32
, hwa
, 1);
1001 iocg
->hweight_inuse
= max_t(u32
, hwi
, 1);
1002 iocg
->hweight_gen
= ioc_gen
;
1005 *hw_activep
= iocg
->hweight_active
;
1007 *hw_inusep
= iocg
->hweight_inuse
;
1010 static void weight_updated(struct ioc_gq
*iocg
)
1012 struct ioc
*ioc
= iocg
->ioc
;
1013 struct blkcg_gq
*blkg
= iocg_to_blkg(iocg
);
1014 struct ioc_cgrp
*iocc
= blkcg_to_iocc(blkg
->blkcg
);
1017 lockdep_assert_held(&ioc
->lock
);
1019 weight
= iocg
->cfg_weight
?: iocc
->dfl_weight
;
1020 if (weight
!= iocg
->weight
&& iocg
->active
)
1021 propagate_active_weight(iocg
, weight
,
1022 DIV64_U64_ROUND_UP(iocg
->inuse
* weight
, iocg
->weight
));
1023 iocg
->weight
= weight
;
1026 static bool iocg_activate(struct ioc_gq
*iocg
, struct ioc_now
*now
)
1028 struct ioc
*ioc
= iocg
->ioc
;
1029 u64 last_period
, cur_period
, max_period_delta
;
1030 u64 vtime
, vmargin
, vmin
;
1034 * If seem to be already active, just update the stamp to tell the
1035 * timer that we're still active. We don't mind occassional races.
1037 if (!list_empty(&iocg
->active_list
)) {
1039 cur_period
= atomic64_read(&ioc
->cur_period
);
1040 if (atomic64_read(&iocg
->active_period
) != cur_period
)
1041 atomic64_set(&iocg
->active_period
, cur_period
);
1045 /* racy check on internal node IOs, treat as root level IOs */
1046 if (iocg
->child_active_sum
)
1049 spin_lock_irq(&ioc
->lock
);
1054 cur_period
= atomic64_read(&ioc
->cur_period
);
1055 last_period
= atomic64_read(&iocg
->active_period
);
1056 atomic64_set(&iocg
->active_period
, cur_period
);
1058 /* already activated or breaking leaf-only constraint? */
1059 if (!list_empty(&iocg
->active_list
))
1060 goto succeed_unlock
;
1061 for (i
= iocg
->level
- 1; i
> 0; i
--)
1062 if (!list_empty(&iocg
->ancestors
[i
]->active_list
))
1065 if (iocg
->child_active_sum
)
1069 * vtime may wrap when vrate is raised substantially due to
1070 * underestimated IO costs. Look at the period and ignore its
1071 * vtime if the iocg has been idle for too long. Also, cap the
1072 * budget it can start with to the margin.
1074 max_period_delta
= DIV64_U64_ROUND_UP(VTIME_VALID_DUR
, ioc
->period_us
);
1075 vtime
= atomic64_read(&iocg
->vtime
);
1076 vmargin
= ioc
->margin_us
* now
->vrate
;
1077 vmin
= now
->vnow
- vmargin
;
1079 if (last_period
+ max_period_delta
< cur_period
||
1080 time_before64(vtime
, vmin
)) {
1081 atomic64_add(vmin
- vtime
, &iocg
->vtime
);
1082 atomic64_add(vmin
- vtime
, &iocg
->done_vtime
);
1087 * Activate, propagate weight and start period timer if not
1088 * running. Reset hweight_gen to avoid accidental match from
1091 iocg
->hweight_gen
= atomic_read(&ioc
->hweight_gen
) - 1;
1092 list_add(&iocg
->active_list
, &ioc
->active_iocgs
);
1093 propagate_active_weight(iocg
, iocg
->weight
,
1094 iocg
->last_inuse
?: iocg
->weight
);
1096 TRACE_IOCG_PATH(iocg_activate
, iocg
, now
,
1097 last_period
, cur_period
, vtime
);
1099 iocg
->last_vtime
= vtime
;
1101 if (ioc
->running
== IOC_IDLE
) {
1102 ioc
->running
= IOC_RUNNING
;
1103 ioc_start_period(ioc
, now
);
1107 spin_unlock_irq(&ioc
->lock
);
1111 spin_unlock_irq(&ioc
->lock
);
1115 static int iocg_wake_fn(struct wait_queue_entry
*wq_entry
, unsigned mode
,
1116 int flags
, void *key
)
1118 struct iocg_wait
*wait
= container_of(wq_entry
, struct iocg_wait
, wait
);
1119 struct iocg_wake_ctx
*ctx
= (struct iocg_wake_ctx
*)key
;
1120 u64 cost
= abs_cost_to_cost(wait
->abs_cost
, ctx
->hw_inuse
);
1122 ctx
->vbudget
-= cost
;
1124 if (ctx
->vbudget
< 0)
1127 iocg_commit_bio(ctx
->iocg
, wait
->bio
, cost
);
1130 * autoremove_wake_function() removes the wait entry only when it
1131 * actually changed the task state. We want the wait always
1132 * removed. Remove explicitly and use default_wake_function().
1134 list_del_init(&wq_entry
->entry
);
1135 wait
->committed
= true;
1137 default_wake_function(wq_entry
, mode
, flags
, key
);
1141 static void iocg_kick_waitq(struct ioc_gq
*iocg
, struct ioc_now
*now
)
1143 struct ioc
*ioc
= iocg
->ioc
;
1144 struct iocg_wake_ctx ctx
= { .iocg
= iocg
};
1145 u64 margin_ns
= (u64
)(ioc
->period_us
*
1146 WAITQ_TIMER_MARGIN_PCT
/ 100) * NSEC_PER_USEC
;
1147 u64 vdebt
, vshortage
, expires
, oexpires
;
1151 lockdep_assert_held(&iocg
->waitq
.lock
);
1153 current_hweight(iocg
, NULL
, &hw_inuse
);
1154 vbudget
= now
->vnow
- atomic64_read(&iocg
->vtime
);
1157 vdebt
= abs_cost_to_cost(iocg
->abs_vdebt
, hw_inuse
);
1158 if (vdebt
&& vbudget
> 0) {
1159 u64 delta
= min_t(u64
, vbudget
, vdebt
);
1160 u64 abs_delta
= min(cost_to_abs_cost(delta
, hw_inuse
),
1163 atomic64_add(delta
, &iocg
->vtime
);
1164 atomic64_add(delta
, &iocg
->done_vtime
);
1165 iocg
->abs_vdebt
-= abs_delta
;
1169 * Wake up the ones which are due and see how much vtime we'll need
1172 ctx
.hw_inuse
= hw_inuse
;
1173 ctx
.vbudget
= vbudget
- vdebt
;
1174 __wake_up_locked_key(&iocg
->waitq
, TASK_NORMAL
, &ctx
);
1175 if (!waitqueue_active(&iocg
->waitq
))
1177 if (WARN_ON_ONCE(ctx
.vbudget
>= 0))
1180 /* determine next wakeup, add a quarter margin to guarantee chunking */
1181 vshortage
= -ctx
.vbudget
;
1182 expires
= now
->now_ns
+
1183 DIV64_U64_ROUND_UP(vshortage
, now
->vrate
) * NSEC_PER_USEC
;
1184 expires
+= margin_ns
/ 4;
1186 /* if already active and close enough, don't bother */
1187 oexpires
= ktime_to_ns(hrtimer_get_softexpires(&iocg
->waitq_timer
));
1188 if (hrtimer_is_queued(&iocg
->waitq_timer
) &&
1189 abs(oexpires
- expires
) <= margin_ns
/ 4)
1192 hrtimer_start_range_ns(&iocg
->waitq_timer
, ns_to_ktime(expires
),
1193 margin_ns
/ 4, HRTIMER_MODE_ABS
);
1196 static enum hrtimer_restart
iocg_waitq_timer_fn(struct hrtimer
*timer
)
1198 struct ioc_gq
*iocg
= container_of(timer
, struct ioc_gq
, waitq_timer
);
1200 unsigned long flags
;
1202 ioc_now(iocg
->ioc
, &now
);
1204 spin_lock_irqsave(&iocg
->waitq
.lock
, flags
);
1205 iocg_kick_waitq(iocg
, &now
);
1206 spin_unlock_irqrestore(&iocg
->waitq
.lock
, flags
);
1208 return HRTIMER_NORESTART
;
1211 static bool iocg_kick_delay(struct ioc_gq
*iocg
, struct ioc_now
*now
)
1213 struct ioc
*ioc
= iocg
->ioc
;
1214 struct blkcg_gq
*blkg
= iocg_to_blkg(iocg
);
1215 u64 vtime
= atomic64_read(&iocg
->vtime
);
1216 u64 vmargin
= ioc
->margin_us
* now
->vrate
;
1217 u64 margin_ns
= ioc
->margin_us
* NSEC_PER_USEC
;
1218 u64 delta_ns
, expires
, oexpires
;
1221 lockdep_assert_held(&iocg
->waitq
.lock
);
1223 /* debt-adjust vtime */
1224 current_hweight(iocg
, NULL
, &hw_inuse
);
1225 vtime
+= abs_cost_to_cost(iocg
->abs_vdebt
, hw_inuse
);
1228 * Clear or maintain depending on the overage. Non-zero vdebt is what
1229 * guarantees that @iocg is online and future iocg_kick_delay() will
1230 * clear use_delay. Don't leave it on when there's no vdebt.
1232 if (!iocg
->abs_vdebt
|| time_before_eq64(vtime
, now
->vnow
)) {
1233 blkcg_clear_delay(blkg
);
1236 if (!atomic_read(&blkg
->use_delay
) &&
1237 time_before_eq64(vtime
, now
->vnow
+ vmargin
))
1241 delta_ns
= DIV64_U64_ROUND_UP(vtime
- now
->vnow
,
1242 now
->vrate
) * NSEC_PER_USEC
;
1243 blkcg_set_delay(blkg
, delta_ns
);
1244 expires
= now
->now_ns
+ delta_ns
;
1246 /* if already active and close enough, don't bother */
1247 oexpires
= ktime_to_ns(hrtimer_get_softexpires(&iocg
->delay_timer
));
1248 if (hrtimer_is_queued(&iocg
->delay_timer
) &&
1249 abs(oexpires
- expires
) <= margin_ns
/ 4)
1252 hrtimer_start_range_ns(&iocg
->delay_timer
, ns_to_ktime(expires
),
1253 margin_ns
/ 4, HRTIMER_MODE_ABS
);
1257 static enum hrtimer_restart
iocg_delay_timer_fn(struct hrtimer
*timer
)
1259 struct ioc_gq
*iocg
= container_of(timer
, struct ioc_gq
, delay_timer
);
1261 unsigned long flags
;
1263 spin_lock_irqsave(&iocg
->waitq
.lock
, flags
);
1264 ioc_now(iocg
->ioc
, &now
);
1265 iocg_kick_delay(iocg
, &now
);
1266 spin_unlock_irqrestore(&iocg
->waitq
.lock
, flags
);
1268 return HRTIMER_NORESTART
;
1271 static void ioc_lat_stat(struct ioc
*ioc
, u32
*missed_ppm_ar
, u32
*rq_wait_pct_p
)
1273 u32 nr_met
[2] = { };
1274 u32 nr_missed
[2] = { };
1278 for_each_online_cpu(cpu
) {
1279 struct ioc_pcpu_stat
*stat
= per_cpu_ptr(ioc
->pcpu_stat
, cpu
);
1280 u64 this_rq_wait_ns
;
1282 for (rw
= READ
; rw
<= WRITE
; rw
++) {
1283 u32 this_met
= local_read(&stat
->missed
[rw
].nr_met
);
1284 u32 this_missed
= local_read(&stat
->missed
[rw
].nr_missed
);
1286 nr_met
[rw
] += this_met
- stat
->missed
[rw
].last_met
;
1287 nr_missed
[rw
] += this_missed
- stat
->missed
[rw
].last_missed
;
1288 stat
->missed
[rw
].last_met
= this_met
;
1289 stat
->missed
[rw
].last_missed
= this_missed
;
1292 this_rq_wait_ns
= local64_read(&stat
->rq_wait_ns
);
1293 rq_wait_ns
+= this_rq_wait_ns
- stat
->last_rq_wait_ns
;
1294 stat
->last_rq_wait_ns
= this_rq_wait_ns
;
1297 for (rw
= READ
; rw
<= WRITE
; rw
++) {
1298 if (nr_met
[rw
] + nr_missed
[rw
])
1300 DIV64_U64_ROUND_UP((u64
)nr_missed
[rw
] * MILLION
,
1301 nr_met
[rw
] + nr_missed
[rw
]);
1303 missed_ppm_ar
[rw
] = 0;
1306 *rq_wait_pct_p
= div64_u64(rq_wait_ns
* 100,
1307 ioc
->period_us
* NSEC_PER_USEC
);
1310 /* was iocg idle this period? */
1311 static bool iocg_is_idle(struct ioc_gq
*iocg
)
1313 struct ioc
*ioc
= iocg
->ioc
;
1315 /* did something get issued this period? */
1316 if (atomic64_read(&iocg
->active_period
) ==
1317 atomic64_read(&ioc
->cur_period
))
1320 /* is something in flight? */
1321 if (atomic64_read(&iocg
->done_vtime
) != atomic64_read(&iocg
->vtime
))
1327 /* returns usage with margin added if surplus is large enough */
1328 static u32
surplus_adjusted_hweight_inuse(u32 usage
, u32 hw_inuse
)
1331 usage
= DIV_ROUND_UP(usage
* SURPLUS_SCALE_PCT
, 100);
1332 usage
+= SURPLUS_SCALE_ABS
;
1334 /* don't bother if the surplus is too small */
1335 if (usage
+ SURPLUS_MIN_ADJ_DELTA
> hw_inuse
)
1341 static void ioc_timer_fn(struct timer_list
*timer
)
1343 struct ioc
*ioc
= container_of(timer
, struct ioc
, timer
);
1344 struct ioc_gq
*iocg
, *tiocg
;
1346 int nr_surpluses
= 0, nr_shortages
= 0, nr_lagging
= 0;
1347 u32 ppm_rthr
= MILLION
- ioc
->params
.qos
[QOS_RPPM
];
1348 u32 ppm_wthr
= MILLION
- ioc
->params
.qos
[QOS_WPPM
];
1349 u32 missed_ppm
[2], rq_wait_pct
;
1351 int prev_busy_level
, i
;
1353 /* how were the latencies during the period? */
1354 ioc_lat_stat(ioc
, missed_ppm
, &rq_wait_pct
);
1356 /* take care of active iocgs */
1357 spin_lock_irq(&ioc
->lock
);
1361 period_vtime
= now
.vnow
- ioc
->period_at_vtime
;
1362 if (WARN_ON_ONCE(!period_vtime
)) {
1363 spin_unlock_irq(&ioc
->lock
);
1368 * Waiters determine the sleep durations based on the vrate they
1369 * saw at the time of sleep. If vrate has increased, some waiters
1370 * could be sleeping for too long. Wake up tardy waiters which
1371 * should have woken up in the last period and expire idle iocgs.
1373 list_for_each_entry_safe(iocg
, tiocg
, &ioc
->active_iocgs
, active_list
) {
1374 if (!waitqueue_active(&iocg
->waitq
) && !iocg
->abs_vdebt
&&
1375 !iocg_is_idle(iocg
))
1378 spin_lock(&iocg
->waitq
.lock
);
1380 if (waitqueue_active(&iocg
->waitq
) || iocg
->abs_vdebt
) {
1381 /* might be oversleeping vtime / hweight changes, kick */
1382 iocg_kick_waitq(iocg
, &now
);
1383 iocg_kick_delay(iocg
, &now
);
1384 } else if (iocg_is_idle(iocg
)) {
1385 /* no waiter and idle, deactivate */
1386 iocg
->last_inuse
= iocg
->inuse
;
1387 __propagate_active_weight(iocg
, 0, 0);
1388 list_del_init(&iocg
->active_list
);
1391 spin_unlock(&iocg
->waitq
.lock
);
1393 commit_active_weights(ioc
);
1395 /* calc usages and see whether some weights need to be moved around */
1396 list_for_each_entry(iocg
, &ioc
->active_iocgs
, active_list
) {
1397 u64 vdone
, vtime
, vusage
, vmargin
, vmin
;
1398 u32 hw_active
, hw_inuse
, usage
;
1401 * Collect unused and wind vtime closer to vnow to prevent
1402 * iocgs from accumulating a large amount of budget.
1404 vdone
= atomic64_read(&iocg
->done_vtime
);
1405 vtime
= atomic64_read(&iocg
->vtime
);
1406 current_hweight(iocg
, &hw_active
, &hw_inuse
);
1409 * Latency QoS detection doesn't account for IOs which are
1410 * in-flight for longer than a period. Detect them by
1411 * comparing vdone against period start. If lagging behind
1412 * IOs from past periods, don't increase vrate.
1414 if ((ppm_rthr
!= MILLION
|| ppm_wthr
!= MILLION
) &&
1415 !atomic_read(&iocg_to_blkg(iocg
)->use_delay
) &&
1416 time_after64(vtime
, vdone
) &&
1417 time_after64(vtime
, now
.vnow
-
1418 MAX_LAGGING_PERIODS
* period_vtime
) &&
1419 time_before64(vdone
, now
.vnow
- period_vtime
))
1422 if (waitqueue_active(&iocg
->waitq
))
1423 vusage
= now
.vnow
- iocg
->last_vtime
;
1424 else if (time_before64(iocg
->last_vtime
, vtime
))
1425 vusage
= vtime
- iocg
->last_vtime
;
1429 iocg
->last_vtime
+= vusage
;
1431 * Factor in in-flight vtime into vusage to avoid
1432 * high-latency completions appearing as idle. This should
1433 * be done after the above ->last_time adjustment.
1435 vusage
= max(vusage
, vtime
- vdone
);
1437 /* calculate hweight based usage ratio and record */
1439 usage
= DIV64_U64_ROUND_UP(vusage
* hw_inuse
,
1441 iocg
->usage_idx
= (iocg
->usage_idx
+ 1) % NR_USAGE_SLOTS
;
1442 iocg
->usages
[iocg
->usage_idx
] = usage
;
1447 /* see whether there's surplus vtime */
1448 vmargin
= ioc
->margin_us
* now
.vrate
;
1449 vmin
= now
.vnow
- vmargin
;
1451 iocg
->has_surplus
= false;
1453 if (!waitqueue_active(&iocg
->waitq
) &&
1454 time_before64(vtime
, vmin
)) {
1455 u64 delta
= vmin
- vtime
;
1457 /* throw away surplus vtime */
1458 atomic64_add(delta
, &iocg
->vtime
);
1459 atomic64_add(delta
, &iocg
->done_vtime
);
1460 iocg
->last_vtime
+= delta
;
1461 /* if usage is sufficiently low, maybe it can donate */
1462 if (surplus_adjusted_hweight_inuse(usage
, hw_inuse
)) {
1463 iocg
->has_surplus
= true;
1466 } else if (hw_inuse
< hw_active
) {
1467 u32 new_hwi
, new_inuse
;
1469 /* was donating but might need to take back some */
1470 if (waitqueue_active(&iocg
->waitq
)) {
1471 new_hwi
= hw_active
;
1473 new_hwi
= max(hw_inuse
,
1474 usage
* SURPLUS_SCALE_PCT
/ 100 +
1478 new_inuse
= div64_u64((u64
)iocg
->inuse
* new_hwi
,
1480 new_inuse
= clamp_t(u32
, new_inuse
, 1, iocg
->active
);
1482 if (new_inuse
> iocg
->inuse
) {
1483 TRACE_IOCG_PATH(inuse_takeback
, iocg
, &now
,
1484 iocg
->inuse
, new_inuse
,
1486 __propagate_active_weight(iocg
, iocg
->weight
,
1490 /* genuninely out of vtime */
1495 if (!nr_shortages
|| !nr_surpluses
)
1496 goto skip_surplus_transfers
;
1498 /* there are both shortages and surpluses, transfer surpluses */
1499 list_for_each_entry(iocg
, &ioc
->active_iocgs
, active_list
) {
1500 u32 usage
, hw_active
, hw_inuse
, new_hwi
, new_inuse
;
1503 if (!iocg
->has_surplus
)
1506 /* base the decision on max historical usage */
1507 for (i
= 0, usage
= 0; i
< NR_USAGE_SLOTS
; i
++) {
1508 if (iocg
->usages
[i
]) {
1509 usage
= max(usage
, iocg
->usages
[i
]);
1513 if (nr_valid
< MIN_VALID_USAGES
)
1516 current_hweight(iocg
, &hw_active
, &hw_inuse
);
1517 new_hwi
= surplus_adjusted_hweight_inuse(usage
, hw_inuse
);
1521 new_inuse
= DIV64_U64_ROUND_UP((u64
)iocg
->inuse
* new_hwi
,
1523 if (new_inuse
< iocg
->inuse
) {
1524 TRACE_IOCG_PATH(inuse_giveaway
, iocg
, &now
,
1525 iocg
->inuse
, new_inuse
,
1527 __propagate_active_weight(iocg
, iocg
->weight
, new_inuse
);
1530 skip_surplus_transfers
:
1531 commit_active_weights(ioc
);
1534 * If q is getting clogged or we're missing too much, we're issuing
1535 * too much IO and should lower vtime rate. If we're not missing
1536 * and experiencing shortages but not surpluses, we're too stingy
1537 * and should increase vtime rate.
1539 prev_busy_level
= ioc
->busy_level
;
1540 if (rq_wait_pct
> RQ_WAIT_BUSY_PCT
||
1541 missed_ppm
[READ
] > ppm_rthr
||
1542 missed_ppm
[WRITE
] > ppm_wthr
) {
1543 /* clearly missing QoS targets, slow down vrate */
1544 ioc
->busy_level
= max(ioc
->busy_level
, 0);
1546 } else if (rq_wait_pct
<= RQ_WAIT_BUSY_PCT
* UNBUSY_THR_PCT
/ 100 &&
1547 missed_ppm
[READ
] <= ppm_rthr
* UNBUSY_THR_PCT
/ 100 &&
1548 missed_ppm
[WRITE
] <= ppm_wthr
* UNBUSY_THR_PCT
/ 100) {
1549 /* QoS targets are being met with >25% margin */
1552 * We're throttling while the device has spare
1553 * capacity. If vrate was being slowed down, stop.
1555 ioc
->busy_level
= min(ioc
->busy_level
, 0);
1558 * If there are IOs spanning multiple periods, wait
1559 * them out before pushing the device harder. If
1560 * there are surpluses, let redistribution work it
1563 if (!nr_lagging
&& !nr_surpluses
)
1567 * Nobody is being throttled and the users aren't
1568 * issuing enough IOs to saturate the device. We
1569 * simply don't know how close the device is to
1570 * saturation. Coast.
1572 ioc
->busy_level
= 0;
1575 /* inside the hysterisis margin, we're good */
1576 ioc
->busy_level
= 0;
1579 ioc
->busy_level
= clamp(ioc
->busy_level
, -1000, 1000);
1581 if (ioc
->busy_level
> 0 || (ioc
->busy_level
< 0 && !nr_lagging
)) {
1582 u64 vrate
= atomic64_read(&ioc
->vtime_rate
);
1583 u64 vrate_min
= ioc
->vrate_min
, vrate_max
= ioc
->vrate_max
;
1585 /* rq_wait signal is always reliable, ignore user vrate_min */
1586 if (rq_wait_pct
> RQ_WAIT_BUSY_PCT
)
1587 vrate_min
= VRATE_MIN
;
1590 * If vrate is out of bounds, apply clamp gradually as the
1591 * bounds can change abruptly. Otherwise, apply busy_level
1594 if (vrate
< vrate_min
) {
1595 vrate
= div64_u64(vrate
* (100 + VRATE_CLAMP_ADJ_PCT
),
1597 vrate
= min(vrate
, vrate_min
);
1598 } else if (vrate
> vrate_max
) {
1599 vrate
= div64_u64(vrate
* (100 - VRATE_CLAMP_ADJ_PCT
),
1601 vrate
= max(vrate
, vrate_max
);
1603 int idx
= min_t(int, abs(ioc
->busy_level
),
1604 ARRAY_SIZE(vrate_adj_pct
) - 1);
1605 u32 adj_pct
= vrate_adj_pct
[idx
];
1607 if (ioc
->busy_level
> 0)
1608 adj_pct
= 100 - adj_pct
;
1610 adj_pct
= 100 + adj_pct
;
1612 vrate
= clamp(DIV64_U64_ROUND_UP(vrate
* adj_pct
, 100),
1613 vrate_min
, vrate_max
);
1616 trace_iocost_ioc_vrate_adj(ioc
, vrate
, missed_ppm
, rq_wait_pct
,
1617 nr_lagging
, nr_shortages
,
1620 atomic64_set(&ioc
->vtime_rate
, vrate
);
1621 ioc
->inuse_margin_vtime
= DIV64_U64_ROUND_UP(
1622 ioc
->period_us
* vrate
* INUSE_MARGIN_PCT
, 100);
1623 } else if (ioc
->busy_level
!= prev_busy_level
|| nr_lagging
) {
1624 trace_iocost_ioc_vrate_adj(ioc
, atomic64_read(&ioc
->vtime_rate
),
1625 missed_ppm
, rq_wait_pct
, nr_lagging
,
1626 nr_shortages
, nr_surpluses
);
1629 ioc_refresh_params(ioc
, false);
1632 * This period is done. Move onto the next one. If nothing's
1633 * going on with the device, stop the timer.
1635 atomic64_inc(&ioc
->cur_period
);
1637 if (ioc
->running
!= IOC_STOP
) {
1638 if (!list_empty(&ioc
->active_iocgs
)) {
1639 ioc_start_period(ioc
, &now
);
1641 ioc
->busy_level
= 0;
1642 ioc
->running
= IOC_IDLE
;
1646 spin_unlock_irq(&ioc
->lock
);
1649 static void calc_vtime_cost_builtin(struct bio
*bio
, struct ioc_gq
*iocg
,
1650 bool is_merge
, u64
*costp
)
1652 struct ioc
*ioc
= iocg
->ioc
;
1653 u64 coef_seqio
, coef_randio
, coef_page
;
1654 u64 pages
= max_t(u64
, bio_sectors(bio
) >> IOC_SECT_TO_PAGE_SHIFT
, 1);
1658 switch (bio_op(bio
)) {
1660 coef_seqio
= ioc
->params
.lcoefs
[LCOEF_RSEQIO
];
1661 coef_randio
= ioc
->params
.lcoefs
[LCOEF_RRANDIO
];
1662 coef_page
= ioc
->params
.lcoefs
[LCOEF_RPAGE
];
1665 coef_seqio
= ioc
->params
.lcoefs
[LCOEF_WSEQIO
];
1666 coef_randio
= ioc
->params
.lcoefs
[LCOEF_WRANDIO
];
1667 coef_page
= ioc
->params
.lcoefs
[LCOEF_WPAGE
];
1674 seek_pages
= abs(bio
->bi_iter
.bi_sector
- iocg
->cursor
);
1675 seek_pages
>>= IOC_SECT_TO_PAGE_SHIFT
;
1679 if (seek_pages
> LCOEF_RANDIO_PAGES
) {
1680 cost
+= coef_randio
;
1685 cost
+= pages
* coef_page
;
1690 static u64
calc_vtime_cost(struct bio
*bio
, struct ioc_gq
*iocg
, bool is_merge
)
1694 calc_vtime_cost_builtin(bio
, iocg
, is_merge
, &cost
);
1698 static void calc_size_vtime_cost_builtin(struct request
*rq
, struct ioc
*ioc
,
1701 unsigned int pages
= blk_rq_stats_sectors(rq
) >> IOC_SECT_TO_PAGE_SHIFT
;
1703 switch (req_op(rq
)) {
1705 *costp
= pages
* ioc
->params
.lcoefs
[LCOEF_RPAGE
];
1708 *costp
= pages
* ioc
->params
.lcoefs
[LCOEF_WPAGE
];
1715 static u64
calc_size_vtime_cost(struct request
*rq
, struct ioc
*ioc
)
1719 calc_size_vtime_cost_builtin(rq
, ioc
, &cost
);
1723 static void ioc_rqos_throttle(struct rq_qos
*rqos
, struct bio
*bio
)
1725 struct blkcg_gq
*blkg
= bio
->bi_blkg
;
1726 struct ioc
*ioc
= rqos_to_ioc(rqos
);
1727 struct ioc_gq
*iocg
= blkg_to_iocg(blkg
);
1729 struct iocg_wait wait
;
1730 u32 hw_active
, hw_inuse
;
1731 u64 abs_cost
, cost
, vtime
;
1733 /* bypass IOs if disabled or for root cgroup */
1734 if (!ioc
->enabled
|| !iocg
->level
)
1737 /* always activate so that even 0 cost IOs get protected to some level */
1738 if (!iocg_activate(iocg
, &now
))
1741 /* calculate the absolute vtime cost */
1742 abs_cost
= calc_vtime_cost(bio
, iocg
, false);
1746 iocg
->cursor
= bio_end_sector(bio
);
1748 vtime
= atomic64_read(&iocg
->vtime
);
1749 current_hweight(iocg
, &hw_active
, &hw_inuse
);
1751 if (hw_inuse
< hw_active
&&
1752 time_after_eq64(vtime
+ ioc
->inuse_margin_vtime
, now
.vnow
)) {
1753 TRACE_IOCG_PATH(inuse_reset
, iocg
, &now
,
1754 iocg
->inuse
, iocg
->weight
, hw_inuse
, hw_active
);
1755 spin_lock_irq(&ioc
->lock
);
1756 propagate_active_weight(iocg
, iocg
->weight
, iocg
->weight
);
1757 spin_unlock_irq(&ioc
->lock
);
1758 current_hweight(iocg
, &hw_active
, &hw_inuse
);
1761 cost
= abs_cost_to_cost(abs_cost
, hw_inuse
);
1764 * If no one's waiting and within budget, issue right away. The
1765 * tests are racy but the races aren't systemic - we only miss once
1766 * in a while which is fine.
1768 if (!waitqueue_active(&iocg
->waitq
) && !iocg
->abs_vdebt
&&
1769 time_before_eq64(vtime
+ cost
, now
.vnow
)) {
1770 iocg_commit_bio(iocg
, bio
, cost
);
1775 * We activated above but w/o any synchronization. Deactivation is
1776 * synchronized with waitq.lock and we won't get deactivated as long
1777 * as we're waiting or has debt, so we're good if we're activated
1778 * here. In the unlikely case that we aren't, just issue the IO.
1780 spin_lock_irq(&iocg
->waitq
.lock
);
1782 if (unlikely(list_empty(&iocg
->active_list
))) {
1783 spin_unlock_irq(&iocg
->waitq
.lock
);
1784 iocg_commit_bio(iocg
, bio
, cost
);
1789 * We're over budget. If @bio has to be issued regardless, remember
1790 * the abs_cost instead of advancing vtime. iocg_kick_waitq() will pay
1791 * off the debt before waking more IOs.
1793 * This way, the debt is continuously paid off each period with the
1794 * actual budget available to the cgroup. If we just wound vtime, we
1795 * would incorrectly use the current hw_inuse for the entire amount
1796 * which, for example, can lead to the cgroup staying blocked for a
1797 * long time even with substantially raised hw_inuse.
1799 * An iocg with vdebt should stay online so that the timer can keep
1800 * deducting its vdebt and [de]activate use_delay mechanism
1801 * accordingly. We don't want to race against the timer trying to
1802 * clear them and leave @iocg inactive w/ dangling use_delay heavily
1803 * penalizing the cgroup and its descendants.
1805 if (bio_issue_as_root_blkg(bio
) || fatal_signal_pending(current
)) {
1806 iocg
->abs_vdebt
+= abs_cost
;
1807 if (iocg_kick_delay(iocg
, &now
))
1808 blkcg_schedule_throttle(rqos
->q
,
1809 (bio
->bi_opf
& REQ_SWAP
) == REQ_SWAP
);
1810 spin_unlock_irq(&iocg
->waitq
.lock
);
1815 * Append self to the waitq and schedule the wakeup timer if we're
1816 * the first waiter. The timer duration is calculated based on the
1817 * current vrate. vtime and hweight changes can make it too short
1818 * or too long. Each wait entry records the absolute cost it's
1819 * waiting for to allow re-evaluation using a custom wait entry.
1821 * If too short, the timer simply reschedules itself. If too long,
1822 * the period timer will notice and trigger wakeups.
1824 * All waiters are on iocg->waitq and the wait states are
1825 * synchronized using waitq.lock.
1827 init_waitqueue_func_entry(&wait
.wait
, iocg_wake_fn
);
1828 wait
.wait
.private = current
;
1830 wait
.abs_cost
= abs_cost
;
1831 wait
.committed
= false; /* will be set true by waker */
1833 __add_wait_queue_entry_tail(&iocg
->waitq
, &wait
.wait
);
1834 iocg_kick_waitq(iocg
, &now
);
1836 spin_unlock_irq(&iocg
->waitq
.lock
);
1839 set_current_state(TASK_UNINTERRUPTIBLE
);
1845 /* waker already committed us, proceed */
1846 finish_wait(&iocg
->waitq
, &wait
.wait
);
1849 static void ioc_rqos_merge(struct rq_qos
*rqos
, struct request
*rq
,
1852 struct ioc_gq
*iocg
= blkg_to_iocg(bio
->bi_blkg
);
1853 struct ioc
*ioc
= iocg
->ioc
;
1854 sector_t bio_end
= bio_end_sector(bio
);
1858 unsigned long flags
;
1860 /* bypass if disabled or for root cgroup */
1861 if (!ioc
->enabled
|| !iocg
->level
)
1864 abs_cost
= calc_vtime_cost(bio
, iocg
, true);
1869 current_hweight(iocg
, NULL
, &hw_inuse
);
1870 cost
= abs_cost_to_cost(abs_cost
, hw_inuse
);
1872 /* update cursor if backmerging into the request at the cursor */
1873 if (blk_rq_pos(rq
) < bio_end
&&
1874 blk_rq_pos(rq
) + blk_rq_sectors(rq
) == iocg
->cursor
)
1875 iocg
->cursor
= bio_end
;
1878 * Charge if there's enough vtime budget and the existing request has
1881 if (rq
->bio
&& rq
->bio
->bi_iocost_cost
&&
1882 time_before_eq64(atomic64_read(&iocg
->vtime
) + cost
, now
.vnow
)) {
1883 iocg_commit_bio(iocg
, bio
, cost
);
1888 * Otherwise, account it as debt if @iocg is online, which it should
1889 * be for the vast majority of cases. See debt handling in
1890 * ioc_rqos_throttle() for details.
1892 spin_lock_irqsave(&iocg
->waitq
.lock
, flags
);
1893 if (likely(!list_empty(&iocg
->active_list
))) {
1894 iocg
->abs_vdebt
+= abs_cost
;
1895 iocg_kick_delay(iocg
, &now
);
1897 iocg_commit_bio(iocg
, bio
, cost
);
1899 spin_unlock_irqrestore(&iocg
->waitq
.lock
, flags
);
1902 static void ioc_rqos_done_bio(struct rq_qos
*rqos
, struct bio
*bio
)
1904 struct ioc_gq
*iocg
= blkg_to_iocg(bio
->bi_blkg
);
1906 if (iocg
&& bio
->bi_iocost_cost
)
1907 atomic64_add(bio
->bi_iocost_cost
, &iocg
->done_vtime
);
1910 static void ioc_rqos_done(struct rq_qos
*rqos
, struct request
*rq
)
1912 struct ioc
*ioc
= rqos_to_ioc(rqos
);
1913 struct ioc_pcpu_stat
*ccs
;
1914 u64 on_q_ns
, rq_wait_ns
, size_nsec
;
1917 if (!ioc
->enabled
|| !rq
->alloc_time_ns
|| !rq
->start_time_ns
)
1920 switch (req_op(rq
) & REQ_OP_MASK
) {
1933 on_q_ns
= ktime_get_ns() - rq
->alloc_time_ns
;
1934 rq_wait_ns
= rq
->start_time_ns
- rq
->alloc_time_ns
;
1935 size_nsec
= div64_u64(calc_size_vtime_cost(rq
, ioc
), VTIME_PER_NSEC
);
1937 ccs
= get_cpu_ptr(ioc
->pcpu_stat
);
1939 if (on_q_ns
<= size_nsec
||
1940 on_q_ns
- size_nsec
<= ioc
->params
.qos
[pidx
] * NSEC_PER_USEC
)
1941 local_inc(&ccs
->missed
[rw
].nr_met
);
1943 local_inc(&ccs
->missed
[rw
].nr_missed
);
1945 local64_add(rq_wait_ns
, &ccs
->rq_wait_ns
);
1950 static void ioc_rqos_queue_depth_changed(struct rq_qos
*rqos
)
1952 struct ioc
*ioc
= rqos_to_ioc(rqos
);
1954 spin_lock_irq(&ioc
->lock
);
1955 ioc_refresh_params(ioc
, false);
1956 spin_unlock_irq(&ioc
->lock
);
1959 static void ioc_rqos_exit(struct rq_qos
*rqos
)
1961 struct ioc
*ioc
= rqos_to_ioc(rqos
);
1963 blkcg_deactivate_policy(rqos
->q
, &blkcg_policy_iocost
);
1965 spin_lock_irq(&ioc
->lock
);
1966 ioc
->running
= IOC_STOP
;
1967 spin_unlock_irq(&ioc
->lock
);
1969 del_timer_sync(&ioc
->timer
);
1970 free_percpu(ioc
->pcpu_stat
);
1974 static struct rq_qos_ops ioc_rqos_ops
= {
1975 .throttle
= ioc_rqos_throttle
,
1976 .merge
= ioc_rqos_merge
,
1977 .done_bio
= ioc_rqos_done_bio
,
1978 .done
= ioc_rqos_done
,
1979 .queue_depth_changed
= ioc_rqos_queue_depth_changed
,
1980 .exit
= ioc_rqos_exit
,
1983 static int blk_iocost_init(struct request_queue
*q
)
1986 struct rq_qos
*rqos
;
1989 ioc
= kzalloc(sizeof(*ioc
), GFP_KERNEL
);
1993 ioc
->pcpu_stat
= alloc_percpu(struct ioc_pcpu_stat
);
1994 if (!ioc
->pcpu_stat
) {
1999 for_each_possible_cpu(cpu
) {
2000 struct ioc_pcpu_stat
*ccs
= per_cpu_ptr(ioc
->pcpu_stat
, cpu
);
2002 for (i
= 0; i
< ARRAY_SIZE(ccs
->missed
); i
++) {
2003 local_set(&ccs
->missed
[i
].nr_met
, 0);
2004 local_set(&ccs
->missed
[i
].nr_missed
, 0);
2006 local64_set(&ccs
->rq_wait_ns
, 0);
2010 rqos
->id
= RQ_QOS_COST
;
2011 rqos
->ops
= &ioc_rqos_ops
;
2014 spin_lock_init(&ioc
->lock
);
2015 timer_setup(&ioc
->timer
, ioc_timer_fn
, 0);
2016 INIT_LIST_HEAD(&ioc
->active_iocgs
);
2018 ioc
->running
= IOC_IDLE
;
2019 atomic64_set(&ioc
->vtime_rate
, VTIME_PER_USEC
);
2020 seqcount_spinlock_init(&ioc
->period_seqcount
, &ioc
->lock
);
2021 ioc
->period_at
= ktime_to_us(ktime_get());
2022 atomic64_set(&ioc
->cur_period
, 0);
2023 atomic_set(&ioc
->hweight_gen
, 0);
2025 spin_lock_irq(&ioc
->lock
);
2026 ioc
->autop_idx
= AUTOP_INVALID
;
2027 ioc_refresh_params(ioc
, true);
2028 spin_unlock_irq(&ioc
->lock
);
2030 rq_qos_add(q
, rqos
);
2031 ret
= blkcg_activate_policy(q
, &blkcg_policy_iocost
);
2033 rq_qos_del(q
, rqos
);
2034 free_percpu(ioc
->pcpu_stat
);
2041 static struct blkcg_policy_data
*ioc_cpd_alloc(gfp_t gfp
)
2043 struct ioc_cgrp
*iocc
;
2045 iocc
= kzalloc(sizeof(struct ioc_cgrp
), gfp
);
2049 iocc
->dfl_weight
= CGROUP_WEIGHT_DFL
;
2053 static void ioc_cpd_free(struct blkcg_policy_data
*cpd
)
2055 kfree(container_of(cpd
, struct ioc_cgrp
, cpd
));
2058 static struct blkg_policy_data
*ioc_pd_alloc(gfp_t gfp
, struct request_queue
*q
,
2059 struct blkcg
*blkcg
)
2061 int levels
= blkcg
->css
.cgroup
->level
+ 1;
2062 struct ioc_gq
*iocg
;
2064 iocg
= kzalloc_node(struct_size(iocg
, ancestors
, levels
), gfp
, q
->node
);
2071 static void ioc_pd_init(struct blkg_policy_data
*pd
)
2073 struct ioc_gq
*iocg
= pd_to_iocg(pd
);
2074 struct blkcg_gq
*blkg
= pd_to_blkg(&iocg
->pd
);
2075 struct ioc
*ioc
= q_to_ioc(blkg
->q
);
2077 struct blkcg_gq
*tblkg
;
2078 unsigned long flags
;
2083 atomic64_set(&iocg
->vtime
, now
.vnow
);
2084 atomic64_set(&iocg
->done_vtime
, now
.vnow
);
2085 atomic64_set(&iocg
->active_period
, atomic64_read(&ioc
->cur_period
));
2086 INIT_LIST_HEAD(&iocg
->active_list
);
2087 iocg
->hweight_active
= HWEIGHT_WHOLE
;
2088 iocg
->hweight_inuse
= HWEIGHT_WHOLE
;
2090 init_waitqueue_head(&iocg
->waitq
);
2091 hrtimer_init(&iocg
->waitq_timer
, CLOCK_MONOTONIC
, HRTIMER_MODE_ABS
);
2092 iocg
->waitq_timer
.function
= iocg_waitq_timer_fn
;
2093 hrtimer_init(&iocg
->delay_timer
, CLOCK_MONOTONIC
, HRTIMER_MODE_ABS
);
2094 iocg
->delay_timer
.function
= iocg_delay_timer_fn
;
2096 iocg
->level
= blkg
->blkcg
->css
.cgroup
->level
;
2098 for (tblkg
= blkg
; tblkg
; tblkg
= tblkg
->parent
) {
2099 struct ioc_gq
*tiocg
= blkg_to_iocg(tblkg
);
2100 iocg
->ancestors
[tiocg
->level
] = tiocg
;
2103 spin_lock_irqsave(&ioc
->lock
, flags
);
2104 weight_updated(iocg
);
2105 spin_unlock_irqrestore(&ioc
->lock
, flags
);
2108 static void ioc_pd_free(struct blkg_policy_data
*pd
)
2110 struct ioc_gq
*iocg
= pd_to_iocg(pd
);
2111 struct ioc
*ioc
= iocg
->ioc
;
2112 unsigned long flags
;
2115 spin_lock_irqsave(&ioc
->lock
, flags
);
2116 if (!list_empty(&iocg
->active_list
)) {
2117 propagate_active_weight(iocg
, 0, 0);
2118 list_del_init(&iocg
->active_list
);
2120 spin_unlock_irqrestore(&ioc
->lock
, flags
);
2122 hrtimer_cancel(&iocg
->waitq_timer
);
2123 hrtimer_cancel(&iocg
->delay_timer
);
2128 static u64
ioc_weight_prfill(struct seq_file
*sf
, struct blkg_policy_data
*pd
,
2131 const char *dname
= blkg_dev_name(pd
->blkg
);
2132 struct ioc_gq
*iocg
= pd_to_iocg(pd
);
2134 if (dname
&& iocg
->cfg_weight
)
2135 seq_printf(sf
, "%s %u\n", dname
, iocg
->cfg_weight
);
2140 static int ioc_weight_show(struct seq_file
*sf
, void *v
)
2142 struct blkcg
*blkcg
= css_to_blkcg(seq_css(sf
));
2143 struct ioc_cgrp
*iocc
= blkcg_to_iocc(blkcg
);
2145 seq_printf(sf
, "default %u\n", iocc
->dfl_weight
);
2146 blkcg_print_blkgs(sf
, blkcg
, ioc_weight_prfill
,
2147 &blkcg_policy_iocost
, seq_cft(sf
)->private, false);
2151 static ssize_t
ioc_weight_write(struct kernfs_open_file
*of
, char *buf
,
2152 size_t nbytes
, loff_t off
)
2154 struct blkcg
*blkcg
= css_to_blkcg(of_css(of
));
2155 struct ioc_cgrp
*iocc
= blkcg_to_iocc(blkcg
);
2156 struct blkg_conf_ctx ctx
;
2157 struct ioc_gq
*iocg
;
2161 if (!strchr(buf
, ':')) {
2162 struct blkcg_gq
*blkg
;
2164 if (!sscanf(buf
, "default %u", &v
) && !sscanf(buf
, "%u", &v
))
2167 if (v
< CGROUP_WEIGHT_MIN
|| v
> CGROUP_WEIGHT_MAX
)
2170 spin_lock(&blkcg
->lock
);
2171 iocc
->dfl_weight
= v
;
2172 hlist_for_each_entry(blkg
, &blkcg
->blkg_list
, blkcg_node
) {
2173 struct ioc_gq
*iocg
= blkg_to_iocg(blkg
);
2176 spin_lock_irq(&iocg
->ioc
->lock
);
2177 weight_updated(iocg
);
2178 spin_unlock_irq(&iocg
->ioc
->lock
);
2181 spin_unlock(&blkcg
->lock
);
2186 ret
= blkg_conf_prep(blkcg
, &blkcg_policy_iocost
, buf
, &ctx
);
2190 iocg
= blkg_to_iocg(ctx
.blkg
);
2192 if (!strncmp(ctx
.body
, "default", 7)) {
2195 if (!sscanf(ctx
.body
, "%u", &v
))
2197 if (v
< CGROUP_WEIGHT_MIN
|| v
> CGROUP_WEIGHT_MAX
)
2201 spin_lock(&iocg
->ioc
->lock
);
2202 iocg
->cfg_weight
= v
;
2203 weight_updated(iocg
);
2204 spin_unlock(&iocg
->ioc
->lock
);
2206 blkg_conf_finish(&ctx
);
2210 blkg_conf_finish(&ctx
);
2214 static u64
ioc_qos_prfill(struct seq_file
*sf
, struct blkg_policy_data
*pd
,
2217 const char *dname
= blkg_dev_name(pd
->blkg
);
2218 struct ioc
*ioc
= pd_to_iocg(pd
)->ioc
;
2223 seq_printf(sf
, "%s enable=%d ctrl=%s rpct=%u.%02u rlat=%u wpct=%u.%02u wlat=%u min=%u.%02u max=%u.%02u\n",
2224 dname
, ioc
->enabled
, ioc
->user_qos_params
? "user" : "auto",
2225 ioc
->params
.qos
[QOS_RPPM
] / 10000,
2226 ioc
->params
.qos
[QOS_RPPM
] % 10000 / 100,
2227 ioc
->params
.qos
[QOS_RLAT
],
2228 ioc
->params
.qos
[QOS_WPPM
] / 10000,
2229 ioc
->params
.qos
[QOS_WPPM
] % 10000 / 100,
2230 ioc
->params
.qos
[QOS_WLAT
],
2231 ioc
->params
.qos
[QOS_MIN
] / 10000,
2232 ioc
->params
.qos
[QOS_MIN
] % 10000 / 100,
2233 ioc
->params
.qos
[QOS_MAX
] / 10000,
2234 ioc
->params
.qos
[QOS_MAX
] % 10000 / 100);
2238 static int ioc_qos_show(struct seq_file
*sf
, void *v
)
2240 struct blkcg
*blkcg
= css_to_blkcg(seq_css(sf
));
2242 blkcg_print_blkgs(sf
, blkcg
, ioc_qos_prfill
,
2243 &blkcg_policy_iocost
, seq_cft(sf
)->private, false);
2247 static const match_table_t qos_ctrl_tokens
= {
2248 { QOS_ENABLE
, "enable=%u" },
2249 { QOS_CTRL
, "ctrl=%s" },
2250 { NR_QOS_CTRL_PARAMS
, NULL
},
2253 static const match_table_t qos_tokens
= {
2254 { QOS_RPPM
, "rpct=%s" },
2255 { QOS_RLAT
, "rlat=%u" },
2256 { QOS_WPPM
, "wpct=%s" },
2257 { QOS_WLAT
, "wlat=%u" },
2258 { QOS_MIN
, "min=%s" },
2259 { QOS_MAX
, "max=%s" },
2260 { NR_QOS_PARAMS
, NULL
},
2263 static ssize_t
ioc_qos_write(struct kernfs_open_file
*of
, char *input
,
2264 size_t nbytes
, loff_t off
)
2266 struct gendisk
*disk
;
2268 u32 qos
[NR_QOS_PARAMS
];
2273 disk
= blkcg_conf_get_disk(&input
);
2275 return PTR_ERR(disk
);
2277 ioc
= q_to_ioc(disk
->queue
);
2279 ret
= blk_iocost_init(disk
->queue
);
2282 ioc
= q_to_ioc(disk
->queue
);
2285 spin_lock_irq(&ioc
->lock
);
2286 memcpy(qos
, ioc
->params
.qos
, sizeof(qos
));
2287 enable
= ioc
->enabled
;
2288 user
= ioc
->user_qos_params
;
2289 spin_unlock_irq(&ioc
->lock
);
2291 while ((p
= strsep(&input
, " \t\n"))) {
2292 substring_t args
[MAX_OPT_ARGS
];
2300 switch (match_token(p
, qos_ctrl_tokens
, args
)) {
2302 match_u64(&args
[0], &v
);
2306 match_strlcpy(buf
, &args
[0], sizeof(buf
));
2307 if (!strcmp(buf
, "auto"))
2309 else if (!strcmp(buf
, "user"))
2316 tok
= match_token(p
, qos_tokens
, args
);
2320 if (match_strlcpy(buf
, &args
[0], sizeof(buf
)) >=
2323 if (cgroup_parse_float(buf
, 2, &v
))
2325 if (v
< 0 || v
> 10000)
2331 if (match_u64(&args
[0], &v
))
2337 if (match_strlcpy(buf
, &args
[0], sizeof(buf
)) >=
2340 if (cgroup_parse_float(buf
, 2, &v
))
2344 qos
[tok
] = clamp_t(s64
, v
* 100,
2345 VRATE_MIN_PPM
, VRATE_MAX_PPM
);
2353 if (qos
[QOS_MIN
] > qos
[QOS_MAX
])
2356 spin_lock_irq(&ioc
->lock
);
2359 blk_stat_enable_accounting(ioc
->rqos
.q
);
2360 blk_queue_flag_set(QUEUE_FLAG_RQ_ALLOC_TIME
, ioc
->rqos
.q
);
2361 ioc
->enabled
= true;
2363 blk_queue_flag_clear(QUEUE_FLAG_RQ_ALLOC_TIME
, ioc
->rqos
.q
);
2364 ioc
->enabled
= false;
2368 memcpy(ioc
->params
.qos
, qos
, sizeof(qos
));
2369 ioc
->user_qos_params
= true;
2371 ioc
->user_qos_params
= false;
2374 ioc_refresh_params(ioc
, true);
2375 spin_unlock_irq(&ioc
->lock
);
2377 put_disk_and_module(disk
);
2382 put_disk_and_module(disk
);
2386 static u64
ioc_cost_model_prfill(struct seq_file
*sf
,
2387 struct blkg_policy_data
*pd
, int off
)
2389 const char *dname
= blkg_dev_name(pd
->blkg
);
2390 struct ioc
*ioc
= pd_to_iocg(pd
)->ioc
;
2391 u64
*u
= ioc
->params
.i_lcoefs
;
2396 seq_printf(sf
, "%s ctrl=%s model=linear "
2397 "rbps=%llu rseqiops=%llu rrandiops=%llu "
2398 "wbps=%llu wseqiops=%llu wrandiops=%llu\n",
2399 dname
, ioc
->user_cost_model
? "user" : "auto",
2400 u
[I_LCOEF_RBPS
], u
[I_LCOEF_RSEQIOPS
], u
[I_LCOEF_RRANDIOPS
],
2401 u
[I_LCOEF_WBPS
], u
[I_LCOEF_WSEQIOPS
], u
[I_LCOEF_WRANDIOPS
]);
2405 static int ioc_cost_model_show(struct seq_file
*sf
, void *v
)
2407 struct blkcg
*blkcg
= css_to_blkcg(seq_css(sf
));
2409 blkcg_print_blkgs(sf
, blkcg
, ioc_cost_model_prfill
,
2410 &blkcg_policy_iocost
, seq_cft(sf
)->private, false);
2414 static const match_table_t cost_ctrl_tokens
= {
2415 { COST_CTRL
, "ctrl=%s" },
2416 { COST_MODEL
, "model=%s" },
2417 { NR_COST_CTRL_PARAMS
, NULL
},
2420 static const match_table_t i_lcoef_tokens
= {
2421 { I_LCOEF_RBPS
, "rbps=%u" },
2422 { I_LCOEF_RSEQIOPS
, "rseqiops=%u" },
2423 { I_LCOEF_RRANDIOPS
, "rrandiops=%u" },
2424 { I_LCOEF_WBPS
, "wbps=%u" },
2425 { I_LCOEF_WSEQIOPS
, "wseqiops=%u" },
2426 { I_LCOEF_WRANDIOPS
, "wrandiops=%u" },
2427 { NR_I_LCOEFS
, NULL
},
2430 static ssize_t
ioc_cost_model_write(struct kernfs_open_file
*of
, char *input
,
2431 size_t nbytes
, loff_t off
)
2433 struct gendisk
*disk
;
2440 disk
= blkcg_conf_get_disk(&input
);
2442 return PTR_ERR(disk
);
2444 ioc
= q_to_ioc(disk
->queue
);
2446 ret
= blk_iocost_init(disk
->queue
);
2449 ioc
= q_to_ioc(disk
->queue
);
2452 spin_lock_irq(&ioc
->lock
);
2453 memcpy(u
, ioc
->params
.i_lcoefs
, sizeof(u
));
2454 user
= ioc
->user_cost_model
;
2455 spin_unlock_irq(&ioc
->lock
);
2457 while ((p
= strsep(&input
, " \t\n"))) {
2458 substring_t args
[MAX_OPT_ARGS
];
2466 switch (match_token(p
, cost_ctrl_tokens
, args
)) {
2468 match_strlcpy(buf
, &args
[0], sizeof(buf
));
2469 if (!strcmp(buf
, "auto"))
2471 else if (!strcmp(buf
, "user"))
2477 match_strlcpy(buf
, &args
[0], sizeof(buf
));
2478 if (strcmp(buf
, "linear"))
2483 tok
= match_token(p
, i_lcoef_tokens
, args
);
2484 if (tok
== NR_I_LCOEFS
)
2486 if (match_u64(&args
[0], &v
))
2492 spin_lock_irq(&ioc
->lock
);
2494 memcpy(ioc
->params
.i_lcoefs
, u
, sizeof(u
));
2495 ioc
->user_cost_model
= true;
2497 ioc
->user_cost_model
= false;
2499 ioc_refresh_params(ioc
, true);
2500 spin_unlock_irq(&ioc
->lock
);
2502 put_disk_and_module(disk
);
2508 put_disk_and_module(disk
);
2512 static struct cftype ioc_files
[] = {
2515 .flags
= CFTYPE_NOT_ON_ROOT
,
2516 .seq_show
= ioc_weight_show
,
2517 .write
= ioc_weight_write
,
2521 .flags
= CFTYPE_ONLY_ON_ROOT
,
2522 .seq_show
= ioc_qos_show
,
2523 .write
= ioc_qos_write
,
2526 .name
= "cost.model",
2527 .flags
= CFTYPE_ONLY_ON_ROOT
,
2528 .seq_show
= ioc_cost_model_show
,
2529 .write
= ioc_cost_model_write
,
2534 static struct blkcg_policy blkcg_policy_iocost
= {
2535 .dfl_cftypes
= ioc_files
,
2536 .cpd_alloc_fn
= ioc_cpd_alloc
,
2537 .cpd_free_fn
= ioc_cpd_free
,
2538 .pd_alloc_fn
= ioc_pd_alloc
,
2539 .pd_init_fn
= ioc_pd_init
,
2540 .pd_free_fn
= ioc_pd_free
,
2543 static int __init
ioc_init(void)
2545 return blkcg_policy_register(&blkcg_policy_iocost
);
2548 static void __exit
ioc_exit(void)
2550 return blkcg_policy_unregister(&blkcg_policy_iocost
);
2553 module_init(ioc_init
);
2554 module_exit(ioc_exit
);