]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * Block stat tracking code | |
3 | * | |
4 | * Copyright (C) 2016 Jens Axboe | |
5 | */ | |
6 | #include <linux/kernel.h> | |
7 | #include <linux/blk-mq.h> | |
8 | ||
9 | #include "blk-stat.h" | |
10 | #include "blk-mq.h" | |
11 | ||
12 | static void blk_stat_flush_batch(struct blk_rq_stat *stat) | |
13 | { | |
14 | const s32 nr_batch = READ_ONCE(stat->nr_batch); | |
15 | const s32 nr_samples = READ_ONCE(stat->nr_samples); | |
16 | ||
17 | if (!nr_batch) | |
18 | return; | |
19 | if (!nr_samples) | |
20 | stat->mean = div64_s64(stat->batch, nr_batch); | |
21 | else { | |
22 | stat->mean = div64_s64((stat->mean * nr_samples) + | |
23 | stat->batch, | |
24 | nr_batch + nr_samples); | |
25 | } | |
26 | ||
27 | stat->nr_samples += nr_batch; | |
28 | stat->nr_batch = stat->batch = 0; | |
29 | } | |
30 | ||
31 | static void blk_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src) | |
32 | { | |
33 | if (!src->nr_samples) | |
34 | return; | |
35 | ||
36 | blk_stat_flush_batch(src); | |
37 | ||
38 | dst->min = min(dst->min, src->min); | |
39 | dst->max = max(dst->max, src->max); | |
40 | ||
41 | if (!dst->nr_samples) | |
42 | dst->mean = src->mean; | |
43 | else { | |
44 | dst->mean = div64_s64((src->mean * src->nr_samples) + | |
45 | (dst->mean * dst->nr_samples), | |
46 | dst->nr_samples + src->nr_samples); | |
47 | } | |
48 | dst->nr_samples += src->nr_samples; | |
49 | } | |
50 | ||
51 | static void blk_mq_stat_get(struct request_queue *q, struct blk_rq_stat *dst) | |
52 | { | |
53 | struct blk_mq_hw_ctx *hctx; | |
54 | struct blk_mq_ctx *ctx; | |
55 | uint64_t latest = 0; | |
56 | int i, j, nr; | |
57 | ||
58 | blk_stat_init(&dst[BLK_STAT_READ]); | |
59 | blk_stat_init(&dst[BLK_STAT_WRITE]); | |
60 | ||
61 | nr = 0; | |
62 | do { | |
63 | uint64_t newest = 0; | |
64 | ||
65 | queue_for_each_hw_ctx(q, hctx, i) { | |
66 | hctx_for_each_ctx(hctx, ctx, j) { | |
67 | if (!ctx->stat[BLK_STAT_READ].nr_samples && | |
68 | !ctx->stat[BLK_STAT_WRITE].nr_samples) | |
69 | continue; | |
70 | if (ctx->stat[BLK_STAT_READ].time > newest) | |
71 | newest = ctx->stat[BLK_STAT_READ].time; | |
72 | if (ctx->stat[BLK_STAT_WRITE].time > newest) | |
73 | newest = ctx->stat[BLK_STAT_WRITE].time; | |
74 | } | |
75 | } | |
76 | ||
77 | /* | |
78 | * No samples | |
79 | */ | |
80 | if (!newest) | |
81 | break; | |
82 | ||
83 | if (newest > latest) | |
84 | latest = newest; | |
85 | ||
86 | queue_for_each_hw_ctx(q, hctx, i) { | |
87 | hctx_for_each_ctx(hctx, ctx, j) { | |
88 | if (ctx->stat[BLK_STAT_READ].time == newest) { | |
89 | blk_stat_sum(&dst[BLK_STAT_READ], | |
90 | &ctx->stat[BLK_STAT_READ]); | |
91 | nr++; | |
92 | } | |
93 | if (ctx->stat[BLK_STAT_WRITE].time == newest) { | |
94 | blk_stat_sum(&dst[BLK_STAT_WRITE], | |
95 | &ctx->stat[BLK_STAT_WRITE]); | |
96 | nr++; | |
97 | } | |
98 | } | |
99 | } | |
100 | /* | |
101 | * If we race on finding an entry, just loop back again. | |
102 | * Should be very rare. | |
103 | */ | |
104 | } while (!nr); | |
105 | ||
106 | dst[BLK_STAT_READ].time = dst[BLK_STAT_WRITE].time = latest; | |
107 | } | |
108 | ||
109 | void blk_queue_stat_get(struct request_queue *q, struct blk_rq_stat *dst) | |
110 | { | |
111 | if (q->mq_ops) | |
112 | blk_mq_stat_get(q, dst); | |
113 | else { | |
114 | memcpy(&dst[BLK_STAT_READ], &q->rq_stats[BLK_STAT_READ], | |
115 | sizeof(struct blk_rq_stat)); | |
116 | memcpy(&dst[BLK_STAT_WRITE], &q->rq_stats[BLK_STAT_WRITE], | |
117 | sizeof(struct blk_rq_stat)); | |
118 | } | |
119 | } | |
120 | ||
121 | void blk_hctx_stat_get(struct blk_mq_hw_ctx *hctx, struct blk_rq_stat *dst) | |
122 | { | |
123 | struct blk_mq_ctx *ctx; | |
124 | unsigned int i, nr; | |
125 | ||
126 | nr = 0; | |
127 | do { | |
128 | uint64_t newest = 0; | |
129 | ||
130 | hctx_for_each_ctx(hctx, ctx, i) { | |
131 | if (!ctx->stat[BLK_STAT_READ].nr_samples && | |
132 | !ctx->stat[BLK_STAT_WRITE].nr_samples) | |
133 | continue; | |
134 | ||
135 | if (ctx->stat[BLK_STAT_READ].time > newest) | |
136 | newest = ctx->stat[BLK_STAT_READ].time; | |
137 | if (ctx->stat[BLK_STAT_WRITE].time > newest) | |
138 | newest = ctx->stat[BLK_STAT_WRITE].time; | |
139 | } | |
140 | ||
141 | if (!newest) | |
142 | break; | |
143 | ||
144 | hctx_for_each_ctx(hctx, ctx, i) { | |
145 | if (ctx->stat[BLK_STAT_READ].time == newest) { | |
146 | blk_stat_sum(&dst[BLK_STAT_READ], | |
147 | &ctx->stat[BLK_STAT_READ]); | |
148 | nr++; | |
149 | } | |
150 | if (ctx->stat[BLK_STAT_WRITE].time == newest) { | |
151 | blk_stat_sum(&dst[BLK_STAT_WRITE], | |
152 | &ctx->stat[BLK_STAT_WRITE]); | |
153 | nr++; | |
154 | } | |
155 | } | |
156 | /* | |
157 | * If we race on finding an entry, just loop back again. | |
158 | * Should be very rare, as the window is only updated | |
159 | * occasionally | |
160 | */ | |
161 | } while (!nr); | |
162 | } | |
163 | ||
164 | static void __blk_stat_init(struct blk_rq_stat *stat, s64 time_now) | |
165 | { | |
166 | stat->min = -1ULL; | |
167 | stat->max = stat->nr_samples = stat->mean = 0; | |
168 | stat->batch = stat->nr_batch = 0; | |
169 | stat->time = time_now & BLK_STAT_NSEC_MASK; | |
170 | } | |
171 | ||
172 | void blk_stat_init(struct blk_rq_stat *stat) | |
173 | { | |
174 | __blk_stat_init(stat, ktime_to_ns(ktime_get())); | |
175 | } | |
176 | ||
177 | static bool __blk_stat_is_current(struct blk_rq_stat *stat, s64 now) | |
178 | { | |
179 | return (now & BLK_STAT_NSEC_MASK) == (stat->time & BLK_STAT_NSEC_MASK); | |
180 | } | |
181 | ||
182 | bool blk_stat_is_current(struct blk_rq_stat *stat) | |
183 | { | |
184 | return __blk_stat_is_current(stat, ktime_to_ns(ktime_get())); | |
185 | } | |
186 | ||
187 | void blk_stat_add(struct blk_rq_stat *stat, struct request *rq) | |
188 | { | |
189 | s64 now, value; | |
190 | ||
191 | now = __blk_stat_time(ktime_to_ns(ktime_get())); | |
192 | if (now < blk_stat_time(&rq->issue_stat)) | |
193 | return; | |
194 | ||
195 | if (!__blk_stat_is_current(stat, now)) | |
196 | __blk_stat_init(stat, now); | |
197 | ||
198 | value = now - blk_stat_time(&rq->issue_stat); | |
199 | if (value > stat->max) | |
200 | stat->max = value; | |
201 | if (value < stat->min) | |
202 | stat->min = value; | |
203 | ||
204 | if (stat->batch + value < stat->batch || | |
205 | stat->nr_batch + 1 == BLK_RQ_STAT_BATCH) | |
206 | blk_stat_flush_batch(stat); | |
207 | ||
208 | stat->batch += value; | |
209 | stat->nr_batch++; | |
210 | } | |
211 | ||
212 | void blk_stat_clear(struct request_queue *q) | |
213 | { | |
214 | if (q->mq_ops) { | |
215 | struct blk_mq_hw_ctx *hctx; | |
216 | struct blk_mq_ctx *ctx; | |
217 | int i, j; | |
218 | ||
219 | queue_for_each_hw_ctx(q, hctx, i) { | |
220 | hctx_for_each_ctx(hctx, ctx, j) { | |
221 | blk_stat_init(&ctx->stat[BLK_STAT_READ]); | |
222 | blk_stat_init(&ctx->stat[BLK_STAT_WRITE]); | |
223 | } | |
224 | } | |
225 | } else { | |
226 | blk_stat_init(&q->rq_stats[BLK_STAT_READ]); | |
227 | blk_stat_init(&q->rq_stats[BLK_STAT_WRITE]); | |
228 | } | |
229 | } | |
230 | ||
231 | void blk_stat_set_issue_time(struct blk_issue_stat *stat) | |
232 | { | |
233 | stat->time = (stat->time & BLK_STAT_MASK) | | |
234 | (ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK); | |
235 | } | |
236 | ||
237 | /* | |
238 | * Enable stat tracking, return whether it was enabled | |
239 | */ | |
240 | bool blk_stat_enable(struct request_queue *q) | |
241 | { | |
242 | if (!test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) { | |
243 | set_bit(QUEUE_FLAG_STATS, &q->queue_flags); | |
244 | return false; | |
245 | } | |
246 | ||
247 | return true; | |
248 | } |