]>
Commit | Line | Data |
---|---|---|
cf43e6be JA |
1 | #ifndef BLK_STAT_H |
2 | #define BLK_STAT_H | |
3 | ||
34dbad5d OS |
4 | #include <linux/kernel.h> |
5 | #include <linux/blkdev.h> | |
6 | #include <linux/ktime.h> | |
7 | #include <linux/rcupdate.h> | |
8 | #include <linux/timer.h> | |
cf43e6be JA |
9 | |
10 | /* | |
88eeca49 SL |
11 | * from upper: |
12 | * 3 bits: reserved for other usage | |
13 | * 12 bits: size | |
14 | * 49 bits: time | |
cf43e6be JA |
15 | */ |
16 | #define BLK_STAT_RES_BITS 3 | |
88eeca49 SL |
17 | #define BLK_STAT_SIZE_BITS 12 |
18 | #define BLK_STAT_RES_SHIFT (64 - BLK_STAT_RES_BITS) | |
19 | #define BLK_STAT_SIZE_SHIFT (BLK_STAT_RES_SHIFT - BLK_STAT_SIZE_BITS) | |
20 | #define BLK_STAT_TIME_MASK ((1ULL << BLK_STAT_SIZE_SHIFT) - 1) | |
21 | #define BLK_STAT_SIZE_MASK \ | |
22 | (((1ULL << BLK_STAT_SIZE_BITS) - 1) << BLK_STAT_SIZE_SHIFT) | |
23 | #define BLK_STAT_RES_MASK (~((1ULL << BLK_STAT_RES_SHIFT) - 1)) | |
cf43e6be | 24 | |
34dbad5d OS |
25 | /** |
26 | * struct blk_stat_callback - Block statistics callback. | |
27 | * | |
28 | * A &struct blk_stat_callback is associated with a &struct request_queue. While | |
29 | * @timer is active, that queue's request completion latencies are sorted into | |
30 | * buckets by @bucket_fn and added to a per-cpu buffer, @cpu_stat. When the | |
31 | * timer fires, @cpu_stat is flushed to @stat and @timer_fn is invoked. | |
32 | */ | |
33 | struct blk_stat_callback { | |
34 | /* | |
35 | * @list: RCU list of callbacks for a &struct request_queue. | |
36 | */ | |
37 | struct list_head list; | |
38 | ||
39 | /** | |
40 | * @timer: Timer for the next callback invocation. | |
41 | */ | |
42 | struct timer_list timer; | |
43 | ||
44 | /** | |
45 | * @cpu_stat: Per-cpu statistics buckets. | |
46 | */ | |
47 | struct blk_rq_stat __percpu *cpu_stat; | |
48 | ||
49 | /** | |
50 | * @bucket_fn: Given a request, returns which statistics bucket it | |
51 | * should be accounted under. | |
52 | */ | |
53 | unsigned int (*bucket_fn)(const struct request *); | |
54 | ||
55 | /** | |
56 | * @buckets: Number of statistics buckets. | |
57 | */ | |
58 | unsigned int buckets; | |
59 | ||
60 | /** | |
61 | * @stat: Array of statistics buckets. | |
62 | */ | |
63 | struct blk_rq_stat *stat; | |
64 | ||
65 | /** | |
66 | * @fn: Callback function. | |
67 | */ | |
68 | void (*timer_fn)(struct blk_stat_callback *); | |
69 | ||
70 | /** | |
71 | * @data: Private pointer for the user. | |
72 | */ | |
73 | void *data; | |
74 | ||
75 | struct rcu_head rcu; | |
76 | }; | |
77 | ||
78 | struct blk_queue_stats *blk_alloc_queue_stats(void); | |
79 | void blk_free_queue_stats(struct blk_queue_stats *); | |
80 | ||
81 | void blk_stat_add(struct request *); | |
82 | ||
cf43e6be JA |
83 | static inline u64 __blk_stat_time(u64 time) |
84 | { | |
85 | return time & BLK_STAT_TIME_MASK; | |
86 | } | |
87 | ||
88 | static inline u64 blk_stat_time(struct blk_issue_stat *stat) | |
89 | { | |
88eeca49 SL |
90 | return __blk_stat_time(stat->stat); |
91 | } | |
92 | ||
93 | static inline sector_t blk_capped_size(sector_t size) | |
94 | { | |
95 | return size & ((1ULL << BLK_STAT_SIZE_BITS) - 1); | |
96 | } | |
97 | ||
98 | static inline sector_t blk_stat_size(struct blk_issue_stat *stat) | |
99 | { | |
100 | return (stat->stat & BLK_STAT_SIZE_MASK) >> BLK_STAT_SIZE_SHIFT; | |
101 | } | |
102 | ||
103 | static inline void blk_stat_set_issue(struct blk_issue_stat *stat, | |
104 | sector_t size) | |
105 | { | |
106 | stat->stat = (stat->stat & BLK_STAT_RES_MASK) | | |
107 | (ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK) | | |
108 | (((u64)blk_capped_size(size)) << BLK_STAT_SIZE_SHIFT); | |
cf43e6be JA |
109 | } |
110 | ||
b9147dd1 SL |
111 | /* record time/size info in request but not add a callback */ |
112 | void blk_stat_enable_accounting(struct request_queue *q); | |
113 | ||
34dbad5d OS |
114 | /* |
115 | * blk_stat_rq_ddir() - Bucket callback function for the request data direction. | |
116 | * @rq: Request. | |
117 | * | |
118 | * This is the same as rq_data_dir() but as a function so it can be used as | |
119 | * @bucket_fn for blk_stat_alloc_callback(). | |
120 | * | |
121 | * Return: Data direction of the request, either READ or WRITE. | |
122 | */ | |
123 | unsigned int blk_stat_rq_ddir(const struct request *rq); | |
124 | ||
125 | /** | |
126 | * blk_stat_alloc_callback() - Allocate a block statistics callback. | |
127 | * @timer_fn: Timer callback function. | |
128 | * @bucket_fn: Bucket callback function. | |
129 | * @buckets: Number of statistics buckets. | |
130 | * @data: Value for the @data field of the &struct blk_stat_callback. | |
131 | * | |
132 | * See &struct blk_stat_callback for details on the callback functions. | |
133 | * | |
134 | * Return: &struct blk_stat_callback on success or NULL on ENOMEM. | |
135 | */ | |
136 | struct blk_stat_callback * | |
137 | blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *), | |
138 | unsigned int (*bucket_fn)(const struct request *), | |
139 | unsigned int buckets, void *data); | |
140 | ||
141 | /** | |
142 | * blk_stat_add_callback() - Add a block statistics callback to be run on a | |
143 | * request queue. | |
144 | * @q: The request queue. | |
145 | * @cb: The callback. | |
146 | * | |
147 | * Note that a single &struct blk_stat_callback can only be added to a single | |
148 | * &struct request_queue. | |
149 | */ | |
150 | void blk_stat_add_callback(struct request_queue *q, | |
151 | struct blk_stat_callback *cb); | |
152 | ||
153 | /** | |
154 | * blk_stat_remove_callback() - Remove a block statistics callback from a | |
155 | * request queue. | |
156 | * @q: The request queue. | |
157 | * @cb: The callback. | |
158 | * | |
159 | * When this returns, the callback is not running on any CPUs and will not be | |
160 | * called again unless readded. | |
161 | */ | |
162 | void blk_stat_remove_callback(struct request_queue *q, | |
163 | struct blk_stat_callback *cb); | |
164 | ||
165 | /** | |
166 | * blk_stat_free_callback() - Free a block statistics callback. | |
167 | * @cb: The callback. | |
168 | * | |
169 | * @cb may be NULL, in which case this does nothing. If it is not NULL, @cb must | |
170 | * not be associated with a request queue. I.e., if it was previously added with | |
171 | * blk_stat_add_callback(), it must also have been removed since then with | |
172 | * blk_stat_remove_callback(). | |
173 | */ | |
174 | void blk_stat_free_callback(struct blk_stat_callback *cb); | |
175 | ||
176 | /** | |
177 | * blk_stat_is_active() - Check if a block statistics callback is currently | |
178 | * gathering statistics. | |
179 | * @cb: The callback. | |
180 | */ | |
181 | static inline bool blk_stat_is_active(struct blk_stat_callback *cb) | |
182 | { | |
183 | return timer_pending(&cb->timer); | |
184 | } | |
185 | ||
186 | /** | |
187 | * blk_stat_activate_nsecs() - Gather block statistics during a time window in | |
188 | * nanoseconds. | |
189 | * @cb: The callback. | |
190 | * @nsecs: Number of nanoseconds to gather statistics for. | |
191 | * | |
192 | * The timer callback will be called when the window expires. | |
193 | */ | |
194 | static inline void blk_stat_activate_nsecs(struct blk_stat_callback *cb, | |
195 | u64 nsecs) | |
196 | { | |
197 | mod_timer(&cb->timer, jiffies + nsecs_to_jiffies(nsecs)); | |
198 | } | |
199 | ||
200 | /** | |
201 | * blk_stat_activate_msecs() - Gather block statistics during a time window in | |
202 | * milliseconds. | |
203 | * @cb: The callback. | |
204 | * @msecs: Number of milliseconds to gather statistics for. | |
205 | * | |
206 | * The timer callback will be called when the window expires. | |
207 | */ | |
208 | static inline void blk_stat_activate_msecs(struct blk_stat_callback *cb, | |
209 | unsigned int msecs) | |
210 | { | |
211 | mod_timer(&cb->timer, jiffies + msecs_to_jiffies(msecs)); | |
212 | } | |
213 | ||
cf43e6be | 214 | #endif |