]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
e34cbd30 JA |
2 | #ifndef WB_THROTTLE_H |
3 | #define WB_THROTTLE_H | |
4 | ||
5 | #include <linux/kernel.h> | |
6 | #include <linux/atomic.h> | |
7 | #include <linux/wait.h> | |
8 | #include <linux/timer.h> | |
9 | #include <linux/ktime.h> | |
10 | ||
11 | #include "blk-stat.h" | |
12 | ||
13 | enum wbt_flags { | |
14 | WBT_TRACKED = 1, /* write, tracked for throttling */ | |
15 | WBT_READ = 2, /* read */ | |
16 | WBT_KSWAPD = 4, /* write, from kswapd */ | |
17 | ||
18 | WBT_NR_BITS = 3, /* number of bits */ | |
19 | }; | |
20 | ||
21 | enum { | |
d9204a0b JA |
22 | WBT_RWQ_BG = 0, |
23 | WBT_RWQ_KSWAPD, | |
24 | WBT_NUM_RWQ, | |
e34cbd30 JA |
25 | }; |
26 | ||
d62118b6 JA |
27 | /* |
28 | * Enable states. Either off, or on by default (done at init time), | |
29 | * or on through manual setup in sysfs. | |
30 | */ | |
31 | enum { | |
32 | WBT_STATE_ON_DEFAULT = 1, | |
33 | WBT_STATE_ON_MANUAL = 2, | |
34 | }; | |
35 | ||
e34cbd30 JA |
36 | static inline void wbt_clear_state(struct blk_issue_stat *stat) |
37 | { | |
88eeca49 | 38 | stat->stat &= ~BLK_STAT_RES_MASK; |
e34cbd30 JA |
39 | } |
40 | ||
41 | static inline enum wbt_flags wbt_stat_to_mask(struct blk_issue_stat *stat) | |
42 | { | |
88eeca49 | 43 | return (stat->stat & BLK_STAT_RES_MASK) >> BLK_STAT_RES_SHIFT; |
e34cbd30 JA |
44 | } |
45 | ||
46 | static inline void wbt_track(struct blk_issue_stat *stat, enum wbt_flags wb_acct) | |
47 | { | |
88eeca49 | 48 | stat->stat |= ((u64) wb_acct) << BLK_STAT_RES_SHIFT; |
e34cbd30 JA |
49 | } |
50 | ||
51 | static inline bool wbt_is_tracked(struct blk_issue_stat *stat) | |
52 | { | |
88eeca49 | 53 | return (stat->stat >> BLK_STAT_RES_SHIFT) & WBT_TRACKED; |
e34cbd30 JA |
54 | } |
55 | ||
56 | static inline bool wbt_is_read(struct blk_issue_stat *stat) | |
57 | { | |
88eeca49 | 58 | return (stat->stat >> BLK_STAT_RES_SHIFT) & WBT_READ; |
e34cbd30 JA |
59 | } |
60 | ||
e34cbd30 JA |
61 | struct rq_wait { |
62 | wait_queue_head_t wait; | |
63 | atomic_t inflight; | |
64 | }; | |
65 | ||
66 | struct rq_wb { | |
67 | /* | |
68 | * Settings that govern how we throttle | |
69 | */ | |
70 | unsigned int wb_background; /* background writeback */ | |
71 | unsigned int wb_normal; /* normal writeback */ | |
72 | unsigned int wb_max; /* max throughput writeback */ | |
73 | int scale_step; | |
74 | bool scaled_max; | |
75 | ||
d62118b6 JA |
76 | short enable_state; /* WBT_STATE_* */ |
77 | ||
e34cbd30 JA |
78 | /* |
79 | * Number of consecutive periods where we don't have enough | |
80 | * information to make a firm scale up/down decision. | |
81 | */ | |
82 | unsigned int unknown_cnt; | |
83 | ||
84 | u64 win_nsec; /* default window size */ | |
85 | u64 cur_win_nsec; /* current window size */ | |
86 | ||
34dbad5d | 87 | struct blk_stat_callback *cb; |
e34cbd30 JA |
88 | |
89 | s64 sync_issue; | |
90 | void *sync_cookie; | |
91 | ||
92 | unsigned int wc; | |
93 | unsigned int queue_depth; | |
94 | ||
95 | unsigned long last_issue; /* last non-throttled issue */ | |
96 | unsigned long last_comp; /* last non-throttled comp */ | |
97 | unsigned long min_lat_nsec; | |
d8a0cbfd | 98 | struct request_queue *queue; |
e34cbd30 | 99 | struct rq_wait rq_wait[WBT_NUM_RWQ]; |
e34cbd30 JA |
100 | }; |
101 | ||
102 | static inline unsigned int wbt_inflight(struct rq_wb *rwb) | |
103 | { | |
104 | unsigned int i, ret = 0; | |
105 | ||
106 | for (i = 0; i < WBT_NUM_RWQ; i++) | |
107 | ret += atomic_read(&rwb->rq_wait[i].inflight); | |
108 | ||
109 | return ret; | |
110 | } | |
111 | ||
e34cbd30 JA |
112 | #ifdef CONFIG_BLK_WBT |
113 | ||
114 | void __wbt_done(struct rq_wb *, enum wbt_flags); | |
115 | void wbt_done(struct rq_wb *, struct blk_issue_stat *); | |
116 | enum wbt_flags wbt_wait(struct rq_wb *, struct bio *, spinlock_t *); | |
8054b89f | 117 | int wbt_init(struct request_queue *); |
e34cbd30 JA |
118 | void wbt_exit(struct request_queue *); |
119 | void wbt_update_limits(struct rq_wb *); | |
120 | void wbt_requeue(struct rq_wb *, struct blk_issue_stat *); | |
121 | void wbt_issue(struct rq_wb *, struct blk_issue_stat *); | |
fa224eed | 122 | void wbt_disable_default(struct request_queue *); |
8330cdb0 | 123 | void wbt_enable_default(struct request_queue *); |
e34cbd30 JA |
124 | |
125 | void wbt_set_queue_depth(struct rq_wb *, unsigned int); | |
126 | void wbt_set_write_cache(struct rq_wb *, bool); | |
127 | ||
80e091d1 JA |
128 | u64 wbt_default_latency_nsec(struct request_queue *); |
129 | ||
e34cbd30 JA |
130 | #else |
131 | ||
132 | static inline void __wbt_done(struct rq_wb *rwb, enum wbt_flags flags) | |
133 | { | |
134 | } | |
135 | static inline void wbt_done(struct rq_wb *rwb, struct blk_issue_stat *stat) | |
136 | { | |
137 | } | |
138 | static inline enum wbt_flags wbt_wait(struct rq_wb *rwb, struct bio *bio, | |
139 | spinlock_t *lock) | |
140 | { | |
141 | return 0; | |
142 | } | |
8054b89f | 143 | static inline int wbt_init(struct request_queue *q) |
e34cbd30 JA |
144 | { |
145 | return -EINVAL; | |
146 | } | |
147 | static inline void wbt_exit(struct request_queue *q) | |
148 | { | |
149 | } | |
150 | static inline void wbt_update_limits(struct rq_wb *rwb) | |
151 | { | |
152 | } | |
153 | static inline void wbt_requeue(struct rq_wb *rwb, struct blk_issue_stat *stat) | |
154 | { | |
155 | } | |
156 | static inline void wbt_issue(struct rq_wb *rwb, struct blk_issue_stat *stat) | |
157 | { | |
158 | } | |
fa224eed | 159 | static inline void wbt_disable_default(struct request_queue *q) |
e34cbd30 JA |
160 | { |
161 | } | |
8330cdb0 JK |
162 | static inline void wbt_enable_default(struct request_queue *q) |
163 | { | |
164 | } | |
e34cbd30 JA |
165 | static inline void wbt_set_queue_depth(struct rq_wb *rwb, unsigned int depth) |
166 | { | |
167 | } | |
168 | static inline void wbt_set_write_cache(struct rq_wb *rwb, bool wc) | |
169 | { | |
170 | } | |
80e091d1 JA |
171 | static inline u64 wbt_default_latency_nsec(struct request_queue *q) |
172 | { | |
173 | return 0; | |
174 | } | |
e34cbd30 JA |
175 | |
176 | #endif /* CONFIG_BLK_WBT */ | |
177 | ||
178 | #endif |