]>
Commit | Line | Data |
---|---|---|
bd9d3e69 KM |
1 | /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ |
2 | /* Copyright (c) 2019 Mellanox Technologies. */ | |
3 | ||
4 | #ifndef DIM_H | |
5 | #define DIM_H | |
6 | ||
7 | #include <linux/module.h> | |
8 | ||
9 | /** | |
10 | * Number of events between DIM iterations. | |
11 | * Causes a moderation of the algorithm run. | |
12 | */ | |
13 | #define DIM_NEVENTS 64 | |
14 | ||
15 | /** | |
16 | * Is a difference between values justifies taking an action. | |
17 | * We consider 10% difference as significant. | |
18 | */ | |
19 | #define IS_SIGNIFICANT_DIFF(val, ref) \ | |
20 | (((100UL * abs((val) - (ref))) / (ref)) > 10) | |
21 | ||
22 | /** | |
23 | * Calculate the gap between two values. | |
24 | * Take wrap-around and variable size into consideration. | |
25 | */ | |
26 | #define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) \ | |
27 | & (BIT_ULL(bits) - 1)) | |
28 | ||
29 | /** | |
30 | * Structure for CQ moderation values. | |
31 | * Used for communications between DIM and its consumer. | |
32 | * | |
33 | * @usec: CQ timer suggestion (by DIM) | |
34 | * @pkts: CQ packet counter suggestion (by DIM) | |
35 | * @cq_period_mode: CQ priod count mode (from CQE/EQE) | |
36 | */ | |
37 | struct dim_cq_moder { | |
38 | u16 usec; | |
39 | u16 pkts; | |
40 | u16 comps; | |
41 | u8 cq_period_mode; | |
42 | }; | |
43 | ||
44 | /** | |
45 | * Structure for DIM sample data. | |
46 | * Used for communications between DIM and its consumer. | |
47 | * | |
48 | * @time: Sample timestamp | |
49 | * @pkt_ctr: Number of packets | |
50 | * @byte_ctr: Number of bytes | |
51 | * @event_ctr: Number of events | |
52 | */ | |
53 | struct dim_sample { | |
54 | ktime_t time; | |
55 | u32 pkt_ctr; | |
56 | u32 byte_ctr; | |
57 | u16 event_ctr; | |
58 | u32 comp_ctr; | |
59 | }; | |
60 | ||
61 | /** | |
62 | * Structure for DIM stats. | |
63 | * Used for holding current measured rates. | |
64 | * | |
65 | * @ppms: Packets per msec | |
66 | * @bpms: Bytes per msec | |
67 | * @epms: Events per msec | |
68 | */ | |
69 | struct dim_stats { | |
70 | int ppms; /* packets per msec */ | |
71 | int bpms; /* bytes per msec */ | |
72 | int epms; /* events per msec */ | |
73 | int cpms; /* completions per msec */ | |
74 | int cpe_ratio; /* ratio of completions to events */ | |
75 | }; | |
76 | ||
77 | /** | |
78 | * Main structure for dynamic interrupt moderation (DIM). | |
79 | * Used for holding all information about a specific DIM instance. | |
80 | * | |
81 | * @state: Algorithm state (see below) | |
82 | * @prev_stats: Measured rates from previous iteration (for comparison) | |
83 | * @start_sample: Sampled data at start of current iteration | |
84 | * @work: Work to perform on action required | |
85 | * @priv: A pointer to the struct that points to dim | |
86 | * @profile_ix: Current moderation profile | |
87 | * @mode: CQ period count mode | |
88 | * @tune_state: Algorithm tuning state (see below) | |
89 | * @steps_right: Number of steps taken towards higher moderation | |
90 | * @steps_left: Number of steps taken towards lower moderation | |
91 | * @tired: Parking depth counter | |
92 | */ | |
93 | struct dim { | |
94 | u8 state; | |
95 | struct dim_stats prev_stats; | |
96 | struct dim_sample start_sample; | |
97 | struct dim_sample measuring_sample; | |
98 | struct work_struct work; | |
99 | void *priv; | |
100 | u8 profile_ix; | |
101 | u8 mode; | |
102 | u8 tune_state; | |
103 | u8 steps_right; | |
104 | u8 steps_left; | |
105 | u8 tired; | |
106 | }; | |
107 | ||
108 | /** | |
109 | * enum dim_cq_period_mode | |
110 | * | |
111 | * These are the modes for CQ period count. | |
112 | * | |
113 | * @DIM_CQ_PERIOD_MODE_START_FROM_EQE: Start counting from EQE | |
114 | * @DIM_CQ_PERIOD_MODE_START_FROM_CQE: Start counting from CQE (implies timer reset) | |
115 | * @DIM_CQ_PERIOD_NUM_MODES: Number of modes | |
116 | */ | |
117 | enum { | |
118 | DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0, | |
119 | DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1, | |
120 | DIM_CQ_PERIOD_NUM_MODES | |
121 | }; | |
122 | ||
123 | /** | |
124 | * enum dim_state | |
125 | * | |
126 | * These are the DIM algorithm states. | |
127 | * These will determine if the algorithm is in a valid state to start an iteration. | |
128 | * | |
129 | * @DIM_START_MEASURE: This is the first iteration (also after applying a new profile) | |
130 | * @DIM_MEASURE_IN_PROGRESS: Algorithm is already in progress - check if | |
131 | * need to perform an action | |
132 | * @DIM_APPLY_NEW_PROFILE: DIM consumer is currently applying a profile - no need to measure | |
133 | */ | |
134 | enum { | |
135 | DIM_START_MEASURE, | |
136 | DIM_MEASURE_IN_PROGRESS, | |
137 | DIM_APPLY_NEW_PROFILE, | |
138 | }; | |
139 | ||
140 | /** | |
141 | * enum dim_tune_state | |
142 | * | |
143 | * These are the DIM algorithm tune states. | |
144 | * These will determine which action the algorithm should perform. | |
145 | * | |
146 | * @DIM_PARKING_ON_TOP: Algorithm found a local top point - exit on significant difference | |
147 | * @DIM_PARKING_TIRED: Algorithm found a deep top point - don't exit if tired > 0 | |
148 | * @DIM_GOING_RIGHT: Algorithm is currently trying higher moderation levels | |
149 | * @DIM_GOING_LEFT: Algorithm is currently trying lower moderation levels | |
150 | */ | |
151 | enum { | |
152 | DIM_PARKING_ON_TOP, | |
153 | DIM_PARKING_TIRED, | |
154 | DIM_GOING_RIGHT, | |
155 | DIM_GOING_LEFT, | |
156 | }; | |
157 | ||
158 | /** | |
159 | * enum dim_stats_state | |
160 | * | |
161 | * These are the DIM algorithm statistics states. | |
162 | * These will determine the verdict of current iteration. | |
163 | * | |
164 | * @DIM_STATS_WORSE: Current iteration shows worse performance than before | |
165 | * @DIM_STATS_WORSE: Current iteration shows same performance than before | |
166 | * @DIM_STATS_WORSE: Current iteration shows better performance than before | |
167 | */ | |
168 | enum { | |
169 | DIM_STATS_WORSE, | |
170 | DIM_STATS_SAME, | |
171 | DIM_STATS_BETTER, | |
172 | }; | |
173 | ||
174 | /** | |
175 | * enum dim_step_result | |
176 | * | |
177 | * These are the DIM algorithm step results. | |
178 | * These describe the result of a step. | |
179 | * | |
180 | * @DIM_STEPPED: Performed a regular step | |
181 | * @DIM_TOO_TIRED: Same kind of step was done multiple times - should go to | |
182 | * tired parking | |
183 | * @DIM_ON_EDGE: Stepped to the most left/right profile | |
184 | */ | |
185 | enum { | |
186 | DIM_STEPPED, | |
187 | DIM_TOO_TIRED, | |
188 | DIM_ON_EDGE, | |
189 | }; | |
190 | ||
191 | /** | |
192 | * dim_on_top - check if current state is a good place to stop (top location) | |
193 | * @dim: DIM context | |
194 | * | |
195 | * Check if current profile is a good place to park at. | |
196 | * This will result in reducing the DIM checks frequency as we assume we | |
197 | * shouldn't probably change profiles, unless traffic pattern wasn't changed. | |
198 | */ | |
199 | bool dim_on_top(struct dim *dim); | |
200 | ||
201 | /** | |
202 | * dim_turn - change profile alterning direction | |
203 | * @dim: DIM context | |
204 | * | |
205 | * Go left if we were going right and vice-versa. | |
206 | * Do nothing if currently parking. | |
207 | */ | |
208 | void dim_turn(struct dim *dim); | |
209 | ||
210 | /** | |
211 | * dim_park_on_top - enter a parking state on a top location | |
212 | * @dim: DIM context | |
213 | * | |
214 | * Enter parking state. | |
215 | * Clear all movement history. | |
216 | */ | |
217 | void dim_park_on_top(struct dim *dim); | |
218 | ||
219 | /** | |
220 | * dim_park_tired - enter a tired parking state | |
221 | * @dim: DIM context | |
222 | * | |
223 | * Enter parking state. | |
224 | * Clear all movement history and cause DIM checks frequency to reduce. | |
225 | */ | |
226 | void dim_park_tired(struct dim *dim); | |
227 | ||
228 | /** | |
229 | * dim_calc_stats - calculate the difference between two samples | |
230 | * @start: start sample | |
231 | * @end: end sample | |
232 | * @curr_stats: delta between samples | |
233 | * | |
234 | * Calculate the delta between two samples (in data rates). | |
235 | * Takes into consideration counter wrap-around. | |
236 | */ | |
237 | void dim_calc_stats(struct dim_sample *start, struct dim_sample *end, | |
238 | struct dim_stats *curr_stats); | |
239 | ||
240 | /** | |
241 | * dim_update_sample - set a sample's fields with give values | |
242 | * @event_ctr: number of events to set | |
243 | * @packets: number of packets to set | |
244 | * @bytes: number of bytes to set | |
245 | * @s: DIM sample | |
246 | */ | |
247 | static inline void | |
248 | dim_update_sample(u16 event_ctr, u64 packets, u64 bytes, struct dim_sample *s) | |
249 | { | |
250 | s->time = ktime_get(); | |
251 | s->pkt_ctr = packets; | |
252 | s->byte_ctr = bytes; | |
253 | s->event_ctr = event_ctr; | |
254 | } | |
255 | ||
256 | /** | |
257 | * dim_update_sample_with_comps - set a sample's fields with given | |
258 | * values including the completion parameter | |
259 | * @event_ctr: number of events to set | |
260 | * @packets: number of packets to set | |
261 | * @bytes: number of bytes to set | |
262 | * @comps: number of completions to set | |
263 | * @s: DIM sample | |
264 | */ | |
265 | static inline void | |
266 | dim_update_sample_with_comps(u16 event_ctr, u64 packets, u64 bytes, u64 comps, | |
267 | struct dim_sample *s) | |
268 | { | |
269 | dim_update_sample(event_ctr, packets, bytes, s); | |
270 | s->comp_ctr = comps; | |
271 | } | |
272 | ||
273 | /* Net DIM */ | |
274 | ||
275 | /** | |
276 | * net_dim_get_rx_moderation - provide a CQ moderation object for the given RX profile | |
277 | * @cq_period_mode: CQ period mode | |
278 | * @ix: Profile index | |
279 | */ | |
280 | struct dim_cq_moder net_dim_get_rx_moderation(u8 cq_period_mode, int ix); | |
281 | ||
282 | /** | |
283 | * net_dim_get_def_rx_moderation - provide the default RX moderation | |
284 | * @cq_period_mode: CQ period mode | |
285 | */ | |
286 | struct dim_cq_moder net_dim_get_def_rx_moderation(u8 cq_period_mode); | |
287 | ||
288 | /** | |
289 | * net_dim_get_tx_moderation - provide a CQ moderation object for the given TX profile | |
290 | * @cq_period_mode: CQ period mode | |
291 | * @ix: Profile index | |
292 | */ | |
293 | struct dim_cq_moder net_dim_get_tx_moderation(u8 cq_period_mode, int ix); | |
294 | ||
295 | /** | |
296 | * net_dim_get_def_tx_moderation - provide the default TX moderation | |
297 | * @cq_period_mode: CQ period mode | |
298 | */ | |
299 | struct dim_cq_moder net_dim_get_def_tx_moderation(u8 cq_period_mode); | |
300 | ||
301 | /** | |
302 | * net_dim - main DIM algorithm entry point | |
303 | * @dim: DIM instance information | |
304 | * @end_sample: Current data measurement | |
305 | * | |
306 | * Called by the consumer. | |
307 | * This is the main logic of the algorithm, where data is processed in order to decide on next | |
308 | * required action. | |
309 | */ | |
310 | void net_dim(struct dim *dim, struct dim_sample end_sample); | |
311 | ||
312 | /* RDMA DIM */ | |
313 | ||
314 | /* | |
315 | * RDMA DIM profile: | |
316 | * profile size must be of RDMA_DIM_PARAMS_NUM_PROFILES. | |
317 | */ | |
318 | #define RDMA_DIM_PARAMS_NUM_PROFILES 9 | |
319 | #define RDMA_DIM_START_PROFILE 0 | |
320 | ||
321 | static const struct dim_cq_moder | |
322 | rdma_dim_prof[RDMA_DIM_PARAMS_NUM_PROFILES] = { | |
323 | {1, 0, 1, 0}, | |
324 | {1, 0, 4, 0}, | |
325 | {2, 0, 4, 0}, | |
326 | {2, 0, 8, 0}, | |
327 | {4, 0, 8, 0}, | |
328 | {16, 0, 8, 0}, | |
329 | {16, 0, 16, 0}, | |
330 | {32, 0, 16, 0}, | |
331 | {32, 0, 32, 0}, | |
332 | }; | |
333 | ||
334 | /** | |
335 | * rdma_dim - Runs the adaptive moderation. | |
336 | * @dim: The moderation struct. | |
337 | * @completions: The number of completions collected in this round. | |
338 | * | |
339 | * Each call to rdma_dim takes the latest amount of completions that | |
340 | * have been collected and counts them as a new event. | |
341 | * Once enough events have been collected the algorithm decides a new | |
342 | * moderation level. | |
343 | */ | |
344 | void rdma_dim(struct dim *dim, u64 completions); | |
345 | ||
346 | #endif /* DIM_H */ |