2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2017-2018, Broadcom Limited. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
37 #include <linux/module.h>
39 struct net_dim_cq_moder
{
45 struct net_dim_sample
{
52 struct net_dim_stats
{
53 int ppms
; /* packets per msec */
54 int bpms
; /* bytes per msec */
55 int epms
; /* events per msec */
58 struct net_dim
{ /* Adaptive Moderation */
60 struct net_dim_stats prev_stats
;
61 struct net_dim_sample start_sample
;
62 struct work_struct work
;
72 NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE
= 0x0,
73 NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE
= 0x1,
74 NET_DIM_CQ_PERIOD_NUM_MODES
77 /* Adaptive moderation logic */
79 NET_DIM_START_MEASURE
,
80 NET_DIM_MEASURE_IN_PROGRESS
,
81 NET_DIM_APPLY_NEW_PROFILE
,
85 NET_DIM_PARKING_ON_TOP
,
86 NET_DIM_PARKING_TIRED
,
103 #define NET_DIM_PARAMS_NUM_PROFILES 5
104 /* Adaptive moderation profiles */
105 #define NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256
106 #define NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE 128
107 #define NET_DIM_DEF_PROFILE_CQE 1
108 #define NET_DIM_DEF_PROFILE_EQE 1
110 /* All profiles sizes must be NET_PARAMS_DIM_NUM_PROFILES */
111 #define NET_DIM_RX_EQE_PROFILES { \
112 {1, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
113 {8, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
114 {64, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
115 {128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
116 {256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
119 #define NET_DIM_RX_CQE_PROFILES { \
127 #define NET_DIM_TX_EQE_PROFILES { \
128 {1, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
129 {8, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
130 {32, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
131 {64, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
132 {128, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE} \
135 #define NET_DIM_TX_CQE_PROFILES { \
143 static const struct net_dim_cq_moder
144 rx_profile
[NET_DIM_CQ_PERIOD_NUM_MODES
][NET_DIM_PARAMS_NUM_PROFILES
] = {
145 NET_DIM_RX_EQE_PROFILES
,
146 NET_DIM_RX_CQE_PROFILES
,
149 static const struct net_dim_cq_moder
150 tx_profile
[NET_DIM_CQ_PERIOD_NUM_MODES
][NET_DIM_PARAMS_NUM_PROFILES
] = {
151 NET_DIM_TX_EQE_PROFILES
,
152 NET_DIM_TX_CQE_PROFILES
,
155 static inline struct net_dim_cq_moder
156 net_dim_get_rx_moderation(u8 cq_period_mode
, int ix
)
158 struct net_dim_cq_moder cq_moder
= rx_profile
[cq_period_mode
][ix
];
160 cq_moder
.cq_period_mode
= cq_period_mode
;
164 static inline struct net_dim_cq_moder
165 net_dim_get_def_rx_moderation(u8 cq_period_mode
)
167 u8 profile_ix
= cq_period_mode
== NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE
?
168 NET_DIM_DEF_PROFILE_CQE
: NET_DIM_DEF_PROFILE_EQE
;
170 return net_dim_get_rx_moderation(cq_period_mode
, profile_ix
);
173 static inline struct net_dim_cq_moder
174 net_dim_get_tx_moderation(u8 cq_period_mode
, int ix
)
176 struct net_dim_cq_moder cq_moder
= tx_profile
[cq_period_mode
][ix
];
178 cq_moder
.cq_period_mode
= cq_period_mode
;
182 static inline struct net_dim_cq_moder
183 net_dim_get_def_tx_moderation(u8 cq_period_mode
)
185 u8 profile_ix
= cq_period_mode
== NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE
?
186 NET_DIM_DEF_PROFILE_CQE
: NET_DIM_DEF_PROFILE_EQE
;
188 return net_dim_get_tx_moderation(cq_period_mode
, profile_ix
);
191 static inline bool net_dim_on_top(struct net_dim
*dim
)
193 switch (dim
->tune_state
) {
194 case NET_DIM_PARKING_ON_TOP
:
195 case NET_DIM_PARKING_TIRED
:
197 case NET_DIM_GOING_RIGHT
:
198 return (dim
->steps_left
> 1) && (dim
->steps_right
== 1);
199 default: /* NET_DIM_GOING_LEFT */
200 return (dim
->steps_right
> 1) && (dim
->steps_left
== 1);
204 static inline void net_dim_turn(struct net_dim
*dim
)
206 switch (dim
->tune_state
) {
207 case NET_DIM_PARKING_ON_TOP
:
208 case NET_DIM_PARKING_TIRED
:
210 case NET_DIM_GOING_RIGHT
:
211 dim
->tune_state
= NET_DIM_GOING_LEFT
;
214 case NET_DIM_GOING_LEFT
:
215 dim
->tune_state
= NET_DIM_GOING_RIGHT
;
216 dim
->steps_right
= 0;
221 static inline int net_dim_step(struct net_dim
*dim
)
223 if (dim
->tired
== (NET_DIM_PARAMS_NUM_PROFILES
* 2))
224 return NET_DIM_TOO_TIRED
;
226 switch (dim
->tune_state
) {
227 case NET_DIM_PARKING_ON_TOP
:
228 case NET_DIM_PARKING_TIRED
:
230 case NET_DIM_GOING_RIGHT
:
231 if (dim
->profile_ix
== (NET_DIM_PARAMS_NUM_PROFILES
- 1))
232 return NET_DIM_ON_EDGE
;
236 case NET_DIM_GOING_LEFT
:
237 if (dim
->profile_ix
== 0)
238 return NET_DIM_ON_EDGE
;
245 return NET_DIM_STEPPED
;
248 static inline void net_dim_park_on_top(struct net_dim
*dim
)
250 dim
->steps_right
= 0;
253 dim
->tune_state
= NET_DIM_PARKING_ON_TOP
;
256 static inline void net_dim_park_tired(struct net_dim
*dim
)
258 dim
->steps_right
= 0;
260 dim
->tune_state
= NET_DIM_PARKING_TIRED
;
263 static inline void net_dim_exit_parking(struct net_dim
*dim
)
265 dim
->tune_state
= dim
->profile_ix
? NET_DIM_GOING_LEFT
:
270 #define IS_SIGNIFICANT_DIFF(val, ref) \
271 (((100UL * abs((val) - (ref))) / (ref)) > 10) /* more than 10% difference */
273 static inline int net_dim_stats_compare(struct net_dim_stats
*curr
,
274 struct net_dim_stats
*prev
)
277 return curr
->bpms
? NET_DIM_STATS_BETTER
:
280 if (IS_SIGNIFICANT_DIFF(curr
->bpms
, prev
->bpms
))
281 return (curr
->bpms
> prev
->bpms
) ? NET_DIM_STATS_BETTER
:
285 return curr
->ppms
? NET_DIM_STATS_BETTER
:
288 if (IS_SIGNIFICANT_DIFF(curr
->ppms
, prev
->ppms
))
289 return (curr
->ppms
> prev
->ppms
) ? NET_DIM_STATS_BETTER
:
293 return NET_DIM_STATS_SAME
;
295 if (IS_SIGNIFICANT_DIFF(curr
->epms
, prev
->epms
))
296 return (curr
->epms
< prev
->epms
) ? NET_DIM_STATS_BETTER
:
299 return NET_DIM_STATS_SAME
;
302 static inline bool net_dim_decision(struct net_dim_stats
*curr_stats
,
305 int prev_state
= dim
->tune_state
;
306 int prev_ix
= dim
->profile_ix
;
310 switch (dim
->tune_state
) {
311 case NET_DIM_PARKING_ON_TOP
:
312 stats_res
= net_dim_stats_compare(curr_stats
, &dim
->prev_stats
);
313 if (stats_res
!= NET_DIM_STATS_SAME
)
314 net_dim_exit_parking(dim
);
317 case NET_DIM_PARKING_TIRED
:
320 net_dim_exit_parking(dim
);
323 case NET_DIM_GOING_RIGHT
:
324 case NET_DIM_GOING_LEFT
:
325 stats_res
= net_dim_stats_compare(curr_stats
, &dim
->prev_stats
);
326 if (stats_res
!= NET_DIM_STATS_BETTER
)
329 if (net_dim_on_top(dim
)) {
330 net_dim_park_on_top(dim
);
334 step_res
= net_dim_step(dim
);
336 case NET_DIM_ON_EDGE
:
337 net_dim_park_on_top(dim
);
339 case NET_DIM_TOO_TIRED
:
340 net_dim_park_tired(dim
);
347 if ((prev_state
!= NET_DIM_PARKING_ON_TOP
) ||
348 (dim
->tune_state
!= NET_DIM_PARKING_ON_TOP
))
349 dim
->prev_stats
= *curr_stats
;
351 return dim
->profile_ix
!= prev_ix
;
354 static inline void net_dim_sample(u16 event_ctr
,
357 struct net_dim_sample
*s
)
359 s
->time
= ktime_get();
360 s
->pkt_ctr
= packets
;
362 s
->event_ctr
= event_ctr
;
365 #define NET_DIM_NEVENTS 64
366 #define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1))
368 static inline void net_dim_calc_stats(struct net_dim_sample
*start
,
369 struct net_dim_sample
*end
,
370 struct net_dim_stats
*curr_stats
)
372 /* u32 holds up to 71 minutes, should be enough */
373 u32 delta_us
= ktime_us_delta(end
->time
, start
->time
);
374 u32 npkts
= BIT_GAP(BITS_PER_TYPE(u32
), end
->pkt_ctr
, start
->pkt_ctr
);
375 u32 nbytes
= BIT_GAP(BITS_PER_TYPE(u32
), end
->byte_ctr
,
381 curr_stats
->ppms
= DIV_ROUND_UP(npkts
* USEC_PER_MSEC
, delta_us
);
382 curr_stats
->bpms
= DIV_ROUND_UP(nbytes
* USEC_PER_MSEC
, delta_us
);
383 curr_stats
->epms
= DIV_ROUND_UP(NET_DIM_NEVENTS
* USEC_PER_MSEC
,
387 static inline void net_dim(struct net_dim
*dim
,
388 struct net_dim_sample end_sample
)
390 struct net_dim_stats curr_stats
;
393 switch (dim
->state
) {
394 case NET_DIM_MEASURE_IN_PROGRESS
:
395 nevents
= BIT_GAP(BITS_PER_TYPE(u16
),
396 end_sample
.event_ctr
,
397 dim
->start_sample
.event_ctr
);
398 if (nevents
< NET_DIM_NEVENTS
)
400 net_dim_calc_stats(&dim
->start_sample
, &end_sample
,
402 if (net_dim_decision(&curr_stats
, dim
)) {
403 dim
->state
= NET_DIM_APPLY_NEW_PROFILE
;
404 schedule_work(&dim
->work
);
408 case NET_DIM_START_MEASURE
:
409 net_dim_sample(end_sample
.event_ctr
, end_sample
.pkt_ctr
, end_sample
.byte_ctr
,
411 dim
->state
= NET_DIM_MEASURE_IN_PROGRESS
;
413 case NET_DIM_APPLY_NEW_PROFILE
:
418 #endif /* NET_DIM_H */