drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c

   1 /*
   2  * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include "en.h"
  34
  35 /* Adaptive moderation profiles */
  36 #define MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256
  37 #define MLX5E_RX_AM_DEF_PROFILE_CQE 1
  38 #define MLX5E_RX_AM_DEF_PROFILE_EQE 1
  39 #define MLX5E_PARAMS_AM_NUM_PROFILES 5
  40
  41 /* All profiles sizes must be MLX5E_PARAMS_AM_NUM_PROFILES */
  42 #define MLX5_AM_EQE_PROFILES { \
  43         {1,   MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  44         {8,   MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  45         {64,  MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  46         {128, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  47         {256, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
  48 }
  49
  50 #define MLX5_AM_CQE_PROFILES { \
  51         {2,  256},             \
  52         {8,  128},             \
  53         {16, 64},              \
  54         {32, 64},              \
  55         {64, 64}               \
  56 }
  57
  58 static const struct mlx5e_cq_moder
  59 profile[MLX5_CQ_PERIOD_NUM_MODES][MLX5E_PARAMS_AM_NUM_PROFILES] = {
  60         MLX5_AM_EQE_PROFILES,
  61         MLX5_AM_CQE_PROFILES,
  62 };
  63
  64 static inline struct mlx5e_cq_moder mlx5e_am_get_profile(u8 cq_period_mode, int ix)
  65 {
  66         return profile[cq_period_mode][ix];
  67 }
  68
  69 struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode)
  70 {
  71         int default_profile_ix;
  72
  73         if (rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
  74                 default_profile_ix = MLX5E_RX_AM_DEF_PROFILE_CQE;
  75         else /* MLX5_CQ_PERIOD_MODE_START_FROM_EQE */
  76                 default_profile_ix = MLX5E_RX_AM_DEF_PROFILE_EQE;
  77
  78         return profile[rx_cq_period_mode][default_profile_ix];
  79 }
  80
  81 /* Adaptive moderation logic */
  82 enum {
  83         MLX5E_AM_START_MEASURE,
  84         MLX5E_AM_MEASURE_IN_PROGRESS,
  85         MLX5E_AM_APPLY_NEW_PROFILE,
  86 };
  87
  88 enum {
  89         MLX5E_AM_PARKING_ON_TOP,
  90         MLX5E_AM_PARKING_TIRED,
  91         MLX5E_AM_GOING_RIGHT,
  92         MLX5E_AM_GOING_LEFT,
  93 };
  94
  95 enum {
  96         MLX5E_AM_STATS_WORSE,
  97         MLX5E_AM_STATS_SAME,
  98         MLX5E_AM_STATS_BETTER,
  99 };
 100
 101 enum {
 102         MLX5E_AM_STEPPED,
 103         MLX5E_AM_TOO_TIRED,
 104         MLX5E_AM_ON_EDGE,
 105 };
 106
 107 static bool mlx5e_am_on_top(struct mlx5e_rx_am *am)
 108 {
 109         switch (am->tune_state) {
 110         case MLX5E_AM_PARKING_ON_TOP:
 111         case MLX5E_AM_PARKING_TIRED:
 112                 return true;
 113         case MLX5E_AM_GOING_RIGHT:
 114                 return (am->steps_left > 1) && (am->steps_right == 1);
 115         default: /* MLX5E_AM_GOING_LEFT */
 116                 return (am->steps_right > 1) && (am->steps_left == 1);
 117         }
 118 }
 119
 120 static void mlx5e_am_turn(struct mlx5e_rx_am *am)
 121 {
 122         switch (am->tune_state) {
 123         case MLX5E_AM_PARKING_ON_TOP:
 124         case MLX5E_AM_PARKING_TIRED:
 125                 break;
 126         case MLX5E_AM_GOING_RIGHT:
 127                 am->tune_state = MLX5E_AM_GOING_LEFT;
 128                 am->steps_left = 0;
 129                 break;
 130         case MLX5E_AM_GOING_LEFT:
 131                 am->tune_state = MLX5E_AM_GOING_RIGHT;
 132                 am->steps_right = 0;
 133                 break;
 134         }
 135 }
 136
 137 static int mlx5e_am_step(struct mlx5e_rx_am *am)
 138 {
 139         if (am->tired == (MLX5E_PARAMS_AM_NUM_PROFILES * 2))
 140                 return MLX5E_AM_TOO_TIRED;
 141
 142         switch (am->tune_state) {
 143         case MLX5E_AM_PARKING_ON_TOP:
 144         case MLX5E_AM_PARKING_TIRED:
 145                 break;
 146         case MLX5E_AM_GOING_RIGHT:
 147                 if (am->profile_ix == (MLX5E_PARAMS_AM_NUM_PROFILES - 1))
 148                         return MLX5E_AM_ON_EDGE;
 149                 am->profile_ix++;
 150                 am->steps_right++;
 151                 break;
 152         case MLX5E_AM_GOING_LEFT:
 153                 if (am->profile_ix == 0)
 154                         return MLX5E_AM_ON_EDGE;
 155                 am->profile_ix--;
 156                 am->steps_left++;
 157                 break;
 158         }
 159
 160         am->tired++;
 161         return MLX5E_AM_STEPPED;
 162 }
 163
 164 static void mlx5e_am_park_on_top(struct mlx5e_rx_am *am)
 165 {
 166         am->steps_right  = 0;
 167         am->steps_left   = 0;
 168         am->tired        = 0;
 169         am->tune_state   = MLX5E_AM_PARKING_ON_TOP;
 170 }
 171
 172 static void mlx5e_am_park_tired(struct mlx5e_rx_am *am)
 173 {
 174         am->steps_right  = 0;
 175         am->steps_left   = 0;
 176         am->tune_state   = MLX5E_AM_PARKING_TIRED;
 177 }
 178
 179 static void mlx5e_am_exit_parking(struct mlx5e_rx_am *am)
 180 {
 181         am->tune_state = am->profile_ix ? MLX5E_AM_GOING_LEFT :
 182                                           MLX5E_AM_GOING_RIGHT;
 183         mlx5e_am_step(am);
 184 }
 185
 186 static int mlx5e_am_stats_compare(struct mlx5e_rx_am_stats *curr,
 187                                   struct mlx5e_rx_am_stats *prev)
 188 {
 189         int diff;
 190
 191         if (!prev->ppms)
 192                 return curr->ppms ? MLX5E_AM_STATS_BETTER :
 193                                     MLX5E_AM_STATS_SAME;
 194
 195         diff = curr->ppms - prev->ppms;
 196         if (((100 * abs(diff)) / prev->ppms) > 10) /* more than 10% diff */
 197                 return (diff > 0) ? MLX5E_AM_STATS_BETTER :
 198                                     MLX5E_AM_STATS_WORSE;
 199
 200         if (!prev->epms)
 201                 return curr->epms ? MLX5E_AM_STATS_WORSE :
 202                                     MLX5E_AM_STATS_SAME;
 203
 204         diff = curr->epms - prev->epms;
 205         if (((100 * abs(diff)) / prev->epms) > 10) /* more than 10% diff */
 206                 return (diff < 0) ? MLX5E_AM_STATS_BETTER :
 207                                     MLX5E_AM_STATS_WORSE;
 208
 209         return MLX5E_AM_STATS_SAME;
 210 }
 211
 212 static bool mlx5e_am_decision(struct mlx5e_rx_am_stats *curr_stats,
 213                               struct mlx5e_rx_am *am)
 214 {
 215         int prev_state = am->tune_state;
 216         int prev_ix = am->profile_ix;
 217         int stats_res;
 218         int step_res;
 219
 220         switch (am->tune_state) {
 221         case MLX5E_AM_PARKING_ON_TOP:
 222                 stats_res = mlx5e_am_stats_compare(curr_stats, &am->prev_stats);
 223                 if (stats_res != MLX5E_AM_STATS_SAME)
 224                         mlx5e_am_exit_parking(am);
 225                 break;
 226
 227         case MLX5E_AM_PARKING_TIRED:
 228                 am->tired--;
 229                 if (!am->tired)
 230                         mlx5e_am_exit_parking(am);
 231                 break;
 232
 233         case MLX5E_AM_GOING_RIGHT:
 234         case MLX5E_AM_GOING_LEFT:
 235                 stats_res = mlx5e_am_stats_compare(curr_stats, &am->prev_stats);
 236                 if (stats_res != MLX5E_AM_STATS_BETTER)
 237                         mlx5e_am_turn(am);
 238
 239                 if (mlx5e_am_on_top(am)) {
 240                         mlx5e_am_park_on_top(am);
 241                         break;
 242                 }
 243
 244                 step_res = mlx5e_am_step(am);
 245                 switch (step_res) {
 246                 case MLX5E_AM_ON_EDGE:
 247                         mlx5e_am_park_on_top(am);
 248                         break;
 249                 case MLX5E_AM_TOO_TIRED:
 250                         mlx5e_am_park_tired(am);
 251                         break;
 252                 }
 253
 254                 break;
 255         }
 256
 257         if ((prev_state     != MLX5E_AM_PARKING_ON_TOP) ||
 258             (am->tune_state != MLX5E_AM_PARKING_ON_TOP))
 259                 am->prev_stats = *curr_stats;
 260
 261         return am->profile_ix != prev_ix;
 262 }
 263
 264 static void mlx5e_am_sample(struct mlx5e_rq *rq,
 265                             struct mlx5e_rx_am_sample *s)
 266 {
 267         s->time      = ktime_get();
 268         s->pkt_ctr   = rq->stats.packets;
 269         s->event_ctr = rq->cq.event_ctr;
 270 }
 271
 272 #define MLX5E_AM_NEVENTS 64
 273
 274 static void mlx5e_am_calc_stats(struct mlx5e_rx_am_sample *start,
 275                                 struct mlx5e_rx_am_sample *end,
 276                                 struct mlx5e_rx_am_stats *curr_stats)
 277 {
 278         /* u32 holds up to 71 minutes, should be enough */
 279         u32 delta_us = ktime_us_delta(end->time, start->time);
 280         unsigned int npkts = end->pkt_ctr - start->pkt_ctr;
 281
 282         if (!delta_us)
 283                 return;
 284
 285         curr_stats->ppms =            (npkts * USEC_PER_MSEC) / delta_us;
 286         curr_stats->epms = (MLX5E_AM_NEVENTS * USEC_PER_MSEC) / delta_us;
 287 }
 288
 289 void mlx5e_rx_am_work(struct work_struct *work)
 290 {
 291         struct mlx5e_rx_am *am = container_of(work, struct mlx5e_rx_am,
 292                                               work);
 293         struct mlx5e_rq *rq = container_of(am, struct mlx5e_rq, am);
 294         struct mlx5e_cq_moder cur_profile = profile[am->mode][am->profile_ix];
 295
 296         mlx5_core_modify_cq_moderation(rq->mdev, &rq->cq.mcq,
 297                                        cur_profile.usec, cur_profile.pkts);
 298
 299         am->state = MLX5E_AM_START_MEASURE;
 300 }
 301
 302 void mlx5e_rx_am(struct mlx5e_rq *rq)
 303 {
 304         struct mlx5e_rx_am *am = &rq->am;
 305         struct mlx5e_rx_am_sample end_sample;
 306         struct mlx5e_rx_am_stats curr_stats;
 307         u16 nevents;
 308
 309         switch (am->state) {
 310         case MLX5E_AM_MEASURE_IN_PROGRESS:
 311                 nevents = rq->cq.event_ctr - am->start_sample.event_ctr;
 312                 if (nevents < MLX5E_AM_NEVENTS)
 313                         break;
 314                 mlx5e_am_sample(rq, &end_sample);
 315                 mlx5e_am_calc_stats(&am->start_sample, &end_sample,
 316                                     &curr_stats);
 317                 if (mlx5e_am_decision(&curr_stats, am)) {
 318                         am->state = MLX5E_AM_APPLY_NEW_PROFILE;
 319                         schedule_work(&am->work);
 320                         break;
 321                 }
 322                 /* fall through */
 323         case MLX5E_AM_START_MEASURE:
 324                 mlx5e_am_sample(rq, &am->start_sample);
 325                 am->state = MLX5E_AM_MEASURE_IN_PROGRESS;
 326                 break;
 327         case MLX5E_AM_APPLY_NEW_PROFILE:
 328                 break;
 329         }
 330 }