drivers/net/ethernet/mellanox/mlx5/core/en/params.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /* Copyright (c) 2019 Mellanox Technologies. */
   3
   4 #include "en/params.h"
   5 #include "en/txrx.h"
   6 #include "en/port.h"
   7 #include "en_accel/en_accel.h"
   8 #include "accel/ipsec.h"
   9 #include "fpga/ipsec.h"
  10
  11 static bool mlx5e_rx_is_xdp(struct mlx5e_params *params,
  12                             struct mlx5e_xsk_param *xsk)
  13 {
  14         return params->xdp_prog || xsk;
  15 }
  16
  17 u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params,
  18                                  struct mlx5e_xsk_param *xsk)
  19 {
  20         u16 headroom;
  21
  22         if (xsk)
  23                 return xsk->headroom;
  24
  25         headroom = NET_IP_ALIGN;
  26         if (mlx5e_rx_is_xdp(params, xsk))
  27                 headroom += XDP_PACKET_HEADROOM;
  28         else
  29                 headroom += MLX5_RX_HEADROOM;
  30
  31         return headroom;
  32 }
  33
  34 u32 mlx5e_rx_get_min_frag_sz(struct mlx5e_params *params,
  35                              struct mlx5e_xsk_param *xsk)
  36 {
  37         u32 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
  38         u16 linear_rq_headroom = mlx5e_get_linear_rq_headroom(params, xsk);
  39
  40         return linear_rq_headroom + hw_mtu;
  41 }
  42
  43 static u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params,
  44                                        struct mlx5e_xsk_param *xsk)
  45 {
  46         u32 frag_sz = mlx5e_rx_get_min_frag_sz(params, xsk);
  47
  48         /* AF_XDP doesn't build SKBs in place. */
  49         if (!xsk)
  50                 frag_sz = MLX5_SKB_FRAG_SZ(frag_sz);
  51
  52         /* XDP in mlx5e doesn't support multiple packets per page. AF_XDP is a
  53          * special case. It can run with frames smaller than a page, as it
  54          * doesn't allocate pages dynamically. However, here we pretend that
  55          * fragments are page-sized: it allows to treat XSK frames like pages
  56          * by redirecting alloc and free operations to XSK rings and by using
  57          * the fact there are no multiple packets per "page" (which is a frame).
  58          * The latter is important, because frames may come in a random order,
  59          * and we will have trouble assemblying a real page of multiple frames.
  60          */
  61         if (mlx5e_rx_is_xdp(params, xsk))
  62                 frag_sz = max_t(u32, frag_sz, PAGE_SIZE);
  63
  64         /* Even if we can go with a smaller fragment size, we must not put
  65          * multiple packets into a single frame.
  66          */
  67         if (xsk)
  68                 frag_sz = max_t(u32, frag_sz, xsk->chunk_size);
  69
  70         return frag_sz;
  71 }
  72
  73 u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params,
  74                                 struct mlx5e_xsk_param *xsk)
  75 {
  76         u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params, xsk);
  77
  78         return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz);
  79 }
  80
  81 bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params,
  82                             struct mlx5e_xsk_param *xsk)
  83 {
  84         /* AF_XDP allocates SKBs on XDP_PASS - ensure they don't occupy more
  85          * than one page. For this, check both with and without xsk.
  86          */
  87         u32 linear_frag_sz = max(mlx5e_rx_get_linear_frag_sz(params, xsk),
  88                                  mlx5e_rx_get_linear_frag_sz(params, NULL));
  89
  90         return !params->lro_en && linear_frag_sz <= PAGE_SIZE;
  91 }
  92
  93 bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
  94                                    u8 log_stride_sz, u8 log_num_strides)
  95 {
  96         if (log_stride_sz + log_num_strides != MLX5_MPWRQ_LOG_WQE_SZ)
  97                 return false;
  98
  99         if (log_stride_sz < MLX5_MPWQE_LOG_STRIDE_SZ_BASE ||
 100             log_stride_sz > MLX5_MPWQE_LOG_STRIDE_SZ_MAX)
 101                 return false;
 102
 103         if (log_num_strides > MLX5_MPWQE_LOG_NUM_STRIDES_MAX)
 104                 return false;
 105
 106         if (MLX5_CAP_GEN(mdev, ext_stride_num_range))
 107                 return log_num_strides >= MLX5_MPWQE_LOG_NUM_STRIDES_EXT_BASE;
 108
 109         return log_num_strides >= MLX5_MPWQE_LOG_NUM_STRIDES_BASE;
 110 }
 111
 112 bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
 113                                   struct mlx5e_params *params,
 114                                   struct mlx5e_xsk_param *xsk)
 115 {
 116         s8 log_num_strides;
 117         u8 log_stride_sz;
 118
 119         if (!mlx5e_rx_is_linear_skb(params, xsk))
 120                 return false;
 121
 122         log_stride_sz = order_base_2(mlx5e_rx_get_linear_frag_sz(params, xsk));
 123         log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - log_stride_sz;
 124
 125         return mlx5e_verify_rx_mpwqe_strides(mdev, log_stride_sz, log_num_strides);
 126 }
 127
 128 u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params,
 129                                struct mlx5e_xsk_param *xsk)
 130 {
 131         u8 log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(params, xsk);
 132
 133         /* Numbers are unsigned, don't subtract to avoid underflow. */
 134         if (params->log_rq_mtu_frames <
 135             log_pkts_per_wqe + MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW)
 136                 return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW;
 137
 138         return params->log_rq_mtu_frames - log_pkts_per_wqe;
 139 }
 140
 141 u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
 142                                    struct mlx5e_params *params,
 143                                    struct mlx5e_xsk_param *xsk)
 144 {
 145         if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk))
 146                 return order_base_2(mlx5e_rx_get_linear_frag_sz(params, xsk));
 147
 148         return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev);
 149 }
 150
 151 u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
 152                                    struct mlx5e_params *params,
 153                                    struct mlx5e_xsk_param *xsk)
 154 {
 155         return MLX5_MPWRQ_LOG_WQE_SZ -
 156                 mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
 157 }
 158
 159 u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
 160                           struct mlx5e_params *params,
 161                           struct mlx5e_xsk_param *xsk)
 162 {
 163         bool is_linear_skb = (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) ?
 164                 mlx5e_rx_is_linear_skb(params, xsk) :
 165                 mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk);
 166
 167         return is_linear_skb ? mlx5e_get_linear_rq_headroom(params, xsk) : 0;
 168 }
 169
 170 struct mlx5e_lro_param mlx5e_get_lro_param(struct mlx5e_params *params)
 171 {
 172         struct mlx5e_lro_param lro_param;
 173
 174         lro_param = (struct mlx5e_lro_param) {
 175                 .enabled = params->lro_en,
 176                 .timeout = params->packet_merge_timeout,
 177         };
 178
 179         return lro_param;
 180 }
 181
 182 u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
 183 {
 184         bool is_mpwqe = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE);
 185         u16 stop_room;
 186
 187         stop_room  = mlx5e_tls_get_stop_room(mdev, params);
 188         stop_room += mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS);
 189         if (is_mpwqe)
 190                 /* A MPWQE can take up to the maximum-sized WQE + all the normal
 191                  * stop room can be taken if a new packet breaks the active
 192                  * MPWQE session and allocates its WQEs right away.
 193                  */
 194                 stop_room += mlx5e_stop_room_for_wqe(MLX5_SEND_WQE_MAX_WQEBBS);
 195
 196         return stop_room;
 197 }
 198
 199 int mlx5e_validate_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
 200 {
 201         size_t sq_size = 1 << params->log_sq_size;
 202         u16 stop_room;
 203
 204         stop_room = mlx5e_calc_sq_stop_room(mdev, params);
 205         if (stop_room >= sq_size) {
 206                 mlx5_core_err(mdev, "Stop room %u is bigger than the SQ size %zu\n",
 207                               stop_room, sq_size);
 208                 return -EINVAL;
 209         }
 210
 211         return 0;
 212 }
 213
 214 static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode)
 215 {
 216         struct dim_cq_moder moder = {};
 217
 218         moder.cq_period_mode = cq_period_mode;
 219         moder.pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
 220         moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
 221         if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
 222                 moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE;
 223
 224         return moder;
 225 }
 226
 227 static struct dim_cq_moder mlx5e_get_def_rx_moderation(u8 cq_period_mode)
 228 {
 229         struct dim_cq_moder moder = {};
 230
 231         moder.cq_period_mode = cq_period_mode;
 232         moder.pkts = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
 233         moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
 234         if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
 235                 moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
 236
 237         return moder;
 238 }
 239
 240 static u8 mlx5_to_net_dim_cq_period_mode(u8 cq_period_mode)
 241 {
 242         return cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE ?
 243                 DIM_CQ_PERIOD_MODE_START_FROM_CQE :
 244                 DIM_CQ_PERIOD_MODE_START_FROM_EQE;
 245 }
 246
 247 void mlx5e_reset_tx_moderation(struct mlx5e_params *params, u8 cq_period_mode)
 248 {
 249         if (params->tx_dim_enabled) {
 250                 u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode);
 251
 252                 params->tx_cq_moderation = net_dim_get_def_tx_moderation(dim_period_mode);
 253         } else {
 254                 params->tx_cq_moderation = mlx5e_get_def_tx_moderation(cq_period_mode);
 255         }
 256 }
 257
 258 void mlx5e_reset_rx_moderation(struct mlx5e_params *params, u8 cq_period_mode)
 259 {
 260         if (params->rx_dim_enabled) {
 261                 u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode);
 262
 263                 params->rx_cq_moderation = net_dim_get_def_rx_moderation(dim_period_mode);
 264         } else {
 265                 params->rx_cq_moderation = mlx5e_get_def_rx_moderation(cq_period_mode);
 266         }
 267 }
 268
 269 void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
 270 {
 271         mlx5e_reset_tx_moderation(params, cq_period_mode);
 272         MLX5E_SET_PFLAG(params, MLX5E_PFLAG_TX_CQE_BASED_MODER,
 273                         params->tx_cq_moderation.cq_period_mode ==
 274                                 MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
 275 }
 276
 277 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
 278 {
 279         mlx5e_reset_rx_moderation(params, cq_period_mode);
 280         MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_BASED_MODER,
 281                         params->rx_cq_moderation.cq_period_mode ==
 282                                 MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
 283 }
 284
 285 bool slow_pci_heuristic(struct mlx5_core_dev *mdev)
 286 {
 287         u32 link_speed = 0;
 288         u32 pci_bw = 0;
 289
 290         mlx5e_port_max_linkspeed(mdev, &link_speed);
 291         pci_bw = pcie_bandwidth_available(mdev->pdev, NULL, NULL, NULL);
 292         mlx5_core_dbg_once(mdev, "Max link speed = %d, PCI BW = %d\n",
 293                            link_speed, pci_bw);
 294
 295 #define MLX5E_SLOW_PCI_RATIO (2)
 296
 297         return link_speed && pci_bw &&
 298                 link_speed > MLX5E_SLOW_PCI_RATIO * pci_bw;
 299 }
 300
 301 bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev,
 302                                 struct mlx5e_params *params)
 303 {
 304         if (!mlx5e_check_fragmented_striding_rq_cap(mdev))
 305                 return false;
 306
 307         if (mlx5_fpga_is_ipsec_device(mdev))
 308                 return false;
 309
 310         if (params->xdp_prog) {
 311                 /* XSK params are not considered here. If striding RQ is in use,
 312                  * and an XSK is being opened, mlx5e_rx_mpwqe_is_linear_skb will
 313                  * be called with the known XSK params.
 314                  */
 315                 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL))
 316                         return false;
 317         }
 318
 319         return true;
 320 }
 321
 322 void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,
 323                                struct mlx5e_params *params)
 324 {
 325         params->log_rq_mtu_frames = is_kdump_kernel() ?
 326                 MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE :
 327                 MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
 328
 329         mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
 330                        params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
 331                        params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ?
 332                        BIT(mlx5e_mpwqe_get_log_rq_size(params, NULL)) :
 333                        BIT(params->log_rq_mtu_frames),
 334                        BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params, NULL)),
 335                        MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS));
 336 }
 337
 338 void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
 339 {
 340         params->rq_wq_type = mlx5e_striding_rq_possible(mdev, params) &&
 341                 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) ?
 342                 MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
 343                 MLX5_WQ_TYPE_CYCLIC;
 344 }
 345
 346 void mlx5e_build_rq_params(struct mlx5_core_dev *mdev,
 347                            struct mlx5e_params *params)
 348 {
 349         /* Prefer Striding RQ, unless any of the following holds:
 350          * - Striding RQ configuration is not possible/supported.
 351          * - Slow PCI heuristic.
 352          * - Legacy RQ would use linear SKB while Striding RQ would use non-linear.
 353          *
 354          * No XSK params: checking the availability of striding RQ in general.
 355          */
 356         if (!slow_pci_heuristic(mdev) &&
 357             mlx5e_striding_rq_possible(mdev, params) &&
 358             (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) ||
 359              !mlx5e_rx_is_linear_skb(params, NULL)))
 360                 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true);
 361         mlx5e_set_rq_type(mdev, params);
 362         mlx5e_init_rq_type_params(mdev, params);
 363 }
 364
 365 /* Build queue parameters */
 366
 367 void mlx5e_build_create_cq_param(struct mlx5e_create_cq_param *ccp, struct mlx5e_channel *c)
 368 {
 369         *ccp = (struct mlx5e_create_cq_param) {
 370                 .napi = &c->napi,
 371                 .ch_stats = c->stats,
 372                 .node = cpu_to_node(c->cpu),
 373                 .ix = c->ix,
 374         };
 375 }
 376
 377 #define DEFAULT_FRAG_SIZE (2048)
 378
 379 static void mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
 380                                       struct mlx5e_params *params,
 381                                       struct mlx5e_xsk_param *xsk,
 382                                       struct mlx5e_rq_frags_info *info)
 383 {
 384         u32 byte_count = MLX5E_SW2HW_MTU(params, params->sw_mtu);
 385         int frag_size_max = DEFAULT_FRAG_SIZE;
 386         u32 buf_size = 0;
 387         int i;
 388
 389         if (mlx5_fpga_is_ipsec_device(mdev))
 390                 byte_count += MLX5E_METADATA_ETHER_LEN;
 391
 392         if (mlx5e_rx_is_linear_skb(params, xsk)) {
 393                 int frag_stride;
 394
 395                 frag_stride = mlx5e_rx_get_linear_frag_sz(params, xsk);
 396                 frag_stride = roundup_pow_of_two(frag_stride);
 397
 398                 info->arr[0].frag_size = byte_count;
 399                 info->arr[0].frag_stride = frag_stride;
 400                 info->num_frags = 1;
 401                 info->wqe_bulk = PAGE_SIZE / frag_stride;
 402                 goto out;
 403         }
 404
 405         if (byte_count > PAGE_SIZE +
 406             (MLX5E_MAX_RX_FRAGS - 1) * frag_size_max)
 407                 frag_size_max = PAGE_SIZE;
 408
 409         i = 0;
 410         while (buf_size < byte_count) {
 411                 int frag_size = byte_count - buf_size;
 412
 413                 if (i < MLX5E_MAX_RX_FRAGS - 1)
 414                         frag_size = min(frag_size, frag_size_max);
 415
 416                 info->arr[i].frag_size = frag_size;
 417                 info->arr[i].frag_stride = roundup_pow_of_two(frag_size);
 418
 419                 buf_size += frag_size;
 420                 i++;
 421         }
 422         info->num_frags = i;
 423         /* number of different wqes sharing a page */
 424         info->wqe_bulk = 1 + (info->num_frags % 2);
 425
 426 out:
 427         info->wqe_bulk = max_t(u8, info->wqe_bulk, 8);
 428         info->log_num_frags = order_base_2(info->num_frags);
 429 }
 430
 431 static u8 mlx5e_get_rqwq_log_stride(u8 wq_type, int ndsegs)
 432 {
 433         int sz = sizeof(struct mlx5_wqe_data_seg) * ndsegs;
 434
 435         switch (wq_type) {
 436         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 437                 sz += sizeof(struct mlx5e_rx_wqe_ll);
 438                 break;
 439         default: /* MLX5_WQ_TYPE_CYCLIC */
 440                 sz += sizeof(struct mlx5e_rx_wqe_cyc);
 441         }
 442
 443         return order_base_2(sz);
 444 }
 445
 446 static void mlx5e_build_common_cq_param(struct mlx5_core_dev *mdev,
 447                                         struct mlx5e_cq_param *param)
 448 {
 449         void *cqc = param->cqc;
 450
 451         MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index);
 452         if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128)
 453                 MLX5_SET(cqc, cqc, cqe_sz, CQE_STRIDE_128_PAD);
 454 }
 455
 456 static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev,
 457                                     struct mlx5e_params *params,
 458                                     struct mlx5e_xsk_param *xsk,
 459                                     struct mlx5e_cq_param *param)
 460 {
 461         bool hw_stridx = false;
 462         void *cqc = param->cqc;
 463         u8 log_cq_size;
 464
 465         switch (params->rq_wq_type) {
 466         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 467                 log_cq_size = mlx5e_mpwqe_get_log_rq_size(params, xsk) +
 468                         mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
 469                 hw_stridx = MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index);
 470                 break;
 471         default: /* MLX5_WQ_TYPE_CYCLIC */
 472                 log_cq_size = params->log_rq_mtu_frames;
 473         }
 474
 475         MLX5_SET(cqc, cqc, log_cq_size, log_cq_size);
 476         if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) {
 477                 MLX5_SET(cqc, cqc, mini_cqe_res_format, hw_stridx ?
 478                          MLX5_CQE_FORMAT_CSUM_STRIDX : MLX5_CQE_FORMAT_CSUM);
 479                 MLX5_SET(cqc, cqc, cqe_comp_en, 1);
 480         }
 481
 482         mlx5e_build_common_cq_param(mdev, param);
 483         param->cq_period_mode = params->rx_cq_moderation.cq_period_mode;
 484 }
 485
 486 static u8 rq_end_pad_mode(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
 487 {
 488         bool ro = pcie_relaxed_ordering_enabled(mdev->pdev) &&
 489                 MLX5_CAP_GEN(mdev, relaxed_ordering_write);
 490
 491         return ro && params->lro_en ?
 492                 MLX5_WQ_END_PAD_MODE_NONE : MLX5_WQ_END_PAD_MODE_ALIGN;
 493 }
 494
 495 int mlx5e_build_rq_param(struct mlx5_core_dev *mdev,
 496                          struct mlx5e_params *params,
 497                          struct mlx5e_xsk_param *xsk,
 498                          u16 q_counter,
 499                          struct mlx5e_rq_param *param)
 500 {
 501         void *rqc = param->rqc;
 502         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
 503         int ndsegs = 1;
 504
 505         switch (params->rq_wq_type) {
 506         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: {
 507                 u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
 508                 u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
 509
 510                 if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size,
 511                                                    log_wqe_num_of_strides)) {
 512                         mlx5_core_err(mdev,
 513                                       "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u\n",
 514                                       log_wqe_stride_size, log_wqe_num_of_strides);
 515                         return -EINVAL;
 516                 }
 517
 518                 MLX5_SET(wq, wq, log_wqe_num_of_strides,
 519                          log_wqe_num_of_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE);
 520                 MLX5_SET(wq, wq, log_wqe_stride_size,
 521                          log_wqe_stride_size - MLX5_MPWQE_LOG_STRIDE_SZ_BASE);
 522                 MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(params, xsk));
 523                 break;
 524         }
 525         default: /* MLX5_WQ_TYPE_CYCLIC */
 526                 MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames);
 527                 mlx5e_build_rq_frags_info(mdev, params, xsk, &param->frags_info);
 528                 ndsegs = param->frags_info.num_frags;
 529         }
 530
 531         MLX5_SET(wq, wq, wq_type,          params->rq_wq_type);
 532         MLX5_SET(wq, wq, end_padding_mode, rq_end_pad_mode(mdev, params));
 533         MLX5_SET(wq, wq, log_wq_stride,
 534                  mlx5e_get_rqwq_log_stride(params->rq_wq_type, ndsegs));
 535         MLX5_SET(wq, wq, pd,               mdev->mlx5e_res.hw_objs.pdn);
 536         MLX5_SET(rqc, rqc, counter_set_id, q_counter);
 537         MLX5_SET(rqc, rqc, vsd,            params->vlan_strip_disable);
 538         MLX5_SET(rqc, rqc, scatter_fcs,    params->scatter_fcs_en);
 539
 540         param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
 541         mlx5e_build_rx_cq_param(mdev, params, xsk, &param->cqp);
 542
 543         return 0;
 544 }
 545
 546 void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev,
 547                                u16 q_counter,
 548                                struct mlx5e_rq_param *param)
 549 {
 550         void *rqc = param->rqc;
 551         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
 552
 553         MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
 554         MLX5_SET(wq, wq, log_wq_stride,
 555                  mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1));
 556         MLX5_SET(rqc, rqc, counter_set_id, q_counter);
 557
 558         param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
 559 }
 560
 561 void mlx5e_build_tx_cq_param(struct mlx5_core_dev *mdev,
 562                              struct mlx5e_params *params,
 563                              struct mlx5e_cq_param *param)
 564 {
 565         void *cqc = param->cqc;
 566
 567         MLX5_SET(cqc, cqc, log_cq_size, params->log_sq_size);
 568
 569         mlx5e_build_common_cq_param(mdev, param);
 570         param->cq_period_mode = params->tx_cq_moderation.cq_period_mode;
 571 }
 572
 573 void mlx5e_build_sq_param_common(struct mlx5_core_dev *mdev,
 574                                  struct mlx5e_sq_param *param)
 575 {
 576         void *sqc = param->sqc;
 577         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
 578
 579         MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
 580         MLX5_SET(wq, wq, pd,            mdev->mlx5e_res.hw_objs.pdn);
 581
 582         param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
 583 }
 584
 585 void mlx5e_build_sq_param(struct mlx5_core_dev *mdev,
 586                           struct mlx5e_params *params,
 587                           struct mlx5e_sq_param *param)
 588 {
 589         void *sqc = param->sqc;
 590         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
 591         bool allow_swp;
 592
 593         allow_swp = mlx5_geneve_tx_allowed(mdev) ||
 594                     !!MLX5_IPSEC_DEV(mdev);
 595         mlx5e_build_sq_param_common(mdev, param);
 596         MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
 597         MLX5_SET(sqc, sqc, allow_swp, allow_swp);
 598         param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE);
 599         param->stop_room = mlx5e_calc_sq_stop_room(mdev, params);
 600         mlx5e_build_tx_cq_param(mdev, params, &param->cqp);
 601 }
 602
 603 static void mlx5e_build_ico_cq_param(struct mlx5_core_dev *mdev,
 604                                      u8 log_wq_size,
 605                                      struct mlx5e_cq_param *param)
 606 {
 607         void *cqc = param->cqc;
 608
 609         MLX5_SET(cqc, cqc, log_cq_size, log_wq_size);
 610
 611         mlx5e_build_common_cq_param(mdev, param);
 612
 613         param->cq_period_mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
 614 }
 615
 616 static u8 mlx5e_get_rq_log_wq_sz(void *rqc)
 617 {
 618         void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
 619
 620         return MLX5_GET(wq, wq, log_wq_sz);
 621 }
 622
 623 static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5e_params *params,
 624                                       struct mlx5e_rq_param *rqp)
 625 {
 626         switch (params->rq_wq_type) {
 627         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 628                 return max_t(u8, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE,
 629                              order_base_2(MLX5E_UMR_WQEBBS) +
 630                              mlx5e_get_rq_log_wq_sz(rqp->rqc));
 631         default: /* MLX5_WQ_TYPE_CYCLIC */
 632                 return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
 633         }
 634 }
 635
 636 static u8 mlx5e_build_async_icosq_log_wq_sz(struct mlx5_core_dev *mdev)
 637 {
 638         if (mlx5e_accel_is_ktls_rx(mdev))
 639                 return MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
 640
 641         return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
 642 }
 643
 644 static void mlx5e_build_icosq_param(struct mlx5_core_dev *mdev,
 645                                     u8 log_wq_size,
 646                                     struct mlx5e_sq_param *param)
 647 {
 648         void *sqc = param->sqc;
 649         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
 650
 651         mlx5e_build_sq_param_common(mdev, param);
 652
 653         MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
 654         MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(mdev, reg_umr_sq));
 655         mlx5e_build_ico_cq_param(mdev, log_wq_size, &param->cqp);
 656 }
 657
 658 static void mlx5e_build_async_icosq_param(struct mlx5_core_dev *mdev,
 659                                           u8 log_wq_size,
 660                                           struct mlx5e_sq_param *param)
 661 {
 662         void *sqc = param->sqc;
 663         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
 664
 665         mlx5e_build_sq_param_common(mdev, param);
 666         param->stop_room = mlx5e_stop_room_for_wqe(1); /* for XSK NOP */
 667         param->is_tls = mlx5e_accel_is_ktls_rx(mdev);
 668         if (param->is_tls)
 669                 param->stop_room += mlx5e_stop_room_for_wqe(1); /* for TLS RX resync NOP */
 670         MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(mdev, reg_umr_sq));
 671         MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
 672         mlx5e_build_ico_cq_param(mdev, log_wq_size, &param->cqp);
 673 }
 674
 675 void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev,
 676                              struct mlx5e_params *params,
 677                              struct mlx5e_sq_param *param)
 678 {
 679         void *sqc = param->sqc;
 680         void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
 681
 682         mlx5e_build_sq_param_common(mdev, param);
 683         MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
 684         param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE);
 685         mlx5e_build_tx_cq_param(mdev, params, &param->cqp);
 686 }
 687
 688 int mlx5e_build_channel_param(struct mlx5_core_dev *mdev,
 689                               struct mlx5e_params *params,
 690                               u16 q_counter,
 691                               struct mlx5e_channel_param *cparam)
 692 {
 693         u8 icosq_log_wq_sz, async_icosq_log_wq_sz;
 694         int err;
 695
 696         err = mlx5e_build_rq_param(mdev, params, NULL, q_counter, &cparam->rq);
 697         if (err)
 698                 return err;
 699
 700         icosq_log_wq_sz = mlx5e_build_icosq_log_wq_sz(params, &cparam->rq);
 701         async_icosq_log_wq_sz = mlx5e_build_async_icosq_log_wq_sz(mdev);
 702
 703         mlx5e_build_sq_param(mdev, params, &cparam->txq_sq);
 704         mlx5e_build_xdpsq_param(mdev, params, &cparam->xdp_sq);
 705         mlx5e_build_icosq_param(mdev, icosq_log_wq_sz, &cparam->icosq);
 706         mlx5e_build_async_icosq_param(mdev, async_icosq_log_wq_sz, &cparam->async_icosq);
 707
 708         return 0;
 709 }