drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h

   1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
   2 /* Copyright (c) 2019 Mellanox Technologies. */
   3
   4 #ifndef __MLX5_EN_TXRX_H___
   5 #define __MLX5_EN_TXRX_H___
   6
   7 #include "en.h"
   8 #include <linux/indirect_call_wrapper.h>
   9
  10 #define MLX5E_TX_WQE_EMPTY_DS_COUNT (sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS)
  11
  12 /* The mult of MLX5_SEND_WQE_MAX_WQEBBS * MLX5_SEND_WQEBB_NUM_DS
  13  * (16 * 4 == 64) does not fit in the 6-bit DS field of Ctrl Segment.
  14  * We use a bound lower that MLX5_SEND_WQE_MAX_WQEBBS to let a
  15  * full-session WQE be cache-aligned.
  16  */
  17 #if L1_CACHE_BYTES < 128
  18 #define MLX5E_TX_MPW_MAX_WQEBBS (MLX5_SEND_WQE_MAX_WQEBBS - 1)
  19 #else
  20 #define MLX5E_TX_MPW_MAX_WQEBBS (MLX5_SEND_WQE_MAX_WQEBBS - 2)
  21 #endif
  22
  23 #define MLX5E_TX_MPW_MAX_NUM_DS (MLX5E_TX_MPW_MAX_WQEBBS * MLX5_SEND_WQEBB_NUM_DS)
  24
  25 #define INL_HDR_START_SZ (sizeof(((struct mlx5_wqe_eth_seg *)NULL)->inline_hdr.start))
  26
  27 #define MLX5E_RX_ERR_CQE(cqe) (get_cqe_opcode(cqe) != MLX5_CQE_RESP_SEND)
  28
  29 enum mlx5e_icosq_wqe_type {
  30         MLX5E_ICOSQ_WQE_NOP,
  31         MLX5E_ICOSQ_WQE_UMR_RX,
  32 #ifdef CONFIG_MLX5_EN_TLS
  33         MLX5E_ICOSQ_WQE_UMR_TLS,
  34         MLX5E_ICOSQ_WQE_SET_PSV_TLS,
  35         MLX5E_ICOSQ_WQE_GET_PSV_TLS,
  36 #endif
  37 };
  38
  39 /* General */
  40 static inline bool mlx5e_skb_is_multicast(struct sk_buff *skb)
  41 {
  42         return skb->pkt_type == PACKET_MULTICAST || skb->pkt_type == PACKET_BROADCAST;
  43 }
  44
  45 void mlx5e_trigger_irq(struct mlx5e_icosq *sq);
  46 void mlx5e_completion_event(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe);
  47 void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event);
  48 int mlx5e_napi_poll(struct napi_struct *napi, int budget);
  49 int mlx5e_poll_ico_cq(struct mlx5e_cq *cq);
  50
  51 /* RX */
  52 void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info);
  53 void mlx5e_page_release_dynamic(struct mlx5e_rq *rq,
  54                                 struct mlx5e_dma_info *dma_info,
  55                                 bool recycle);
  56 INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq));
  57 INDIRECT_CALLABLE_DECLARE(bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq));
  58 int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget);
  59 void mlx5e_free_rx_descs(struct mlx5e_rq *rq);
  60 void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq *rq);
  61
  62 /* TX */
  63 u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
  64                        struct net_device *sb_dev);
  65 netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev);
  66 bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget);
  67 void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq);
  68
  69 static inline bool
  70 mlx5e_wqc_has_room_for(struct mlx5_wq_cyc *wq, u16 cc, u16 pc, u16 n)
  71 {
  72         return (mlx5_wq_cyc_ctr2ix(wq, cc - pc) >= n) || (cc == pc);
  73 }
  74
  75 static inline void *mlx5e_fetch_wqe(struct mlx5_wq_cyc *wq, u16 pi, size_t wqe_size)
  76 {
  77         void *wqe;
  78
  79         wqe = mlx5_wq_cyc_get_wqe(wq, pi);
  80         memset(wqe, 0, wqe_size);
  81
  82         return wqe;
  83 }
  84
  85 #define MLX5E_TX_FETCH_WQE(sq, pi) \
  86         ((struct mlx5e_tx_wqe *)mlx5e_fetch_wqe(&(sq)->wq, pi, sizeof(struct mlx5e_tx_wqe)))
  87
  88 static inline struct mlx5e_tx_wqe *
  89 mlx5e_post_nop(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc)
  90 {
  91         u16                         pi   = mlx5_wq_cyc_ctr2ix(wq, *pc);
  92         struct mlx5e_tx_wqe        *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);
  93         struct mlx5_wqe_ctrl_seg   *cseg = &wqe->ctrl;
  94
  95         memset(cseg, 0, sizeof(*cseg));
  96
  97         cseg->opmod_idx_opcode = cpu_to_be32((*pc << 8) | MLX5_OPCODE_NOP);
  98         cseg->qpn_ds           = cpu_to_be32((sqn << 8) | 0x01);
  99
 100         (*pc)++;
 101
 102         return wqe;
 103 }
 104
 105 static inline struct mlx5e_tx_wqe *
 106 mlx5e_post_nop_fence(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc)
 107 {
 108         u16                         pi   = mlx5_wq_cyc_ctr2ix(wq, *pc);
 109         struct mlx5e_tx_wqe        *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);
 110         struct mlx5_wqe_ctrl_seg   *cseg = &wqe->ctrl;
 111
 112         memset(cseg, 0, sizeof(*cseg));
 113
 114         cseg->opmod_idx_opcode = cpu_to_be32((*pc << 8) | MLX5_OPCODE_NOP);
 115         cseg->qpn_ds           = cpu_to_be32((sqn << 8) | 0x01);
 116         cseg->fm_ce_se         = MLX5_FENCE_MODE_INITIATOR_SMALL;
 117
 118         (*pc)++;
 119
 120         return wqe;
 121 }
 122
 123 struct mlx5e_tx_wqe_info {
 124         struct sk_buff *skb;
 125         u32 num_bytes;
 126         u8 num_wqebbs;
 127         u8 num_dma;
 128         u8 num_fifo_pkts;
 129 #ifdef CONFIG_MLX5_EN_TLS
 130         struct page *resync_dump_frag_page;
 131 #endif
 132 };
 133
 134 static inline u16 mlx5e_txqsq_get_next_pi(struct mlx5e_txqsq *sq, u16 size)
 135 {
 136         struct mlx5_wq_cyc *wq = &sq->wq;
 137         u16 pi, contig_wqebbs;
 138
 139         pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
 140         contig_wqebbs = mlx5_wq_cyc_get_contig_wqebbs(wq, pi);
 141         if (unlikely(contig_wqebbs < size)) {
 142                 struct mlx5e_tx_wqe_info *wi, *edge_wi;
 143
 144                 wi = &sq->db.wqe_info[pi];
 145                 edge_wi = wi + contig_wqebbs;
 146
 147                 /* Fill SQ frag edge with NOPs to avoid WQE wrapping two pages. */
 148                 for (; wi < edge_wi; wi++) {
 149                         *wi = (struct mlx5e_tx_wqe_info) {
 150                                 .num_wqebbs = 1,
 151                         };
 152                         mlx5e_post_nop(wq, sq->sqn, &sq->pc);
 153                 }
 154                 sq->stats->nop += contig_wqebbs;
 155
 156                 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
 157         }
 158
 159         return pi;
 160 }
 161
 162 struct mlx5e_icosq_wqe_info {
 163         u8 wqe_type;
 164         u8 num_wqebbs;
 165
 166         /* Auxiliary data for different wqe types. */
 167         union {
 168                 struct {
 169                         struct mlx5e_rq *rq;
 170                 } umr;
 171 #ifdef CONFIG_MLX5_EN_TLS
 172                 struct {
 173                         struct mlx5e_ktls_offload_context_rx *priv_rx;
 174                 } tls_set_params;
 175                 struct {
 176                         struct mlx5e_ktls_rx_resync_buf *buf;
 177                 } tls_get_params;
 178 #endif
 179         };
 180 };
 181
 182 void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq);
 183
 184 static inline u16 mlx5e_icosq_get_next_pi(struct mlx5e_icosq *sq, u16 size)
 185 {
 186         struct mlx5_wq_cyc *wq = &sq->wq;
 187         u16 pi, contig_wqebbs;
 188
 189         pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
 190         contig_wqebbs = mlx5_wq_cyc_get_contig_wqebbs(wq, pi);
 191         if (unlikely(contig_wqebbs < size)) {
 192                 struct mlx5e_icosq_wqe_info *wi, *edge_wi;
 193
 194                 wi = &sq->db.wqe_info[pi];
 195                 edge_wi = wi + contig_wqebbs;
 196
 197                 /* Fill SQ frag edge with NOPs to avoid WQE wrapping two pages. */
 198                 for (; wi < edge_wi; wi++) {
 199                         *wi = (struct mlx5e_icosq_wqe_info) {
 200                                 .wqe_type   = MLX5E_ICOSQ_WQE_NOP,
 201                                 .num_wqebbs = 1,
 202                         };
 203                         mlx5e_post_nop(wq, sq->sqn, &sq->pc);
 204                 }
 205
 206                 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
 207         }
 208
 209         return pi;
 210 }
 211
 212 static inline void
 213 mlx5e_notify_hw(struct mlx5_wq_cyc *wq, u16 pc, void __iomem *uar_map,
 214                 struct mlx5_wqe_ctrl_seg *ctrl)
 215 {
 216         ctrl->fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE;
 217         /* ensure wqe is visible to device before updating doorbell record */
 218         dma_wmb();
 219
 220         *wq->db = cpu_to_be32(pc);
 221
 222         /* ensure doorbell record is visible to device before ringing the
 223          * doorbell
 224          */
 225         wmb();
 226
 227         mlx5_write64((__be32 *)ctrl, uar_map);
 228 }
 229
 230 static inline void mlx5e_cq_arm(struct mlx5e_cq *cq)
 231 {
 232         struct mlx5_core_cq *mcq;
 233
 234         mcq = &cq->mcq;
 235         mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, cq->wq.cc);
 236 }
 237
 238 static inline struct mlx5e_sq_dma *
 239 mlx5e_dma_get(struct mlx5e_txqsq *sq, u32 i)
 240 {
 241         return &sq->db.dma_fifo[i & sq->dma_fifo_mask];
 242 }
 243
 244 static inline void
 245 mlx5e_dma_push(struct mlx5e_txqsq *sq, dma_addr_t addr, u32 size,
 246                enum mlx5e_dma_map_type map_type)
 247 {
 248         struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, sq->dma_fifo_pc++);
 249
 250         dma->addr = addr;
 251         dma->size = size;
 252         dma->type = map_type;
 253 }
 254
 255 static inline
 256 struct sk_buff **mlx5e_skb_fifo_get(struct mlx5e_skb_fifo *fifo, u16 i)
 257 {
 258         return &fifo->fifo[i & fifo->mask];
 259 }
 260
 261 static inline
 262 void mlx5e_skb_fifo_push(struct mlx5e_skb_fifo *fifo, struct sk_buff *skb)
 263 {
 264         struct sk_buff **skb_item = mlx5e_skb_fifo_get(fifo, (*fifo->pc)++);
 265
 266         *skb_item = skb;
 267 }
 268
 269 static inline
 270 struct sk_buff *mlx5e_skb_fifo_pop(struct mlx5e_skb_fifo *fifo)
 271 {
 272         return *mlx5e_skb_fifo_get(fifo, (*fifo->cc)++);
 273 }
 274
 275 static inline void
 276 mlx5e_tx_dma_unmap(struct device *pdev, struct mlx5e_sq_dma *dma)
 277 {
 278         switch (dma->type) {
 279         case MLX5E_DMA_MAP_SINGLE:
 280                 dma_unmap_single(pdev, dma->addr, dma->size, DMA_TO_DEVICE);
 281                 break;
 282         case MLX5E_DMA_MAP_PAGE:
 283                 dma_unmap_page(pdev, dma->addr, dma->size, DMA_TO_DEVICE);
 284                 break;
 285         default:
 286                 WARN_ONCE(true, "mlx5e_tx_dma_unmap unknown DMA type!\n");
 287         }
 288 }
 289
 290 void mlx5e_sq_xmit_simple(struct mlx5e_txqsq *sq, struct sk_buff *skb, bool xmit_more);
 291 void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq);
 292
 293 static inline bool mlx5e_tx_mpwqe_is_full(struct mlx5e_tx_mpwqe *session)
 294 {
 295         return session->ds_count == MLX5E_TX_MPW_MAX_NUM_DS;
 296 }
 297
 298 static inline void mlx5e_rqwq_reset(struct mlx5e_rq *rq)
 299 {
 300         if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
 301                 mlx5_wq_ll_reset(&rq->mpwqe.wq);
 302                 rq->mpwqe.actual_wq_head = 0;
 303         } else {
 304                 mlx5_wq_cyc_reset(&rq->wqe.wq);
 305         }
 306 }
 307
 308 static inline void mlx5e_dump_error_cqe(struct mlx5e_cq *cq, u32 qn,
 309                                         struct mlx5_err_cqe *err_cqe)
 310 {
 311         struct mlx5_cqwq *wq = &cq->wq;
 312         u32 ci;
 313
 314         ci = mlx5_cqwq_ctr2ix(wq, wq->cc - 1);
 315
 316         netdev_err(cq->netdev,
 317                    "Error cqe on cqn 0x%x, ci 0x%x, qn 0x%x, opcode 0x%x, syndrome 0x%x, vendor syndrome 0x%x\n",
 318                    cq->mcq.cqn, ci, qn,
 319                    get_cqe_opcode((struct mlx5_cqe64 *)err_cqe),
 320                    err_cqe->syndrome, err_cqe->vendor_err_synd);
 321         mlx5_dump_err_cqe(cq->mdev, err_cqe);
 322 }
 323
 324 static inline u32 mlx5e_rqwq_get_size(struct mlx5e_rq *rq)
 325 {
 326         switch (rq->wq_type) {
 327         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 328                 return mlx5_wq_ll_get_size(&rq->mpwqe.wq);
 329         default:
 330                 return mlx5_wq_cyc_get_size(&rq->wqe.wq);
 331         }
 332 }
 333
 334 static inline u32 mlx5e_rqwq_get_cur_sz(struct mlx5e_rq *rq)
 335 {
 336         switch (rq->wq_type) {
 337         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 338                 return rq->mpwqe.wq.cur_sz;
 339         default:
 340                 return rq->wqe.wq.cur_sz;
 341         }
 342 }
 343
 344 static inline u16 mlx5e_rqwq_get_head(struct mlx5e_rq *rq)
 345 {
 346         switch (rq->wq_type) {
 347         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 348                 return mlx5_wq_ll_get_head(&rq->mpwqe.wq);
 349         default:
 350                 return mlx5_wq_cyc_get_head(&rq->wqe.wq);
 351         }
 352 }
 353
 354 static inline u16 mlx5e_rqwq_get_wqe_counter(struct mlx5e_rq *rq)
 355 {
 356         switch (rq->wq_type) {
 357         case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 358                 return mlx5_wq_ll_get_counter(&rq->mpwqe.wq);
 359         default:
 360                 return mlx5_wq_cyc_get_counter(&rq->wqe.wq);
 361         }
 362 }
 363
 364 /* SW parser related functions */
 365
 366 struct mlx5e_swp_spec {
 367         __be16 l3_proto;
 368         u8 l4_proto;
 369         u8 is_tun;
 370         __be16 tun_l3_proto;
 371         u8 tun_l4_proto;
 372 };
 373
 374 static inline void
 375 mlx5e_set_eseg_swp(struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg,
 376                    struct mlx5e_swp_spec *swp_spec)
 377 {
 378         /* SWP offsets are in 2-bytes words */
 379         eseg->swp_outer_l3_offset = skb_network_offset(skb) / 2;
 380         if (swp_spec->l3_proto == htons(ETH_P_IPV6))
 381                 eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L3_IPV6;
 382         if (swp_spec->l4_proto) {
 383                 eseg->swp_outer_l4_offset = skb_transport_offset(skb) / 2;
 384                 if (swp_spec->l4_proto == IPPROTO_UDP)
 385                         eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L4_UDP;
 386         }
 387
 388         if (swp_spec->is_tun) {
 389                 eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2;
 390                 if (swp_spec->tun_l3_proto == htons(ETH_P_IPV6))
 391                         eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6;
 392         } else { /* typically for ipsec when xfrm mode != XFRM_MODE_TUNNEL */
 393                 eseg->swp_inner_l3_offset = skb_network_offset(skb) / 2;
 394                 if (swp_spec->l3_proto == htons(ETH_P_IPV6))
 395                         eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6;
 396         }
 397         switch (swp_spec->tun_l4_proto) {
 398         case IPPROTO_UDP:
 399                 eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP;
 400                 fallthrough;
 401         case IPPROTO_TCP:
 402                 eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2;
 403                 break;
 404         }
 405 }
 406
 407 static inline u16 mlx5e_stop_room_for_wqe(u16 wqe_size)
 408 {
 409         BUILD_BUG_ON(PAGE_SIZE / MLX5_SEND_WQE_BB < MLX5_SEND_WQE_MAX_WQEBBS);
 410
 411         /* A WQE must not cross the page boundary, hence two conditions:
 412          * 1. Its size must not exceed the page size.
 413          * 2. If the WQE size is X, and the space remaining in a page is less
 414          *    than X, this space needs to be padded with NOPs. So, one WQE of
 415          *    size X may require up to X-1 WQEBBs of padding, which makes the
 416          *    stop room of X-1 + X.
 417          * WQE size is also limited by the hardware limit.
 418          */
 419
 420         if (__builtin_constant_p(wqe_size))
 421                 BUILD_BUG_ON(wqe_size > MLX5_SEND_WQE_MAX_WQEBBS);
 422         else
 423                 WARN_ON_ONCE(wqe_size > MLX5_SEND_WQE_MAX_WQEBBS);
 424
 425         return wqe_size * 2 - 1;
 426 }
 427
 428 #endif