ceph/src/spdk/dpdk/drivers/net/ark/ark_ethdev_tx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2015-2018 Atomic Rules LLC
   3  */
   4
   5 #include <unistd.h>
   6
   7 #include "ark_ethdev_tx.h"
   8 #include "ark_global.h"
   9 #include "ark_mpu.h"
  10 #include "ark_ddm.h"
  11 #include "ark_logs.h"
  12
  13 #define ARK_TX_META_SIZE   32
  14 #define ARK_TX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_TX_META_SIZE)
  15 #define ARK_TX_MAX_NOCHAIN (RTE_MBUF_DEFAULT_DATAROOM)
  16
  17
  18 /* ************************************************************************* */
  19 struct ark_tx_queue {
  20         struct ark_tx_meta *meta_q;
  21         struct rte_mbuf **bufs;
  22
  23         /* handles for hw objects */
  24         struct ark_mpu_t *mpu;
  25         struct ark_ddm_t *ddm;
  26
  27         /* Stats HW tracks bytes and packets, need to count send errors */
  28         uint64_t tx_errors;
  29
  30         uint32_t queue_size;
  31         uint32_t queue_mask;
  32
  33         /* 3 indexes to the paired data rings. */
  34         uint32_t prod_index;            /* where to put the next one */
  35         uint32_t free_index;            /* mbuf has been freed */
  36
  37         /* The queue Id is used to identify the HW Q */
  38         uint16_t phys_qid;
  39         /* The queue Index within the dpdk device structures */
  40         uint16_t queue_index;
  41
  42         uint32_t pad[1];
  43
  44         /* second cache line - fields only used in slow path */
  45         MARKER cacheline1 __rte_cache_min_aligned;
  46         uint32_t cons_index;            /* hw is done, can be freed */
  47 } __rte_cache_aligned;
  48
  49 /* Forward declarations */
  50 static uint32_t eth_ark_tx_jumbo(struct ark_tx_queue *queue,
  51                                  struct rte_mbuf *mbuf);
  52 static int eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue);
  53 static void free_completed_tx(struct ark_tx_queue *queue);
  54
  55 static inline void
  56 ark_tx_hw_queue_stop(struct ark_tx_queue *queue)
  57 {
  58         ark_mpu_stop(queue->mpu);
  59 }
  60
  61 /* ************************************************************************* */
  62 static inline void
  63 eth_ark_tx_meta_from_mbuf(struct ark_tx_meta *meta,
  64                           const struct rte_mbuf *mbuf,
  65                           uint8_t flags)
  66 {
  67         meta->physaddr = rte_mbuf_data_iova(mbuf);
  68         meta->user1 = (uint32_t)mbuf->udata64;
  69         meta->data_len = rte_pktmbuf_data_len(mbuf);
  70         meta->flags = flags;
  71 }
  72
  73 /* ************************************************************************* */
  74 uint16_t
  75 eth_ark_xmit_pkts_noop(void *vtxq __rte_unused,
  76                        struct rte_mbuf **tx_pkts __rte_unused,
  77                        uint16_t nb_pkts __rte_unused)
  78 {
  79         return 0;
  80 }
  81
  82 /* ************************************************************************* */
  83 uint16_t
  84 eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
  85 {
  86         struct ark_tx_queue *queue;
  87         struct rte_mbuf *mbuf;
  88         struct ark_tx_meta *meta;
  89
  90         uint32_t idx;
  91         uint32_t prod_index_limit;
  92         int stat;
  93         uint16_t nb;
  94
  95         queue = (struct ark_tx_queue *)vtxq;
  96
  97         /* free any packets after the HW is done with them */
  98         free_completed_tx(queue);
  99
 100         prod_index_limit = queue->queue_size + queue->free_index;
 101
 102         for (nb = 0;
 103              (nb < nb_pkts) && (queue->prod_index != prod_index_limit);
 104              ++nb) {
 105                 mbuf = tx_pkts[nb];
 106
 107                 if (ARK_TX_PAD_TO_60) {
 108                         if (unlikely(rte_pktmbuf_pkt_len(mbuf) < 60)) {
 109                                 /* this packet even if it is small can be split,
 110                                  * be sure to add to the end mbuf
 111                                  */
 112                                 uint16_t to_add =
 113                                         60 - rte_pktmbuf_pkt_len(mbuf);
 114                                 char *appended =
 115                                         rte_pktmbuf_append(mbuf, to_add);
 116
 117                                 if (appended == 0) {
 118                                         /* This packet is in error,
 119                                          * we cannot send it so just
 120                                          * count it and delete it.
 121                                          */
 122                                         queue->tx_errors += 1;
 123                                         rte_pktmbuf_free(mbuf);
 124                                         continue;
 125                                 }
 126                                 memset(appended, 0, to_add);
 127                         }
 128                 }
 129
 130                 if (unlikely(mbuf->nb_segs != 1)) {
 131                         stat = eth_ark_tx_jumbo(queue, mbuf);
 132                         if (unlikely(stat != 0))
 133                                 break;          /* Queue is full */
 134                 } else {
 135                         idx = queue->prod_index & queue->queue_mask;
 136                         queue->bufs[idx] = mbuf;
 137                         meta = &queue->meta_q[idx];
 138                         eth_ark_tx_meta_from_mbuf(meta,
 139                                                   mbuf,
 140                                                   ARK_DDM_SOP |
 141                                                   ARK_DDM_EOP);
 142                         queue->prod_index++;
 143                 }
 144         }
 145
 146         if (ARK_TX_DEBUG && (nb != nb_pkts)) {
 147                 PMD_TX_LOG(DEBUG, "TX: Failure to send:"
 148                            " req: %" PRIU32
 149                            " sent: %" PRIU32
 150                            " prod: %" PRIU32
 151                            " cons: %" PRIU32
 152                            " free: %" PRIU32 "\n",
 153                            nb_pkts, nb,
 154                            queue->prod_index,
 155                            queue->cons_index,
 156                            queue->free_index);
 157                 ark_mpu_dump(queue->mpu,
 158                              "TX Failure MPU: ",
 159                              queue->phys_qid);
 160         }
 161
 162         /* let FPGA know producer index.  */
 163         if (likely(nb != 0))
 164                 ark_mpu_set_producer(queue->mpu, queue->prod_index);
 165
 166         return nb;
 167 }
 168
 169 /* ************************************************************************* */
 170 static uint32_t
 171 eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf)
 172 {
 173         struct rte_mbuf *next;
 174         struct ark_tx_meta *meta;
 175         uint32_t free_queue_space;
 176         uint32_t idx;
 177         uint8_t flags = ARK_DDM_SOP;
 178
 179         free_queue_space = queue->queue_mask -
 180                 (queue->prod_index - queue->free_index);
 181         if (unlikely(free_queue_space < mbuf->nb_segs))
 182                 return -1;
 183
 184         while (mbuf != NULL) {
 185                 next = mbuf->next;
 186
 187                 idx = queue->prod_index & queue->queue_mask;
 188                 queue->bufs[idx] = mbuf;
 189                 meta = &queue->meta_q[idx];
 190
 191                 flags |= (next == NULL) ? ARK_DDM_EOP : 0;
 192                 eth_ark_tx_meta_from_mbuf(meta, mbuf, flags);
 193                 queue->prod_index++;
 194
 195                 flags &= ~ARK_DDM_SOP;  /* drop SOP flags */
 196                 mbuf = next;
 197         }
 198
 199         return 0;
 200 }
 201
 202 /* ************************************************************************* */
 203 int
 204 eth_ark_tx_queue_setup(struct rte_eth_dev *dev,
 205                        uint16_t queue_idx,
 206                        uint16_t nb_desc,
 207                        unsigned int socket_id,
 208                        const struct rte_eth_txconf *tx_conf __rte_unused)
 209 {
 210         struct ark_adapter *ark = (struct ark_adapter *)dev->data->dev_private;
 211         struct ark_tx_queue *queue;
 212         int status;
 213
 214         /* Future: divide the Q's evenly with multi-ports */
 215         int port = dev->data->port_id;
 216         int qidx = port + queue_idx;
 217
 218         if (!rte_is_power_of_2(nb_desc)) {
 219                 PMD_DRV_LOG(ERR,
 220                             "DPDK Arkville configuration queue size"
 221                             " must be power of two %u (%s)\n",
 222                             nb_desc, __func__);
 223                 return -1;
 224         }
 225
 226         /* Allocate queue struct */
 227         queue = rte_zmalloc_socket("Ark_txqueue",
 228                                    sizeof(struct ark_tx_queue),
 229                                    64,
 230                                    socket_id);
 231         if (queue == 0) {
 232                 PMD_DRV_LOG(ERR, "Failed to allocate tx "
 233                             "queue memory in %s\n",
 234                             __func__);
 235                 return -ENOMEM;
 236         }
 237
 238         /* we use zmalloc no need to initialize fields */
 239         queue->queue_size = nb_desc;
 240         queue->queue_mask = nb_desc - 1;
 241         queue->phys_qid = qidx;
 242         queue->queue_index = queue_idx;
 243         dev->data->tx_queues[queue_idx] = queue;
 244
 245         queue->meta_q =
 246                 rte_zmalloc_socket("Ark_txqueue meta",
 247                                    nb_desc * sizeof(struct ark_tx_meta),
 248                                    64,
 249                                    socket_id);
 250         queue->bufs =
 251                 rte_zmalloc_socket("Ark_txqueue bufs",
 252                                    nb_desc * sizeof(struct rte_mbuf *),
 253                                    64,
 254                                    socket_id);
 255
 256         if (queue->meta_q == 0 || queue->bufs == 0) {
 257                 PMD_DRV_LOG(ERR, "Failed to allocate "
 258                             "queue memory in %s\n", __func__);
 259                 rte_free(queue->meta_q);
 260                 rte_free(queue->bufs);
 261                 rte_free(queue);
 262                 return -ENOMEM;
 263         }
 264
 265         queue->ddm = RTE_PTR_ADD(ark->ddm.v, qidx * ARK_DDM_QOFFSET);
 266         queue->mpu = RTE_PTR_ADD(ark->mputx.v, qidx * ARK_MPU_QOFFSET);
 267
 268         status = eth_ark_tx_hw_queue_config(queue);
 269
 270         if (unlikely(status != 0)) {
 271                 rte_free(queue->meta_q);
 272                 rte_free(queue->bufs);
 273                 rte_free(queue);
 274                 return -1;              /* ERROR CODE */
 275         }
 276
 277         return 0;
 278 }
 279
 280 /* ************************************************************************* */
 281 static int
 282 eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue)
 283 {
 284         rte_iova_t queue_base, ring_base, cons_index_addr;
 285         uint32_t write_interval_ns;
 286
 287         /* Verify HW -- MPU */
 288         if (ark_mpu_verify(queue->mpu, sizeof(struct ark_tx_meta)))
 289                 return -1;
 290
 291         queue_base = rte_malloc_virt2iova(queue);
 292         ring_base = rte_malloc_virt2iova(queue->meta_q);
 293         cons_index_addr =
 294                 queue_base + offsetof(struct ark_tx_queue, cons_index);
 295
 296         ark_mpu_stop(queue->mpu);
 297         ark_mpu_reset(queue->mpu);
 298
 299         /* Stop and Reset and configure MPU */
 300         ark_mpu_configure(queue->mpu, ring_base, queue->queue_size, 1);
 301
 302         /*
 303          * Adjust the write interval based on queue size --
 304          * increase pcie traffic  when low mbuf count
 305          * Queue sizes less than 128 are not allowed
 306          */
 307         switch (queue->queue_size) {
 308         case 128:
 309                 write_interval_ns = 500;
 310                 break;
 311         case 256:
 312                 write_interval_ns = 500;
 313                 break;
 314         case 512:
 315                 write_interval_ns = 1000;
 316                 break;
 317         default:
 318                 write_interval_ns = 2000;
 319                 break;
 320         }
 321
 322         /* Completion address in UDM */
 323         ark_ddm_setup(queue->ddm, cons_index_addr, write_interval_ns);
 324
 325         return 0;
 326 }
 327
 328 /* ************************************************************************* */
 329 void
 330 eth_ark_tx_queue_release(void *vtx_queue)
 331 {
 332         struct ark_tx_queue *queue;
 333
 334         queue = (struct ark_tx_queue *)vtx_queue;
 335
 336         ark_tx_hw_queue_stop(queue);
 337
 338         queue->cons_index = queue->prod_index;
 339         free_completed_tx(queue);
 340
 341         rte_free(queue->meta_q);
 342         rte_free(queue->bufs);
 343         rte_free(queue);
 344 }
 345
 346 /* ************************************************************************* */
 347 int
 348 eth_ark_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
 349 {
 350         struct ark_tx_queue *queue;
 351         int cnt = 0;
 352
 353         queue = dev->data->tx_queues[queue_id];
 354
 355         /* Wait for DDM to send out all packets. */
 356         while (queue->cons_index != queue->prod_index) {
 357                 usleep(100);
 358                 if (cnt++ > 10000)
 359                         return -1;
 360         }
 361
 362         ark_mpu_stop(queue->mpu);
 363         free_completed_tx(queue);
 364
 365         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 366
 367         return 0;
 368 }
 369
 370 int
 371 eth_ark_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
 372 {
 373         struct ark_tx_queue *queue;
 374
 375         queue = dev->data->tx_queues[queue_id];
 376         if (dev->data->tx_queue_state[queue_id] == RTE_ETH_QUEUE_STATE_STARTED)
 377                 return 0;
 378
 379         ark_mpu_start(queue->mpu);
 380         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 381
 382         return 0;
 383 }
 384
 385 /* ************************************************************************* */
 386 static void
 387 free_completed_tx(struct ark_tx_queue *queue)
 388 {
 389         struct rte_mbuf *mbuf;
 390         struct ark_tx_meta *meta;
 391         uint32_t top_index;
 392
 393         top_index = queue->cons_index;  /* read once */
 394         while (queue->free_index != top_index) {
 395                 meta = &queue->meta_q[queue->free_index & queue->queue_mask];
 396                 mbuf = queue->bufs[queue->free_index & queue->queue_mask];
 397
 398                 if (likely((meta->flags & ARK_DDM_SOP) != 0)) {
 399                         /* ref count of the mbuf is checked in this call. */
 400                         rte_pktmbuf_free(mbuf);
 401                 }
 402                 queue->free_index++;
 403         }
 404 }
 405
 406 /* ************************************************************************* */
 407 void
 408 eth_tx_queue_stats_get(void *vqueue, struct rte_eth_stats *stats)
 409 {
 410         struct ark_tx_queue *queue;
 411         struct ark_ddm_t *ddm;
 412         uint64_t bytes, pkts;
 413
 414         queue = vqueue;
 415         ddm = queue->ddm;
 416
 417         bytes = ark_ddm_queue_byte_count(ddm);
 418         pkts = ark_ddm_queue_pkt_count(ddm);
 419
 420         stats->q_opackets[queue->queue_index] = pkts;
 421         stats->q_obytes[queue->queue_index] = bytes;
 422         stats->opackets += pkts;
 423         stats->obytes += bytes;
 424         stats->oerrors += queue->tx_errors;
 425 }
 426
 427 void
 428 eth_tx_queue_stats_reset(void *vqueue)
 429 {
 430         struct ark_tx_queue *queue;
 431         struct ark_ddm_t *ddm;
 432
 433         queue = vqueue;
 434         ddm = queue->ddm;
 435
 436         ark_ddm_queue_reset_stats(ddm);
 437         queue->tx_errors = 0;
 438 }