ceph/src/seastar/dpdk/drivers/net/octeontx/base/octeontx_pkovf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Cavium, Inc
   3  */
   4
   5 #include <stdbool.h>
   6 #include <string.h>
   7 #include <stdio.h>
   8
   9 #include <rte_eal.h>
  10 #include <rte_cycles.h>
  11 #include <rte_malloc.h>
  12 #include <rte_memory.h>
  13 #include <rte_bus_pci.h>
  14 #include <rte_spinlock.h>
  15
  16 #include "../octeontx_logs.h"
  17 #include "octeontx_io.h"
  18 #include "octeontx_pkovf.h"
  19
  20 struct octeontx_pko_iomem {
  21         uint8_t         *va;
  22         rte_iova_t      iova;
  23         size_t          size;
  24 };
  25
  26 #define PKO_IOMEM_NULL (struct octeontx_pko_iomem){0, 0, 0}
  27
  28 struct octeontx_pko_fc_ctl_s {
  29         int64_t buf_cnt;
  30         int64_t padding[(PKO_DQ_FC_STRIDE / 8) - 1];
  31 };
  32
  33 struct octeontx_pkovf {
  34         uint8_t         *bar0;
  35         uint8_t         *bar2;
  36         uint16_t        domain;
  37         uint16_t        vfid;
  38 };
  39
  40 struct octeontx_pko_vf_ctl_s {
  41         rte_spinlock_t lock;
  42
  43         struct octeontx_pko_iomem fc_iomem;
  44         struct octeontx_pko_fc_ctl_s *fc_ctl;
  45         struct octeontx_pkovf pko[PKO_VF_MAX];
  46         struct {
  47                 uint64_t chanid;
  48         } dq_map[PKO_VF_MAX * PKO_VF_NUM_DQ];
  49 };
  50
  51 static struct octeontx_pko_vf_ctl_s pko_vf_ctl;
  52
  53 static void *
  54 octeontx_pko_dq_vf_bar0(uint16_t txq)
  55 {
  56         int vf_ix;
  57
  58         vf_ix = txq / PKO_VF_NUM_DQ;
  59         return pko_vf_ctl.pko[vf_ix].bar0;
  60 }
  61
  62 static int
  63 octeontx_pko_dq_gdq(uint16_t txq)
  64 {
  65         return txq % PKO_VF_NUM_DQ;
  66 }
  67
  68 /**
  69  * Open a PKO DQ.
  70  */
  71 static inline
  72 int octeontx_pko_dq_open(uint16_t txq)
  73 {
  74         unsigned int reg_off;
  75         uint8_t *vf_bar0;
  76         uint64_t rtn;
  77         int gdq;
  78
  79         vf_bar0 = octeontx_pko_dq_vf_bar0(txq);
  80         gdq = octeontx_pko_dq_gdq(txq);
  81
  82         if (unlikely(gdq < 0 || vf_bar0 == NULL))
  83                 return -EINVAL;
  84         *(volatile int64_t*)(pko_vf_ctl.fc_ctl + txq) =
  85                 PKO_DQ_FC_DEPTH_PAGES - PKO_DQ_FC_SKID;
  86
  87         rte_wmb();
  88
  89         octeontx_write64(PKO_DQ_FC_DEPTH_PAGES,
  90                          vf_bar0 + PKO_VF_DQ_FC_STATUS(gdq));
  91
  92         /* Set the register to return descriptor (packet) count as DEPTH */
  93         /* KIND=1, NCB_QUERY_RSP=0 */
  94         octeontx_write64(1ull << PKO_DQ_KIND_BIT,
  95                                 vf_bar0 + PKO_VF_DQ_WM_CTL(gdq));
  96         reg_off = PKO_VF_DQ_OP_OPEN(gdq);
  97
  98         rtn = octeontx_reg_ldadd_u64(vf_bar0 + reg_off, 0);
  99
 100         /* PKO_DQOP_E::OPEN */
 101         if (((rtn >> PKO_DQ_OP_BIT) & 0x3) != 0x1)
 102                 return -EIO;
 103
 104         switch (rtn >> PKO_DQ_STATUS_BIT) {
 105         case 0xC:       /* DQALREADYCREATED */
 106         case 0x0:       /* PASS */
 107                 break;
 108         default:
 109                 return -EIO;
 110         }
 111
 112         /* DRAIN=0, DRAIN_NULL_LINK=0, SW_XOFF=0 */
 113         octeontx_write64(0, vf_bar0 + PKO_VF_DQ_SW_XOFF(gdq));
 114
 115         return rtn & ((1ull << PKO_DQ_OP_BIT) - 1);
 116 }
 117
 118 /**
 119  * Close a PKO DQ
 120  * Flush all packets pending.
 121  */
 122 static inline
 123 int octeontx_pko_dq_close(uint16_t txq)
 124 {
 125         unsigned int reg_off;
 126         uint8_t *vf_bar0;
 127         uint64_t rtn;
 128         int res;
 129
 130         vf_bar0 = octeontx_pko_dq_vf_bar0(txq);
 131         res = octeontx_pko_dq_gdq(txq);
 132
 133         if (unlikely(res < 0 || vf_bar0 == NULL))
 134                 return -EINVAL;
 135
 136         reg_off = PKO_VF_DQ_OP_CLOSE(res);
 137
 138         rtn = octeontx_reg_ldadd_u64(vf_bar0 + reg_off, 0);
 139
 140         /* PKO_DQOP_E::CLOSE */
 141         if (((rtn >> PKO_DQ_OP_BIT) & 0x3) != 0x2)
 142                 return -EIO;
 143
 144         switch (rtn >> PKO_DQ_STATUS_BIT) {
 145         case 0xD:       /* DQNOTCREATED */
 146         case 0x0:       /* PASS */
 147                 break;
 148         default:
 149                 return -EIO;
 150         }
 151
 152         res = rtn & ((1ull << PKO_DQ_OP_BIT) - 1); /* DEPTH */
 153         return res;
 154 }
 155
 156 /* Flush all packets pending on a DQ */
 157 static inline
 158 int octeontx_pko_dq_drain(uint16_t txq)
 159 {
 160         unsigned int gdq;
 161         uint8_t *vf_bar0;
 162         uint64_t reg;
 163         int res, timo = PKO_DQ_DRAIN_TO;
 164
 165         vf_bar0 = octeontx_pko_dq_vf_bar0(txq);
 166         res = octeontx_pko_dq_gdq(txq);
 167         gdq = res;
 168
 169          /* DRAIN=1, DRAIN_NULL_LINK=0, SW_XOFF=1 */
 170          octeontx_write64(0x3, vf_bar0 + PKO_VF_DQ_SW_XOFF(gdq));
 171         /* Wait until buffers leave DQs */
 172         reg = octeontx_read64(vf_bar0 + PKO_VF_DQ_WM_CNT(gdq));
 173         while (reg && timo > 0) {
 174                 rte_delay_us(100);
 175                 timo--;
 176                 reg = octeontx_read64(vf_bar0 + PKO_VF_DQ_WM_CNT(gdq));
 177         }
 178         /* DRAIN=0, DRAIN_NULL_LINK=0, SW_XOFF=0 */
 179         octeontx_write64(0, vf_bar0 + PKO_VF_DQ_SW_XOFF(gdq));
 180
 181         return reg;
 182 }
 183
 184 static inline int
 185 octeontx_pko_dq_range_lookup(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid,
 186                              unsigned int dq_num, unsigned int dq_from)
 187 {
 188         unsigned int dq, dq_cnt;
 189         unsigned int dq_base;
 190
 191         dq_cnt = 0;
 192         dq = dq_from;
 193         while (dq < RTE_DIM(ctl->dq_map)) {
 194                 dq_base = dq;
 195                 dq_cnt = 0;
 196                 while (ctl->dq_map[dq].chanid == ~chanid &&
 197                         dq < RTE_DIM(ctl->dq_map)) {
 198                         dq_cnt++;
 199                         if (dq_cnt == dq_num)
 200                                 return dq_base;
 201                         dq++;
 202                 }
 203                 dq++;
 204         }
 205         return -1;
 206 }
 207
 208 static inline void
 209 octeontx_pko_dq_range_assign(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid,
 210                              unsigned int dq_base, unsigned int dq_num)
 211 {
 212         unsigned int dq, dq_cnt;
 213
 214         dq_cnt = 0;
 215         while (dq_cnt < dq_num) {
 216                 dq = dq_base + dq_cnt;
 217
 218                 octeontx_log_dbg("DQ# %u assigned to CHAN# %" PRIx64 "", dq,
 219                         chanid);
 220
 221                 ctl->dq_map[dq].chanid = ~chanid;
 222                 dq_cnt++;
 223         }
 224 }
 225
 226 static inline int
 227 octeontx_pko_dq_claim(struct octeontx_pko_vf_ctl_s *ctl, unsigned int dq_base,
 228                       unsigned int dq_num, uint64_t chanid)
 229 {
 230         const uint64_t null_chanid = ~0ull;
 231         int dq;
 232
 233         rte_spinlock_lock(&ctl->lock);
 234
 235         dq = octeontx_pko_dq_range_lookup(ctl, null_chanid, dq_num, dq_base);
 236         if (dq < 0 || (unsigned int)dq != dq_base) {
 237                 rte_spinlock_unlock(&ctl->lock);
 238                 return -1;
 239         }
 240         octeontx_pko_dq_range_assign(ctl, chanid, dq_base, dq_num);
 241
 242         rte_spinlock_unlock(&ctl->lock);
 243
 244         return 0;
 245 }
 246
 247 static inline int
 248 octeontx_pko_dq_free(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid)
 249 {
 250         const uint64_t null_chanid = ~0ull;
 251         unsigned int dq = 0, dq_cnt = 0;
 252
 253         rte_spinlock_lock(&ctl->lock);
 254         while (dq < RTE_DIM(ctl->dq_map)) {
 255                 if (ctl->dq_map[dq].chanid == ~chanid) {
 256                         ctl->dq_map[dq].chanid = ~null_chanid;
 257                         dq_cnt++;
 258                 }
 259                 dq++;
 260         }
 261         rte_spinlock_unlock(&ctl->lock);
 262
 263         return dq_cnt > 0 ? 0 : -EINVAL;
 264 }
 265
 266 int
 267 octeontx_pko_channel_open(int dq_base, int dq_num, int chanid)
 268 {
 269         struct octeontx_pko_vf_ctl_s *ctl = &pko_vf_ctl;
 270         int res;
 271
 272         res = octeontx_pko_dq_claim(ctl, dq_base, dq_num, chanid);
 273         if (res < 0)
 274                 return -1;
 275
 276         return 0;
 277 }
 278
 279 int
 280 octeontx_pko_channel_close(int chanid)
 281 {
 282         struct octeontx_pko_vf_ctl_s *ctl = &pko_vf_ctl;
 283         int res;
 284
 285         res = octeontx_pko_dq_free(ctl, chanid);
 286         if (res < 0)
 287                 return -1;
 288
 289         return 0;
 290 }
 291
 292 static inline int
 293 octeontx_pko_chan_start(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid)
 294 {
 295         unsigned int dq_vf;
 296         unsigned int dq, dq_cnt;
 297
 298         dq_cnt = 0;
 299         dq = 0;
 300         while (dq < RTE_DIM(ctl->dq_map)) {
 301                 dq_vf = dq / PKO_VF_NUM_DQ;
 302
 303                 if (!ctl->pko[dq_vf].bar0) {
 304                         dq += PKO_VF_NUM_DQ;
 305                         continue;
 306                 }
 307
 308                 if (ctl->dq_map[dq].chanid != ~chanid) {
 309                         dq++;
 310                         continue;
 311                 }
 312
 313                 if (octeontx_pko_dq_open(dq) < 0)
 314                         break;
 315
 316                 dq_cnt++;
 317                 dq++;
 318         }
 319
 320         return dq_cnt;
 321 }
 322
 323 int
 324 octeontx_pko_channel_start(int chanid)
 325 {
 326         struct octeontx_pko_vf_ctl_s *ctl = &pko_vf_ctl;
 327         int dq_cnt;
 328
 329         dq_cnt = octeontx_pko_chan_start(ctl, chanid);
 330         if (dq_cnt < 0)
 331                 return -1;
 332
 333         return dq_cnt;
 334 }
 335
 336 static inline int
 337 octeontx_pko_chan_stop(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid)
 338 {
 339         unsigned int dq, dq_cnt, dq_vf;
 340         int res;
 341
 342         dq_cnt = 0;
 343         dq = 0;
 344         while (dq < RTE_DIM(ctl->dq_map)) {
 345                 dq_vf = dq / PKO_VF_NUM_DQ;
 346
 347                 if (!ctl->pko[dq_vf].bar0) {
 348                         dq += PKO_VF_NUM_DQ;
 349                         continue;
 350                 }
 351
 352                 if (ctl->dq_map[dq].chanid != ~chanid) {
 353                         dq++;
 354                         continue;
 355                 }
 356
 357                 res = octeontx_pko_dq_drain(dq);
 358                 if (res > 0)
 359                         octeontx_log_err("draining DQ%d, buffers left: %x",
 360                                          dq, res);
 361
 362                 res = octeontx_pko_dq_close(dq);
 363                 if (res < 0)
 364                         octeontx_log_err("closing DQ%d failed\n", dq);
 365
 366                 dq_cnt++;
 367                 dq++;
 368         }
 369         return dq_cnt;
 370 }
 371
 372 int
 373 octeontx_pko_channel_stop(int chanid)
 374 {
 375         struct octeontx_pko_vf_ctl_s *ctl = &pko_vf_ctl;
 376
 377         octeontx_pko_chan_stop(ctl, chanid);
 378         return 0;
 379 }
 380
 381 static inline int
 382 octeontx_pko_channel_query(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid,
 383                            void *out, size_t out_elem_size,
 384                            size_t dq_num, octeontx_pko_dq_getter_t getter)
 385 {
 386         octeontx_dq_t curr;
 387         unsigned int dq_vf;
 388         unsigned int dq;
 389
 390         RTE_SET_USED(out_elem_size);
 391         memset(&curr, 0, sizeof(octeontx_dq_t));
 392
 393         dq_vf = dq_num / PKO_VF_NUM_DQ;
 394         dq = dq_num % PKO_VF_NUM_DQ;
 395
 396         if (!ctl->pko[dq_vf].bar0)
 397                 return -EINVAL;
 398
 399         if (ctl->dq_map[dq_num].chanid != ~chanid)
 400                 return -EINVAL;
 401
 402         uint8_t *iter = (uint8_t *)out;
 403         curr.lmtline_va = ctl->pko[dq_vf].bar2;
 404         curr.ioreg_va = (void *)((uintptr_t)ctl->pko[dq_vf].bar0
 405                 + PKO_VF_DQ_OP_SEND((dq), 0));
 406         curr.fc_status_va = ctl->fc_ctl + dq;
 407
 408         octeontx_log_dbg("lmtline=%p ioreg_va=%p fc_status_va=%p",
 409                          curr.lmtline_va, curr.ioreg_va,
 410                          curr.fc_status_va);
 411
 412         getter(&curr, (void *)iter);
 413         return 0;
 414 }
 415
 416 int
 417 octeontx_pko_channel_query_dqs(int chanid, void *out, size_t out_elem_size,
 418                                 size_t dq_num, octeontx_pko_dq_getter_t getter)
 419 {
 420         struct octeontx_pko_vf_ctl_s *ctl = &pko_vf_ctl;
 421         int dq_cnt;
 422
 423         dq_cnt = octeontx_pko_channel_query(ctl, chanid, out, out_elem_size,
 424                                                 dq_num, getter);
 425         if (dq_cnt < 0)
 426                 return -1;
 427
 428         return dq_cnt;
 429 }
 430
 431 int
 432 octeontx_pko_vf_count(void)
 433 {
 434         int vf_cnt;
 435
 436         vf_cnt = 0;
 437         while (pko_vf_ctl.pko[vf_cnt].bar0)
 438                 vf_cnt++;
 439
 440         return vf_cnt;
 441 }
 442
 443 int
 444 octeontx_pko_init_fc(const size_t pko_vf_count)
 445 {
 446         int dq_ix;
 447         uint64_t reg;
 448         uint8_t *vf_bar0;
 449         size_t vf_idx;
 450         size_t fc_mem_size;
 451
 452         fc_mem_size = sizeof(struct octeontx_pko_fc_ctl_s) *
 453                         pko_vf_count * PKO_VF_NUM_DQ;
 454
 455         pko_vf_ctl.fc_iomem.va = rte_malloc(NULL, fc_mem_size, 128);
 456         if (unlikely(!pko_vf_ctl.fc_iomem.va)) {
 457                 octeontx_log_err("fc_iomem: not enough memory");
 458                 return -ENOMEM;
 459         }
 460
 461         pko_vf_ctl.fc_iomem.iova = rte_malloc_virt2iova((void *)
 462                                                         pko_vf_ctl.fc_iomem.va);
 463         pko_vf_ctl.fc_iomem.size = fc_mem_size;
 464
 465         pko_vf_ctl.fc_ctl =
 466                 (struct octeontx_pko_fc_ctl_s *)pko_vf_ctl.fc_iomem.va;
 467
 468         /* Configure Flow-Control feature for all DQs of open VFs */
 469         for (vf_idx = 0; vf_idx < pko_vf_count; vf_idx++) {
 470                 dq_ix = vf_idx * PKO_VF_NUM_DQ;
 471
 472                 vf_bar0 = pko_vf_ctl.pko[vf_idx].bar0;
 473
 474                 reg = (pko_vf_ctl.fc_iomem.iova +
 475                         (sizeof(struct octeontx_pko_fc_ctl_s) * dq_ix)) & ~0x7F;
 476                 reg |=                  /* BASE */
 477                     (0x2 << 3) |        /* HYST_BITS */
 478                     (((PKO_DQ_FC_STRIDE == PKO_DQ_FC_STRIDE_16) ? 1 : 0) << 2) |
 479                     (0x1 << 0);         /* ENABLE */
 480
 481                 octeontx_write64(reg, vf_bar0 + PKO_VF_DQ_FC_CONFIG);
 482
 483                 octeontx_log_dbg("PKO: bar0 %p VF_idx %d DQ_FC_CFG=%" PRIx64 "",
 484                                  vf_bar0, (int)vf_idx, reg);
 485         }
 486         return 0;
 487 }
 488
 489 void
 490 octeontx_pko_fc_free(void)
 491 {
 492         rte_free(pko_vf_ctl.fc_iomem.va);
 493 }
 494
 495 static void
 496 octeontx_pkovf_setup(void)
 497 {
 498         static bool init_once;
 499
 500         if (!init_once) {
 501                 unsigned int i;
 502
 503                 rte_spinlock_init(&pko_vf_ctl.lock);
 504
 505                 pko_vf_ctl.fc_iomem = PKO_IOMEM_NULL;
 506                 pko_vf_ctl.fc_ctl = NULL;
 507
 508                 for (i = 0; i < PKO_VF_MAX; i++) {
 509                         pko_vf_ctl.pko[i].bar0 = NULL;
 510                         pko_vf_ctl.pko[i].bar2 = NULL;
 511                         pko_vf_ctl.pko[i].domain = ~(uint16_t)0;
 512                         pko_vf_ctl.pko[i].vfid = ~(uint16_t)0;
 513                 }
 514
 515                 for (i = 0; i < (PKO_VF_MAX * PKO_VF_NUM_DQ); i++)
 516                         pko_vf_ctl.dq_map[i].chanid = 0;
 517
 518                 init_once = true;
 519         }
 520 }
 521
 522 /* PKOVF pcie device*/
 523 static int
 524 pkovf_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 525 {
 526         uint64_t val;
 527         uint16_t vfid;
 528         uint16_t domain;
 529         uint8_t *bar0;
 530         uint8_t *bar2;
 531         struct octeontx_pkovf *res;
 532
 533         RTE_SET_USED(pci_drv);
 534
 535         /* For secondary processes, the primary has done all the work */
 536         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 537                 return 0;
 538
 539         if (pci_dev->mem_resource[0].addr == NULL ||
 540             pci_dev->mem_resource[2].addr == NULL) {
 541                 octeontx_log_err("Empty bars %p %p",
 542                         pci_dev->mem_resource[0].addr,
 543                         pci_dev->mem_resource[2].addr);
 544                 return -ENODEV;
 545         }
 546         bar0 = pci_dev->mem_resource[0].addr;
 547         bar2 = pci_dev->mem_resource[2].addr;
 548
 549         octeontx_pkovf_setup();
 550
 551         /* get vfid and domain */
 552         val = octeontx_read64(bar0 + PKO_VF_DQ_FC_CONFIG);
 553         domain = (val >> 7) & 0xffff;
 554         vfid = (val >> 23) & 0xffff;
 555
 556         if (unlikely(vfid >= PKO_VF_MAX)) {
 557                 octeontx_log_err("pko: Invalid vfid %d", vfid);
 558                 return -EINVAL;
 559         }
 560
 561         res = &pko_vf_ctl.pko[vfid];
 562         res->vfid = vfid;
 563         res->domain = domain;
 564         res->bar0 = bar0;
 565         res->bar2 = bar2;
 566
 567         octeontx_log_dbg("Domain=%d group=%d", res->domain, res->vfid);
 568         return 0;
 569 }
 570
 571 #define PCI_VENDOR_ID_CAVIUM               0x177D
 572 #define PCI_DEVICE_ID_OCTEONTX_PKO_VF      0xA049
 573
 574 static const struct rte_pci_id pci_pkovf_map[] = {
 575         {
 576                 RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM,
 577                                 PCI_DEVICE_ID_OCTEONTX_PKO_VF)
 578         },
 579         {
 580                 .vendor_id = 0,
 581         },
 582 };
 583
 584 static struct rte_pci_driver pci_pkovf = {
 585         .id_table = pci_pkovf_map,
 586         .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 587         .probe = pkovf_probe,
 588 };
 589
 590 RTE_PMD_REGISTER_PCI(octeontx_pkovf, pci_pkovf);