ceph/src/spdk/dpdk/drivers/bus/pci/pci_common.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation.
   3  * Copyright 2013-2014 6WIND S.A.
   4  */
   5
   6 #include <string.h>
   7 #include <inttypes.h>
   8 #include <stdint.h>
   9 #include <stdlib.h>
  10 #include <stdio.h>
  11 #include <sys/queue.h>
  12 #include <sys/mman.h>
  13
  14 #include <rte_errno.h>
  15 #include <rte_interrupts.h>
  16 #include <rte_log.h>
  17 #include <rte_bus.h>
  18 #include <rte_pci.h>
  19 #include <rte_bus_pci.h>
  20 #include <rte_per_lcore.h>
  21 #include <rte_memory.h>
  22 #include <rte_eal.h>
  23 #include <rte_string_fns.h>
  24 #include <rte_common.h>
  25 #include <rte_devargs.h>
  26
  27 #include "private.h"
  28
  29
  30 extern struct rte_pci_bus rte_pci_bus;
  31
  32 #define SYSFS_PCI_DEVICES "/sys/bus/pci/devices"
  33
  34 const char *rte_pci_get_sysfs_path(void)
  35 {
  36         const char *path = NULL;
  37
  38         path = getenv("SYSFS_PCI_DEVICES");
  39         if (path == NULL)
  40                 return SYSFS_PCI_DEVICES;
  41
  42         return path;
  43 }
  44
  45 static struct rte_devargs *pci_devargs_lookup(struct rte_pci_device *dev)
  46 {
  47         struct rte_devargs *devargs;
  48         struct rte_pci_addr addr;
  49
  50         RTE_EAL_DEVARGS_FOREACH("pci", devargs) {
  51                 devargs->bus->parse(devargs->name, &addr);
  52                 if (!rte_pci_addr_cmp(&dev->addr, &addr))
  53                         return devargs;
  54         }
  55         return NULL;
  56 }
  57
  58 void
  59 pci_name_set(struct rte_pci_device *dev)
  60 {
  61         struct rte_devargs *devargs;
  62
  63         /* Each device has its internal, canonical name set. */
  64         rte_pci_device_name(&dev->addr,
  65                         dev->name, sizeof(dev->name));
  66         devargs = pci_devargs_lookup(dev);
  67         dev->device.devargs = devargs;
  68         /* In blacklist mode, if the device is not blacklisted, no
  69          * rte_devargs exists for it.
  70          */
  71         if (devargs != NULL)
  72                 /* If an rte_devargs exists, the generic rte_device uses the
  73                  * given name as its name.
  74                  */
  75                 dev->device.name = dev->device.devargs->name;
  76         else
  77                 /* Otherwise, it uses the internal, canonical form. */
  78                 dev->device.name = dev->name;
  79 }
  80
  81 /*
  82  * Match the PCI Driver and Device using the ID Table
  83  */
  84 int
  85 rte_pci_match(const struct rte_pci_driver *pci_drv,
  86               const struct rte_pci_device *pci_dev)
  87 {
  88         const struct rte_pci_id *id_table;
  89
  90         for (id_table = pci_drv->id_table; id_table->vendor_id != 0;
  91              id_table++) {
  92                 /* check if device's identifiers match the driver's ones */
  93                 if (id_table->vendor_id != pci_dev->id.vendor_id &&
  94                                 id_table->vendor_id != PCI_ANY_ID)
  95                         continue;
  96                 if (id_table->device_id != pci_dev->id.device_id &&
  97                                 id_table->device_id != PCI_ANY_ID)
  98                         continue;
  99                 if (id_table->subsystem_vendor_id !=
 100                     pci_dev->id.subsystem_vendor_id &&
 101                     id_table->subsystem_vendor_id != PCI_ANY_ID)
 102                         continue;
 103                 if (id_table->subsystem_device_id !=
 104                     pci_dev->id.subsystem_device_id &&
 105                     id_table->subsystem_device_id != PCI_ANY_ID)
 106                         continue;
 107                 if (id_table->class_id != pci_dev->id.class_id &&
 108                                 id_table->class_id != RTE_CLASS_ANY_ID)
 109                         continue;
 110
 111                 return 1;
 112         }
 113
 114         return 0;
 115 }
 116
 117 /*
 118  * If vendor/device ID match, call the probe() function of the
 119  * driver.
 120  */
 121 static int
 122 rte_pci_probe_one_driver(struct rte_pci_driver *dr,
 123                          struct rte_pci_device *dev)
 124 {
 125         int ret;
 126         struct rte_pci_addr *loc;
 127
 128         if ((dr == NULL) || (dev == NULL))
 129                 return -EINVAL;
 130
 131         loc = &dev->addr;
 132
 133         /* The device is not blacklisted; Check if driver supports it */
 134         if (!rte_pci_match(dr, dev))
 135                 /* Match of device and driver failed */
 136                 return 1;
 137
 138         RTE_LOG(INFO, EAL, "PCI device "PCI_PRI_FMT" on NUMA socket %i\n",
 139                         loc->domain, loc->bus, loc->devid, loc->function,
 140                         dev->device.numa_node);
 141
 142         /* no initialization when blacklisted, return without error */
 143         if (dev->device.devargs != NULL &&
 144                 dev->device.devargs->policy ==
 145                         RTE_DEV_BLACKLISTED) {
 146                 RTE_LOG(INFO, EAL, "  Device is blacklisted, not"
 147                         " initializing\n");
 148                 return 1;
 149         }
 150
 151         if (dev->device.numa_node < 0) {
 152                 RTE_LOG(WARNING, EAL, "  Invalid NUMA socket, default to 0\n");
 153                 dev->device.numa_node = 0;
 154         }
 155
 156         RTE_LOG(INFO, EAL, "  probe driver: %x:%x %s\n", dev->id.vendor_id,
 157                 dev->id.device_id, dr->driver.name);
 158
 159         /*
 160          * reference driver structure
 161          * This needs to be before rte_pci_map_device(), as it enables to use
 162          * driver flags for adjusting configuration.
 163          */
 164         dev->driver = dr;
 165         dev->device.driver = &dr->driver;
 166
 167         if (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) {
 168                 /* map resources for devices that use igb_uio */
 169                 ret = rte_pci_map_device(dev);
 170                 if (ret != 0) {
 171                         dev->driver = NULL;
 172                         dev->device.driver = NULL;
 173                         return ret;
 174                 }
 175         }
 176
 177         /* call the driver probe() function */
 178         ret = dr->probe(dr, dev);
 179         if (ret) {
 180                 dev->driver = NULL;
 181                 dev->device.driver = NULL;
 182                 if ((dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) &&
 183                         /* Don't unmap if device is unsupported and
 184                          * driver needs mapped resources.
 185                          */
 186                         !(ret > 0 &&
 187                                 (dr->drv_flags & RTE_PCI_DRV_KEEP_MAPPED_RES)))
 188                         rte_pci_unmap_device(dev);
 189         }
 190
 191         return ret;
 192 }
 193
 194 /*
 195  * If vendor/device ID match, call the remove() function of the
 196  * driver.
 197  */
 198 static int
 199 rte_pci_detach_dev(struct rte_pci_device *dev)
 200 {
 201         struct rte_pci_addr *loc;
 202         struct rte_pci_driver *dr;
 203         int ret = 0;
 204
 205         if (dev == NULL)
 206                 return -EINVAL;
 207
 208         dr = dev->driver;
 209         loc = &dev->addr;
 210
 211         RTE_LOG(DEBUG, EAL, "PCI device "PCI_PRI_FMT" on NUMA socket %i\n",
 212                         loc->domain, loc->bus, loc->devid,
 213                         loc->function, dev->device.numa_node);
 214
 215         RTE_LOG(DEBUG, EAL, "  remove driver: %x:%x %s\n", dev->id.vendor_id,
 216                         dev->id.device_id, dr->driver.name);
 217
 218         if (dr->remove) {
 219                 ret = dr->remove(dev);
 220                 if (ret < 0)
 221                         return ret;
 222         }
 223
 224         /* clear driver structure */
 225         dev->driver = NULL;
 226
 227         if (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING)
 228                 /* unmap resources for devices that use igb_uio */
 229                 rte_pci_unmap_device(dev);
 230
 231         return 0;
 232 }
 233
 234 /*
 235  * If vendor/device ID match, call the probe() function of all
 236  * registered driver for the given device. Return -1 if initialization
 237  * failed, return 1 if no driver is found for this device.
 238  */
 239 static int
 240 pci_probe_all_drivers(struct rte_pci_device *dev)
 241 {
 242         struct rte_pci_driver *dr = NULL;
 243         int rc = 0;
 244
 245         if (dev == NULL)
 246                 return -1;
 247
 248         /* Check if a driver is already loaded */
 249         if (dev->driver != NULL)
 250                 return 0;
 251
 252         FOREACH_DRIVER_ON_PCIBUS(dr) {
 253                 rc = rte_pci_probe_one_driver(dr, dev);
 254                 if (rc < 0)
 255                         /* negative value is an error */
 256                         return -1;
 257                 if (rc > 0)
 258                         /* positive value means driver doesn't support it */
 259                         continue;
 260                 return 0;
 261         }
 262         return 1;
 263 }
 264
 265 /*
 266  * Scan the content of the PCI bus, and call the probe() function for
 267  * all registered drivers that have a matching entry in its id_table
 268  * for discovered devices.
 269  */
 270 int
 271 rte_pci_probe(void)
 272 {
 273         struct rte_pci_device *dev = NULL;
 274         size_t probed = 0, failed = 0;
 275         struct rte_devargs *devargs;
 276         int probe_all = 0;
 277         int ret = 0;
 278
 279         if (rte_pci_bus.bus.conf.scan_mode != RTE_BUS_SCAN_WHITELIST)
 280                 probe_all = 1;
 281
 282         FOREACH_DEVICE_ON_PCIBUS(dev) {
 283                 probed++;
 284
 285                 devargs = dev->device.devargs;
 286                 /* probe all or only whitelisted devices */
 287                 if (probe_all)
 288                         ret = pci_probe_all_drivers(dev);
 289                 else if (devargs != NULL &&
 290                         devargs->policy == RTE_DEV_WHITELISTED)
 291                         ret = pci_probe_all_drivers(dev);
 292                 if (ret < 0) {
 293                         RTE_LOG(ERR, EAL, "Requested device " PCI_PRI_FMT
 294                                  " cannot be used\n", dev->addr.domain, dev->addr.bus,
 295                                  dev->addr.devid, dev->addr.function);
 296                         rte_errno = errno;
 297                         failed++;
 298                         ret = 0;
 299                 }
 300         }
 301
 302         return (probed && probed == failed) ? -1 : 0;
 303 }
 304
 305 /* dump one device */
 306 static int
 307 pci_dump_one_device(FILE *f, struct rte_pci_device *dev)
 308 {
 309         int i;
 310
 311         fprintf(f, PCI_PRI_FMT, dev->addr.domain, dev->addr.bus,
 312                dev->addr.devid, dev->addr.function);
 313         fprintf(f, " - vendor:%x device:%x\n", dev->id.vendor_id,
 314                dev->id.device_id);
 315
 316         for (i = 0; i != sizeof(dev->mem_resource) /
 317                 sizeof(dev->mem_resource[0]); i++) {
 318                 fprintf(f, "   %16.16"PRIx64" %16.16"PRIx64"\n",
 319                         dev->mem_resource[i].phys_addr,
 320                         dev->mem_resource[i].len);
 321         }
 322         return 0;
 323 }
 324
 325 /* dump devices on the bus */
 326 void
 327 rte_pci_dump(FILE *f)
 328 {
 329         struct rte_pci_device *dev = NULL;
 330
 331         FOREACH_DEVICE_ON_PCIBUS(dev) {
 332                 pci_dump_one_device(f, dev);
 333         }
 334 }
 335
 336 static int
 337 pci_parse(const char *name, void *addr)
 338 {
 339         struct rte_pci_addr *out = addr;
 340         struct rte_pci_addr pci_addr;
 341         bool parse;
 342
 343         parse = (rte_pci_addr_parse(name, &pci_addr) == 0);
 344         if (parse && addr != NULL)
 345                 *out = pci_addr;
 346         return parse == false;
 347 }
 348
 349 /* register a driver */
 350 void
 351 rte_pci_register(struct rte_pci_driver *driver)
 352 {
 353         TAILQ_INSERT_TAIL(&rte_pci_bus.driver_list, driver, next);
 354         driver->bus = &rte_pci_bus;
 355 }
 356
 357 /* unregister a driver */
 358 void
 359 rte_pci_unregister(struct rte_pci_driver *driver)
 360 {
 361         TAILQ_REMOVE(&rte_pci_bus.driver_list, driver, next);
 362         driver->bus = NULL;
 363 }
 364
 365 /* Add a device to PCI bus */
 366 void
 367 rte_pci_add_device(struct rte_pci_device *pci_dev)
 368 {
 369         TAILQ_INSERT_TAIL(&rte_pci_bus.device_list, pci_dev, next);
 370 }
 371
 372 /* Insert a device into a predefined position in PCI bus */
 373 void
 374 rte_pci_insert_device(struct rte_pci_device *exist_pci_dev,
 375                       struct rte_pci_device *new_pci_dev)
 376 {
 377         TAILQ_INSERT_BEFORE(exist_pci_dev, new_pci_dev, next);
 378 }
 379
 380 /* Remove a device from PCI bus */
 381 static void
 382 rte_pci_remove_device(struct rte_pci_device *pci_dev)
 383 {
 384         TAILQ_REMOVE(&rte_pci_bus.device_list, pci_dev, next);
 385 }
 386
 387 static struct rte_device *
 388 pci_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
 389                 const void *data)
 390 {
 391         const struct rte_pci_device *pstart;
 392         struct rte_pci_device *pdev;
 393
 394         if (start != NULL) {
 395                 pstart = RTE_DEV_TO_PCI_CONST(start);
 396                 pdev = TAILQ_NEXT(pstart, next);
 397         } else {
 398                 pdev = TAILQ_FIRST(&rte_pci_bus.device_list);
 399         }
 400         while (pdev != NULL) {
 401                 if (cmp(&pdev->device, data) == 0)
 402                         return &pdev->device;
 403                 pdev = TAILQ_NEXT(pdev, next);
 404         }
 405         return NULL;
 406 }
 407
 408 static int
 409 pci_plug(struct rte_device *dev)
 410 {
 411         return pci_probe_all_drivers(RTE_DEV_TO_PCI(dev));
 412 }
 413
 414 static int
 415 pci_unplug(struct rte_device *dev)
 416 {
 417         struct rte_pci_device *pdev;
 418         int ret;
 419
 420         pdev = RTE_DEV_TO_PCI(dev);
 421         ret = rte_pci_detach_dev(pdev);
 422         if (ret == 0) {
 423                 rte_pci_remove_device(pdev);
 424                 free(pdev);
 425         }
 426         return ret;
 427 }
 428
 429 struct rte_pci_bus rte_pci_bus = {
 430         .bus = {
 431                 .scan = rte_pci_scan,
 432                 .probe = rte_pci_probe,
 433                 .find_device = pci_find_device,
 434                 .plug = pci_plug,
 435                 .unplug = pci_unplug,
 436                 .parse = pci_parse,
 437                 .get_iommu_class = rte_pci_get_iommu_class,
 438         },
 439         .device_list = TAILQ_HEAD_INITIALIZER(rte_pci_bus.device_list),
 440         .driver_list = TAILQ_HEAD_INITIALIZER(rte_pci_bus.driver_list),
 441 };
 442
 443 RTE_REGISTER_BUS(pci, rte_pci_bus.bus);