ceph/src/spdk/dpdk/drivers/bus/vdev/vdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016 RehiveTech. All rights reserved.
   3  */
   4
   5 #include <string.h>
   6 #include <inttypes.h>
   7 #include <stdio.h>
   8 #include <stdlib.h>
   9 #include <stdint.h>
  10 #include <stdbool.h>
  11 #include <sys/queue.h>
  12
  13 #include <rte_eal.h>
  14 #include <rte_dev.h>
  15 #include <rte_bus.h>
  16 #include <rte_common.h>
  17 #include <rte_devargs.h>
  18 #include <rte_memory.h>
  19 #include <rte_tailq.h>
  20 #include <rte_spinlock.h>
  21 #include <rte_string_fns.h>
  22 #include <rte_errno.h>
  23
  24 #include "rte_bus_vdev.h"
  25 #include "vdev_logs.h"
  26 #include "vdev_private.h"
  27
  28 #define VDEV_MP_KEY     "bus_vdev_mp"
  29
  30 int vdev_logtype_bus;
  31
  32 /* Forward declare to access virtual bus name */
  33 static struct rte_bus rte_vdev_bus;
  34
  35 /** Double linked list of virtual device drivers. */
  36 TAILQ_HEAD(vdev_device_list, rte_vdev_device);
  37
  38 static struct vdev_device_list vdev_device_list =
  39         TAILQ_HEAD_INITIALIZER(vdev_device_list);
  40 /* The lock needs to be recursive because a vdev can manage another vdev. */
  41 static rte_spinlock_recursive_t vdev_device_list_lock =
  42         RTE_SPINLOCK_RECURSIVE_INITIALIZER;
  43
  44 static struct vdev_driver_list vdev_driver_list =
  45         TAILQ_HEAD_INITIALIZER(vdev_driver_list);
  46
  47 struct vdev_custom_scan {
  48         TAILQ_ENTRY(vdev_custom_scan) next;
  49         rte_vdev_scan_callback callback;
  50         void *user_arg;
  51 };
  52 TAILQ_HEAD(vdev_custom_scans, vdev_custom_scan);
  53 static struct vdev_custom_scans vdev_custom_scans =
  54         TAILQ_HEAD_INITIALIZER(vdev_custom_scans);
  55 static rte_spinlock_t vdev_custom_scan_lock = RTE_SPINLOCK_INITIALIZER;
  56
  57 /* register a driver */
  58 void
  59 rte_vdev_register(struct rte_vdev_driver *driver)
  60 {
  61         TAILQ_INSERT_TAIL(&vdev_driver_list, driver, next);
  62 }
  63
  64 /* unregister a driver */
  65 void
  66 rte_vdev_unregister(struct rte_vdev_driver *driver)
  67 {
  68         TAILQ_REMOVE(&vdev_driver_list, driver, next);
  69 }
  70
  71 int
  72 rte_vdev_add_custom_scan(rte_vdev_scan_callback callback, void *user_arg)
  73 {
  74         struct vdev_custom_scan *custom_scan;
  75
  76         rte_spinlock_lock(&vdev_custom_scan_lock);
  77
  78         /* check if already registered */
  79         TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) {
  80                 if (custom_scan->callback == callback &&
  81                                 custom_scan->user_arg == user_arg)
  82                         break;
  83         }
  84
  85         if (custom_scan == NULL) {
  86                 custom_scan = malloc(sizeof(struct vdev_custom_scan));
  87                 if (custom_scan != NULL) {
  88                         custom_scan->callback = callback;
  89                         custom_scan->user_arg = user_arg;
  90                         TAILQ_INSERT_TAIL(&vdev_custom_scans, custom_scan, next);
  91                 }
  92         }
  93
  94         rte_spinlock_unlock(&vdev_custom_scan_lock);
  95
  96         return (custom_scan == NULL) ? -1 : 0;
  97 }
  98
  99 int
 100 rte_vdev_remove_custom_scan(rte_vdev_scan_callback callback, void *user_arg)
 101 {
 102         struct vdev_custom_scan *custom_scan, *tmp_scan;
 103
 104         rte_spinlock_lock(&vdev_custom_scan_lock);
 105         TAILQ_FOREACH_SAFE(custom_scan, &vdev_custom_scans, next, tmp_scan) {
 106                 if (custom_scan->callback != callback ||
 107                                 (custom_scan->user_arg != (void *)-1 &&
 108                                 custom_scan->user_arg != user_arg))
 109                         continue;
 110                 TAILQ_REMOVE(&vdev_custom_scans, custom_scan, next);
 111                 free(custom_scan);
 112         }
 113         rte_spinlock_unlock(&vdev_custom_scan_lock);
 114
 115         return 0;
 116 }
 117
 118 static int
 119 vdev_parse(const char *name, void *addr)
 120 {
 121         struct rte_vdev_driver **out = addr;
 122         struct rte_vdev_driver *driver = NULL;
 123
 124         TAILQ_FOREACH(driver, &vdev_driver_list, next) {
 125                 if (strncmp(driver->driver.name, name,
 126                             strlen(driver->driver.name)) == 0)
 127                         break;
 128                 if (driver->driver.alias &&
 129                     strncmp(driver->driver.alias, name,
 130                             strlen(driver->driver.alias)) == 0)
 131                         break;
 132         }
 133         if (driver != NULL &&
 134             addr != NULL)
 135                 *out = driver;
 136         return driver == NULL;
 137 }
 138
 139 static int
 140 vdev_probe_all_drivers(struct rte_vdev_device *dev)
 141 {
 142         const char *name;
 143         struct rte_vdev_driver *driver;
 144         int ret;
 145
 146         if (rte_dev_is_probed(&dev->device))
 147                 return -EEXIST;
 148
 149         name = rte_vdev_device_name(dev);
 150         VDEV_LOG(DEBUG, "Search driver to probe device %s", name);
 151
 152         if (vdev_parse(name, &driver))
 153                 return -1;
 154         ret = driver->probe(dev);
 155         if (ret == 0)
 156                 dev->device.driver = &driver->driver;
 157         return ret;
 158 }
 159
 160 /* The caller shall be responsible for thread-safe */
 161 static struct rte_vdev_device *
 162 find_vdev(const char *name)
 163 {
 164         struct rte_vdev_device *dev;
 165
 166         if (!name)
 167                 return NULL;
 168
 169         TAILQ_FOREACH(dev, &vdev_device_list, next) {
 170                 const char *devname = rte_vdev_device_name(dev);
 171
 172                 if (!strcmp(devname, name))
 173                         return dev;
 174         }
 175
 176         return NULL;
 177 }
 178
 179 static struct rte_devargs *
 180 alloc_devargs(const char *name, const char *args)
 181 {
 182         struct rte_devargs *devargs;
 183         int ret;
 184
 185         devargs = calloc(1, sizeof(*devargs));
 186         if (!devargs)
 187                 return NULL;
 188
 189         devargs->bus = &rte_vdev_bus;
 190         if (args)
 191                 devargs->args = strdup(args);
 192         else
 193                 devargs->args = strdup("");
 194
 195         ret = strlcpy(devargs->name, name, sizeof(devargs->name));
 196         if (ret < 0 || ret >= (int)sizeof(devargs->name)) {
 197                 free(devargs->args);
 198                 free(devargs);
 199                 return NULL;
 200         }
 201
 202         return devargs;
 203 }
 204
 205 static int
 206 insert_vdev(const char *name, const char *args,
 207                 struct rte_vdev_device **p_dev,
 208                 bool init)
 209 {
 210         struct rte_vdev_device *dev;
 211         struct rte_devargs *devargs;
 212         int ret;
 213
 214         if (name == NULL)
 215                 return -EINVAL;
 216
 217         devargs = alloc_devargs(name, args);
 218         if (!devargs)
 219                 return -ENOMEM;
 220
 221         dev = calloc(1, sizeof(*dev));
 222         if (!dev) {
 223                 ret = -ENOMEM;
 224                 goto fail;
 225         }
 226
 227         dev->device.bus = &rte_vdev_bus;
 228         dev->device.numa_node = SOCKET_ID_ANY;
 229         dev->device.name = devargs->name;
 230
 231         if (find_vdev(name)) {
 232                 /*
 233                  * A vdev is expected to have only one port.
 234                  * So there is no reason to try probing again,
 235                  * even with new arguments.
 236                  */
 237                 ret = -EEXIST;
 238                 goto fail;
 239         }
 240
 241         if (init)
 242                 rte_devargs_insert(&devargs);
 243         dev->device.devargs = devargs;
 244         TAILQ_INSERT_TAIL(&vdev_device_list, dev, next);
 245
 246         if (p_dev)
 247                 *p_dev = dev;
 248
 249         return 0;
 250 fail:
 251         free(devargs->args);
 252         free(devargs);
 253         free(dev);
 254         return ret;
 255 }
 256
 257 int
 258 rte_vdev_init(const char *name, const char *args)
 259 {
 260         struct rte_vdev_device *dev;
 261         int ret;
 262
 263         rte_spinlock_recursive_lock(&vdev_device_list_lock);
 264         ret = insert_vdev(name, args, &dev, true);
 265         if (ret == 0) {
 266                 ret = vdev_probe_all_drivers(dev);
 267                 if (ret) {
 268                         if (ret > 0)
 269                                 VDEV_LOG(ERR, "no driver found for %s", name);
 270                         /* If fails, remove it from vdev list */
 271                         TAILQ_REMOVE(&vdev_device_list, dev, next);
 272                         rte_devargs_remove(dev->device.devargs);
 273                         free(dev);
 274                 }
 275         }
 276         rte_spinlock_recursive_unlock(&vdev_device_list_lock);
 277         return ret;
 278 }
 279
 280 static int
 281 vdev_remove_driver(struct rte_vdev_device *dev)
 282 {
 283         const char *name = rte_vdev_device_name(dev);
 284         const struct rte_vdev_driver *driver;
 285
 286         if (!dev->device.driver) {
 287                 VDEV_LOG(DEBUG, "no driver attach to device %s", name);
 288                 return 1;
 289         }
 290
 291         driver = container_of(dev->device.driver, const struct rte_vdev_driver,
 292                 driver);
 293         return driver->remove(dev);
 294 }
 295
 296 int
 297 rte_vdev_uninit(const char *name)
 298 {
 299         struct rte_vdev_device *dev;
 300         int ret;
 301
 302         if (name == NULL)
 303                 return -EINVAL;
 304
 305         rte_spinlock_recursive_lock(&vdev_device_list_lock);
 306
 307         dev = find_vdev(name);
 308         if (!dev) {
 309                 ret = -ENOENT;
 310                 goto unlock;
 311         }
 312
 313         ret = vdev_remove_driver(dev);
 314         if (ret)
 315                 goto unlock;
 316
 317         TAILQ_REMOVE(&vdev_device_list, dev, next);
 318         rte_devargs_remove(dev->device.devargs);
 319         free(dev);
 320
 321 unlock:
 322         rte_spinlock_recursive_unlock(&vdev_device_list_lock);
 323         return ret;
 324 }
 325
 326 struct vdev_param {
 327 #define VDEV_SCAN_REQ   1
 328 #define VDEV_SCAN_ONE   2
 329 #define VDEV_SCAN_REP   3
 330         int type;
 331         int num;
 332         char name[RTE_DEV_NAME_MAX_LEN];
 333 };
 334
 335 static int vdev_plug(struct rte_device *dev);
 336
 337 /**
 338  * This function works as the action for both primary and secondary process
 339  * for static vdev discovery when a secondary process is booting.
 340  *
 341  * step 1, secondary process sends a sync request to ask for vdev in primary;
 342  * step 2, primary process receives the request, and send vdevs one by one;
 343  * step 3, primary process sends back reply, which indicates how many vdevs
 344  * are sent.
 345  */
 346 static int
 347 vdev_action(const struct rte_mp_msg *mp_msg, const void *peer)
 348 {
 349         struct rte_vdev_device *dev;
 350         struct rte_mp_msg mp_resp;
 351         struct vdev_param *ou = (struct vdev_param *)&mp_resp.param;
 352         const struct vdev_param *in = (const struct vdev_param *)mp_msg->param;
 353         const char *devname;
 354         int num;
 355         int ret;
 356
 357         strlcpy(mp_resp.name, VDEV_MP_KEY, sizeof(mp_resp.name));
 358         mp_resp.len_param = sizeof(*ou);
 359         mp_resp.num_fds = 0;
 360
 361         switch (in->type) {
 362         case VDEV_SCAN_REQ:
 363                 ou->type = VDEV_SCAN_ONE;
 364                 ou->num = 1;
 365                 num = 0;
 366
 367                 rte_spinlock_recursive_lock(&vdev_device_list_lock);
 368                 TAILQ_FOREACH(dev, &vdev_device_list, next) {
 369                         devname = rte_vdev_device_name(dev);
 370                         if (strlen(devname) == 0) {
 371                                 VDEV_LOG(INFO, "vdev with no name is not sent");
 372                                 continue;
 373                         }
 374                         VDEV_LOG(INFO, "send vdev, %s", devname);
 375                         strlcpy(ou->name, devname, RTE_DEV_NAME_MAX_LEN);
 376                         if (rte_mp_sendmsg(&mp_resp) < 0)
 377                                 VDEV_LOG(ERR, "send vdev, %s, failed, %s",
 378                                          devname, strerror(rte_errno));
 379                         num++;
 380                 }
 381                 rte_spinlock_recursive_unlock(&vdev_device_list_lock);
 382
 383                 ou->type = VDEV_SCAN_REP;
 384                 ou->num = num;
 385                 if (rte_mp_reply(&mp_resp, peer) < 0)
 386                         VDEV_LOG(ERR, "Failed to reply a scan request");
 387                 break;
 388         case VDEV_SCAN_ONE:
 389                 VDEV_LOG(INFO, "receive vdev, %s", in->name);
 390                 ret = insert_vdev(in->name, NULL, NULL, false);
 391                 if (ret == -EEXIST)
 392                         VDEV_LOG(DEBUG, "device already exist, %s", in->name);
 393                 else if (ret < 0)
 394                         VDEV_LOG(ERR, "failed to add vdev, %s", in->name);
 395                 break;
 396         default:
 397                 VDEV_LOG(ERR, "vdev cannot recognize this message");
 398         }
 399
 400         return 0;
 401 }
 402
 403 static int
 404 vdev_scan(void)
 405 {
 406         struct rte_vdev_device *dev;
 407         struct rte_devargs *devargs;
 408         struct vdev_custom_scan *custom_scan;
 409
 410         if (rte_mp_action_register(VDEV_MP_KEY, vdev_action) < 0 &&
 411             rte_errno != EEXIST) {
 412                 VDEV_LOG(ERR, "Failed to add vdev mp action");
 413                 return -1;
 414         }
 415
 416         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 417                 struct rte_mp_msg mp_req, *mp_rep;
 418                 struct rte_mp_reply mp_reply;
 419                 struct timespec ts = {.tv_sec = 5, .tv_nsec = 0};
 420                 struct vdev_param *req = (struct vdev_param *)mp_req.param;
 421                 struct vdev_param *resp;
 422
 423                 strlcpy(mp_req.name, VDEV_MP_KEY, sizeof(mp_req.name));
 424                 mp_req.len_param = sizeof(*req);
 425                 mp_req.num_fds = 0;
 426                 req->type = VDEV_SCAN_REQ;
 427                 if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 &&
 428                     mp_reply.nb_received == 1) {
 429                         mp_rep = &mp_reply.msgs[0];
 430                         resp = (struct vdev_param *)mp_rep->param;
 431                         VDEV_LOG(INFO, "Received %d vdevs", resp->num);
 432                         free(mp_reply.msgs);
 433                 } else
 434                         VDEV_LOG(ERR, "Failed to request vdev from primary");
 435
 436                 /* Fall through to allow private vdevs in secondary process */
 437         }
 438
 439         /* call custom scan callbacks if any */
 440         rte_spinlock_lock(&vdev_custom_scan_lock);
 441         TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) {
 442                 if (custom_scan->callback != NULL)
 443                         /*
 444                          * the callback should update devargs list
 445                          * by calling rte_devargs_insert() with
 446                          *     devargs.bus = rte_bus_find_by_name("vdev");
 447                          *     devargs.type = RTE_DEVTYPE_VIRTUAL;
 448                          *     devargs.policy = RTE_DEV_WHITELISTED;
 449                          */
 450                         custom_scan->callback(custom_scan->user_arg);
 451         }
 452         rte_spinlock_unlock(&vdev_custom_scan_lock);
 453
 454         /* for virtual devices we scan the devargs_list populated via cmdline */
 455         RTE_EAL_DEVARGS_FOREACH("vdev", devargs) {
 456
 457                 dev = calloc(1, sizeof(*dev));
 458                 if (!dev)
 459                         return -1;
 460
 461                 rte_spinlock_recursive_lock(&vdev_device_list_lock);
 462
 463                 if (find_vdev(devargs->name)) {
 464                         rte_spinlock_recursive_unlock(&vdev_device_list_lock);
 465                         free(dev);
 466                         continue;
 467                 }
 468
 469                 dev->device.bus = &rte_vdev_bus;
 470                 dev->device.devargs = devargs;
 471                 dev->device.numa_node = SOCKET_ID_ANY;
 472                 dev->device.name = devargs->name;
 473
 474                 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next);
 475
 476                 rte_spinlock_recursive_unlock(&vdev_device_list_lock);
 477         }
 478
 479         return 0;
 480 }
 481
 482 static int
 483 vdev_probe(void)
 484 {
 485         struct rte_vdev_device *dev;
 486         int r, ret = 0;
 487
 488         /* call the init function for each virtual device */
 489         TAILQ_FOREACH(dev, &vdev_device_list, next) {
 490                 /* we don't use the vdev lock here, as it's only used in DPDK
 491                  * initialization; and we don't want to hold such a lock when
 492                  * we call each driver probe.
 493                  */
 494
 495                 r = vdev_probe_all_drivers(dev);
 496                 if (r != 0) {
 497                         if (r == -EEXIST)
 498                                 continue;
 499                         VDEV_LOG(ERR, "failed to initialize %s device",
 500                                 rte_vdev_device_name(dev));
 501                         ret = -1;
 502                 }
 503         }
 504
 505         return ret;
 506 }
 507
 508 struct rte_device *
 509 rte_vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
 510                      const void *data)
 511 {
 512         const struct rte_vdev_device *vstart;
 513         struct rte_vdev_device *dev;
 514
 515         rte_spinlock_recursive_lock(&vdev_device_list_lock);
 516         if (start != NULL) {
 517                 vstart = RTE_DEV_TO_VDEV_CONST(start);
 518                 dev = TAILQ_NEXT(vstart, next);
 519         } else {
 520                 dev = TAILQ_FIRST(&vdev_device_list);
 521         }
 522         while (dev != NULL) {
 523                 if (cmp(&dev->device, data) == 0)
 524                         break;
 525                 dev = TAILQ_NEXT(dev, next);
 526         }
 527         rte_spinlock_recursive_unlock(&vdev_device_list_lock);
 528
 529         return dev ? &dev->device : NULL;
 530 }
 531
 532 static int
 533 vdev_plug(struct rte_device *dev)
 534 {
 535         return vdev_probe_all_drivers(RTE_DEV_TO_VDEV(dev));
 536 }
 537
 538 static int
 539 vdev_unplug(struct rte_device *dev)
 540 {
 541         return rte_vdev_uninit(dev->name);
 542 }
 543
 544 static struct rte_bus rte_vdev_bus = {
 545         .scan = vdev_scan,
 546         .probe = vdev_probe,
 547         .find_device = rte_vdev_find_device,
 548         .plug = vdev_plug,
 549         .unplug = vdev_unplug,
 550         .parse = vdev_parse,
 551         .dev_iterate = rte_vdev_dev_iterate,
 552 };
 553
 554 RTE_REGISTER_BUS(vdev, rte_vdev_bus);
 555
 556 RTE_INIT(vdev_init_log)
 557 {
 558         vdev_logtype_bus = rte_log_register("bus.vdev");
 559         if (vdev_logtype_bus >= 0)
 560                 rte_log_set_level(vdev_logtype_bus, RTE_LOG_NOTICE);
 561 }