ceph/src/seastar/dpdk/examples/vhost_crypto/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017-2018 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <string.h>
   8 #include <unistd.h>
   9 #include <stdbool.h>
  10 #include <assert.h>
  11 #include <getopt.h>
  12
  13 #include <rte_malloc.h>
  14 #include <rte_cycles.h>
  15 #include <rte_vhost.h>
  16 #include <rte_cryptodev.h>
  17 #include <rte_vhost_crypto.h>
  18 #include <rte_string_fns.h>
  19
  20 #include <cmdline_rdline.h>
  21 #include <cmdline_parse.h>
  22 #include <cmdline_parse_string.h>
  23 #include <cmdline.h>
  24
  25 #define NB_VIRTIO_QUEUES                (1)
  26 #define MAX_PKT_BURST                   (64)
  27 #define MAX_IV_LEN                      (32)
  28 #define NB_MEMPOOL_OBJS                 (8192)
  29 #define NB_CRYPTO_DESCRIPTORS           (4096)
  30 #define NB_CACHE_OBJS                   (128)
  31 #define SESSION_MAP_ENTRIES             (1024)
  32 #define REFRESH_TIME_SEC                (3)
  33
  34 #define MAX_NB_SOCKETS                  (4)
  35 #define MAX_NB_WORKER_CORES             (16)
  36
  37 struct lcore_option {
  38         uint32_t lcore_id;
  39         char *socket_files[MAX_NB_SOCKETS];
  40         uint32_t nb_sockets;
  41         uint8_t cid;
  42         uint16_t qid;
  43 };
  44
  45 struct vhost_crypto_info {
  46         int vids[MAX_NB_SOCKETS];
  47         uint32_t nb_vids;
  48         struct rte_mempool *sess_pool;
  49         struct rte_mempool *sess_priv_pool;
  50         struct rte_mempool *cop_pool;
  51         uint8_t cid;
  52         uint32_t qid;
  53         uint32_t nb_inflight_ops;
  54         volatile uint32_t initialized[MAX_NB_SOCKETS];
  55 } __rte_cache_aligned;
  56
  57 struct vhost_crypto_options {
  58         struct lcore_option los[MAX_NB_WORKER_CORES];
  59         struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES];
  60         uint32_t nb_los;
  61         uint32_t zero_copy;
  62         uint32_t guest_polling;
  63 } options;
  64
  65 #define CONFIG_KEYWORD          "config"
  66 #define SOCKET_FILE_KEYWORD     "socket-file"
  67 #define ZERO_COPY_KEYWORD       "zero-copy"
  68 #define POLLING_KEYWORD         "guest-polling"
  69
  70 #define NB_SOCKET_FIELDS        (2)
  71
  72 static uint32_t
  73 find_lo(uint32_t lcore_id)
  74 {
  75         uint32_t i;
  76
  77         for (i = 0; i < options.nb_los; i++)
  78                 if (options.los[i].lcore_id == lcore_id)
  79                         return i;
  80
  81         return UINT32_MAX;
  82 }
  83
  84 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */
  85 static int
  86 parse_socket_arg(char *arg)
  87 {
  88         uint32_t nb_sockets;
  89         uint32_t lcore_id;
  90         char *str_fld[NB_SOCKET_FIELDS];
  91         struct lcore_option *lo;
  92         uint32_t idx;
  93         char *end;
  94
  95         if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') !=
  96                                 NB_SOCKET_FIELDS) {
  97                 RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg);
  98                 return -EINVAL;
  99         }
 100
 101         errno = 0;
 102         lcore_id = strtoul(str_fld[0], &end, 0);
 103         if (errno != 0 || end == str_fld[0] || lcore_id > 255)
 104                 return -EINVAL;
 105
 106         idx = find_lo(lcore_id);
 107         if (idx == UINT32_MAX) {
 108                 if (options.nb_los == MAX_NB_WORKER_CORES)
 109                         return -ENOMEM;
 110                 lo = &options.los[options.nb_los];
 111                 lo->lcore_id = lcore_id;
 112                 options.nb_los++;
 113         } else
 114                 lo = &options.los[idx];
 115
 116         nb_sockets = lo->nb_sockets;
 117
 118         if (nb_sockets >= MAX_NB_SOCKETS) {
 119                 RTE_LOG(ERR, USER1, "Too many socket files!\n");
 120                 return -ENOMEM;
 121         }
 122
 123         lo->socket_files[nb_sockets] = strdup(str_fld[1]);
 124         if (!lo->socket_files[nb_sockets]) {
 125                 RTE_LOG(ERR, USER1, "Insufficient memory\n");
 126                 return -ENOMEM;
 127         }
 128
 129         lo->nb_sockets++;
 130
 131         return 0;
 132 }
 133
 134 static int
 135 parse_config(char *q_arg)
 136 {
 137         struct lcore_option *lo;
 138         char s[256];
 139         const char *p, *p0 = q_arg;
 140         char *end;
 141         enum fieldnames {
 142                 FLD_LCORE = 0,
 143                 FLD_CID,
 144                 FLD_QID,
 145                 _NUM_FLD
 146         };
 147         uint32_t flds[_NUM_FLD];
 148         char *str_fld[_NUM_FLD];
 149         uint32_t i;
 150         uint32_t size;
 151
 152         while ((p = strchr(p0, '(')) != NULL) {
 153                 ++p;
 154                 p0 = strchr(p, ')');
 155                 if (p0 == NULL)
 156                         return -1;
 157
 158                 size = p0 - p;
 159                 if (size >= sizeof(s))
 160                         return -1;
 161
 162                 snprintf(s, sizeof(s), "%.*s", size, p);
 163                 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
 164                                 _NUM_FLD)
 165                         return -1;
 166                 for (i = 0; i < _NUM_FLD; i++) {
 167                         errno = 0;
 168                         flds[i] = strtoul(str_fld[i], &end, 0);
 169                         if (errno != 0 || end == str_fld[i] || flds[i] > 255)
 170                                 return -EINVAL;
 171                 }
 172
 173                 if (flds[FLD_LCORE] > RTE_MAX_LCORE)
 174                         return -EINVAL;
 175
 176                 i = find_lo(flds[FLD_LCORE]);
 177                 if (i == UINT32_MAX) {
 178                         if (options.nb_los == MAX_NB_WORKER_CORES)
 179                                 return -ENOMEM;
 180                         lo = &options.los[options.nb_los];
 181                         options.nb_los++;
 182                 } else
 183                         lo = &options.los[i];
 184
 185                 lo->lcore_id = flds[FLD_LCORE];
 186                 lo->cid = flds[FLD_CID];
 187                 lo->qid = flds[FLD_QID];
 188         }
 189
 190         return 0;
 191 }
 192
 193 static void
 194 vhost_crypto_usage(const char *prgname)
 195 {
 196         printf("%s [EAL options] --\n"
 197                 "  --%s <lcore>,SOCKET-FILE-PATH\n"
 198                 "  --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]"
 199                 "  --%s: zero copy\n"
 200                 "  --%s: guest polling\n",
 201                 prgname, SOCKET_FILE_KEYWORD, CONFIG_KEYWORD,
 202                 ZERO_COPY_KEYWORD, POLLING_KEYWORD);
 203 }
 204
 205 static int
 206 vhost_crypto_parse_args(int argc, char **argv)
 207 {
 208         int opt, ret;
 209         char *prgname = argv[0];
 210         char **argvopt;
 211         int option_index;
 212         struct option lgopts[] = {
 213                         {SOCKET_FILE_KEYWORD, required_argument, 0, 0},
 214                         {CONFIG_KEYWORD, required_argument, 0, 0},
 215                         {ZERO_COPY_KEYWORD, no_argument, 0, 0},
 216                         {POLLING_KEYWORD, no_argument, 0, 0},
 217                         {NULL, 0, 0, 0}
 218         };
 219
 220         argvopt = argv;
 221
 222         while ((opt = getopt_long(argc, argvopt, "s:",
 223                                   lgopts, &option_index)) != EOF) {
 224
 225                 switch (opt) {
 226                 case 0:
 227                         if (strcmp(lgopts[option_index].name,
 228                                         SOCKET_FILE_KEYWORD) == 0) {
 229                                 ret = parse_socket_arg(optarg);
 230                                 if (ret < 0) {
 231                                         vhost_crypto_usage(prgname);
 232                                         return ret;
 233                                 }
 234                         } else if (strcmp(lgopts[option_index].name,
 235                                         CONFIG_KEYWORD) == 0) {
 236                                 ret = parse_config(optarg);
 237                                 if (ret < 0) {
 238                                         vhost_crypto_usage(prgname);
 239                                         return ret;
 240                                 }
 241                         } else if (strcmp(lgopts[option_index].name,
 242                                         ZERO_COPY_KEYWORD) == 0) {
 243                                 options.zero_copy =
 244                                         RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE;
 245                         } else if (strcmp(lgopts[option_index].name,
 246                                         POLLING_KEYWORD) == 0) {
 247                                 options.guest_polling = 1;
 248                         } else {
 249                                 vhost_crypto_usage(prgname);
 250                                 return -EINVAL;
 251                         }
 252                         break;
 253                 default:
 254                         return -1;
 255                 }
 256         }
 257
 258         return 0;
 259 }
 260
 261 static int
 262 new_device(int vid)
 263 {
 264         struct vhost_crypto_info *info = NULL;
 265         char path[PATH_MAX];
 266         uint32_t i, j;
 267         int ret;
 268
 269         ret = rte_vhost_get_ifname(vid, path, PATH_MAX);
 270         if (ret) {
 271                 RTE_LOG(ERR, USER1, "Cannot find matched socket\n");
 272                 return ret;
 273         }
 274
 275         for (i = 0; i < options.nb_los; i++) {
 276                 for (j = 0; j < options.los[i].nb_sockets; j++) {
 277                         if (strcmp(path, options.los[i].socket_files[j]) == 0) {
 278                                 info = options.infos[i];
 279                                 break;
 280                         }
 281                 }
 282
 283                 if (info)
 284                         break;
 285         }
 286
 287         if (!info) {
 288                 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n");
 289                 return -ENOENT;
 290         }
 291
 292         ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool,
 293                         info->sess_priv_pool,
 294                         rte_lcore_to_socket_id(options.los[i].lcore_id));
 295         if (ret) {
 296                 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n");
 297                 return ret;
 298         }
 299
 300         ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy);
 301         if (ret) {
 302                 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n",
 303                                 options.zero_copy == 1 ? "enable" : "disable");
 304                 return ret;
 305         }
 306
 307         info->vids[j] = vid;
 308         info->initialized[j] = 1;
 309
 310         rte_wmb();
 311
 312         RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path,
 313                         vid);
 314         return 0;
 315 }
 316
 317 static void
 318 destroy_device(int vid)
 319 {
 320         struct vhost_crypto_info *info = NULL;
 321         uint32_t i, j;
 322
 323         for (i = 0; i < options.nb_los; i++) {
 324                 for (j = 0; j < options.los[i].nb_sockets; j++) {
 325                         if (options.infos[i]->vids[j] == vid) {
 326                                 info = options.infos[i];
 327                                 break;
 328                         }
 329                 }
 330                 if (info)
 331                         break;
 332         }
 333
 334         if (!info) {
 335                 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n");
 336                 return;
 337         }
 338
 339         do {
 340
 341         } while (info->nb_inflight_ops);
 342
 343         info->initialized[j] = 0;
 344
 345         rte_wmb();
 346
 347         rte_vhost_crypto_free(vid);
 348
 349         RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid);
 350 }
 351
 352 static const struct vhost_device_ops virtio_crypto_device_ops = {
 353         .new_device =  new_device,
 354         .destroy_device = destroy_device,
 355 };
 356
 357 __attribute__((unused))
 358 static void clrscr(void)
 359 {
 360         system("@cls||clear");
 361 }
 362
 363 static int
 364 vhost_crypto_worker(void *arg)
 365 {
 366         struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
 367         struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
 368         struct vhost_crypto_info *info = arg;
 369         uint16_t nb_callfds;
 370         int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS];
 371         uint32_t lcore_id = rte_lcore_id();
 372         uint32_t burst_size = MAX_PKT_BURST;
 373         uint32_t i, j, k;
 374         uint32_t to_fetch, fetched;
 375
 376         int ret = 0;
 377
 378         RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id);
 379
 380         for (i = 0; i < NB_VIRTIO_QUEUES; i++) {
 381                 if (rte_crypto_op_bulk_alloc(info->cop_pool,
 382                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i],
 383                                 burst_size) < burst_size) {
 384                         RTE_LOG(ERR, USER1, "Failed to alloc cops\n");
 385                         ret = -1;
 386                         goto exit;
 387                 }
 388         }
 389
 390         while (1) {
 391                 for (i = 0; i < info->nb_vids; i++) {
 392                         if (unlikely(info->initialized[i] == 0))
 393                                 continue;
 394
 395                         for (j = 0; j < NB_VIRTIO_QUEUES; j++) {
 396                                 to_fetch = RTE_MIN(burst_size,
 397                                                 (NB_CRYPTO_DESCRIPTORS -
 398                                                 info->nb_inflight_ops));
 399                                 fetched = rte_vhost_crypto_fetch_requests(
 400                                                 info->vids[i], j, ops[j],
 401                                                 to_fetch);
 402                                 info->nb_inflight_ops +=
 403                                                 rte_cryptodev_enqueue_burst(
 404                                                 info->cid, info->qid, ops[j],
 405                                                 fetched);
 406                                 if (unlikely(rte_crypto_op_bulk_alloc(
 407                                                 info->cop_pool,
 408                                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 409                                                 ops[j], fetched) < fetched)) {
 410                                         RTE_LOG(ERR, USER1, "Failed realloc\n");
 411                                         return -1;
 412                                 }
 413
 414                                 fetched = rte_cryptodev_dequeue_burst(
 415                                                 info->cid, info->qid,
 416                                                 ops_deq[j], RTE_MIN(burst_size,
 417                                                 info->nb_inflight_ops));
 418                                 fetched = rte_vhost_crypto_finalize_requests(
 419                                                 ops_deq[j], fetched, callfds,
 420                                                 &nb_callfds);
 421
 422                                 info->nb_inflight_ops -= fetched;
 423
 424                                 if (!options.guest_polling) {
 425                                         for (k = 0; k < nb_callfds; k++)
 426                                                 eventfd_write(callfds[k],
 427                                                                 (eventfd_t)1);
 428                                 }
 429
 430                                 rte_mempool_put_bulk(info->cop_pool,
 431                                                 (void **)ops_deq[j], fetched);
 432                         }
 433                 }
 434         }
 435 exit:
 436         return ret;
 437 }
 438
 439 static void
 440 free_resource(void)
 441 {
 442         uint32_t i, j;
 443
 444         for (i = 0; i < options.nb_los; i++) {
 445                 struct lcore_option *lo = &options.los[i];
 446                 struct vhost_crypto_info *info = options.infos[i];
 447
 448                 if (!info)
 449                         continue;
 450
 451                 rte_mempool_free(info->cop_pool);
 452                 rte_mempool_free(info->sess_pool);
 453                 rte_mempool_free(info->sess_priv_pool);
 454
 455                 for (j = 0; j < lo->nb_sockets; j++) {
 456                         rte_vhost_driver_unregister(lo->socket_files[i]);
 457                         free(lo->socket_files[i]);
 458                 }
 459
 460                 rte_free(info);
 461         }
 462
 463         memset(&options, 0, sizeof(options));
 464 }
 465
 466 int
 467 main(int argc, char *argv[])
 468 {
 469         struct rte_cryptodev_qp_conf qp_conf;
 470         struct rte_cryptodev_config config;
 471         struct rte_cryptodev_info dev_info;
 472         char name[128];
 473         uint32_t i, j, lcore;
 474         int ret;
 475
 476         ret = rte_eal_init(argc, argv);
 477         if (ret < 0)
 478                 return -1;
 479         argc -= ret;
 480         argv += ret;
 481
 482         ret = vhost_crypto_parse_args(argc, argv);
 483         if (ret < 0)
 484                 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n");
 485
 486         for (i = 0; i < options.nb_los; i++) {
 487                 struct lcore_option *lo = &options.los[i];
 488                 struct vhost_crypto_info *info;
 489
 490                 info = rte_zmalloc_socket(NULL, sizeof(*info),
 491                                 RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id(
 492                                                 lo->lcore_id));
 493                 if (!info) {
 494                         ret = -ENOMEM;
 495                         goto error_exit;
 496                 }
 497
 498                 info->cid = lo->cid;
 499                 info->qid = lo->qid;
 500                 info->nb_vids = lo->nb_sockets;
 501
 502                 rte_cryptodev_info_get(info->cid, &dev_info);
 503                 if (options.zero_copy == RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE) {
 504 #define VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD     crypto_aesni_mb
 505 #define VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD    crypto_aesni_gcm
 506                         if (strstr(dev_info.driver_name,
 507                                 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD)) ||
 508                                 strstr(dev_info.driver_name,
 509                                 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD))) {
 510                                 RTE_LOG(ERR, USER1, "Cannot enable zero-copy in %s\n",
 511                                         dev_info.driver_name);
 512                                 ret = -EPERM;
 513                                 goto error_exit;
 514                         }
 515                 }
 516
 517                 if (dev_info.max_nb_queue_pairs < info->qid + 1) {
 518                         RTE_LOG(ERR, USER1, "Number of queues cannot over %u",
 519                                         dev_info.max_nb_queue_pairs);
 520                         goto error_exit;
 521                 }
 522
 523                 config.nb_queue_pairs = dev_info.max_nb_queue_pairs;
 524                 config.socket_id = rte_lcore_to_socket_id(lo->lcore_id);
 525
 526                 ret = rte_cryptodev_configure(info->cid, &config);
 527                 if (ret < 0) {
 528                         RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u",
 529                                         info->cid);
 530                         goto error_exit;
 531                 }
 532
 533                 snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id);
 534                 info->sess_pool = rte_cryptodev_sym_session_pool_create(name,
 535                                 SESSION_MAP_ENTRIES, 0, 0, 0,
 536                                 rte_lcore_to_socket_id(lo->lcore_id));
 537
 538                 snprintf(name, 127, "SESS_POOL_PRIV_%u", lo->lcore_id);
 539                 info->sess_priv_pool = rte_mempool_create(name,
 540                                 SESSION_MAP_ENTRIES,
 541                                 rte_cryptodev_sym_get_private_session_size(
 542                                 info->cid), 64, 0, NULL, NULL, NULL, NULL,
 543                                 rte_lcore_to_socket_id(lo->lcore_id), 0);
 544                 if (!info->sess_priv_pool || !info->sess_pool) {
 545                         RTE_LOG(ERR, USER1, "Failed to create mempool");
 546                         goto error_exit;
 547                 }
 548
 549                 snprintf(name, 127, "COPPOOL_%u", lo->lcore_id);
 550                 info->cop_pool = rte_crypto_op_pool_create(name,
 551                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS,
 552                                 NB_CACHE_OBJS, 0,
 553                                 rte_lcore_to_socket_id(lo->lcore_id));
 554
 555                 if (!info->cop_pool) {
 556                         RTE_LOG(ERR, USER1, "Failed to create crypto pool");
 557                         ret = -ENOMEM;
 558                         goto error_exit;
 559                 }
 560
 561                 options.infos[i] = info;
 562
 563                 qp_conf.nb_descriptors = NB_CRYPTO_DESCRIPTORS;
 564                 qp_conf.mp_session = info->sess_pool;
 565                 qp_conf.mp_session_private = info->sess_priv_pool;
 566
 567                 for (j = 0; j < dev_info.max_nb_queue_pairs; j++) {
 568                         ret = rte_cryptodev_queue_pair_setup(info->cid, j,
 569                                         &qp_conf, rte_lcore_to_socket_id(
 570                                                         lo->lcore_id));
 571                         if (ret < 0) {
 572                                 RTE_LOG(ERR, USER1, "Failed to configure qp\n");
 573                                 goto error_exit;
 574                         }
 575                 }
 576         }
 577
 578         for (i = 0; i < options.nb_los; i++) {
 579                 struct lcore_option *lo = &options.los[i];
 580                 struct vhost_crypto_info *info = options.infos[i];
 581
 582                 ret = rte_cryptodev_start(lo->cid);
 583                 if (ret < 0) {
 584                         RTE_LOG(ERR, USER1, "Failed to start cryptodev\n");
 585                         goto error_exit;
 586                 }
 587
 588                 if (rte_eal_remote_launch(vhost_crypto_worker, info,
 589                                 lo->lcore_id) < 0) {
 590                         RTE_LOG(ERR, USER1, "Failed to start worker lcore");
 591                         goto error_exit;
 592                 }
 593
 594                 for (j = 0; j < lo->nb_sockets; j++) {
 595                         ret = rte_vhost_driver_register(lo->socket_files[j],
 596                                 RTE_VHOST_USER_DEQUEUE_ZERO_COPY);
 597                         if (ret < 0) {
 598                                 RTE_LOG(ERR, USER1, "socket %s already exists\n",
 599                                         lo->socket_files[j]);
 600                                 goto error_exit;
 601                         }
 602
 603                         rte_vhost_driver_callback_register(lo->socket_files[j],
 604                                 &virtio_crypto_device_ops);
 605
 606                         ret = rte_vhost_driver_start(lo->socket_files[j]);
 607                         if (ret < 0)  {
 608                                 RTE_LOG(ERR, USER1, "failed to start vhost.\n");
 609                                 goto error_exit;
 610                         }
 611                 }
 612         }
 613
 614         RTE_LCORE_FOREACH(lcore)
 615                 rte_eal_wait_lcore(lcore);
 616
 617         free_resource();
 618
 619         return 0;
 620
 621 error_exit:
 622
 623         free_resource();
 624
 625         return -1;
 626 }