ceph/src/spdk/dpdk/examples/vhost_crypto/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017-2018 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <unistd.h>
   8 #include <stdbool.h>
   9 #include <assert.h>
  10 #include <getopt.h>
  11
  12 #include <rte_malloc.h>
  13 #include <rte_cycles.h>
  14 #include <rte_vhost.h>
  15 #include <rte_cryptodev.h>
  16 #include <rte_vhost_crypto.h>
  17
  18 #include <cmdline_rdline.h>
  19 #include <cmdline_parse.h>
  20 #include <cmdline_parse_string.h>
  21 #include <cmdline.h>
  22
  23 #define NB_VIRTIO_QUEUES                (1)
  24 #define MAX_PKT_BURST                   (64)
  25 #define MAX_IV_LEN                      (32)
  26 #define NB_MEMPOOL_OBJS                 (8192)
  27 #define NB_CRYPTO_DESCRIPTORS           (4096)
  28 #define NB_CACHE_OBJS                   (128)
  29 #define SESSION_MAP_ENTRIES             (1024)
  30 #define REFRESH_TIME_SEC                (3)
  31
  32 #define MAX_NB_SOCKETS                  (32)
  33 #define DEF_SOCKET_FILE                 "/tmp/vhost_crypto1.socket"
  34
  35 struct vhost_crypto_options {
  36         char *socket_files[MAX_NB_SOCKETS];
  37         uint32_t nb_sockets;
  38         uint8_t cid;
  39         uint16_t qid;
  40         uint32_t zero_copy;
  41         uint32_t guest_polling;
  42 } options;
  43
  44 struct vhost_crypto_info {
  45         int vids[MAX_NB_SOCKETS];
  46         struct rte_mempool *sess_pool;
  47         struct rte_mempool *cop_pool;
  48         uint32_t lcore_id;
  49         uint8_t cid;
  50         uint32_t qid;
  51         uint32_t nb_vids;
  52         volatile uint32_t initialized[MAX_NB_SOCKETS];
  53
  54 } info;
  55
  56 #define SOCKET_FILE_KEYWORD     "socket-file"
  57 #define CRYPTODEV_ID_KEYWORD    "cdev-id"
  58 #define CRYPTODEV_QUEUE_KEYWORD "cdev-queue-id"
  59 #define ZERO_COPY_KEYWORD       "zero-copy"
  60 #define POLLING_KEYWORD         "guest-polling"
  61
  62 uint64_t vhost_cycles[2], last_v_cycles[2];
  63 uint64_t outpkt_amount;
  64
  65 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */
  66 static int
  67 parse_socket_arg(char *arg)
  68 {
  69         uint32_t nb_sockets = options.nb_sockets;
  70         size_t len = strlen(arg);
  71
  72         if (nb_sockets >= MAX_NB_SOCKETS) {
  73                 RTE_LOG(ERR, USER1, "Too many socket files!\n");
  74                 return -ENOMEM;
  75         }
  76
  77         options.socket_files[nb_sockets] = rte_malloc(NULL, len, 0);
  78         if (!options.socket_files[nb_sockets]) {
  79                 RTE_LOG(ERR, USER1, "Insufficient memory\n");
  80                 return -ENOMEM;
  81         }
  82
  83         rte_memcpy(options.socket_files[nb_sockets], arg, len);
  84
  85         options.nb_sockets++;
  86
  87         return 0;
  88 }
  89
  90 static int
  91 parse_cryptodev_id(const char *q_arg)
  92 {
  93         char *end = NULL;
  94         uint64_t pm;
  95
  96         /* parse decimal string */
  97         pm = strtoul(q_arg, &end, 10);
  98         if (pm > rte_cryptodev_count()) {
  99                 RTE_LOG(ERR, USER1, "Invalid Cryptodev ID %s\n", q_arg);
 100                 return -1;
 101         }
 102
 103         options.cid = (uint8_t)pm;
 104
 105         return 0;
 106 }
 107
 108 static int
 109 parse_cdev_queue_id(const char *q_arg)
 110 {
 111         char *end = NULL;
 112         uint64_t pm;
 113
 114         /* parse decimal string */
 115         pm = strtoul(q_arg, &end, 10);
 116         if (pm == UINT64_MAX) {
 117                 RTE_LOG(ERR, USER1, "Invalid Cryptodev Queue ID %s\n", q_arg);
 118                 return -1;
 119         }
 120
 121         options.qid = (uint16_t)pm;
 122
 123         return 0;
 124 }
 125
 126 static void
 127 vhost_crypto_usage(const char *prgname)
 128 {
 129         printf("%s [EAL options] --\n"
 130                 "  --%s SOCKET-FILE-PATH\n"
 131                 "  --%s CRYPTODEV_ID: crypto device id\n"
 132                 "  --%s CDEV_QUEUE_ID: crypto device queue id\n"
 133                 "  --%s: zero copy\n"
 134                 "  --%s: guest polling\n",
 135                 prgname, SOCKET_FILE_KEYWORD, CRYPTODEV_ID_KEYWORD,
 136                 CRYPTODEV_QUEUE_KEYWORD, ZERO_COPY_KEYWORD, POLLING_KEYWORD);
 137 }
 138
 139 static int
 140 vhost_crypto_parse_args(int argc, char **argv)
 141 {
 142         int opt, ret;
 143         char *prgname = argv[0];
 144         char **argvopt;
 145         int option_index;
 146         struct option lgopts[] = {
 147                         {SOCKET_FILE_KEYWORD, required_argument, 0, 0},
 148                         {CRYPTODEV_ID_KEYWORD, required_argument, 0, 0},
 149                         {CRYPTODEV_QUEUE_KEYWORD, required_argument, 0, 0},
 150                         {ZERO_COPY_KEYWORD, no_argument, 0, 0},
 151                         {POLLING_KEYWORD, no_argument, 0, 0},
 152                         {NULL, 0, 0, 0}
 153         };
 154
 155         options.cid = 0;
 156         options.qid = 0;
 157         options.nb_sockets = 0;
 158         options.guest_polling = 0;
 159         options.zero_copy = RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE;
 160
 161         argvopt = argv;
 162
 163         while ((opt = getopt_long(argc, argvopt, "s:",
 164                                   lgopts, &option_index)) != EOF) {
 165
 166                 switch (opt) {
 167                 case 0:
 168                         if (strcmp(lgopts[option_index].name,
 169                                         SOCKET_FILE_KEYWORD) == 0) {
 170                                 ret = parse_socket_arg(optarg);
 171                                 if (ret < 0) {
 172                                         vhost_crypto_usage(prgname);
 173                                         return ret;
 174                                 }
 175                         } else if (strcmp(lgopts[option_index].name,
 176                                         CRYPTODEV_ID_KEYWORD) == 0) {
 177                                 ret = parse_cryptodev_id(optarg);
 178                                 if (ret < 0) {
 179                                         vhost_crypto_usage(prgname);
 180                                         return ret;
 181                                 }
 182                         } else if (strcmp(lgopts[option_index].name,
 183                                         CRYPTODEV_QUEUE_KEYWORD) == 0) {
 184                                 ret = parse_cdev_queue_id(optarg);
 185                                 if (ret < 0) {
 186                                         vhost_crypto_usage(prgname);
 187                                         return ret;
 188                                 }
 189                         } else if (strcmp(lgopts[option_index].name,
 190                                         ZERO_COPY_KEYWORD) == 0) {
 191                                 options.zero_copy =
 192                                         RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE;
 193                         } else if (strcmp(lgopts[option_index].name,
 194                                         POLLING_KEYWORD) == 0) {
 195                                 options.guest_polling = 1;
 196                         } else {
 197                                 vhost_crypto_usage(prgname);
 198                                 return -EINVAL;
 199                         }
 200                         break;
 201                 default:
 202                         return -1;
 203                 }
 204         }
 205
 206         if (options.nb_sockets == 0) {
 207                 options.socket_files[0] = strdup(DEF_SOCKET_FILE);
 208                 options.nb_sockets = 1;
 209                 RTE_LOG(INFO, USER1,
 210                                 "VHOST-CRYPTO: use default socket file %s\n",
 211                                 DEF_SOCKET_FILE);
 212         }
 213
 214         return 0;
 215 }
 216
 217 static int
 218 new_device(int vid)
 219 {
 220         char path[PATH_MAX];
 221         uint32_t idx, i;
 222         int ret;
 223
 224         ret = rte_vhost_get_ifname(vid, path, PATH_MAX);
 225         if (ret) {
 226                 RTE_LOG(ERR, USER1, "Cannot find matched socket\n");
 227                 return ret;
 228         }
 229
 230         for (idx = 0; idx < options.nb_sockets; idx++) {
 231                 if (strcmp(path, options.socket_files[idx]) == 0)
 232                         break;
 233         }
 234
 235         if (idx == options.nb_sockets) {
 236                 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n");
 237                 return -ENOENT;
 238         }
 239
 240         for (i = 0; i < 2; i++) {
 241                 vhost_cycles[i] = 0;
 242                 last_v_cycles[i] = 0;
 243         }
 244
 245         ret = rte_vhost_crypto_create(vid, info.cid, info.sess_pool,
 246                         rte_lcore_to_socket_id(info.lcore_id));
 247         if (ret) {
 248                 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n");
 249                 return ret;
 250         }
 251
 252         ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy);
 253         if (ret) {
 254                 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n",
 255                                 options.zero_copy == 1 ? "enable" : "disable");
 256                 return ret;
 257         }
 258
 259         info.vids[idx] = vid;
 260         info.initialized[idx] = 1;
 261
 262         rte_wmb();
 263
 264         RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path,
 265                         vid);
 266         return 0;
 267 }
 268
 269 static void
 270 destroy_device(int vid)
 271 {
 272         uint32_t i;
 273
 274         for (i = 0; i < info.nb_vids; i++) {
 275                 if (vid == info.vids[i])
 276                         break;
 277         }
 278
 279         if (i == info.nb_vids) {
 280                 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n");
 281                 return;
 282         }
 283
 284         info.initialized[i] = 0;
 285
 286         rte_wmb();
 287
 288         rte_vhost_crypto_free(vid);
 289
 290         RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid);
 291 }
 292
 293 static const struct vhost_device_ops virtio_crypto_device_ops = {
 294         .new_device =  new_device,
 295         .destroy_device = destroy_device,
 296 };
 297
 298 __attribute__((unused))
 299 static void clrscr(void)
 300 {
 301         system("@cls||clear");
 302 }
 303
 304 static int
 305 vhost_crypto_worker(__rte_unused void *arg)
 306 {
 307         struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
 308         struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
 309         uint32_t nb_inflight_ops = 0;
 310         uint16_t nb_callfds;
 311         int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS];
 312         uint32_t lcore_id = rte_lcore_id();
 313         uint32_t burst_size = MAX_PKT_BURST;
 314         uint32_t i, j, k;
 315         uint32_t to_fetch, fetched;
 316         uint64_t t_start, t_end, interval;
 317
 318         int ret = 0;
 319
 320         RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id);
 321
 322         for (i = 0; i < NB_VIRTIO_QUEUES; i++) {
 323                 if (rte_crypto_op_bulk_alloc(info.cop_pool,
 324                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i],
 325                                 burst_size) < burst_size) {
 326                         RTE_LOG(ERR, USER1, "Failed to alloc cops\n");
 327                         ret = -1;
 328                         goto exit;
 329                 }
 330         }
 331
 332         while (1) {
 333                 for (i = 0; i < info.nb_vids; i++) {
 334                         if (unlikely(info.initialized[i] == 0))
 335                                 continue;
 336
 337                         for (j = 0; j < NB_VIRTIO_QUEUES; j++) {
 338                                 t_start = rte_rdtsc_precise();
 339
 340                                 to_fetch = RTE_MIN(burst_size,
 341                                                 (NB_CRYPTO_DESCRIPTORS -
 342                                                 nb_inflight_ops));
 343                                 fetched = rte_vhost_crypto_fetch_requests(
 344                                                 info.vids[i], j, ops[j],
 345                                                 to_fetch);
 346                                 nb_inflight_ops += rte_cryptodev_enqueue_burst(
 347                                                 info.cid, info.qid, ops[j],
 348                                                 fetched);
 349                                 if (unlikely(rte_crypto_op_bulk_alloc(
 350                                                 info.cop_pool,
 351                                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 352                                                 ops[j], fetched) < fetched)) {
 353                                         RTE_LOG(ERR, USER1, "Failed realloc\n");
 354                                         return -1;
 355                                 }
 356                                 t_end = rte_rdtsc_precise();
 357                                 interval = t_end - t_start;
 358
 359                                 vhost_cycles[fetched > 0] += interval;
 360
 361                                 t_start = t_end;
 362                                 fetched = rte_cryptodev_dequeue_burst(
 363                                                 info.cid, info.qid,
 364                                                 ops_deq[j], RTE_MIN(burst_size,
 365                                                 nb_inflight_ops));
 366                                 fetched = rte_vhost_crypto_finalize_requests(
 367                                                 ops_deq[j], fetched, callfds,
 368                                                 &nb_callfds);
 369
 370                                 nb_inflight_ops -= fetched;
 371                                 outpkt_amount += fetched;
 372
 373                                 if (!options.guest_polling) {
 374                                         for (k = 0; k < nb_callfds; k++)
 375                                                 eventfd_write(callfds[k],
 376                                                                 (eventfd_t)1);
 377                                 }
 378
 379                                 rte_mempool_put_bulk(info.cop_pool,
 380                                                 (void **)ops_deq[j], fetched);
 381                                 interval = rte_rdtsc_precise() - t_start;
 382
 383                                 vhost_cycles[fetched > 0] += interval;
 384                         }
 385                 }
 386         }
 387 exit:
 388         return ret;
 389 }
 390
 391
 392 static void
 393 unregister_drivers(int socket_num)
 394 {
 395         int ret;
 396
 397         ret = rte_vhost_driver_unregister(options.socket_files[socket_num]);
 398         if (ret != 0)
 399                 RTE_LOG(ERR, USER1,
 400                         "Fail to unregister vhost driver for %s.\n",
 401                         options.socket_files[socket_num]);
 402 }
 403
 404 int
 405 main(int argc, char *argv[])
 406 {
 407         struct rte_cryptodev_qp_conf qp_conf = {NB_CRYPTO_DESCRIPTORS};
 408         struct rte_cryptodev_config config;
 409         struct rte_cryptodev_info dev_info;
 410         uint32_t cryptodev_id;
 411         uint32_t worker_lcore;
 412         char name[128];
 413         uint32_t i = 0;
 414         int ret;
 415
 416         ret = rte_eal_init(argc, argv);
 417         if (ret < 0)
 418                 return -1;
 419         argc -= ret;
 420         argv += ret;
 421
 422         ret = vhost_crypto_parse_args(argc, argv);
 423         if (ret < 0)
 424                 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n");
 425
 426         info.cid = options.cid;
 427         info.qid = options.qid;
 428
 429         worker_lcore = rte_get_next_lcore(0, 1, 0);
 430         if (worker_lcore == RTE_MAX_LCORE)
 431                 rte_exit(EXIT_FAILURE, "Not enough lcore\n");
 432
 433         cryptodev_id = info.cid;
 434         rte_cryptodev_info_get(cryptodev_id, &dev_info);
 435         if (dev_info.max_nb_queue_pairs < info.qid + 1) {
 436                 RTE_LOG(ERR, USER1, "Number of queues cannot over %u",
 437                                 dev_info.max_nb_queue_pairs);
 438                 goto error_exit;
 439         }
 440
 441         config.nb_queue_pairs = dev_info.max_nb_queue_pairs;
 442         config.socket_id = rte_lcore_to_socket_id(worker_lcore);
 443
 444         ret = rte_cryptodev_configure(cryptodev_id, &config);
 445         if (ret < 0) {
 446                 RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u",
 447                                 cryptodev_id);
 448                 goto error_exit;
 449         }
 450
 451         snprintf(name, 127, "SESS_POOL_%u", worker_lcore);
 452         info.sess_pool = rte_mempool_create(name, SESSION_MAP_ENTRIES,
 453                         rte_cryptodev_sym_get_private_session_size(
 454                         cryptodev_id), 64, 0, NULL, NULL, NULL, NULL,
 455                         rte_lcore_to_socket_id(worker_lcore), 0);
 456         if (!info.sess_pool) {
 457                 RTE_LOG(ERR, USER1, "Failed to create mempool");
 458                 goto error_exit;
 459         }
 460
 461         snprintf(name, 127, "COPPOOL_%u", worker_lcore);
 462         info.cop_pool = rte_crypto_op_pool_create(name,
 463                         RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS,
 464                         NB_CACHE_OBJS, 0, rte_lcore_to_socket_id(worker_lcore));
 465
 466         if (!info.cop_pool) {
 467                 RTE_LOG(ERR, USER1, "Lcore %u failed to create crypto pool",
 468                                 worker_lcore);
 469                 ret = -1;
 470                 goto error_exit;
 471         }
 472
 473         info.nb_vids = options.nb_sockets;
 474         for (i = 0; i < MAX_NB_SOCKETS; i++)
 475                 info.vids[i] = -1;
 476
 477         for (i = 0; i < dev_info.max_nb_queue_pairs; i++) {
 478                 ret = rte_cryptodev_queue_pair_setup(cryptodev_id, i,
 479                                 &qp_conf, rte_lcore_to_socket_id(worker_lcore),
 480                                 info.sess_pool);
 481                 if (ret < 0) {
 482                         RTE_LOG(ERR, USER1, "Failed to configure qp %u\n",
 483                                         info.cid);
 484                         goto error_exit;
 485                 }
 486         }
 487
 488         ret = rte_cryptodev_start(cryptodev_id);
 489         if (ret < 0) {
 490                 RTE_LOG(ERR, USER1, "Failed to start cryptodev %u\n", info.cid);
 491                 goto error_exit;
 492         }
 493
 494         info.cid = cryptodev_id;
 495         info.lcore_id = worker_lcore;
 496
 497         if (rte_eal_remote_launch(vhost_crypto_worker, NULL, worker_lcore)
 498                         < 0) {
 499                 RTE_LOG(ERR, USER1, "Failed to start worker lcore");
 500                 goto error_exit;
 501         }
 502
 503         for (i = 0; i < options.nb_sockets; i++) {
 504                 if (rte_vhost_driver_register(options.socket_files[i],
 505                                 RTE_VHOST_USER_DEQUEUE_ZERO_COPY) < 0) {
 506                         RTE_LOG(ERR, USER1, "socket %s already exists\n",
 507                                         options.socket_files[i]);
 508                         goto error_exit;
 509                 }
 510
 511                 rte_vhost_driver_callback_register(options.socket_files[i],
 512                                 &virtio_crypto_device_ops);
 513
 514                 if (rte_vhost_driver_start(options.socket_files[i]) < 0) {
 515                         RTE_LOG(ERR, USER1, "failed to start vhost driver.\n");
 516                         goto error_exit;
 517                 }
 518         }
 519
 520         RTE_LCORE_FOREACH(worker_lcore)
 521                 rte_eal_wait_lcore(worker_lcore);
 522
 523         rte_mempool_free(info.sess_pool);
 524         rte_mempool_free(info.cop_pool);
 525
 526         return 0;
 527
 528 error_exit:
 529         for (i = 0; i < options.nb_sockets; i++)
 530                 unregister_drivers(i);
 531
 532         rte_mempool_free(info.cop_pool);
 533         rte_mempool_free(info.sess_pool);
 534
 535         return -1;
 536 }