ceph/src/dpdk/lib/librte_port/rte_port_fd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33 #include <string.h>
  34 #include <stdint.h>
  35 #include <unistd.h>
  36
  37 #include <rte_mbuf.h>
  38 #include <rte_malloc.h>
  39
  40 #include "rte_port_fd.h"
  41
  42 /*
  43  * Port FD Reader
  44  */
  45 #ifdef RTE_PORT_STATS_COLLECT
  46
  47 #define RTE_PORT_FD_READER_STATS_PKTS_IN_ADD(port, val) \
  48         do { port->stats.n_pkts_in += val; } while (0)
  49 #define RTE_PORT_FD_READER_STATS_PKTS_DROP_ADD(port, val) \
  50         do { port->stats.n_pkts_drop += val; } while (0)
  51
  52 #else
  53
  54 #define RTE_PORT_FD_READER_STATS_PKTS_IN_ADD(port, val)
  55 #define RTE_PORT_FD_READER_STATS_PKTS_DROP_ADD(port, val)
  56
  57 #endif
  58
  59 struct rte_port_fd_reader {
  60         struct rte_port_in_stats stats;
  61         int fd;
  62         uint32_t mtu;
  63         struct rte_mempool *mempool;
  64 };
  65
  66 static void *
  67 rte_port_fd_reader_create(void *params, int socket_id)
  68 {
  69         struct rte_port_fd_reader_params *conf =
  70                         (struct rte_port_fd_reader_params *) params;
  71         struct rte_port_fd_reader *port;
  72
  73         /* Check input parameters */
  74         if (conf == NULL) {
  75                 RTE_LOG(ERR, PORT, "%s: params is NULL\n", __func__);
  76                 return NULL;
  77         }
  78         if (conf->fd < 0) {
  79                 RTE_LOG(ERR, PORT, "%s: Invalid file descriptor\n", __func__);
  80                 return NULL;
  81         }
  82         if (conf->mtu == 0) {
  83                 RTE_LOG(ERR, PORT, "%s: Invalid MTU\n", __func__);
  84                 return NULL;
  85         }
  86         if (conf->mempool == NULL) {
  87                 RTE_LOG(ERR, PORT, "%s: Invalid mempool\n", __func__);
  88                 return NULL;
  89         }
  90
  91         /* Memory allocation */
  92         port = rte_zmalloc_socket("PORT", sizeof(*port),
  93                         RTE_CACHE_LINE_SIZE, socket_id);
  94         if (port == NULL) {
  95                 RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
  96                 return NULL;
  97         }
  98
  99         /* Initialization */
 100         port->fd = conf->fd;
 101         port->mtu = conf->mtu;
 102         port->mempool = conf->mempool;
 103
 104         return port;
 105 }
 106
 107 static int
 108 rte_port_fd_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts)
 109 {
 110         struct rte_port_fd_reader *p = (struct rte_port_fd_reader *) port;
 111         uint32_t i;
 112
 113         if (rte_mempool_get_bulk(p->mempool, (void **) pkts, n_pkts) != 0)
 114                 return 0;
 115
 116         for (i = 0; i < n_pkts; i++) {
 117                 rte_mbuf_refcnt_set(pkts[i], 1);
 118                 rte_pktmbuf_reset(pkts[i]);
 119         }
 120
 121         for (i = 0; i < n_pkts; i++) {
 122                 struct rte_mbuf *pkt = pkts[i];
 123                 void *pkt_data = rte_pktmbuf_mtod(pkt, void *);
 124                 ssize_t n_bytes;
 125
 126                 n_bytes = read(p->fd, pkt_data, (size_t) p->mtu);
 127                 if (n_bytes <= 0)
 128                         break;
 129
 130                 pkt->data_len = n_bytes;
 131                 pkt->pkt_len = n_bytes;
 132         }
 133
 134         for ( ; i < n_pkts; i++)
 135                 rte_pktmbuf_free(pkts[i]);
 136
 137         RTE_PORT_FD_READER_STATS_PKTS_IN_ADD(p, i);
 138
 139         return n_pkts;
 140 }
 141
 142 static int
 143 rte_port_fd_reader_free(void *port)
 144 {
 145         if (port == NULL) {
 146                 RTE_LOG(ERR, PORT, "%s: port is NULL\n", __func__);
 147                 return -EINVAL;
 148         }
 149
 150         rte_free(port);
 151
 152         return 0;
 153 }
 154
 155 static int rte_port_fd_reader_stats_read(void *port,
 156                 struct rte_port_in_stats *stats, int clear)
 157 {
 158         struct rte_port_fd_reader *p =
 159                         (struct rte_port_fd_reader *) port;
 160
 161         if (stats != NULL)
 162                 memcpy(stats, &p->stats, sizeof(p->stats));
 163
 164         if (clear)
 165                 memset(&p->stats, 0, sizeof(p->stats));
 166
 167         return 0;
 168 }
 169
 170 /*
 171  * Port FD Writer
 172  */
 173 #ifdef RTE_PORT_STATS_COLLECT
 174
 175 #define RTE_PORT_FD_WRITER_STATS_PKTS_IN_ADD(port, val) \
 176         do { port->stats.n_pkts_in += val; } while (0)
 177 #define RTE_PORT_FD_WRITER_STATS_PKTS_DROP_ADD(port, val) \
 178         do { port->stats.n_pkts_drop += val; } while (0)
 179
 180 #else
 181
 182 #define RTE_PORT_FD_WRITER_STATS_PKTS_IN_ADD(port, val)
 183 #define RTE_PORT_FD_WRITER_STATS_PKTS_DROP_ADD(port, val)
 184
 185 #endif
 186
 187 struct rte_port_fd_writer {
 188         struct rte_port_out_stats stats;
 189
 190         struct rte_mbuf *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX];
 191         uint32_t tx_burst_sz;
 192         uint16_t tx_buf_count;
 193         uint32_t fd;
 194 };
 195
 196 static void *
 197 rte_port_fd_writer_create(void *params, int socket_id)
 198 {
 199         struct rte_port_fd_writer_params *conf =
 200                 (struct rte_port_fd_writer_params *) params;
 201         struct rte_port_fd_writer *port;
 202
 203         /* Check input parameters */
 204         if ((conf == NULL) ||
 205                 (conf->tx_burst_sz == 0) ||
 206                 (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) ||
 207                 (!rte_is_power_of_2(conf->tx_burst_sz))) {
 208                 RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__);
 209                 return NULL;
 210         }
 211
 212         /* Memory allocation */
 213         port = rte_zmalloc_socket("PORT", sizeof(*port),
 214                 RTE_CACHE_LINE_SIZE, socket_id);
 215         if (port == NULL) {
 216                 RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
 217                 return NULL;
 218         }
 219
 220         /* Initialization */
 221         port->fd = conf->fd;
 222         port->tx_burst_sz = conf->tx_burst_sz;
 223         port->tx_buf_count = 0;
 224
 225         return port;
 226 }
 227
 228 static inline void
 229 send_burst(struct rte_port_fd_writer *p)
 230 {
 231         uint32_t i;
 232
 233         for (i = 0; i < p->tx_buf_count; i++) {
 234                 struct rte_mbuf *pkt = p->tx_buf[i];
 235                 void *pkt_data = rte_pktmbuf_mtod(pkt, void*);
 236                 size_t n_bytes = rte_pktmbuf_data_len(pkt);
 237                 ssize_t ret;
 238
 239                 ret = write(p->fd, pkt_data, n_bytes);
 240                 if (ret < 0)
 241                         break;
 242         }
 243
 244         RTE_PORT_FD_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - i);
 245
 246         for (i = 0; i < p->tx_buf_count; i++)
 247                 rte_pktmbuf_free(p->tx_buf[i]);
 248
 249         p->tx_buf_count = 0;
 250 }
 251
 252 static int
 253 rte_port_fd_writer_tx(void *port, struct rte_mbuf *pkt)
 254 {
 255         struct rte_port_fd_writer *p =
 256                 (struct rte_port_fd_writer *) port;
 257
 258         p->tx_buf[p->tx_buf_count++] = pkt;
 259         RTE_PORT_FD_WRITER_STATS_PKTS_IN_ADD(p, 1);
 260         if (p->tx_buf_count >= p->tx_burst_sz)
 261                 send_burst(p);
 262
 263         return 0;
 264 }
 265
 266 static int
 267 rte_port_fd_writer_tx_bulk(void *port,
 268         struct rte_mbuf **pkts,
 269         uint64_t pkts_mask)
 270 {
 271         struct rte_port_fd_writer *p =
 272                 (struct rte_port_fd_writer *) port;
 273         uint32_t tx_buf_count = p->tx_buf_count;
 274
 275         if ((pkts_mask & (pkts_mask + 1)) == 0) {
 276                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
 277                 uint32_t i;
 278
 279                 for (i = 0; i < n_pkts; i++)
 280                         p->tx_buf[tx_buf_count++] = pkts[i];
 281                 RTE_PORT_FD_WRITER_STATS_PKTS_IN_ADD(p, n_pkts);
 282         } else
 283                 for ( ; pkts_mask; ) {
 284                         uint32_t pkt_index = __builtin_ctzll(pkts_mask);
 285                         uint64_t pkt_mask = 1LLU << pkt_index;
 286                         struct rte_mbuf *pkt = pkts[pkt_index];
 287
 288                         p->tx_buf[tx_buf_count++] = pkt;
 289                         RTE_PORT_FD_WRITER_STATS_PKTS_IN_ADD(p, 1);
 290                         pkts_mask &= ~pkt_mask;
 291                 }
 292
 293         p->tx_buf_count = tx_buf_count;
 294         if (tx_buf_count >= p->tx_burst_sz)
 295                 send_burst(p);
 296
 297         return 0;
 298 }
 299
 300 static int
 301 rte_port_fd_writer_flush(void *port)
 302 {
 303         struct rte_port_fd_writer *p =
 304                 (struct rte_port_fd_writer *) port;
 305
 306         if (p->tx_buf_count > 0)
 307                 send_burst(p);
 308
 309         return 0;
 310 }
 311
 312 static int
 313 rte_port_fd_writer_free(void *port)
 314 {
 315         if (port == NULL) {
 316                 RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__);
 317                 return -EINVAL;
 318         }
 319
 320         rte_port_fd_writer_flush(port);
 321         rte_free(port);
 322
 323         return 0;
 324 }
 325
 326 static int rte_port_fd_writer_stats_read(void *port,
 327                 struct rte_port_out_stats *stats, int clear)
 328 {
 329         struct rte_port_fd_writer *p =
 330                 (struct rte_port_fd_writer *) port;
 331
 332         if (stats != NULL)
 333                 memcpy(stats, &p->stats, sizeof(p->stats));
 334
 335         if (clear)
 336                 memset(&p->stats, 0, sizeof(p->stats));
 337
 338         return 0;
 339 }
 340
 341 /*
 342  * Port FD Writer Nodrop
 343  */
 344 #ifdef RTE_PORT_STATS_COLLECT
 345
 346 #define RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) \
 347         do { port->stats.n_pkts_in += val; } while (0)
 348 #define RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) \
 349         do { port->stats.n_pkts_drop += val; } while (0)
 350
 351 #else
 352
 353 #define RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val)
 354 #define RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val)
 355
 356 #endif
 357
 358 struct rte_port_fd_writer_nodrop {
 359         struct rte_port_out_stats stats;
 360
 361         struct rte_mbuf *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX];
 362         uint32_t tx_burst_sz;
 363         uint16_t tx_buf_count;
 364         uint64_t n_retries;
 365         uint32_t fd;
 366 };
 367
 368 static void *
 369 rte_port_fd_writer_nodrop_create(void *params, int socket_id)
 370 {
 371         struct rte_port_fd_writer_nodrop_params *conf =
 372                         (struct rte_port_fd_writer_nodrop_params *) params;
 373         struct rte_port_fd_writer_nodrop *port;
 374
 375         /* Check input parameters */
 376         if ((conf == NULL) ||
 377                 (conf->fd < 0) ||
 378                 (conf->tx_burst_sz == 0) ||
 379                 (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) ||
 380                 (!rte_is_power_of_2(conf->tx_burst_sz))) {
 381                 RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__);
 382                 return NULL;
 383         }
 384
 385         /* Memory allocation */
 386         port = rte_zmalloc_socket("PORT", sizeof(*port),
 387                 RTE_CACHE_LINE_SIZE, socket_id);
 388         if (port == NULL) {
 389                 RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
 390                 return NULL;
 391         }
 392
 393         /* Initialization */
 394         port->fd = conf->fd;
 395         port->tx_burst_sz = conf->tx_burst_sz;
 396         port->tx_buf_count = 0;
 397
 398         /*
 399          * When n_retries is 0 it means that we should wait for every packet to
 400          * send no matter how many retries should it take. To limit number of
 401          * branches in fast path, we use UINT64_MAX instead of branching.
 402          */
 403         port->n_retries = (conf->n_retries == 0) ? UINT64_MAX : conf->n_retries;
 404
 405         return port;
 406 }
 407
 408 static inline void
 409 send_burst_nodrop(struct rte_port_fd_writer_nodrop *p)
 410 {
 411         uint64_t n_retries;
 412         uint32_t i;
 413
 414         n_retries = 0;
 415         for (i = 0; (i < p->tx_buf_count) && (n_retries < p->n_retries); i++) {
 416                 struct rte_mbuf *pkt = p->tx_buf[i];
 417                 void *pkt_data = rte_pktmbuf_mtod(pkt, void*);
 418                 size_t n_bytes = rte_pktmbuf_data_len(pkt);
 419
 420                 for ( ; n_retries < p->n_retries; n_retries++) {
 421                         ssize_t ret;
 422
 423                         ret = write(p->fd, pkt_data, n_bytes);
 424                         if (ret)
 425                                 break;
 426                 }
 427         }
 428
 429         RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - i);
 430
 431         for (i = 0; i < p->tx_buf_count; i++)
 432                 rte_pktmbuf_free(p->tx_buf[i]);
 433
 434         p->tx_buf_count = 0;
 435 }
 436
 437 static int
 438 rte_port_fd_writer_nodrop_tx(void *port, struct rte_mbuf *pkt)
 439 {
 440         struct rte_port_fd_writer_nodrop *p =
 441                 (struct rte_port_fd_writer_nodrop *) port;
 442
 443         p->tx_buf[p->tx_buf_count++] = pkt;
 444         RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 1);
 445         if (p->tx_buf_count >= p->tx_burst_sz)
 446                 send_burst_nodrop(p);
 447
 448         return 0;
 449 }
 450
 451 static int
 452 rte_port_fd_writer_nodrop_tx_bulk(void *port,
 453         struct rte_mbuf **pkts,
 454         uint64_t pkts_mask)
 455 {
 456         struct rte_port_fd_writer_nodrop *p =
 457                 (struct rte_port_fd_writer_nodrop *) port;
 458         uint32_t tx_buf_count = p->tx_buf_count;
 459
 460         if ((pkts_mask & (pkts_mask + 1)) == 0) {
 461                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
 462                 uint32_t i;
 463
 464                 for (i = 0; i < n_pkts; i++)
 465                         p->tx_buf[tx_buf_count++] = pkts[i];
 466                 RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts);
 467         } else
 468                 for ( ; pkts_mask; ) {
 469                         uint32_t pkt_index = __builtin_ctzll(pkts_mask);
 470                         uint64_t pkt_mask = 1LLU << pkt_index;
 471                         struct rte_mbuf *pkt = pkts[pkt_index];
 472
 473                         p->tx_buf[tx_buf_count++] = pkt;
 474                         RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 1);
 475                         pkts_mask &= ~pkt_mask;
 476                 }
 477
 478         p->tx_buf_count = tx_buf_count;
 479         if (tx_buf_count >= p->tx_burst_sz)
 480                 send_burst_nodrop(p);
 481
 482         return 0;
 483 }
 484
 485 static int
 486 rte_port_fd_writer_nodrop_flush(void *port)
 487 {
 488         struct rte_port_fd_writer_nodrop *p =
 489                 (struct rte_port_fd_writer_nodrop *) port;
 490
 491         if (p->tx_buf_count > 0)
 492                 send_burst_nodrop(p);
 493
 494         return 0;
 495 }
 496
 497 static int
 498 rte_port_fd_writer_nodrop_free(void *port)
 499 {
 500         if (port == NULL) {
 501                 RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__);
 502                 return -EINVAL;
 503         }
 504
 505         rte_port_fd_writer_nodrop_flush(port);
 506         rte_free(port);
 507
 508 return 0;
 509 }
 510
 511 static int rte_port_fd_writer_nodrop_stats_read(void *port,
 512                 struct rte_port_out_stats *stats, int clear)
 513 {
 514         struct rte_port_fd_writer_nodrop *p =
 515                 (struct rte_port_fd_writer_nodrop *) port;
 516
 517         if (stats != NULL)
 518                 memcpy(stats, &p->stats, sizeof(p->stats));
 519
 520         if (clear)
 521                 memset(&p->stats, 0, sizeof(p->stats));
 522
 523         return 0;
 524 }
 525
 526 /*
 527  * Summary of port operations
 528  */
 529 struct rte_port_in_ops rte_port_fd_reader_ops = {
 530         .f_create = rte_port_fd_reader_create,
 531         .f_free = rte_port_fd_reader_free,
 532         .f_rx = rte_port_fd_reader_rx,
 533         .f_stats = rte_port_fd_reader_stats_read,
 534 };
 535
 536 struct rte_port_out_ops rte_port_fd_writer_ops = {
 537         .f_create = rte_port_fd_writer_create,
 538         .f_free = rte_port_fd_writer_free,
 539         .f_tx = rte_port_fd_writer_tx,
 540         .f_tx_bulk = rte_port_fd_writer_tx_bulk,
 541         .f_flush = rte_port_fd_writer_flush,
 542         .f_stats = rte_port_fd_writer_stats_read,
 543 };
 544
 545 struct rte_port_out_ops rte_port_fd_writer_nodrop_ops = {
 546         .f_create = rte_port_fd_writer_nodrop_create,
 547         .f_free = rte_port_fd_writer_nodrop_free,
 548         .f_tx = rte_port_fd_writer_nodrop_tx,
 549         .f_tx_bulk = rte_port_fd_writer_nodrop_tx_bulk,
 550         .f_flush = rte_port_fd_writer_nodrop_flush,
 551         .f_stats = rte_port_fd_writer_nodrop_stats_read,
 552 };