ceph/src/dpdk/lib/librte_port/rte_port_ras.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33 #include <string.h>
  34
  35 #include <rte_ether.h>
  36 #include <rte_ip_frag.h>
  37 #include <rte_cycles.h>
  38 #include <rte_log.h>
  39
  40 #include "rte_port_ras.h"
  41
  42 #ifndef RTE_PORT_RAS_N_BUCKETS
  43 #define RTE_PORT_RAS_N_BUCKETS                                 4094
  44 #endif
  45
  46 #ifndef RTE_PORT_RAS_N_ENTRIES_PER_BUCKET
  47 #define RTE_PORT_RAS_N_ENTRIES_PER_BUCKET                      8
  48 #endif
  49
  50 #ifndef RTE_PORT_RAS_N_ENTRIES
  51 #define RTE_PORT_RAS_N_ENTRIES (RTE_PORT_RAS_N_BUCKETS * RTE_PORT_RAS_N_ENTRIES_PER_BUCKET)
  52 #endif
  53
  54 #ifdef RTE_PORT_STATS_COLLECT
  55
  56 #define RTE_PORT_RING_WRITER_RAS_STATS_PKTS_IN_ADD(port, val) \
  57         port->stats.n_pkts_in += val
  58 #define RTE_PORT_RING_WRITER_RAS_STATS_PKTS_DROP_ADD(port, val) \
  59         port->stats.n_pkts_drop += val
  60
  61 #else
  62
  63 #define RTE_PORT_RING_WRITER_RAS_STATS_PKTS_IN_ADD(port, val)
  64 #define RTE_PORT_RING_WRITER_RAS_STATS_PKTS_DROP_ADD(port, val)
  65
  66 #endif
  67
  68 struct rte_port_ring_writer_ras;
  69
  70 typedef void (*ras_op)(
  71                 struct rte_port_ring_writer_ras *p,
  72                 struct rte_mbuf *pkt);
  73
  74 static void
  75 process_ipv4(struct rte_port_ring_writer_ras *p, struct rte_mbuf *pkt);
  76 static void
  77 process_ipv6(struct rte_port_ring_writer_ras *p, struct rte_mbuf *pkt);
  78
  79 struct rte_port_ring_writer_ras {
  80         struct rte_port_out_stats stats;
  81
  82         struct rte_mbuf *tx_buf[RTE_PORT_IN_BURST_SIZE_MAX];
  83         struct rte_ring *ring;
  84         uint32_t tx_burst_sz;
  85         uint32_t tx_buf_count;
  86         struct rte_ip_frag_tbl *frag_tbl;
  87         struct rte_ip_frag_death_row death_row;
  88
  89         ras_op f_ras;
  90 };
  91
  92 static void *
  93 rte_port_ring_writer_ras_create(void *params, int socket_id, int is_ipv4)
  94 {
  95         struct rte_port_ring_writer_ras_params *conf =
  96                         (struct rte_port_ring_writer_ras_params *) params;
  97         struct rte_port_ring_writer_ras *port;
  98         uint64_t frag_cycles;
  99
 100         /* Check input parameters */
 101         if (conf == NULL) {
 102                 RTE_LOG(ERR, PORT, "%s: Parameter conf is NULL\n", __func__);
 103                 return NULL;
 104         }
 105         if (conf->ring == NULL) {
 106                 RTE_LOG(ERR, PORT, "%s: Parameter ring is NULL\n", __func__);
 107                 return NULL;
 108         }
 109         if ((conf->tx_burst_sz == 0) ||
 110             (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX)) {
 111                 RTE_LOG(ERR, PORT, "%s: Parameter tx_burst_sz is invalid\n",
 112                         __func__);
 113                 return NULL;
 114         }
 115
 116         /* Memory allocation */
 117         port = rte_zmalloc_socket("PORT", sizeof(*port),
 118                         RTE_CACHE_LINE_SIZE, socket_id);
 119         if (port == NULL) {
 120                 RTE_LOG(ERR, PORT, "%s: Failed to allocate socket\n", __func__);
 121                 return NULL;
 122         }
 123
 124         /* Create fragmentation table */
 125         frag_cycles = (rte_get_tsc_hz() + MS_PER_S - 1) / MS_PER_S * MS_PER_S;
 126         frag_cycles *= 100;
 127
 128         port->frag_tbl = rte_ip_frag_table_create(
 129                 RTE_PORT_RAS_N_BUCKETS,
 130                 RTE_PORT_RAS_N_ENTRIES_PER_BUCKET,
 131                 RTE_PORT_RAS_N_ENTRIES,
 132                 frag_cycles,
 133                 socket_id);
 134
 135         if (port->frag_tbl == NULL) {
 136                 RTE_LOG(ERR, PORT, "%s: rte_ip_frag_table_create failed\n",
 137                         __func__);
 138                 rte_free(port);
 139                 return NULL;
 140         }
 141
 142         /* Initialization */
 143         port->ring = conf->ring;
 144         port->tx_burst_sz = conf->tx_burst_sz;
 145         port->tx_buf_count = 0;
 146
 147         port->f_ras = (is_ipv4 == 1) ? process_ipv4 : process_ipv6;
 148
 149         return port;
 150 }
 151
 152 static void *
 153 rte_port_ring_writer_ipv4_ras_create(void *params, int socket_id)
 154 {
 155         return rte_port_ring_writer_ras_create(params, socket_id, 1);
 156 }
 157
 158 static void *
 159 rte_port_ring_writer_ipv6_ras_create(void *params, int socket_id)
 160 {
 161         return rte_port_ring_writer_ras_create(params, socket_id, 0);
 162 }
 163
 164 static inline void
 165 send_burst(struct rte_port_ring_writer_ras *p)
 166 {
 167         uint32_t nb_tx;
 168
 169         nb_tx = rte_ring_sp_enqueue_burst(p->ring, (void **)p->tx_buf,
 170                         p->tx_buf_count);
 171
 172         RTE_PORT_RING_WRITER_RAS_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - nb_tx);
 173         for ( ; nb_tx < p->tx_buf_count; nb_tx++)
 174                 rte_pktmbuf_free(p->tx_buf[nb_tx]);
 175
 176         p->tx_buf_count = 0;
 177 }
 178
 179 static void
 180 process_ipv4(struct rte_port_ring_writer_ras *p, struct rte_mbuf *pkt)
 181 {
 182         /* Assume there is no ethernet header */
 183         struct ipv4_hdr *pkt_hdr = rte_pktmbuf_mtod(pkt, struct ipv4_hdr *);
 184
 185         /* Get "More fragments" flag and fragment offset */
 186         uint16_t frag_field = rte_be_to_cpu_16(pkt_hdr->fragment_offset);
 187         uint16_t frag_offset = (uint16_t)(frag_field & IPV4_HDR_OFFSET_MASK);
 188         uint16_t frag_flag = (uint16_t)(frag_field & IPV4_HDR_MF_FLAG);
 189
 190         /* If it is a fragmented packet, then try to reassemble */
 191         if ((frag_flag == 0) && (frag_offset == 0))
 192                 p->tx_buf[p->tx_buf_count++] = pkt;
 193         else {
 194                 struct rte_mbuf *mo;
 195                 struct rte_ip_frag_tbl *tbl = p->frag_tbl;
 196                 struct rte_ip_frag_death_row *dr = &p->death_row;
 197
 198                 pkt->l3_len = sizeof(*pkt_hdr);
 199
 200                 /* Process this fragment */
 201                 mo = rte_ipv4_frag_reassemble_packet(tbl, dr, pkt, rte_rdtsc(),
 202                                 pkt_hdr);
 203                 if (mo != NULL)
 204                         p->tx_buf[p->tx_buf_count++] = mo;
 205
 206                 rte_ip_frag_free_death_row(&p->death_row, 3);
 207         }
 208 }
 209
 210 static void
 211 process_ipv6(struct rte_port_ring_writer_ras *p, struct rte_mbuf *pkt)
 212 {
 213         /* Assume there is no ethernet header */
 214         struct ipv6_hdr *pkt_hdr = rte_pktmbuf_mtod(pkt, struct ipv6_hdr *);
 215
 216         struct ipv6_extension_fragment *frag_hdr;
 217         uint16_t frag_data = 0;
 218         frag_hdr = rte_ipv6_frag_get_ipv6_fragment_header(pkt_hdr);
 219         if (frag_hdr != NULL)
 220                 frag_data = rte_be_to_cpu_16(frag_hdr->frag_data);
 221
 222         /* If it is a fragmented packet, then try to reassemble */
 223         if ((frag_data & RTE_IPV6_FRAG_USED_MASK) == 0)
 224                 p->tx_buf[p->tx_buf_count++] = pkt;
 225         else {
 226                 struct rte_mbuf *mo;
 227                 struct rte_ip_frag_tbl *tbl = p->frag_tbl;
 228                 struct rte_ip_frag_death_row *dr = &p->death_row;
 229
 230                 pkt->l3_len = sizeof(*pkt_hdr) + sizeof(*frag_hdr);
 231
 232                 /* Process this fragment */
 233                 mo = rte_ipv6_frag_reassemble_packet(tbl, dr, pkt, rte_rdtsc(), pkt_hdr,
 234                                 frag_hdr);
 235                 if (mo != NULL)
 236                         p->tx_buf[p->tx_buf_count++] = mo;
 237
 238                 rte_ip_frag_free_death_row(&p->death_row, 3);
 239         }
 240 }
 241
 242 static int
 243 rte_port_ring_writer_ras_tx(void *port, struct rte_mbuf *pkt)
 244 {
 245         struct rte_port_ring_writer_ras *p =
 246                         (struct rte_port_ring_writer_ras *) port;
 247
 248         RTE_PORT_RING_WRITER_RAS_STATS_PKTS_IN_ADD(p, 1);
 249         p->f_ras(p, pkt);
 250         if (p->tx_buf_count >= p->tx_burst_sz)
 251                 send_burst(p);
 252
 253         return 0;
 254 }
 255
 256 static int
 257 rte_port_ring_writer_ras_tx_bulk(void *port,
 258                 struct rte_mbuf **pkts,
 259                 uint64_t pkts_mask)
 260 {
 261         struct rte_port_ring_writer_ras *p =
 262                         (struct rte_port_ring_writer_ras *) port;
 263
 264         if ((pkts_mask & (pkts_mask + 1)) == 0) {
 265                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
 266                 uint32_t i;
 267
 268                 for (i = 0; i < n_pkts; i++) {
 269                         struct rte_mbuf *pkt = pkts[i];
 270
 271                         RTE_PORT_RING_WRITER_RAS_STATS_PKTS_IN_ADD(p, 1);
 272                         p->f_ras(p, pkt);
 273                         if (p->tx_buf_count >= p->tx_burst_sz)
 274                                 send_burst(p);
 275                 }
 276         } else {
 277                 for ( ; pkts_mask; ) {
 278                         uint32_t pkt_index = __builtin_ctzll(pkts_mask);
 279                         uint64_t pkt_mask = 1LLU << pkt_index;
 280                         struct rte_mbuf *pkt = pkts[pkt_index];
 281
 282                         RTE_PORT_RING_WRITER_RAS_STATS_PKTS_IN_ADD(p, 1);
 283                         p->f_ras(p, pkt);
 284                         if (p->tx_buf_count >= p->tx_burst_sz)
 285                                 send_burst(p);
 286
 287                         pkts_mask &= ~pkt_mask;
 288                 }
 289         }
 290
 291         return 0;
 292 }
 293
 294 static int
 295 rte_port_ring_writer_ras_flush(void *port)
 296 {
 297         struct rte_port_ring_writer_ras *p =
 298                         (struct rte_port_ring_writer_ras *) port;
 299
 300         if (p->tx_buf_count > 0)
 301                 send_burst(p);
 302
 303         return 0;
 304 }
 305
 306 static int
 307 rte_port_ring_writer_ras_free(void *port)
 308 {
 309         struct rte_port_ring_writer_ras *p =
 310                         (struct rte_port_ring_writer_ras *) port;
 311
 312         if (port == NULL) {
 313                 RTE_LOG(ERR, PORT, "%s: Parameter port is NULL\n", __func__);
 314                 return -1;
 315         }
 316
 317         rte_port_ring_writer_ras_flush(port);
 318         rte_ip_frag_table_destroy(p->frag_tbl);
 319         rte_free(port);
 320
 321         return 0;
 322 }
 323
 324 static int
 325 rte_port_ras_writer_stats_read(void *port,
 326                 struct rte_port_out_stats *stats, int clear)
 327 {
 328         struct rte_port_ring_writer_ras *p =
 329                 (struct rte_port_ring_writer_ras *) port;
 330
 331         if (stats != NULL)
 332                 memcpy(stats, &p->stats, sizeof(p->stats));
 333
 334         if (clear)
 335                 memset(&p->stats, 0, sizeof(p->stats));
 336
 337         return 0;
 338 }
 339
 340 /*
 341  * Summary of port operations
 342  */
 343 struct rte_port_out_ops rte_port_ring_writer_ipv4_ras_ops = {
 344         .f_create = rte_port_ring_writer_ipv4_ras_create,
 345         .f_free = rte_port_ring_writer_ras_free,
 346         .f_tx = rte_port_ring_writer_ras_tx,
 347         .f_tx_bulk = rte_port_ring_writer_ras_tx_bulk,
 348         .f_flush = rte_port_ring_writer_ras_flush,
 349         .f_stats = rte_port_ras_writer_stats_read,
 350 };
 351
 352 struct rte_port_out_ops rte_port_ring_writer_ipv6_ras_ops = {
 353         .f_create = rte_port_ring_writer_ipv6_ras_create,
 354         .f_free = rte_port_ring_writer_ras_free,
 355         .f_tx = rte_port_ring_writer_ras_tx,
 356         .f_tx_bulk = rte_port_ring_writer_ras_tx_bulk,
 357         .f_flush = rte_port_ring_writer_ras_flush,
 358         .f_stats = rte_port_ras_writer_stats_read,
 359 };