ceph/src/dpdk/lib/librte_table/rte_table_lpm.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <string.h>
  35 #include <stdio.h>
  36
  37 #include <rte_common.h>
  38 #include <rte_mbuf.h>
  39 #include <rte_memory.h>
  40 #include <rte_malloc.h>
  41 #include <rte_byteorder.h>
  42 #include <rte_log.h>
  43 #include <rte_lpm.h>
  44
  45 #include "rte_table_lpm.h"
  46
  47 #ifndef RTE_TABLE_LPM_MAX_NEXT_HOPS
  48 #define RTE_TABLE_LPM_MAX_NEXT_HOPS                        65536
  49 #endif
  50
  51 #ifdef RTE_TABLE_STATS_COLLECT
  52
  53 #define RTE_TABLE_LPM_STATS_PKTS_IN_ADD(table, val) \
  54         table->stats.n_pkts_in += val
  55 #define RTE_TABLE_LPM_STATS_PKTS_LOOKUP_MISS(table, val) \
  56         table->stats.n_pkts_lookup_miss += val
  57
  58 #else
  59
  60 #define RTE_TABLE_LPM_STATS_PKTS_IN_ADD(table, val)
  61 #define RTE_TABLE_LPM_STATS_PKTS_LOOKUP_MISS(table, val)
  62
  63 #endif
  64
  65 struct rte_table_lpm {
  66         struct rte_table_stats stats;
  67
  68         /* Input parameters */
  69         uint32_t entry_size;
  70         uint32_t entry_unique_size;
  71         uint32_t n_rules;
  72         uint32_t offset;
  73
  74         /* Handle to low-level LPM table */
  75         struct rte_lpm *lpm;
  76
  77         /* Next Hop Table (NHT) */
  78         uint32_t nht_users[RTE_TABLE_LPM_MAX_NEXT_HOPS];
  79         uint8_t nht[0] __rte_cache_aligned;
  80 };
  81
  82 static void *
  83 rte_table_lpm_create(void *params, int socket_id, uint32_t entry_size)
  84 {
  85         struct rte_table_lpm_params *p = (struct rte_table_lpm_params *) params;
  86         struct rte_table_lpm *lpm;
  87         struct rte_lpm_config lpm_config;
  88
  89         uint32_t total_size, nht_size;
  90
  91         /* Check input parameters */
  92         if (p == NULL) {
  93                 RTE_LOG(ERR, TABLE, "%s: NULL input parameters\n", __func__);
  94                 return NULL;
  95         }
  96         if (p->n_rules == 0) {
  97                 RTE_LOG(ERR, TABLE, "%s: Invalid n_rules\n", __func__);
  98                 return NULL;
  99         }
 100         if (p->number_tbl8s == 0) {
 101                 RTE_LOG(ERR, TABLE, "%s: Invalid number_tbl8s\n", __func__);
 102                 return NULL;
 103         }
 104         if (p->entry_unique_size == 0) {
 105                 RTE_LOG(ERR, TABLE, "%s: Invalid entry_unique_size\n",
 106                         __func__);
 107                 return NULL;
 108         }
 109         if (p->entry_unique_size > entry_size) {
 110                 RTE_LOG(ERR, TABLE, "%s: Invalid entry_unique_size\n",
 111                         __func__);
 112                 return NULL;
 113         }
 114         if (p->name == NULL) {
 115                 RTE_LOG(ERR, TABLE, "%s: Table name is NULL\n",
 116                         __func__);
 117                 return NULL;
 118         }
 119         entry_size = RTE_ALIGN(entry_size, sizeof(uint64_t));
 120
 121         /* Memory allocation */
 122         nht_size = RTE_TABLE_LPM_MAX_NEXT_HOPS * entry_size;
 123         total_size = sizeof(struct rte_table_lpm) + nht_size;
 124         lpm = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE,
 125                 socket_id);
 126         if (lpm == NULL) {
 127                 RTE_LOG(ERR, TABLE,
 128                         "%s: Cannot allocate %u bytes for LPM table\n",
 129                         __func__, total_size);
 130                 return NULL;
 131         }
 132
 133         /* LPM low-level table creation */
 134         lpm_config.max_rules = p->n_rules;
 135         lpm_config.number_tbl8s = p->number_tbl8s;
 136         lpm_config.flags = p->flags;
 137         lpm->lpm = rte_lpm_create(p->name, socket_id, &lpm_config);
 138
 139         if (lpm->lpm == NULL) {
 140                 rte_free(lpm);
 141                 RTE_LOG(ERR, TABLE, "Unable to create low-level LPM table\n");
 142                 return NULL;
 143         }
 144
 145         /* Memory initialization */
 146         lpm->entry_size = entry_size;
 147         lpm->entry_unique_size = p->entry_unique_size;
 148         lpm->n_rules = p->n_rules;
 149         lpm->offset = p->offset;
 150
 151         return lpm;
 152 }
 153
 154 static int
 155 rte_table_lpm_free(void *table)
 156 {
 157         struct rte_table_lpm *lpm = (struct rte_table_lpm *) table;
 158
 159         /* Check input parameters */
 160         if (lpm == NULL) {
 161                 RTE_LOG(ERR, TABLE, "%s: table parameter is NULL\n", __func__);
 162                 return -EINVAL;
 163         }
 164
 165         /* Free previously allocated resources */
 166         rte_lpm_free(lpm->lpm);
 167         rte_free(lpm);
 168
 169         return 0;
 170 }
 171
 172 static int
 173 nht_find_free(struct rte_table_lpm *lpm, uint32_t *pos)
 174 {
 175         uint32_t i;
 176
 177         for (i = 0; i < RTE_TABLE_LPM_MAX_NEXT_HOPS; i++) {
 178                 if (lpm->nht_users[i] == 0) {
 179                         *pos = i;
 180                         return 1;
 181                 }
 182         }
 183
 184         return 0;
 185 }
 186
 187 static int
 188 nht_find_existing(struct rte_table_lpm *lpm, void *entry, uint32_t *pos)
 189 {
 190         uint32_t i;
 191
 192         for (i = 0; i < RTE_TABLE_LPM_MAX_NEXT_HOPS; i++) {
 193                 uint8_t *nht_entry = &lpm->nht[i * lpm->entry_size];
 194
 195                 if ((lpm->nht_users[i] > 0) && (memcmp(nht_entry, entry,
 196                         lpm->entry_unique_size) == 0)) {
 197                         *pos = i;
 198                         return 1;
 199                 }
 200         }
 201
 202         return 0;
 203 }
 204
 205 static int
 206 rte_table_lpm_entry_add(
 207         void *table,
 208         void *key,
 209         void *entry,
 210         int *key_found,
 211         void **entry_ptr)
 212 {
 213         struct rte_table_lpm *lpm = (struct rte_table_lpm *) table;
 214         struct rte_table_lpm_key *ip_prefix = (struct rte_table_lpm_key *) key;
 215         uint32_t nht_pos, nht_pos0_valid;
 216         int status;
 217         uint32_t nht_pos0 = 0;
 218
 219         /* Check input parameters */
 220         if (lpm == NULL) {
 221                 RTE_LOG(ERR, TABLE, "%s: table parameter is NULL\n", __func__);
 222                 return -EINVAL;
 223         }
 224         if (ip_prefix == NULL) {
 225                 RTE_LOG(ERR, TABLE, "%s: ip_prefix parameter is NULL\n",
 226                         __func__);
 227                 return -EINVAL;
 228         }
 229         if (entry == NULL) {
 230                 RTE_LOG(ERR, TABLE, "%s: entry parameter is NULL\n", __func__);
 231                 return -EINVAL;
 232         }
 233
 234         if ((ip_prefix->depth == 0) || (ip_prefix->depth > 32)) {
 235                 RTE_LOG(ERR, TABLE, "%s: invalid depth (%d)\n",
 236                         __func__, ip_prefix->depth);
 237                 return -EINVAL;
 238         }
 239
 240         /* Check if rule is already present in the table */
 241         status = rte_lpm_is_rule_present(lpm->lpm, ip_prefix->ip,
 242                 ip_prefix->depth, &nht_pos0);
 243         nht_pos0_valid = status > 0;
 244
 245         /* Find existing or free NHT entry */
 246         if (nht_find_existing(lpm, entry, &nht_pos) == 0) {
 247                 uint8_t *nht_entry;
 248
 249                 if (nht_find_free(lpm, &nht_pos) == 0) {
 250                         RTE_LOG(ERR, TABLE, "%s: NHT full\n", __func__);
 251                         return -1;
 252                 }
 253
 254                 nht_entry = &lpm->nht[nht_pos * lpm->entry_size];
 255                 memcpy(nht_entry, entry, lpm->entry_size);
 256         }
 257
 258         /* Add rule to low level LPM table */
 259         if (rte_lpm_add(lpm->lpm, ip_prefix->ip, ip_prefix->depth, nht_pos) < 0) {
 260                 RTE_LOG(ERR, TABLE, "%s: LPM rule add failed\n", __func__);
 261                 return -1;
 262         }
 263
 264         /* Commit NHT changes */
 265         lpm->nht_users[nht_pos]++;
 266         lpm->nht_users[nht_pos0] -= nht_pos0_valid;
 267
 268         *key_found = nht_pos0_valid;
 269         *entry_ptr = (void *) &lpm->nht[nht_pos * lpm->entry_size];
 270         return 0;
 271 }
 272
 273 static int
 274 rte_table_lpm_entry_delete(
 275         void *table,
 276         void *key,
 277         int *key_found,
 278         void *entry)
 279 {
 280         struct rte_table_lpm *lpm = (struct rte_table_lpm *) table;
 281         struct rte_table_lpm_key *ip_prefix = (struct rte_table_lpm_key *) key;
 282         uint32_t nht_pos;
 283         int status;
 284
 285         /* Check input parameters */
 286         if (lpm == NULL) {
 287                 RTE_LOG(ERR, TABLE, "%s: table parameter is NULL\n", __func__);
 288                 return -EINVAL;
 289         }
 290         if (ip_prefix == NULL) {
 291                 RTE_LOG(ERR, TABLE, "%s: ip_prefix parameter is NULL\n",
 292                         __func__);
 293                 return -EINVAL;
 294         }
 295         if ((ip_prefix->depth == 0) || (ip_prefix->depth > 32)) {
 296                 RTE_LOG(ERR, TABLE, "%s: invalid depth (%d)\n", __func__,
 297                         ip_prefix->depth);
 298                 return -EINVAL;
 299         }
 300
 301         /* Return if rule is not present in the table */
 302         status = rte_lpm_is_rule_present(lpm->lpm, ip_prefix->ip,
 303                 ip_prefix->depth, &nht_pos);
 304         if (status < 0) {
 305                 RTE_LOG(ERR, TABLE, "%s: LPM algorithmic error\n", __func__);
 306                 return -1;
 307         }
 308         if (status == 0) {
 309                 *key_found = 0;
 310                 return 0;
 311         }
 312
 313         /* Delete rule from the low-level LPM table */
 314         status = rte_lpm_delete(lpm->lpm, ip_prefix->ip, ip_prefix->depth);
 315         if (status) {
 316                 RTE_LOG(ERR, TABLE, "%s: LPM rule delete failed\n", __func__);
 317                 return -1;
 318         }
 319
 320         /* Commit NHT changes */
 321         lpm->nht_users[nht_pos]--;
 322
 323         *key_found = 1;
 324         if (entry)
 325                 memcpy(entry, &lpm->nht[nht_pos * lpm->entry_size],
 326                         lpm->entry_size);
 327
 328         return 0;
 329 }
 330
 331 static int
 332 rte_table_lpm_lookup(
 333         void *table,
 334         struct rte_mbuf **pkts,
 335         uint64_t pkts_mask,
 336         uint64_t *lookup_hit_mask,
 337         void **entries)
 338 {
 339         struct rte_table_lpm *lpm = (struct rte_table_lpm *) table;
 340         uint64_t pkts_out_mask = 0;
 341         uint32_t i;
 342
 343         __rte_unused uint32_t n_pkts_in = __builtin_popcountll(pkts_mask);
 344         RTE_TABLE_LPM_STATS_PKTS_IN_ADD(lpm, n_pkts_in);
 345
 346         pkts_out_mask = 0;
 347         for (i = 0; i < (uint32_t)(RTE_PORT_IN_BURST_SIZE_MAX -
 348                 __builtin_clzll(pkts_mask)); i++) {
 349                 uint64_t pkt_mask = 1LLU << i;
 350
 351                 if (pkt_mask & pkts_mask) {
 352                         struct rte_mbuf *pkt = pkts[i];
 353                         uint32_t ip = rte_bswap32(
 354                                 RTE_MBUF_METADATA_UINT32(pkt, lpm->offset));
 355                         int status;
 356                         uint32_t nht_pos;
 357
 358                         status = rte_lpm_lookup(lpm->lpm, ip, &nht_pos);
 359                         if (status == 0) {
 360                                 pkts_out_mask |= pkt_mask;
 361                                 entries[i] = (void *) &lpm->nht[nht_pos *
 362                                         lpm->entry_size];
 363                         }
 364                 }
 365         }
 366
 367         *lookup_hit_mask = pkts_out_mask;
 368         RTE_TABLE_LPM_STATS_PKTS_LOOKUP_MISS(lpm, n_pkts_in - __builtin_popcountll(pkts_out_mask));
 369         return 0;
 370 }
 371
 372 static int
 373 rte_table_lpm_stats_read(void *table, struct rte_table_stats *stats, int clear)
 374 {
 375         struct rte_table_lpm *t = (struct rte_table_lpm *) table;
 376
 377         if (stats != NULL)
 378                 memcpy(stats, &t->stats, sizeof(t->stats));
 379
 380         if (clear)
 381                 memset(&t->stats, 0, sizeof(t->stats));
 382
 383         return 0;
 384 }
 385
 386 struct rte_table_ops rte_table_lpm_ops = {
 387         .f_create = rte_table_lpm_create,
 388         .f_free = rte_table_lpm_free,
 389         .f_add = rte_table_lpm_entry_add,
 390         .f_delete = rte_table_lpm_entry_delete,
 391         .f_add_bulk = NULL,
 392         .f_delete_bulk = NULL,
 393         .f_lookup = rte_table_lpm_lookup,
 394         .f_stats = rte_table_lpm_stats_read,
 395 };