ceph/src/spdk/dpdk/lib/librte_acl/acl_run.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #ifndef _ACL_RUN_H_
   6 #define _ACL_RUN_H_
   7
   8 #include <rte_acl.h>
   9 #include "acl.h"
  10
  11 #define MAX_SEARCHES_AVX16      16
  12 #define MAX_SEARCHES_SSE8       8
  13 #define MAX_SEARCHES_ALTIVEC8   8
  14 #define MAX_SEARCHES_SSE4       4
  15 #define MAX_SEARCHES_ALTIVEC4   4
  16 #define MAX_SEARCHES_SCALAR     2
  17
  18 #define GET_NEXT_4BYTES(prm, idx)       \
  19         (*((const int32_t *)((prm)[(idx)].data + *(prm)[idx].data_index++)))
  20
  21
  22 #define RTE_ACL_NODE_INDEX      ((uint32_t)~RTE_ACL_NODE_TYPE)
  23
  24 #define SCALAR_QRANGE_MULT      0x01010101
  25 #define SCALAR_QRANGE_MASK      0x7f7f7f7f
  26 #define SCALAR_QRANGE_MIN       0x80808080
  27
  28 /*
  29  * Structure to manage N parallel trie traversals.
  30  * The runtime trie traversal routines can process 8, 4, or 2 tries
  31  * in parallel. Each packet may require multiple trie traversals (up to 4).
  32  * This structure is used to fill the slots (0 to n-1) for parallel processing
  33  * with the trie traversals needed for each packet.
  34  */
  35 struct acl_flow_data {
  36         uint32_t            num_packets;
  37         /* number of packets processed */
  38         uint32_t            started;
  39         /* number of trie traversals in progress */
  40         uint32_t            trie;
  41         /* current trie index (0 to N-1) */
  42         uint32_t            cmplt_size;
  43         /* maximum number of packets to process */
  44         uint32_t            total_packets;
  45         /* number of result categories per packet. */
  46         uint32_t            categories;
  47         const uint64_t     *trans;
  48         const uint8_t     **data;
  49         uint32_t           *results;
  50         struct completion  *last_cmplt;
  51         struct completion  *cmplt_array;
  52 };
  53
  54 /*
  55  * Structure to maintain running results for
  56  * a single packet (up to 4 tries).
  57  */
  58 struct completion {
  59         uint32_t *results;                          /* running results. */
  60         int32_t   priority[RTE_ACL_MAX_CATEGORIES]; /* running priorities. */
  61         uint32_t  count;                            /* num of remaining tries */
  62         /* true for allocated struct */
  63 } __rte_aligned(XMM_SIZE);
  64
  65 /*
  66  * One parms structure for each slot in the search engine.
  67  */
  68 struct parms {
  69         const uint8_t              *data;
  70         /* input data for this packet */
  71         const uint32_t             *data_index;
  72         /* data indirection for this trie */
  73         struct completion          *cmplt;
  74         /* completion data for this packet */
  75 };
  76
  77 /*
  78  * Define an global idle node for unused engine slots
  79  */
  80 static const uint32_t idle[UINT8_MAX + 1];
  81
  82 /*
  83  * Allocate a completion structure to manage the tries for a packet.
  84  */
  85 static inline struct completion *
  86 alloc_completion(struct completion *p, uint32_t size, uint32_t tries,
  87         uint32_t *results)
  88 {
  89         uint32_t n;
  90
  91         for (n = 0; n < size; n++) {
  92
  93                 if (p[n].count == 0) {
  94
  95                         /* mark as allocated and set number of tries. */
  96                         p[n].count = tries;
  97                         p[n].results = results;
  98                         return &(p[n]);
  99                 }
 100         }
 101
 102         /* should never get here */
 103         return NULL;
 104 }
 105
 106 /*
 107  * Resolve priority for a single result trie.
 108  */
 109 static inline void
 110 resolve_single_priority(uint64_t transition, int n,
 111         const struct rte_acl_ctx *ctx, struct parms *parms,
 112         const struct rte_acl_match_results *p)
 113 {
 114         if (parms[n].cmplt->count == ctx->num_tries ||
 115                         parms[n].cmplt->priority[0] <=
 116                         p[transition].priority[0]) {
 117
 118                 parms[n].cmplt->priority[0] = p[transition].priority[0];
 119                 parms[n].cmplt->results[0] = p[transition].results[0];
 120         }
 121 }
 122
 123 /*
 124  * Routine to fill a slot in the parallel trie traversal array (parms) from
 125  * the list of packets (flows).
 126  */
 127 static inline uint64_t
 128 acl_start_next_trie(struct acl_flow_data *flows, struct parms *parms, int n,
 129         const struct rte_acl_ctx *ctx)
 130 {
 131         uint64_t transition;
 132
 133         /* if there are any more packets to process */
 134         if (flows->num_packets < flows->total_packets) {
 135                 parms[n].data = flows->data[flows->num_packets];
 136                 parms[n].data_index = ctx->trie[flows->trie].data_index;
 137
 138                 /* if this is the first trie for this packet */
 139                 if (flows->trie == 0) {
 140                         flows->last_cmplt = alloc_completion(flows->cmplt_array,
 141                                 flows->cmplt_size, ctx->num_tries,
 142                                 flows->results +
 143                                 flows->num_packets * flows->categories);
 144                 }
 145
 146                 /* set completion parameters and starting index for this slot */
 147                 parms[n].cmplt = flows->last_cmplt;
 148                 transition =
 149                         flows->trans[parms[n].data[*parms[n].data_index++] +
 150                         ctx->trie[flows->trie].root_index];
 151
 152                 /*
 153                  * if this is the last trie for this packet,
 154                  * then setup next packet.
 155                  */
 156                 flows->trie++;
 157                 if (flows->trie >= ctx->num_tries) {
 158                         flows->trie = 0;
 159                         flows->num_packets++;
 160                 }
 161
 162                 /* keep track of number of active trie traversals */
 163                 flows->started++;
 164
 165         /* no more tries to process, set slot to an idle position */
 166         } else {
 167                 transition = ctx->idle;
 168                 parms[n].data = (const uint8_t *)idle;
 169                 parms[n].data_index = idle;
 170         }
 171         return transition;
 172 }
 173
 174 static inline void
 175 acl_set_flow(struct acl_flow_data *flows, struct completion *cmplt,
 176         uint32_t cmplt_size, const uint8_t **data, uint32_t *results,
 177         uint32_t data_num, uint32_t categories, const uint64_t *trans)
 178 {
 179         flows->num_packets = 0;
 180         flows->started = 0;
 181         flows->trie = 0;
 182         flows->last_cmplt = NULL;
 183         flows->cmplt_array = cmplt;
 184         flows->total_packets = data_num;
 185         flows->categories = categories;
 186         flows->cmplt_size = cmplt_size;
 187         flows->data = data;
 188         flows->results = results;
 189         flows->trans = trans;
 190 }
 191
 192 typedef void (*resolve_priority_t)
 193 (uint64_t transition, int n, const struct rte_acl_ctx *ctx,
 194         struct parms *parms, const struct rte_acl_match_results *p,
 195         uint32_t categories);
 196
 197 /*
 198  * Detect matches. If a match node transition is found, then this trie
 199  * traversal is complete and fill the slot with the next trie
 200  * to be processed.
 201  */
 202 static inline uint64_t
 203 acl_match_check(uint64_t transition, int slot,
 204         const struct rte_acl_ctx *ctx, struct parms *parms,
 205         struct acl_flow_data *flows, resolve_priority_t resolve_priority)
 206 {
 207         const struct rte_acl_match_results *p;
 208
 209         p = (const struct rte_acl_match_results *)
 210                 (flows->trans + ctx->match_index);
 211
 212         if (transition & RTE_ACL_NODE_MATCH) {
 213
 214                 /* Remove flags from index and decrement active traversals */
 215                 transition &= RTE_ACL_NODE_INDEX;
 216                 flows->started--;
 217
 218                 /* Resolve priorities for this trie and running results */
 219                 if (flows->categories == 1)
 220                         resolve_single_priority(transition, slot, ctx,
 221                                 parms, p);
 222                 else
 223                         resolve_priority(transition, slot, ctx, parms,
 224                                 p, flows->categories);
 225
 226                 /* Count down completed tries for this search request */
 227                 parms[slot].cmplt->count--;
 228
 229                 /* Fill the slot with the next trie or idle trie */
 230                 transition = acl_start_next_trie(flows, parms, slot, ctx);
 231         }
 232
 233         return transition;
 234 }
 235
 236 #endif /* _ACL_RUN_H_ */