ceph/src/spdk/dpdk/lib/librte_ring/rte_ring.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2010-2015 Intel Corporation
   4  * Copyright (c) 2007,2008 Kip Macy kmacy@freebsd.org
   5  * All rights reserved.
   6  * Derived from FreeBSD's bufring.h
   7  * Used as BSD-3 Licensed with permission from Kip Macy.
   8  */
   9
  10 #include <stdio.h>
  11 #include <stdarg.h>
  12 #include <string.h>
  13 #include <stdint.h>
  14 #include <inttypes.h>
  15 #include <errno.h>
  16 #include <sys/queue.h>
  17
  18 #include <rte_common.h>
  19 #include <rte_log.h>
  20 #include <rte_memory.h>
  21 #include <rte_memzone.h>
  22 #include <rte_malloc.h>
  23 #include <rte_launch.h>
  24 #include <rte_eal.h>
  25 #include <rte_eal_memconfig.h>
  26 #include <rte_atomic.h>
  27 #include <rte_per_lcore.h>
  28 #include <rte_lcore.h>
  29 #include <rte_branch_prediction.h>
  30 #include <rte_errno.h>
  31 #include <rte_string_fns.h>
  32 #include <rte_spinlock.h>
  33
  34 #include "rte_ring.h"
  35
  36 TAILQ_HEAD(rte_ring_list, rte_tailq_entry);
  37
  38 static struct rte_tailq_elem rte_ring_tailq = {
  39         .name = RTE_TAILQ_RING_NAME,
  40 };
  41 EAL_REGISTER_TAILQ(rte_ring_tailq)
  42
  43 /* true if x is a power of 2 */
  44 #define POWEROF2(x) ((((x)-1) & (x)) == 0)
  45
  46 /* return the size of memory occupied by a ring */
  47 ssize_t
  48 rte_ring_get_memsize(unsigned count)
  49 {
  50         ssize_t sz;
  51
  52         /* count must be a power of 2 */
  53         if ((!POWEROF2(count)) || (count > RTE_RING_SZ_MASK )) {
  54                 RTE_LOG(ERR, RING,
  55                         "Requested size is invalid, must be power of 2, and "
  56                         "do not exceed the size limit %u\n", RTE_RING_SZ_MASK);
  57                 return -EINVAL;
  58         }
  59
  60         sz = sizeof(struct rte_ring) + count * sizeof(void *);
  61         sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE);
  62         return sz;
  63 }
  64
  65 int
  66 rte_ring_init(struct rte_ring *r, const char *name, unsigned count,
  67         unsigned flags)
  68 {
  69         int ret;
  70
  71         /* compilation-time checks */
  72         RTE_BUILD_BUG_ON((sizeof(struct rte_ring) &
  73                           RTE_CACHE_LINE_MASK) != 0);
  74         RTE_BUILD_BUG_ON((offsetof(struct rte_ring, cons) &
  75                           RTE_CACHE_LINE_MASK) != 0);
  76         RTE_BUILD_BUG_ON((offsetof(struct rte_ring, prod) &
  77                           RTE_CACHE_LINE_MASK) != 0);
  78
  79         /* init the ring structure */
  80         memset(r, 0, sizeof(*r));
  81         ret = snprintf(r->name, sizeof(r->name), "%s", name);
  82         if (ret < 0 || ret >= (int)sizeof(r->name))
  83                 return -ENAMETOOLONG;
  84         r->flags = flags;
  85         r->prod.single = (flags & RING_F_SP_ENQ) ? __IS_SP : __IS_MP;
  86         r->cons.single = (flags & RING_F_SC_DEQ) ? __IS_SC : __IS_MC;
  87
  88         if (flags & RING_F_EXACT_SZ) {
  89                 r->size = rte_align32pow2(count + 1);
  90                 r->mask = r->size - 1;
  91                 r->capacity = count;
  92         } else {
  93                 if ((!POWEROF2(count)) || (count > RTE_RING_SZ_MASK)) {
  94                         RTE_LOG(ERR, RING,
  95                                 "Requested size is invalid, must be power of 2, and not exceed the size limit %u\n",
  96                                 RTE_RING_SZ_MASK);
  97                         return -EINVAL;
  98                 }
  99                 r->size = count;
 100                 r->mask = count - 1;
 101                 r->capacity = r->mask;
 102         }
 103         r->prod.head = r->cons.head = 0;
 104         r->prod.tail = r->cons.tail = 0;
 105
 106         return 0;
 107 }
 108
 109 /* create the ring */
 110 struct rte_ring *
 111 rte_ring_create(const char *name, unsigned count, int socket_id,
 112                 unsigned flags)
 113 {
 114         char mz_name[RTE_MEMZONE_NAMESIZE];
 115         struct rte_ring *r;
 116         struct rte_tailq_entry *te;
 117         const struct rte_memzone *mz;
 118         ssize_t ring_size;
 119         int mz_flags = 0;
 120         struct rte_ring_list* ring_list = NULL;
 121         const unsigned int requested_count = count;
 122         int ret;
 123
 124         ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list);
 125
 126         /* for an exact size ring, round up from count to a power of two */
 127         if (flags & RING_F_EXACT_SZ)
 128                 count = rte_align32pow2(count + 1);
 129
 130         ring_size = rte_ring_get_memsize(count);
 131         if (ring_size < 0) {
 132                 rte_errno = ring_size;
 133                 return NULL;
 134         }
 135
 136         ret = snprintf(mz_name, sizeof(mz_name), "%s%s",
 137                 RTE_RING_MZ_PREFIX, name);
 138         if (ret < 0 || ret >= (int)sizeof(mz_name)) {
 139                 rte_errno = ENAMETOOLONG;
 140                 return NULL;
 141         }
 142
 143         te = rte_zmalloc("RING_TAILQ_ENTRY", sizeof(*te), 0);
 144         if (te == NULL) {
 145                 RTE_LOG(ERR, RING, "Cannot reserve memory for tailq\n");
 146                 rte_errno = ENOMEM;
 147                 return NULL;
 148         }
 149
 150         rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
 151
 152         /* reserve a memory zone for this ring. If we can't get rte_config or
 153          * we are secondary process, the memzone_reserve function will set
 154          * rte_errno for us appropriately - hence no check in this this function */
 155         mz = rte_memzone_reserve_aligned(mz_name, ring_size, socket_id,
 156                                          mz_flags, __alignof__(*r));
 157         if (mz != NULL) {
 158                 r = mz->addr;
 159                 /* no need to check return value here, we already checked the
 160                  * arguments above */
 161                 rte_ring_init(r, name, requested_count, flags);
 162
 163                 te->data = (void *) r;
 164                 r->memzone = mz;
 165
 166                 TAILQ_INSERT_TAIL(ring_list, te, next);
 167         } else {
 168                 r = NULL;
 169                 RTE_LOG(ERR, RING, "Cannot reserve memory\n");
 170                 rte_free(te);
 171         }
 172         rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
 173
 174         return r;
 175 }
 176
 177 /* free the ring */
 178 void
 179 rte_ring_free(struct rte_ring *r)
 180 {
 181         struct rte_ring_list *ring_list = NULL;
 182         struct rte_tailq_entry *te;
 183
 184         if (r == NULL)
 185                 return;
 186
 187         /*
 188          * Ring was not created with rte_ring_create,
 189          * therefore, there is no memzone to free.
 190          */
 191         if (r->memzone == NULL) {
 192                 RTE_LOG(ERR, RING, "Cannot free ring (not created with rte_ring_create()");
 193                 return;
 194         }
 195
 196         if (rte_memzone_free(r->memzone) != 0) {
 197                 RTE_LOG(ERR, RING, "Cannot free memory\n");
 198                 return;
 199         }
 200
 201         ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list);
 202         rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
 203
 204         /* find out tailq entry */
 205         TAILQ_FOREACH(te, ring_list, next) {
 206                 if (te->data == (void *) r)
 207                         break;
 208         }
 209
 210         if (te == NULL) {
 211                 rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
 212                 return;
 213         }
 214
 215         TAILQ_REMOVE(ring_list, te, next);
 216
 217         rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
 218
 219         rte_free(te);
 220 }
 221
 222 /* dump the status of the ring on the console */
 223 void
 224 rte_ring_dump(FILE *f, const struct rte_ring *r)
 225 {
 226         fprintf(f, "ring <%s>@%p\n", r->name, r);
 227         fprintf(f, "  flags=%x\n", r->flags);
 228         fprintf(f, "  size=%"PRIu32"\n", r->size);
 229         fprintf(f, "  capacity=%"PRIu32"\n", r->capacity);
 230         fprintf(f, "  ct=%"PRIu32"\n", r->cons.tail);
 231         fprintf(f, "  ch=%"PRIu32"\n", r->cons.head);
 232         fprintf(f, "  pt=%"PRIu32"\n", r->prod.tail);
 233         fprintf(f, "  ph=%"PRIu32"\n", r->prod.head);
 234         fprintf(f, "  used=%u\n", rte_ring_count(r));
 235         fprintf(f, "  avail=%u\n", rte_ring_free_count(r));
 236 }
 237
 238 /* dump the status of all rings on the console */
 239 void
 240 rte_ring_list_dump(FILE *f)
 241 {
 242         const struct rte_tailq_entry *te;
 243         struct rte_ring_list *ring_list;
 244
 245         ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list);
 246
 247         rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK);
 248
 249         TAILQ_FOREACH(te, ring_list, next) {
 250                 rte_ring_dump(f, (struct rte_ring *) te->data);
 251         }
 252
 253         rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK);
 254 }
 255
 256 /* search a ring from its name */
 257 struct rte_ring *
 258 rte_ring_lookup(const char *name)
 259 {
 260         struct rte_tailq_entry *te;
 261         struct rte_ring *r = NULL;
 262         struct rte_ring_list *ring_list;
 263
 264         ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list);
 265
 266         rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK);
 267
 268         TAILQ_FOREACH(te, ring_list, next) {
 269                 r = (struct rte_ring *) te->data;
 270                 if (strncmp(name, r->name, RTE_RING_NAMESIZE) == 0)
 271                         break;
 272         }
 273
 274         rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK);
 275
 276         if (te == NULL) {
 277                 rte_errno = ENOENT;
 278                 return NULL;
 279         }
 280
 281         return r;
 282 }