ceph/src/spdk/lib/env_dpdk/env.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (c) Intel Corporation.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include "spdk/stdinc.h"
  35
  36 #include "spdk/env.h"
  37
  38 #include <rte_config.h>
  39 #include <rte_cycles.h>
  40 #include <rte_malloc.h>
  41 #include <rte_mempool.h>
  42 #include <rte_memzone.h>
  43 #include <rte_version.h>
  44
  45 static uint64_t
  46 virt_to_phys(void *vaddr)
  47 {
  48         uint64_t ret;
  49
  50 #if RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 3)
  51         ret = rte_malloc_virt2iova(vaddr);
  52         if (ret != RTE_BAD_IOVA) {
  53                 return ret;
  54         }
  55 #else
  56         ret = rte_malloc_virt2phy(vaddr);
  57         if (ret != RTE_BAD_PHYS_ADDR) {
  58                 return ret;
  59         }
  60 #endif
  61
  62         return spdk_vtophys(vaddr);
  63 }
  64
  65 void *
  66 spdk_malloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags)
  67 {
  68         if (flags == 0) {
  69                 return NULL;
  70         }
  71
  72         void *buf = rte_malloc_socket(NULL, size, align, socket_id);
  73         if (buf && phys_addr) {
  74                 *phys_addr = virt_to_phys(buf);
  75         }
  76         return buf;
  77 }
  78
  79 void *
  80 spdk_zmalloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags)
  81 {
  82         void *buf = spdk_malloc(size, align, phys_addr, socket_id, flags);
  83         if (buf) {
  84                 memset(buf, 0, size);
  85         }
  86         return buf;
  87 }
  88
  89 void
  90 spdk_free(void *buf)
  91 {
  92         rte_free(buf);
  93 }
  94
  95 void *
  96 spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id)
  97 {
  98         return spdk_malloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE));
  99 }
 100
 101 void *
 102 spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id)
 103 {
 104         return spdk_zmalloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE));
 105 }
 106
 107 void *
 108 spdk_dma_malloc(size_t size, size_t align, uint64_t *phys_addr)
 109 {
 110         return spdk_dma_malloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY);
 111 }
 112
 113 void *
 114 spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr)
 115 {
 116         return spdk_dma_zmalloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY);
 117 }
 118
 119 void *
 120 spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *phys_addr)
 121 {
 122         void *new_buf = rte_realloc(buf, size, align);
 123         if (new_buf && phys_addr) {
 124                 *phys_addr = virt_to_phys(new_buf);
 125         }
 126         return new_buf;
 127 }
 128
 129 void
 130 spdk_dma_free(void *buf)
 131 {
 132         spdk_free(buf);
 133 }
 134
 135 void *
 136 spdk_memzone_reserve_aligned(const char *name, size_t len, int socket_id,
 137                              unsigned flags, unsigned align)
 138 {
 139         const struct rte_memzone *mz;
 140         unsigned dpdk_flags = 0;
 141
 142 #if RTE_VERSION >= RTE_VERSION_NUM(18, 05, 0, 0)
 143         /* Older DPDKs do not offer such flag since their
 144          * memzones are iova-contiguous by default.
 145          */
 146         if ((flags & SPDK_MEMZONE_NO_IOVA_CONTIG) == 0) {
 147                 dpdk_flags |= RTE_MEMZONE_IOVA_CONTIG;
 148         }
 149 #endif
 150
 151         if (socket_id == SPDK_ENV_SOCKET_ID_ANY) {
 152                 socket_id = SOCKET_ID_ANY;
 153         }
 154
 155         mz = rte_memzone_reserve_aligned(name, len, socket_id, dpdk_flags, align);
 156
 157         if (mz != NULL) {
 158                 memset(mz->addr, 0, len);
 159                 return mz->addr;
 160         } else {
 161                 return NULL;
 162         }
 163 }
 164
 165 void *
 166 spdk_memzone_reserve(const char *name, size_t len, int socket_id, unsigned flags)
 167 {
 168         return spdk_memzone_reserve_aligned(name, len, socket_id, flags,
 169                                             RTE_CACHE_LINE_SIZE);
 170 }
 171
 172 void *
 173 spdk_memzone_lookup(const char *name)
 174 {
 175         const struct rte_memzone *mz = rte_memzone_lookup(name);
 176
 177         if (mz != NULL) {
 178                 return mz->addr;
 179         } else {
 180                 return NULL;
 181         }
 182 }
 183
 184 int
 185 spdk_memzone_free(const char *name)
 186 {
 187         const struct rte_memzone *mz = rte_memzone_lookup(name);
 188
 189         if (mz != NULL) {
 190                 return rte_memzone_free(mz);
 191         }
 192
 193         return -1;
 194 }
 195
 196 void
 197 spdk_memzone_dump(FILE *f)
 198 {
 199         rte_memzone_dump(f);
 200 }
 201
 202 struct spdk_mempool *
 203 spdk_mempool_create_ctor(const char *name, size_t count,
 204                          size_t ele_size, size_t cache_size, int socket_id,
 205                          spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg)
 206 {
 207         struct rte_mempool *mp;
 208         size_t tmp;
 209
 210         if (socket_id == SPDK_ENV_SOCKET_ID_ANY) {
 211                 socket_id = SOCKET_ID_ANY;
 212         }
 213
 214         /* No more than half of all elements can be in cache */
 215         tmp = (count / 2) / rte_lcore_count();
 216         if (cache_size > tmp) {
 217                 cache_size = tmp;
 218         }
 219
 220         if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE) {
 221                 cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE;
 222         }
 223
 224         mp = rte_mempool_create(name, count, ele_size, cache_size,
 225                                 0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg,
 226                                 socket_id, MEMPOOL_F_NO_PHYS_CONTIG);
 227
 228         return (struct spdk_mempool *)mp;
 229 }
 230
 231
 232 struct spdk_mempool *
 233 spdk_mempool_create(const char *name, size_t count,
 234                     size_t ele_size, size_t cache_size, int socket_id)
 235 {
 236         return spdk_mempool_create_ctor(name, count, ele_size, cache_size, socket_id,
 237                                         NULL, NULL);
 238 }
 239
 240 char *
 241 spdk_mempool_get_name(struct spdk_mempool *mp)
 242 {
 243         return ((struct rte_mempool *)mp)->name;
 244 }
 245
 246 void
 247 spdk_mempool_free(struct spdk_mempool *mp)
 248 {
 249 #if RTE_VERSION >= RTE_VERSION_NUM(16, 7, 0, 1)
 250         rte_mempool_free((struct rte_mempool *)mp);
 251 #endif
 252 }
 253
 254 void *
 255 spdk_mempool_get(struct spdk_mempool *mp)
 256 {
 257         void *ele = NULL;
 258         int rc;
 259
 260         rc = rte_mempool_get((struct rte_mempool *)mp, &ele);
 261         if (rc != 0) {
 262                 return NULL;
 263         }
 264         return ele;
 265 }
 266
 267 int
 268 spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count)
 269 {
 270         return rte_mempool_get_bulk((struct rte_mempool *)mp, ele_arr, count);
 271 }
 272
 273 void
 274 spdk_mempool_put(struct spdk_mempool *mp, void *ele)
 275 {
 276         rte_mempool_put((struct rte_mempool *)mp, ele);
 277 }
 278
 279 void
 280 spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count)
 281 {
 282         rte_mempool_put_bulk((struct rte_mempool *)mp, ele_arr, count);
 283 }
 284
 285 size_t
 286 spdk_mempool_count(const struct spdk_mempool *pool)
 287 {
 288 #if RTE_VERSION < RTE_VERSION_NUM(16, 7, 0, 1)
 289         return rte_mempool_count((struct rte_mempool *)pool);
 290 #else
 291         return rte_mempool_avail_count((struct rte_mempool *)pool);
 292 #endif
 293 }
 294
 295 bool
 296 spdk_process_is_primary(void)
 297 {
 298         return (rte_eal_process_type() == RTE_PROC_PRIMARY);
 299 }
 300
 301 uint64_t spdk_get_ticks(void)
 302 {
 303         return rte_get_timer_cycles();
 304 }
 305
 306 uint64_t spdk_get_ticks_hz(void)
 307 {
 308         return rte_get_timer_hz();
 309 }
 310
 311 void spdk_delay_us(unsigned int us)
 312 {
 313         rte_delay_us(us);
 314 }
 315
 316 void
 317 spdk_unaffinitize_thread(void)
 318 {
 319         rte_cpuset_t new_cpuset;
 320         long num_cores, i;
 321
 322         CPU_ZERO(&new_cpuset);
 323
 324         num_cores = sysconf(_SC_NPROCESSORS_CONF);
 325
 326         /* Create a mask containing all CPUs */
 327         for (i = 0; i < num_cores; i++) {
 328                 CPU_SET(i, &new_cpuset);
 329         }
 330
 331         rte_thread_set_affinity(&new_cpuset);
 332 }
 333
 334 void *
 335 spdk_call_unaffinitized(void *cb(void *arg), void *arg)
 336 {
 337         rte_cpuset_t orig_cpuset;
 338         void *ret;
 339
 340         if (cb == NULL) {
 341                 return NULL;
 342         }
 343
 344         rte_thread_get_affinity(&orig_cpuset);
 345
 346         spdk_unaffinitize_thread();
 347
 348         ret = cb(arg);
 349
 350         rte_thread_set_affinity(&orig_cpuset);
 351
 352         return ret;
 353 }
 354
 355 struct spdk_ring *
 356 spdk_ring_create(enum spdk_ring_type type, size_t count, int socket_id)
 357 {
 358         char ring_name[64];
 359         static uint32_t ring_num = 0;
 360         unsigned flags = 0;
 361
 362         switch (type) {
 363         case SPDK_RING_TYPE_SP_SC:
 364                 flags = RING_F_SP_ENQ | RING_F_SC_DEQ;
 365                 break;
 366         case SPDK_RING_TYPE_MP_SC:
 367                 flags = RING_F_SC_DEQ;
 368                 break;
 369         case SPDK_RING_TYPE_MP_MC:
 370                 flags = 0;
 371                 break;
 372         default:
 373                 return NULL;
 374         }
 375
 376         snprintf(ring_name, sizeof(ring_name), "ring_%u_%d",
 377                  __sync_fetch_and_add(&ring_num, 1), getpid());
 378
 379         return (struct spdk_ring *)rte_ring_create(ring_name, count, socket_id, flags);
 380 }
 381
 382 void
 383 spdk_ring_free(struct spdk_ring *ring)
 384 {
 385         rte_ring_free((struct rte_ring *)ring);
 386 }
 387
 388 size_t
 389 spdk_ring_count(struct spdk_ring *ring)
 390 {
 391         return rte_ring_count((struct rte_ring *)ring);
 392 }
 393
 394 size_t
 395 spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count)
 396 {
 397         int rc;
 398 #if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0)
 399         rc = rte_ring_enqueue_bulk((struct rte_ring *)ring, objs, count);
 400         if (rc == 0) {
 401                 return count;
 402         }
 403
 404         return 0;
 405 #else
 406         rc = rte_ring_enqueue_bulk((struct rte_ring *)ring, objs, count, NULL);
 407         return rc;
 408 #endif
 409 }
 410
 411 size_t
 412 spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count)
 413 {
 414 #if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0)
 415         return rte_ring_dequeue_burst((struct rte_ring *)ring, objs, count);
 416 #else
 417         return rte_ring_dequeue_burst((struct rte_ring *)ring, objs, count, NULL);
 418 #endif
 419 }