ceph/src/dpdk/examples/performance-thread/common/lthread_tls.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <string.h>
  37 #include <stdint.h>
  38 #include <limits.h>
  39 #include <inttypes.h>
  40 #include <unistd.h>
  41 #include <pthread.h>
  42 #include <fcntl.h>
  43 #include <sys/time.h>
  44 #include <sys/mman.h>
  45 #include <execinfo.h>
  46 #include <sched.h>
  47
  48 #include <rte_malloc.h>
  49 #include <rte_log.h>
  50 #include <rte_ring.h>
  51 #include <rte_atomic_64.h>
  52
  53 #include "lthread_tls.h"
  54 #include "lthread_queue.h"
  55 #include "lthread_objcache.h"
  56 #include "lthread_sched.h"
  57
  58 static struct rte_ring *key_pool;
  59 static uint64_t key_pool_init;
  60
  61 /* needed to cause section start and end to be defined */
  62 RTE_DEFINE_PER_LTHREAD(void *, dummy);
  63
  64 static struct lthread_key key_table[LTHREAD_MAX_KEYS];
  65
  66 void lthread_tls_ctor(void) __attribute__((constructor));
  67
  68 void lthread_tls_ctor(void)
  69 {
  70         key_pool = NULL;
  71         key_pool_init = 0;
  72 }
  73
  74 /*
  75  * Initialize a pool of keys
  76  * These are unique tokens that can be obtained by threads
  77  * calling lthread_key_create()
  78  */
  79 void _lthread_key_pool_init(void)
  80 {
  81         static struct rte_ring *pool;
  82         struct lthread_key *new_key;
  83         char name[MAX_LTHREAD_NAME_SIZE];
  84
  85         bzero(key_table, sizeof(key_table));
  86
  87         /* only one lcore should do this */
  88         if (rte_atomic64_cmpset(&key_pool_init, 0, 1)) {
  89
  90                 snprintf(name,
  91                         MAX_LTHREAD_NAME_SIZE,
  92                         "lthread_key_pool_%d",
  93                         getpid());
  94
  95                 pool = rte_ring_create(name,
  96                                         LTHREAD_MAX_KEYS, 0, 0);
  97                 RTE_ASSERT(pool);
  98
  99                 int i;
 100
 101                 for (i = 1; i < LTHREAD_MAX_KEYS; i++) {
 102                         new_key = &key_table[i];
 103                         rte_ring_mp_enqueue((struct rte_ring *)pool,
 104                                                 (void *)new_key);
 105                 }
 106                 key_pool = pool;
 107         }
 108         /* other lcores wait here till done */
 109         while (key_pool == NULL) {
 110                 rte_compiler_barrier();
 111                 sched_yield();
 112         };
 113 }
 114
 115 /*
 116  * Create a key
 117  * this means getting a key from the the pool
 118  */
 119 int lthread_key_create(unsigned int *key, tls_destructor_func destructor)
 120 {
 121         if (key == NULL)
 122                 return POSIX_ERRNO(EINVAL);
 123
 124         struct lthread_key *new_key;
 125
 126         if (rte_ring_mc_dequeue((struct rte_ring *)key_pool, (void **)&new_key)
 127             == 0) {
 128                 new_key->destructor = destructor;
 129                 *key = (new_key - key_table);
 130
 131                 return 0;
 132         }
 133         return POSIX_ERRNO(EAGAIN);
 134 }
 135
 136
 137 /*
 138  * Delete a key
 139  */
 140 int lthread_key_delete(unsigned int k)
 141 {
 142         struct lthread_key *key;
 143
 144         key = (struct lthread_key *) &key_table[k];
 145
 146         if (k > LTHREAD_MAX_KEYS)
 147                 return POSIX_ERRNO(EINVAL);
 148
 149         key->destructor = NULL;
 150         rte_ring_mp_enqueue((struct rte_ring *)key_pool,
 151                                         (void *)key);
 152         return 0;
 153 }
 154
 155
 156
 157 /*
 158  * Break association for all keys in use by this thread
 159  * invoke the destructor if available.
 160  * Since a destructor can create keys we could enter an infinite loop
 161  * therefore we give up after LTHREAD_DESTRUCTOR_ITERATIONS
 162  * the behavior is modelled on pthread
 163  */
 164 void _lthread_tls_destroy(struct lthread *lt)
 165 {
 166         int i, k;
 167         int nb_keys;
 168         void *data;
 169
 170         for (i = 0; i < LTHREAD_DESTRUCTOR_ITERATIONS; i++) {
 171
 172                 for (k = 1; k < LTHREAD_MAX_KEYS; k++) {
 173
 174                         /* no keys in use ? */
 175                         nb_keys = lt->tls->nb_keys_inuse;
 176                         if (nb_keys == 0)
 177                                 return;
 178
 179                         /* this key not in use ? */
 180                         if (lt->tls->data[k] == NULL)
 181                                 continue;
 182
 183                         /* remove this key */
 184                         data = lt->tls->data[k];
 185                         lt->tls->data[k] = NULL;
 186                         lt->tls->nb_keys_inuse = nb_keys-1;
 187
 188                         /* invoke destructor */
 189                         if (key_table[k].destructor != NULL)
 190                                 key_table[k].destructor(data);
 191                 }
 192         }
 193 }
 194
 195 /*
 196  * Return the pointer associated with a key
 197  * If the key is no longer valid return NULL
 198  */
 199 void
 200 *lthread_getspecific(unsigned int k)
 201 {
 202
 203         if (k > LTHREAD_MAX_KEYS)
 204                 return NULL;
 205
 206         return THIS_LTHREAD->tls->data[k];
 207 }
 208
 209 /*
 210  * Set a value against a key
 211  * If the key is no longer valid return an error
 212  * when storing value
 213  */
 214 int lthread_setspecific(unsigned int k, const void *data)
 215 {
 216         if (k > LTHREAD_MAX_KEYS)
 217                 return POSIX_ERRNO(EINVAL);
 218
 219         int n = THIS_LTHREAD->tls->nb_keys_inuse;
 220
 221         /* discard const qualifier */
 222         char *p = (char *) (uintptr_t) data;
 223
 224
 225         if (data != NULL) {
 226                 if (THIS_LTHREAD->tls->data[k] == NULL)
 227                         THIS_LTHREAD->tls->nb_keys_inuse = n+1;
 228         }
 229
 230         THIS_LTHREAD->tls->data[k] = (void *) p;
 231         return 0;
 232 }
 233
 234 /*
 235  * Allocate data for TLS cache
 236 */
 237 void _lthread_tls_alloc(struct lthread *lt)
 238 {
 239         struct lthread_tls *tls;
 240
 241         tls = _lthread_objcache_alloc((THIS_SCHED)->tls_cache);
 242
 243         RTE_ASSERT(tls != NULL);
 244
 245         tls->root_sched = (THIS_SCHED);
 246         lt->tls = tls;
 247
 248         /* allocate data for TLS varaiables using RTE_PER_LTHREAD macros */
 249         if (sizeof(void *) < (uint64_t)RTE_PER_LTHREAD_SECTION_SIZE) {
 250                 lt->per_lthread_data =
 251                     _lthread_objcache_alloc((THIS_SCHED)->per_lthread_cache);
 252         }
 253 }