src/jemalloc/include/jemalloc/internal/tsd.h

   1 /******************************************************************************/
   2 #ifdef JEMALLOC_H_TYPES
   3
   4 /* Maximum number of malloc_tsd users with cleanup functions. */
   5 #define MALLOC_TSD_CLEANUPS_MAX 2
   6
   7 typedef bool (*malloc_tsd_cleanup_t)(void);
   8
   9 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
  10     !defined(_WIN32))
  11 typedef struct tsd_init_block_s tsd_init_block_t;
  12 typedef struct tsd_init_head_s tsd_init_head_t;
  13 #endif
  14
  15 typedef struct tsd_s tsd_t;
  16
  17 typedef enum {
  18         tsd_state_uninitialized,
  19         tsd_state_nominal,
  20         tsd_state_purgatory,
  21         tsd_state_reincarnated
  22 } tsd_state_t;
  23
  24 /*
  25  * TLS/TSD-agnostic macro-based implementation of thread-specific data.  There
  26  * are five macros that support (at least) three use cases: file-private,
  27  * library-private, and library-private inlined.  Following is an example
  28  * library-private tsd variable:
  29  *
  30  * In example.h:
  31  *   typedef struct {
  32  *           int x;
  33  *           int y;
  34  *   } example_t;
  35  *   #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0})
  36  *   malloc_tsd_types(example_, example_t)
  37  *   malloc_tsd_protos(, example_, example_t)
  38  *   malloc_tsd_externs(example_, example_t)
  39  * In example.c:
  40  *   malloc_tsd_data(, example_, example_t, EX_INITIALIZER)
  41  *   malloc_tsd_funcs(, example_, example_t, EX_INITIALIZER,
  42  *       example_tsd_cleanup)
  43  *
  44  * The result is a set of generated functions, e.g.:
  45  *
  46  *   bool example_tsd_boot(void) {...}
  47  *   example_t *example_tsd_get() {...}
  48  *   void example_tsd_set(example_t *val) {...}
  49  *
  50  * Note that all of the functions deal in terms of (a_type *) rather than
  51  * (a_type) so that it is possible to support non-pointer types (unlike
  52  * pthreads TSD).  example_tsd_cleanup() is passed an (a_type *) pointer that is
  53  * cast to (void *).  This means that the cleanup function needs to cast the
  54  * function argument to (a_type *), then dereference the resulting pointer to
  55  * access fields, e.g.
  56  *
  57  *   void
  58  *   example_tsd_cleanup(void *arg)
  59  *   {
  60  *           example_t *example = (example_t *)arg;
  61  *
  62  *           example->x = 42;
  63  *           [...]
  64  *           if ([want the cleanup function to be called again])
  65  *                   example_tsd_set(example);
  66  *   }
  67  *
  68  * If example_tsd_set() is called within example_tsd_cleanup(), it will be
  69  * called again.  This is similar to how pthreads TSD destruction works, except
  70  * that pthreads only calls the cleanup function again if the value was set to
  71  * non-NULL.
  72  */
  73
  74 /* malloc_tsd_types(). */
  75 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
  76 #define malloc_tsd_types(a_name, a_type)
  77 #elif (defined(JEMALLOC_TLS))
  78 #define malloc_tsd_types(a_name, a_type)
  79 #elif (defined(_WIN32))
  80 #define malloc_tsd_types(a_name, a_type)                                \
  81 typedef struct {                                                        \
  82         bool    initialized;                                            \
  83         a_type  val;                                                    \
  84 } a_name##tsd_wrapper_t;
  85 #else
  86 #define malloc_tsd_types(a_name, a_type)                                \
  87 typedef struct {                                                        \
  88         bool    initialized;                                            \
  89         a_type  val;                                                    \
  90 } a_name##tsd_wrapper_t;
  91 #endif
  92
  93 /* malloc_tsd_protos(). */
  94 #define malloc_tsd_protos(a_attr, a_name, a_type)                       \
  95 a_attr bool                                                             \
  96 a_name##tsd_boot0(void);                                                \
  97 a_attr void                                                             \
  98 a_name##tsd_boot1(void);                                                \
  99 a_attr bool                                                             \
 100 a_name##tsd_boot(void);                                                 \
 101 a_attr a_type *                                                         \
 102 a_name##tsd_get(void);                                                  \
 103 a_attr void                                                             \
 104 a_name##tsd_set(a_type *val);
 105
 106 /* malloc_tsd_externs(). */
 107 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
 108 #define malloc_tsd_externs(a_name, a_type)                              \
 109 extern __thread a_type  a_name##tsd_tls;                                \
 110 extern __thread bool    a_name##tsd_initialized;                        \
 111 extern bool             a_name##tsd_booted;
 112 #elif (defined(JEMALLOC_TLS))
 113 #define malloc_tsd_externs(a_name, a_type)                              \
 114 extern __thread a_type  a_name##tsd_tls;                                \
 115 extern pthread_key_t    a_name##tsd_tsd;                                \
 116 extern bool             a_name##tsd_booted;
 117 #elif (defined(_WIN32))
 118 #define malloc_tsd_externs(a_name, a_type)                              \
 119 extern DWORD            a_name##tsd_tsd;                                \
 120 extern a_name##tsd_wrapper_t    a_name##tsd_boot_wrapper;               \
 121 extern bool             a_name##tsd_booted;
 122 #else
 123 #define malloc_tsd_externs(a_name, a_type)                              \
 124 extern pthread_key_t    a_name##tsd_tsd;                                \
 125 extern tsd_init_head_t  a_name##tsd_init_head;                          \
 126 extern a_name##tsd_wrapper_t    a_name##tsd_boot_wrapper;               \
 127 extern bool             a_name##tsd_booted;
 128 #endif
 129
 130 /* malloc_tsd_data(). */
 131 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
 132 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer)          \
 133 a_attr __thread a_type JEMALLOC_TLS_MODEL                               \
 134     a_name##tsd_tls = a_initializer;                                    \
 135 a_attr __thread bool JEMALLOC_TLS_MODEL                                 \
 136     a_name##tsd_initialized = false;                                    \
 137 a_attr bool             a_name##tsd_booted = false;
 138 #elif (defined(JEMALLOC_TLS))
 139 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer)          \
 140 a_attr __thread a_type JEMALLOC_TLS_MODEL                               \
 141     a_name##tsd_tls = a_initializer;                                    \
 142 a_attr pthread_key_t    a_name##tsd_tsd;                                \
 143 a_attr bool             a_name##tsd_booted = false;
 144 #elif (defined(_WIN32))
 145 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer)          \
 146 a_attr DWORD            a_name##tsd_tsd;                                \
 147 a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = {               \
 148         false,                                                          \
 149         a_initializer                                                   \
 150 };                                                                      \
 151 a_attr bool             a_name##tsd_booted = false;
 152 #else
 153 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer)          \
 154 a_attr pthread_key_t    a_name##tsd_tsd;                                \
 155 a_attr tsd_init_head_t  a_name##tsd_init_head = {                       \
 156         ql_head_initializer(blocks),                                    \
 157         MALLOC_MUTEX_INITIALIZER                                        \
 158 };                                                                      \
 159 a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = {               \
 160         false,                                                          \
 161         a_initializer                                                   \
 162 };                                                                      \
 163 a_attr bool             a_name##tsd_booted = false;
 164 #endif
 165
 166 /* malloc_tsd_funcs(). */
 167 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
 168 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,         \
 169     a_cleanup)                                                          \
 170 /* Initialization/cleanup. */                                           \
 171 a_attr bool                                                             \
 172 a_name##tsd_cleanup_wrapper(void)                                       \
 173 {                                                                       \
 174                                                                         \
 175         if (a_name##tsd_initialized) {                                  \
 176                 a_name##tsd_initialized = false;                        \
 177                 a_cleanup(&a_name##tsd_tls);                            \
 178         }                                                               \
 179         return (a_name##tsd_initialized);                               \
 180 }                                                                       \
 181 a_attr bool                                                             \
 182 a_name##tsd_boot0(void)                                                 \
 183 {                                                                       \
 184                                                                         \
 185         if (a_cleanup != malloc_tsd_no_cleanup) {                       \
 186                 malloc_tsd_cleanup_register(                            \
 187                     &a_name##tsd_cleanup_wrapper);                      \
 188         }                                                               \
 189         a_name##tsd_booted = true;                                      \
 190         return (false);                                                 \
 191 }                                                                       \
 192 a_attr void                                                             \
 193 a_name##tsd_boot1(void)                                                 \
 194 {                                                                       \
 195                                                                         \
 196         /* Do nothing. */                                               \
 197 }                                                                       \
 198 a_attr bool                                                             \
 199 a_name##tsd_boot(void)                                                  \
 200 {                                                                       \
 201                                                                         \
 202         return (a_name##tsd_boot0());                                   \
 203 }                                                                       \
 204 /* Get/set. */                                                          \
 205 a_attr a_type *                                                         \
 206 a_name##tsd_get(void)                                                   \
 207 {                                                                       \
 208                                                                         \
 209         assert(a_name##tsd_booted);                                     \
 210         return (&a_name##tsd_tls);                                      \
 211 }                                                                       \
 212 a_attr void                                                             \
 213 a_name##tsd_set(a_type *val)                                            \
 214 {                                                                       \
 215                                                                         \
 216         assert(a_name##tsd_booted);                                     \
 217         a_name##tsd_tls = (*val);                                       \
 218         if (a_cleanup != malloc_tsd_no_cleanup)                         \
 219                 a_name##tsd_initialized = true;                         \
 220 }
 221 #elif (defined(JEMALLOC_TLS))
 222 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,         \
 223     a_cleanup)                                                          \
 224 /* Initialization/cleanup. */                                           \
 225 a_attr bool                                                             \
 226 a_name##tsd_boot0(void)                                                 \
 227 {                                                                       \
 228                                                                         \
 229         if (a_cleanup != malloc_tsd_no_cleanup) {                       \
 230                 if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) !=  \
 231                     0)                                                  \
 232                         return (true);                                  \
 233         }                                                               \
 234         a_name##tsd_booted = true;                                      \
 235         return (false);                                                 \
 236 }                                                                       \
 237 a_attr void                                                             \
 238 a_name##tsd_boot1(void)                                                 \
 239 {                                                                       \
 240                                                                         \
 241         /* Do nothing. */                                               \
 242 }                                                                       \
 243 a_attr bool                                                             \
 244 a_name##tsd_boot(void)                                                  \
 245 {                                                                       \
 246                                                                         \
 247         return (a_name##tsd_boot0());                                   \
 248 }                                                                       \
 249 /* Get/set. */                                                          \
 250 a_attr a_type *                                                         \
 251 a_name##tsd_get(void)                                                   \
 252 {                                                                       \
 253                                                                         \
 254         assert(a_name##tsd_booted);                                     \
 255         return (&a_name##tsd_tls);                                      \
 256 }                                                                       \
 257 a_attr void                                                             \
 258 a_name##tsd_set(a_type *val)                                            \
 259 {                                                                       \
 260                                                                         \
 261         assert(a_name##tsd_booted);                                     \
 262         a_name##tsd_tls = (*val);                                       \
 263         if (a_cleanup != malloc_tsd_no_cleanup) {                       \
 264                 if (pthread_setspecific(a_name##tsd_tsd,                \
 265                     (void *)(&a_name##tsd_tls))) {                      \
 266                         malloc_write("<jemalloc>: Error"                \
 267                             " setting TSD for "#a_name"\n");            \
 268                         if (opt_abort)                                  \
 269                                 abort();                                \
 270                 }                                                       \
 271         }                                                               \
 272 }
 273 #elif (defined(_WIN32))
 274 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,         \
 275     a_cleanup)                                                          \
 276 /* Initialization/cleanup. */                                           \
 277 a_attr bool                                                             \
 278 a_name##tsd_cleanup_wrapper(void)                                       \
 279 {                                                                       \
 280         DWORD error = GetLastError();                                   \
 281         a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)      \
 282             TlsGetValue(a_name##tsd_tsd);                               \
 283         SetLastError(error);                                            \
 284                                                                         \
 285         if (wrapper == NULL)                                            \
 286                 return (false);                                         \
 287         if (a_cleanup != malloc_tsd_no_cleanup &&                       \
 288             wrapper->initialized) {                                     \
 289                 wrapper->initialized = false;                           \
 290                 a_cleanup(&wrapper->val);                               \
 291                 if (wrapper->initialized) {                             \
 292                         /* Trigger another cleanup round. */            \
 293                         return (true);                                  \
 294                 }                                                       \
 295         }                                                               \
 296         malloc_tsd_dalloc(wrapper);                                     \
 297         return (false);                                                 \
 298 }                                                                       \
 299 a_attr void                                                             \
 300 a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper)                 \
 301 {                                                                       \
 302                                                                         \
 303         if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) {           \
 304                 malloc_write("<jemalloc>: Error setting"                \
 305                     " TSD for "#a_name"\n");                            \
 306                 abort();                                                \
 307         }                                                               \
 308 }                                                                       \
 309 a_attr a_name##tsd_wrapper_t *                                          \
 310 a_name##tsd_wrapper_get(void)                                           \
 311 {                                                                       \
 312         DWORD error = GetLastError();                                   \
 313         a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)      \
 314             TlsGetValue(a_name##tsd_tsd);                               \
 315         SetLastError(error);                                            \
 316                                                                         \
 317         if (unlikely(wrapper == NULL)) {                                \
 318                 wrapper = (a_name##tsd_wrapper_t *)                     \
 319                     malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));   \
 320                 if (wrapper == NULL) {                                  \
 321                         malloc_write("<jemalloc>: Error allocating"     \
 322                             " TSD for "#a_name"\n");                    \
 323                         abort();                                        \
 324                 } else {                                                \
 325                         wrapper->initialized = false;                   \
 326                         wrapper->val = a_initializer;                   \
 327                 }                                                       \
 328                 a_name##tsd_wrapper_set(wrapper);                       \
 329         }                                                               \
 330         return (wrapper);                                               \
 331 }                                                                       \
 332 a_attr bool                                                             \
 333 a_name##tsd_boot0(void)                                                 \
 334 {                                                                       \
 335                                                                         \
 336         a_name##tsd_tsd = TlsAlloc();                                   \
 337         if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES)                      \
 338                 return (true);                                          \
 339         if (a_cleanup != malloc_tsd_no_cleanup) {                       \
 340                 malloc_tsd_cleanup_register(                            \
 341                     &a_name##tsd_cleanup_wrapper);                      \
 342         }                                                               \
 343         a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper);             \
 344         a_name##tsd_booted = true;                                      \
 345         return (false);                                                 \
 346 }                                                                       \
 347 a_attr void                                                             \
 348 a_name##tsd_boot1(void)                                                 \
 349 {                                                                       \
 350         a_name##tsd_wrapper_t *wrapper;                                 \
 351         wrapper = (a_name##tsd_wrapper_t *)                             \
 352             malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));           \
 353         if (wrapper == NULL) {                                          \
 354                 malloc_write("<jemalloc>: Error allocating"             \
 355                     " TSD for "#a_name"\n");                            \
 356                 abort();                                                \
 357         }                                                               \
 358         memcpy(wrapper, &a_name##tsd_boot_wrapper,                      \
 359             sizeof(a_name##tsd_wrapper_t));                             \
 360         a_name##tsd_wrapper_set(wrapper);                               \
 361 }                                                                       \
 362 a_attr bool                                                             \
 363 a_name##tsd_boot(void)                                                  \
 364 {                                                                       \
 365                                                                         \
 366         if (a_name##tsd_boot0())                                        \
 367                 return (true);                                          \
 368         a_name##tsd_boot1();                                            \
 369         return (false);                                                 \
 370 }                                                                       \
 371 /* Get/set. */                                                          \
 372 a_attr a_type *                                                         \
 373 a_name##tsd_get(void)                                                   \
 374 {                                                                       \
 375         a_name##tsd_wrapper_t *wrapper;                                 \
 376                                                                         \
 377         assert(a_name##tsd_booted);                                     \
 378         wrapper = a_name##tsd_wrapper_get();                            \
 379         return (&wrapper->val);                                         \
 380 }                                                                       \
 381 a_attr void                                                             \
 382 a_name##tsd_set(a_type *val)                                            \
 383 {                                                                       \
 384         a_name##tsd_wrapper_t *wrapper;                                 \
 385                                                                         \
 386         assert(a_name##tsd_booted);                                     \
 387         wrapper = a_name##tsd_wrapper_get();                            \
 388         wrapper->val = *(val);                                          \
 389         if (a_cleanup != malloc_tsd_no_cleanup)                         \
 390                 wrapper->initialized = true;                            \
 391 }
 392 #else
 393 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer,         \
 394     a_cleanup)                                                          \
 395 /* Initialization/cleanup. */                                           \
 396 a_attr void                                                             \
 397 a_name##tsd_cleanup_wrapper(void *arg)                                  \
 398 {                                                                       \
 399         a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg;  \
 400                                                                         \
 401         if (a_cleanup != malloc_tsd_no_cleanup &&                       \
 402             wrapper->initialized) {                                     \
 403                 wrapper->initialized = false;                           \
 404                 a_cleanup(&wrapper->val);                               \
 405                 if (wrapper->initialized) {                             \
 406                         /* Trigger another cleanup round. */            \
 407                         if (pthread_setspecific(a_name##tsd_tsd,        \
 408                             (void *)wrapper)) {                         \
 409                                 malloc_write("<jemalloc>: Error"        \
 410                                     " setting TSD for "#a_name"\n");    \
 411                                 if (opt_abort)                          \
 412                                         abort();                        \
 413                         }                                               \
 414                         return;                                         \
 415                 }                                                       \
 416         }                                                               \
 417         malloc_tsd_dalloc(wrapper);                                     \
 418 }                                                                       \
 419 a_attr void                                                             \
 420 a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper)                 \
 421 {                                                                       \
 422                                                                         \
 423         if (pthread_setspecific(a_name##tsd_tsd,                        \
 424             (void *)wrapper)) {                                         \
 425                 malloc_write("<jemalloc>: Error setting"                \
 426                     " TSD for "#a_name"\n");                            \
 427                 abort();                                                \
 428         }                                                               \
 429 }                                                                       \
 430 a_attr a_name##tsd_wrapper_t *                                          \
 431 a_name##tsd_wrapper_get(void)                                           \
 432 {                                                                       \
 433         a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)      \
 434             pthread_getspecific(a_name##tsd_tsd);                       \
 435                                                                         \
 436         if (unlikely(wrapper == NULL)) {                                \
 437                 tsd_init_block_t block;                                 \
 438                 wrapper = tsd_init_check_recursion(                     \
 439                     &a_name##tsd_init_head, &block);                    \
 440                 if (wrapper)                                            \
 441                     return (wrapper);                                   \
 442                 wrapper = (a_name##tsd_wrapper_t *)                     \
 443                     malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));   \
 444                 block.data = wrapper;                                   \
 445                 if (wrapper == NULL) {                                  \
 446                         malloc_write("<jemalloc>: Error allocating"     \
 447                             " TSD for "#a_name"\n");                    \
 448                         abort();                                        \
 449                 } else {                                                \
 450                         wrapper->initialized = false;                   \
 451                         wrapper->val = a_initializer;                   \
 452                 }                                                       \
 453                 a_name##tsd_wrapper_set(wrapper);                       \
 454                 tsd_init_finish(&a_name##tsd_init_head, &block);        \
 455         }                                                               \
 456         return (wrapper);                                               \
 457 }                                                                       \
 458 a_attr bool                                                             \
 459 a_name##tsd_boot0(void)                                                 \
 460 {                                                                       \
 461                                                                         \
 462         if (pthread_key_create(&a_name##tsd_tsd,                        \
 463             a_name##tsd_cleanup_wrapper) != 0)                          \
 464                 return (true);                                          \
 465         a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper);             \
 466         a_name##tsd_booted = true;                                      \
 467         return (false);                                                 \
 468 }                                                                       \
 469 a_attr void                                                             \
 470 a_name##tsd_boot1(void)                                                 \
 471 {                                                                       \
 472         a_name##tsd_wrapper_t *wrapper;                                 \
 473         wrapper = (a_name##tsd_wrapper_t *)                             \
 474             malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));           \
 475         if (wrapper == NULL) {                                          \
 476                 malloc_write("<jemalloc>: Error allocating"             \
 477                     " TSD for "#a_name"\n");                            \
 478                 abort();                                                \
 479         }                                                               \
 480         memcpy(wrapper, &a_name##tsd_boot_wrapper,                      \
 481             sizeof(a_name##tsd_wrapper_t));                             \
 482         a_name##tsd_wrapper_set(wrapper);                               \
 483 }                                                                       \
 484 a_attr bool                                                             \
 485 a_name##tsd_boot(void)                                                  \
 486 {                                                                       \
 487                                                                         \
 488         if (a_name##tsd_boot0())                                        \
 489                 return (true);                                          \
 490         a_name##tsd_boot1();                                            \
 491         return (false);                                                 \
 492 }                                                                       \
 493 /* Get/set. */                                                          \
 494 a_attr a_type *                                                         \
 495 a_name##tsd_get(void)                                                   \
 496 {                                                                       \
 497         a_name##tsd_wrapper_t *wrapper;                                 \
 498                                                                         \
 499         assert(a_name##tsd_booted);                                     \
 500         wrapper = a_name##tsd_wrapper_get();                            \
 501         return (&wrapper->val);                                         \
 502 }                                                                       \
 503 a_attr void                                                             \
 504 a_name##tsd_set(a_type *val)                                            \
 505 {                                                                       \
 506         a_name##tsd_wrapper_t *wrapper;                                 \
 507                                                                         \
 508         assert(a_name##tsd_booted);                                     \
 509         wrapper = a_name##tsd_wrapper_get();                            \
 510         wrapper->val = *(val);                                          \
 511         if (a_cleanup != malloc_tsd_no_cleanup)                         \
 512                 wrapper->initialized = true;                            \
 513 }
 514 #endif
 515
 516 #endif /* JEMALLOC_H_TYPES */
 517 /******************************************************************************/
 518 #ifdef JEMALLOC_H_STRUCTS
 519
 520 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
 521     !defined(_WIN32))
 522 struct tsd_init_block_s {
 523         ql_elm(tsd_init_block_t)        link;
 524         pthread_t                       thread;
 525         void                            *data;
 526 };
 527 struct tsd_init_head_s {
 528         ql_head(tsd_init_block_t)       blocks;
 529         malloc_mutex_t                  lock;
 530 };
 531 #endif
 532
 533 #define MALLOC_TSD                                                      \
 534 /*  O(name,                     type) */                                \
 535     O(tcache,                   tcache_t *)                             \
 536     O(thread_allocated,         uint64_t)                               \
 537     O(thread_deallocated,       uint64_t)                               \
 538     O(prof_tdata,               prof_tdata_t *)                         \
 539     O(arena,                    arena_t *)                              \
 540     O(arenas_tdata,             arena_tdata_t *)                        \
 541     O(narenas_tdata,            unsigned)                               \
 542     O(arenas_tdata_bypass,      bool)                                   \
 543     O(tcache_enabled,           tcache_enabled_t)                       \
 544     O(quarantine,               quarantine_t *)                         \
 545
 546 #define TSD_INITIALIZER {                                               \
 547     tsd_state_uninitialized,                                            \
 548     NULL,                                                               \
 549     0,                                                                  \
 550     0,                                                                  \
 551     NULL,                                                               \
 552     NULL,                                                               \
 553     NULL,                                                               \
 554     0,                                                                  \
 555     false,                                                              \
 556     tcache_enabled_default,                                             \
 557     NULL                                                                \
 558 }
 559
 560 struct tsd_s {
 561         tsd_state_t     state;
 562 #define O(n, t)                                                         \
 563         t               n;
 564 MALLOC_TSD
 565 #undef O
 566 };
 567
 568 static const tsd_t tsd_initializer = TSD_INITIALIZER;
 569
 570 malloc_tsd_types(, tsd_t)
 571
 572 #endif /* JEMALLOC_H_STRUCTS */
 573 /******************************************************************************/
 574 #ifdef JEMALLOC_H_EXTERNS
 575
 576 void    *malloc_tsd_malloc(size_t size);
 577 void    malloc_tsd_dalloc(void *wrapper);
 578 void    malloc_tsd_no_cleanup(void *arg);
 579 void    malloc_tsd_cleanup_register(bool (*f)(void));
 580 bool    malloc_tsd_boot0(void);
 581 void    malloc_tsd_boot1(void);
 582 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
 583     !defined(_WIN32))
 584 void    *tsd_init_check_recursion(tsd_init_head_t *head,
 585     tsd_init_block_t *block);
 586 void    tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
 587 #endif
 588 void    tsd_cleanup(void *arg);
 589
 590 #endif /* JEMALLOC_H_EXTERNS */
 591 /******************************************************************************/
 592 #ifdef JEMALLOC_H_INLINES
 593
 594 #ifndef JEMALLOC_ENABLE_INLINE
 595 malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t)
 596
 597 tsd_t   *tsd_fetch(void);
 598 bool    tsd_nominal(tsd_t *tsd);
 599 #define O(n, t)                                                         \
 600 t       *tsd_##n##p_get(tsd_t *tsd);                                    \
 601 t       tsd_##n##_get(tsd_t *tsd);                                      \
 602 void    tsd_##n##_set(tsd_t *tsd, t n);
 603 MALLOC_TSD
 604 #undef O
 605 #endif
 606
 607 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_))
 608 malloc_tsd_externs(, tsd_t)
 609 malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup)
 610
 611 JEMALLOC_ALWAYS_INLINE tsd_t *
 612 tsd_fetch(void)
 613 {
 614         tsd_t *tsd = tsd_get();
 615
 616         if (unlikely(tsd->state != tsd_state_nominal)) {
 617                 if (tsd->state == tsd_state_uninitialized) {
 618                         tsd->state = tsd_state_nominal;
 619                         /* Trigger cleanup handler registration. */
 620                         tsd_set(tsd);
 621                 } else if (tsd->state == tsd_state_purgatory) {
 622                         tsd->state = tsd_state_reincarnated;
 623                         tsd_set(tsd);
 624                 } else
 625                         assert(tsd->state == tsd_state_reincarnated);
 626         }
 627
 628         return (tsd);
 629 }
 630
 631 JEMALLOC_INLINE bool
 632 tsd_nominal(tsd_t *tsd)
 633 {
 634
 635         return (tsd->state == tsd_state_nominal);
 636 }
 637
 638 #define O(n, t)                                                         \
 639 JEMALLOC_ALWAYS_INLINE t *                                              \
 640 tsd_##n##p_get(tsd_t *tsd)                                              \
 641 {                                                                       \
 642                                                                         \
 643         return (&tsd->n);                                               \
 644 }                                                                       \
 645                                                                         \
 646 JEMALLOC_ALWAYS_INLINE t                                                \
 647 tsd_##n##_get(tsd_t *tsd)                                               \
 648 {                                                                       \
 649                                                                         \
 650         return (*tsd_##n##p_get(tsd));                                  \
 651 }                                                                       \
 652                                                                         \
 653 JEMALLOC_ALWAYS_INLINE void                                             \
 654 tsd_##n##_set(tsd_t *tsd, t n)                                          \
 655 {                                                                       \
 656                                                                         \
 657         assert(tsd->state == tsd_state_nominal);                        \
 658         tsd->n = n;                                                     \
 659 }
 660 MALLOC_TSD
 661 #undef O
 662 #endif
 663
 664 #endif /* JEMALLOC_H_INLINES */
 665 /******************************************************************************/