1 /******************************************************************************/
2 #ifdef JEMALLOC_H_TYPES
4 /* Maximum number of malloc_tsd users with cleanup functions. */
5 #define MALLOC_TSD_CLEANUPS_MAX 2
7 typedef bool (*malloc_tsd_cleanup_t
)(void);
9 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
11 typedef struct tsd_init_block_s tsd_init_block_t
;
12 typedef struct tsd_init_head_s tsd_init_head_t
;
15 typedef struct tsd_s tsd_t
;
18 tsd_state_uninitialized
,
21 tsd_state_reincarnated
25 * TLS/TSD-agnostic macro-based implementation of thread-specific data. There
26 * are five macros that support (at least) three use cases: file-private,
27 * library-private, and library-private inlined. Following is an example
28 * library-private tsd variable:
35 * #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0})
36 * malloc_tsd_types(example_, example_t)
37 * malloc_tsd_protos(, example_, example_t)
38 * malloc_tsd_externs(example_, example_t)
40 * malloc_tsd_data(, example_, example_t, EX_INITIALIZER)
41 * malloc_tsd_funcs(, example_, example_t, EX_INITIALIZER,
42 * example_tsd_cleanup)
44 * The result is a set of generated functions, e.g.:
46 * bool example_tsd_boot(void) {...}
47 * example_t *example_tsd_get() {...}
48 * void example_tsd_set(example_t *val) {...}
50 * Note that all of the functions deal in terms of (a_type *) rather than
51 * (a_type) so that it is possible to support non-pointer types (unlike
52 * pthreads TSD). example_tsd_cleanup() is passed an (a_type *) pointer that is
53 * cast to (void *). This means that the cleanup function needs to cast the
54 * function argument to (a_type *), then dereference the resulting pointer to
58 * example_tsd_cleanup(void *arg)
60 * example_t *example = (example_t *)arg;
64 * if ([want the cleanup function to be called again])
65 * example_tsd_set(example);
68 * If example_tsd_set() is called within example_tsd_cleanup(), it will be
69 * called again. This is similar to how pthreads TSD destruction works, except
70 * that pthreads only calls the cleanup function again if the value was set to
74 /* malloc_tsd_types(). */
75 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
76 #define malloc_tsd_types(a_name, a_type)
77 #elif (defined(JEMALLOC_TLS))
78 #define malloc_tsd_types(a_name, a_type)
79 #elif (defined(_WIN32))
80 #define malloc_tsd_types(a_name, a_type) \
84 } a_name##tsd_wrapper_t;
86 #define malloc_tsd_types(a_name, a_type) \
90 } a_name##tsd_wrapper_t;
93 /* malloc_tsd_protos(). */
94 #define malloc_tsd_protos(a_attr, a_name, a_type) \
96 a_name##tsd_boot0(void); \
98 a_name##tsd_boot1(void); \
100 a_name##tsd_boot(void); \
102 a_name##tsd_get(void); \
104 a_name##tsd_set(a_type *val);
106 /* malloc_tsd_externs(). */
107 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
108 #define malloc_tsd_externs(a_name, a_type) \
109 extern __thread a_type a_name##tsd_tls; \
110 extern __thread bool a_name##tsd_initialized; \
111 extern bool a_name##tsd_booted;
112 #elif (defined(JEMALLOC_TLS))
113 #define malloc_tsd_externs(a_name, a_type) \
114 extern __thread a_type a_name##tsd_tls; \
115 extern pthread_key_t a_name##tsd_tsd; \
116 extern bool a_name##tsd_booted;
117 #elif (defined(_WIN32))
118 #define malloc_tsd_externs(a_name, a_type) \
119 extern DWORD a_name##tsd_tsd; \
120 extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \
121 extern bool a_name##tsd_booted;
123 #define malloc_tsd_externs(a_name, a_type) \
124 extern pthread_key_t a_name##tsd_tsd; \
125 extern tsd_init_head_t a_name##tsd_init_head; \
126 extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \
127 extern bool a_name##tsd_booted;
130 /* malloc_tsd_data(). */
131 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
132 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
133 a_attr __thread a_type JEMALLOC_TLS_MODEL \
134 a_name##tsd_tls = a_initializer; \
135 a_attr __thread bool JEMALLOC_TLS_MODEL \
136 a_name##tsd_initialized = false; \
137 a_attr bool a_name##tsd_booted = false;
138 #elif (defined(JEMALLOC_TLS))
139 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
140 a_attr __thread a_type JEMALLOC_TLS_MODEL \
141 a_name##tsd_tls = a_initializer; \
142 a_attr pthread_key_t a_name##tsd_tsd; \
143 a_attr bool a_name##tsd_booted = false;
144 #elif (defined(_WIN32))
145 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
146 a_attr DWORD a_name##tsd_tsd; \
147 a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \
151 a_attr bool a_name##tsd_booted = false;
153 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
154 a_attr pthread_key_t a_name##tsd_tsd; \
155 a_attr tsd_init_head_t a_name##tsd_init_head = { \
156 ql_head_initializer(blocks), \
157 MALLOC_MUTEX_INITIALIZER \
159 a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \
163 a_attr bool a_name##tsd_booted = false;
166 /* malloc_tsd_funcs(). */
167 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
168 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
170 /* Initialization/cleanup. */ \
172 a_name##tsd_cleanup_wrapper(void) \
175 if (a_name##tsd_initialized) { \
176 a_name##tsd_initialized = false; \
177 a_cleanup(&a_name##tsd_tls); \
179 return (a_name##tsd_initialized); \
182 a_name##tsd_boot0(void) \
185 if (a_cleanup != malloc_tsd_no_cleanup) { \
186 malloc_tsd_cleanup_register( \
187 &a_name##tsd_cleanup_wrapper); \
189 a_name##tsd_booted = true; \
193 a_name##tsd_boot1(void) \
199 a_name##tsd_boot(void) \
202 return (a_name##tsd_boot0()); \
206 a_name##tsd_get(void) \
209 assert(a_name##tsd_booted); \
210 return (&a_name##tsd_tls); \
213 a_name##tsd_set(a_type *val) \
216 assert(a_name##tsd_booted); \
217 a_name##tsd_tls = (*val); \
218 if (a_cleanup != malloc_tsd_no_cleanup) \
219 a_name##tsd_initialized = true; \
221 #elif (defined(JEMALLOC_TLS))
222 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
224 /* Initialization/cleanup. */ \
226 a_name##tsd_boot0(void) \
229 if (a_cleanup != malloc_tsd_no_cleanup) { \
230 if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) != \
234 a_name##tsd_booted = true; \
238 a_name##tsd_boot1(void) \
244 a_name##tsd_boot(void) \
247 return (a_name##tsd_boot0()); \
251 a_name##tsd_get(void) \
254 assert(a_name##tsd_booted); \
255 return (&a_name##tsd_tls); \
258 a_name##tsd_set(a_type *val) \
261 assert(a_name##tsd_booted); \
262 a_name##tsd_tls = (*val); \
263 if (a_cleanup != malloc_tsd_no_cleanup) { \
264 if (pthread_setspecific(a_name##tsd_tsd, \
265 (void *)(&a_name##tsd_tls))) { \
266 malloc_write("<jemalloc>: Error" \
267 " setting TSD for "#a_name"\n"); \
273 #elif (defined(_WIN32))
274 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
276 /* Initialization/cleanup. */ \
278 a_name##tsd_cleanup_wrapper(void) \
280 DWORD error = GetLastError(); \
281 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
282 TlsGetValue(a_name##tsd_tsd); \
283 SetLastError(error); \
285 if (wrapper == NULL) \
287 if (a_cleanup != malloc_tsd_no_cleanup && \
288 wrapper->initialized) { \
289 wrapper->initialized = false; \
290 a_cleanup(&wrapper->val); \
291 if (wrapper->initialized) { \
292 /* Trigger another cleanup round. */ \
296 malloc_tsd_dalloc(wrapper); \
300 a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \
303 if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) { \
304 malloc_write("<jemalloc>: Error setting" \
305 " TSD for "#a_name"\n"); \
309 a_attr a_name##tsd_wrapper_t * \
310 a_name##tsd_wrapper_get(void) \
312 DWORD error = GetLastError(); \
313 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
314 TlsGetValue(a_name##tsd_tsd); \
315 SetLastError(error); \
317 if (unlikely(wrapper == NULL)) { \
318 wrapper = (a_name##tsd_wrapper_t *) \
319 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
320 if (wrapper == NULL) { \
321 malloc_write("<jemalloc>: Error allocating" \
322 " TSD for "#a_name"\n"); \
325 wrapper->initialized = false; \
326 wrapper->val = a_initializer; \
328 a_name##tsd_wrapper_set(wrapper); \
333 a_name##tsd_boot0(void) \
336 a_name##tsd_tsd = TlsAlloc(); \
337 if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES) \
339 if (a_cleanup != malloc_tsd_no_cleanup) { \
340 malloc_tsd_cleanup_register( \
341 &a_name##tsd_cleanup_wrapper); \
343 a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \
344 a_name##tsd_booted = true; \
348 a_name##tsd_boot1(void) \
350 a_name##tsd_wrapper_t *wrapper; \
351 wrapper = (a_name##tsd_wrapper_t *) \
352 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
353 if (wrapper == NULL) { \
354 malloc_write("<jemalloc>: Error allocating" \
355 " TSD for "#a_name"\n"); \
358 memcpy(wrapper, &a_name##tsd_boot_wrapper, \
359 sizeof(a_name##tsd_wrapper_t)); \
360 a_name##tsd_wrapper_set(wrapper); \
363 a_name##tsd_boot(void) \
366 if (a_name##tsd_boot0()) \
368 a_name##tsd_boot1(); \
373 a_name##tsd_get(void) \
375 a_name##tsd_wrapper_t *wrapper; \
377 assert(a_name##tsd_booted); \
378 wrapper = a_name##tsd_wrapper_get(); \
379 return (&wrapper->val); \
382 a_name##tsd_set(a_type *val) \
384 a_name##tsd_wrapper_t *wrapper; \
386 assert(a_name##tsd_booted); \
387 wrapper = a_name##tsd_wrapper_get(); \
388 wrapper->val = *(val); \
389 if (a_cleanup != malloc_tsd_no_cleanup) \
390 wrapper->initialized = true; \
393 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
395 /* Initialization/cleanup. */ \
397 a_name##tsd_cleanup_wrapper(void *arg) \
399 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg; \
401 if (a_cleanup != malloc_tsd_no_cleanup && \
402 wrapper->initialized) { \
403 wrapper->initialized = false; \
404 a_cleanup(&wrapper->val); \
405 if (wrapper->initialized) { \
406 /* Trigger another cleanup round. */ \
407 if (pthread_setspecific(a_name##tsd_tsd, \
408 (void *)wrapper)) { \
409 malloc_write("<jemalloc>: Error" \
410 " setting TSD for "#a_name"\n"); \
417 malloc_tsd_dalloc(wrapper); \
420 a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \
423 if (pthread_setspecific(a_name##tsd_tsd, \
424 (void *)wrapper)) { \
425 malloc_write("<jemalloc>: Error setting" \
426 " TSD for "#a_name"\n"); \
430 a_attr a_name##tsd_wrapper_t * \
431 a_name##tsd_wrapper_get(void) \
433 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
434 pthread_getspecific(a_name##tsd_tsd); \
436 if (unlikely(wrapper == NULL)) { \
437 tsd_init_block_t block; \
438 wrapper = tsd_init_check_recursion( \
439 &a_name##tsd_init_head, &block); \
442 wrapper = (a_name##tsd_wrapper_t *) \
443 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
444 block.data = wrapper; \
445 if (wrapper == NULL) { \
446 malloc_write("<jemalloc>: Error allocating" \
447 " TSD for "#a_name"\n"); \
450 wrapper->initialized = false; \
451 wrapper->val = a_initializer; \
453 a_name##tsd_wrapper_set(wrapper); \
454 tsd_init_finish(&a_name##tsd_init_head, &block); \
459 a_name##tsd_boot0(void) \
462 if (pthread_key_create(&a_name##tsd_tsd, \
463 a_name##tsd_cleanup_wrapper) != 0) \
465 a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \
466 a_name##tsd_booted = true; \
470 a_name##tsd_boot1(void) \
472 a_name##tsd_wrapper_t *wrapper; \
473 wrapper = (a_name##tsd_wrapper_t *) \
474 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
475 if (wrapper == NULL) { \
476 malloc_write("<jemalloc>: Error allocating" \
477 " TSD for "#a_name"\n"); \
480 memcpy(wrapper, &a_name##tsd_boot_wrapper, \
481 sizeof(a_name##tsd_wrapper_t)); \
482 a_name##tsd_wrapper_set(wrapper); \
485 a_name##tsd_boot(void) \
488 if (a_name##tsd_boot0()) \
490 a_name##tsd_boot1(); \
495 a_name##tsd_get(void) \
497 a_name##tsd_wrapper_t *wrapper; \
499 assert(a_name##tsd_booted); \
500 wrapper = a_name##tsd_wrapper_get(); \
501 return (&wrapper->val); \
504 a_name##tsd_set(a_type *val) \
506 a_name##tsd_wrapper_t *wrapper; \
508 assert(a_name##tsd_booted); \
509 wrapper = a_name##tsd_wrapper_get(); \
510 wrapper->val = *(val); \
511 if (a_cleanup != malloc_tsd_no_cleanup) \
512 wrapper->initialized = true; \
516 #endif /* JEMALLOC_H_TYPES */
517 /******************************************************************************/
518 #ifdef JEMALLOC_H_STRUCTS
520 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
522 struct tsd_init_block_s
{
523 ql_elm(tsd_init_block_t
) link
;
527 struct tsd_init_head_s
{
528 ql_head(tsd_init_block_t
) blocks
;
534 /* O(name, type) */ \
535 O(tcache, tcache_t *) \
536 O(thread_allocated, uint64_t) \
537 O(thread_deallocated, uint64_t) \
538 O(prof_tdata, prof_tdata_t *) \
539 O(arena, arena_t *) \
540 O(arenas_tdata, arena_tdata_t *) \
541 O(narenas_tdata, unsigned) \
542 O(arenas_tdata_bypass, bool) \
543 O(tcache_enabled, tcache_enabled_t) \
544 O(quarantine, quarantine_t *) \
546 #define TSD_INITIALIZER { \
547 tsd_state_uninitialized, \
556 tcache_enabled_default, \
568 static const tsd_t tsd_initializer
= TSD_INITIALIZER
;
570 malloc_tsd_types(, tsd_t
)
572 #endif /* JEMALLOC_H_STRUCTS */
573 /******************************************************************************/
574 #ifdef JEMALLOC_H_EXTERNS
576 void *malloc_tsd_malloc(size_t size
);
577 void malloc_tsd_dalloc(void *wrapper
);
578 void malloc_tsd_no_cleanup(void *arg
);
579 void malloc_tsd_cleanup_register(bool (*f
)(void));
580 bool malloc_tsd_boot0(void);
581 void malloc_tsd_boot1(void);
582 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
584 void *tsd_init_check_recursion(tsd_init_head_t
*head
,
585 tsd_init_block_t
*block
);
586 void tsd_init_finish(tsd_init_head_t
*head
, tsd_init_block_t
*block
);
588 void tsd_cleanup(void *arg
);
590 #endif /* JEMALLOC_H_EXTERNS */
591 /******************************************************************************/
592 #ifdef JEMALLOC_H_INLINES
594 #ifndef JEMALLOC_ENABLE_INLINE
595 malloc_tsd_protos(JEMALLOC_ATTR(unused
), , tsd_t
)
597 tsd_t
*tsd_fetch(void);
598 bool tsd_nominal(tsd_t
*tsd
);
600 t *tsd_##n##p_get(tsd_t *tsd); \
601 t tsd_##n##_get(tsd_t *tsd); \
602 void tsd_##n##_set(tsd_t *tsd, t n);
607 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_))
608 malloc_tsd_externs(, tsd_t
)
609 malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE
, , tsd_t
, tsd_initializer
, tsd_cleanup
)
611 JEMALLOC_ALWAYS_INLINE tsd_t
*
614 tsd_t
*tsd
= tsd_get();
616 if (unlikely(tsd
->state
!= tsd_state_nominal
)) {
617 if (tsd
->state
== tsd_state_uninitialized
) {
618 tsd
->state
= tsd_state_nominal
;
619 /* Trigger cleanup handler registration. */
621 } else if (tsd
->state
== tsd_state_purgatory
) {
622 tsd
->state
= tsd_state_reincarnated
;
625 assert(tsd
->state
== tsd_state_reincarnated
);
632 tsd_nominal(tsd_t
*tsd
)
635 return (tsd
->state
== tsd_state_nominal
);
639 JEMALLOC_ALWAYS_INLINE t * \
640 tsd_##n##p_get(tsd_t *tsd) \
646 JEMALLOC_ALWAYS_INLINE t \
647 tsd_##n##_get(tsd_t *tsd) \
650 return (*tsd_##n##p_get(tsd)); \
653 JEMALLOC_ALWAYS_INLINE void \
654 tsd_##n##_set(tsd_t *tsd, t n) \
657 assert(tsd->state == tsd_state_nominal); \
664 #endif /* JEMALLOC_H_INLINES */
665 /******************************************************************************/