]> git.proxmox.com Git - rustc.git/blame - src/jemalloc/include/jemalloc/internal/tsd.h
projects / rustc.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
New upstream version 1.22.1+dfsg1
[rustc.git] / src / jemalloc / include / jemalloc / internal / tsd.h
CommitLineData
1a4d82fc
JJ		 1/******************************************************************************/
2#ifdef JEMALLOC_H_TYPES
3
4/* Maximum number of malloc_tsd users with cleanup functions. */
54a0048b 5#define MALLOC_TSD_CLEANUPS_MAX 2
1a4d82fc
JJ		 6
7typedef bool (*malloc_tsd_cleanup_t)(void);
8
9#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
10 !defined(_WIN32))
11typedef struct tsd_init_block_s tsd_init_block_t;
12typedef struct tsd_init_head_s tsd_init_head_t;
13#endif
14
15typedef struct tsd_s tsd_t;
3b2f2976
XL		 16typedef struct tsdn_s tsdn_t;
17
18#define TSDN_NULL ((tsdn_t *)0)
1a4d82fc
JJ		 19
20typedef enum {
21 tsd_state_uninitialized,
22 tsd_state_nominal,
23 tsd_state_purgatory,
24 tsd_state_reincarnated
25} tsd_state_t;
26
27/*
28 * TLS/TSD-agnostic macro-based implementation of thread-specific data. There
54a0048b 29 * are five macros that support (at least) three use cases: file-private,
1a4d82fc
JJ		 30 * library-private, and library-private inlined. Following is an example
31 * library-private tsd variable:
32 *
33 * In example.h:
34 * typedef struct {
35 * int x;
36 * int y;
37 * } example_t;
38 * #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0})
54a0048b
SL		 39 * malloc_tsd_types(example_, example_t)
40 * malloc_tsd_protos(, example_, example_t)
41 * malloc_tsd_externs(example_, example_t)
1a4d82fc 42 * In example.c:
54a0048b
SL		 43 * malloc_tsd_data(, example_, example_t, EX_INITIALIZER)
44 * malloc_tsd_funcs(, example_, example_t, EX_INITIALIZER,
1a4d82fc
JJ		 45 * example_tsd_cleanup)
46 *
47 * The result is a set of generated functions, e.g.:
48 *
49 * bool example_tsd_boot(void) {...}
3b2f2976
XL		 50 * bool example_tsd_booted_get(void) {...}
51 * example_t *example_tsd_get(bool init) {...}
54a0048b 52 * void example_tsd_set(example_t *val) {...}
1a4d82fc
JJ		 53 *
54 * Note that all of the functions deal in terms of (a_type *) rather than
54a0048b 55 * (a_type) so that it is possible to support non-pointer types (unlike
1a4d82fc
JJ		 56 * pthreads TSD). example_tsd_cleanup() is passed an (a_type *) pointer that is
57 * cast to (void *). This means that the cleanup function needs to cast the
58 * function argument to (a_type *), then dereference the resulting pointer to
59 * access fields, e.g.
60 *
61 * void
62 * example_tsd_cleanup(void *arg)
63 * {
64 * example_t *example = (example_t *)arg;
65 *
66 * example->x = 42;
67 * [...]
68 * if ([want the cleanup function to be called again])
69 * example_tsd_set(example);
70 * }
71 *
72 * If example_tsd_set() is called within example_tsd_cleanup(), it will be
73 * called again. This is similar to how pthreads TSD destruction works, except
74 * that pthreads only calls the cleanup function again if the value was set to
75 * non-NULL.
76 */
77
54a0048b
SL		 78/* malloc_tsd_types(). */
79#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
80#define malloc_tsd_types(a_name, a_type)
81#elif (defined(JEMALLOC_TLS))
82#define malloc_tsd_types(a_name, a_type)
83#elif (defined(_WIN32))
84#define malloc_tsd_types(a_name, a_type) \
85typedef struct { \
86 bool initialized; \
87 a_type val; \
88} a_name##tsd_wrapper_t;
89#else
90#define malloc_tsd_types(a_name, a_type) \
91typedef struct { \
92 bool initialized; \
93 a_type val; \
94} a_name##tsd_wrapper_t;
95#endif
96
1a4d82fc
JJ		 97/* malloc_tsd_protos(). */
98#define malloc_tsd_protos(a_attr, a_name, a_type) \
99a_attr bool \
54a0048b
SL		 100a_name##tsd_boot0(void); \
101a_attr void \
102a_name##tsd_boot1(void); \
103a_attr bool \
1a4d82fc 104a_name##tsd_boot(void); \
3b2f2976
XL		 105a_attr bool \
106a_name##tsd_booted_get(void); \
1a4d82fc 107a_attr a_type * \
3b2f2976 108a_name##tsd_get(bool init); \
1a4d82fc
JJ		 109a_attr void \
110a_name##tsd_set(a_type *val);
111
112/* malloc_tsd_externs(). */
113#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
114#define malloc_tsd_externs(a_name, a_type) \
115extern __thread a_type a_name##tsd_tls; \
116extern __thread bool a_name##tsd_initialized; \
117extern bool a_name##tsd_booted;
118#elif (defined(JEMALLOC_TLS))
119#define malloc_tsd_externs(a_name, a_type) \
120extern __thread a_type a_name##tsd_tls; \
121extern pthread_key_t a_name##tsd_tsd; \
122extern bool a_name##tsd_booted;
123#elif (defined(_WIN32))
124#define malloc_tsd_externs(a_name, a_type) \
125extern DWORD a_name##tsd_tsd; \
54a0048b 126extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \
1a4d82fc
JJ		 127extern bool a_name##tsd_booted;
128#else
129#define malloc_tsd_externs(a_name, a_type) \
130extern pthread_key_t a_name##tsd_tsd; \
131extern tsd_init_head_t a_name##tsd_init_head; \
54a0048b 132extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \
1a4d82fc
JJ		 133extern bool a_name##tsd_booted;
134#endif
135
136/* malloc_tsd_data(). */
137#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
138#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
139a_attr __thread a_type JEMALLOC_TLS_MODEL \
140 a_name##tsd_tls = a_initializer; \
141a_attr __thread bool JEMALLOC_TLS_MODEL \
142 a_name##tsd_initialized = false; \
143a_attr bool a_name##tsd_booted = false;
144#elif (defined(JEMALLOC_TLS))
145#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
146a_attr __thread a_type JEMALLOC_TLS_MODEL \
147 a_name##tsd_tls = a_initializer; \
148a_attr pthread_key_t a_name##tsd_tsd; \
149a_attr bool a_name##tsd_booted = false;
150#elif (defined(_WIN32))
151#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
152a_attr DWORD a_name##tsd_tsd; \
54a0048b
SL		 153a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \
154 false, \
155 a_initializer \
156}; \
1a4d82fc
JJ		 157a_attr bool a_name##tsd_booted = false;
158#else
159#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
160a_attr pthread_key_t a_name##tsd_tsd; \
161a_attr tsd_init_head_t a_name##tsd_init_head = { \
162 ql_head_initializer(blocks), \
163 MALLOC_MUTEX_INITIALIZER \
164}; \
54a0048b
SL		 165a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \
166 false, \
167 a_initializer \
168}; \
1a4d82fc
JJ		 169a_attr bool a_name##tsd_booted = false;
170#endif
171
172/* malloc_tsd_funcs(). */
173#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
174#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
175 a_cleanup) \
176/* Initialization/cleanup. */ \
177a_attr bool \
178a_name##tsd_cleanup_wrapper(void) \
179{ \
180 \
181 if (a_name##tsd_initialized) { \
182 a_name##tsd_initialized = false; \
183 a_cleanup(&a_name##tsd_tls); \
184 } \
185 return (a_name##tsd_initialized); \
186} \
187a_attr bool \
54a0048b 188a_name##tsd_boot0(void) \
1a4d82fc
JJ		 189{ \
190 \
191 if (a_cleanup != malloc_tsd_no_cleanup) { \
192 malloc_tsd_cleanup_register( \
193 &a_name##tsd_cleanup_wrapper); \
194 } \
195 a_name##tsd_booted = true; \
196 return (false); \
197} \
54a0048b
SL		 198a_attr void \
199a_name##tsd_boot1(void) \
200{ \
201 \
202 /* Do nothing. */ \
203} \
204a_attr bool \
205a_name##tsd_boot(void) \
206{ \
207 \
208 return (a_name##tsd_boot0()); \
209} \
3b2f2976
XL		 210a_attr bool \
211a_name##tsd_booted_get(void) \
212{ \
213 \
214 return (a_name##tsd_booted); \
215} \
216a_attr bool \
217a_name##tsd_get_allocates(void) \
218{ \
219 \
220 return (false); \
221} \
1a4d82fc
JJ		 222/* Get/set. */ \
223a_attr a_type * \
3b2f2976 224a_name##tsd_get(bool init) \
1a4d82fc
JJ		 225{ \
226 \
227 assert(a_name##tsd_booted); \
228 return (&a_name##tsd_tls); \
229} \
230a_attr void \
231a_name##tsd_set(a_type *val) \
232{ \
233 \
234 assert(a_name##tsd_booted); \
235 a_name##tsd_tls = (*val); \
236 if (a_cleanup != malloc_tsd_no_cleanup) \
237 a_name##tsd_initialized = true; \
238}
239#elif (defined(JEMALLOC_TLS))
240#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
241 a_cleanup) \
242/* Initialization/cleanup. */ \
243a_attr bool \
54a0048b 244a_name##tsd_boot0(void) \
1a4d82fc
JJ		 245{ \
246 \
247 if (a_cleanup != malloc_tsd_no_cleanup) { \
248 if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) != \
249 0) \
250 return (true); \
251 } \
252 a_name##tsd_booted = true; \
253 return (false); \
254} \
54a0048b
SL		 255a_attr void \
256a_name##tsd_boot1(void) \
257{ \
258 \
259 /* Do nothing. */ \
260} \
261a_attr bool \
262a_name##tsd_boot(void) \
263{ \
264 \
265 return (a_name##tsd_boot0()); \
266} \
3b2f2976
XL		 267a_attr bool \
268a_name##tsd_booted_get(void) \
269{ \
270 \
271 return (a_name##tsd_booted); \
272} \
273a_attr bool \
274a_name##tsd_get_allocates(void) \
275{ \
276 \
277 return (false); \
278} \
1a4d82fc
JJ		 279/* Get/set. */ \
280a_attr a_type * \
3b2f2976 281a_name##tsd_get(bool init) \
1a4d82fc
JJ		 282{ \
283 \
284 assert(a_name##tsd_booted); \
285 return (&a_name##tsd_tls); \
286} \
287a_attr void \
288a_name##tsd_set(a_type *val) \
289{ \
290 \
291 assert(a_name##tsd_booted); \
292 a_name##tsd_tls = (*val); \
293 if (a_cleanup != malloc_tsd_no_cleanup) { \
294 if (pthread_setspecific(a_name##tsd_tsd, \
295 (void *)(&a_name##tsd_tls))) { \
296 malloc_write("<jemalloc>: Error" \
297 " setting TSD for "#a_name"\n"); \
298 if (opt_abort) \
299 abort(); \
300 } \
301 } \
302}
303#elif (defined(_WIN32))
304#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
305 a_cleanup) \
1a4d82fc
JJ		 306/* Initialization/cleanup. */ \
307a_attr bool \
308a_name##tsd_cleanup_wrapper(void) \
309{ \
54a0048b
SL		 310 DWORD error = GetLastError(); \
311 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
312 TlsGetValue(a_name##tsd_tsd); \
313 SetLastError(error); \
1a4d82fc 314 \
1a4d82fc
JJ		 315 if (wrapper == NULL) \
316 return (false); \
317 if (a_cleanup != malloc_tsd_no_cleanup && \
318 wrapper->initialized) { \
319 wrapper->initialized = false; \
320 a_cleanup(&wrapper->val); \
321 if (wrapper->initialized) { \
322 /* Trigger another cleanup round. */ \
323 return (true); \
324 } \
325 } \
326 malloc_tsd_dalloc(wrapper); \
327 return (false); \
328} \
54a0048b
SL		 329a_attr void \
330a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \
1a4d82fc
JJ		 331{ \
332 \
54a0048b
SL		 333 if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) { \
334 malloc_write("<jemalloc>: Error setting" \
335 " TSD for "#a_name"\n"); \
336 abort(); \
1a4d82fc 337 } \
1a4d82fc 338} \
1a4d82fc 339a_attr a_name##tsd_wrapper_t * \
3b2f2976 340a_name##tsd_wrapper_get(bool init) \
1a4d82fc 341{ \
54a0048b 342 DWORD error = GetLastError(); \
1a4d82fc
JJ		 343 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
344 TlsGetValue(a_name##tsd_tsd); \
54a0048b 345 SetLastError(error); \
1a4d82fc 346 \
3b2f2976 347 if (init && unlikely(wrapper == NULL)) { \
1a4d82fc
JJ		 348 wrapper = (a_name##tsd_wrapper_t *) \
349 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
350 if (wrapper == NULL) { \
351 malloc_write("<jemalloc>: Error allocating" \
352 " TSD for "#a_name"\n"); \
353 abort(); \
354 } else { \
355 wrapper->initialized = false; \
356 wrapper->val = a_initializer; \
357 } \
54a0048b 358 a_name##tsd_wrapper_set(wrapper); \
1a4d82fc
JJ		 359 } \
360 return (wrapper); \
361} \
54a0048b
SL		 362a_attr bool \
363a_name##tsd_boot0(void) \
364{ \
365 \
366 a_name##tsd_tsd = TlsAlloc(); \
367 if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES) \
368 return (true); \
369 if (a_cleanup != malloc_tsd_no_cleanup) { \
370 malloc_tsd_cleanup_register( \
371 &a_name##tsd_cleanup_wrapper); \
372 } \
373 a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \
374 a_name##tsd_booted = true; \
375 return (false); \
376} \
377a_attr void \
378a_name##tsd_boot1(void) \
379{ \
380 a_name##tsd_wrapper_t *wrapper; \
381 wrapper = (a_name##tsd_wrapper_t *) \
382 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
383 if (wrapper == NULL) { \
384 malloc_write("<jemalloc>: Error allocating" \
385 " TSD for "#a_name"\n"); \
386 abort(); \
387 } \
388 memcpy(wrapper, &a_name##tsd_boot_wrapper, \
389 sizeof(a_name##tsd_wrapper_t)); \
390 a_name##tsd_wrapper_set(wrapper); \
391} \
392a_attr bool \
393a_name##tsd_boot(void) \
394{ \
395 \
396 if (a_name##tsd_boot0()) \
397 return (true); \
398 a_name##tsd_boot1(); \
399 return (false); \
400} \
3b2f2976
XL		 401a_attr bool \
402a_name##tsd_booted_get(void) \
403{ \
404 \
405 return (a_name##tsd_booted); \
406} \
407a_attr bool \
408a_name##tsd_get_allocates(void) \
409{ \
410 \
411 return (true); \
412} \
54a0048b 413/* Get/set. */ \
1a4d82fc 414a_attr a_type * \
3b2f2976 415a_name##tsd_get(bool init) \
1a4d82fc
JJ		 416{ \
417 a_name##tsd_wrapper_t *wrapper; \
418 \
419 assert(a_name##tsd_booted); \
3b2f2976
XL		 420 wrapper = a_name##tsd_wrapper_get(init); \
421 if (a_name##tsd_get_allocates() && !init && wrapper == NULL) \
422 return (NULL); \
1a4d82fc
JJ		 423 return (&wrapper->val); \
424} \
425a_attr void \
426a_name##tsd_set(a_type *val) \
427{ \
428 a_name##tsd_wrapper_t *wrapper; \
429 \
430 assert(a_name##tsd_booted); \
3b2f2976 431 wrapper = a_name##tsd_wrapper_get(true); \
1a4d82fc
JJ		 432 wrapper->val = *(val); \
433 if (a_cleanup != malloc_tsd_no_cleanup) \
434 wrapper->initialized = true; \
435}
436#else
437#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
438 a_cleanup) \
1a4d82fc
JJ		 439/* Initialization/cleanup. */ \
440a_attr void \
441a_name##tsd_cleanup_wrapper(void *arg) \
442{ \
443 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg; \
444 \
445 if (a_cleanup != malloc_tsd_no_cleanup && \
446 wrapper->initialized) { \
447 wrapper->initialized = false; \
448 a_cleanup(&wrapper->val); \
449 if (wrapper->initialized) { \
450 /* Trigger another cleanup round. */ \
451 if (pthread_setspecific(a_name##tsd_tsd, \
452 (void *)wrapper)) { \
453 malloc_write("<jemalloc>: Error" \
454 " setting TSD for "#a_name"\n"); \
455 if (opt_abort) \
456 abort(); \
457 } \
458 return; \
459 } \
460 } \
461 malloc_tsd_dalloc(wrapper); \
462} \
54a0048b
SL		 463a_attr void \
464a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \
1a4d82fc
JJ		 465{ \
466 \
54a0048b
SL		 467 if (pthread_setspecific(a_name##tsd_tsd, \
468 (void *)wrapper)) { \
469 malloc_write("<jemalloc>: Error setting" \
470 " TSD for "#a_name"\n"); \
471 abort(); \
472 } \
1a4d82fc 473} \
1a4d82fc 474a_attr a_name##tsd_wrapper_t * \
3b2f2976 475a_name##tsd_wrapper_get(bool init) \
1a4d82fc
JJ		 476{ \
477 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
478 pthread_getspecific(a_name##tsd_tsd); \
479 \
3b2f2976 480 if (init && unlikely(wrapper == NULL)) { \
1a4d82fc 481 tsd_init_block_t block; \
3b2f2976
XL		 482 wrapper = (a_name##tsd_wrapper_t *) \
483 tsd_init_check_recursion(&a_name##tsd_init_head, \
484 &block); \
1a4d82fc
JJ		 485 if (wrapper) \
486 return (wrapper); \
487 wrapper = (a_name##tsd_wrapper_t *) \
488 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
3b2f2976 489 block.data = (void *)wrapper; \
1a4d82fc
JJ		 490 if (wrapper == NULL) { \
491 malloc_write("<jemalloc>: Error allocating" \
492 " TSD for "#a_name"\n"); \
493 abort(); \
494 } else { \
495 wrapper->initialized = false; \
496 wrapper->val = a_initializer; \
497 } \
54a0048b 498 a_name##tsd_wrapper_set(wrapper); \
1a4d82fc
JJ		 499 tsd_init_finish(&a_name##tsd_init_head, &block); \
500 } \
501 return (wrapper); \
502} \
54a0048b
SL		 503a_attr bool \
504a_name##tsd_boot0(void) \
505{ \
506 \
507 if (pthread_key_create(&a_name##tsd_tsd, \
508 a_name##tsd_cleanup_wrapper) != 0) \
509 return (true); \
510 a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \
511 a_name##tsd_booted = true; \
512 return (false); \
513} \
514a_attr void \
515a_name##tsd_boot1(void) \
516{ \
517 a_name##tsd_wrapper_t *wrapper; \
518 wrapper = (a_name##tsd_wrapper_t *) \
519 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
520 if (wrapper == NULL) { \
521 malloc_write("<jemalloc>: Error allocating" \
522 " TSD for "#a_name"\n"); \
523 abort(); \
524 } \
525 memcpy(wrapper, &a_name##tsd_boot_wrapper, \
526 sizeof(a_name##tsd_wrapper_t)); \
527 a_name##tsd_wrapper_set(wrapper); \
528} \
529a_attr bool \
530a_name##tsd_boot(void) \
531{ \
532 \
533 if (a_name##tsd_boot0()) \
534 return (true); \
535 a_name##tsd_boot1(); \
536 return (false); \
537} \
3b2f2976
XL		 538a_attr bool \
539a_name##tsd_booted_get(void) \
540{ \
541 \
542 return (a_name##tsd_booted); \
543} \
544a_attr bool \
545a_name##tsd_get_allocates(void) \
546{ \
547 \
548 return (true); \
549} \
54a0048b 550/* Get/set. */ \
1a4d82fc 551a_attr a_type * \
3b2f2976 552a_name##tsd_get(bool init) \
1a4d82fc
JJ		 553{ \
554 a_name##tsd_wrapper_t *wrapper; \
555 \
556 assert(a_name##tsd_booted); \
3b2f2976
XL		 557 wrapper = a_name##tsd_wrapper_get(init); \
558 if (a_name##tsd_get_allocates() && !init && wrapper == NULL) \
559 return (NULL); \
1a4d82fc
JJ		 560 return (&wrapper->val); \
561} \
562a_attr void \
563a_name##tsd_set(a_type *val) \
564{ \
565 a_name##tsd_wrapper_t *wrapper; \
566 \
567 assert(a_name##tsd_booted); \
3b2f2976 568 wrapper = a_name##tsd_wrapper_get(true); \
1a4d82fc
JJ		 569 wrapper->val = *(val); \
570 if (a_cleanup != malloc_tsd_no_cleanup) \
571 wrapper->initialized = true; \
572}
573#endif
574
575#endif /* JEMALLOC_H_TYPES */
576/******************************************************************************/
577#ifdef JEMALLOC_H_STRUCTS
578
579#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
580 !defined(_WIN32))
581struct tsd_init_block_s {
582 ql_elm(tsd_init_block_t) link;
583 pthread_t thread;
584 void *data;
585};
586struct tsd_init_head_s {
587 ql_head(tsd_init_block_t) blocks;
588 malloc_mutex_t lock;
589};
590#endif
591
592#define MALLOC_TSD \
593/* O(name, type) */ \
594 O(tcache, tcache_t *) \
595 O(thread_allocated, uint64_t) \
596 O(thread_deallocated, uint64_t) \
597 O(prof_tdata, prof_tdata_t *) \
3b2f2976 598 O(iarena, arena_t *) \
1a4d82fc 599 O(arena, arena_t *) \
54a0048b
SL		 600 O(arenas_tdata, arena_tdata_t *) \
601 O(narenas_tdata, unsigned) \
602 O(arenas_tdata_bypass, bool) \
1a4d82fc
JJ		 603 O(tcache_enabled, tcache_enabled_t) \
604 O(quarantine, quarantine_t *) \
3b2f2976
XL		 605 O(witnesses, witness_list_t) \
606 O(witness_fork, bool) \
1a4d82fc
JJ		 607
608#define TSD_INITIALIZER { \
609 tsd_state_uninitialized, \
610 NULL, \
611 0, \
612 0, \
613 NULL, \
614 NULL, \
54a0048b 615 NULL, \
3b2f2976 616 NULL, \
54a0048b
SL		 617 0, \
618 false, \
1a4d82fc 619 tcache_enabled_default, \
3b2f2976
XL		 620 NULL, \
621 ql_head_initializer(witnesses), \
622 false \
1a4d82fc
JJ		 623}
624
625struct tsd_s {
626 tsd_state_t state;
627#define O(n, t) \
628 t n;
629MALLOC_TSD
630#undef O
631};
632
3b2f2976
XL		 633/*
634 * Wrapper around tsd_t that makes it possible to avoid implicit conversion
635 * between tsd_t and tsdn_t, where tsdn_t is "nullable" and has to be
636 * explicitly converted to tsd_t, which is non-nullable.
637 */
638struct tsdn_s {
639 tsd_t tsd;
640};
641
1a4d82fc
JJ		 642static const tsd_t tsd_initializer = TSD_INITIALIZER;
643
54a0048b
SL		 644malloc_tsd_types(, tsd_t)
645
1a4d82fc
JJ		 646#endif /* JEMALLOC_H_STRUCTS */
647/******************************************************************************/
648#ifdef JEMALLOC_H_EXTERNS
649
650void *malloc_tsd_malloc(size_t size);
651void malloc_tsd_dalloc(void *wrapper);
652void malloc_tsd_no_cleanup(void *arg);
653void malloc_tsd_cleanup_register(bool (*f)(void));
3b2f2976 654tsd_t *malloc_tsd_boot0(void);
54a0048b 655void malloc_tsd_boot1(void);
1a4d82fc
JJ		 656#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
657 !defined(_WIN32))
658void *tsd_init_check_recursion(tsd_init_head_t *head,
659 tsd_init_block_t *block);
660void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
661#endif
662void tsd_cleanup(void *arg);
663
664#endif /* JEMALLOC_H_EXTERNS */
665/******************************************************************************/
666#ifdef JEMALLOC_H_INLINES
667
668#ifndef JEMALLOC_ENABLE_INLINE
669malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t)
670
3b2f2976 671tsd_t *tsd_fetch_impl(bool init);
1a4d82fc 672tsd_t *tsd_fetch(void);
3b2f2976 673tsdn_t *tsd_tsdn(tsd_t *tsd);
1a4d82fc
JJ		 674bool tsd_nominal(tsd_t *tsd);
675#define O(n, t) \
676t *tsd_##n##p_get(tsd_t *tsd); \
677t tsd_##n##_get(tsd_t *tsd); \
678void tsd_##n##_set(tsd_t *tsd, t n);
679MALLOC_TSD
680#undef O
3b2f2976
XL		 681tsdn_t *tsdn_fetch(void);
682bool tsdn_null(const tsdn_t *tsdn);
683tsd_t *tsdn_tsd(tsdn_t *tsdn);
1a4d82fc
JJ		 684#endif
685
686#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_))
687malloc_tsd_externs(, tsd_t)
688malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup)
689
690JEMALLOC_ALWAYS_INLINE tsd_t *
3b2f2976 691tsd_fetch_impl(bool init)
1a4d82fc 692{
3b2f2976
XL		 693 tsd_t *tsd = tsd_get(init);
694
695 if (!init && tsd_get_allocates() && tsd == NULL)
696 return (NULL);
697 assert(tsd != NULL);
1a4d82fc
JJ		 698
699 if (unlikely(tsd->state != tsd_state_nominal)) {
700 if (tsd->state == tsd_state_uninitialized) {
701 tsd->state = tsd_state_nominal;
702 /* Trigger cleanup handler registration. */
703 tsd_set(tsd);
704 } else if (tsd->state == tsd_state_purgatory) {
705 tsd->state = tsd_state_reincarnated;
706 tsd_set(tsd);
707 } else
708 assert(tsd->state == tsd_state_reincarnated);
709 }
710
711 return (tsd);
712}
713
3b2f2976
XL		 714JEMALLOC_ALWAYS_INLINE tsd_t *
715tsd_fetch(void)
716{
717
718 return (tsd_fetch_impl(true));
719}
720
721JEMALLOC_ALWAYS_INLINE tsdn_t *
722tsd_tsdn(tsd_t *tsd)
723{
724
725 return ((tsdn_t *)tsd);
726}
727
1a4d82fc
JJ		 728JEMALLOC_INLINE bool
729tsd_nominal(tsd_t *tsd)
730{
731
732 return (tsd->state == tsd_state_nominal);
733}
734
735#define O(n, t) \
736JEMALLOC_ALWAYS_INLINE t * \
737tsd_##n##p_get(tsd_t *tsd) \
738{ \
739 \
740 return (&tsd->n); \
741} \
742 \
743JEMALLOC_ALWAYS_INLINE t \
744tsd_##n##_get(tsd_t *tsd) \
745{ \
746 \
747 return (*tsd_##n##p_get(tsd)); \
748} \
749 \
750JEMALLOC_ALWAYS_INLINE void \
751tsd_##n##_set(tsd_t *tsd, t n) \
752{ \
753 \
754 assert(tsd->state == tsd_state_nominal); \
755 tsd->n = n; \
756}
757MALLOC_TSD
758#undef O
3b2f2976
XL		 759
760JEMALLOC_ALWAYS_INLINE tsdn_t *
761tsdn_fetch(void)
762{
763
764 if (!tsd_booted_get())
765 return (NULL);
766
767 return (tsd_tsdn(tsd_fetch_impl(false)));
768}
769
770JEMALLOC_ALWAYS_INLINE bool
771tsdn_null(const tsdn_t *tsdn)
772{
773
774 return (tsdn == NULL);
775}
776
777JEMALLOC_ALWAYS_INLINE tsd_t *
778tsdn_tsd(tsdn_t *tsdn)
779{
780
781 assert(!tsdn_null(tsdn));
782
783 return (&tsdn->tsd);
784}
1a4d82fc
JJ		 785#endif
786
787#endif /* JEMALLOC_H_INLINES */
788/******************************************************************************/
backport for Debian buster