]> git.proxmox.com Git - rustc.git/blob - src/jemalloc/src/prof.c
projects / rustc.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Imported Upstream version 1.8.0+dfsg1
[rustc.git] / src / jemalloc / src / prof.c
1 #define JEMALLOC_PROF_C_
2 #include "jemalloc/internal/jemalloc_internal.h"
3 /******************************************************************************/
4
5 #ifdef JEMALLOC_PROF_LIBUNWIND
6 #define UNW_LOCAL_ONLY
7 #include <libunwind.h>
8 #endif
9
10 #ifdef JEMALLOC_PROF_LIBGCC
11 #include <unwind.h>
12 #endif
13
14 /******************************************************************************/
15 /* Data. */
16
17 bool opt_prof = false;
18 bool opt_prof_active = true;
19 bool opt_prof_thread_active_init = true;
20 size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT;
21 ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT;
22 bool opt_prof_gdump = false;
23 bool opt_prof_final = true;
24 bool opt_prof_leak = false;
25 bool opt_prof_accum = false;
26 char opt_prof_prefix[
27 /* Minimize memory bloat for non-prof builds. */
28 #ifdef JEMALLOC_PROF
29 PATH_MAX +
30 #endif
31 1];
32
33 /*
34 * Initialized as opt_prof_active, and accessed via
35 * prof_active_[gs]et{_unlocked,}().
36 */
37 bool prof_active;
38 static malloc_mutex_t prof_active_mtx;
39
40 /*
41 * Initialized as opt_prof_thread_active_init, and accessed via
42 * prof_thread_active_init_[gs]et().
43 */
44 static bool prof_thread_active_init;
45 static malloc_mutex_t prof_thread_active_init_mtx;
46
47 uint64_t prof_interval = 0;
48
49 size_t lg_prof_sample;
50
51 /*
52 * Table of mutexes that are shared among gctx's. These are leaf locks, so
53 * there is no problem with using them for more than one gctx at the same time.
54 * The primary motivation for this sharing though is that gctx's are ephemeral,
55 * and destroying mutexes causes complications for systems that allocate when
56 * creating/destroying mutexes.
57 */
58 static malloc_mutex_t *gctx_locks;
59 static unsigned cum_gctxs; /* Atomic counter. */
60
61 /*
62 * Table of mutexes that are shared among tdata's. No operations require
63 * holding multiple tdata locks, so there is no problem with using them for more
64 * than one tdata at the same time, even though a gctx lock may be acquired
65 * while holding a tdata lock.
66 */
67 static malloc_mutex_t *tdata_locks;
68
69 /*
70 * Global hash of (prof_bt_t *)-->(prof_gctx_t *). This is the master data
71 * structure that knows about all backtraces currently captured.
72 */
73 static ckh_t bt2gctx;
74 static malloc_mutex_t bt2gctx_mtx;
75
76 /*
77 * Tree of all extant prof_tdata_t structures, regardless of state,
78 * {attached,detached,expired}.
79 */
80 static prof_tdata_tree_t tdatas;
81 static malloc_mutex_t tdatas_mtx;
82
83 static uint64_t next_thr_uid;
84 static malloc_mutex_t next_thr_uid_mtx;
85
86 static malloc_mutex_t prof_dump_seq_mtx;
87 static uint64_t prof_dump_seq;
88 static uint64_t prof_dump_iseq;
89 static uint64_t prof_dump_mseq;
90 static uint64_t prof_dump_useq;
91
92 /*
93 * This buffer is rather large for stack allocation, so use a single buffer for
94 * all profile dumps.
95 */
96 static malloc_mutex_t prof_dump_mtx;
97 static char prof_dump_buf[
98 /* Minimize memory bloat for non-prof builds. */
99 #ifdef JEMALLOC_PROF
100 PROF_DUMP_BUFSIZE
101 #else
102 1
103 #endif
104 ];
105 static unsigned prof_dump_buf_end;
106 static int prof_dump_fd;
107
108 /* Do not dump any profiles until bootstrapping is complete. */
109 static bool prof_booted = false;
110
111 /******************************************************************************/
112 /*
113 * Function prototypes for static functions that are referenced prior to
114 * definition.
115 */
116
117 static bool prof_tctx_should_destroy(prof_tctx_t *tctx);
118 static void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx);
119 static bool prof_tdata_should_destroy(prof_tdata_t *tdata,
120 bool even_if_attached);
121 static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata,
122 bool even_if_attached);
123 static char *prof_thread_name_alloc(tsd_t *tsd, const char *thread_name);
124
125 /******************************************************************************/
126 /* Red-black trees. */
127
128 JEMALLOC_INLINE_C int
129 prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b)
130 {
131 uint64_t a_uid = a->tdata->thr_uid;
132 uint64_t b_uid = b->tdata->thr_uid;
133
134 return ((a_uid > b_uid) - (a_uid < b_uid));
135 }
136
137 rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t,
138 tctx_link, prof_tctx_comp)
139
140 JEMALLOC_INLINE_C int
141 prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b)
142 {
143 unsigned a_len = a->bt.len;
144 unsigned b_len = b->bt.len;
145 unsigned comp_len = (a_len < b_len) ? a_len : b_len;
146 int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *));
147 if (ret == 0)
148 ret = (a_len > b_len) - (a_len < b_len);
149 return (ret);
150 }
151
152 rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link,
153 prof_gctx_comp)
154
155 JEMALLOC_INLINE_C int
156 prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b)
157 {
158 int ret;
159 uint64_t a_uid = a->thr_uid;
160 uint64_t b_uid = b->thr_uid;
161
162 ret = ((a_uid > b_uid) - (a_uid < b_uid));
163 if (ret == 0) {
164 uint64_t a_discrim = a->thr_discrim;
165 uint64_t b_discrim = b->thr_discrim;
166
167 ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim));
168 }
169 return (ret);
170 }
171
172 rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link,
173 prof_tdata_comp)
174
175 /******************************************************************************/
176
177 void
178 prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated)
179 {
180 prof_tdata_t *tdata;
181
182 cassert(config_prof);
183
184 if (updated) {
185 /*
186 * Compute a new sample threshold. This isn't very important in
187 * practice, because this function is rarely executed, so the
188 * potential for sample bias is minimal except in contrived
189 * programs.
190 */
191 tdata = prof_tdata_get(tsd, true);
192 if (tdata != NULL)
193 prof_sample_threshold_update(tctx->tdata);
194 }
195
196 if ((uintptr_t)tctx > (uintptr_t)1U) {
197 malloc_mutex_lock(tctx->tdata->lock);
198 tctx->prepared = false;
199 if (prof_tctx_should_destroy(tctx))
200 prof_tctx_destroy(tsd, tctx);
201 else
202 malloc_mutex_unlock(tctx->tdata->lock);
203 }
204 }
205
206 void
207 prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx) {
208 prof_tctx_set(ptr, tctx);
209
210 malloc_mutex_lock(tctx->tdata->lock);
211 tctx->cnts.curobjs++;
212 tctx->cnts.curbytes += usize;
213 if (opt_prof_accum) {
214 tctx->cnts.accumobjs++;
215 tctx->cnts.accumbytes += usize;
216 }
217 tctx->prepared = false;
218 malloc_mutex_unlock(tctx->tdata->lock);
219 }
220
221 void
222 prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx)
223 {
224
225 malloc_mutex_lock(tctx->tdata->lock);
226 assert(tctx->cnts.curobjs > 0);
227 assert(tctx->cnts.curbytes >= usize);
228 tctx->cnts.curobjs--;
229 tctx->cnts.curbytes -= usize;
230
231 if (prof_tctx_should_destroy(tctx))
232 prof_tctx_destroy(tsd, tctx);
233 else
234 malloc_mutex_unlock(tctx->tdata->lock);
235 }
236
237 void
238 bt_init(prof_bt_t *bt, void **vec)
239 {
240
241 cassert(config_prof);
242
243 bt->vec = vec;
244 bt->len = 0;
245 }
246
247 static inline void
248 prof_enter(prof_tdata_t *tdata)
249 {
250
251 cassert(config_prof);
252
253 assert(!tdata->enq);
254 tdata->enq = true;
255
256 malloc_mutex_lock(&bt2gctx_mtx);
257 }
258
259 static inline void
260 prof_leave(prof_tdata_t *tdata)
261 {
262 bool idump, gdump;
263
264 cassert(config_prof);
265
266 malloc_mutex_unlock(&bt2gctx_mtx);
267
268 assert(tdata->enq);
269 tdata->enq = false;
270 idump = tdata->enq_idump;
271 tdata->enq_idump = false;
272 gdump = tdata->enq_gdump;
273 tdata->enq_gdump = false;
274
275 if (idump)
276 prof_idump();
277 if (gdump)
278 prof_gdump();
279 }
280
281 #ifdef JEMALLOC_PROF_LIBUNWIND
282 void
283 prof_backtrace(prof_bt_t *bt)
284 {
285 int nframes;
286
287 cassert(config_prof);
288 assert(bt->len == 0);
289 assert(bt->vec != NULL);
290
291 nframes = unw_backtrace(bt->vec, PROF_BT_MAX);
292 if (nframes <= 0)
293 return;
294 bt->len = nframes;
295 }
296 #elif (defined(JEMALLOC_PROF_LIBGCC))
297 static _Unwind_Reason_Code
298 prof_unwind_init_callback(struct _Unwind_Context *context, void *arg)
299 {
300
301 cassert(config_prof);
302
303 return (_URC_NO_REASON);
304 }
305
306 static _Unwind_Reason_Code
307 prof_unwind_callback(struct _Unwind_Context *context, void *arg)
308 {
309 prof_unwind_data_t *data = (prof_unwind_data_t *)arg;
310 void *ip;
311
312 cassert(config_prof);
313
314 ip = (void *)_Unwind_GetIP(context);
315 if (ip == NULL)
316 return (_URC_END_OF_STACK);
317 data->bt->vec[data->bt->len] = ip;
318 data->bt->len++;
319 if (data->bt->len == data->max)
320 return (_URC_END_OF_STACK);
321
322 return (_URC_NO_REASON);
323 }
324
325 void
326 prof_backtrace(prof_bt_t *bt)
327 {
328 prof_unwind_data_t data = {bt, PROF_BT_MAX};
329
330 cassert(config_prof);
331
332 _Unwind_Backtrace(prof_unwind_callback, &data);
333 }
334 #elif (defined(JEMALLOC_PROF_GCC))
335 void
336 prof_backtrace(prof_bt_t *bt)
337 {
338 #define BT_FRAME(i) \
339 if ((i) < PROF_BT_MAX) { \
340 void *p; \
341 if (__builtin_frame_address(i) == 0) \
342 return; \
343 p = __builtin_return_address(i); \
344 if (p == NULL) \
345 return; \
346 bt->vec[(i)] = p; \
347 bt->len = (i) + 1; \
348 } else \
349 return;
350
351 cassert(config_prof);
352
353 BT_FRAME(0)
354 BT_FRAME(1)
355 BT_FRAME(2)
356 BT_FRAME(3)
357 BT_FRAME(4)
358 BT_FRAME(5)
359 BT_FRAME(6)
360 BT_FRAME(7)
361 BT_FRAME(8)
362 BT_FRAME(9)
363
364 BT_FRAME(10)
365 BT_FRAME(11)
366 BT_FRAME(12)
367 BT_FRAME(13)
368 BT_FRAME(14)
369 BT_FRAME(15)
370 BT_FRAME(16)
371 BT_FRAME(17)
372 BT_FRAME(18)
373 BT_FRAME(19)
374
375 BT_FRAME(20)
376 BT_FRAME(21)
377 BT_FRAME(22)
378 BT_FRAME(23)
379 BT_FRAME(24)
380 BT_FRAME(25)
381 BT_FRAME(26)
382 BT_FRAME(27)
383 BT_FRAME(28)
384 BT_FRAME(29)
385
386 BT_FRAME(30)
387 BT_FRAME(31)
388 BT_FRAME(32)
389 BT_FRAME(33)
390 BT_FRAME(34)
391 BT_FRAME(35)
392 BT_FRAME(36)
393 BT_FRAME(37)
394 BT_FRAME(38)
395 BT_FRAME(39)
396
397 BT_FRAME(40)
398 BT_FRAME(41)
399 BT_FRAME(42)
400 BT_FRAME(43)
401 BT_FRAME(44)
402 BT_FRAME(45)
403 BT_FRAME(46)
404 BT_FRAME(47)
405 BT_FRAME(48)
406 BT_FRAME(49)
407
408 BT_FRAME(50)
409 BT_FRAME(51)
410 BT_FRAME(52)
411 BT_FRAME(53)
412 BT_FRAME(54)
413 BT_FRAME(55)
414 BT_FRAME(56)
415 BT_FRAME(57)
416 BT_FRAME(58)
417 BT_FRAME(59)
418
419 BT_FRAME(60)
420 BT_FRAME(61)
421 BT_FRAME(62)
422 BT_FRAME(63)
423 BT_FRAME(64)
424 BT_FRAME(65)
425 BT_FRAME(66)
426 BT_FRAME(67)
427 BT_FRAME(68)
428 BT_FRAME(69)
429
430 BT_FRAME(70)
431 BT_FRAME(71)
432 BT_FRAME(72)
433 BT_FRAME(73)
434 BT_FRAME(74)
435 BT_FRAME(75)
436 BT_FRAME(76)
437 BT_FRAME(77)
438 BT_FRAME(78)
439 BT_FRAME(79)
440
441 BT_FRAME(80)
442 BT_FRAME(81)
443 BT_FRAME(82)
444 BT_FRAME(83)
445 BT_FRAME(84)
446 BT_FRAME(85)
447 BT_FRAME(86)
448 BT_FRAME(87)
449 BT_FRAME(88)
450 BT_FRAME(89)
451
452 BT_FRAME(90)
453 BT_FRAME(91)
454 BT_FRAME(92)
455 BT_FRAME(93)
456 BT_FRAME(94)
457 BT_FRAME(95)
458 BT_FRAME(96)
459 BT_FRAME(97)
460 BT_FRAME(98)
461 BT_FRAME(99)
462
463 BT_FRAME(100)
464 BT_FRAME(101)
465 BT_FRAME(102)
466 BT_FRAME(103)
467 BT_FRAME(104)
468 BT_FRAME(105)
469 BT_FRAME(106)
470 BT_FRAME(107)
471 BT_FRAME(108)
472 BT_FRAME(109)
473
474 BT_FRAME(110)
475 BT_FRAME(111)
476 BT_FRAME(112)
477 BT_FRAME(113)
478 BT_FRAME(114)
479 BT_FRAME(115)
480 BT_FRAME(116)
481 BT_FRAME(117)
482 BT_FRAME(118)
483 BT_FRAME(119)
484
485 BT_FRAME(120)
486 BT_FRAME(121)
487 BT_FRAME(122)
488 BT_FRAME(123)
489 BT_FRAME(124)
490 BT_FRAME(125)
491 BT_FRAME(126)
492 BT_FRAME(127)
493 #undef BT_FRAME
494 }
495 #else
496 void
497 prof_backtrace(prof_bt_t *bt)
498 {
499
500 cassert(config_prof);
501 not_reached();
502 }
503 #endif
504
505 static malloc_mutex_t *
506 prof_gctx_mutex_choose(void)
507 {
508 unsigned ngctxs = atomic_add_u(&cum_gctxs, 1);
509
510 return (&gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS]);
511 }
512
513 static malloc_mutex_t *
514 prof_tdata_mutex_choose(uint64_t thr_uid)
515 {
516
517 return (&tdata_locks[thr_uid % PROF_NTDATA_LOCKS]);
518 }
519
520 static prof_gctx_t *
521 prof_gctx_create(tsd_t *tsd, prof_bt_t *bt)
522 {
523 /*
524 * Create a single allocation that has space for vec of length bt->len.
525 */
526 prof_gctx_t *gctx = (prof_gctx_t *)imalloc(tsd, offsetof(prof_gctx_t,
527 vec) + (bt->len * sizeof(void *)));
528 if (gctx == NULL)
529 return (NULL);
530 gctx->lock = prof_gctx_mutex_choose();
531 /*
532 * Set nlimbo to 1, in order to avoid a race condition with
533 * prof_tctx_destroy()/prof_gctx_try_destroy().
534 */
535 gctx->nlimbo = 1;
536 tctx_tree_new(&gctx->tctxs);
537 /* Duplicate bt. */
538 memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *));
539 gctx->bt.vec = gctx->vec;
540 gctx->bt.len = bt->len;
541 return (gctx);
542 }
543
544 static void
545 prof_gctx_try_destroy(tsd_t *tsd, prof_gctx_t *gctx, prof_tdata_t *tdata)
546 {
547
548 cassert(config_prof);
549
550 /*
551 * Check that gctx is still unused by any thread cache before destroying
552 * it. prof_lookup() increments gctx->nlimbo in order to avoid a race
553 * condition with this function, as does prof_tctx_destroy() in order to
554 * avoid a race between the main body of prof_tctx_destroy() and entry
555 * into this function.
556 */
557 prof_enter(tdata);
558 malloc_mutex_lock(gctx->lock);
559 assert(gctx->nlimbo != 0);
560 if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) {
561 /* Remove gctx from bt2gctx. */
562 if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL))
563 not_reached();
564 prof_leave(tdata);
565 /* Destroy gctx. */
566 malloc_mutex_unlock(gctx->lock);
567 idalloc(tsd, gctx);
568 } else {
569 /*
570 * Compensate for increment in prof_tctx_destroy() or
571 * prof_lookup().
572 */
573 gctx->nlimbo--;
574 malloc_mutex_unlock(gctx->lock);
575 prof_leave(tdata);
576 }
577 }
578
579 /* tctx->tdata->lock must be held. */
580 static bool
581 prof_tctx_should_destroy(prof_tctx_t *tctx)
582 {
583
584 if (opt_prof_accum)
585 return (false);
586 if (tctx->cnts.curobjs != 0)
587 return (false);
588 if (tctx->prepared)
589 return (false);
590 return (true);
591 }
592
593 static bool
594 prof_gctx_should_destroy(prof_gctx_t *gctx)
595 {
596
597 if (opt_prof_accum)
598 return (false);
599 if (!tctx_tree_empty(&gctx->tctxs))
600 return (false);
601 if (gctx->nlimbo != 0)
602 return (false);
603 return (true);
604 }
605
606 /* tctx->tdata->lock is held upon entry, and released before return. */
607 static void
608 prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx)
609 {
610 prof_tdata_t *tdata = tctx->tdata;
611 prof_gctx_t *gctx = tctx->gctx;
612 bool destroy_tdata, destroy_gctx;
613
614 assert(tctx->cnts.curobjs == 0);
615 assert(tctx->cnts.curbytes == 0);
616 assert(!opt_prof_accum);
617 assert(tctx->cnts.accumobjs == 0);
618 assert(tctx->cnts.accumbytes == 0);
619
620 ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);
621 destroy_tdata = prof_tdata_should_destroy(tdata, false);
622 malloc_mutex_unlock(tdata->lock);
623
624 malloc_mutex_lock(gctx->lock);
625 tctx_tree_remove(&gctx->tctxs, tctx);
626 if (prof_gctx_should_destroy(gctx)) {
627 /*
628 * Increment gctx->nlimbo in order to keep another thread from
629 * winning the race to destroy gctx while this one has
630 * gctx->lock dropped. Without this, it would be possible for
631 * another thread to:
632 *
633 * 1) Sample an allocation associated with gctx.
634 * 2) Deallocate the sampled object.
635 * 3) Successfully prof_gctx_try_destroy(gctx).
636 *
637 * The result would be that gctx no longer exists by the time
638 * this thread accesses it in prof_gctx_try_destroy().
639 */
640 gctx->nlimbo++;
641 destroy_gctx = true;
642 } else
643 destroy_gctx = false;
644 malloc_mutex_unlock(gctx->lock);
645 if (destroy_gctx)
646 prof_gctx_try_destroy(tsd, gctx, tdata);
647
648 if (destroy_tdata)
649 prof_tdata_destroy(tsd, tdata, false);
650
651 idalloc(tsd, tctx);
652 }
653
654 static bool
655 prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
656 void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx)
657 {
658 union {
659 prof_gctx_t *p;
660 void *v;
661 } gctx;
662 union {
663 prof_bt_t *p;
664 void *v;
665 } btkey;
666 bool new_gctx;
667
668 prof_enter(tdata);
669 if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
670 /* bt has never been seen before. Insert it. */
671 gctx.p = prof_gctx_create(tsd, bt);
672 if (gctx.v == NULL) {
673 prof_leave(tdata);
674 return (true);
675 }
676 btkey.p = &gctx.p->bt;
677 if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) {
678 /* OOM. */
679 prof_leave(tdata);
680 idalloc(tsd, gctx.v);
681 return (true);
682 }
683 new_gctx = true;
684 } else {
685 /*
686 * Increment nlimbo, in order to avoid a race condition with
687 * prof_tctx_destroy()/prof_gctx_try_destroy().
688 */
689 malloc_mutex_lock(gctx.p->lock);
690 gctx.p->nlimbo++;
691 malloc_mutex_unlock(gctx.p->lock);
692 new_gctx = false;
693 }
694 prof_leave(tdata);
695
696 *p_btkey = btkey.v;
697 *p_gctx = gctx.p;
698 *p_new_gctx = new_gctx;
699 return (false);
700 }
701
702 prof_tctx_t *
703 prof_lookup(tsd_t *tsd, prof_bt_t *bt)
704 {
705 union {
706 prof_tctx_t *p;
707 void *v;
708 } ret;
709 prof_tdata_t *tdata;
710 bool not_found;
711
712 cassert(config_prof);
713
714 tdata = prof_tdata_get(tsd, false);
715 if (tdata == NULL)
716 return (NULL);
717
718 malloc_mutex_lock(tdata->lock);
719 not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v);
720 if (!not_found) /* Note double negative! */
721 ret.p->prepared = true;
722 malloc_mutex_unlock(tdata->lock);
723 if (not_found) {
724 void *btkey;
725 prof_gctx_t *gctx;
726 bool new_gctx, error;
727
728 /*
729 * This thread's cache lacks bt. Look for it in the global
730 * cache.
731 */
732 if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx,
733 &new_gctx))
734 return (NULL);
735
736 /* Link a prof_tctx_t into gctx for this thread. */
737 ret.v = imalloc(tsd, sizeof(prof_tctx_t));
738 if (ret.p == NULL) {
739 if (new_gctx)
740 prof_gctx_try_destroy(tsd, gctx, tdata);
741 return (NULL);
742 }
743 ret.p->tdata = tdata;
744 memset(&ret.p->cnts, 0, sizeof(prof_cnt_t));
745 ret.p->gctx = gctx;
746 ret.p->prepared = true;
747 ret.p->state = prof_tctx_state_initializing;
748 malloc_mutex_lock(tdata->lock);
749 error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);
750 malloc_mutex_unlock(tdata->lock);
751 if (error) {
752 if (new_gctx)
753 prof_gctx_try_destroy(tsd, gctx, tdata);
754 idalloc(tsd, ret.v);
755 return (NULL);
756 }
757 malloc_mutex_lock(gctx->lock);
758 ret.p->state = prof_tctx_state_nominal;
759 tctx_tree_insert(&gctx->tctxs, ret.p);
760 gctx->nlimbo--;
761 malloc_mutex_unlock(gctx->lock);
762 }
763
764 return (ret.p);
765 }
766
767 void
768 prof_sample_threshold_update(prof_tdata_t *tdata)
769 {
770 /*
771 * The body of this function is compiled out unless heap profiling is
772 * enabled, so that it is possible to compile jemalloc with floating
773 * point support completely disabled. Avoiding floating point code is
774 * important on memory-constrained systems, but it also enables a
775 * workaround for versions of glibc that don't properly save/restore
776 * floating point registers during dynamic lazy symbol loading (which
777 * internally calls into whatever malloc implementation happens to be
778 * integrated into the application). Note that some compilers (e.g.
779 * gcc 4.8) may use floating point registers for fast memory moves, so
780 * jemalloc must be compiled with such optimizations disabled (e.g.
781 * -mno-sse) in order for the workaround to be complete.
782 */
783 #ifdef JEMALLOC_PROF
784 uint64_t r;
785 double u;
786
787 if (!config_prof)
788 return;
789
790 if (lg_prof_sample == 0) {
791 tdata->bytes_until_sample = 0;
792 return;
793 }
794
795 /*
796 * Compute sample interval as a geometrically distributed random
797 * variable with mean (2^lg_prof_sample).
798 *
799 * __ __
800 * | log(u) | 1
801 * tdata->bytes_until_sample = | -------- |, where p = ---------------
802 * | log(1-p) | lg_prof_sample
803 * 2
804 *
805 * For more information on the math, see:
806 *
807 * Non-Uniform Random Variate Generation
808 * Luc Devroye
809 * Springer-Verlag, New York, 1986
810 * pp 500
811 * (http://luc.devroye.org/rnbookindex.html)
812 */
813 prng64(r, 53, tdata->prng_state, UINT64_C(6364136223846793005),
814 UINT64_C(1442695040888963407));
815 u = (double)r * (1.0/9007199254740992.0L);
816 tdata->bytes_until_sample = (uint64_t)(log(u) /
817 log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample))))
818 + (uint64_t)1U;
819 #endif
820 }
821
822 #ifdef JEMALLOC_JET
823 static prof_tdata_t *
824 prof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
825 {
826 size_t *tdata_count = (size_t *)arg;
827
828 (*tdata_count)++;
829
830 return (NULL);
831 }
832
833 size_t
834 prof_tdata_count(void)
835 {
836 size_t tdata_count = 0;
837
838 malloc_mutex_lock(&tdatas_mtx);
839 tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter,
840 (void *)&tdata_count);
841 malloc_mutex_unlock(&tdatas_mtx);
842
843 return (tdata_count);
844 }
845 #endif
846
847 #ifdef JEMALLOC_JET
848 size_t
849 prof_bt_count(void)
850 {
851 size_t bt_count;
852 tsd_t *tsd;
853 prof_tdata_t *tdata;
854
855 tsd = tsd_fetch();
856 tdata = prof_tdata_get(tsd, false);
857 if (tdata == NULL)
858 return (0);
859
860 prof_enter(tdata);
861 bt_count = ckh_count(&bt2gctx);
862 prof_leave(tdata);
863
864 return (bt_count);
865 }
866 #endif
867
868 #ifdef JEMALLOC_JET
869 #undef prof_dump_open
870 #define prof_dump_open JEMALLOC_N(prof_dump_open_impl)
871 #endif
872 static int
873 prof_dump_open(bool propagate_err, const char *filename)
874 {
875 int fd;
876
877 fd = creat(filename, 0644);
878 if (fd == -1 && !propagate_err) {
879 malloc_printf("<jemalloc>: creat(\"%s\"), 0644) failed\n",
880 filename);
881 if (opt_abort)
882 abort();
883 }
884
885 return (fd);
886 }
887 #ifdef JEMALLOC_JET
888 #undef prof_dump_open
889 #define prof_dump_open JEMALLOC_N(prof_dump_open)
890 prof_dump_open_t *prof_dump_open = JEMALLOC_N(prof_dump_open_impl);
891 #endif
892
893 static bool
894 prof_dump_flush(bool propagate_err)
895 {
896 bool ret = false;
897 ssize_t err;
898
899 cassert(config_prof);
900
901 err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end);
902 if (err == -1) {
903 if (!propagate_err) {
904 malloc_write("<jemalloc>: write() failed during heap "
905 "profile flush\n");
906 if (opt_abort)
907 abort();
908 }
909 ret = true;
910 }
911 prof_dump_buf_end = 0;
912
913 return (ret);
914 }
915
916 static bool
917 prof_dump_close(bool propagate_err)
918 {
919 bool ret;
920
921 assert(prof_dump_fd != -1);
922 ret = prof_dump_flush(propagate_err);
923 close(prof_dump_fd);
924 prof_dump_fd = -1;
925
926 return (ret);
927 }
928
929 static bool
930 prof_dump_write(bool propagate_err, const char *s)
931 {
932 unsigned i, slen, n;
933
934 cassert(config_prof);
935
936 i = 0;
937 slen = strlen(s);
938 while (i < slen) {
939 /* Flush the buffer if it is full. */
940 if (prof_dump_buf_end == PROF_DUMP_BUFSIZE)
941 if (prof_dump_flush(propagate_err) && propagate_err)
942 return (true);
943
944 if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) {
945 /* Finish writing. */
946 n = slen - i;
947 } else {
948 /* Write as much of s as will fit. */
949 n = PROF_DUMP_BUFSIZE - prof_dump_buf_end;
950 }
951 memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n);
952 prof_dump_buf_end += n;
953 i += n;
954 }
955
956 return (false);
957 }
958
959 JEMALLOC_ATTR(format(printf, 2, 3))
960 static bool
961 prof_dump_printf(bool propagate_err, const char *format, ...)
962 {
963 bool ret;
964 va_list ap;
965 char buf[PROF_PRINTF_BUFSIZE];
966
967 va_start(ap, format);
968 malloc_vsnprintf(buf, sizeof(buf), format, ap);
969 va_end(ap);
970 ret = prof_dump_write(propagate_err, buf);
971
972 return (ret);
973 }
974
975 /* tctx->tdata->lock is held. */
976 static void
977 prof_tctx_merge_tdata(prof_tctx_t *tctx, prof_tdata_t *tdata)
978 {
979
980 malloc_mutex_lock(tctx->gctx->lock);
981 if (tctx->state == prof_tctx_state_initializing) {
982 malloc_mutex_unlock(tctx->gctx->lock);
983 return;
984 }
985 assert(tctx->state == prof_tctx_state_nominal);
986 tctx->state = prof_tctx_state_dumping;
987 malloc_mutex_unlock(tctx->gctx->lock);
988
989 memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t));
990
991 tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
992 tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
993 if (opt_prof_accum) {
994 tdata->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs;
995 tdata->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes;
996 }
997 }
998
999 /* gctx->lock is held. */
1000 static void
1001 prof_tctx_merge_gctx(prof_tctx_t *tctx, prof_gctx_t *gctx)
1002 {
1003
1004 gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
1005 gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
1006 if (opt_prof_accum) {
1007 gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs;
1008 gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes;
1009 }
1010 }
1011
1012 /* tctx->gctx is held. */
1013 static prof_tctx_t *
1014 prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
1015 {
1016
1017 switch (tctx->state) {
1018 case prof_tctx_state_nominal:
1019 /* New since dumping started; ignore. */
1020 break;
1021 case prof_tctx_state_dumping:
1022 case prof_tctx_state_purgatory:
1023 prof_tctx_merge_gctx(tctx, tctx->gctx);
1024 break;
1025 default:
1026 not_reached();
1027 }
1028
1029 return (NULL);
1030 }
1031
1032 /* gctx->lock is held. */
1033 static prof_tctx_t *
1034 prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
1035 {
1036 bool propagate_err = *(bool *)arg;
1037
1038 if (prof_dump_printf(propagate_err,
1039 " t%"PRIu64": %"PRIu64": %"PRIu64" [%"PRIu64": %"PRIu64"]\n",
1040 tctx->tdata->thr_uid, tctx->dump_cnts.curobjs,
1041 tctx->dump_cnts.curbytes, tctx->dump_cnts.accumobjs,
1042 tctx->dump_cnts.accumbytes))
1043 return (tctx);
1044 return (NULL);
1045 }
1046
1047 /* tctx->gctx is held. */
1048 static prof_tctx_t *
1049 prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
1050 {
1051 prof_tctx_t *ret;
1052
1053 switch (tctx->state) {
1054 case prof_tctx_state_nominal:
1055 /* New since dumping started; ignore. */
1056 break;
1057 case prof_tctx_state_dumping:
1058 tctx->state = prof_tctx_state_nominal;
1059 break;
1060 case prof_tctx_state_purgatory:
1061 ret = tctx;
1062 goto label_return;
1063 default:
1064 not_reached();
1065 }
1066
1067 ret = NULL;
1068 label_return:
1069 return (ret);
1070 }
1071
1072 static void
1073 prof_dump_gctx_prep(prof_gctx_t *gctx, prof_gctx_tree_t *gctxs)
1074 {
1075
1076 cassert(config_prof);
1077
1078 malloc_mutex_lock(gctx->lock);
1079
1080 /*
1081 * Increment nlimbo so that gctx won't go away before dump.
1082 * Additionally, link gctx into the dump list so that it is included in
1083 * prof_dump()'s second pass.
1084 */
1085 gctx->nlimbo++;
1086 gctx_tree_insert(gctxs, gctx);
1087
1088 memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t));
1089
1090 malloc_mutex_unlock(gctx->lock);
1091 }
1092
1093 static prof_gctx_t *
1094 prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg)
1095 {
1096 size_t *leak_ngctx = (size_t *)arg;
1097
1098 malloc_mutex_lock(gctx->lock);
1099 tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter, NULL);
1100 if (gctx->cnt_summed.curobjs != 0)
1101 (*leak_ngctx)++;
1102 malloc_mutex_unlock(gctx->lock);
1103
1104 return (NULL);
1105 }
1106
1107 static void
1108 prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs)
1109 {
1110 prof_tdata_t *tdata = prof_tdata_get(tsd, false);
1111 prof_gctx_t *gctx;
1112
1113 /*
1114 * Standard tree iteration won't work here, because as soon as we
1115 * decrement gctx->nlimbo and unlock gctx, another thread can
1116 * concurrently destroy it, which will corrupt the tree. Therefore,
1117 * tear down the tree one node at a time during iteration.
1118 */
1119 while ((gctx = gctx_tree_first(gctxs)) != NULL) {
1120 gctx_tree_remove(gctxs, gctx);
1121 malloc_mutex_lock(gctx->lock);
1122 {
1123 prof_tctx_t *next;
1124
1125 next = NULL;
1126 do {
1127 prof_tctx_t *to_destroy =
1128 tctx_tree_iter(&gctx->tctxs, next,
1129 prof_tctx_finish_iter, NULL);
1130 if (to_destroy != NULL) {
1131 next = tctx_tree_next(&gctx->tctxs,
1132 to_destroy);
1133 tctx_tree_remove(&gctx->tctxs,
1134 to_destroy);
1135 idalloc(tsd, to_destroy);
1136 } else
1137 next = NULL;
1138 } while (next != NULL);
1139 }
1140 gctx->nlimbo--;
1141 if (prof_gctx_should_destroy(gctx)) {
1142 gctx->nlimbo++;
1143 malloc_mutex_unlock(gctx->lock);
1144 prof_gctx_try_destroy(tsd, gctx, tdata);
1145 } else
1146 malloc_mutex_unlock(gctx->lock);
1147 }
1148 }
1149
1150 static prof_tdata_t *
1151 prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
1152 {
1153 prof_cnt_t *cnt_all = (prof_cnt_t *)arg;
1154
1155 malloc_mutex_lock(tdata->lock);
1156 if (!tdata->expired) {
1157 size_t tabind;
1158 union {
1159 prof_tctx_t *p;
1160 void *v;
1161 } tctx;
1162
1163 tdata->dumping = true;
1164 memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t));
1165 for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL,
1166 &tctx.v);)
1167 prof_tctx_merge_tdata(tctx.p, tdata);
1168
1169 cnt_all->curobjs += tdata->cnt_summed.curobjs;
1170 cnt_all->curbytes += tdata->cnt_summed.curbytes;
1171 if (opt_prof_accum) {
1172 cnt_all->accumobjs += tdata->cnt_summed.accumobjs;
1173 cnt_all->accumbytes += tdata->cnt_summed.accumbytes;
1174 }
1175 } else
1176 tdata->dumping = false;
1177 malloc_mutex_unlock(tdata->lock);
1178
1179 return (NULL);
1180 }
1181
1182 static prof_tdata_t *
1183 prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
1184 {
1185 bool propagate_err = *(bool *)arg;
1186
1187 if (!tdata->dumping)
1188 return (NULL);
1189
1190 if (prof_dump_printf(propagate_err,
1191 " t%"PRIu64": %"PRIu64": %"PRIu64" [%"PRIu64": %"PRIu64"]%s%s\n",
1192 tdata->thr_uid, tdata->cnt_summed.curobjs,
1193 tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs,
1194 tdata->cnt_summed.accumbytes,
1195 (tdata->thread_name != NULL) ? " " : "",
1196 (tdata->thread_name != NULL) ? tdata->thread_name : ""))
1197 return (tdata);
1198 return (NULL);
1199 }
1200
1201 #ifdef JEMALLOC_JET
1202 #undef prof_dump_header
1203 #define prof_dump_header JEMALLOC_N(prof_dump_header_impl)
1204 #endif
1205 static bool
1206 prof_dump_header(bool propagate_err, const prof_cnt_t *cnt_all)
1207 {
1208 bool ret;
1209
1210 if (prof_dump_printf(propagate_err,
1211 "heap_v2/%"PRIu64"\n"
1212 " t*: %"PRIu64": %"PRIu64" [%"PRIu64": %"PRIu64"]\n",
1213 ((uint64_t)1U << lg_prof_sample), cnt_all->curobjs,
1214 cnt_all->curbytes, cnt_all->accumobjs, cnt_all->accumbytes))
1215 return (true);
1216
1217 malloc_mutex_lock(&tdatas_mtx);
1218 ret = (tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter,
1219 (void *)&propagate_err) != NULL);
1220 malloc_mutex_unlock(&tdatas_mtx);
1221 return (ret);
1222 }
1223 #ifdef JEMALLOC_JET
1224 #undef prof_dump_header
1225 #define prof_dump_header JEMALLOC_N(prof_dump_header)
1226 prof_dump_header_t *prof_dump_header = JEMALLOC_N(prof_dump_header_impl);
1227 #endif
1228
1229 /* gctx->lock is held. */
1230 static bool
1231 prof_dump_gctx(bool propagate_err, prof_gctx_t *gctx, const prof_bt_t *bt,
1232 prof_gctx_tree_t *gctxs)
1233 {
1234 bool ret;
1235 unsigned i;
1236
1237 cassert(config_prof);
1238
1239 /* Avoid dumping such gctx's that have no useful data. */
1240 if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) ||
1241 (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) {
1242 assert(gctx->cnt_summed.curobjs == 0);
1243 assert(gctx->cnt_summed.curbytes == 0);
1244 assert(gctx->cnt_summed.accumobjs == 0);
1245 assert(gctx->cnt_summed.accumbytes == 0);
1246 ret = false;
1247 goto label_return;
1248 }
1249
1250 if (prof_dump_printf(propagate_err, "@")) {
1251 ret = true;
1252 goto label_return;
1253 }
1254 for (i = 0; i < bt->len; i++) {
1255 if (prof_dump_printf(propagate_err, " %#"PRIxPTR,
1256 (uintptr_t)bt->vec[i])) {
1257 ret = true;
1258 goto label_return;
1259 }
1260 }
1261
1262 if (prof_dump_printf(propagate_err,
1263 "\n"
1264 " t*: %"PRIu64": %"PRIu64" [%"PRIu64": %"PRIu64"]\n",
1265 gctx->cnt_summed.curobjs, gctx->cnt_summed.curbytes,
1266 gctx->cnt_summed.accumobjs, gctx->cnt_summed.accumbytes)) {
1267 ret = true;
1268 goto label_return;
1269 }
1270
1271 if (tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter,
1272 (void *)&propagate_err) != NULL) {
1273 ret = true;
1274 goto label_return;
1275 }
1276
1277 ret = false;
1278 label_return:
1279 return (ret);
1280 }
1281
1282 static bool
1283 prof_dump_maps(bool propagate_err)
1284 {
1285 bool ret;
1286 int mfd;
1287 char filename[PATH_MAX + 1];
1288
1289 cassert(config_prof);
1290 #ifdef __FreeBSD__
1291 malloc_snprintf(filename, sizeof(filename), "/proc/curproc/map");
1292 #else
1293 malloc_snprintf(filename, sizeof(filename), "/proc/%d/maps",
1294 (int)getpid());
1295 #endif
1296 mfd = open(filename, O_RDONLY);
1297 if (mfd != -1) {
1298 ssize_t nread;
1299
1300 if (prof_dump_write(propagate_err, "\nMAPPED_LIBRARIES:\n") &&
1301 propagate_err) {
1302 ret = true;
1303 goto label_return;
1304 }
1305 nread = 0;
1306 do {
1307 prof_dump_buf_end += nread;
1308 if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) {
1309 /* Make space in prof_dump_buf before read(). */
1310 if (prof_dump_flush(propagate_err) &&
1311 propagate_err) {
1312 ret = true;
1313 goto label_return;
1314 }
1315 }
1316 nread = read(mfd, &prof_dump_buf[prof_dump_buf_end],
1317 PROF_DUMP_BUFSIZE - prof_dump_buf_end);
1318 } while (nread > 0);
1319 } else {
1320 ret = true;
1321 goto label_return;
1322 }
1323
1324 ret = false;
1325 label_return:
1326 if (mfd != -1)
1327 close(mfd);
1328 return (ret);
1329 }
1330
1331 static void
1332 prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx,
1333 const char *filename)
1334 {
1335
1336 if (cnt_all->curbytes != 0) {
1337 malloc_printf("<jemalloc>: Leak summary: %"PRIu64" byte%s, %"
1338 PRIu64" object%s, %zu context%s\n",
1339 cnt_all->curbytes, (cnt_all->curbytes != 1) ? "s" : "",
1340 cnt_all->curobjs, (cnt_all->curobjs != 1) ? "s" : "",
1341 leak_ngctx, (leak_ngctx != 1) ? "s" : "");
1342 malloc_printf(
1343 "<jemalloc>: Run pprof on \"%s\" for leak detail\n",
1344 filename);
1345 }
1346 }
1347
1348 static prof_gctx_t *
1349 prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg)
1350 {
1351 prof_gctx_t *ret;
1352 bool propagate_err = *(bool *)arg;
1353
1354 malloc_mutex_lock(gctx->lock);
1355
1356 if (prof_dump_gctx(propagate_err, gctx, &gctx->bt, gctxs)) {
1357 ret = gctx;
1358 goto label_return;
1359 }
1360
1361 ret = NULL;
1362 label_return:
1363 malloc_mutex_unlock(gctx->lock);
1364 return (ret);
1365 }
1366
1367 static bool
1368 prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck)
1369 {
1370 prof_tdata_t *tdata;
1371 prof_cnt_t cnt_all;
1372 size_t tabind;
1373 union {
1374 prof_gctx_t *p;
1375 void *v;
1376 } gctx;
1377 size_t leak_ngctx;
1378 prof_gctx_tree_t gctxs;
1379
1380 cassert(config_prof);
1381
1382 tdata = prof_tdata_get(tsd, true);
1383 if (tdata == NULL)
1384 return (true);
1385
1386 malloc_mutex_lock(&prof_dump_mtx);
1387 prof_enter(tdata);
1388
1389 /*
1390 * Put gctx's in limbo and clear their counters in preparation for
1391 * summing.
1392 */
1393 gctx_tree_new(&gctxs);
1394 for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);)
1395 prof_dump_gctx_prep(gctx.p, &gctxs);
1396
1397 /*
1398 * Iterate over tdatas, and for the non-expired ones snapshot their tctx
1399 * stats and merge them into the associated gctx's.
1400 */
1401 memset(&cnt_all, 0, sizeof(prof_cnt_t));
1402 malloc_mutex_lock(&tdatas_mtx);
1403 tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter, (void *)&cnt_all);
1404 malloc_mutex_unlock(&tdatas_mtx);
1405
1406 /* Merge tctx stats into gctx's. */
1407 leak_ngctx = 0;
1408 gctx_tree_iter(&gctxs, NULL, prof_gctx_merge_iter, (void *)&leak_ngctx);
1409
1410 prof_leave(tdata);
1411
1412 /* Create dump file. */
1413 if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1)
1414 goto label_open_close_error;
1415
1416 /* Dump profile header. */
1417 if (prof_dump_header(propagate_err, &cnt_all))
1418 goto label_write_error;
1419
1420 /* Dump per gctx profile stats. */
1421 if (gctx_tree_iter(&gctxs, NULL, prof_gctx_dump_iter,
1422 (void *)&propagate_err) != NULL)
1423 goto label_write_error;
1424
1425 /* Dump /proc/<pid>/maps if possible. */
1426 if (prof_dump_maps(propagate_err))
1427 goto label_write_error;
1428
1429 if (prof_dump_close(propagate_err))
1430 goto label_open_close_error;
1431
1432 prof_gctx_finish(tsd, &gctxs);
1433 malloc_mutex_unlock(&prof_dump_mtx);
1434
1435 if (leakcheck)
1436 prof_leakcheck(&cnt_all, leak_ngctx, filename);
1437
1438 return (false);
1439 label_write_error:
1440 prof_dump_close(propagate_err);
1441 label_open_close_error:
1442 prof_gctx_finish(tsd, &gctxs);
1443 malloc_mutex_unlock(&prof_dump_mtx);
1444 return (true);
1445 }
1446
1447 #define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1)
1448 #define VSEQ_INVALID UINT64_C(0xffffffffffffffff)
1449 static void
1450 prof_dump_filename(char *filename, char v, uint64_t vseq)
1451 {
1452
1453 cassert(config_prof);
1454
1455 if (vseq != VSEQ_INVALID) {
1456 /* "<prefix>.<pid>.<seq>.v<vseq>.heap" */
1457 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
1458 "%s.%d.%"PRIu64".%c%"PRIu64".heap",
1459 opt_prof_prefix, (int)getpid(), prof_dump_seq, v, vseq);
1460 } else {
1461 /* "<prefix>.<pid>.<seq>.<v>.heap" */
1462 malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE,
1463 "%s.%d.%"PRIu64".%c.heap",
1464 opt_prof_prefix, (int)getpid(), prof_dump_seq, v);
1465 }
1466 prof_dump_seq++;
1467 }
1468
1469 static void
1470 prof_fdump(void)
1471 {
1472 tsd_t *tsd;
1473 char filename[DUMP_FILENAME_BUFSIZE];
1474
1475 cassert(config_prof);
1476
1477 if (!prof_booted)
1478 return;
1479 tsd = tsd_fetch();
1480
1481 if (opt_prof_final && opt_prof_prefix[0] != '\0') {
1482 malloc_mutex_lock(&prof_dump_seq_mtx);
1483 prof_dump_filename(filename, 'f', VSEQ_INVALID);
1484 malloc_mutex_unlock(&prof_dump_seq_mtx);
1485 prof_dump(tsd, false, filename, opt_prof_leak);
1486 }
1487 }
1488
1489 void
1490 prof_idump(void)
1491 {
1492 tsd_t *tsd;
1493 prof_tdata_t *tdata;
1494 char filename[PATH_MAX + 1];
1495
1496 cassert(config_prof);
1497
1498 if (!prof_booted)
1499 return;
1500 tsd = tsd_fetch();
1501 tdata = prof_tdata_get(tsd, false);
1502 if (tdata == NULL)
1503 return;
1504 if (tdata->enq) {
1505 tdata->enq_idump = true;
1506 return;
1507 }
1508
1509 if (opt_prof_prefix[0] != '\0') {
1510 malloc_mutex_lock(&prof_dump_seq_mtx);
1511 prof_dump_filename(filename, 'i', prof_dump_iseq);
1512 prof_dump_iseq++;
1513 malloc_mutex_unlock(&prof_dump_seq_mtx);
1514 prof_dump(tsd, false, filename, false);
1515 }
1516 }
1517
1518 bool
1519 prof_mdump(const char *filename)
1520 {
1521 tsd_t *tsd;
1522 char filename_buf[DUMP_FILENAME_BUFSIZE];
1523
1524 cassert(config_prof);
1525
1526 if (!opt_prof || !prof_booted)
1527 return (true);
1528 tsd = tsd_fetch();
1529
1530 if (filename == NULL) {
1531 /* No filename specified, so automatically generate one. */
1532 if (opt_prof_prefix[0] == '\0')
1533 return (true);
1534 malloc_mutex_lock(&prof_dump_seq_mtx);
1535 prof_dump_filename(filename_buf, 'm', prof_dump_mseq);
1536 prof_dump_mseq++;
1537 malloc_mutex_unlock(&prof_dump_seq_mtx);
1538 filename = filename_buf;
1539 }
1540 return (prof_dump(tsd, true, filename, false));
1541 }
1542
1543 void
1544 prof_gdump(void)
1545 {
1546 tsd_t *tsd;
1547 prof_tdata_t *tdata;
1548 char filename[DUMP_FILENAME_BUFSIZE];
1549
1550 cassert(config_prof);
1551
1552 if (!prof_booted)
1553 return;
1554 tsd = tsd_fetch();
1555 tdata = prof_tdata_get(tsd, false);
1556 if (tdata == NULL)
1557 return;
1558 if (tdata->enq) {
1559 tdata->enq_gdump = true;
1560 return;
1561 }
1562
1563 if (opt_prof_prefix[0] != '\0') {
1564 malloc_mutex_lock(&prof_dump_seq_mtx);
1565 prof_dump_filename(filename, 'u', prof_dump_useq);
1566 prof_dump_useq++;
1567 malloc_mutex_unlock(&prof_dump_seq_mtx);
1568 prof_dump(tsd, false, filename, false);
1569 }
1570 }
1571
1572 static void
1573 prof_bt_hash(const void *key, size_t r_hash[2])
1574 {
1575 prof_bt_t *bt = (prof_bt_t *)key;
1576
1577 cassert(config_prof);
1578
1579 hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash);
1580 }
1581
1582 static bool
1583 prof_bt_keycomp(const void *k1, const void *k2)
1584 {
1585 const prof_bt_t *bt1 = (prof_bt_t *)k1;
1586 const prof_bt_t *bt2 = (prof_bt_t *)k2;
1587
1588 cassert(config_prof);
1589
1590 if (bt1->len != bt2->len)
1591 return (false);
1592 return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0);
1593 }
1594
1595 JEMALLOC_INLINE_C uint64_t
1596 prof_thr_uid_alloc(void)
1597 {
1598 uint64_t thr_uid;
1599
1600 malloc_mutex_lock(&next_thr_uid_mtx);
1601 thr_uid = next_thr_uid;
1602 next_thr_uid++;
1603 malloc_mutex_unlock(&next_thr_uid_mtx);
1604
1605 return (thr_uid);
1606 }
1607
1608 static prof_tdata_t *
1609 prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim,
1610 char *thread_name, bool active)
1611 {
1612 prof_tdata_t *tdata;
1613
1614 cassert(config_prof);
1615
1616 /* Initialize an empty cache for this thread. */
1617 tdata = (prof_tdata_t *)imalloc(tsd, sizeof(prof_tdata_t));
1618 if (tdata == NULL)
1619 return (NULL);
1620
1621 tdata->lock = prof_tdata_mutex_choose(thr_uid);
1622 tdata->thr_uid = thr_uid;
1623 tdata->thr_discrim = thr_discrim;
1624 tdata->thread_name = thread_name;
1625 tdata->attached = true;
1626 tdata->expired = false;
1627
1628 if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS,
1629 prof_bt_hash, prof_bt_keycomp)) {
1630 idalloc(tsd, tdata);
1631 return (NULL);
1632 }
1633
1634 tdata->prng_state = (uint64_t)(uintptr_t)tdata;
1635 prof_sample_threshold_update(tdata);
1636
1637 tdata->enq = false;
1638 tdata->enq_idump = false;
1639 tdata->enq_gdump = false;
1640
1641 tdata->dumping = false;
1642 tdata->active = active;
1643
1644 malloc_mutex_lock(&tdatas_mtx);
1645 tdata_tree_insert(&tdatas, tdata);
1646 malloc_mutex_unlock(&tdatas_mtx);
1647
1648 return (tdata);
1649 }
1650
1651 prof_tdata_t *
1652 prof_tdata_init(tsd_t *tsd)
1653 {
1654
1655 return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc(), 0, NULL,
1656 prof_thread_active_init_get()));
1657 }
1658
1659 /* tdata->lock must be held. */
1660 static bool
1661 prof_tdata_should_destroy(prof_tdata_t *tdata, bool even_if_attached)
1662 {
1663
1664 if (tdata->attached && !even_if_attached)
1665 return (false);
1666 if (ckh_count(&tdata->bt2tctx) != 0)
1667 return (false);
1668 return (true);
1669 }
1670
1671 /* tdatas_mtx must be held. */
1672 static void
1673 prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata,
1674 bool even_if_attached)
1675 {
1676
1677 assert(prof_tdata_should_destroy(tdata, even_if_attached));
1678 assert(tsd_prof_tdata_get(tsd) != tdata);
1679
1680 tdata_tree_remove(&tdatas, tdata);
1681
1682 if (tdata->thread_name != NULL)
1683 idalloc(tsd, tdata->thread_name);
1684 ckh_delete(tsd, &tdata->bt2tctx);
1685 idalloc(tsd, tdata);
1686 }
1687
1688 static void
1689 prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached)
1690 {
1691
1692 malloc_mutex_lock(&tdatas_mtx);
1693 prof_tdata_destroy_locked(tsd, tdata, even_if_attached);
1694 malloc_mutex_unlock(&tdatas_mtx);
1695 }
1696
1697 static void
1698 prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata)
1699 {
1700 bool destroy_tdata;
1701
1702 malloc_mutex_lock(tdata->lock);
1703 if (tdata->attached) {
1704 destroy_tdata = prof_tdata_should_destroy(tdata, true);
1705 /*
1706 * Only detach if !destroy_tdata, because detaching would allow
1707 * another thread to win the race to destroy tdata.
1708 */
1709 if (!destroy_tdata)
1710 tdata->attached = false;
1711 tsd_prof_tdata_set(tsd, NULL);
1712 } else
1713 destroy_tdata = false;
1714 malloc_mutex_unlock(tdata->lock);
1715 if (destroy_tdata)
1716 prof_tdata_destroy(tsd, tdata, true);
1717 }
1718
1719 prof_tdata_t *
1720 prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata)
1721 {
1722 uint64_t thr_uid = tdata->thr_uid;
1723 uint64_t thr_discrim = tdata->thr_discrim + 1;
1724 char *thread_name = (tdata->thread_name != NULL) ?
1725 prof_thread_name_alloc(tsd, tdata->thread_name) : NULL;
1726 bool active = tdata->active;
1727
1728 prof_tdata_detach(tsd, tdata);
1729 return (prof_tdata_init_impl(tsd, thr_uid, thr_discrim, thread_name,
1730 active));
1731 }
1732
1733 static bool
1734 prof_tdata_expire(prof_tdata_t *tdata)
1735 {
1736 bool destroy_tdata;
1737
1738 malloc_mutex_lock(tdata->lock);
1739 if (!tdata->expired) {
1740 tdata->expired = true;
1741 destroy_tdata = tdata->attached ? false :
1742 prof_tdata_should_destroy(tdata, false);
1743 } else
1744 destroy_tdata = false;
1745 malloc_mutex_unlock(tdata->lock);
1746
1747 return (destroy_tdata);
1748 }
1749
1750 static prof_tdata_t *
1751 prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
1752 {
1753
1754 return (prof_tdata_expire(tdata) ? tdata : NULL);
1755 }
1756
1757 void
1758 prof_reset(tsd_t *tsd, size_t lg_sample)
1759 {
1760 prof_tdata_t *next;
1761
1762 assert(lg_sample < (sizeof(uint64_t) << 3));
1763
1764 malloc_mutex_lock(&prof_dump_mtx);
1765 malloc_mutex_lock(&tdatas_mtx);
1766
1767 lg_prof_sample = lg_sample;
1768
1769 next = NULL;
1770 do {
1771 prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next,
1772 prof_tdata_reset_iter, NULL);
1773 if (to_destroy != NULL) {
1774 next = tdata_tree_next(&tdatas, to_destroy);
1775 prof_tdata_destroy_locked(tsd, to_destroy, false);
1776 } else
1777 next = NULL;
1778 } while (next != NULL);
1779
1780 malloc_mutex_unlock(&tdatas_mtx);
1781 malloc_mutex_unlock(&prof_dump_mtx);
1782 }
1783
1784 void
1785 prof_tdata_cleanup(tsd_t *tsd)
1786 {
1787 prof_tdata_t *tdata;
1788
1789 if (!config_prof)
1790 return;
1791
1792 tdata = tsd_prof_tdata_get(tsd);
1793 if (tdata != NULL)
1794 prof_tdata_detach(tsd, tdata);
1795 }
1796
1797 bool
1798 prof_active_get(void)
1799 {
1800 bool prof_active_current;
1801
1802 malloc_mutex_lock(&prof_active_mtx);
1803 prof_active_current = prof_active;
1804 malloc_mutex_unlock(&prof_active_mtx);
1805 return (prof_active_current);
1806 }
1807
1808 bool
1809 prof_active_set(bool active)
1810 {
1811 bool prof_active_old;
1812
1813 malloc_mutex_lock(&prof_active_mtx);
1814 prof_active_old = prof_active;
1815 prof_active = active;
1816 malloc_mutex_unlock(&prof_active_mtx);
1817 return (prof_active_old);
1818 }
1819
1820 const char *
1821 prof_thread_name_get(void)
1822 {
1823 tsd_t *tsd;
1824 prof_tdata_t *tdata;
1825
1826 tsd = tsd_fetch();
1827 tdata = prof_tdata_get(tsd, true);
1828 if (tdata == NULL)
1829 return ("");
1830 return (tdata->thread_name != NULL ? tdata->thread_name : "");
1831 }
1832
1833 static char *
1834 prof_thread_name_alloc(tsd_t *tsd, const char *thread_name)
1835 {
1836 char *ret;
1837 size_t size;
1838
1839 if (thread_name == NULL)
1840 return (NULL);
1841
1842 size = strlen(thread_name) + 1;
1843 if (size == 1)
1844 return ("");
1845
1846 ret = imalloc(tsd, size);
1847 if (ret == NULL)
1848 return (NULL);
1849 memcpy(ret, thread_name, size);
1850 return (ret);
1851 }
1852
1853 int
1854 prof_thread_name_set(tsd_t *tsd, const char *thread_name)
1855 {
1856 prof_tdata_t *tdata;
1857 unsigned i;
1858 char *s;
1859
1860 tdata = prof_tdata_get(tsd, true);
1861 if (tdata == NULL)
1862 return (EAGAIN);
1863
1864 /* Validate input. */
1865 if (thread_name == NULL)
1866 return (EFAULT);
1867 for (i = 0; thread_name[i] != '\0'; i++) {
1868 char c = thread_name[i];
1869 if (!isgraph(c) && !isblank(c))
1870 return (EFAULT);
1871 }
1872
1873 s = prof_thread_name_alloc(tsd, thread_name);
1874 if (s == NULL)
1875 return (EAGAIN);
1876
1877 if (tdata->thread_name != NULL) {
1878 idalloc(tsd, tdata->thread_name);
1879 tdata->thread_name = NULL;
1880 }
1881 if (strlen(s) > 0)
1882 tdata->thread_name = s;
1883 return (0);
1884 }
1885
1886 bool
1887 prof_thread_active_get(void)
1888 {
1889 tsd_t *tsd;
1890 prof_tdata_t *tdata;
1891
1892 tsd = tsd_fetch();
1893 tdata = prof_tdata_get(tsd, true);
1894 if (tdata == NULL)
1895 return (false);
1896 return (tdata->active);
1897 }
1898
1899 bool
1900 prof_thread_active_set(bool active)
1901 {
1902 tsd_t *tsd;
1903 prof_tdata_t *tdata;
1904
1905 tsd = tsd_fetch();
1906 tdata = prof_tdata_get(tsd, true);
1907 if (tdata == NULL)
1908 return (true);
1909 tdata->active = active;
1910 return (false);
1911 }
1912
1913 bool
1914 prof_thread_active_init_get(void)
1915 {
1916 bool active_init;
1917
1918 malloc_mutex_lock(&prof_thread_active_init_mtx);
1919 active_init = prof_thread_active_init;
1920 malloc_mutex_unlock(&prof_thread_active_init_mtx);
1921 return (active_init);
1922 }
1923
1924 bool
1925 prof_thread_active_init_set(bool active_init)
1926 {
1927 bool active_init_old;
1928
1929 malloc_mutex_lock(&prof_thread_active_init_mtx);
1930 active_init_old = prof_thread_active_init;
1931 prof_thread_active_init = active_init;
1932 malloc_mutex_unlock(&prof_thread_active_init_mtx);
1933 return (active_init_old);
1934 }
1935
1936 void
1937 prof_boot0(void)
1938 {
1939
1940 cassert(config_prof);
1941
1942 memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT,
1943 sizeof(PROF_PREFIX_DEFAULT));
1944 }
1945
1946 void
1947 prof_boot1(void)
1948 {
1949
1950 cassert(config_prof);
1951
1952 /*
1953 * opt_prof must be in its final state before any arenas are
1954 * initialized, so this function must be executed early.
1955 */
1956
1957 if (opt_prof_leak && !opt_prof) {
1958 /*
1959 * Enable opt_prof, but in such a way that profiles are never
1960 * automatically dumped.
1961 */
1962 opt_prof = true;
1963 opt_prof_gdump = false;
1964 } else if (opt_prof) {
1965 if (opt_lg_prof_interval >= 0) {
1966 prof_interval = (((uint64_t)1U) <<
1967 opt_lg_prof_interval);
1968 }
1969 }
1970 }
1971
1972 bool
1973 prof_boot2(void)
1974 {
1975
1976 cassert(config_prof);
1977
1978 if (opt_prof) {
1979 tsd_t *tsd;
1980 unsigned i;
1981
1982 lg_prof_sample = opt_lg_prof_sample;
1983
1984 prof_active = opt_prof_active;
1985 if (malloc_mutex_init(&prof_active_mtx))
1986 return (true);
1987
1988 prof_thread_active_init = opt_prof_thread_active_init;
1989 if (malloc_mutex_init(&prof_thread_active_init_mtx))
1990 return (true);
1991
1992 tsd = tsd_fetch();
1993 if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash,
1994 prof_bt_keycomp))
1995 return (true);
1996 if (malloc_mutex_init(&bt2gctx_mtx))
1997 return (true);
1998
1999 tdata_tree_new(&tdatas);
2000 if (malloc_mutex_init(&tdatas_mtx))
2001 return (true);
2002
2003 next_thr_uid = 0;
2004 if (malloc_mutex_init(&next_thr_uid_mtx))
2005 return (true);
2006
2007 if (malloc_mutex_init(&prof_dump_seq_mtx))
2008 return (true);
2009 if (malloc_mutex_init(&prof_dump_mtx))
2010 return (true);
2011
2012 if (atexit(prof_fdump) != 0) {
2013 malloc_write("<jemalloc>: Error in atexit()\n");
2014 if (opt_abort)
2015 abort();
2016 }
2017
2018 gctx_locks = (malloc_mutex_t *)base_alloc(PROF_NCTX_LOCKS *
2019 sizeof(malloc_mutex_t));
2020 if (gctx_locks == NULL)
2021 return (true);
2022 for (i = 0; i < PROF_NCTX_LOCKS; i++) {
2023 if (malloc_mutex_init(&gctx_locks[i]))
2024 return (true);
2025 }
2026
2027 tdata_locks = (malloc_mutex_t *)base_alloc(PROF_NTDATA_LOCKS *
2028 sizeof(malloc_mutex_t));
2029 if (tdata_locks == NULL)
2030 return (true);
2031 for (i = 0; i < PROF_NTDATA_LOCKS; i++) {
2032 if (malloc_mutex_init(&tdata_locks[i]))
2033 return (true);
2034 }
2035 }
2036
2037 #ifdef JEMALLOC_PROF_LIBGCC
2038 /*
2039 * Cause the backtracing machinery to allocate its internal state
2040 * before enabling profiling.
2041 */
2042 _Unwind_Backtrace(prof_unwind_init_callback, NULL);
2043 #endif
2044
2045 prof_booted = true;
2046
2047 return (false);
2048 }
2049
2050 void
2051 prof_prefork(void)
2052 {
2053
2054 if (opt_prof) {
2055 unsigned i;
2056
2057 malloc_mutex_prefork(&tdatas_mtx);
2058 malloc_mutex_prefork(&bt2gctx_mtx);
2059 malloc_mutex_prefork(&next_thr_uid_mtx);
2060 malloc_mutex_prefork(&prof_dump_seq_mtx);
2061 for (i = 0; i < PROF_NCTX_LOCKS; i++)
2062 malloc_mutex_prefork(&gctx_locks[i]);
2063 for (i = 0; i < PROF_NTDATA_LOCKS; i++)
2064 malloc_mutex_prefork(&tdata_locks[i]);
2065 }
2066 }
2067
2068 void
2069 prof_postfork_parent(void)
2070 {
2071
2072 if (opt_prof) {
2073 unsigned i;
2074
2075 for (i = 0; i < PROF_NTDATA_LOCKS; i++)
2076 malloc_mutex_postfork_parent(&tdata_locks[i]);
2077 for (i = 0; i < PROF_NCTX_LOCKS; i++)
2078 malloc_mutex_postfork_parent(&gctx_locks[i]);
2079 malloc_mutex_postfork_parent(&prof_dump_seq_mtx);
2080 malloc_mutex_postfork_parent(&next_thr_uid_mtx);
2081 malloc_mutex_postfork_parent(&bt2gctx_mtx);
2082 malloc_mutex_postfork_parent(&tdatas_mtx);
2083 }
2084 }
2085
2086 void
2087 prof_postfork_child(void)
2088 {
2089
2090 if (opt_prof) {
2091 unsigned i;
2092
2093 for (i = 0; i < PROF_NTDATA_LOCKS; i++)
2094 malloc_mutex_postfork_child(&tdata_locks[i]);
2095 for (i = 0; i < PROF_NCTX_LOCKS; i++)
2096 malloc_mutex_postfork_child(&gctx_locks[i]);
2097 malloc_mutex_postfork_child(&prof_dump_seq_mtx);
2098 malloc_mutex_postfork_child(&next_thr_uid_mtx);
2099 malloc_mutex_postfork_child(&bt2gctx_mtx);
2100 malloc_mutex_postfork_child(&tdatas_mtx);
2101 }
2102 }
2103
2104 /******************************************************************************/
backport for Debian buster