[rustc.git] / src / jemalloc / include / jemalloc / internal / prof.h

/******************************************************************************/
#ifdef JEMALLOC_H_TYPES

typedef struct prof_bt_s prof_bt_t;
typedef struct prof_cnt_s prof_cnt_t;
typedef struct prof_tctx_s prof_tctx_t;
typedef struct prof_gctx_s prof_gctx_t;
typedef struct prof_tdata_s prof_tdata_t;

/* Option defaults. */
#ifdef JEMALLOC_PROF
#  define PROF_PREFIX_DEFAULT		"jeprof"
#else
#  define PROF_PREFIX_DEFAULT		""
#endif
#define	LG_PROF_SAMPLE_DEFAULT		19
#define	LG_PROF_INTERVAL_DEFAULT	-1

/*
 * Hard limit on stack backtrace depth.  The version of prof_backtrace() that
 * is based on __builtin_return_address() necessarily has a hard-coded number
 * of backtrace frame handlers, and should be kept in sync with this setting.
 */
#define	PROF_BT_MAX			128

/* Initial hash table size. */
#define	PROF_CKH_MINITEMS		64

/* Size of memory buffer to use when writing dump files. */
#define	PROF_DUMP_BUFSIZE		65536

/* Size of stack-allocated buffer used by prof_printf(). */
#define	PROF_PRINTF_BUFSIZE		128

/*
 * Number of mutexes shared among all gctx's.  No space is allocated for these
 * unless profiling is enabled, so it's okay to over-provision.
 */
#define	PROF_NCTX_LOCKS			1024

/*
 * Number of mutexes shared among all tdata's.  No space is allocated for these
 * unless profiling is enabled, so it's okay to over-provision.
 */
#define	PROF_NTDATA_LOCKS		256

/*
 * prof_tdata pointers close to NULL are used to encode state information that
 * is used for cleaning up during thread shutdown.
 */
#define	PROF_TDATA_STATE_REINCARNATED	((prof_tdata_t *)(uintptr_t)1)
#define	PROF_TDATA_STATE_PURGATORY	((prof_tdata_t *)(uintptr_t)2)
#define	PROF_TDATA_STATE_MAX		PROF_TDATA_STATE_PURGATORY

#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS

struct prof_bt_s {
	/* Backtrace, stored as len program counters. */
	void		**vec;
	unsigned	len;
};

#ifdef JEMALLOC_PROF_LIBGCC
/* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */
typedef struct {
	prof_bt_t	*bt;
	unsigned	max;
} prof_unwind_data_t;
#endif

struct prof_cnt_s {
	/* Profiling counters. */
	uint64_t	curobjs;
	uint64_t	curbytes;
	uint64_t	accumobjs;
	uint64_t	accumbytes;
};

typedef enum {
	prof_tctx_state_initializing,
	prof_tctx_state_nominal,
	prof_tctx_state_dumping,
	prof_tctx_state_purgatory /* Dumper must finish destroying. */
} prof_tctx_state_t;

struct prof_tctx_s {
	/* Thread data for thread that performed the allocation. */
	prof_tdata_t		*tdata;

	/*
	 * Copy of tdata->thr_{uid,discrim}, necessary because tdata may be
	 * defunct during teardown.
	 */
	uint64_t		thr_uid;
	uint64_t		thr_discrim;

	/* Profiling counters, protected by tdata->lock. */
	prof_cnt_t		cnts;

	/* Associated global context. */
	prof_gctx_t		*gctx;

	/*
	 * UID that distinguishes multiple tctx's created by the same thread,
	 * but coexisting in gctx->tctxs.  There are two ways that such
	 * coexistence can occur:
	 * - A dumper thread can cause a tctx to be retained in the purgatory
	 *   state.
	 * - Although a single "producer" thread must create all tctx's which
	 *   share the same thr_uid, multiple "consumers" can each concurrently
	 *   execute portions of prof_tctx_destroy().  prof_tctx_destroy() only
	 *   gets called once each time cnts.cur{objs,bytes} drop to 0, but this
	 *   threshold can be hit again before the first consumer finishes
	 *   executing prof_tctx_destroy().
	 */
	uint64_t		tctx_uid;

	/* Linkage into gctx's tctxs. */
	rb_node(prof_tctx_t)	tctx_link;

	/*
	 * True during prof_alloc_prep()..prof_malloc_sample_object(), prevents
	 * sample vs destroy race.
	 */
	bool			prepared;

	/* Current dump-related state, protected by gctx->lock. */
	prof_tctx_state_t	state;

	/*
	 * Copy of cnts snapshotted during early dump phase, protected by
	 * dump_mtx.
	 */
	prof_cnt_t		dump_cnts;
};
typedef rb_tree(prof_tctx_t) prof_tctx_tree_t;

struct prof_gctx_s {
	/* Protects nlimbo, cnt_summed, and tctxs. */
	malloc_mutex_t		*lock;

	/*
	 * Number of threads that currently cause this gctx to be in a state of
	 * limbo due to one of:
	 *   - Initializing this gctx.
	 *   - Initializing per thread counters associated with this gctx.
	 *   - Preparing to destroy this gctx.
	 *   - Dumping a heap profile that includes this gctx.
	 * nlimbo must be 1 (single destroyer) in order to safely destroy the
	 * gctx.
	 */
	unsigned		nlimbo;

	/*
	 * Tree of profile counters, one for each thread that has allocated in
	 * this context.
	 */
	prof_tctx_tree_t	tctxs;

	/* Linkage for tree of contexts to be dumped. */
	rb_node(prof_gctx_t)	dump_link;

	/* Temporary storage for summation during dump. */
	prof_cnt_t		cnt_summed;

	/* Associated backtrace. */
	prof_bt_t		bt;

	/* Backtrace vector, variable size, referred to by bt. */
	void			*vec[1];
};
typedef rb_tree(prof_gctx_t) prof_gctx_tree_t;

struct prof_tdata_s {
	malloc_mutex_t		*lock;

	/* Monotonically increasing unique thread identifier. */
	uint64_t		thr_uid;

	/*
	 * Monotonically increasing discriminator among tdata structures
	 * associated with the same thr_uid.
	 */
	uint64_t		thr_discrim;

	/* Included in heap profile dumps if non-NULL. */
	char			*thread_name;

	bool			attached;
	bool			expired;

	rb_node(prof_tdata_t)	tdata_link;

	/*
	 * Counter used to initialize prof_tctx_t's tctx_uid.  No locking is
	 * necessary when incrementing this field, because only one thread ever
	 * does so.
	 */
	uint64_t		tctx_uid_next;

	/*
	 * Hash of (prof_bt_t *)-->(prof_tctx_t *).  Each thread tracks
	 * backtraces for which it has non-zero allocation/deallocation counters
	 * associated with thread-specific prof_tctx_t objects.  Other threads
	 * may write to prof_tctx_t contents when freeing associated objects.
	 */
	ckh_t			bt2tctx;

	/* Sampling state. */
	uint64_t		prng_state;
	uint64_t		bytes_until_sample;

	/* State used to avoid dumping while operating on prof internals. */
	bool			enq;
	bool			enq_idump;
	bool			enq_gdump;

	/*
	 * Set to true during an early dump phase for tdata's which are
	 * currently being dumped.  New threads' tdata's have this initialized
	 * to false so that they aren't accidentally included in later dump
	 * phases.
	 */
	bool			dumping;

	/*
	 * True if profiling is active for this tdata's thread
	 * (thread.prof.active mallctl).
	 */
	bool			active;

	/* Temporary storage for summation during dump. */
	prof_cnt_t		cnt_summed;

	/* Backtrace vector, used for calls to prof_backtrace(). */
	void			*vec[PROF_BT_MAX];
};
typedef rb_tree(prof_tdata_t) prof_tdata_tree_t;

#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS

extern bool	opt_prof;
extern bool	opt_prof_active;
extern bool	opt_prof_thread_active_init;
extern size_t	opt_lg_prof_sample;   /* Mean bytes between samples. */
extern ssize_t	opt_lg_prof_interval; /* lg(prof_interval). */
extern bool	opt_prof_gdump;       /* High-water memory dumping. */
extern bool	opt_prof_final;       /* Final profile dumping. */
extern bool	opt_prof_leak;        /* Dump leak summary at exit. */
extern bool	opt_prof_accum;       /* Report cumulative bytes. */
extern char	opt_prof_prefix[
    /* Minimize memory bloat for non-prof builds. */
#ifdef JEMALLOC_PROF
    PATH_MAX +
#endif
    1];

/* Accessed via prof_active_[gs]et{_unlocked,}(). */
extern bool	prof_active;

/* Accessed via prof_gdump_[gs]et{_unlocked,}(). */
extern bool	prof_gdump_val;

/*
 * Profile dump interval, measured in bytes allocated.  Each arena triggers a
 * profile dump when it reaches this threshold.  The effect is that the
 * interval between profile dumps averages prof_interval, though the actual
 * interval between dumps will tend to be sporadic, and the interval will be a
 * maximum of approximately (prof_interval * narenas).
 */
extern uint64_t	prof_interval;

/*
 * Initialized as opt_lg_prof_sample, and potentially modified during profiling
 * resets.
 */
extern size_t	lg_prof_sample;

void	prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated);
void	prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize,
    prof_tctx_t *tctx);
void	prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx);
void	bt_init(prof_bt_t *bt, void **vec);
void	prof_backtrace(prof_bt_t *bt);
prof_tctx_t	*prof_lookup(tsd_t *tsd, prof_bt_t *bt);
#ifdef JEMALLOC_JET
size_t	prof_tdata_count(void);
size_t	prof_bt_count(void);
const prof_cnt_t *prof_cnt_all(void);
typedef int (prof_dump_open_t)(bool, const char *);
extern prof_dump_open_t *prof_dump_open;
typedef bool (prof_dump_header_t)(tsdn_t *, bool, const prof_cnt_t *);
extern prof_dump_header_t *prof_dump_header;
#endif
void	prof_idump(tsdn_t *tsdn);
bool	prof_mdump(tsd_t *tsd, const char *filename);
void	prof_gdump(tsdn_t *tsdn);
prof_tdata_t	*prof_tdata_init(tsd_t *tsd);
prof_tdata_t	*prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
void	prof_reset(tsd_t *tsd, size_t lg_sample);
void	prof_tdata_cleanup(tsd_t *tsd);
bool	prof_active_get(tsdn_t *tsdn);
bool	prof_active_set(tsdn_t *tsdn, bool active);
const char	*prof_thread_name_get(tsd_t *tsd);
int	prof_thread_name_set(tsd_t *tsd, const char *thread_name);
bool	prof_thread_active_get(tsd_t *tsd);
bool	prof_thread_active_set(tsd_t *tsd, bool active);
bool	prof_thread_active_init_get(tsdn_t *tsdn);
bool	prof_thread_active_init_set(tsdn_t *tsdn, bool active_init);
bool	prof_gdump_get(tsdn_t *tsdn);
bool	prof_gdump_set(tsdn_t *tsdn, bool active);
void	prof_boot0(void);
void	prof_boot1(void);
bool	prof_boot2(tsd_t *tsd);
void	prof_prefork0(tsdn_t *tsdn);
void	prof_prefork1(tsdn_t *tsdn);
void	prof_postfork_parent(tsdn_t *tsdn);
void	prof_postfork_child(tsdn_t *tsdn);
void	prof_sample_threshold_update(prof_tdata_t *tdata);

#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES

#ifndef JEMALLOC_ENABLE_INLINE
bool	prof_active_get_unlocked(void);
bool	prof_gdump_get_unlocked(void);
prof_tdata_t	*prof_tdata_get(tsd_t *tsd, bool create);
prof_tctx_t	*prof_tctx_get(tsdn_t *tsdn, const void *ptr);
void	prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize,
    prof_tctx_t *tctx);
void	prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize,
    const void *old_ptr, prof_tctx_t *tctx);
bool	prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit,
    prof_tdata_t **tdata_out);
prof_tctx_t	*prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active,
    bool update);
void	prof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize,
    prof_tctx_t *tctx);
void	prof_realloc(tsd_t *tsd, const void *ptr, size_t usize,
    prof_tctx_t *tctx, bool prof_active, bool updated, const void *old_ptr,
    size_t old_usize, prof_tctx_t *old_tctx);
void	prof_free(tsd_t *tsd, const void *ptr, size_t usize);
#endif

#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_))
JEMALLOC_ALWAYS_INLINE bool
prof_active_get_unlocked(void)
{

	/*
	 * Even if opt_prof is true, sampling can be temporarily disabled by
	 * setting prof_active to false.  No locking is used when reading
	 * prof_active in the fast path, so there are no guarantees regarding
	 * how long it will take for all threads to notice state changes.
	 */
	return (prof_active);
}

JEMALLOC_ALWAYS_INLINE bool
prof_gdump_get_unlocked(void)
{

	/*
	 * No locking is used when reading prof_gdump_val in the fast path, so
	 * there are no guarantees regarding how long it will take for all
	 * threads to notice state changes.
	 */
	return (prof_gdump_val);
}

JEMALLOC_ALWAYS_INLINE prof_tdata_t *
prof_tdata_get(tsd_t *tsd, bool create)
{
	prof_tdata_t *tdata;

	cassert(config_prof);

	tdata = tsd_prof_tdata_get(tsd);
	if (create) {
		if (unlikely(tdata == NULL)) {
			if (tsd_nominal(tsd)) {
				tdata = prof_tdata_init(tsd);
				tsd_prof_tdata_set(tsd, tdata);
			}
		} else if (unlikely(tdata->expired)) {
			tdata = prof_tdata_reinit(tsd, tdata);
			tsd_prof_tdata_set(tsd, tdata);
		}
		assert(tdata == NULL || tdata->attached);
	}

	return (tdata);
}

JEMALLOC_ALWAYS_INLINE prof_tctx_t *
prof_tctx_get(tsdn_t *tsdn, const void *ptr)
{

	cassert(config_prof);
	assert(ptr != NULL);

	return (arena_prof_tctx_get(tsdn, ptr));
}

JEMALLOC_ALWAYS_INLINE void
prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx)
{

	cassert(config_prof);
	assert(ptr != NULL);

	arena_prof_tctx_set(tsdn, ptr, usize, tctx);
}

JEMALLOC_ALWAYS_INLINE void
prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize, const void *old_ptr,
    prof_tctx_t *old_tctx)
{

	cassert(config_prof);
	assert(ptr != NULL);

	arena_prof_tctx_reset(tsdn, ptr, usize, old_ptr, old_tctx);
}

JEMALLOC_ALWAYS_INLINE bool
prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update,
    prof_tdata_t **tdata_out)
{
	prof_tdata_t *tdata;

	cassert(config_prof);

	tdata = prof_tdata_get(tsd, true);
	if (unlikely((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX))
		tdata = NULL;

	if (tdata_out != NULL)
		*tdata_out = tdata;

	if (unlikely(tdata == NULL))
		return (true);

	if (likely(tdata->bytes_until_sample >= usize)) {
		if (update)
			tdata->bytes_until_sample -= usize;
		return (true);
	} else {
		/* Compute new sample threshold. */
		if (update)
			prof_sample_threshold_update(tdata);
		return (!tdata->active);
	}
}

JEMALLOC_ALWAYS_INLINE prof_tctx_t *
prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, bool update)
{
	prof_tctx_t *ret;
	prof_tdata_t *tdata;
	prof_bt_t bt;

	assert(usize == s2u(usize));

	if (!prof_active || likely(prof_sample_accum_update(tsd, usize, update,
	    &tdata)))
		ret = (prof_tctx_t *)(uintptr_t)1U;
	else {
		bt_init(&bt, tdata->vec);
		prof_backtrace(&bt);
		ret = prof_lookup(tsd, &bt);
	}

	return (ret);
}

JEMALLOC_ALWAYS_INLINE void
prof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx)
{

	cassert(config_prof);
	assert(ptr != NULL);
	assert(usize == isalloc(tsdn, ptr, true));

	if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
		prof_malloc_sample_object(tsdn, ptr, usize, tctx);
	else
		prof_tctx_set(tsdn, ptr, usize, (prof_tctx_t *)(uintptr_t)1U);
}

JEMALLOC_ALWAYS_INLINE void
prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx,
    bool prof_active, bool updated, const void *old_ptr, size_t old_usize,
    prof_tctx_t *old_tctx)
{
	bool sampled, old_sampled;

	cassert(config_prof);
	assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U);

	if (prof_active && !updated && ptr != NULL) {
		assert(usize == isalloc(tsd_tsdn(tsd), ptr, true));
		if (prof_sample_accum_update(tsd, usize, true, NULL)) {
			/*
			 * Don't sample.  The usize passed to prof_alloc_prep()
			 * was larger than what actually got allocated, so a
			 * backtrace was captured for this allocation, even
			 * though its actual usize was insufficient to cross the
			 * sample threshold.
			 */
			prof_alloc_rollback(tsd, tctx, true);
			tctx = (prof_tctx_t *)(uintptr_t)1U;
		}
	}

	sampled = ((uintptr_t)tctx > (uintptr_t)1U);
	old_sampled = ((uintptr_t)old_tctx > (uintptr_t)1U);

	if (unlikely(sampled))
		prof_malloc_sample_object(tsd_tsdn(tsd), ptr, usize, tctx);
	else
		prof_tctx_reset(tsd_tsdn(tsd), ptr, usize, old_ptr, old_tctx);

	if (unlikely(old_sampled))
		prof_free_sampled_object(tsd, old_usize, old_tctx);
}

JEMALLOC_ALWAYS_INLINE void
prof_free(tsd_t *tsd, const void *ptr, size_t usize)
{
	prof_tctx_t *tctx = prof_tctx_get(tsd_tsdn(tsd), ptr);

	cassert(config_prof);
	assert(usize == isalloc(tsd_tsdn(tsd), ptr, true));

	if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
		prof_free_sampled_object(tsd, usize, tctx);
}
#endif

#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
Commit	Line	Data
1a4d82fc JJ	1	/******************************************************************************/
	2	#ifdef JEMALLOC_H_TYPES
	3
	4	typedef struct prof_bt_s prof_bt_t;
	5	typedef struct prof_cnt_s prof_cnt_t;
	6	typedef struct prof_tctx_s prof_tctx_t;
	7	typedef struct prof_gctx_s prof_gctx_t;
	8	typedef struct prof_tdata_s prof_tdata_t;
	9
	10	/* Option defaults. */
	11	#ifdef JEMALLOC_PROF
	12	# define PROF_PREFIX_DEFAULT "jeprof"
	13	#else
	14	# define PROF_PREFIX_DEFAULT ""
	15	#endif
	16	#define LG_PROF_SAMPLE_DEFAULT 19
	17	#define LG_PROF_INTERVAL_DEFAULT -1
	18
	19	/*
	20	* Hard limit on stack backtrace depth. The version of prof_backtrace() that
	21	* is based on __builtin_return_address() necessarily has a hard-coded number
	22	* of backtrace frame handlers, and should be kept in sync with this setting.
	23	*/
	24	#define PROF_BT_MAX 128
	25
	26	/* Initial hash table size. */
	27	#define PROF_CKH_MINITEMS 64
	28
	29	/* Size of memory buffer to use when writing dump files. */
	30	#define PROF_DUMP_BUFSIZE 65536
	31
	32	/* Size of stack-allocated buffer used by prof_printf(). */
	33	#define PROF_PRINTF_BUFSIZE 128
	34
	35	/*
	36	* Number of mutexes shared among all gctx's. No space is allocated for these
	37	* unless profiling is enabled, so it's okay to over-provision.
	38	*/
	39	#define PROF_NCTX_LOCKS 1024
	40
	41	/*
	42	* Number of mutexes shared among all tdata's. No space is allocated for these
	43	* unless profiling is enabled, so it's okay to over-provision.
	44	*/
	45	#define PROF_NTDATA_LOCKS 256
	46
	47	/*
	48	* prof_tdata pointers close to NULL are used to encode state information that
	49	* is used for cleaning up during thread shutdown.
	50	*/
	51	#define PROF_TDATA_STATE_REINCARNATED ((prof_tdata_t *)(uintptr_t)1)
	52	#define PROF_TDATA_STATE_PURGATORY ((prof_tdata_t *)(uintptr_t)2)
	53	#define PROF_TDATA_STATE_MAX PROF_TDATA_STATE_PURGATORY
	54
	55	#endif /* JEMALLOC_H_TYPES */
	56	/******************************************************************************/
	57	#ifdef JEMALLOC_H_STRUCTS
	58
	59	struct prof_bt_s {
	60	/* Backtrace, stored as len program counters. */
	61	void **vec;
	62	unsigned len;
	63	};
	64
65	#ifdef JEMALLOC_PROF_LIBGCC
66	/* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */
67	typedef struct {
68	prof_bt_t *bt;
69	unsigned max;
70	} prof_unwind_data_t;
71	#endif
72
73	struct prof_cnt_s {
74	/* Profiling counters. */
75	uint64_t curobjs;
76	uint64_t curbytes;
77	uint64_t accumobjs;
78	uint64_t accumbytes;
79	};
80
81	typedef enum {
82	prof_tctx_state_initializing,
83	prof_tctx_state_nominal,
84	prof_tctx_state_dumping,
85	prof_tctx_state_purgatory /* Dumper must finish destroying. */
86	} prof_tctx_state_t;
87
88	struct prof_tctx_s {
89	/* Thread data for thread that performed the allocation. */
90	prof_tdata_t *tdata;
91
54a0048b SL	92	/*
	93	* Copy of tdata->thr_{uid,discrim}, necessary because tdata may be
	94	* defunct during teardown.
	95	*/
	96	uint64_t thr_uid;
	97	uint64_t thr_discrim;
	98
1a4d82fc JJ	99	/* Profiling counters, protected by tdata->lock. */
	100	prof_cnt_t cnts;
	101
	102	/* Associated global context. */
	103	prof_gctx_t *gctx;
	104
54a0048b SL	105	/*
	106	* UID that distinguishes multiple tctx's created by the same thread,
	107	* but coexisting in gctx->tctxs. There are two ways that such
	108	* coexistence can occur:
	109	* - A dumper thread can cause a tctx to be retained in the purgatory
	110	* state.
	111	* - Although a single "producer" thread must create all tctx's which
	112	* share the same thr_uid, multiple "consumers" can each concurrently
	113	* execute portions of prof_tctx_destroy(). prof_tctx_destroy() only
	114	* gets called once each time cnts.cur{objs,bytes} drop to 0, but this
	115	* threshold can be hit again before the first consumer finishes
	116	* executing prof_tctx_destroy().
	117	*/
	118	uint64_t tctx_uid;
	119
1a4d82fc JJ	120	/* Linkage into gctx's tctxs. */
	121	rb_node(prof_tctx_t) tctx_link;
	122
	123	/*
	124	* True during prof_alloc_prep()..prof_malloc_sample_object(), prevents
	125	* sample vs destroy race.
	126	*/
	127	bool prepared;
	128
	129	/* Current dump-related state, protected by gctx->lock. */
	130	prof_tctx_state_t state;
	131
	132	/*
	133	* Copy of cnts snapshotted during early dump phase, protected by
	134	* dump_mtx.
	135	*/
	136	prof_cnt_t dump_cnts;
	137	};
	138	typedef rb_tree(prof_tctx_t) prof_tctx_tree_t;
	139
	140	struct prof_gctx_s {
	141	/* Protects nlimbo, cnt_summed, and tctxs. */
	142	malloc_mutex_t *lock;
	143
	144	/*
	145	* Number of threads that currently cause this gctx to be in a state of
	146	* limbo due to one of:
	147	* - Initializing this gctx.
	148	* - Initializing per thread counters associated with this gctx.
	149	* - Preparing to destroy this gctx.
	150	* - Dumping a heap profile that includes this gctx.
	151	* nlimbo must be 1 (single destroyer) in order to safely destroy the
	152	* gctx.
	153	*/
	154	unsigned nlimbo;
	155
	156	/*
	157	* Tree of profile counters, one for each thread that has allocated in
	158	* this context.
	159	*/
	160	prof_tctx_tree_t tctxs;
	161
	162	/* Linkage for tree of contexts to be dumped. */
	163	rb_node(prof_gctx_t) dump_link;
	164
	165	/* Temporary storage for summation during dump. */
	166	prof_cnt_t cnt_summed;
	167
	168	/* Associated backtrace. */
	169	prof_bt_t bt;
	170
	171	/* Backtrace vector, variable size, referred to by bt. */
	172	void *vec[1];
	173	};
	174	typedef rb_tree(prof_gctx_t) prof_gctx_tree_t;
	175
	176	struct prof_tdata_s {
	177	malloc_mutex_t *lock;
	178
	179	/* Monotonically increasing unique thread identifier. */
	180	uint64_t thr_uid;
	181
	182	/*
	183	* Monotonically increasing discriminator among tdata structures
184	* associated with the same thr_uid.
185	*/
186	uint64_t thr_discrim;
187
188	/* Included in heap profile dumps if non-NULL. */
189	char *thread_name;
190
191	bool attached;
192	bool expired;
193
194	rb_node(prof_tdata_t) tdata_link;
195
54a0048b SL	196	/*
	197	* Counter used to initialize prof_tctx_t's tctx_uid. No locking is
	198	* necessary when incrementing this field, because only one thread ever
	199	* does so.
	200	*/
	201	uint64_t tctx_uid_next;
	202
1a4d82fc JJ	203	/*
	204	* Hash of (prof_bt_t )-->(prof_tctx_t ). Each thread tracks
	205	* backtraces for which it has non-zero allocation/deallocation counters
	206	* associated with thread-specific prof_tctx_t objects. Other threads
	207	* may write to prof_tctx_t contents when freeing associated objects.
	208	*/
	209	ckh_t bt2tctx;
	210
	211	/* Sampling state. */
	212	uint64_t prng_state;
	213	uint64_t bytes_until_sample;
	214
	215	/* State used to avoid dumping while operating on prof internals. */
	216	bool enq;
	217	bool enq_idump;
	218	bool enq_gdump;
	219
	220	/*
	221	* Set to true during an early dump phase for tdata's which are
	222	* currently being dumped. New threads' tdata's have this initialized
	223	* to false so that they aren't accidentally included in later dump
	224	* phases.
	225	*/
	226	bool dumping;
	227
	228	/*
	229	* True if profiling is active for this tdata's thread
	230	* (thread.prof.active mallctl).
	231	*/
	232	bool active;
	233
	234	/* Temporary storage for summation during dump. */
	235	prof_cnt_t cnt_summed;
	236
	237	/* Backtrace vector, used for calls to prof_backtrace(). */
	238	void *vec[PROF_BT_MAX];
	239	};
	240	typedef rb_tree(prof_tdata_t) prof_tdata_tree_t;
	241
	242	#endif /* JEMALLOC_H_STRUCTS */
	243	/******************************************************************************/
	244	#ifdef JEMALLOC_H_EXTERNS
	245
	246	extern bool opt_prof;
	247	extern bool opt_prof_active;
	248	extern bool opt_prof_thread_active_init;
	249	extern size_t opt_lg_prof_sample; /* Mean bytes between samples. */
	250	extern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */
	251	extern bool opt_prof_gdump; /* High-water memory dumping. */
	252	extern bool opt_prof_final; /* Final profile dumping. */
	253	extern bool opt_prof_leak; /* Dump leak summary at exit. */
	254	extern bool opt_prof_accum; /* Report cumulative bytes. */
	255	extern char opt_prof_prefix[
	256	/* Minimize memory bloat for non-prof builds. */
	257	#ifdef JEMALLOC_PROF
	258	PATH_MAX +
	259	#endif
	260	1];
	261
	262	/* Accessed via prof_active_[gs]et{_unlocked,}(). */
	263	extern bool prof_active;
	264
54a0048b SL	265	/* Accessed via prof_gdump_[gs]et{_unlocked,}(). */
	266	extern bool prof_gdump_val;
	267
1a4d82fc JJ	268	/*
	269	* Profile dump interval, measured in bytes allocated. Each arena triggers a
	270	* profile dump when it reaches this threshold. The effect is that the
	271	* interval between profile dumps averages prof_interval, though the actual
	272	* interval between dumps will tend to be sporadic, and the interval will be a
	273	* maximum of approximately (prof_interval * narenas).
	274	*/
	275	extern uint64_t prof_interval;
	276
	277	/*
	278	* Initialized as opt_lg_prof_sample, and potentially modified during profiling
	279	* resets.
	280	*/
	281	extern size_t lg_prof_sample;
	282
	283	void prof_alloc_rollback(tsd_t tsd, prof_tctx_t tctx, bool updated);
3b2f2976	284	void prof_malloc_sample_object(tsdn_t tsdn, const void ptr, size_t usize,
1a4d82fc JJ	285	prof_tctx_t *tctx);
	286	void prof_free_sampled_object(tsd_t tsd, size_t usize, prof_tctx_t tctx);
	287	void bt_init(prof_bt_t bt, void *vec);
	288	void prof_backtrace(prof_bt_t *bt);
	289	prof_tctx_t prof_lookup(tsd_t tsd, prof_bt_t *bt);
	290	#ifdef JEMALLOC_JET
	291	size_t prof_tdata_count(void);
	292	size_t prof_bt_count(void);
	293	const prof_cnt_t *prof_cnt_all(void);
	294	typedef int (prof_dump_open_t)(bool, const char *);
	295	extern prof_dump_open_t *prof_dump_open;
3b2f2976	296	typedef bool (prof_dump_header_t)(tsdn_t , bool, const prof_cnt_t );
1a4d82fc JJ	297	extern prof_dump_header_t *prof_dump_header;
1a4d82fc JJ	298	#endif
3b2f2976 XL	299	void prof_idump(tsdn_t *tsdn);
	300	bool prof_mdump(tsd_t tsd, const char filename);
	301	void prof_gdump(tsdn_t *tsdn);
1a4d82fc JJ	302	prof_tdata_t prof_tdata_init(tsd_t tsd);
	303	prof_tdata_t prof_tdata_reinit(tsd_t tsd, prof_tdata_t *tdata);
	304	void prof_reset(tsd_t *tsd, size_t lg_sample);
	305	void prof_tdata_cleanup(tsd_t *tsd);
3b2f2976 XL	306	bool prof_active_get(tsdn_t *tsdn);
	307	bool prof_active_set(tsdn_t *tsdn, bool active);
	308	const char prof_thread_name_get(tsd_t tsd);
1a4d82fc	309	int prof_thread_name_set(tsd_t tsd, const char thread_name);
3b2f2976 XL	310	bool prof_thread_active_get(tsd_t *tsd);
	311	bool prof_thread_active_set(tsd_t *tsd, bool active);
	312	bool prof_thread_active_init_get(tsdn_t *tsdn);
	313	bool prof_thread_active_init_set(tsdn_t *tsdn, bool active_init);
	314	bool prof_gdump_get(tsdn_t *tsdn);
	315	bool prof_gdump_set(tsdn_t *tsdn, bool active);
1a4d82fc JJ	316	void prof_boot0(void);
1a4d82fc JJ	317	void prof_boot1(void);
3b2f2976 XL	318	bool prof_boot2(tsd_t *tsd);
	319	void prof_prefork0(tsdn_t *tsdn);
	320	void prof_prefork1(tsdn_t *tsdn);
	321	void prof_postfork_parent(tsdn_t *tsdn);
	322	void prof_postfork_child(tsdn_t *tsdn);
1a4d82fc JJ	323	void prof_sample_threshold_update(prof_tdata_t *tdata);
	324
	325	#endif /* JEMALLOC_H_EXTERNS */
	326	/******************************************************************************/
	327	#ifdef JEMALLOC_H_INLINES
	328
	329	#ifndef JEMALLOC_ENABLE_INLINE
	330	bool prof_active_get_unlocked(void);
54a0048b	331	bool prof_gdump_get_unlocked(void);
1a4d82fc	332	prof_tdata_t prof_tdata_get(tsd_t tsd, bool create);
3b2f2976 XL	333	prof_tctx_t prof_tctx_get(tsdn_t tsdn, const void *ptr);
	334	void prof_tctx_set(tsdn_t tsdn, const void ptr, size_t usize,
	335	prof_tctx_t *tctx);
	336	void prof_tctx_reset(tsdn_t tsdn, const void ptr, size_t usize,
	337	const void old_ptr, prof_tctx_t tctx);
1a4d82fc JJ	338	bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit,
1a4d82fc JJ	339	prof_tdata_t **tdata_out);
54a0048b SL	340	prof_tctx_t prof_alloc_prep(tsd_t tsd, size_t usize, bool prof_active,
54a0048b SL	341	bool update);
3b2f2976	342	void prof_malloc(tsdn_t tsdn, const void ptr, size_t usize,
1a4d82fc	343	prof_tctx_t *tctx);
1a4d82fc	344	void prof_realloc(tsd_t tsd, const void ptr, size_t usize,
54a0048b SL	345	prof_tctx_t tctx, bool prof_active, bool updated, const void old_ptr,
54a0048b SL	346	size_t old_usize, prof_tctx_t *old_tctx);
1a4d82fc JJ	347	void prof_free(tsd_t tsd, const void ptr, size_t usize);
	348	#endif
	349
	350	#if (defined(JEMALLOC_ENABLE_INLINE) \|\| defined(JEMALLOC_PROF_C_))
	351	JEMALLOC_ALWAYS_INLINE bool
	352	prof_active_get_unlocked(void)
	353	{
	354
	355	/*
	356	* Even if opt_prof is true, sampling can be temporarily disabled by
	357	* setting prof_active to false. No locking is used when reading
	358	* prof_active in the fast path, so there are no guarantees regarding
	359	* how long it will take for all threads to notice state changes.
	360	*/
	361	return (prof_active);
	362	}
	363
54a0048b SL	364	JEMALLOC_ALWAYS_INLINE bool
	365	prof_gdump_get_unlocked(void)
	366	{
	367
	368	/*
	369	* No locking is used when reading prof_gdump_val in the fast path, so
	370	* there are no guarantees regarding how long it will take for all
	371	* threads to notice state changes.
	372	*/
	373	return (prof_gdump_val);
	374	}
	375
1a4d82fc JJ	376	JEMALLOC_ALWAYS_INLINE prof_tdata_t *
	377	prof_tdata_get(tsd_t *tsd, bool create)
	378	{
	379	prof_tdata_t *tdata;
	380
	381	cassert(config_prof);
	382
	383	tdata = tsd_prof_tdata_get(tsd);
	384	if (create) {
	385	if (unlikely(tdata == NULL)) {
	386	if (tsd_nominal(tsd)) {
	387	tdata = prof_tdata_init(tsd);
	388	tsd_prof_tdata_set(tsd, tdata);
	389	}
	390	} else if (unlikely(tdata->expired)) {
	391	tdata = prof_tdata_reinit(tsd, tdata);
	392	tsd_prof_tdata_set(tsd, tdata);
	393	}
	394	assert(tdata == NULL \|\| tdata->attached);
	395	}
	396
	397	return (tdata);
	398	}
	399
	400	JEMALLOC_ALWAYS_INLINE prof_tctx_t *
3b2f2976	401	prof_tctx_get(tsdn_t tsdn, const void ptr)
1a4d82fc	402	{
1a4d82fc JJ	403
	404	cassert(config_prof);
	405	assert(ptr != NULL);
	406
3b2f2976	407	return (arena_prof_tctx_get(tsdn, ptr));
54a0048b	408	}
1a4d82fc	409
54a0048b	410	JEMALLOC_ALWAYS_INLINE void
3b2f2976	411	prof_tctx_set(tsdn_t tsdn, const void ptr, size_t usize, prof_tctx_t *tctx)
54a0048b SL	412	{
	413
	414	cassert(config_prof);
	415	assert(ptr != NULL);
	416
3b2f2976	417	arena_prof_tctx_set(tsdn, ptr, usize, tctx);
1a4d82fc JJ	418	}
	419
	420	JEMALLOC_ALWAYS_INLINE void
3b2f2976	421	prof_tctx_reset(tsdn_t tsdn, const void ptr, size_t usize, const void *old_ptr,
54a0048b	422	prof_tctx_t *old_tctx)
1a4d82fc	423	{
1a4d82fc JJ	424
	425	cassert(config_prof);
	426	assert(ptr != NULL);
	427
3b2f2976	428	arena_prof_tctx_reset(tsdn, ptr, usize, old_ptr, old_tctx);
1a4d82fc JJ	429	}
	430
	431	JEMALLOC_ALWAYS_INLINE bool
	432	prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update,
	433	prof_tdata_t **tdata_out)
	434	{
	435	prof_tdata_t *tdata;
	436
	437	cassert(config_prof);
	438
	439	tdata = prof_tdata_get(tsd, true);
54a0048b	440	if (unlikely((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX))
1a4d82fc JJ	441	tdata = NULL;
	442
	443	if (tdata_out != NULL)
	444	*tdata_out = tdata;
	445
54a0048b	446	if (unlikely(tdata == NULL))
1a4d82fc JJ	447	return (true);
1a4d82fc JJ	448
54a0048b	449	if (likely(tdata->bytes_until_sample >= usize)) {
1a4d82fc JJ	450	if (update)
	451	tdata->bytes_until_sample -= usize;
	452	return (true);
	453	} else {
	454	/* Compute new sample threshold. */
	455	if (update)
	456	prof_sample_threshold_update(tdata);
	457	return (!tdata->active);
	458	}
	459	}
	460
	461	JEMALLOC_ALWAYS_INLINE prof_tctx_t *
54a0048b	462	prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, bool update)
1a4d82fc JJ	463	{
	464	prof_tctx_t *ret;
	465	prof_tdata_t *tdata;
	466	prof_bt_t bt;
	467
	468	assert(usize == s2u(usize));
	469
54a0048b SL	470	if (!prof_active \|\| likely(prof_sample_accum_update(tsd, usize, update,
54a0048b SL	471	&tdata)))
1a4d82fc JJ	472	ret = (prof_tctx_t *)(uintptr_t)1U;
	473	else {
	474	bt_init(&bt, tdata->vec);
	475	prof_backtrace(&bt);
	476	ret = prof_lookup(tsd, &bt);
	477	}
	478
	479	return (ret);
	480	}
	481
	482	JEMALLOC_ALWAYS_INLINE void
3b2f2976	483	prof_malloc(tsdn_t tsdn, const void ptr, size_t usize, prof_tctx_t *tctx)
1a4d82fc JJ	484	{
	485
	486	cassert(config_prof);
	487	assert(ptr != NULL);
3b2f2976	488	assert(usize == isalloc(tsdn, ptr, true));
1a4d82fc JJ	489
1a4d82fc JJ	490	if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
3b2f2976	491	prof_malloc_sample_object(tsdn, ptr, usize, tctx);
1a4d82fc	492	else
3b2f2976	493	prof_tctx_set(tsdn, ptr, usize, (prof_tctx_t *)(uintptr_t)1U);
1a4d82fc JJ	494	}
	495
	496	JEMALLOC_ALWAYS_INLINE void
	497	prof_realloc(tsd_t tsd, const void ptr, size_t usize, prof_tctx_t *tctx,
54a0048b SL	498	bool prof_active, bool updated, const void *old_ptr, size_t old_usize,
54a0048b SL	499	prof_tctx_t *old_tctx)
1a4d82fc	500	{
54a0048b	501	bool sampled, old_sampled;
1a4d82fc JJ	502
	503	cassert(config_prof);
	504	assert(ptr != NULL \|\| (uintptr_t)tctx <= (uintptr_t)1U);
	505
54a0048b	506	if (prof_active && !updated && ptr != NULL) {
3b2f2976	507	assert(usize == isalloc(tsd_tsdn(tsd), ptr, true));
1a4d82fc JJ	508	if (prof_sample_accum_update(tsd, usize, true, NULL)) {
1a4d82fc JJ	509	/*
54a0048b	510	* Don't sample. The usize passed to prof_alloc_prep()
1a4d82fc JJ	511	* was larger than what actually got allocated, so a
	512	* backtrace was captured for this allocation, even
	513	* though its actual usize was insufficient to cross the
	514	* sample threshold.
	515	*/
3b2f2976	516	prof_alloc_rollback(tsd, tctx, true);
1a4d82fc JJ	517	tctx = (prof_tctx_t *)(uintptr_t)1U;
	518	}
	519	}
	520
54a0048b SL	521	sampled = ((uintptr_t)tctx > (uintptr_t)1U);
	522	old_sampled = ((uintptr_t)old_tctx > (uintptr_t)1U);
	523
	524	if (unlikely(sampled))
3b2f2976	525	prof_malloc_sample_object(tsd_tsdn(tsd), ptr, usize, tctx);
1a4d82fc	526	else
3b2f2976	527	prof_tctx_reset(tsd_tsdn(tsd), ptr, usize, old_ptr, old_tctx);
54a0048b SL	528
	529	if (unlikely(old_sampled))
	530	prof_free_sampled_object(tsd, old_usize, old_tctx);
1a4d82fc JJ	531	}
	532
	533	JEMALLOC_ALWAYS_INLINE void
	534	prof_free(tsd_t tsd, const void ptr, size_t usize)
	535	{
3b2f2976	536	prof_tctx_t *tctx = prof_tctx_get(tsd_tsdn(tsd), ptr);
1a4d82fc JJ	537
1a4d82fc JJ	538	cassert(config_prof);
3b2f2976	539	assert(usize == isalloc(tsd_tsdn(tsd), ptr, true));
1a4d82fc JJ	540
	541	if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
	542	prof_free_sampled_object(tsd, usize, tctx);
	543	}
	544	#endif
	545
	546	#endif /* JEMALLOC_H_INLINES */
	547	/******************************************************************************/