[rustc.git] / src / jemalloc / include / jemalloc / internal / tcache.h

/******************************************************************************/
#ifdef JEMALLOC_H_TYPES

typedef struct tcache_bin_info_s tcache_bin_info_t;
typedef struct tcache_bin_s tcache_bin_t;
typedef struct tcache_s tcache_t;
typedef struct tcaches_s tcaches_t;

/*
 * tcache pointers close to NULL are used to encode state information that is
 * used for two purposes: preventing thread caching on a per thread basis and
 * cleaning up during thread shutdown.
 */
#define	TCACHE_STATE_DISABLED		((tcache_t *)(uintptr_t)1)
#define	TCACHE_STATE_REINCARNATED	((tcache_t *)(uintptr_t)2)
#define	TCACHE_STATE_PURGATORY		((tcache_t *)(uintptr_t)3)
#define	TCACHE_STATE_MAX		TCACHE_STATE_PURGATORY

/*
 * Absolute minimum number of cache slots for each small bin.
 */
#define	TCACHE_NSLOTS_SMALL_MIN		20

/*
 * Absolute maximum number of cache slots for each small bin in the thread
 * cache.  This is an additional constraint beyond that imposed as: twice the
 * number of regions per run for this size class.
 *
 * This constant must be an even number.
 */
#define	TCACHE_NSLOTS_SMALL_MAX		200

/* Number of cache slots for large size classes. */
#define	TCACHE_NSLOTS_LARGE		20

/* (1U << opt_lg_tcache_max) is used to compute tcache_maxclass. */
#define	LG_TCACHE_MAXCLASS_DEFAULT	15

/*
 * TCACHE_GC_SWEEP is the approximate number of allocation events between
 * full GC sweeps.  Integer rounding may cause the actual number to be
 * slightly higher, since GC is performed incrementally.
 */
#define	TCACHE_GC_SWEEP			8192

/* Number of tcache allocation/deallocation events between incremental GCs. */
#define	TCACHE_GC_INCR							\
    ((TCACHE_GC_SWEEP / NBINS) + ((TCACHE_GC_SWEEP / NBINS == 0) ? 0 : 1))

#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS

typedef enum {
	tcache_enabled_false   = 0, /* Enable cast to/from bool. */
	tcache_enabled_true    = 1,
	tcache_enabled_default = 2
} tcache_enabled_t;

/*
 * Read-only information associated with each element of tcache_t's tbins array
 * is stored separately, mainly to reduce memory usage.
 */
struct tcache_bin_info_s {
	unsigned	ncached_max;	/* Upper limit on ncached. */
};

struct tcache_bin_s {
	tcache_bin_stats_t tstats;
	int		low_water;	/* Min # cached since last GC. */
	unsigned	lg_fill_div;	/* Fill (ncached_max >> lg_fill_div). */
	unsigned	ncached;	/* # of cached objects. */
	/*
	 * To make use of adjacent cacheline prefetch, the items in the avail
	 * stack goes to higher address for newer allocations.  avail points
	 * just above the available space, which means that
	 * avail[-ncached, ... -1] are available items and the lowest item will
	 * be allocated first.
	 */
	void		**avail;	/* Stack of available objects. */
};

struct tcache_s {
	ql_elm(tcache_t) link;		/* Used for aggregating stats. */
	uint64_t	prof_accumbytes;/* Cleared after arena_prof_accum(). */
	ticker_t	gc_ticker;	/* Drives incremental GC. */
	szind_t		next_gc_bin;	/* Next bin to GC. */
	tcache_bin_t	tbins[1];	/* Dynamically sized. */
	/*
	 * The pointer stacks associated with tbins follow as a contiguous
	 * array.  During tcache initialization, the avail pointer in each
	 * element of tbins is initialized to point to the proper offset within
	 * this array.
	 */
};

/* Linkage for list of available (previously used) explicit tcache IDs. */
struct tcaches_s {
	union {
		tcache_t	*tcache;
		tcaches_t	*next;
	};
};

#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS

extern bool	opt_tcache;
extern ssize_t	opt_lg_tcache_max;

extern tcache_bin_info_t	*tcache_bin_info;

/*
 * Number of tcache bins.  There are NBINS small-object bins, plus 0 or more
 * large-object bins.
 */
extern unsigned	nhbins;

/* Maximum cached size class. */
extern size_t	tcache_maxclass;

/*
 * Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
 * usable via the MALLOCX_TCACHE() flag.  The automatic per thread tcaches are
 * completely disjoint from this data structure.  tcaches starts off as a sparse
 * array, so it has no physical memory footprint until individual pages are
 * touched.  This allows the entire array to be allocated the first time an
 * explicit tcache is created without a disproportionate impact on memory usage.
 */
extern tcaches_t	*tcaches;

size_t	tcache_salloc(const void *ptr);
void	tcache_event_hard(tsd_t *tsd, tcache_t *tcache);
void	*tcache_alloc_small_hard(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
    tcache_bin_t *tbin, szind_t binind, bool *tcache_success);
void	tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
    szind_t binind, unsigned rem);
void	tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
    unsigned rem, tcache_t *tcache);
void	tcache_arena_associate(tcache_t *tcache, arena_t *arena);
void	tcache_arena_reassociate(tcache_t *tcache, arena_t *oldarena,
    arena_t *newarena);
void	tcache_arena_dissociate(tcache_t *tcache, arena_t *arena);
tcache_t *tcache_get_hard(tsd_t *tsd);
tcache_t *tcache_create(tsd_t *tsd, arena_t *arena);
void	tcache_cleanup(tsd_t *tsd);
void	tcache_enabled_cleanup(tsd_t *tsd);
void	tcache_stats_merge(tcache_t *tcache, arena_t *arena);
bool	tcaches_create(tsd_t *tsd, unsigned *r_ind);
void	tcaches_flush(tsd_t *tsd, unsigned ind);
void	tcaches_destroy(tsd_t *tsd, unsigned ind);
bool	tcache_boot(void);

#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES

#ifndef JEMALLOC_ENABLE_INLINE
void	tcache_event(tsd_t *tsd, tcache_t *tcache);
void	tcache_flush(void);
bool	tcache_enabled_get(void);
tcache_t *tcache_get(tsd_t *tsd, bool create);
void	tcache_enabled_set(bool enabled);
void	*tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success);
void	*tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
    size_t size, szind_t ind, bool zero, bool slow_path);
void	*tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
    size_t size, szind_t ind, bool zero, bool slow_path);
void	tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr,
    szind_t binind, bool slow_path);
void	tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr,
    size_t size, bool slow_path);
tcache_t	*tcaches_get(tsd_t *tsd, unsigned ind);
#endif

#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_))
JEMALLOC_INLINE void
tcache_flush(void)
{
	tsd_t *tsd;

	cassert(config_tcache);

	tsd = tsd_fetch();
	tcache_cleanup(tsd);
}

JEMALLOC_INLINE bool
tcache_enabled_get(void)
{
	tsd_t *tsd;
	tcache_enabled_t tcache_enabled;

	cassert(config_tcache);

	tsd = tsd_fetch();
	tcache_enabled = tsd_tcache_enabled_get(tsd);
	if (tcache_enabled == tcache_enabled_default) {
		tcache_enabled = (tcache_enabled_t)opt_tcache;
		tsd_tcache_enabled_set(tsd, tcache_enabled);
	}

	return ((bool)tcache_enabled);
}

JEMALLOC_INLINE void
tcache_enabled_set(bool enabled)
{
	tsd_t *tsd;
	tcache_enabled_t tcache_enabled;

	cassert(config_tcache);

	tsd = tsd_fetch();

	tcache_enabled = (tcache_enabled_t)enabled;
	tsd_tcache_enabled_set(tsd, tcache_enabled);

	if (!enabled)
		tcache_cleanup(tsd);
}

JEMALLOC_ALWAYS_INLINE tcache_t *
tcache_get(tsd_t *tsd, bool create)
{
	tcache_t *tcache;

	if (!config_tcache)
		return (NULL);

	tcache = tsd_tcache_get(tsd);
	if (!create)
		return (tcache);
	if (unlikely(tcache == NULL) && tsd_nominal(tsd)) {
		tcache = tcache_get_hard(tsd);
		tsd_tcache_set(tsd, tcache);
	}

	return (tcache);
}

JEMALLOC_ALWAYS_INLINE void
tcache_event(tsd_t *tsd, tcache_t *tcache)
{

	if (TCACHE_GC_INCR == 0)
		return;

	if (unlikely(ticker_tick(&tcache->gc_ticker)))
		tcache_event_hard(tsd, tcache);
}

JEMALLOC_ALWAYS_INLINE void *
tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success)
{
	void *ret;

	if (unlikely(tbin->ncached == 0)) {
		tbin->low_water = -1;
		*tcache_success = false;
		return (NULL);
	}
	/*
	 * tcache_success (instead of ret) should be checked upon the return of
	 * this function.  We avoid checking (ret == NULL) because there is
	 * never a null stored on the avail stack (which is unknown to the
	 * compiler), and eagerly checking ret would cause pipeline stall
	 * (waiting for the cacheline).
	 */
	*tcache_success = true;
	ret = *(tbin->avail - tbin->ncached);
	tbin->ncached--;

	if (unlikely((int)tbin->ncached < tbin->low_water))
		tbin->low_water = tbin->ncached;

	return (ret);
}

JEMALLOC_ALWAYS_INLINE void *
tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
    szind_t binind, bool zero, bool slow_path)
{
	void *ret;
	tcache_bin_t *tbin;
	bool tcache_success;
	size_t usize JEMALLOC_CC_SILENCE_INIT(0);

	assert(binind < NBINS);
	tbin = &tcache->tbins[binind];
	ret = tcache_alloc_easy(tbin, &tcache_success);
	assert(tcache_success == (ret != NULL));
	if (unlikely(!tcache_success)) {
		bool tcache_hard_success;
		arena = arena_choose(tsd, arena);
		if (unlikely(arena == NULL))
			return (NULL);

		ret = tcache_alloc_small_hard(tsd, arena, tcache, tbin, binind,
			&tcache_hard_success);
		if (tcache_hard_success == false)
			return (NULL);
	}

	assert(ret);
	/*
	 * Only compute usize if required.  The checks in the following if
	 * statement are all static.
	 */
	if (config_prof || (slow_path && config_fill) || unlikely(zero)) {
		usize = index2size(binind);
		assert(tcache_salloc(ret) == usize);
	}

	if (likely(!zero)) {
		if (slow_path && config_fill) {
			if (unlikely(opt_junk_alloc)) {
				arena_alloc_junk_small(ret,
				    &arena_bin_info[binind], false);
			} else if (unlikely(opt_zero))
				memset(ret, 0, usize);
		}
	} else {
		if (slow_path && config_fill && unlikely(opt_junk_alloc)) {
			arena_alloc_junk_small(ret, &arena_bin_info[binind],
			    true);
		}
		memset(ret, 0, usize);
	}

	if (config_stats)
		tbin->tstats.nrequests++;
	if (config_prof)
		tcache->prof_accumbytes += usize;
	tcache_event(tsd, tcache);
	return (ret);
}

JEMALLOC_ALWAYS_INLINE void *
tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
    szind_t binind, bool zero, bool slow_path)
{
	void *ret;
	tcache_bin_t *tbin;
	bool tcache_success;

	assert(binind < nhbins);
	tbin = &tcache->tbins[binind];
	ret = tcache_alloc_easy(tbin, &tcache_success);
	assert(tcache_success == (ret != NULL));
	if (unlikely(!tcache_success)) {
		/*
		 * Only allocate one large object at a time, because it's quite
		 * expensive to create one and not use it.
		 */
		arena = arena_choose(tsd, arena);
		if (unlikely(arena == NULL))
			return (NULL);

		ret = arena_malloc_large(tsd, arena, binind, zero);
		if (ret == NULL)
			return (NULL);
	} else {
		size_t usize JEMALLOC_CC_SILENCE_INIT(0);

		/* Only compute usize on demand */
		if (config_prof || (slow_path && config_fill) ||
		    unlikely(zero)) {
			usize = index2size(binind);
			assert(usize <= tcache_maxclass);
		}

		if (config_prof && usize == LARGE_MINCLASS) {
			arena_chunk_t *chunk =
			    (arena_chunk_t *)CHUNK_ADDR2BASE(ret);
			size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >>
			    LG_PAGE);
			arena_mapbits_large_binind_set(chunk, pageind,
			    BININD_INVALID);
		}
		if (likely(!zero)) {
			if (slow_path && config_fill) {
				if (unlikely(opt_junk_alloc))
					memset(ret, 0xa5, usize);
				else if (unlikely(opt_zero))
					memset(ret, 0, usize);
			}
		} else
			memset(ret, 0, usize);

		if (config_stats)
			tbin->tstats.nrequests++;
		if (config_prof)
			tcache->prof_accumbytes += usize;
	}

	tcache_event(tsd, tcache);
	return (ret);
}

JEMALLOC_ALWAYS_INLINE void
tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
    bool slow_path)
{
	tcache_bin_t *tbin;
	tcache_bin_info_t *tbin_info;

	assert(tcache_salloc(ptr) <= SMALL_MAXCLASS);

	if (slow_path && config_fill && unlikely(opt_junk_free))
		arena_dalloc_junk_small(ptr, &arena_bin_info[binind]);

	tbin = &tcache->tbins[binind];
	tbin_info = &tcache_bin_info[binind];
	if (unlikely(tbin->ncached == tbin_info->ncached_max)) {
		tcache_bin_flush_small(tsd, tcache, tbin, binind,
		    (tbin_info->ncached_max >> 1));
	}
	assert(tbin->ncached < tbin_info->ncached_max);
	tbin->ncached++;
	*(tbin->avail - tbin->ncached) = ptr;

	tcache_event(tsd, tcache);
}

JEMALLOC_ALWAYS_INLINE void
tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, size_t size,
    bool slow_path)
{
	szind_t binind;
	tcache_bin_t *tbin;
	tcache_bin_info_t *tbin_info;

	assert((size & PAGE_MASK) == 0);
	assert(tcache_salloc(ptr) > SMALL_MAXCLASS);
	assert(tcache_salloc(ptr) <= tcache_maxclass);

	binind = size2index(size);

	if (slow_path && config_fill && unlikely(opt_junk_free))
		arena_dalloc_junk_large(ptr, size);

	tbin = &tcache->tbins[binind];
	tbin_info = &tcache_bin_info[binind];
	if (unlikely(tbin->ncached == tbin_info->ncached_max)) {
		tcache_bin_flush_large(tsd, tbin, binind,
		    (tbin_info->ncached_max >> 1), tcache);
	}
	assert(tbin->ncached < tbin_info->ncached_max);
	tbin->ncached++;
	*(tbin->avail - tbin->ncached) = ptr;

	tcache_event(tsd, tcache);
}

JEMALLOC_ALWAYS_INLINE tcache_t *
tcaches_get(tsd_t *tsd, unsigned ind)
{
	tcaches_t *elm = &tcaches[ind];
	if (unlikely(elm->tcache == NULL))
		elm->tcache = tcache_create(tsd, arena_choose(tsd, NULL));
	return (elm->tcache);
}
#endif

#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
Commit	Line	Data
970d7e83 LB	1	/******************************************************************************/
	2	#ifdef JEMALLOC_H_TYPES
	3
	4	typedef struct tcache_bin_info_s tcache_bin_info_t;
	5	typedef struct tcache_bin_s tcache_bin_t;
	6	typedef struct tcache_s tcache_t;
54a0048b	7	typedef struct tcaches_s tcaches_t;
970d7e83 LB	8
	9	/*
	10	* tcache pointers close to NULL are used to encode state information that is
	11	* used for two purposes: preventing thread caching on a per thread basis and
	12	* cleaning up during thread shutdown.
	13	*/
	14	#define TCACHE_STATE_DISABLED ((tcache_t *)(uintptr_t)1)
	15	#define TCACHE_STATE_REINCARNATED ((tcache_t *)(uintptr_t)2)
	16	#define TCACHE_STATE_PURGATORY ((tcache_t *)(uintptr_t)3)
	17	#define TCACHE_STATE_MAX TCACHE_STATE_PURGATORY
	18
54a0048b SL	19	/*
	20	* Absolute minimum number of cache slots for each small bin.
	21	*/
	22	#define TCACHE_NSLOTS_SMALL_MIN 20
	23
970d7e83 LB	24	/*
	25	* Absolute maximum number of cache slots for each small bin in the thread
	26	* cache. This is an additional constraint beyond that imposed as: twice the
	27	* number of regions per run for this size class.
	28	*
	29	* This constant must be an even number.
	30	*/
	31	#define TCACHE_NSLOTS_SMALL_MAX 200
	32
	33	/* Number of cache slots for large size classes. */
	34	#define TCACHE_NSLOTS_LARGE 20
	35
	36	/* (1U << opt_lg_tcache_max) is used to compute tcache_maxclass. */
	37	#define LG_TCACHE_MAXCLASS_DEFAULT 15
	38
	39	/*
	40	* TCACHE_GC_SWEEP is the approximate number of allocation events between
	41	* full GC sweeps. Integer rounding may cause the actual number to be
	42	* slightly higher, since GC is performed incrementally.
	43	*/
	44	#define TCACHE_GC_SWEEP 8192
	45
	46	/* Number of tcache allocation/deallocation events between incremental GCs. */
	47	#define TCACHE_GC_INCR \
	48	((TCACHE_GC_SWEEP / NBINS) + ((TCACHE_GC_SWEEP / NBINS == 0) ? 0 : 1))
	49
	50	#endif /* JEMALLOC_H_TYPES */
	51	/******************************************************************************/
	52	#ifdef JEMALLOC_H_STRUCTS
	53
	54	typedef enum {
	55	tcache_enabled_false = 0, /* Enable cast to/from bool. */
	56	tcache_enabled_true = 1,
	57	tcache_enabled_default = 2
	58	} tcache_enabled_t;
	59
	60	/*
	61	* Read-only information associated with each element of tcache_t's tbins array
	62	* is stored separately, mainly to reduce memory usage.
	63	*/
	64	struct tcache_bin_info_s {
	65	unsigned ncached_max; /* Upper limit on ncached. */
	66	};
	67
	68	struct tcache_bin_s {
	69	tcache_bin_stats_t tstats;
	70	int low_water; /* Min # cached since last GC. */
	71	unsigned lg_fill_div; /* Fill (ncached_max >> lg_fill_div). */
	72	unsigned ncached; /* # of cached objects. */
54a0048b SL	73	/*
	74	* To make use of adjacent cacheline prefetch, the items in the avail
	75	* stack goes to higher address for newer allocations. avail points
	76	* just above the available space, which means that
	77	* avail[-ncached, ... -1] are available items and the lowest item will
	78	* be allocated first.
	79	*/
970d7e83 LB	80	void *avail; / Stack of available objects. */
	81	};
	82
	83	struct tcache_s {
	84	ql_elm(tcache_t) link; /* Used for aggregating stats. */
54a0048b SL	85	uint64_t prof_accumbytes;/* Cleared after arena_prof_accum(). */
	86	ticker_t gc_ticker; /* Drives incremental GC. */
	87	szind_t next_gc_bin; /* Next bin to GC. */
970d7e83 LB	88	tcache_bin_t tbins[1]; /* Dynamically sized. */
	89	/*
	90	* The pointer stacks associated with tbins follow as a contiguous
	91	* array. During tcache initialization, the avail pointer in each
	92	* element of tbins is initialized to point to the proper offset within
	93	* this array.
	94	*/
	95	};
	96
54a0048b SL	97	/* Linkage for list of available (previously used) explicit tcache IDs. */
	98	struct tcaches_s {
	99	union {
	100	tcache_t *tcache;
	101	tcaches_t *next;
	102	};
	103	};
	104
970d7e83 LB	105	#endif /* JEMALLOC_H_STRUCTS */
	106	/******************************************************************************/
	107	#ifdef JEMALLOC_H_EXTERNS
	108
	109	extern bool opt_tcache;
	110	extern ssize_t opt_lg_tcache_max;
	111
	112	extern tcache_bin_info_t *tcache_bin_info;
	113
	114	/*
	115	* Number of tcache bins. There are NBINS small-object bins, plus 0 or more
	116	* large-object bins.
	117	*/
54a0048b	118	extern unsigned nhbins;
970d7e83 LB	119
970d7e83 LB	120	/* Maximum cached size class. */
54a0048b SL	121	extern size_t tcache_maxclass;
	122
	123	/*
	124	* Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
	125	* usable via the MALLOCX_TCACHE() flag. The automatic per thread tcaches are
	126	* completely disjoint from this data structure. tcaches starts off as a sparse
	127	* array, so it has no physical memory footprint until individual pages are
	128	* touched. This allows the entire array to be allocated the first time an
	129	* explicit tcache is created without a disproportionate impact on memory usage.
	130	*/
	131	extern tcaches_t *tcaches;
970d7e83 LB	132
970d7e83 LB	133	size_t tcache_salloc(const void *ptr);
54a0048b SL	134	void tcache_event_hard(tsd_t tsd, tcache_t tcache);
	135	void tcache_alloc_small_hard(tsd_t tsd, arena_t arena, tcache_t tcache,
	136	tcache_bin_t tbin, szind_t binind, bool tcache_success);
	137	void tcache_bin_flush_small(tsd_t tsd, tcache_t tcache, tcache_bin_t *tbin,
	138	szind_t binind, unsigned rem);
	139	void tcache_bin_flush_large(tsd_t tsd, tcache_bin_t tbin, szind_t binind,
	140	unsigned rem, tcache_t *tcache);
970d7e83	141	void tcache_arena_associate(tcache_t tcache, arena_t arena);
54a0048b SL	142	void tcache_arena_reassociate(tcache_t tcache, arena_t oldarena,
	143	arena_t *newarena);
	144	void tcache_arena_dissociate(tcache_t tcache, arena_t arena);
1a4d82fc	145	tcache_t tcache_get_hard(tsd_t tsd);
54a0048b	146	tcache_t tcache_create(tsd_t tsd, arena_t *arena);
1a4d82fc JJ	147	void tcache_cleanup(tsd_t *tsd);
1a4d82fc JJ	148	void tcache_enabled_cleanup(tsd_t *tsd);
970d7e83	149	void tcache_stats_merge(tcache_t tcache, arena_t arena);
54a0048b SL	150	bool tcaches_create(tsd_t tsd, unsigned r_ind);
	151	void tcaches_flush(tsd_t *tsd, unsigned ind);
	152	void tcaches_destroy(tsd_t *tsd, unsigned ind);
1a4d82fc	153	bool tcache_boot(void);
970d7e83 LB	154
	155	#endif /* JEMALLOC_H_EXTERNS */
	156	/******************************************************************************/
	157	#ifdef JEMALLOC_H_INLINES
	158
	159	#ifndef JEMALLOC_ENABLE_INLINE
54a0048b	160	void tcache_event(tsd_t tsd, tcache_t tcache);
970d7e83 LB	161	void tcache_flush(void);
970d7e83 LB	162	bool tcache_enabled_get(void);
1a4d82fc	163	tcache_t tcache_get(tsd_t tsd, bool create);
970d7e83	164	void tcache_enabled_set(bool enabled);
54a0048b SL	165	void tcache_alloc_easy(tcache_bin_t tbin, bool *tcache_success);
	166	void tcache_alloc_small(tsd_t tsd, arena_t arena, tcache_t tcache,
	167	size_t size, szind_t ind, bool zero, bool slow_path);
	168	void tcache_alloc_large(tsd_t tsd, arena_t arena, tcache_t tcache,
	169	size_t size, szind_t ind, bool zero, bool slow_path);
	170	void tcache_dalloc_small(tsd_t tsd, tcache_t tcache, void *ptr,
	171	szind_t binind, bool slow_path);
	172	void tcache_dalloc_large(tsd_t tsd, tcache_t tcache, void *ptr,
	173	size_t size, bool slow_path);
	174	tcache_t tcaches_get(tsd_t tsd, unsigned ind);
970d7e83 LB	175	#endif
	176
	177	#if (defined(JEMALLOC_ENABLE_INLINE) \|\| defined(JEMALLOC_TCACHE_C_))
970d7e83 LB	178	JEMALLOC_INLINE void
	179	tcache_flush(void)
	180	{
1a4d82fc	181	tsd_t *tsd;
970d7e83 LB	182
	183	cassert(config_tcache);
	184
1a4d82fc JJ	185	tsd = tsd_fetch();
1a4d82fc JJ	186	tcache_cleanup(tsd);
970d7e83 LB	187	}
	188
	189	JEMALLOC_INLINE bool
	190	tcache_enabled_get(void)
	191	{
1a4d82fc	192	tsd_t *tsd;
970d7e83 LB	193	tcache_enabled_t tcache_enabled;
	194
	195	cassert(config_tcache);
	196
1a4d82fc JJ	197	tsd = tsd_fetch();
1a4d82fc JJ	198	tcache_enabled = tsd_tcache_enabled_get(tsd);
970d7e83 LB	199	if (tcache_enabled == tcache_enabled_default) {
970d7e83 LB	200	tcache_enabled = (tcache_enabled_t)opt_tcache;
1a4d82fc	201	tsd_tcache_enabled_set(tsd, tcache_enabled);
970d7e83 LB	202	}
	203
	204	return ((bool)tcache_enabled);
	205	}
	206
	207	JEMALLOC_INLINE void
	208	tcache_enabled_set(bool enabled)
	209	{
1a4d82fc	210	tsd_t *tsd;
970d7e83	211	tcache_enabled_t tcache_enabled;
970d7e83 LB	212
	213	cassert(config_tcache);
	214
1a4d82fc JJ	215	tsd = tsd_fetch();
1a4d82fc JJ	216
970d7e83	217	tcache_enabled = (tcache_enabled_t)enabled;
1a4d82fc JJ	218	tsd_tcache_enabled_set(tsd, tcache_enabled);
	219
	220	if (!enabled)
	221	tcache_cleanup(tsd);
970d7e83 LB	222	}
	223
	224	JEMALLOC_ALWAYS_INLINE tcache_t *
1a4d82fc	225	tcache_get(tsd_t *tsd, bool create)
970d7e83 LB	226	{
	227	tcache_t *tcache;
	228
1a4d82fc	229	if (!config_tcache)
970d7e83 LB	230	return (NULL);
970d7e83 LB	231
1a4d82fc JJ	232	tcache = tsd_tcache_get(tsd);
	233	if (!create)
	234	return (tcache);
	235	if (unlikely(tcache == NULL) && tsd_nominal(tsd)) {
	236	tcache = tcache_get_hard(tsd);
	237	tsd_tcache_set(tsd, tcache);
970d7e83 LB	238	}
	239
	240	return (tcache);
	241	}
	242
	243	JEMALLOC_ALWAYS_INLINE void
54a0048b	244	tcache_event(tsd_t tsd, tcache_t tcache)
970d7e83 LB	245	{
	246
	247	if (TCACHE_GC_INCR == 0)
	248	return;
	249
54a0048b SL	250	if (unlikely(ticker_tick(&tcache->gc_ticker)))
54a0048b SL	251	tcache_event_hard(tsd, tcache);
970d7e83 LB	252	}
	253
	254	JEMALLOC_ALWAYS_INLINE void *
54a0048b	255	tcache_alloc_easy(tcache_bin_t tbin, bool tcache_success)
970d7e83 LB	256	{
	257	void *ret;
	258
1a4d82fc	259	if (unlikely(tbin->ncached == 0)) {
970d7e83	260	tbin->low_water = -1;
54a0048b	261	*tcache_success = false;
970d7e83 LB	262	return (NULL);
970d7e83 LB	263	}
54a0048b SL	264	/*
	265	* tcache_success (instead of ret) should be checked upon the return of
	266	* this function. We avoid checking (ret == NULL) because there is
	267	* never a null stored on the avail stack (which is unknown to the
	268	* compiler), and eagerly checking ret would cause pipeline stall
	269	* (waiting for the cacheline).
	270	*/
	271	*tcache_success = true;
	272	ret = *(tbin->avail - tbin->ncached);
970d7e83	273	tbin->ncached--;
54a0048b	274
1a4d82fc	275	if (unlikely((int)tbin->ncached < tbin->low_water))
970d7e83	276	tbin->low_water = tbin->ncached;
54a0048b	277
970d7e83 LB	278	return (ret);
	279	}
	280
	281	JEMALLOC_ALWAYS_INLINE void *
54a0048b SL	282	tcache_alloc_small(tsd_t tsd, arena_t arena, tcache_t *tcache, size_t size,
54a0048b SL	283	szind_t binind, bool zero, bool slow_path)
970d7e83 LB	284	{
970d7e83 LB	285	void *ret;
970d7e83	286	tcache_bin_t *tbin;
54a0048b SL	287	bool tcache_success;
54a0048b SL	288	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
970d7e83	289
970d7e83 LB	290	assert(binind < NBINS);
970d7e83 LB	291	tbin = &tcache->tbins[binind];
54a0048b SL	292	ret = tcache_alloc_easy(tbin, &tcache_success);
	293	assert(tcache_success == (ret != NULL));
	294	if (unlikely(!tcache_success)) {
	295	bool tcache_hard_success;
	296	arena = arena_choose(tsd, arena);
	297	if (unlikely(arena == NULL))
970d7e83	298	return (NULL);
54a0048b SL	299
	300	ret = tcache_alloc_small_hard(tsd, arena, tcache, tbin, binind,
	301	&tcache_hard_success);
	302	if (tcache_hard_success == false)
	303	return (NULL);
	304	}
	305
	306	assert(ret);
	307	/*
	308	* Only compute usize if required. The checks in the following if
	309	* statement are all static.
	310	*/
	311	if (config_prof \|\| (slow_path && config_fill) \|\| unlikely(zero)) {
	312	usize = index2size(binind);
	313	assert(tcache_salloc(ret) == usize);
970d7e83	314	}
970d7e83	315
1a4d82fc	316	if (likely(!zero)) {
54a0048b SL	317	if (slow_path && config_fill) {
54a0048b SL	318	if (unlikely(opt_junk_alloc)) {
970d7e83 LB	319	arena_alloc_junk_small(ret,
970d7e83 LB	320	&arena_bin_info[binind], false);
1a4d82fc	321	} else if (unlikely(opt_zero))
54a0048b	322	memset(ret, 0, usize);
970d7e83 LB	323	}
970d7e83 LB	324	} else {
54a0048b	325	if (slow_path && config_fill && unlikely(opt_junk_alloc)) {
970d7e83 LB	326	arena_alloc_junk_small(ret, &arena_bin_info[binind],
	327	true);
	328	}
54a0048b	329	memset(ret, 0, usize);
970d7e83	330	}
970d7e83 LB	331
	332	if (config_stats)
	333	tbin->tstats.nrequests++;
	334	if (config_prof)
54a0048b SL	335	tcache->prof_accumbytes += usize;
54a0048b SL	336	tcache_event(tsd, tcache);
970d7e83 LB	337	return (ret);
	338	}
	339
	340	JEMALLOC_ALWAYS_INLINE void *
54a0048b SL	341	tcache_alloc_large(tsd_t tsd, arena_t arena, tcache_t *tcache, size_t size,
54a0048b SL	342	szind_t binind, bool zero, bool slow_path)
970d7e83 LB	343	{
970d7e83 LB	344	void *ret;
970d7e83	345	tcache_bin_t *tbin;
54a0048b	346	bool tcache_success;
970d7e83	347
970d7e83 LB	348	assert(binind < nhbins);
970d7e83 LB	349	tbin = &tcache->tbins[binind];
54a0048b SL	350	ret = tcache_alloc_easy(tbin, &tcache_success);
	351	assert(tcache_success == (ret != NULL));
	352	if (unlikely(!tcache_success)) {
970d7e83 LB	353	/*
	354	* Only allocate one large object at a time, because it's quite
	355	* expensive to create one and not use it.
	356	*/
54a0048b SL	357	arena = arena_choose(tsd, arena);
	358	if (unlikely(arena == NULL))
	359	return (NULL);
	360
	361	ret = arena_malloc_large(tsd, arena, binind, zero);
970d7e83 LB	362	if (ret == NULL)
	363	return (NULL);
	364	} else {
54a0048b SL	365	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
	366
	367	/* Only compute usize on demand */
	368	if (config_prof \|\| (slow_path && config_fill) \|\|
	369	unlikely(zero)) {
	370	usize = index2size(binind);
	371	assert(usize <= tcache_maxclass);
	372	}
	373
	374	if (config_prof && usize == LARGE_MINCLASS) {
970d7e83 LB	375	arena_chunk_t *chunk =
	376	(arena_chunk_t *)CHUNK_ADDR2BASE(ret);
	377	size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >>
	378	LG_PAGE);
	379	arena_mapbits_large_binind_set(chunk, pageind,
	380	BININD_INVALID);
	381	}
1a4d82fc	382	if (likely(!zero)) {
54a0048b SL	383	if (slow_path && config_fill) {
	384	if (unlikely(opt_junk_alloc))
	385	memset(ret, 0xa5, usize);
1a4d82fc	386	else if (unlikely(opt_zero))
54a0048b	387	memset(ret, 0, usize);
970d7e83	388	}
1a4d82fc	389	} else
54a0048b	390	memset(ret, 0, usize);
970d7e83 LB	391
	392	if (config_stats)
	393	tbin->tstats.nrequests++;
	394	if (config_prof)
54a0048b	395	tcache->prof_accumbytes += usize;
970d7e83 LB	396	}
970d7e83 LB	397
54a0048b	398	tcache_event(tsd, tcache);
970d7e83 LB	399	return (ret);
	400	}
	401
	402	JEMALLOC_ALWAYS_INLINE void
54a0048b SL	403	tcache_dalloc_small(tsd_t tsd, tcache_t tcache, void *ptr, szind_t binind,
54a0048b SL	404	bool slow_path)
970d7e83 LB	405	{
	406	tcache_bin_t *tbin;
	407	tcache_bin_info_t *tbin_info;
	408
	409	assert(tcache_salloc(ptr) <= SMALL_MAXCLASS);
	410
54a0048b	411	if (slow_path && config_fill && unlikely(opt_junk_free))
970d7e83 LB	412	arena_dalloc_junk_small(ptr, &arena_bin_info[binind]);
	413
	414	tbin = &tcache->tbins[binind];
	415	tbin_info = &tcache_bin_info[binind];
1a4d82fc	416	if (unlikely(tbin->ncached == tbin_info->ncached_max)) {
54a0048b SL	417	tcache_bin_flush_small(tsd, tcache, tbin, binind,
54a0048b SL	418	(tbin_info->ncached_max >> 1));
970d7e83 LB	419	}
970d7e83 LB	420	assert(tbin->ncached < tbin_info->ncached_max);
970d7e83	421	tbin->ncached++;
54a0048b	422	*(tbin->avail - tbin->ncached) = ptr;
970d7e83	423
54a0048b	424	tcache_event(tsd, tcache);
970d7e83 LB	425	}
	426
	427	JEMALLOC_ALWAYS_INLINE void
54a0048b SL	428	tcache_dalloc_large(tsd_t tsd, tcache_t tcache, void *ptr, size_t size,
54a0048b SL	429	bool slow_path)
970d7e83	430	{
54a0048b	431	szind_t binind;
970d7e83 LB	432	tcache_bin_t *tbin;
	433	tcache_bin_info_t *tbin_info;
	434
	435	assert((size & PAGE_MASK) == 0);
	436	assert(tcache_salloc(ptr) > SMALL_MAXCLASS);
	437	assert(tcache_salloc(ptr) <= tcache_maxclass);
	438
54a0048b	439	binind = size2index(size);
970d7e83	440
54a0048b SL	441	if (slow_path && config_fill && unlikely(opt_junk_free))
54a0048b SL	442	arena_dalloc_junk_large(ptr, size);
970d7e83 LB	443
	444	tbin = &tcache->tbins[binind];
	445	tbin_info = &tcache_bin_info[binind];
1a4d82fc	446	if (unlikely(tbin->ncached == tbin_info->ncached_max)) {
54a0048b SL	447	tcache_bin_flush_large(tsd, tbin, binind,
54a0048b SL	448	(tbin_info->ncached_max >> 1), tcache);
970d7e83 LB	449	}
970d7e83 LB	450	assert(tbin->ncached < tbin_info->ncached_max);
970d7e83	451	tbin->ncached++;
54a0048b	452	*(tbin->avail - tbin->ncached) = ptr;
970d7e83	453
54a0048b SL	454	tcache_event(tsd, tcache);
	455	}
	456
	457	JEMALLOC_ALWAYS_INLINE tcache_t *
	458	tcaches_get(tsd_t *tsd, unsigned ind)
	459	{
	460	tcaches_t *elm = &tcaches[ind];
	461	if (unlikely(elm->tcache == NULL))
	462	elm->tcache = tcache_create(tsd, arena_choose(tsd, NULL));
	463	return (elm->tcache);
970d7e83 LB	464	}
	465	#endif
	466
	467	#endif /* JEMALLOC_H_INLINES */
	468	/******************************************************************************/