[rustc.git] / src / jemalloc / include / jemalloc / internal / mutex.h

/******************************************************************************/
#ifdef JEMALLOC_H_TYPES

typedef struct malloc_mutex_s malloc_mutex_t;

#ifdef _WIN32
#  define MALLOC_MUTEX_INITIALIZER
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
#  define MALLOC_MUTEX_INITIALIZER					\
     {OS_UNFAIR_LOCK_INIT, WITNESS_INITIALIZER(WITNESS_RANK_OMIT)}
#elif (defined(JEMALLOC_OSSPIN))
#  define MALLOC_MUTEX_INITIALIZER {0, WITNESS_INITIALIZER(WITNESS_RANK_OMIT)}
#elif (defined(JEMALLOC_MUTEX_INIT_CB))
#  define MALLOC_MUTEX_INITIALIZER					\
    {PTHREAD_MUTEX_INITIALIZER, NULL, WITNESS_INITIALIZER(WITNESS_RANK_OMIT)}
#else
#  if (defined(JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP) &&		\
       defined(PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP))
#    define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_ADAPTIVE_NP
#    define MALLOC_MUTEX_INITIALIZER					\
       {PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP,				\
        WITNESS_INITIALIZER(WITNESS_RANK_OMIT)}
#  else
#    define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT
#    define MALLOC_MUTEX_INITIALIZER					\
       {PTHREAD_MUTEX_INITIALIZER, WITNESS_INITIALIZER(WITNESS_RANK_OMIT)}
#  endif
#endif

#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS

struct malloc_mutex_s {
#ifdef _WIN32
#  if _WIN32_WINNT >= 0x0600
	SRWLOCK         	lock;
#  else
	CRITICAL_SECTION	lock;
#  endif
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
	os_unfair_lock		lock;
#elif (defined(JEMALLOC_OSSPIN))
	OSSpinLock		lock;
#elif (defined(JEMALLOC_MUTEX_INIT_CB))
	pthread_mutex_t		lock;
	malloc_mutex_t		*postponed_next;
#else
	pthread_mutex_t		lock;
#endif
	witness_t		witness;
};

#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS

#ifdef JEMALLOC_LAZY_LOCK
extern bool isthreaded;
#else
#  undef isthreaded /* Undo private_namespace.h definition. */
#  define isthreaded true
#endif

bool	malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
    witness_rank_t rank);
void	malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex);
void	malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex);
void	malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex);
bool	malloc_mutex_boot(void);

#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES

#ifndef JEMALLOC_ENABLE_INLINE
void	malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex);
void	malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex);
void	malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex);
void	malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex);
#endif

#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_))
JEMALLOC_INLINE void
malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex)
{

	witness_assert_not_owner(tsdn, &mutex->witness);
	if (isthreaded) {
#ifdef _WIN32
#  if _WIN32_WINNT >= 0x0600
		AcquireSRWLockExclusive(&mutex->lock);
#  else
		EnterCriticalSection(&mutex->lock);
#  endif
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
		os_unfair_lock_lock(&mutex->lock);
#elif (defined(JEMALLOC_OSSPIN))
		OSSpinLockLock(&mutex->lock);
#else
		pthread_mutex_lock(&mutex->lock);
#endif
	}
	witness_lock(tsdn, &mutex->witness);
}

JEMALLOC_INLINE void
malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex)
{

	witness_unlock(tsdn, &mutex->witness);
	if (isthreaded) {
#ifdef _WIN32
#  if _WIN32_WINNT >= 0x0600
		ReleaseSRWLockExclusive(&mutex->lock);
#  else
		LeaveCriticalSection(&mutex->lock);
#  endif
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
		os_unfair_lock_unlock(&mutex->lock);
#elif (defined(JEMALLOC_OSSPIN))
		OSSpinLockUnlock(&mutex->lock);
#else
		pthread_mutex_unlock(&mutex->lock);
#endif
	}
}

JEMALLOC_INLINE void
malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex)
{

	witness_assert_owner(tsdn, &mutex->witness);
}

JEMALLOC_INLINE void
malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex)
{

	witness_assert_not_owner(tsdn, &mutex->witness);
}
#endif

#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
Commit	Line	Data
970d7e83 LB	1	/******************************************************************************/
	2	#ifdef JEMALLOC_H_TYPES
	3
	4	typedef struct malloc_mutex_s malloc_mutex_t;
	5
	6	#ifdef _WIN32
	7	# define MALLOC_MUTEX_INITIALIZER
3b2f2976 XL	8	#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
	9	# define MALLOC_MUTEX_INITIALIZER \
	10	{OS_UNFAIR_LOCK_INIT, WITNESS_INITIALIZER(WITNESS_RANK_OMIT)}
970d7e83	11	#elif (defined(JEMALLOC_OSSPIN))
3b2f2976	12	# define MALLOC_MUTEX_INITIALIZER {0, WITNESS_INITIALIZER(WITNESS_RANK_OMIT)}
970d7e83	13	#elif (defined(JEMALLOC_MUTEX_INIT_CB))
3b2f2976 XL	14	# define MALLOC_MUTEX_INITIALIZER \
3b2f2976 XL	15	{PTHREAD_MUTEX_INITIALIZER, NULL, WITNESS_INITIALIZER(WITNESS_RANK_OMIT)}
970d7e83	16	#else
1a4d82fc	17	# if (defined(JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP) && \
970d7e83 LB	18	defined(PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP))
970d7e83 LB	19	# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_ADAPTIVE_NP
3b2f2976 XL	20	# define MALLOC_MUTEX_INITIALIZER \
	21	{PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP, \
	22	WITNESS_INITIALIZER(WITNESS_RANK_OMIT)}
970d7e83 LB	23	# else
970d7e83 LB	24	# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT
3b2f2976 XL	25	# define MALLOC_MUTEX_INITIALIZER \
3b2f2976 XL	26	{PTHREAD_MUTEX_INITIALIZER, WITNESS_INITIALIZER(WITNESS_RANK_OMIT)}
970d7e83 LB	27	# endif
	28	#endif
	29
	30	#endif /* JEMALLOC_H_TYPES */
	31	/******************************************************************************/
	32	#ifdef JEMALLOC_H_STRUCTS
	33
	34	struct malloc_mutex_s {
	35	#ifdef _WIN32
54a0048b SL	36	# if _WIN32_WINNT >= 0x0600
	37	SRWLOCK lock;
	38	# else
970d7e83	39	CRITICAL_SECTION lock;
54a0048b	40	# endif
3b2f2976 XL	41	#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
3b2f2976 XL	42	os_unfair_lock lock;
970d7e83 LB	43	#elif (defined(JEMALLOC_OSSPIN))
	44	OSSpinLock lock;
	45	#elif (defined(JEMALLOC_MUTEX_INIT_CB))
	46	pthread_mutex_t lock;
	47	malloc_mutex_t *postponed_next;
	48	#else
	49	pthread_mutex_t lock;
	50	#endif
3b2f2976	51	witness_t witness;
970d7e83 LB	52	};
	53
	54	#endif /* JEMALLOC_H_STRUCTS */
	55	/******************************************************************************/
	56	#ifdef JEMALLOC_H_EXTERNS
	57
	58	#ifdef JEMALLOC_LAZY_LOCK
	59	extern bool isthreaded;
	60	#else
	61	# undef isthreaded /* Undo private_namespace.h definition. */
	62	# define isthreaded true
	63	#endif
	64
3b2f2976 XL	65	bool malloc_mutex_init(malloc_mutex_t mutex, const char name,
	66	witness_rank_t rank);
	67	void malloc_mutex_prefork(tsdn_t tsdn, malloc_mutex_t mutex);
	68	void malloc_mutex_postfork_parent(tsdn_t tsdn, malloc_mutex_t mutex);
	69	void malloc_mutex_postfork_child(tsdn_t tsdn, malloc_mutex_t mutex);
	70	bool malloc_mutex_boot(void);
970d7e83 LB	71
	72	#endif /* JEMALLOC_H_EXTERNS */
	73	/******************************************************************************/
	74	#ifdef JEMALLOC_H_INLINES
	75
	76	#ifndef JEMALLOC_ENABLE_INLINE
3b2f2976 XL	77	void malloc_mutex_lock(tsdn_t tsdn, malloc_mutex_t mutex);
	78	void malloc_mutex_unlock(tsdn_t tsdn, malloc_mutex_t mutex);
	79	void malloc_mutex_assert_owner(tsdn_t tsdn, malloc_mutex_t mutex);
	80	void malloc_mutex_assert_not_owner(tsdn_t tsdn, malloc_mutex_t mutex);
970d7e83 LB	81	#endif
	82
	83	#if (defined(JEMALLOC_ENABLE_INLINE) \|\| defined(JEMALLOC_MUTEX_C_))
	84	JEMALLOC_INLINE void
3b2f2976	85	malloc_mutex_lock(tsdn_t tsdn, malloc_mutex_t mutex)
970d7e83 LB	86	{
970d7e83 LB	87
3b2f2976	88	witness_assert_not_owner(tsdn, &mutex->witness);
970d7e83 LB	89	if (isthreaded) {
970d7e83 LB	90	#ifdef _WIN32
54a0048b SL	91	# if _WIN32_WINNT >= 0x0600
	92	AcquireSRWLockExclusive(&mutex->lock);
	93	# else
970d7e83	94	EnterCriticalSection(&mutex->lock);
54a0048b	95	# endif
3b2f2976 XL	96	#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
3b2f2976 XL	97	os_unfair_lock_lock(&mutex->lock);
970d7e83 LB	98	#elif (defined(JEMALLOC_OSSPIN))
	99	OSSpinLockLock(&mutex->lock);
	100	#else
	101	pthread_mutex_lock(&mutex->lock);
	102	#endif
	103	}
3b2f2976	104	witness_lock(tsdn, &mutex->witness);
970d7e83 LB	105	}
	106
	107	JEMALLOC_INLINE void
3b2f2976	108	malloc_mutex_unlock(tsdn_t tsdn, malloc_mutex_t mutex)
970d7e83 LB	109	{
970d7e83 LB	110
3b2f2976	111	witness_unlock(tsdn, &mutex->witness);
970d7e83 LB	112	if (isthreaded) {
970d7e83 LB	113	#ifdef _WIN32
54a0048b SL	114	# if _WIN32_WINNT >= 0x0600
	115	ReleaseSRWLockExclusive(&mutex->lock);
	116	# else
970d7e83	117	LeaveCriticalSection(&mutex->lock);
54a0048b	118	# endif
3b2f2976 XL	119	#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
3b2f2976 XL	120	os_unfair_lock_unlock(&mutex->lock);
970d7e83 LB	121	#elif (defined(JEMALLOC_OSSPIN))
	122	OSSpinLockUnlock(&mutex->lock);
	123	#else
	124	pthread_mutex_unlock(&mutex->lock);
	125	#endif
	126	}
	127	}
3b2f2976 XL	128
	129	JEMALLOC_INLINE void
	130	malloc_mutex_assert_owner(tsdn_t tsdn, malloc_mutex_t mutex)
	131	{
	132
	133	witness_assert_owner(tsdn, &mutex->witness);
	134	}
	135
	136	JEMALLOC_INLINE void
	137	malloc_mutex_assert_not_owner(tsdn_t tsdn, malloc_mutex_t mutex)
	138	{
	139
	140	witness_assert_not_owner(tsdn, &mutex->witness);
	141	}
970d7e83 LB	142	#endif
	143
	144	#endif /* JEMALLOC_H_INLINES */
	145	/******************************************************************************/