[mirror_spl.git] / modules / spl / spl-mutex.c

/*
 *  This file is part of the SPL: Solaris Porting Layer.
 *
 *  Copyright (c) 2008 Lawrence Livermore National Security, LLC.
 *  Produced at Lawrence Livermore National Laboratory
 *  Written by:
 *          Brian Behlendorf <behlendorf1@llnl.gov>,
 *          Herb Wartens <wartens2@llnl.gov>,
 *          Jim Garlick <garlick@llnl.gov>
 *  UCRL-CODE-235197
 *
 *  This is free software; you can redistribute it and/or modify it
 *  under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This is distributed in the hope that it will be useful, but WITHOUT
 *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 *  for more details.
 *
 *  You should have received a copy of the GNU General Public License along
 *  with this program; if not, write to the Free Software Foundation, Inc.,
 *  51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
 */

#include <sys/mutex.h>

#ifdef DEBUG_SUBSYSTEM
#undef DEBUG_SUBSYSTEM
#endif

#define DEBUG_SUBSYSTEM S_MUTEX

/* Mutex implementation based on those found in Solaris.  This means
 * they the MUTEX_DEFAULT type is an adaptive mutex.  When calling
 * mutex_enter() your process will spin waiting for the lock if it's
 * likely the lock will be free'd shortly.  If it looks like the
 * lock will be held for a longer time we schedule and sleep waiting
 * for it.  This determination is made by checking if the holder of
 * the lock is currently running on cpu or sleeping waiting to be
 * scheduled.  If the holder is currently running it's likely the
 * lock will be shortly dropped.
 *
 * XXX: This is basically a rough implementation to see if this
 * helps our performance.  If it does a more careful implementation
 * should be done, perhaps in assembly.
 */

/*  0:         Never spin when trying to aquire lock
 * -1:         Spin until aquired or holder yeilds without dropping lock
 *  1-MAX_INT: Spin for N attempts before sleeping for lock
 */
int mutex_spin_max = 0;

#ifdef DEBUG_MUTEX
int mutex_stats[MUTEX_STATS_SIZE] = { 0 };
spinlock_t mutex_stats_lock;
struct list_head mutex_stats_list;
#endif

int
__spl_mutex_init(kmutex_t *mp, char *name, int type, void *ibc)
{
	int flags = KM_SLEEP;

	ASSERT(mp);
	ASSERT(name);
	ASSERT(ibc == NULL);
	ASSERT(mp->km_magic != KM_MAGIC); /* Never double init */

	mp->km_name = NULL;
	mp->km_name_size = strlen(name) + 1;

	switch (type) {
		case MUTEX_DEFAULT:
			mp->km_type = MUTEX_ADAPTIVE;
			break;
		case MUTEX_SPIN:
		case MUTEX_ADAPTIVE:
			mp->km_type = type;
			break;
		default:
			SBUG();
	}

	/* We may be called when there is a non-zero preempt_count or
	 * interrupts are disabled is which case we must not sleep.
	 */
        if (current_thread_info()->preempt_count || irqs_disabled())
		flags = KM_NOSLEEP;

	/* Semaphore kmem_alloc'ed to keep struct size down (<64b) */
	mp->km_sem = kmem_alloc(sizeof(struct semaphore), flags);
	if (mp->km_sem == NULL)
		return -ENOMEM;

	mp->km_name = kmem_alloc(mp->km_name_size, flags);
	if (mp->km_name == NULL) {
		kmem_free(mp->km_sem, sizeof(struct semaphore));
		return -ENOMEM;
	}

	sema_init(mp->km_sem, 1);
	strncpy(mp->km_name, name, mp->km_name_size);

#ifdef DEBUG_MUTEX
	mp->km_stats = kmem_zalloc(sizeof(int) * MUTEX_STATS_SIZE, flags);
        if (mp->km_stats == NULL) {
		kmem_free(mp->km_name, mp->km_name_size);
		kmem_free(mp->km_sem, sizeof(struct semaphore));
		return -ENOMEM;
	}

	/* XXX - This appears to be a much more contended lock than I
	 * would have expected.  To run with this debugging enabled and
	 * get reasonable performance we may need to be more clever and
	 * do something like hash the mutex ptr on to one of several
	 * lists to ease this single point of contention.
	 */
	spin_lock(&mutex_stats_lock);
	list_add_tail(&mp->km_list, &mutex_stats_list);
	spin_unlock(&mutex_stats_lock);
#endif
	mp->km_magic = KM_MAGIC;
	mp->km_owner = NULL;

	return 0;
}
EXPORT_SYMBOL(__spl_mutex_init);

void
__spl_mutex_destroy(kmutex_t *mp)
{
	ASSERT(mp);
	ASSERT(mp->km_magic == KM_MAGIC);

#ifdef DEBUG_MUTEX
	spin_lock(&mutex_stats_lock);
	list_del_init(&mp->km_list);
	spin_unlock(&mutex_stats_lock);

	kmem_free(mp->km_stats, sizeof(int) * MUTEX_STATS_SIZE);
#endif
	kmem_free(mp->km_name, mp->km_name_size);
	kmem_free(mp->km_sem, sizeof(struct semaphore));

	memset(mp, KM_POISON, sizeof(*mp));
}
EXPORT_SYMBOL(__spl_mutex_destroy);

/* Return 1 if we acquired the mutex, else zero.  */
int
__mutex_tryenter(kmutex_t *mp)
{
	int rc;
	ENTRY;

	ASSERT(mp);
	ASSERT(mp->km_magic == KM_MAGIC);
	MUTEX_STAT_INC(mutex_stats, MUTEX_TRYENTER_TOTAL);
	MUTEX_STAT_INC(mp->km_stats, MUTEX_TRYENTER_TOTAL);

	rc = down_trylock(mp->km_sem);
	if (rc == 0) {
		ASSERT(mp->km_owner == NULL);
		mp->km_owner = current;
		MUTEX_STAT_INC(mutex_stats, MUTEX_TRYENTER_NOT_HELD);
		MUTEX_STAT_INC(mp->km_stats, MUTEX_TRYENTER_NOT_HELD);
	}

	RETURN(!rc);
}
EXPORT_SYMBOL(__mutex_tryenter);

#ifndef HAVE_TASK_CURR
#define task_curr(owner)                0
#endif


static void
mutex_enter_adaptive(kmutex_t *mp)
{
	struct task_struct *owner;
	int count = 0;

	/* Lock is not held so we expect to aquire the lock */
	if ((owner = mp->km_owner) == NULL) {
		down(mp->km_sem);
		MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_NOT_HELD);
		MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_NOT_HELD);
	} else {
		/* The lock is held by a currently running task which
		 * we expect will drop the lock before leaving the
		 * head of the runqueue.  So the ideal thing to do
		 * is spin until we aquire the lock and avoid a
		 * context switch.  However it is also possible the
		 * task holding the lock yields the processor with
		 * out dropping lock.  In which case, we know it's
		 * going to be a while so we stop spinning and go
		 * to sleep waiting for the lock to be available.
		 * This should strike the optimum balance between
		 * spinning and sleeping waiting for a lock.
		 */
		while (task_curr(owner) && (count <= mutex_spin_max)) {
			if (down_trylock(mp->km_sem) == 0) {
				MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_SPIN);
				MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_SPIN);
				GOTO(out, count);
			}
			count++;
		}

		/* The lock is held by a sleeping task so it's going to
		 * cost us minimally one context switch.  We might as
		 * well sleep and yield the processor to other tasks.
		 */
		down(mp->km_sem);
		MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_SLEEP);
		MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_SLEEP);
	}
out:
	MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_TOTAL);
	MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_TOTAL);
}

void
__mutex_enter(kmutex_t *mp)
{
	ENTRY;
	ASSERT(mp);
	ASSERT(mp->km_magic == KM_MAGIC);

	switch (mp->km_type) {
		case MUTEX_SPIN:
			while (down_trylock(mp->km_sem));
			MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_SPIN);
			MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_SPIN);
			break;
		case MUTEX_ADAPTIVE:
			mutex_enter_adaptive(mp);
			break;
	}

	ASSERT(mp->km_owner == NULL);
	mp->km_owner = current;

	EXIT;
}
EXPORT_SYMBOL(__mutex_enter);

void
__mutex_exit(kmutex_t *mp)
{
	ENTRY;
	ASSERT(mp);
	ASSERT(mp->km_magic == KM_MAGIC);
	ASSERT(mp->km_owner == current);
	mp->km_owner = NULL;
	up(mp->km_sem);
	EXIT;
}
EXPORT_SYMBOL(__mutex_exit);

/* Return 1 if mutex is held by current process, else zero.  */
int
__mutex_owned(kmutex_t *mp)
{
	ENTRY;
	ASSERT(mp);
	ASSERT(mp->km_magic == KM_MAGIC);
	RETURN(mp->km_owner == current);
}
EXPORT_SYMBOL(__mutex_owned);

/* Return owner if mutex is owned, else NULL.  */
kthread_t *
__spl_mutex_owner(kmutex_t *mp)
{
	ENTRY;
	ASSERT(mp);
	ASSERT(mp->km_magic == KM_MAGIC);
	RETURN(mp->km_owner);
}
EXPORT_SYMBOL(__spl_mutex_owner);

int
spl_mutex_init(void)
{
	ENTRY;
#ifdef DEBUG_MUTEX
	spin_lock_init(&mutex_stats_lock);
        INIT_LIST_HEAD(&mutex_stats_list);
#endif
	RETURN(0);
}

void
spl_mutex_fini(void)
{
        ENTRY;
#ifdef DEBUG_MUTEX
	ASSERT(list_empty(&mutex_stats_list));
#endif
        EXIT;
}

module_param(mutex_spin_max, int, 0644);
MODULE_PARM_DESC(mutex_spin_max, "Spin a maximum of N times to aquire lock");
Commit	Line	Data
715f6251	1	/*
	2	* This file is part of the SPL: Solaris Porting Layer.
	3	*
	4	* Copyright (c) 2008 Lawrence Livermore National Security, LLC.
	5	* Produced at Lawrence Livermore National Laboratory
	6	* Written by:
	7	* Brian Behlendorf <behlendorf1@llnl.gov>,
	8	* Herb Wartens <wartens2@llnl.gov>,
	9	* Jim Garlick <garlick@llnl.gov>
	10	* UCRL-CODE-235197
	11	*
	12	* This is free software; you can redistribute it and/or modify it
	13	* under the terms of the GNU General Public License as published by
	14	* the Free Software Foundation; either version 2 of the License, or
	15	* (at your option) any later version.
	16	*
	17	* This is distributed in the hope that it will be useful, but WITHOUT
	18	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	19	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	20	* for more details.
	21	*
	22	* You should have received a copy of the GNU General Public License along
	23	* with this program; if not, write to the Free Software Foundation, Inc.,
	24	* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	25	*/
	26
9ab1ac14	27	#include <sys/mutex.h>
	28
	29	#ifdef DEBUG_SUBSYSTEM
	30	#undef DEBUG_SUBSYSTEM
	31	#endif
	32
	33	#define DEBUG_SUBSYSTEM S_MUTEX
	34
	35	/* Mutex implementation based on those found in Solaris. This means
	36	* they the MUTEX_DEFAULT type is an adaptive mutex. When calling
	37	* mutex_enter() your process will spin waiting for the lock if it's
	38	* likely the lock will be free'd shortly. If it looks like the
	39	* lock will be held for a longer time we schedule and sleep waiting
	40	* for it. This determination is made by checking if the holder of
	41	* the lock is currently running on cpu or sleeping waiting to be
	42	* scheduled. If the holder is currently running it's likely the
	43	* lock will be shortly dropped.
	44	*
	45	* XXX: This is basically a rough implementation to see if this
	46	* helps our performance. If it does a more careful implementation
	47	* should be done, perhaps in assembly.
	48	*/
	49
	50	/* 0: Never spin when trying to aquire lock
	51	* -1: Spin until aquired or holder yeilds without dropping lock
	52	* 1-MAX_INT: Spin for N attempts before sleeping for lock
	53	*/
56f92453	54	int mutex_spin_max = 0;
9ab1ac14	55
	56	#ifdef DEBUG_MUTEX
	57	int mutex_stats[MUTEX_STATS_SIZE] = { 0 };
404992e3	58	spinlock_t mutex_stats_lock;
4f86a887	59	struct list_head mutex_stats_list;
9ab1ac14	60	#endif
9ab1ac14	61
c30df9c8	62	int
9ab1ac14	63	__spl_mutex_init(kmutex_t mp, char name, int type, void *ibc)
9ab1ac14	64	{
d6a26c6a	65	int flags = KM_SLEEP;
d6a26c6a	66
9ab1ac14	67	ASSERT(mp);
	68	ASSERT(name);
	69	ASSERT(ibc == NULL);
	70	ASSERT(mp->km_magic != KM_MAGIC); /* Never double init */
	71
9ab1ac14	72	mp->km_name = NULL;
	73	mp->km_name_size = strlen(name) + 1;
	74
	75	switch (type) {
	76	case MUTEX_DEFAULT:
	77	mp->km_type = MUTEX_ADAPTIVE;
	78	break;
	79	case MUTEX_SPIN:
	80	case MUTEX_ADAPTIVE:
	81	mp->km_type = type;
	82	break;
	83	default:
	84	SBUG();
	85	}
	86
d6a26c6a	87	/* We may be called when there is a non-zero preempt_count or
	88	* interrupts are disabled is which case we must not sleep.
	89	*/
	90	if (current_thread_info()->preempt_count \|\| irqs_disabled())
	91	flags = KM_NOSLEEP;
	92
9ab1ac14	93	/* Semaphore kmem_alloc'ed to keep struct size down (<64b) */
d6a26c6a	94	mp->km_sem = kmem_alloc(sizeof(struct semaphore), flags);
9ab1ac14	95	if (mp->km_sem == NULL)
c30df9c8	96	return -ENOMEM;
9ab1ac14	97
d6a26c6a	98	mp->km_name = kmem_alloc(mp->km_name_size, flags);
9ab1ac14	99	if (mp->km_name == NULL) {
9ab1ac14	100	kmem_free(mp->km_sem, sizeof(struct semaphore));
c30df9c8	101	return -ENOMEM;
9ab1ac14	102	}
	103
	104	sema_init(mp->km_sem, 1);
a97df54e	105	strncpy(mp->km_name, name, mp->km_name_size);
9ab1ac14	106
9ab1ac14	107	#ifdef DEBUG_MUTEX
d6a26c6a	108	mp->km_stats = kmem_zalloc(sizeof(int) * MUTEX_STATS_SIZE, flags);
9ab1ac14	109	if (mp->km_stats == NULL) {
	110	kmem_free(mp->km_name, mp->km_name_size);
	111	kmem_free(mp->km_sem, sizeof(struct semaphore));
c30df9c8	112	return -ENOMEM;
9ab1ac14	113	}
9ab1ac14	114
c6dc93d6	115	/* XXX - This appears to be a much more contended lock than I
	116	* would have expected. To run with this debugging enabled and
	117	* get reasonable performance we may need to be more clever and
	118	* do something like hash the mutex ptr on to one of several
	119	* lists to ease this single point of contention.
	120	*/
404992e3	121	spin_lock(&mutex_stats_lock);
9ab1ac14	122	list_add_tail(&mp->km_list, &mutex_stats_list);
404992e3	123	spin_unlock(&mutex_stats_lock);
9ab1ac14	124	#endif
c30df9c8	125	mp->km_magic = KM_MAGIC;
	126	mp->km_owner = NULL;
	127
	128	return 0;
9ab1ac14	129	}
	130	EXPORT_SYMBOL(__spl_mutex_init);
	131
	132	void
	133	__spl_mutex_destroy(kmutex_t *mp)
	134	{
	135	ASSERT(mp);
	136	ASSERT(mp->km_magic == KM_MAGIC);
	137
	138	#ifdef DEBUG_MUTEX
404992e3	139	spin_lock(&mutex_stats_lock);
9ab1ac14	140	list_del_init(&mp->km_list);
404992e3	141	spin_unlock(&mutex_stats_lock);
9ab1ac14	142
	143	kmem_free(mp->km_stats, sizeof(int) * MUTEX_STATS_SIZE);
	144	#endif
	145	kmem_free(mp->km_name, mp->km_name_size);
	146	kmem_free(mp->km_sem, sizeof(struct semaphore));
	147
	148	memset(mp, KM_POISON, sizeof(*mp));
	149	}
	150	EXPORT_SYMBOL(__spl_mutex_destroy);
	151
	152	/* Return 1 if we acquired the mutex, else zero. */
	153	int
	154	__mutex_tryenter(kmutex_t *mp)
	155	{
	156	int rc;
	157	ENTRY;
	158
	159	ASSERT(mp);
	160	ASSERT(mp->km_magic == KM_MAGIC);
	161	MUTEX_STAT_INC(mutex_stats, MUTEX_TRYENTER_TOTAL);
	162	MUTEX_STAT_INC(mp->km_stats, MUTEX_TRYENTER_TOTAL);
	163
	164	rc = down_trylock(mp->km_sem);
	165	if (rc == 0) {
	166	ASSERT(mp->km_owner == NULL);
	167	mp->km_owner = current;
	168	MUTEX_STAT_INC(mutex_stats, MUTEX_TRYENTER_NOT_HELD);
	169	MUTEX_STAT_INC(mp->km_stats, MUTEX_TRYENTER_NOT_HELD);
	170	}
	171
	172	RETURN(!rc);
	173	}
	174	EXPORT_SYMBOL(__mutex_tryenter);
	175
57d86234	176	#ifndef HAVE_TASK_CURR
	177	#define task_curr(owner) 0
	178	#endif
	179
	180
9ab1ac14	181	static void
	182	mutex_enter_adaptive(kmutex_t *mp)
	183	{
	184	struct task_struct *owner;
	185	int count = 0;
	186
	187	/* Lock is not held so we expect to aquire the lock */
	188	if ((owner = mp->km_owner) == NULL) {
	189	down(mp->km_sem);
	190	MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_NOT_HELD);
	191	MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_NOT_HELD);
	192	} else {
	193	/* The lock is held by a currently running task which
	194	* we expect will drop the lock before leaving the
	195	* head of the runqueue. So the ideal thing to do
	196	* is spin until we aquire the lock and avoid a
	197	* context switch. However it is also possible the
	198	* task holding the lock yields the processor with
	199	* out dropping lock. In which case, we know it's
	200	* going to be a while so we stop spinning and go
	201	* to sleep waiting for the lock to be available.
	202	* This should strike the optimum balance between
	203	* spinning and sleeping waiting for a lock.
	204	*/
	205	while (task_curr(owner) && (count <= mutex_spin_max)) {
	206	if (down_trylock(mp->km_sem) == 0) {
	207	MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_SPIN);
	208	MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_SPIN);
	209	GOTO(out, count);
	210	}
	211	count++;
	212	}
	213
	214	/* The lock is held by a sleeping task so it's going to
	215	* cost us minimally one context switch. We might as
	216	* well sleep and yield the processor to other tasks.
	217	*/
	218	down(mp->km_sem);
	219	MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_SLEEP);
	220	MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_SLEEP);
	221	}
	222	out:
	223	MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_TOTAL);
	224	MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_TOTAL);
	225	}
	226
	227	void
	228	__mutex_enter(kmutex_t *mp)
	229	{
	230	ENTRY;
	231	ASSERT(mp);
	232	ASSERT(mp->km_magic == KM_MAGIC);
	233
	234	switch (mp->km_type) {
	235	case MUTEX_SPIN:
	236	while (down_trylock(mp->km_sem));
	237	MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_SPIN);
	238	MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_SPIN);
	239	break;
	240	case MUTEX_ADAPTIVE:
	241	mutex_enter_adaptive(mp);
	242	break;
	243	}
	244
245	ASSERT(mp->km_owner == NULL);
246	mp->km_owner = current;
247
248	EXIT;
249	}
250	EXPORT_SYMBOL(__mutex_enter);
251
252	void
253	__mutex_exit(kmutex_t *mp)
254	{
255	ENTRY;
256	ASSERT(mp);
257	ASSERT(mp->km_magic == KM_MAGIC);
258	ASSERT(mp->km_owner == current);
259	mp->km_owner = NULL;
260	up(mp->km_sem);
261	EXIT;
262	}
263	EXPORT_SYMBOL(__mutex_exit);
264
265	/* Return 1 if mutex is held by current process, else zero. */
266	int
267	__mutex_owned(kmutex_t *mp)
268	{
269	ENTRY;
270	ASSERT(mp);
271	ASSERT(mp->km_magic == KM_MAGIC);
272	RETURN(mp->km_owner == current);
273	}
274	EXPORT_SYMBOL(__mutex_owned);
275
276	/* Return owner if mutex is owned, else NULL. */
277	kthread_t *
278	__spl_mutex_owner(kmutex_t *mp)
279	{
280	ENTRY;
281	ASSERT(mp);
282	ASSERT(mp->km_magic == KM_MAGIC);
283	RETURN(mp->km_owner);
284	}
285	EXPORT_SYMBOL(__spl_mutex_owner);
286
287	int
288	spl_mutex_init(void)
289	{
290	ENTRY;
4f86a887	291	#ifdef DEBUG_MUTEX
404992e3	292	spin_lock_init(&mutex_stats_lock);
4f86a887	293	INIT_LIST_HEAD(&mutex_stats_list);
4f86a887	294	#endif
9ab1ac14	295	RETURN(0);
	296	}
	297
	298	void
	299	spl_mutex_fini(void)
	300	{
	301	ENTRY;
	302	#ifdef DEBUG_MUTEX
	303	ASSERT(list_empty(&mutex_stats_list));
	304	#endif
	305	EXIT;
	306	}
	307
56f92453	308	module_param(mutex_spin_max, int, 0644);
56f92453	309	MODULE_PARM_DESC(mutex_spin_max, "Spin a maximum of N times to aquire lock");