[mirror_spl-debian.git] / modules / spl / spl-mutex.c

/*
 *  This file is part of the SPL: Solaris Porting Layer.
 *
 *  Copyright (c) 2008 Lawrence Livermore National Security, LLC.
 *  Produced at Lawrence Livermore National Laboratory
 *  Written by:
 *          Brian Behlendorf <behlendorf1@llnl.gov>,
 *          Herb Wartens <wartens2@llnl.gov>,
 *          Jim Garlick <garlick@llnl.gov>
 *  UCRL-CODE-235197
 *
 *  This is free software; you can redistribute it and/or modify it
 *  under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This is distributed in the hope that it will be useful, but WITHOUT
 *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 *  for more details.
 *
 *  You should have received a copy of the GNU General Public License along
 *  with this program; if not, write to the Free Software Foundation, Inc.,
 *  51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
 */

#include <sys/mutex.h>

#ifdef DEBUG_SUBSYSTEM
#undef DEBUG_SUBSYSTEM
#endif

#define DEBUG_SUBSYSTEM S_MUTEX

/* Mutex implementation based on those found in Solaris.  This means
 * they the MUTEX_DEFAULT type is an adaptive mutex.  When calling
 * mutex_enter() your process will spin waiting for the lock if it's
 * likely the lock will be free'd shortly.  If it looks like the
 * lock will be held for a longer time we schedule and sleep waiting
 * for it.  This determination is made by checking if the holder of
 * the lock is currently running on cpu or sleeping waiting to be
 * scheduled.  If the holder is currently running it's likely the
 * lock will be shortly dropped.
 *
 * XXX: This is basically a rough implementation to see if this
 * helps our performance.  If it does a more careful implementation
 * should be done, perhaps in assembly.
 */

/*  0:         Never spin when trying to aquire lock
 * -1:         Spin until aquired or holder yeilds without dropping lock
 *  1-MAX_INT: Spin for N attempts before sleeping for lock
 */
int mutex_spin_max = 0;

#ifdef DEBUG_MUTEX
int mutex_stats[MUTEX_STATS_SIZE] = { 0 };
spinlock_t mutex_stats_lock;
struct list_head mutex_stats_list;
#endif

void
__spl_mutex_init(kmutex_t *mp, char *name, int type, void *ibc)
{
	int flags = KM_SLEEP;

	ASSERT(mp);
	ASSERT(name);
	ASSERT(ibc == NULL);
	ASSERT(mp->km_magic != KM_MAGIC); /* Never double init */

	mp->km_magic = KM_MAGIC;
	mp->km_owner = NULL;
	mp->km_name = NULL;
	mp->km_name_size = strlen(name) + 1;

	switch (type) {
		case MUTEX_DEFAULT:
			mp->km_type = MUTEX_ADAPTIVE;
			break;
		case MUTEX_SPIN:
		case MUTEX_ADAPTIVE:
			mp->km_type = type;
			break;
		default:
			SBUG();
	}

	/* We may be called when there is a non-zero preempt_count or
	 * interrupts are disabled is which case we must not sleep.
	 */
        if (current_thread_info()->preempt_count || irqs_disabled())
		flags = KM_NOSLEEP;

	/* Semaphore kmem_alloc'ed to keep struct size down (<64b) */
	mp->km_sem = kmem_alloc(sizeof(struct semaphore), flags);
	if (mp->km_sem == NULL)
		return;

	mp->km_name = kmem_alloc(mp->km_name_size, flags);
	if (mp->km_name == NULL) {
		kmem_free(mp->km_sem, sizeof(struct semaphore));
		return;
	}

	sema_init(mp->km_sem, 1);
	strncpy(mp->km_name, name, mp->km_name_size);

#ifdef DEBUG_MUTEX
	mp->km_stats = kmem_zalloc(sizeof(int) * MUTEX_STATS_SIZE, flags);
        if (mp->km_stats == NULL) {
		kmem_free(mp->km_name, mp->km_name_size);
		kmem_free(mp->km_sem, sizeof(struct semaphore));
		return;
	}

	/* XXX - This appears to be a much more contended lock than I
	 * would have expected.  To run with this debugging enabled and
	 * get reasonable performance we may need to be more clever and
	 * do something like hash the mutex ptr on to one of several
	 * lists to ease this single point of contention.
	 */
	spin_lock(&mutex_stats_lock);
	list_add_tail(&mp->km_list, &mutex_stats_list);
	spin_unlock(&mutex_stats_lock);
#endif
}
EXPORT_SYMBOL(__spl_mutex_init);

void
__spl_mutex_destroy(kmutex_t *mp)
{
	ASSERT(mp);
	ASSERT(mp->km_magic == KM_MAGIC);

#ifdef DEBUG_MUTEX
	spin_lock(&mutex_stats_lock);
	list_del_init(&mp->km_list);
	spin_unlock(&mutex_stats_lock);

	kmem_free(mp->km_stats, sizeof(int) * MUTEX_STATS_SIZE);
#endif
	kmem_free(mp->km_name, mp->km_name_size);
	kmem_free(mp->km_sem, sizeof(struct semaphore));

	memset(mp, KM_POISON, sizeof(*mp));
}
EXPORT_SYMBOL(__spl_mutex_destroy);

/* Return 1 if we acquired the mutex, else zero.  */
int
__mutex_tryenter(kmutex_t *mp)
{
	int rc;
	ENTRY;

	ASSERT(mp);
	ASSERT(mp->km_magic == KM_MAGIC);
	MUTEX_STAT_INC(mutex_stats, MUTEX_TRYENTER_TOTAL);
	MUTEX_STAT_INC(mp->km_stats, MUTEX_TRYENTER_TOTAL);

	rc = down_trylock(mp->km_sem);
	if (rc == 0) {
		ASSERT(mp->km_owner == NULL);
		mp->km_owner = current;
		MUTEX_STAT_INC(mutex_stats, MUTEX_TRYENTER_NOT_HELD);
		MUTEX_STAT_INC(mp->km_stats, MUTEX_TRYENTER_NOT_HELD);
	}

	RETURN(!rc);
}
EXPORT_SYMBOL(__mutex_tryenter);

static void
mutex_enter_adaptive(kmutex_t *mp)
{
	struct task_struct *owner;
	int count = 0;

	/* Lock is not held so we expect to aquire the lock */
	if ((owner = mp->km_owner) == NULL) {
		down(mp->km_sem);
		MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_NOT_HELD);
		MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_NOT_HELD);
	} else {
		/* The lock is held by a currently running task which
		 * we expect will drop the lock before leaving the
		 * head of the runqueue.  So the ideal thing to do
		 * is spin until we aquire the lock and avoid a
		 * context switch.  However it is also possible the
		 * task holding the lock yields the processor with
		 * out dropping lock.  In which case, we know it's
		 * going to be a while so we stop spinning and go
		 * to sleep waiting for the lock to be available.
		 * This should strike the optimum balance between
		 * spinning and sleeping waiting for a lock.
		 */
		while (task_curr(owner) && (count <= mutex_spin_max)) {
			if (down_trylock(mp->km_sem) == 0) {
				MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_SPIN);
				MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_SPIN);
				GOTO(out, count);
			}
			count++;
		}

		/* The lock is held by a sleeping task so it's going to
		 * cost us minimally one context switch.  We might as
		 * well sleep and yield the processor to other tasks.
		 */
		down(mp->km_sem);
		MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_SLEEP);
		MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_SLEEP);
	}
out:
	MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_TOTAL);
	MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_TOTAL);
}

void
__mutex_enter(kmutex_t *mp)
{
	ENTRY;
	ASSERT(mp);
	ASSERT(mp->km_magic == KM_MAGIC);

	switch (mp->km_type) {
		case MUTEX_SPIN:
			while (down_trylock(mp->km_sem));
			MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_SPIN);
			MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_SPIN);
			break;
		case MUTEX_ADAPTIVE:
			mutex_enter_adaptive(mp);
			break;
	}

	ASSERT(mp->km_owner == NULL);
	mp->km_owner = current;

	EXIT;
}
EXPORT_SYMBOL(__mutex_enter);

void
__mutex_exit(kmutex_t *mp)
{
	ENTRY;
	ASSERT(mp);
	ASSERT(mp->km_magic == KM_MAGIC);
	ASSERT(mp->km_owner == current);
	mp->km_owner = NULL;
	up(mp->km_sem);
	EXIT;
}
EXPORT_SYMBOL(__mutex_exit);

/* Return 1 if mutex is held by current process, else zero.  */
int
__mutex_owned(kmutex_t *mp)
{
	ENTRY;
	ASSERT(mp);
	ASSERT(mp->km_magic == KM_MAGIC);
	RETURN(mp->km_owner == current);
}
EXPORT_SYMBOL(__mutex_owned);

/* Return owner if mutex is owned, else NULL.  */
kthread_t *
__spl_mutex_owner(kmutex_t *mp)
{
	ENTRY;
	ASSERT(mp);
	ASSERT(mp->km_magic == KM_MAGIC);
	RETURN(mp->km_owner);
}
EXPORT_SYMBOL(__spl_mutex_owner);

int
spl_mutex_init(void)
{
	ENTRY;
#ifdef DEBUG_MUTEX
	spin_lock_init(&mutex_stats_lock);
        INIT_LIST_HEAD(&mutex_stats_list);
#endif
	RETURN(0);
}

void
spl_mutex_fini(void)
{
        ENTRY;
#ifdef DEBUG_MUTEX
	ASSERT(list_empty(&mutex_stats_list));
#endif
        EXIT;
}

module_param(mutex_spin_max, int, 0644);
MODULE_PARM_DESC(mutex_spin_max, "Spin a maximum of N times to aquire lock");
Commit	Line	Data
715f6251	1	/*
	2	* This file is part of the SPL: Solaris Porting Layer.
	3	*
	4	* Copyright (c) 2008 Lawrence Livermore National Security, LLC.
	5	* Produced at Lawrence Livermore National Laboratory
	6	* Written by:
	7	* Brian Behlendorf <behlendorf1@llnl.gov>,
	8	* Herb Wartens <wartens2@llnl.gov>,
	9	* Jim Garlick <garlick@llnl.gov>
	10	* UCRL-CODE-235197
	11	*
	12	* This is free software; you can redistribute it and/or modify it
	13	* under the terms of the GNU General Public License as published by
	14	* the Free Software Foundation; either version 2 of the License, or
	15	* (at your option) any later version.
	16	*
	17	* This is distributed in the hope that it will be useful, but WITHOUT
	18	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	19	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	20	* for more details.
	21	*
	22	* You should have received a copy of the GNU General Public License along
	23	* with this program; if not, write to the Free Software Foundation, Inc.,
	24	* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	25	*/
	26
9ab1ac14	27	#include <sys/mutex.h>
	28
	29	#ifdef DEBUG_SUBSYSTEM
	30	#undef DEBUG_SUBSYSTEM
	31	#endif
	32
	33	#define DEBUG_SUBSYSTEM S_MUTEX
	34
	35	/* Mutex implementation based on those found in Solaris. This means
	36	* they the MUTEX_DEFAULT type is an adaptive mutex. When calling
	37	* mutex_enter() your process will spin waiting for the lock if it's
	38	* likely the lock will be free'd shortly. If it looks like the
	39	* lock will be held for a longer time we schedule and sleep waiting
	40	* for it. This determination is made by checking if the holder of
	41	* the lock is currently running on cpu or sleeping waiting to be
	42	* scheduled. If the holder is currently running it's likely the
	43	* lock will be shortly dropped.
	44	*
	45	* XXX: This is basically a rough implementation to see if this
	46	* helps our performance. If it does a more careful implementation
	47	* should be done, perhaps in assembly.
	48	*/
	49
	50	/* 0: Never spin when trying to aquire lock
	51	* -1: Spin until aquired or holder yeilds without dropping lock
	52	* 1-MAX_INT: Spin for N attempts before sleeping for lock
	53	*/
56f92453	54	int mutex_spin_max = 0;
9ab1ac14	55
	56	#ifdef DEBUG_MUTEX
	57	int mutex_stats[MUTEX_STATS_SIZE] = { 0 };
404992e3	58	spinlock_t mutex_stats_lock;
4f86a887	59	struct list_head mutex_stats_list;
9ab1ac14	60	#endif
	61
	62	void
	63	__spl_mutex_init(kmutex_t mp, char name, int type, void *ibc)
	64	{
d6a26c6a	65	int flags = KM_SLEEP;
d6a26c6a	66
9ab1ac14	67	ASSERT(mp);
	68	ASSERT(name);
	69	ASSERT(ibc == NULL);
	70	ASSERT(mp->km_magic != KM_MAGIC); /* Never double init */
	71
	72	mp->km_magic = KM_MAGIC;
	73	mp->km_owner = NULL;
	74	mp->km_name = NULL;
	75	mp->km_name_size = strlen(name) + 1;
	76
	77	switch (type) {
	78	case MUTEX_DEFAULT:
	79	mp->km_type = MUTEX_ADAPTIVE;
	80	break;
	81	case MUTEX_SPIN:
	82	case MUTEX_ADAPTIVE:
	83	mp->km_type = type;
	84	break;
	85	default:
	86	SBUG();
	87	}
	88
d6a26c6a	89	/* We may be called when there is a non-zero preempt_count or
	90	* interrupts are disabled is which case we must not sleep.
	91	*/
	92	if (current_thread_info()->preempt_count \|\| irqs_disabled())
	93	flags = KM_NOSLEEP;
	94
9ab1ac14	95	/* Semaphore kmem_alloc'ed to keep struct size down (<64b) */
d6a26c6a	96	mp->km_sem = kmem_alloc(sizeof(struct semaphore), flags);
9ab1ac14	97	if (mp->km_sem == NULL)
	98	return;
	99
d6a26c6a	100	mp->km_name = kmem_alloc(mp->km_name_size, flags);
9ab1ac14	101	if (mp->km_name == NULL) {
	102	kmem_free(mp->km_sem, sizeof(struct semaphore));
	103	return;
	104	}
	105
	106	sema_init(mp->km_sem, 1);
a97df54e	107	strncpy(mp->km_name, name, mp->km_name_size);
9ab1ac14	108
9ab1ac14	109	#ifdef DEBUG_MUTEX
d6a26c6a	110	mp->km_stats = kmem_zalloc(sizeof(int) * MUTEX_STATS_SIZE, flags);
9ab1ac14	111	if (mp->km_stats == NULL) {
	112	kmem_free(mp->km_name, mp->km_name_size);
	113	kmem_free(mp->km_sem, sizeof(struct semaphore));
	114	return;
	115	}
	116
c6dc93d6	117	/* XXX - This appears to be a much more contended lock than I
	118	* would have expected. To run with this debugging enabled and
	119	* get reasonable performance we may need to be more clever and
	120	* do something like hash the mutex ptr on to one of several
	121	* lists to ease this single point of contention.
	122	*/
404992e3	123	spin_lock(&mutex_stats_lock);
9ab1ac14	124	list_add_tail(&mp->km_list, &mutex_stats_list);
404992e3	125	spin_unlock(&mutex_stats_lock);
9ab1ac14	126	#endif
	127	}
	128	EXPORT_SYMBOL(__spl_mutex_init);
	129
	130	void
	131	__spl_mutex_destroy(kmutex_t *mp)
	132	{
	133	ASSERT(mp);
	134	ASSERT(mp->km_magic == KM_MAGIC);
	135
	136	#ifdef DEBUG_MUTEX
404992e3	137	spin_lock(&mutex_stats_lock);
9ab1ac14	138	list_del_init(&mp->km_list);
404992e3	139	spin_unlock(&mutex_stats_lock);
9ab1ac14	140
	141	kmem_free(mp->km_stats, sizeof(int) * MUTEX_STATS_SIZE);
	142	#endif
	143	kmem_free(mp->km_name, mp->km_name_size);
	144	kmem_free(mp->km_sem, sizeof(struct semaphore));
	145
	146	memset(mp, KM_POISON, sizeof(*mp));
	147	}
	148	EXPORT_SYMBOL(__spl_mutex_destroy);
	149
	150	/* Return 1 if we acquired the mutex, else zero. */
	151	int
	152	__mutex_tryenter(kmutex_t *mp)
	153	{
	154	int rc;
	155	ENTRY;
	156
	157	ASSERT(mp);
	158	ASSERT(mp->km_magic == KM_MAGIC);
	159	MUTEX_STAT_INC(mutex_stats, MUTEX_TRYENTER_TOTAL);
	160	MUTEX_STAT_INC(mp->km_stats, MUTEX_TRYENTER_TOTAL);
	161
	162	rc = down_trylock(mp->km_sem);
	163	if (rc == 0) {
	164	ASSERT(mp->km_owner == NULL);
	165	mp->km_owner = current;
	166	MUTEX_STAT_INC(mutex_stats, MUTEX_TRYENTER_NOT_HELD);
	167	MUTEX_STAT_INC(mp->km_stats, MUTEX_TRYENTER_NOT_HELD);
	168	}
	169
	170	RETURN(!rc);
	171	}
	172	EXPORT_SYMBOL(__mutex_tryenter);
	173
	174	static void
	175	mutex_enter_adaptive(kmutex_t *mp)
	176	{
	177	struct task_struct *owner;
	178	int count = 0;
	179
	180	/* Lock is not held so we expect to aquire the lock */
	181	if ((owner = mp->km_owner) == NULL) {
	182	down(mp->km_sem);
	183	MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_NOT_HELD);
	184	MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_NOT_HELD);
	185	} else {
	186	/* The lock is held by a currently running task which
	187	* we expect will drop the lock before leaving the
	188	* head of the runqueue. So the ideal thing to do
	189	* is spin until we aquire the lock and avoid a
	190	* context switch. However it is also possible the
	191	* task holding the lock yields the processor with
	192	* out dropping lock. In which case, we know it's
	193	* going to be a while so we stop spinning and go
	194	* to sleep waiting for the lock to be available.
	195	* This should strike the optimum balance between
	196	* spinning and sleeping waiting for a lock.
	197	*/
	198	while (task_curr(owner) && (count <= mutex_spin_max)) {
	199	if (down_trylock(mp->km_sem) == 0) {
	200	MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_SPIN);
	201	MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_SPIN);
	202	GOTO(out, count);
	203	}
204	count++;
205	}
206
207	/* The lock is held by a sleeping task so it's going to
208	* cost us minimally one context switch. We might as
209	* well sleep and yield the processor to other tasks.
210	*/
211	down(mp->km_sem);
212	MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_SLEEP);
213	MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_SLEEP);
214	}
215	out:
216	MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_TOTAL);
217	MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_TOTAL);
218	}
219
220	void
221	__mutex_enter(kmutex_t *mp)
222	{
223	ENTRY;
224	ASSERT(mp);
225	ASSERT(mp->km_magic == KM_MAGIC);
226
227	switch (mp->km_type) {
228	case MUTEX_SPIN:
229	while (down_trylock(mp->km_sem));
230	MUTEX_STAT_INC(mutex_stats, MUTEX_ENTER_SPIN);
231	MUTEX_STAT_INC(mp->km_stats, MUTEX_ENTER_SPIN);
232	break;
233	case MUTEX_ADAPTIVE:
234	mutex_enter_adaptive(mp);
235	break;
236	}
237
238	ASSERT(mp->km_owner == NULL);
239	mp->km_owner = current;
240
241	EXIT;
242	}
243	EXPORT_SYMBOL(__mutex_enter);
244
245	void
246	__mutex_exit(kmutex_t *mp)
247	{
248	ENTRY;
249	ASSERT(mp);
250	ASSERT(mp->km_magic == KM_MAGIC);
251	ASSERT(mp->km_owner == current);
252	mp->km_owner = NULL;
253	up(mp->km_sem);
254	EXIT;
255	}
256	EXPORT_SYMBOL(__mutex_exit);
257
258	/* Return 1 if mutex is held by current process, else zero. */
259	int
260	__mutex_owned(kmutex_t *mp)
261	{
262	ENTRY;
263	ASSERT(mp);
264	ASSERT(mp->km_magic == KM_MAGIC);
265	RETURN(mp->km_owner == current);
266	}
267	EXPORT_SYMBOL(__mutex_owned);
268
269	/* Return owner if mutex is owned, else NULL. */
270	kthread_t *
271	__spl_mutex_owner(kmutex_t *mp)
272	{
273	ENTRY;
274	ASSERT(mp);
275	ASSERT(mp->km_magic == KM_MAGIC);
276	RETURN(mp->km_owner);
277	}
278	EXPORT_SYMBOL(__spl_mutex_owner);
279
280	int
281	spl_mutex_init(void)
282	{
283	ENTRY;
4f86a887	284	#ifdef DEBUG_MUTEX
404992e3	285	spin_lock_init(&mutex_stats_lock);
4f86a887	286	INIT_LIST_HEAD(&mutex_stats_list);
4f86a887	287	#endif
9ab1ac14	288	RETURN(0);
	289	}
	290
	291	void
	292	spl_mutex_fini(void)
	293	{
	294	ENTRY;
	295	#ifdef DEBUG_MUTEX
	296	ASSERT(list_empty(&mutex_stats_list));
	297	#endif
	298	EXIT;
	299	}
	300
56f92453	301	module_param(mutex_spin_max, int, 0644);
56f92453	302	MODULE_PARM_DESC(mutex_spin_max, "Spin a maximum of N times to aquire lock");