[mirror_spl.git] / module / spl / spl-condvar.c

/*****************************************************************************\
 *  Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
 *  Copyright (C) 2007 The Regents of the University of California.
 *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
 *  Written by Brian Behlendorf <behlendorf1@llnl.gov>.
 *  UCRL-CODE-235197
 *
 *  This file is part of the SPL, Solaris Porting Layer.
 *  For details, see <http://zfsonlinux.org/>.
 *
 *  The SPL is free software; you can redistribute it and/or modify it
 *  under the terms of the GNU General Public License as published by the
 *  Free Software Foundation; either version 2 of the License, or (at your
 *  option) any later version.
 *
 *  The SPL is distributed in the hope that it will be useful, but WITHOUT
 *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 *  for more details.
 *
 *  You should have received a copy of the GNU General Public License along
 *  with the SPL.  If not, see <http://www.gnu.org/licenses/>.
 *****************************************************************************
 *  Solaris Porting Layer (SPL) Credential Implementation.
\*****************************************************************************/

#include <sys/condvar.h>
#include <sys/time.h>

void
__cv_init(kcondvar_t *cvp, char *name, kcv_type_t type, void *arg)
{
	ASSERT(cvp);
	ASSERT(name == NULL);
	ASSERT(type == CV_DEFAULT);
	ASSERT(arg == NULL);

	cvp->cv_magic = CV_MAGIC;
	init_waitqueue_head(&cvp->cv_event);
	init_waitqueue_head(&cvp->cv_destroy);
	atomic_set(&cvp->cv_waiters, 0);
	atomic_set(&cvp->cv_refs, 1);
	cvp->cv_mutex = NULL;
}
EXPORT_SYMBOL(__cv_init);

static int
cv_destroy_wakeup(kcondvar_t *cvp)
{
	if (!atomic_read(&cvp->cv_waiters) && !atomic_read(&cvp->cv_refs)) {
		ASSERT(cvp->cv_mutex == NULL);
		ASSERT(!waitqueue_active(&cvp->cv_event));
		return 1;
	}

	return 0;
}

void
__cv_destroy(kcondvar_t *cvp)
{
	ASSERT(cvp);
	ASSERT(cvp->cv_magic == CV_MAGIC);

	cvp->cv_magic = CV_DESTROY;
	atomic_dec(&cvp->cv_refs);

	/* Block until all waiters are woken and references dropped. */
	while (cv_destroy_wakeup(cvp) == 0)
		wait_event_timeout(cvp->cv_destroy, cv_destroy_wakeup(cvp), 1);

	ASSERT3P(cvp->cv_mutex, ==, NULL);
	ASSERT3S(atomic_read(&cvp->cv_refs), ==, 0);
	ASSERT3S(atomic_read(&cvp->cv_waiters), ==, 0);
	ASSERT3S(waitqueue_active(&cvp->cv_event), ==, 0);
}
EXPORT_SYMBOL(__cv_destroy);

static void
cv_wait_common(kcondvar_t *cvp, kmutex_t *mp, int state, int io)
{
	DEFINE_WAIT(wait);

	ASSERT(cvp);
        ASSERT(mp);
	ASSERT(cvp->cv_magic == CV_MAGIC);
	ASSERT(mutex_owned(mp));
	atomic_inc(&cvp->cv_refs);

	if (cvp->cv_mutex == NULL)
		cvp->cv_mutex = mp;

	/* Ensure the same mutex is used by all callers */
	ASSERT(cvp->cv_mutex == mp);

	prepare_to_wait_exclusive(&cvp->cv_event, &wait, state);
	atomic_inc(&cvp->cv_waiters);

	/* Mutex should be dropped after prepare_to_wait() this
	 * ensures we're linked in to the waiters list and avoids the
	 * race where 'cvp->cv_waiters > 0' but the list is empty. */
	mutex_exit(mp);
	if (io)
		io_schedule();
	else
		schedule();
	mutex_enter(mp);

	/* No more waiters a different mutex could be used */
	if (atomic_dec_and_test(&cvp->cv_waiters)) {
		cvp->cv_mutex = NULL;
		wake_up(&cvp->cv_destroy);
	}

	finish_wait(&cvp->cv_event, &wait);
	atomic_dec(&cvp->cv_refs);
}

void
__cv_wait(kcondvar_t *cvp, kmutex_t *mp)
{
	cv_wait_common(cvp, mp, TASK_UNINTERRUPTIBLE, 0);
}
EXPORT_SYMBOL(__cv_wait);

void
__cv_wait_interruptible(kcondvar_t *cvp, kmutex_t *mp)
{
	cv_wait_common(cvp, mp, TASK_INTERRUPTIBLE, 0);
}
EXPORT_SYMBOL(__cv_wait_interruptible);

void
__cv_wait_io(kcondvar_t *cvp, kmutex_t *mp)
{
	cv_wait_common(cvp, mp, TASK_UNINTERRUPTIBLE, 1);
}
EXPORT_SYMBOL(__cv_wait_io);

/* 'expire_time' argument is an absolute wall clock time in jiffies.
 * Return value is time left (expire_time - now) or -1 if timeout occurred.
 */
static clock_t
__cv_timedwait_common(kcondvar_t *cvp, kmutex_t *mp,
		      clock_t expire_time, int state)
{
	DEFINE_WAIT(wait);
	clock_t time_left;

	ASSERT(cvp);
        ASSERT(mp);
	ASSERT(cvp->cv_magic == CV_MAGIC);
	ASSERT(mutex_owned(mp));
	atomic_inc(&cvp->cv_refs);

	if (cvp->cv_mutex == NULL)
		cvp->cv_mutex = mp;

	/* Ensure the same mutex is used by all callers */
	ASSERT(cvp->cv_mutex == mp);

	/* XXX - Does not handle jiffie wrap properly */
	time_left = expire_time - jiffies;
	if (time_left <= 0) {
		atomic_dec(&cvp->cv_refs);
		return (-1);
	}

	prepare_to_wait_exclusive(&cvp->cv_event, &wait, state);
	atomic_inc(&cvp->cv_waiters);

	/* Mutex should be dropped after prepare_to_wait() this
	 * ensures we're linked in to the waiters list and avoids the
	 * race where 'cvp->cv_waiters > 0' but the list is empty. */
	mutex_exit(mp);
	time_left = schedule_timeout(time_left);
	mutex_enter(mp);

	/* No more waiters a different mutex could be used */
	if (atomic_dec_and_test(&cvp->cv_waiters)) {
		cvp->cv_mutex = NULL;
		wake_up(&cvp->cv_destroy);
	}

	finish_wait(&cvp->cv_event, &wait);
	atomic_dec(&cvp->cv_refs);

	return (time_left > 0 ? time_left : -1);
}

clock_t
__cv_timedwait(kcondvar_t *cvp, kmutex_t *mp, clock_t exp_time)
{
	return __cv_timedwait_common(cvp, mp, exp_time, TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(__cv_timedwait);

clock_t
__cv_timedwait_interruptible(kcondvar_t *cvp, kmutex_t *mp, clock_t exp_time)
{
	return __cv_timedwait_common(cvp, mp, exp_time, TASK_INTERRUPTIBLE);
}
EXPORT_SYMBOL(__cv_timedwait_interruptible);

/*
 *'expire_time' argument is an absolute clock time in nanoseconds.
 * Return value is time left (expire_time - now) or -1 if timeout occurred.
 */
static clock_t
__cv_timedwait_hires(kcondvar_t *cvp, kmutex_t *mp,
		     hrtime_t expire_time, int state)
{
	DEFINE_WAIT(wait);
	hrtime_t time_left, now;
	unsigned long time_left_us;

	ASSERT(cvp);
	ASSERT(mp);
	ASSERT(cvp->cv_magic == CV_MAGIC);
	ASSERT(mutex_owned(mp));
	atomic_inc(&cvp->cv_refs);

	if (cvp->cv_mutex == NULL)
		cvp->cv_mutex = mp;

	/* Ensure the same mutex is used by all callers */
	ASSERT(cvp->cv_mutex == mp);

	now = gethrtime();
	time_left = expire_time - now;
	if (time_left <= 0) {
		atomic_dec(&cvp->cv_refs);
		return (-1);
	}
	time_left_us = time_left / NSEC_PER_USEC;

	prepare_to_wait_exclusive(&cvp->cv_event, &wait, state);
	atomic_inc(&cvp->cv_waiters);

	/* Mutex should be dropped after prepare_to_wait() this
	 * ensures we're linked in to the waiters list and avoids the
	 * race where 'cvp->cv_waiters > 0' but the list is empty. */
	mutex_exit(mp);
	/* Allow a 100 us range to give kernel an opportunity to coalesce
	 * interrupts */
	usleep_range(time_left_us, time_left_us + 100);
	mutex_enter(mp);

	/* No more waiters a different mutex could be used */
	if (atomic_dec_and_test(&cvp->cv_waiters)) {
		cvp->cv_mutex = NULL;
		wake_up(&cvp->cv_destroy);
	}

	finish_wait(&cvp->cv_event, &wait);
	atomic_dec(&cvp->cv_refs);

	time_left = expire_time - gethrtime();
	return (time_left > 0 ? time_left : -1);
}

/*
 * Compatibility wrapper for the cv_timedwait_hires() Illumos interface.
 */
clock_t
cv_timedwait_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim,
		   hrtime_t res, int flag)
{
	if (res > 1) {
		/*
		 * Align expiration to the specified resolution.
		 */
		if (flag & CALLOUT_FLAG_ROUNDUP)
			tim += res - 1;
		tim = (tim / res) * res;
	}

	if (!(flag & CALLOUT_FLAG_ABSOLUTE))
		tim += gethrtime();

	return __cv_timedwait_hires(cvp, mp, tim, TASK_UNINTERRUPTIBLE);
}
EXPORT_SYMBOL(cv_timedwait_hires);

void
__cv_signal(kcondvar_t *cvp)
{
	ASSERT(cvp);
	ASSERT(cvp->cv_magic == CV_MAGIC);
	atomic_inc(&cvp->cv_refs);

	/* All waiters are added with WQ_FLAG_EXCLUSIVE so only one
	 * waiter will be set runable with each call to wake_up().
	 * Additionally wake_up() holds a spin_lock assoicated with
	 * the wait queue to ensure we don't race waking up processes. */
	if (atomic_read(&cvp->cv_waiters) > 0)
		wake_up(&cvp->cv_event);

	atomic_dec(&cvp->cv_refs);
}
EXPORT_SYMBOL(__cv_signal);

void
__cv_broadcast(kcondvar_t *cvp)
{
	ASSERT(cvp);
	ASSERT(cvp->cv_magic == CV_MAGIC);
	atomic_inc(&cvp->cv_refs);

	/* Wake_up_all() will wake up all waiters even those which
	 * have the WQ_FLAG_EXCLUSIVE flag set. */
	if (atomic_read(&cvp->cv_waiters) > 0)
		wake_up_all(&cvp->cv_event);

	atomic_dec(&cvp->cv_refs);
}
EXPORT_SYMBOL(__cv_broadcast);
Commit	Line	Data
716154c5 BB	1	/*****************************************************************************\
	2	* Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
	3	* Copyright (C) 2007 The Regents of the University of California.
	4	* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
	5	* Written by Brian Behlendorf <behlendorf1@llnl.gov>.
715f6251	6	* UCRL-CODE-235197
715f6251	7	*
716154c5	8	* This file is part of the SPL, Solaris Porting Layer.
3d6af2dd	9	* For details, see <http://zfsonlinux.org/>.
716154c5 BB	10	*
	11	* The SPL is free software; you can redistribute it and/or modify it
	12	* under the terms of the GNU General Public License as published by the
	13	* Free Software Foundation; either version 2 of the License, or (at your
	14	* option) any later version.
715f6251	15	*
716154c5	16	* The SPL is distributed in the hope that it will be useful, but WITHOUT
715f6251	17	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	18	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	19	* for more details.
	20	*
	21	* You should have received a copy of the GNU General Public License along
716154c5 BB	22	* with the SPL. If not, see <http://www.gnu.org/licenses/>.
	23	*****************************************************************************
	24	* Solaris Porting Layer (SPL) Credential Implementation.
	25	\*****************************************************************************/
715f6251	26
4efd4118	27	#include <sys/condvar.h>
e5b9b344	28	#include <sys/time.h>
4efd4118	29
	30	void
	31	__cv_init(kcondvar_t cvp, char name, kcv_type_t type, void *arg)
	32	{
4efd4118	33	ASSERT(cvp);
b29012b9	34	ASSERT(name == NULL);
4efd4118	35	ASSERT(type == CV_DEFAULT);
	36	ASSERT(arg == NULL);
	37
	38	cvp->cv_magic = CV_MAGIC;
	39	init_waitqueue_head(&cvp->cv_event);
d599e4fa	40	init_waitqueue_head(&cvp->cv_destroy);
4efd4118	41	atomic_set(&cvp->cv_waiters, 0);
d2733258	42	atomic_set(&cvp->cv_refs, 1);
4efd4118	43	cvp->cv_mutex = NULL;
4efd4118	44	}
	45	EXPORT_SYMBOL(__cv_init);
	46
d599e4fa BB	47	static int
	48	cv_destroy_wakeup(kcondvar_t *cvp)
	49	{
d2733258 BB	50	if (!atomic_read(&cvp->cv_waiters) && !atomic_read(&cvp->cv_refs)) {
	51	ASSERT(cvp->cv_mutex == NULL);
	52	ASSERT(!waitqueue_active(&cvp->cv_event));
	53	return 1;
	54	}
d599e4fa	55
d2733258	56	return 0;
d599e4fa BB	57	}
d599e4fa BB	58
4efd4118	59	void
	60	__cv_destroy(kcondvar_t *cvp)
	61	{
4efd4118	62	ASSERT(cvp);
4efd4118	63	ASSERT(cvp->cv_magic == CV_MAGIC);
d599e4fa	64
d2733258 BB	65	cvp->cv_magic = CV_DESTROY;
	66	atomic_dec(&cvp->cv_refs);
	67
	68	/* Block until all waiters are woken and references dropped. */
d599e4fa BB	69	while (cv_destroy_wakeup(cvp) == 0)
	70	wait_event_timeout(cvp->cv_destroy, cv_destroy_wakeup(cvp), 1);
	71
3c60f505	72	ASSERT3P(cvp->cv_mutex, ==, NULL);
d2733258	73	ASSERT3S(atomic_read(&cvp->cv_refs), ==, 0);
3c60f505 BB	74	ASSERT3S(atomic_read(&cvp->cv_waiters), ==, 0);
3c60f505 BB	75	ASSERT3S(waitqueue_active(&cvp->cv_event), ==, 0);
4efd4118	76	}
	77	EXPORT_SYMBOL(__cv_destroy);
	78
f752b46e	79	static void
46a75aad	80	cv_wait_common(kcondvar_t cvp, kmutex_t mp, int state, int io)
4efd4118	81	{
4efd4118	82	DEFINE_WAIT(wait);
4efd4118	83
	84	ASSERT(cvp);
	85	ASSERT(mp);
	86	ASSERT(cvp->cv_magic == CV_MAGIC);
4efd4118	87	ASSERT(mutex_owned(mp));
d2733258	88	atomic_inc(&cvp->cv_refs);
4efd4118	89
	90	if (cvp->cv_mutex == NULL)
	91	cvp->cv_mutex = mp;
	92
	93	/* Ensure the same mutex is used by all callers */
	94	ASSERT(cvp->cv_mutex == mp);
4efd4118	95
f752b46e	96	prepare_to_wait_exclusive(&cvp->cv_event, &wait, state);
4efd4118	97	atomic_inc(&cvp->cv_waiters);
	98
	99	/* Mutex should be dropped after prepare_to_wait() this
	100	* ensures we're linked in to the waiters list and avoids the
	101	* race where 'cvp->cv_waiters > 0' but the list is empty. */
	102	mutex_exit(mp);
46a75aad MJ	103	if (io)
	104	io_schedule();
	105	else
	106	schedule();
4efd4118	107	mutex_enter(mp);
4efd4118	108
058de03c	109	/* No more waiters a different mutex could be used */
d599e4fa	110	if (atomic_dec_and_test(&cvp->cv_waiters)) {
058de03c	111	cvp->cv_mutex = NULL;
d599e4fa BB	112	wake_up(&cvp->cv_destroy);
d599e4fa BB	113	}
058de03c	114
4efd4118	115	finish_wait(&cvp->cv_event, &wait);
d2733258	116	atomic_dec(&cvp->cv_refs);
4efd4118	117	}
f752b46e BB	118
	119	void
	120	__cv_wait(kcondvar_t cvp, kmutex_t mp)
	121	{
46a75aad	122	cv_wait_common(cvp, mp, TASK_UNINTERRUPTIBLE, 0);
f752b46e	123	}
4efd4118	124	EXPORT_SYMBOL(__cv_wait);
4efd4118	125
f752b46e BB	126	void
	127	__cv_wait_interruptible(kcondvar_t cvp, kmutex_t mp)
	128	{
46a75aad	129	cv_wait_common(cvp, mp, TASK_INTERRUPTIBLE, 0);
f752b46e BB	130	}
	131	EXPORT_SYMBOL(__cv_wait_interruptible);
	132
46a75aad MJ	133	void
	134	__cv_wait_io(kcondvar_t cvp, kmutex_t mp)
	135	{
	136	cv_wait_common(cvp, mp, TASK_UNINTERRUPTIBLE, 1);
	137	}
	138	EXPORT_SYMBOL(__cv_wait_io);
	139
4efd4118	140	/* 'expire_time' argument is an absolute wall clock time in jiffies.
	141	* Return value is time left (expire_time - now) or -1 if timeout occurred.
	142	*/
3f688a8c NK	143	static clock_t
	144	__cv_timedwait_common(kcondvar_t cvp, kmutex_t mp,
	145	clock_t expire_time, int state)
4efd4118	146	{
	147	DEFINE_WAIT(wait);
	148	clock_t time_left;
4efd4118	149
	150	ASSERT(cvp);
	151	ASSERT(mp);
	152	ASSERT(cvp->cv_magic == CV_MAGIC);
4efd4118	153	ASSERT(mutex_owned(mp));
d2733258	154	atomic_inc(&cvp->cv_refs);
4efd4118	155
	156	if (cvp->cv_mutex == NULL)
	157	cvp->cv_mutex = mp;
	158
	159	/* Ensure the same mutex is used by all callers */
	160	ASSERT(cvp->cv_mutex == mp);
4efd4118	161
	162	/* XXX - Does not handle jiffie wrap properly */
	163	time_left = expire_time - jiffies;
d2733258 BB	164	if (time_left <= 0) {
d2733258 BB	165	atomic_dec(&cvp->cv_refs);
8d9a23e8	166	return (-1);
d2733258	167	}
4efd4118	168
3f688a8c	169	prepare_to_wait_exclusive(&cvp->cv_event, &wait, state);
4efd4118	170	atomic_inc(&cvp->cv_waiters);
	171
	172	/* Mutex should be dropped after prepare_to_wait() this
	173	* ensures we're linked in to the waiters list and avoids the
	174	* race where 'cvp->cv_waiters > 0' but the list is empty. */
	175	mutex_exit(mp);
	176	time_left = schedule_timeout(time_left);
	177	mutex_enter(mp);
	178
058de03c	179	/* No more waiters a different mutex could be used */
d599e4fa	180	if (atomic_dec_and_test(&cvp->cv_waiters)) {
058de03c	181	cvp->cv_mutex = NULL;
d599e4fa BB	182	wake_up(&cvp->cv_destroy);
d599e4fa BB	183	}
058de03c	184
4efd4118	185	finish_wait(&cvp->cv_event, &wait);
d2733258	186	atomic_dec(&cvp->cv_refs);
4efd4118	187
8d9a23e8	188	return (time_left > 0 ? time_left : -1);
4efd4118	189	}
3f688a8c NK	190
	191	clock_t
	192	__cv_timedwait(kcondvar_t cvp, kmutex_t mp, clock_t exp_time)
	193	{
	194	return __cv_timedwait_common(cvp, mp, exp_time, TASK_UNINTERRUPTIBLE);
	195	}
4efd4118	196	EXPORT_SYMBOL(__cv_timedwait);
4efd4118	197
3f688a8c NK	198	clock_t
	199	__cv_timedwait_interruptible(kcondvar_t cvp, kmutex_t mp, clock_t exp_time)
	200	{
	201	return __cv_timedwait_common(cvp, mp, exp_time, TASK_INTERRUPTIBLE);
	202	}
	203	EXPORT_SYMBOL(__cv_timedwait_interruptible);
	204
184c6873 NB	205	/*
	206	*'expire_time' argument is an absolute clock time in nanoseconds.
	207	* Return value is time left (expire_time - now) or -1 if timeout occurred.
	208	*/
	209	static clock_t
	210	__cv_timedwait_hires(kcondvar_t cvp, kmutex_t mp,
	211	hrtime_t expire_time, int state)
	212	{
	213	DEFINE_WAIT(wait);
	214	hrtime_t time_left, now;
	215	unsigned long time_left_us;
184c6873 NB	216
	217	ASSERT(cvp);
	218	ASSERT(mp);
	219	ASSERT(cvp->cv_magic == CV_MAGIC);
	220	ASSERT(mutex_owned(mp));
	221	atomic_inc(&cvp->cv_refs);
	222
	223	if (cvp->cv_mutex == NULL)
	224	cvp->cv_mutex = mp;
	225
	226	/* Ensure the same mutex is used by all callers */
	227	ASSERT(cvp->cv_mutex == mp);
	228
	229	now = gethrtime();
	230	time_left = expire_time - now;
	231	if (time_left <= 0) {
	232	atomic_dec(&cvp->cv_refs);
8d9a23e8	233	return (-1);
184c6873 NB	234	}
	235	time_left_us = time_left / NSEC_PER_USEC;
	236
	237	prepare_to_wait_exclusive(&cvp->cv_event, &wait, state);
	238	atomic_inc(&cvp->cv_waiters);
	239
	240	/* Mutex should be dropped after prepare_to_wait() this
	241	* ensures we're linked in to the waiters list and avoids the
	242	* race where 'cvp->cv_waiters > 0' but the list is empty. */
	243	mutex_exit(mp);
	244	/* Allow a 100 us range to give kernel an opportunity to coalesce
	245	* interrupts */
	246	usleep_range(time_left_us, time_left_us + 100);
	247	mutex_enter(mp);
	248
	249	/* No more waiters a different mutex could be used */
	250	if (atomic_dec_and_test(&cvp->cv_waiters)) {
	251	cvp->cv_mutex = NULL;
	252	wake_up(&cvp->cv_destroy);
	253	}
	254
	255	finish_wait(&cvp->cv_event, &wait);
	256	atomic_dec(&cvp->cv_refs);
	257
	258	time_left = expire_time - gethrtime();
8d9a23e8	259	return (time_left > 0 ? time_left : -1);
184c6873 NB	260	}
	261
	262	/*
	263	* Compatibility wrapper for the cv_timedwait_hires() Illumos interface.
	264	*/
	265	clock_t
	266	cv_timedwait_hires(kcondvar_t cvp, kmutex_t mp, hrtime_t tim,
	267	hrtime_t res, int flag)
	268	{
	269	if (res > 1) {
	270	/*
	271	* Align expiration to the specified resolution.
	272	*/
	273	if (flag & CALLOUT_FLAG_ROUNDUP)
	274	tim += res - 1;
	275	tim = (tim / res) * res;
	276	}
	277
	278	if (!(flag & CALLOUT_FLAG_ABSOLUTE))
	279	tim += gethrtime();
	280
	281	return __cv_timedwait_hires(cvp, mp, tim, TASK_UNINTERRUPTIBLE);
	282	}
	283	EXPORT_SYMBOL(cv_timedwait_hires);
	284
4efd4118	285	void
	286	__cv_signal(kcondvar_t *cvp)
	287	{
4efd4118	288	ASSERT(cvp);
4efd4118	289	ASSERT(cvp->cv_magic == CV_MAGIC);
d2733258	290	atomic_inc(&cvp->cv_refs);
4efd4118	291
	292	/* All waiters are added with WQ_FLAG_EXCLUSIVE so only one
	293	* waiter will be set runable with each call to wake_up().
	294	* Additionally wake_up() holds a spin_lock assoicated with
	295	* the wait queue to ensure we don't race waking up processes. */
	296	if (atomic_read(&cvp->cv_waiters) > 0)
	297	wake_up(&cvp->cv_event);
	298
d2733258	299	atomic_dec(&cvp->cv_refs);
4efd4118	300	}
	301	EXPORT_SYMBOL(__cv_signal);
	302
	303	void
	304	__cv_broadcast(kcondvar_t *cvp)
	305	{
	306	ASSERT(cvp);
	307	ASSERT(cvp->cv_magic == CV_MAGIC);
d2733258	308	atomic_inc(&cvp->cv_refs);
4efd4118	309
	310	/* Wake_up_all() will wake up all waiters even those which
	311	* have the WQ_FLAG_EXCLUSIVE flag set. */
	312	if (atomic_read(&cvp->cv_waiters) > 0)
	313	wake_up_all(&cvp->cv_event);
	314
d2733258	315	atomic_dec(&cvp->cv_refs);
4efd4118	316	}
4efd4118	317	EXPORT_SYMBOL(__cv_broadcast);