[mirror_zfs.git] / module / os / linux / spl / spl-condvar.c

/*
 *  Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
 *  Copyright (C) 2007 The Regents of the University of California.
 *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
 *  Written by Brian Behlendorf <behlendorf1@llnl.gov>.
 *  UCRL-CODE-235197
 *
 *  This file is part of the SPL, Solaris Porting Layer.
 *  For details, see <http://zfsonlinux.org/>.
 *
 *  The SPL is free software; you can redistribute it and/or modify it
 *  under the terms of the GNU General Public License as published by the
 *  Free Software Foundation; either version 2 of the License, or (at your
 *  option) any later version.
 *
 *  The SPL is distributed in the hope that it will be useful, but WITHOUT
 *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 *  for more details.
 *
 *  You should have received a copy of the GNU General Public License along
 *  with the SPL.  If not, see <http://www.gnu.org/licenses/>.
 *
 *  Solaris Porting Layer (SPL) Credential Implementation.
 */

#include <sys/condvar.h>
#include <sys/time.h>
#include <sys/sysmacros.h>
#include <linux/hrtimer.h>
#include <linux/compiler_compat.h>
#include <linux/mod_compat.h>

#include <linux/sched.h>

#ifdef HAVE_SCHED_SIGNAL_HEADER
#include <linux/sched/signal.h>
#endif

#define	MAX_HRTIMEOUT_SLACK_US	1000
unsigned int spl_schedule_hrtimeout_slack_us = 0;

static int
param_set_hrtimeout_slack(const char *buf, zfs_kernel_param_t *kp)
{
	unsigned long val;
	int error;

	error = kstrtoul(buf, 0, &val);
	if (error)
		return (error);

	if (val > MAX_HRTIMEOUT_SLACK_US)
		return (-EINVAL);

	error = param_set_uint(buf, kp);
	if (error < 0)
		return (error);

	return (0);
}

module_param_call(spl_schedule_hrtimeout_slack_us, param_set_hrtimeout_slack,
	param_get_uint, &spl_schedule_hrtimeout_slack_us, 0644);
MODULE_PARM_DESC(spl_schedule_hrtimeout_slack_us,
	"schedule_hrtimeout_range() delta/slack value in us, default(0)");

void
__cv_init(kcondvar_t *cvp, char *name, kcv_type_t type, void *arg)
{
	ASSERT(cvp);
	ASSERT(name == NULL);
	ASSERT(type == CV_DEFAULT);
	ASSERT(arg == NULL);

	cvp->cv_magic = CV_MAGIC;
	init_waitqueue_head(&cvp->cv_event);
	init_waitqueue_head(&cvp->cv_destroy);
	atomic_set(&cvp->cv_waiters, 0);
	atomic_set(&cvp->cv_refs, 1);
	cvp->cv_mutex = NULL;
}
EXPORT_SYMBOL(__cv_init);

static int
cv_destroy_wakeup(kcondvar_t *cvp)
{
	if (!atomic_read(&cvp->cv_waiters) && !atomic_read(&cvp->cv_refs)) {
		ASSERT(cvp->cv_mutex == NULL);
		ASSERT(!waitqueue_active(&cvp->cv_event));
		return (1);
	}

	return (0);
}

void
__cv_destroy(kcondvar_t *cvp)
{
	ASSERT(cvp);
	ASSERT(cvp->cv_magic == CV_MAGIC);

	cvp->cv_magic = CV_DESTROY;
	atomic_dec(&cvp->cv_refs);

	/* Block until all waiters are woken and references dropped. */
	while (cv_destroy_wakeup(cvp) == 0)
		wait_event_timeout(cvp->cv_destroy, cv_destroy_wakeup(cvp), 1);

	ASSERT3P(cvp->cv_mutex, ==, NULL);
	ASSERT3S(atomic_read(&cvp->cv_refs), ==, 0);
	ASSERT3S(atomic_read(&cvp->cv_waiters), ==, 0);
	ASSERT3S(waitqueue_active(&cvp->cv_event), ==, 0);
}
EXPORT_SYMBOL(__cv_destroy);

static void
cv_wait_common(kcondvar_t *cvp, kmutex_t *mp, int state, int io)
{
	DEFINE_WAIT(wait);
	kmutex_t *m;

	ASSERT(cvp);
	ASSERT(mp);
	ASSERT(cvp->cv_magic == CV_MAGIC);
	ASSERT(mutex_owned(mp));
	atomic_inc(&cvp->cv_refs);

	m = READ_ONCE(cvp->cv_mutex);
	if (!m)
		m = xchg(&cvp->cv_mutex, mp);
	/* Ensure the same mutex is used by all callers */
	ASSERT(m == NULL || m == mp);

	prepare_to_wait_exclusive(&cvp->cv_event, &wait, state);
	atomic_inc(&cvp->cv_waiters);

	/*
	 * Mutex should be dropped after prepare_to_wait() this
	 * ensures we're linked in to the waiters list and avoids the
	 * race where 'cvp->cv_waiters > 0' but the list is empty.
	 */
	mutex_exit(mp);
	if (io)
		io_schedule();
	else
		schedule();

	/* No more waiters a different mutex could be used */
	if (atomic_dec_and_test(&cvp->cv_waiters)) {
		/*
		 * This is set without any lock, so it's racy. But this is
		 * just for debug anyway, so make it best-effort
		 */
		cvp->cv_mutex = NULL;
		wake_up(&cvp->cv_destroy);
	}

	finish_wait(&cvp->cv_event, &wait);
	atomic_dec(&cvp->cv_refs);

	/*
	 * Hold mutex after we release the cvp, otherwise we could dead lock
	 * with a thread holding the mutex and call cv_destroy.
	 */
	mutex_enter(mp);
}

void
__cv_wait(kcondvar_t *cvp, kmutex_t *mp)
{
	cv_wait_common(cvp, mp, TASK_UNINTERRUPTIBLE, 0);
}
EXPORT_SYMBOL(__cv_wait);

void
__cv_wait_io(kcondvar_t *cvp, kmutex_t *mp)
{
	cv_wait_common(cvp, mp, TASK_UNINTERRUPTIBLE, 1);
}
EXPORT_SYMBOL(__cv_wait_io);

int
__cv_wait_io_sig(kcondvar_t *cvp, kmutex_t *mp)
{
	cv_wait_common(cvp, mp, TASK_INTERRUPTIBLE, 1);

	return (signal_pending(current) ? 0 : 1);
}
EXPORT_SYMBOL(__cv_wait_io_sig);

int
__cv_wait_sig(kcondvar_t *cvp, kmutex_t *mp)
{
	cv_wait_common(cvp, mp, TASK_INTERRUPTIBLE, 0);

	return (signal_pending(current) ? 0 : 1);
}
EXPORT_SYMBOL(__cv_wait_sig);

#if defined(HAVE_IO_SCHEDULE_TIMEOUT)
#define	spl_io_schedule_timeout(t)	io_schedule_timeout(t)
#else

struct spl_task_timer {
	struct timer_list timer;
	struct task_struct *task;
};

static void
__cv_wakeup(spl_timer_list_t t)
{
	struct timer_list *tmr = (struct timer_list *)t;
	struct spl_task_timer *task_timer = from_timer(task_timer, tmr, timer);

	wake_up_process(task_timer->task);
}

static long
spl_io_schedule_timeout(long time_left)
{
	long expire_time = jiffies + time_left;
	struct spl_task_timer task_timer;
	struct timer_list *timer = &task_timer.timer;

	task_timer.task = current;

	timer_setup(timer, __cv_wakeup, 0);

	timer->expires = expire_time;
	add_timer(timer);

	io_schedule();

	del_timer_sync(timer);

	time_left = expire_time - jiffies;

	return (time_left < 0 ? 0 : time_left);
}
#endif

/*
 * 'expire_time' argument is an absolute wall clock time in jiffies.
 * Return value is time left (expire_time - now) or -1 if timeout occurred.
 */
static clock_t
__cv_timedwait_common(kcondvar_t *cvp, kmutex_t *mp, clock_t expire_time,
    int state, int io)
{
	DEFINE_WAIT(wait);
	kmutex_t *m;
	clock_t time_left;

	ASSERT(cvp);
	ASSERT(mp);
	ASSERT(cvp->cv_magic == CV_MAGIC);
	ASSERT(mutex_owned(mp));

	/* XXX - Does not handle jiffie wrap properly */
	time_left = expire_time - jiffies;
	if (time_left <= 0)
		return (-1);

	atomic_inc(&cvp->cv_refs);
	m = READ_ONCE(cvp->cv_mutex);
	if (!m)
		m = xchg(&cvp->cv_mutex, mp);
	/* Ensure the same mutex is used by all callers */
	ASSERT(m == NULL || m == mp);

	prepare_to_wait_exclusive(&cvp->cv_event, &wait, state);
	atomic_inc(&cvp->cv_waiters);

	/*
	 * Mutex should be dropped after prepare_to_wait() this
	 * ensures we're linked in to the waiters list and avoids the
	 * race where 'cvp->cv_waiters > 0' but the list is empty.
	 */
	mutex_exit(mp);
	if (io)
		time_left = spl_io_schedule_timeout(time_left);
	else
		time_left = schedule_timeout(time_left);

	/* No more waiters a different mutex could be used */
	if (atomic_dec_and_test(&cvp->cv_waiters)) {
		/*
		 * This is set without any lock, so it's racy. But this is
		 * just for debug anyway, so make it best-effort
		 */
		cvp->cv_mutex = NULL;
		wake_up(&cvp->cv_destroy);
	}

	finish_wait(&cvp->cv_event, &wait);
	atomic_dec(&cvp->cv_refs);

	/*
	 * Hold mutex after we release the cvp, otherwise we could dead lock
	 * with a thread holding the mutex and call cv_destroy.
	 */
	mutex_enter(mp);
	return (time_left > 0 ? 1 : -1);
}

int
__cv_timedwait(kcondvar_t *cvp, kmutex_t *mp, clock_t exp_time)
{
	return (__cv_timedwait_common(cvp, mp, exp_time,
	    TASK_UNINTERRUPTIBLE, 0));
}
EXPORT_SYMBOL(__cv_timedwait);

int
__cv_timedwait_io(kcondvar_t *cvp, kmutex_t *mp, clock_t exp_time)
{
	return (__cv_timedwait_common(cvp, mp, exp_time,
	    TASK_UNINTERRUPTIBLE, 1));
}
EXPORT_SYMBOL(__cv_timedwait_io);

int
__cv_timedwait_sig(kcondvar_t *cvp, kmutex_t *mp, clock_t exp_time)
{
	int rc;

	rc = __cv_timedwait_common(cvp, mp, exp_time, TASK_INTERRUPTIBLE, 0);
	return (signal_pending(current) ? 0 : rc);
}
EXPORT_SYMBOL(__cv_timedwait_sig);

/*
 * 'expire_time' argument is an absolute clock time in nanoseconds.
 * Return value is time left (expire_time - now) or -1 if timeout occurred.
 */
static clock_t
__cv_timedwait_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t expire_time,
    hrtime_t res, int state)
{
	DEFINE_WAIT(wait);
	kmutex_t *m;
	hrtime_t time_left;
	ktime_t ktime_left;
	u64 slack = 0;
	int rc;

	ASSERT(cvp);
	ASSERT(mp);
	ASSERT(cvp->cv_magic == CV_MAGIC);
	ASSERT(mutex_owned(mp));

	time_left = expire_time - gethrtime();
	if (time_left <= 0)
		return (-1);

	atomic_inc(&cvp->cv_refs);
	m = READ_ONCE(cvp->cv_mutex);
	if (!m)
		m = xchg(&cvp->cv_mutex, mp);
	/* Ensure the same mutex is used by all callers */
	ASSERT(m == NULL || m == mp);

	prepare_to_wait_exclusive(&cvp->cv_event, &wait, state);
	atomic_inc(&cvp->cv_waiters);

	/*
	 * Mutex should be dropped after prepare_to_wait() this
	 * ensures we're linked in to the waiters list and avoids the
	 * race where 'cvp->cv_waiters > 0' but the list is empty.
	 */
	mutex_exit(mp);

	ktime_left = ktime_set(0, time_left);
	slack = MIN(MAX(res, spl_schedule_hrtimeout_slack_us * NSEC_PER_USEC),
	    MAX_HRTIMEOUT_SLACK_US * NSEC_PER_USEC);
	rc = schedule_hrtimeout_range(&ktime_left, slack, HRTIMER_MODE_REL);

	/* No more waiters a different mutex could be used */
	if (atomic_dec_and_test(&cvp->cv_waiters)) {
		/*
		 * This is set without any lock, so it's racy. But this is
		 * just for debug anyway, so make it best-effort
		 */
		cvp->cv_mutex = NULL;
		wake_up(&cvp->cv_destroy);
	}

	finish_wait(&cvp->cv_event, &wait);
	atomic_dec(&cvp->cv_refs);

	mutex_enter(mp);
	return (rc == -EINTR ? 1 : -1);
}

/*
 * Compatibility wrapper for the cv_timedwait_hires() Illumos interface.
 */
static int
cv_timedwait_hires_common(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim,
    hrtime_t res, int flag, int state)
{
	if (!(flag & CALLOUT_FLAG_ABSOLUTE))
		tim += gethrtime();

	return (__cv_timedwait_hires(cvp, mp, tim, res, state));
}

int
cv_timedwait_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim, hrtime_t res,
    int flag)
{
	return (cv_timedwait_hires_common(cvp, mp, tim, res, flag,
	    TASK_UNINTERRUPTIBLE));
}
EXPORT_SYMBOL(cv_timedwait_hires);

int
cv_timedwait_sig_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim,
    hrtime_t res, int flag)
{
	int rc;

	rc = cv_timedwait_hires_common(cvp, mp, tim, res, flag,
	    TASK_INTERRUPTIBLE);
	return (signal_pending(current) ? 0 : rc);
}
EXPORT_SYMBOL(cv_timedwait_sig_hires);

void
__cv_signal(kcondvar_t *cvp)
{
	ASSERT(cvp);
	ASSERT(cvp->cv_magic == CV_MAGIC);
	atomic_inc(&cvp->cv_refs);

	/*
	 * All waiters are added with WQ_FLAG_EXCLUSIVE so only one
	 * waiter will be set runnable with each call to wake_up().
	 * Additionally wake_up() holds a spin_lock associated with
	 * the wait queue to ensure we don't race waking up processes.
	 */
	if (atomic_read(&cvp->cv_waiters) > 0)
		wake_up(&cvp->cv_event);

	atomic_dec(&cvp->cv_refs);
}
EXPORT_SYMBOL(__cv_signal);

void
__cv_broadcast(kcondvar_t *cvp)
{
	ASSERT(cvp);
	ASSERT(cvp->cv_magic == CV_MAGIC);
	atomic_inc(&cvp->cv_refs);

	/*
	 * Wake_up_all() will wake up all waiters even those which
	 * have the WQ_FLAG_EXCLUSIVE flag set.
	 */
	if (atomic_read(&cvp->cv_waiters) > 0)
		wake_up_all(&cvp->cv_event);

	atomic_dec(&cvp->cv_refs);
}
EXPORT_SYMBOL(__cv_broadcast);
Commit	Line	Data
23453686	1	/*
716154c5 BB	2	* Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
	3	* Copyright (C) 2007 The Regents of the University of California.
	4	* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
	5	* Written by Brian Behlendorf <behlendorf1@llnl.gov>.
715f6251 BB	6	* UCRL-CODE-235197
715f6251 BB	7	*
716154c5	8	* This file is part of the SPL, Solaris Porting Layer.
3d6af2dd	9	* For details, see <http://zfsonlinux.org/>.
716154c5 BB	10	*
	11	* The SPL is free software; you can redistribute it and/or modify it
	12	* under the terms of the GNU General Public License as published by the
	13	* Free Software Foundation; either version 2 of the License, or (at your
	14	* option) any later version.
715f6251	15	*
716154c5	16	* The SPL is distributed in the hope that it will be useful, but WITHOUT
715f6251 BB	17	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	18	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	19	* for more details.
	20	*
	21	* You should have received a copy of the GNU General Public License along
716154c5	22	* with the SPL. If not, see <http://www.gnu.org/licenses/>.
23453686	23	*
716154c5	24	* Solaris Porting Layer (SPL) Credential Implementation.
23453686	25	*/
715f6251	26
4efd4118	27	#include <sys/condvar.h>
e5b9b344	28	#include <sys/time.h>
8d042842	29	#include <sys/sysmacros.h>
39cd90ef	30	#include <linux/hrtimer.h>
93ce2b4c	31	#include <linux/compiler_compat.h>
8d042842	32	#include <linux/mod_compat.h>
4efd4118	33
186898bb DB	34	#include <linux/sched.h>
	35
	36	#ifdef HAVE_SCHED_SIGNAL_HEADER
	37	#include <linux/sched/signal.h>
	38	#endif
	39
8d042842 TN	40	#define MAX_HRTIMEOUT_SLACK_US 1000
	41	unsigned int spl_schedule_hrtimeout_slack_us = 0;
	42
	43	static int
	44	param_set_hrtimeout_slack(const char buf, zfs_kernel_param_t kp)
	45	{
	46	unsigned long val;
	47	int error;
	48
	49	error = kstrtoul(buf, 0, &val);
	50	if (error)
	51	return (error);
	52
	53	if (val > MAX_HRTIMEOUT_SLACK_US)
	54	return (-EINVAL);
	55
	56	error = param_set_uint(buf, kp);
	57	if (error < 0)
	58	return (error);
	59
	60	return (0);
	61	}
	62
	63	module_param_call(spl_schedule_hrtimeout_slack_us, param_set_hrtimeout_slack,
	64	param_get_uint, &spl_schedule_hrtimeout_slack_us, 0644);
	65	MODULE_PARM_DESC(spl_schedule_hrtimeout_slack_us,
	66	"schedule_hrtimeout_range() delta/slack value in us, default(0)");
	67
4efd4118 BB	68	void
	69	__cv_init(kcondvar_t cvp, char name, kcv_type_t type, void *arg)
	70	{
4efd4118	71	ASSERT(cvp);
b29012b9	72	ASSERT(name == NULL);
4efd4118 BB	73	ASSERT(type == CV_DEFAULT);
	74	ASSERT(arg == NULL);
	75
	76	cvp->cv_magic = CV_MAGIC;
	77	init_waitqueue_head(&cvp->cv_event);
d599e4fa	78	init_waitqueue_head(&cvp->cv_destroy);
4efd4118	79	atomic_set(&cvp->cv_waiters, 0);
d2733258	80	atomic_set(&cvp->cv_refs, 1);
4efd4118	81	cvp->cv_mutex = NULL;
4efd4118 BB	82	}
	83	EXPORT_SYMBOL(__cv_init);
	84
d599e4fa BB	85	static int
	86	cv_destroy_wakeup(kcondvar_t *cvp)
	87	{
d2733258 BB	88	if (!atomic_read(&cvp->cv_waiters) && !atomic_read(&cvp->cv_refs)) {
	89	ASSERT(cvp->cv_mutex == NULL);
	90	ASSERT(!waitqueue_active(&cvp->cv_event));
23453686	91	return (1);
d2733258	92	}
d599e4fa	93
23453686	94	return (0);
d599e4fa BB	95	}
d599e4fa BB	96
4efd4118 BB	97	void
	98	__cv_destroy(kcondvar_t *cvp)
	99	{
4efd4118 BB	100	ASSERT(cvp);
4efd4118 BB	101	ASSERT(cvp->cv_magic == CV_MAGIC);
d599e4fa	102
d2733258 BB	103	cvp->cv_magic = CV_DESTROY;
	104	atomic_dec(&cvp->cv_refs);
	105
	106	/* Block until all waiters are woken and references dropped. */
d599e4fa BB	107	while (cv_destroy_wakeup(cvp) == 0)
	108	wait_event_timeout(cvp->cv_destroy, cv_destroy_wakeup(cvp), 1);
	109
3c60f505	110	ASSERT3P(cvp->cv_mutex, ==, NULL);
d2733258	111	ASSERT3S(atomic_read(&cvp->cv_refs), ==, 0);
3c60f505 BB	112	ASSERT3S(atomic_read(&cvp->cv_waiters), ==, 0);
3c60f505 BB	113	ASSERT3S(waitqueue_active(&cvp->cv_event), ==, 0);
4efd4118 BB	114	}
	115	EXPORT_SYMBOL(__cv_destroy);
	116
f752b46e	117	static void
46a75aad	118	cv_wait_common(kcondvar_t cvp, kmutex_t mp, int state, int io)
4efd4118 BB	119	{
4efd4118 BB	120	DEFINE_WAIT(wait);
e843553d	121	kmutex_t *m;
4efd4118 BB	122
4efd4118 BB	123	ASSERT(cvp);
23453686	124	ASSERT(mp);
4efd4118	125	ASSERT(cvp->cv_magic == CV_MAGIC);
4efd4118	126	ASSERT(mutex_owned(mp));
d2733258	127	atomic_inc(&cvp->cv_refs);
4efd4118	128
93ce2b4c	129	m = READ_ONCE(cvp->cv_mutex);
e843553d CC	130	if (!m)
e843553d CC	131	m = xchg(&cvp->cv_mutex, mp);
4efd4118	132	/* Ensure the same mutex is used by all callers */
e843553d	133	ASSERT(m == NULL \|\| m == mp);
4efd4118	134
f752b46e	135	prepare_to_wait_exclusive(&cvp->cv_event, &wait, state);
4efd4118 BB	136	atomic_inc(&cvp->cv_waiters);
4efd4118 BB	137
23453686 BB	138	/*
23453686 BB	139	* Mutex should be dropped after prepare_to_wait() this
4efd4118	140	* ensures we're linked in to the waiters list and avoids the
23453686 BB	141	* race where 'cvp->cv_waiters > 0' but the list is empty.
23453686 BB	142	*/
4efd4118	143	mutex_exit(mp);
46a75aad MJ	144	if (io)
	145	io_schedule();
	146	else
	147	schedule();
4efd4118	148
058de03c	149	/* No more waiters a different mutex could be used */
d599e4fa	150	if (atomic_dec_and_test(&cvp->cv_waiters)) {
e843553d CC	151	/*
	152	* This is set without any lock, so it's racy. But this is
	153	* just for debug anyway, so make it best-effort
	154	*/
058de03c	155	cvp->cv_mutex = NULL;
d599e4fa BB	156	wake_up(&cvp->cv_destroy);
d599e4fa BB	157	}
058de03c	158
4efd4118	159	finish_wait(&cvp->cv_event, &wait);
d2733258	160	atomic_dec(&cvp->cv_refs);
e843553d CC	161
	162	/*
	163	* Hold mutex after we release the cvp, otherwise we could dead lock
	164	* with a thread holding the mutex and call cv_destroy.
	165	*/
	166	mutex_enter(mp);
4efd4118	167	}
f752b46e BB	168
	169	void
	170	__cv_wait(kcondvar_t cvp, kmutex_t mp)
	171	{
46a75aad	172	cv_wait_common(cvp, mp, TASK_UNINTERRUPTIBLE, 0);
f752b46e	173	}
4efd4118 BB	174	EXPORT_SYMBOL(__cv_wait);
4efd4118 BB	175
23602fdb BB	176	void
	177	__cv_wait_io(kcondvar_t cvp, kmutex_t mp)
	178	{
	179	cv_wait_common(cvp, mp, TASK_UNINTERRUPTIBLE, 1);
	180	}
	181	EXPORT_SYMBOL(__cv_wait_io);
	182
186898bb DB	183	int
	184	__cv_wait_io_sig(kcondvar_t cvp, kmutex_t mp)
	185	{
	186	cv_wait_common(cvp, mp, TASK_INTERRUPTIBLE, 1);
	187
	188	return (signal_pending(current) ? 0 : 1);
	189	}
	190	EXPORT_SYMBOL(__cv_wait_io_sig);
	191
	192	int
23453686	193	__cv_wait_sig(kcondvar_t cvp, kmutex_t mp)
f752b46e	194	{
46a75aad	195	cv_wait_common(cvp, mp, TASK_INTERRUPTIBLE, 0);
186898bb DB	196
186898bb DB	197	return (signal_pending(current) ? 0 : 1);
f752b46e	198	}
23453686	199	EXPORT_SYMBOL(__cv_wait_sig);
f752b46e	200
23602fdb BB	201	#if defined(HAVE_IO_SCHEDULE_TIMEOUT)
	202	#define spl_io_schedule_timeout(t) io_schedule_timeout(t)
	203	#else
8b8b44d0 RK	204
	205	struct spl_task_timer {
	206	struct timer_list timer;
	207	struct task_struct *task;
	208	};
	209
23602fdb	210	static void
8b8b44d0	211	__cv_wakeup(spl_timer_list_t t)
46a75aad	212	{
8b8b44d0 RK	213	struct timer_list tmr = (struct timer_list )t;
	214	struct spl_task_timer *task_timer = from_timer(task_timer, tmr, timer);
	215
	216	wake_up_process(task_timer->task);
46a75aad	217	}
23602fdb BB	218
	219	static long
	220	spl_io_schedule_timeout(long time_left)
	221	{
	222	long expire_time = jiffies + time_left;
8b8b44d0 RK	223	struct spl_task_timer task_timer;
	224	struct timer_list *timer = &task_timer.timer;
	225
	226	task_timer.task = current;
23602fdb	227
8b8b44d0 RK	228	timer_setup(timer, __cv_wakeup, 0);
	229
	230	timer->expires = expire_time;
	231	add_timer(timer);
23602fdb BB	232
	233	io_schedule();
	234
8b8b44d0 RK	235	del_timer_sync(timer);
8b8b44d0 RK	236
23602fdb BB	237	time_left = expire_time - jiffies;
	238
	239	return (time_left < 0 ? 0 : time_left);
	240	}
	241	#endif
46a75aad	242
23453686 BB	243	/*
23453686 BB	244	* 'expire_time' argument is an absolute wall clock time in jiffies.
4efd4118 BB	245	* Return value is time left (expire_time - now) or -1 if timeout occurred.
4efd4118 BB	246	*/
3f688a8c	247	static clock_t
23453686	248	__cv_timedwait_common(kcondvar_t cvp, kmutex_t mp, clock_t expire_time,
23602fdb	249	int state, int io)
4efd4118 BB	250	{
4efd4118 BB	251	DEFINE_WAIT(wait);
e843553d	252	kmutex_t *m;
4efd4118	253	clock_t time_left;
4efd4118 BB	254
4efd4118 BB	255	ASSERT(cvp);
23453686	256	ASSERT(mp);
4efd4118	257	ASSERT(cvp->cv_magic == CV_MAGIC);
4efd4118 BB	258	ASSERT(mutex_owned(mp));
4efd4118 BB	259
2ded1c7e BB	260	/* XXX - Does not handle jiffie wrap properly */
	261	time_left = expire_time - jiffies;
	262	if (time_left <= 0)
	263	return (-1);
	264
	265	atomic_inc(&cvp->cv_refs);
93ce2b4c	266	m = READ_ONCE(cvp->cv_mutex);
e843553d CC	267	if (!m)
e843553d CC	268	m = xchg(&cvp->cv_mutex, mp);
4efd4118	269	/* Ensure the same mutex is used by all callers */
e843553d	270	ASSERT(m == NULL \|\| m == mp);
4efd4118	271
3f688a8c	272	prepare_to_wait_exclusive(&cvp->cv_event, &wait, state);
4efd4118 BB	273	atomic_inc(&cvp->cv_waiters);
4efd4118 BB	274
23453686 BB	275	/*
23453686 BB	276	* Mutex should be dropped after prepare_to_wait() this
4efd4118	277	* ensures we're linked in to the waiters list and avoids the
23453686 BB	278	* race where 'cvp->cv_waiters > 0' but the list is empty.
23453686 BB	279	*/
4efd4118	280	mutex_exit(mp);
23602fdb BB	281	if (io)
	282	time_left = spl_io_schedule_timeout(time_left);
	283	else
	284	time_left = schedule_timeout(time_left);
4efd4118	285
058de03c	286	/* No more waiters a different mutex could be used */
d599e4fa	287	if (atomic_dec_and_test(&cvp->cv_waiters)) {
e843553d CC	288	/*
	289	* This is set without any lock, so it's racy. But this is
	290	* just for debug anyway, so make it best-effort
	291	*/
058de03c	292	cvp->cv_mutex = NULL;
d599e4fa BB	293	wake_up(&cvp->cv_destroy);
d599e4fa BB	294	}
058de03c	295
4efd4118	296	finish_wait(&cvp->cv_event, &wait);
d2733258	297	atomic_dec(&cvp->cv_refs);
4efd4118	298
e843553d CC	299	/*
	300	* Hold mutex after we release the cvp, otherwise we could dead lock
	301	* with a thread holding the mutex and call cv_destroy.
	302	*/
	303	mutex_enter(mp);
8056a756	304	return (time_left > 0 ? 1 : -1);
4efd4118	305	}
3f688a8c	306
8056a756	307	int
3f688a8c NK	308	__cv_timedwait(kcondvar_t cvp, kmutex_t mp, clock_t exp_time)
3f688a8c NK	309	{
23602fdb BB	310	return (__cv_timedwait_common(cvp, mp, exp_time,
23602fdb BB	311	TASK_UNINTERRUPTIBLE, 0));
3f688a8c	312	}
4efd4118 BB	313	EXPORT_SYMBOL(__cv_timedwait);
4efd4118 BB	314
8056a756	315	int
23602fdb BB	316	__cv_timedwait_io(kcondvar_t cvp, kmutex_t mp, clock_t exp_time)
	317	{
	318	return (__cv_timedwait_common(cvp, mp, exp_time,
	319	TASK_UNINTERRUPTIBLE, 1));
	320	}
	321	EXPORT_SYMBOL(__cv_timedwait_io);
	322
8056a756	323	int
23453686	324	__cv_timedwait_sig(kcondvar_t cvp, kmutex_t mp, clock_t exp_time)
3f688a8c	325	{
8056a756 MM	326	int rc;
	327
	328	rc = __cv_timedwait_common(cvp, mp, exp_time, TASK_INTERRUPTIBLE, 0);
	329	return (signal_pending(current) ? 0 : rc);
3f688a8c	330	}
23453686	331	EXPORT_SYMBOL(__cv_timedwait_sig);
3f688a8c	332
184c6873	333	/*
23453686	334	* 'expire_time' argument is an absolute clock time in nanoseconds.
184c6873 NB	335	* Return value is time left (expire_time - now) or -1 if timeout occurred.
	336	*/
	337	static clock_t
23453686	338	__cv_timedwait_hires(kcondvar_t cvp, kmutex_t mp, hrtime_t expire_time,
8d042842	339	hrtime_t res, int state)
184c6873 NB	340	{
184c6873 NB	341	DEFINE_WAIT(wait);
e843553d	342	kmutex_t *m;
2ded1c7e	343	hrtime_t time_left;
39cd90ef	344	ktime_t ktime_left;
8d042842	345	u64 slack = 0;
8056a756	346	int rc;
184c6873 NB	347
	348	ASSERT(cvp);
	349	ASSERT(mp);
	350	ASSERT(cvp->cv_magic == CV_MAGIC);
	351	ASSERT(mutex_owned(mp));
184c6873	352
2ded1c7e BB	353	time_left = expire_time - gethrtime();
	354	if (time_left <= 0)
	355	return (-1);
	356
	357	atomic_inc(&cvp->cv_refs);
93ce2b4c	358	m = READ_ONCE(cvp->cv_mutex);
e843553d CC	359	if (!m)
e843553d CC	360	m = xchg(&cvp->cv_mutex, mp);
184c6873	361	/* Ensure the same mutex is used by all callers */
e843553d	362	ASSERT(m == NULL \|\| m == mp);
184c6873	363
184c6873 NB	364	prepare_to_wait_exclusive(&cvp->cv_event, &wait, state);
	365	atomic_inc(&cvp->cv_waiters);
	366
23453686 BB	367	/*
23453686 BB	368	* Mutex should be dropped after prepare_to_wait() this
184c6873	369	* ensures we're linked in to the waiters list and avoids the
23453686 BB	370	* race where 'cvp->cv_waiters > 0' but the list is empty.
23453686 BB	371	*/
184c6873	372	mutex_exit(mp);
8d042842	373
39cd90ef	374	ktime_left = ktime_set(0, time_left);
8d042842 TN	375	slack = MIN(MAX(res, spl_schedule_hrtimeout_slack_us * NSEC_PER_USEC),
8d042842 TN	376	MAX_HRTIMEOUT_SLACK_US * NSEC_PER_USEC);
8056a756	377	rc = schedule_hrtimeout_range(&ktime_left, slack, HRTIMER_MODE_REL);
184c6873 NB	378
	379	/* No more waiters a different mutex could be used */
	380	if (atomic_dec_and_test(&cvp->cv_waiters)) {
e843553d CC	381	/*
	382	* This is set without any lock, so it's racy. But this is
	383	* just for debug anyway, so make it best-effort
	384	*/
184c6873 NB	385	cvp->cv_mutex = NULL;
	386	wake_up(&cvp->cv_destroy);
	387	}
	388
	389	finish_wait(&cvp->cv_event, &wait);
	390	atomic_dec(&cvp->cv_refs);
	391
e843553d	392	mutex_enter(mp);
8056a756	393	return (rc == -EINTR ? 1 : -1);
184c6873 NB	394	}
	395
	396	/*
	397	* Compatibility wrapper for the cv_timedwait_hires() Illumos interface.
	398	*/
8056a756	399	static int
5461eefe BB	400	cv_timedwait_hires_common(kcondvar_t cvp, kmutex_t mp, hrtime_t tim,
5461eefe BB	401	hrtime_t res, int flag, int state)
184c6873	402	{
872e0cc9 CC	403	if (!(flag & CALLOUT_FLAG_ABSOLUTE))
872e0cc9 CC	404	tim += gethrtime();
39cd90ef	405
8d042842	406	return (__cv_timedwait_hires(cvp, mp, tim, res, state));
39cd90ef	407	}
184c6873	408
8056a756	409	int
39cd90ef CC	410	cv_timedwait_hires(kcondvar_t cvp, kmutex_t mp, hrtime_t tim, hrtime_t res,
	411	int flag)
	412	{
	413	return (cv_timedwait_hires_common(cvp, mp, tim, res, flag,
	414	TASK_UNINTERRUPTIBLE));
184c6873 NB	415	}
	416	EXPORT_SYMBOL(cv_timedwait_hires);
	417
8056a756	418	int
5461eefe BB	419	cv_timedwait_sig_hires(kcondvar_t cvp, kmutex_t mp, hrtime_t tim,
5461eefe BB	420	hrtime_t res, int flag)
39cd90ef	421	{
8056a756 MM	422	int rc;
	423
	424	rc = cv_timedwait_hires_common(cvp, mp, tim, res, flag,
	425	TASK_INTERRUPTIBLE);
	426	return (signal_pending(current) ? 0 : rc);
39cd90ef CC	427	}
	428	EXPORT_SYMBOL(cv_timedwait_sig_hires);
	429
4efd4118 BB	430	void
	431	__cv_signal(kcondvar_t *cvp)
	432	{
4efd4118 BB	433	ASSERT(cvp);
4efd4118 BB	434	ASSERT(cvp->cv_magic == CV_MAGIC);
d2733258	435	atomic_inc(&cvp->cv_refs);
4efd4118	436
23453686 BB	437	/*
23453686 BB	438	* All waiters are added with WQ_FLAG_EXCLUSIVE so only one
9f5c1bc6 AG	439	* waiter will be set runnable with each call to wake_up().
9f5c1bc6 AG	440	* Additionally wake_up() holds a spin_lock associated with
23453686 BB	441	* the wait queue to ensure we don't race waking up processes.
23453686 BB	442	*/
4efd4118 BB	443	if (atomic_read(&cvp->cv_waiters) > 0)
	444	wake_up(&cvp->cv_event);
	445
d2733258	446	atomic_dec(&cvp->cv_refs);
4efd4118 BB	447	}
	448	EXPORT_SYMBOL(__cv_signal);
	449
	450	void
	451	__cv_broadcast(kcondvar_t *cvp)
	452	{
	453	ASSERT(cvp);
	454	ASSERT(cvp->cv_magic == CV_MAGIC);
d2733258	455	atomic_inc(&cvp->cv_refs);
4efd4118	456
23453686 BB	457	/*
	458	* Wake_up_all() will wake up all waiters even those which
	459	* have the WQ_FLAG_EXCLUSIVE flag set.
	460	*/
4efd4118 BB	461	if (atomic_read(&cvp->cv_waiters) > 0)
	462	wake_up_all(&cvp->cv_event);
	463
d2733258	464	atomic_dec(&cvp->cv_refs);
4efd4118 BB	465	}
4efd4118 BB	466	EXPORT_SYMBOL(__cv_broadcast);