[mirror_ubuntu-bionic-kernel.git] / kernel / sched / wait_bit.c

/*
 * The implementation of the wait_bit*() and related waiting APIs:
 */
#include <linux/wait_bit.h>
#include <linux/sched/signal.h>
#include <linux/sched/debug.h>
#include <linux/hash.h>

#define WAIT_TABLE_BITS 8
#define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)

static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned;

wait_queue_head_t *bit_waitqueue(void *word, int bit)
{
	const int shift = BITS_PER_LONG == 32 ? 5 : 6;
	unsigned long val = (unsigned long)word << shift | bit;

	return bit_wait_table + hash_long(val, WAIT_TABLE_BITS);
}
EXPORT_SYMBOL(bit_waitqueue);

int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg)
{
	struct wait_bit_key *key = arg;
	struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);

	if (wait_bit->key.flags != key->flags ||
			wait_bit->key.bit_nr != key->bit_nr ||
			test_bit(key->bit_nr, key->flags))
		return 0;
	else
		return autoremove_wake_function(wq_entry, mode, sync, key);
}
EXPORT_SYMBOL(wake_bit_function);

/*
 * To allow interruptible waiting and asynchronous (i.e. nonblocking)
 * waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are
 * permitted return codes. Nonzero return codes halt waiting and return.
 */
int __sched
__wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
	      wait_bit_action_f *action, unsigned mode)
{
	int ret = 0;

	do {
		prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
		if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags))
			ret = (*action)(&wbq_entry->key, mode);
	} while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret);
	finish_wait(wq_head, &wbq_entry->wq_entry);
	return ret;
}
EXPORT_SYMBOL(__wait_on_bit);

int __sched out_of_line_wait_on_bit(void *word, int bit,
				    wait_bit_action_f *action, unsigned mode)
{
	struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
	DEFINE_WAIT_BIT(wq_entry, word, bit);

	return __wait_on_bit(wq_head, &wq_entry, action, mode);
}
EXPORT_SYMBOL(out_of_line_wait_on_bit);

int __sched out_of_line_wait_on_bit_timeout(
	void *word, int bit, wait_bit_action_f *action,
	unsigned mode, unsigned long timeout)
{
	struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
	DEFINE_WAIT_BIT(wq_entry, word, bit);

	wq_entry.key.timeout = jiffies + timeout;
	return __wait_on_bit(wq_head, &wq_entry, action, mode);
}
EXPORT_SYMBOL_GPL(out_of_line_wait_on_bit_timeout);

int __sched
__wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
			wait_bit_action_f *action, unsigned mode)
{
	int ret = 0;

	for (;;) {
		prepare_to_wait_exclusive(wq_head, &wbq_entry->wq_entry, mode);
		if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
			ret = action(&wbq_entry->key, mode);
			/*
			 * See the comment in prepare_to_wait_event().
			 * finish_wait() does not necessarily takes wwq_head->lock,
			 * but test_and_set_bit() implies mb() which pairs with
			 * smp_mb__after_atomic() before wake_up_page().
			 */
			if (ret)
				finish_wait(wq_head, &wbq_entry->wq_entry);
		}
		if (!test_and_set_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
			if (!ret)
				finish_wait(wq_head, &wbq_entry->wq_entry);
			return 0;
		} else if (ret) {
			return ret;
		}
	}
}
EXPORT_SYMBOL(__wait_on_bit_lock);

int __sched out_of_line_wait_on_bit_lock(void *word, int bit,
					 wait_bit_action_f *action, unsigned mode)
{
	struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
	DEFINE_WAIT_BIT(wq_entry, word, bit);

	return __wait_on_bit_lock(wq_head, &wq_entry, action, mode);
}
EXPORT_SYMBOL(out_of_line_wait_on_bit_lock);

void __wake_up_bit(struct wait_queue_head *wq_head, void *word, int bit)
{
	struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit);
	if (waitqueue_active(wq_head))
		__wake_up(wq_head, TASK_NORMAL, 1, &key);
}
EXPORT_SYMBOL(__wake_up_bit);

/**
 * wake_up_bit - wake up a waiter on a bit
 * @word: the word being waited on, a kernel virtual address
 * @bit: the bit of the word being waited on
 *
 * There is a standard hashed waitqueue table for generic use. This
 * is the part of the hashtable's accessor API that wakes up waiters
 * on a bit. For instance, if one were to have waiters on a bitflag,
 * one would call wake_up_bit() after clearing the bit.
 *
 * In order for this to function properly, as it uses waitqueue_active()
 * internally, some kind of memory barrier must be done prior to calling
 * this. Typically, this will be smp_mb__after_atomic(), but in some
 * cases where bitflags are manipulated non-atomically under a lock, one
 * may need to use a less regular barrier, such fs/inode.c's smp_mb(),
 * because spin_unlock() does not guarantee a memory barrier.
 */
void wake_up_bit(void *word, int bit)
{
	__wake_up_bit(bit_waitqueue(word, bit), word, bit);
}
EXPORT_SYMBOL(wake_up_bit);

/*
 * Manipulate the atomic_t address to produce a better bit waitqueue table hash
 * index (we're keying off bit -1, but that would produce a horrible hash
 * value).
 */
static inline wait_queue_head_t *atomic_t_waitqueue(atomic_t *p)
{
	if (BITS_PER_LONG == 64) {
		unsigned long q = (unsigned long)p;
		return bit_waitqueue((void *)(q & ~1), q & 1);
	}
	return bit_waitqueue(p, 0);
}

static int wake_atomic_t_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync,
				  void *arg)
{
	struct wait_bit_key *key = arg;
	struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
	atomic_t *val = key->flags;

	if (wait_bit->key.flags != key->flags ||
	    wait_bit->key.bit_nr != key->bit_nr ||
	    atomic_read(val) != 0)
		return 0;
	return autoremove_wake_function(wq_entry, mode, sync, key);
}

/*
 * To allow interruptible waiting and asynchronous (i.e. nonblocking) waiting,
 * the actions of __wait_on_atomic_t() are permitted return codes.  Nonzero
 * return codes halt waiting and return.
 */
static __sched
int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
		       wait_atomic_t_action_f action, unsigned int mode)
{
	atomic_t *val;
	int ret = 0;

	do {
		prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
		val = wbq_entry->key.flags;
		if (atomic_read(val) == 0)
			break;
		ret = (*action)(val, mode);
	} while (!ret && atomic_read(val) != 0);
	finish_wait(wq_head, &wbq_entry->wq_entry);
	return ret;
}

#define DEFINE_WAIT_ATOMIC_T(name, p)					\
	struct wait_bit_queue_entry name = {				\
		.key = __WAIT_ATOMIC_T_KEY_INITIALIZER(p),		\
		.wq_entry = {						\
			.private	= current,			\
			.func		= wake_atomic_t_function,	\
			.entry		=				\
				LIST_HEAD_INIT((name).wq_entry.entry),	\
		},							\
	}

__sched int out_of_line_wait_on_atomic_t(atomic_t *p,
					 wait_atomic_t_action_f action,
					 unsigned int mode)
{
	struct wait_queue_head *wq_head = atomic_t_waitqueue(p);
	DEFINE_WAIT_ATOMIC_T(wq_entry, p);

	return __wait_on_atomic_t(wq_head, &wq_entry, action, mode);
}
EXPORT_SYMBOL(out_of_line_wait_on_atomic_t);

__sched int atomic_t_wait(atomic_t *counter, unsigned int mode)
{
	schedule();
	if (signal_pending_state(mode, current))
		return -EINTR;
	return 0;
}
EXPORT_SYMBOL(atomic_t_wait);

/**
 * wake_up_atomic_t - Wake up a waiter on a atomic_t
 * @p: The atomic_t being waited on, a kernel virtual address
 *
 * Wake up anyone waiting for the atomic_t to go to zero.
 *
 * Abuse the bit-waker function and its waitqueue hash table set (the atomic_t
 * check is done by the waiter's wake function, not the by the waker itself).
 */
void wake_up_atomic_t(atomic_t *p)
{
	__wake_up_bit(atomic_t_waitqueue(p), p, WAIT_ATOMIC_T_BIT_NR);
}
EXPORT_SYMBOL(wake_up_atomic_t);

__sched int bit_wait(struct wait_bit_key *word, int mode)
{
	schedule();
	if (signal_pending_state(mode, current))
		return -EINTR;
	return 0;
}
EXPORT_SYMBOL(bit_wait);

__sched int bit_wait_io(struct wait_bit_key *word, int mode)
{
	io_schedule();
	if (signal_pending_state(mode, current))
		return -EINTR;
	return 0;
}
EXPORT_SYMBOL(bit_wait_io);

__sched int bit_wait_timeout(struct wait_bit_key *word, int mode)
{
	unsigned long now = READ_ONCE(jiffies);
	if (time_after_eq(now, word->timeout))
		return -EAGAIN;
	schedule_timeout(word->timeout - now);
	if (signal_pending_state(mode, current))
		return -EINTR;
	return 0;
}
EXPORT_SYMBOL_GPL(bit_wait_timeout);

__sched int bit_wait_io_timeout(struct wait_bit_key *word, int mode)
{
	unsigned long now = READ_ONCE(jiffies);
	if (time_after_eq(now, word->timeout))
		return -EAGAIN;
	io_schedule_timeout(word->timeout - now);
	if (signal_pending_state(mode, current))
		return -EINTR;
	return 0;
}
EXPORT_SYMBOL_GPL(bit_wait_io_timeout);

void __init wait_bit_init(void)
{
	int i;

	for (i = 0; i < WAIT_TABLE_SIZE; i++)
		init_waitqueue_head(bit_wait_table + i);
}
Commit	Line	Data
5dd43ce2 IM	1	/*
	2	* The implementation of the wait_bit*() and related waiting APIs:
	3	*/
	4	#include <linux/wait_bit.h>
	5	#include <linux/sched/signal.h>
	6	#include <linux/sched/debug.h>
5822a454 IM	7	#include <linux/hash.h>
	8
	9	#define WAIT_TABLE_BITS 8
	10	#define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)
	11
	12	static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned;
	13
	14	wait_queue_head_t bit_waitqueue(void word, int bit)
	15	{
	16	const int shift = BITS_PER_LONG == 32 ? 5 : 6;
	17	unsigned long val = (unsigned long)word << shift \| bit;
	18
	19	return bit_wait_table + hash_long(val, WAIT_TABLE_BITS);
	20	}
	21	EXPORT_SYMBOL(bit_waitqueue);
5dd43ce2 IM	22
	23	int wake_bit_function(struct wait_queue_entry wq_entry, unsigned mode, int sync, void arg)
	24	{
	25	struct wait_bit_key *key = arg;
	26	struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
	27
	28	if (wait_bit->key.flags != key->flags \|\|
	29	wait_bit->key.bit_nr != key->bit_nr \|\|
	30	test_bit(key->bit_nr, key->flags))
	31	return 0;
	32	else
	33	return autoremove_wake_function(wq_entry, mode, sync, key);
	34	}
	35	EXPORT_SYMBOL(wake_bit_function);
	36
	37	/*
	38	* To allow interruptible waiting and asynchronous (i.e. nonblocking)
	39	* waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are
	40	* permitted return codes. Nonzero return codes halt waiting and return.
	41	*/
	42	int __sched
	43	__wait_on_bit(struct wait_queue_head wq_head, struct wait_bit_queue_entry wbq_entry,
	44	wait_bit_action_f *action, unsigned mode)
	45	{
	46	int ret = 0;
	47
	48	do {
	49	prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
	50	if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags))
	51	ret = (*action)(&wbq_entry->key, mode);
	52	} while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret);
	53	finish_wait(wq_head, &wbq_entry->wq_entry);
	54	return ret;
	55	}
	56	EXPORT_SYMBOL(__wait_on_bit);
	57
	58	int __sched out_of_line_wait_on_bit(void *word, int bit,
	59	wait_bit_action_f *action, unsigned mode)
	60	{
	61	struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
	62	DEFINE_WAIT_BIT(wq_entry, word, bit);
	63
	64	return __wait_on_bit(wq_head, &wq_entry, action, mode);
	65	}
	66	EXPORT_SYMBOL(out_of_line_wait_on_bit);
	67
	68	int __sched out_of_line_wait_on_bit_timeout(
	69	void word, int bit, wait_bit_action_f action,
	70	unsigned mode, unsigned long timeout)
	71	{
	72	struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
	73	DEFINE_WAIT_BIT(wq_entry, word, bit);
	74
	75	wq_entry.key.timeout = jiffies + timeout;
	76	return __wait_on_bit(wq_head, &wq_entry, action, mode);
	77	}
	78	EXPORT_SYMBOL_GPL(out_of_line_wait_on_bit_timeout);
	79
	80	int __sched
	81	__wait_on_bit_lock(struct wait_queue_head wq_head, struct wait_bit_queue_entry wbq_entry,
	82	wait_bit_action_f *action, unsigned mode)
	83	{
	84	int ret = 0;
	85
86	for (;;) {
87	prepare_to_wait_exclusive(wq_head, &wbq_entry->wq_entry, mode);
88	if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
89	ret = action(&wbq_entry->key, mode);
90	/*
91	* See the comment in prepare_to_wait_event().
92	* finish_wait() does not necessarily takes wwq_head->lock,
93	* but test_and_set_bit() implies mb() which pairs with
94	* smp_mb__after_atomic() before wake_up_page().
95	*/
96	if (ret)
97	finish_wait(wq_head, &wbq_entry->wq_entry);
98	}
99	if (!test_and_set_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
100	if (!ret)
101	finish_wait(wq_head, &wbq_entry->wq_entry);
102	return 0;
103	} else if (ret) {
104	return ret;
105	}
106	}
107	}
108	EXPORT_SYMBOL(__wait_on_bit_lock);
109
110	int __sched out_of_line_wait_on_bit_lock(void *word, int bit,
111	wait_bit_action_f *action, unsigned mode)
112	{
113	struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
114	DEFINE_WAIT_BIT(wq_entry, word, bit);
115
116	return __wait_on_bit_lock(wq_head, &wq_entry, action, mode);
117	}
118	EXPORT_SYMBOL(out_of_line_wait_on_bit_lock);
119
120	void __wake_up_bit(struct wait_queue_head wq_head, void word, int bit)
121	{
122	struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit);
123	if (waitqueue_active(wq_head))
124	__wake_up(wq_head, TASK_NORMAL, 1, &key);
125	}
126	EXPORT_SYMBOL(__wake_up_bit);
127
128	/**
129	* wake_up_bit - wake up a waiter on a bit
130	* @word: the word being waited on, a kernel virtual address
131	* @bit: the bit of the word being waited on
132	*
133	* There is a standard hashed waitqueue table for generic use. This
134	* is the part of the hashtable's accessor API that wakes up waiters
135	* on a bit. For instance, if one were to have waiters on a bitflag,
136	* one would call wake_up_bit() after clearing the bit.
137	*
138	* In order for this to function properly, as it uses waitqueue_active()
139	* internally, some kind of memory barrier must be done prior to calling
140	* this. Typically, this will be smp_mb__after_atomic(), but in some
141	* cases where bitflags are manipulated non-atomically under a lock, one
142	* may need to use a less regular barrier, such fs/inode.c's smp_mb(),
143	* because spin_unlock() does not guarantee a memory barrier.
144	*/
145	void wake_up_bit(void *word, int bit)
146	{
147	__wake_up_bit(bit_waitqueue(word, bit), word, bit);
148	}
149	EXPORT_SYMBOL(wake_up_bit);
150
151	/*
152	* Manipulate the atomic_t address to produce a better bit waitqueue table hash
153	* index (we're keying off bit -1, but that would produce a horrible hash
154	* value).
155	*/
156	static inline wait_queue_head_t atomic_t_waitqueue(atomic_t p)
157	{
158	if (BITS_PER_LONG == 64) {
159	unsigned long q = (unsigned long)p;
160	return bit_waitqueue((void *)(q & ~1), q & 1);
161	}
162	return bit_waitqueue(p, 0);
163	}
164
165	static int wake_atomic_t_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync,
166	void *arg)
167	{
168	struct wait_bit_key *key = arg;
169	struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
170	atomic_t *val = key->flags;
171
172	if (wait_bit->key.flags != key->flags \|\|
173	wait_bit->key.bit_nr != key->bit_nr \|\|
174	atomic_read(val) != 0)
175	return 0;
176	return autoremove_wake_function(wq_entry, mode, sync, key);
177	}
178
179	/*
180	* To allow interruptible waiting and asynchronous (i.e. nonblocking) waiting,
181	* the actions of __wait_on_atomic_t() are permitted return codes. Nonzero
182	* return codes halt waiting and return.
183	*/
184	static __sched
185	int __wait_on_atomic_t(struct wait_queue_head wq_head, struct wait_bit_queue_entry wbq_entry,
5e4def20	186	wait_atomic_t_action_f action, unsigned int mode)
5dd43ce2 IM	187	{
	188	atomic_t *val;
	189	int ret = 0;
	190
	191	do {
	192	prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
	193	val = wbq_entry->key.flags;
	194	if (atomic_read(val) == 0)
	195	break;
5e4def20	196	ret = (*action)(val, mode);
5dd43ce2 IM	197	} while (!ret && atomic_read(val) != 0);
	198	finish_wait(wq_head, &wbq_entry->wq_entry);
	199	return ret;
	200	}
	201
	202	#define DEFINE_WAIT_ATOMIC_T(name, p) \
	203	struct wait_bit_queue_entry name = { \
	204	.key = __WAIT_ATOMIC_T_KEY_INITIALIZER(p), \
	205	.wq_entry = { \
	206	.private = current, \
	207	.func = wake_atomic_t_function, \
2055da97 IM	208	.entry = \
2055da97 IM	209	LIST_HEAD_INIT((name).wq_entry.entry), \
5dd43ce2 IM	210	}, \
	211	}
	212
5e4def20 DH	213	__sched int out_of_line_wait_on_atomic_t(atomic_t *p,
	214	wait_atomic_t_action_f action,
	215	unsigned int mode)
5dd43ce2 IM	216	{
	217	struct wait_queue_head *wq_head = atomic_t_waitqueue(p);
	218	DEFINE_WAIT_ATOMIC_T(wq_entry, p);
	219
	220	return __wait_on_atomic_t(wq_head, &wq_entry, action, mode);
	221	}
	222	EXPORT_SYMBOL(out_of_line_wait_on_atomic_t);
	223
5e4def20 DH	224	__sched int atomic_t_wait(atomic_t *counter, unsigned int mode)
	225	{
	226	schedule();
	227	if (signal_pending_state(mode, current))
	228	return -EINTR;
	229	return 0;
	230	}
	231	EXPORT_SYMBOL(atomic_t_wait);
	232
5dd43ce2 IM	233	/**
	234	* wake_up_atomic_t - Wake up a waiter on a atomic_t
	235	* @p: The atomic_t being waited on, a kernel virtual address
	236	*
	237	* Wake up anyone waiting for the atomic_t to go to zero.
	238	*
	239	* Abuse the bit-waker function and its waitqueue hash table set (the atomic_t
	240	* check is done by the waiter's wake function, not the by the waker itself).
	241	*/
	242	void wake_up_atomic_t(atomic_t *p)
	243	{
	244	__wake_up_bit(atomic_t_waitqueue(p), p, WAIT_ATOMIC_T_BIT_NR);
	245	}
	246	EXPORT_SYMBOL(wake_up_atomic_t);
	247
	248	__sched int bit_wait(struct wait_bit_key *word, int mode)
	249	{
	250	schedule();
	251	if (signal_pending_state(mode, current))
	252	return -EINTR;
	253	return 0;
	254	}
	255	EXPORT_SYMBOL(bit_wait);
	256
	257	__sched int bit_wait_io(struct wait_bit_key *word, int mode)
	258	{
	259	io_schedule();
	260	if (signal_pending_state(mode, current))
	261	return -EINTR;
	262	return 0;
	263	}
	264	EXPORT_SYMBOL(bit_wait_io);
	265
	266	__sched int bit_wait_timeout(struct wait_bit_key *word, int mode)
	267	{
	268	unsigned long now = READ_ONCE(jiffies);
	269	if (time_after_eq(now, word->timeout))
	270	return -EAGAIN;
	271	schedule_timeout(word->timeout - now);
	272	if (signal_pending_state(mode, current))
	273	return -EINTR;
	274	return 0;
	275	}
	276	EXPORT_SYMBOL_GPL(bit_wait_timeout);
	277
	278	__sched int bit_wait_io_timeout(struct wait_bit_key *word, int mode)
	279	{
	280	unsigned long now = READ_ONCE(jiffies);
	281	if (time_after_eq(now, word->timeout))
	282	return -EAGAIN;
	283	io_schedule_timeout(word->timeout - now);
	284	if (signal_pending_state(mode, current))
	285	return -EINTR;
	286	return 0;
	287	}
	288	EXPORT_SYMBOL_GPL(bit_wait_io_timeout);
5822a454 IM	289
	290	void __init wait_bit_init(void)
	291	{
	292	int i;
	293
	294	for (i = 0; i < WAIT_TABLE_SIZE; i++)
	295	init_waitqueue_head(bit_wait_table + i);
	296	}