[mirror_ubuntu-bionic-kernel.git] / include / asm-generic / rwsem.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_GENERIC_RWSEM_H
#define _ASM_GENERIC_RWSEM_H

#ifndef _LINUX_RWSEM_H
#error "Please don't include <asm/rwsem.h> directly, use <linux/rwsem.h> instead."
#endif

#ifdef __KERNEL__

/*
 * R/W semaphores originally for PPC using the stuff in lib/rwsem.c.
 * Adapted largely from include/asm-i386/rwsem.h
 * by Paul Mackerras <paulus@samba.org>.
 */

/*
 * the semaphore definition
 */
#ifdef CONFIG_64BIT
# define RWSEM_ACTIVE_MASK		0xffffffffL
#else
# define RWSEM_ACTIVE_MASK		0x0000ffffL
#endif

#define RWSEM_UNLOCKED_VALUE		0x00000000L
#define RWSEM_ACTIVE_BIAS		0x00000001L
#define RWSEM_WAITING_BIAS		(-RWSEM_ACTIVE_MASK-1)
#define RWSEM_ACTIVE_READ_BIAS		RWSEM_ACTIVE_BIAS
#define RWSEM_ACTIVE_WRITE_BIAS		(RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)

/*
 * lock for reading
 */
static inline void __down_read(struct rw_semaphore *sem)
{
	if (unlikely(atomic_long_inc_return_acquire(&sem->count) <= 0))
		rwsem_down_read_failed(sem);
}

static inline int __down_read_killable(struct rw_semaphore *sem)
{
	if (unlikely(atomic_long_inc_return_acquire(&sem->count) <= 0)) {
		if (IS_ERR(rwsem_down_read_failed_killable(sem)))
			return -EINTR;
	}

	return 0;
}

static inline int __down_read_trylock(struct rw_semaphore *sem)
{
	long tmp;

	while ((tmp = atomic_long_read(&sem->count)) >= 0) {
		if (tmp == atomic_long_cmpxchg_acquire(&sem->count, tmp,
				   tmp + RWSEM_ACTIVE_READ_BIAS)) {
			return 1;
		}
	}
	return 0;
}

/*
 * lock for writing
 */
static inline void __down_write(struct rw_semaphore *sem)
{
	long tmp;

	tmp = atomic_long_add_return_acquire(RWSEM_ACTIVE_WRITE_BIAS,
					     &sem->count);
	if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
		rwsem_down_write_failed(sem);
}

static inline int __down_write_killable(struct rw_semaphore *sem)
{
	long tmp;

	tmp = atomic_long_add_return_acquire(RWSEM_ACTIVE_WRITE_BIAS,
					     &sem->count);
	if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
		if (IS_ERR(rwsem_down_write_failed_killable(sem)))
			return -EINTR;
	return 0;
}

static inline int __down_write_trylock(struct rw_semaphore *sem)
{
	long tmp;

	tmp = atomic_long_cmpxchg_acquire(&sem->count, RWSEM_UNLOCKED_VALUE,
		      RWSEM_ACTIVE_WRITE_BIAS);
	return tmp == RWSEM_UNLOCKED_VALUE;
}

/*
 * unlock after reading
 */
static inline void __up_read(struct rw_semaphore *sem)
{
	long tmp;

	tmp = atomic_long_dec_return_release(&sem->count);
	if (unlikely(tmp < -1 && (tmp & RWSEM_ACTIVE_MASK) == 0))
		rwsem_wake(sem);
}

/*
 * unlock after writing
 */
static inline void __up_write(struct rw_semaphore *sem)
{
	if (unlikely(atomic_long_sub_return_release(RWSEM_ACTIVE_WRITE_BIAS,
						    &sem->count) < 0))
		rwsem_wake(sem);
}

/*
 * downgrade write lock to read lock
 */
static inline void __downgrade_write(struct rw_semaphore *sem)
{
	long tmp;

	/*
	 * When downgrading from exclusive to shared ownership,
	 * anything inside the write-locked region cannot leak
	 * into the read side. In contrast, anything in the
	 * read-locked region is ok to be re-ordered into the
	 * write side. As such, rely on RELEASE semantics.
	 */
	tmp = atomic_long_add_return_release(-RWSEM_WAITING_BIAS, &sem->count);
	if (tmp < 0)
		rwsem_downgrade_wake(sem);
}

#endif	/* __KERNEL__ */
#endif	/* _ASM_GENERIC_RWSEM_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
e172800e WD	2	#ifndef _ASM_GENERIC_RWSEM_H
e172800e WD	3	#define _ASM_GENERIC_RWSEM_H
dd472da3 RK	4
	5	#ifndef _LINUX_RWSEM_H
	6	#error "Please don't include <asm/rwsem.h> directly, use <linux/rwsem.h> instead."
	7	#endif
	8
	9	#ifdef __KERNEL__
	10
	11	/*
e172800e	12	* R/W semaphores originally for PPC using the stuff in lib/rwsem.c.
dd472da3 RK	13	* Adapted largely from include/asm-i386/rwsem.h
	14	* by Paul Mackerras <paulus@samba.org>.
	15	*/
	16
	17	/*
	18	* the semaphore definition
	19	*/
e172800e	20	#ifdef CONFIG_64BIT
dd472da3 RK	21	# define RWSEM_ACTIVE_MASK 0xffffffffL
	22	#else
	23	# define RWSEM_ACTIVE_MASK 0x0000ffffL
	24	#endif
	25
	26	#define RWSEM_UNLOCKED_VALUE 0x00000000L
	27	#define RWSEM_ACTIVE_BIAS 0x00000001L
	28	#define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1)
	29	#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS
	30	#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
	31
	32	/*
	33	* lock for reading
	34	*/
	35	static inline void __down_read(struct rw_semaphore *sem)
	36	{
06321dd2	37	if (unlikely(atomic_long_inc_return_acquire(&sem->count) <= 0))
dd472da3 RK	38	rwsem_down_read_failed(sem);
	39	}
	40
76f8507f KT	41	static inline int __down_read_killable(struct rw_semaphore *sem)
	42	{
	43	if (unlikely(atomic_long_inc_return_acquire(&sem->count) <= 0)) {
	44	if (IS_ERR(rwsem_down_read_failed_killable(sem)))
	45	return -EINTR;
	46	}
	47
	48	return 0;
	49	}
	50
dd472da3 RK	51	static inline int __down_read_trylock(struct rw_semaphore *sem)
	52	{
	53	long tmp;
	54
8ee62b18 JL	55	while ((tmp = atomic_long_read(&sem->count)) >= 0) {
8ee62b18 JL	56	if (tmp == atomic_long_cmpxchg_acquire(&sem->count, tmp,
dd472da3 RK	57	tmp + RWSEM_ACTIVE_READ_BIAS)) {
	58	return 1;
	59	}
	60	}
	61	return 0;
	62	}
	63
	64	/*
	65	* lock for writing
	66	*/
f8e04d85	67	static inline void __down_write(struct rw_semaphore *sem)
dd472da3 RK	68	{
	69	long tmp;
	70
00eb4bab	71	tmp = atomic_long_add_return_acquire(RWSEM_ACTIVE_WRITE_BIAS,
06321dd2	72	&sem->count);
dd472da3 RK	73	if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
	74	rwsem_down_write_failed(sem);
	75	}
	76
d4799608 MH	77	static inline int __down_write_killable(struct rw_semaphore *sem)
	78	{
	79	long tmp;
	80
	81	tmp = atomic_long_add_return_acquire(RWSEM_ACTIVE_WRITE_BIAS,
06321dd2	82	&sem->count);
d4799608 MH	83	if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
	84	if (IS_ERR(rwsem_down_write_failed_killable(sem)))
	85	return -EINTR;
	86	return 0;
	87	}
	88
dd472da3 RK	89	static inline int __down_write_trylock(struct rw_semaphore *sem)
	90	{
	91	long tmp;
	92
8ee62b18	93	tmp = atomic_long_cmpxchg_acquire(&sem->count, RWSEM_UNLOCKED_VALUE,
dd472da3 RK	94	RWSEM_ACTIVE_WRITE_BIAS);
	95	return tmp == RWSEM_UNLOCKED_VALUE;
	96	}
	97
	98	/*
	99	* unlock after reading
	100	*/
	101	static inline void __up_read(struct rw_semaphore *sem)
	102	{
	103	long tmp;
	104
06321dd2	105	tmp = atomic_long_dec_return_release(&sem->count);
dd472da3 RK	106	if (unlikely(tmp < -1 && (tmp & RWSEM_ACTIVE_MASK) == 0))
	107	rwsem_wake(sem);
	108	}
	109
	110	/*
	111	* unlock after writing
	112	*/
	113	static inline void __up_write(struct rw_semaphore *sem)
	114	{
00eb4bab	115	if (unlikely(atomic_long_sub_return_release(RWSEM_ACTIVE_WRITE_BIAS,
06321dd2	116	&sem->count) < 0))
dd472da3 RK	117	rwsem_wake(sem);
	118	}
	119
dd472da3 RK	120	/*
	121	* downgrade write lock to read lock
	122	*/
	123	static inline void __downgrade_write(struct rw_semaphore *sem)
	124	{
	125	long tmp;
	126
00eb4bab DB	127	/*
	128	* When downgrading from exclusive to shared ownership,
	129	* anything inside the write-locked region cannot leak
	130	* into the read side. In contrast, anything in the
	131	* read-locked region is ok to be re-ordered into the
	132	* write side. As such, rely on RELEASE semantics.
	133	*/
06321dd2	134	tmp = atomic_long_add_return_release(-RWSEM_WAITING_BIAS, &sem->count);
dd472da3 RK	135	if (tmp < 0)
	136	rwsem_downgrade_wake(sem);
	137	}
	138
dd472da3	139	#endif /* __KERNEL__ */
e172800e	140	#endif /* _ASM_GENERIC_RWSEM_H */