[mirror_ubuntu-focal-kernel.git] / lib / lockref.c

#include <linux/export.h>
#include <linux/lockref.h>

#if USE_CMPXCHG_LOCKREF

/*
 * Allow weakly-ordered memory architectures to provide barrier-less
 * cmpxchg semantics for lockref updates.
 */
#ifndef cmpxchg64_relaxed
# define cmpxchg64_relaxed cmpxchg64
#endif

/*
 * Note that the "cmpxchg()" reloads the "old" value for the
 * failure case.
 */
#define CMPXCHG_LOOP(CODE, SUCCESS) do {					\
	struct lockref old;							\
	BUILD_BUG_ON(sizeof(old) != 8);						\
	old.lock_count = ACCESS_ONCE(lockref->lock_count);			\
	while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) {  	\
		struct lockref new = old, prev = old;				\
		CODE								\
		old.lock_count = cmpxchg64_relaxed(&lockref->lock_count,	\
						   old.lock_count,		\
						   new.lock_count);		\
		if (likely(old.lock_count == prev.lock_count)) {		\
			SUCCESS;						\
		}								\
		cpu_relax_lowlatency();						\
	}									\
} while (0)

#else

#define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)

#endif

/**
 * lockref_get - Increments reference count unconditionally
 * @lockref: pointer to lockref structure
 *
 * This operation is only valid if you already hold a reference
 * to the object, so you know the count cannot be zero.
 */
void lockref_get(struct lockref *lockref)
{
	CMPXCHG_LOOP(
		new.count++;
	,
		return;
	);

	spin_lock(&lockref->lock);
	lockref->count++;
	spin_unlock(&lockref->lock);
}
EXPORT_SYMBOL(lockref_get);

/**
 * lockref_get_not_zero - Increments count unless the count is 0
 * @lockref: pointer to lockref structure
 * Return: 1 if count updated successfully or 0 if count was zero
 */
int lockref_get_not_zero(struct lockref *lockref)
{
	int retval;

	CMPXCHG_LOOP(
		new.count++;
		if (!old.count)
			return 0;
	,
		return 1;
	);

	spin_lock(&lockref->lock);
	retval = 0;
	if (lockref->count) {
		lockref->count++;
		retval = 1;
	}
	spin_unlock(&lockref->lock);
	return retval;
}
EXPORT_SYMBOL(lockref_get_not_zero);

/**
 * lockref_get_or_lock - Increments count unless the count is 0
 * @lockref: pointer to lockref structure
 * Return: 1 if count updated successfully or 0 if count was zero
 * and we got the lock instead.
 */
int lockref_get_or_lock(struct lockref *lockref)
{
	CMPXCHG_LOOP(
		new.count++;
		if (!old.count)
			break;
	,
		return 1;
	);

	spin_lock(&lockref->lock);
	if (!lockref->count)
		return 0;
	lockref->count++;
	spin_unlock(&lockref->lock);
	return 1;
}
EXPORT_SYMBOL(lockref_get_or_lock);

/**
 * lockref_put_or_lock - decrements count unless count <= 1 before decrement
 * @lockref: pointer to lockref structure
 * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
 */
int lockref_put_or_lock(struct lockref *lockref)
{
	CMPXCHG_LOOP(
		new.count--;
		if (old.count <= 1)
			break;
	,
		return 1;
	);

	spin_lock(&lockref->lock);
	if (lockref->count <= 1)
		return 0;
	lockref->count--;
	spin_unlock(&lockref->lock);
	return 1;
}
EXPORT_SYMBOL(lockref_put_or_lock);

/**
 * lockref_mark_dead - mark lockref dead
 * @lockref: pointer to lockref structure
 */
void lockref_mark_dead(struct lockref *lockref)
{
	assert_spin_locked(&lockref->lock);
	lockref->count = -128;
}
EXPORT_SYMBOL(lockref_mark_dead);

/**
 * lockref_get_not_dead - Increments count unless the ref is dead
 * @lockref: pointer to lockref structure
 * Return: 1 if count updated successfully or 0 if lockref was dead
 */
int lockref_get_not_dead(struct lockref *lockref)
{
	int retval;

	CMPXCHG_LOOP(
		new.count++;
		if ((int)old.count < 0)
			return 0;
	,
		return 1;
	);

	spin_lock(&lockref->lock);
	retval = 0;
	if ((int) lockref->count >= 0) {
		lockref->count++;
		retval = 1;
	}
	spin_unlock(&lockref->lock);
	return retval;
}
EXPORT_SYMBOL(lockref_get_not_dead);
Commit	Line	Data
2f4f12e5 LT	1	#include <linux/export.h>
	2	#include <linux/lockref.h>
	3
57f4257e	4	#if USE_CMPXCHG_LOCKREF
bc08b449	5
d2212b4d WD	6	/*
	7	* Allow weakly-ordered memory architectures to provide barrier-less
	8	* cmpxchg semantics for lockref updates.
	9	*/
	10	#ifndef cmpxchg64_relaxed
	11	# define cmpxchg64_relaxed cmpxchg64
	12	#endif
	13
bc08b449 LT	14	/*
	15	* Note that the "cmpxchg()" reloads the "old" value for the
	16	* failure case.
	17	*/
	18	#define CMPXCHG_LOOP(CODE, SUCCESS) do { \
	19	struct lockref old; \
	20	BUILD_BUG_ON(sizeof(old) != 8); \
	21	old.lock_count = ACCESS_ONCE(lockref->lock_count); \
	22	while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \
	23	struct lockref new = old, prev = old; \
	24	CODE \
d2212b4d WD	25	old.lock_count = cmpxchg64_relaxed(&lockref->lock_count, \
	26	old.lock_count, \
	27	new.lock_count); \
bc08b449 LT	28	if (likely(old.lock_count == prev.lock_count)) { \
	29	SUCCESS; \
	30	} \
3a6bfbc9	31	cpu_relax_lowlatency(); \
bc08b449 LT	32	} \
	33	} while (0)
	34
	35	#else
	36
	37	#define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
	38
	39	#endif
	40
2f4f12e5 LT	41	/**
2f4f12e5 LT	42	* lockref_get - Increments reference count unconditionally
44a0cf92	43	* @lockref: pointer to lockref structure
2f4f12e5 LT	44	*
	45	* This operation is only valid if you already hold a reference
	46	* to the object, so you know the count cannot be zero.
	47	*/
	48	void lockref_get(struct lockref *lockref)
	49	{
bc08b449 LT	50	CMPXCHG_LOOP(
	51	new.count++;
	52	,
	53	return;
	54	);
	55
2f4f12e5 LT	56	spin_lock(&lockref->lock);
	57	lockref->count++;
	58	spin_unlock(&lockref->lock);
	59	}
	60	EXPORT_SYMBOL(lockref_get);
	61
	62	/**
	63	* lockref_get_not_zero - Increments count unless the count is 0
44a0cf92	64	* @lockref: pointer to lockref structure
2f4f12e5 LT	65	* Return: 1 if count updated successfully or 0 if count was zero
	66	*/
	67	int lockref_get_not_zero(struct lockref *lockref)
	68	{
bc08b449 LT	69	int retval;
	70
	71	CMPXCHG_LOOP(
	72	new.count++;
	73	if (!old.count)
	74	return 0;
	75	,
	76	return 1;
	77	);
2f4f12e5 LT	78
2f4f12e5 LT	79	spin_lock(&lockref->lock);
bc08b449	80	retval = 0;
2f4f12e5 LT	81	if (lockref->count) {
	82	lockref->count++;
	83	retval = 1;
	84	}
	85	spin_unlock(&lockref->lock);
	86	return retval;
	87	}
	88	EXPORT_SYMBOL(lockref_get_not_zero);
	89
	90	/**
	91	* lockref_get_or_lock - Increments count unless the count is 0
44a0cf92	92	* @lockref: pointer to lockref structure
2f4f12e5 LT	93	* Return: 1 if count updated successfully or 0 if count was zero
	94	* and we got the lock instead.
	95	*/
	96	int lockref_get_or_lock(struct lockref *lockref)
	97	{
bc08b449 LT	98	CMPXCHG_LOOP(
	99	new.count++;
	100	if (!old.count)
	101	break;
	102	,
	103	return 1;
	104	);
	105
2f4f12e5 LT	106	spin_lock(&lockref->lock);
	107	if (!lockref->count)
	108	return 0;
	109	lockref->count++;
	110	spin_unlock(&lockref->lock);
	111	return 1;
	112	}
	113	EXPORT_SYMBOL(lockref_get_or_lock);
	114
	115	/**
	116	* lockref_put_or_lock - decrements count unless count <= 1 before decrement
44a0cf92	117	* @lockref: pointer to lockref structure
2f4f12e5 LT	118	* Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
	119	*/
	120	int lockref_put_or_lock(struct lockref *lockref)
	121	{
bc08b449 LT	122	CMPXCHG_LOOP(
	123	new.count--;
	124	if (old.count <= 1)
	125	break;
	126	,
	127	return 1;
	128	);
	129
2f4f12e5 LT	130	spin_lock(&lockref->lock);
	131	if (lockref->count <= 1)
	132	return 0;
	133	lockref->count--;
	134	spin_unlock(&lockref->lock);
	135	return 1;
	136	}
	137	EXPORT_SYMBOL(lockref_put_or_lock);
e7d33bb5 LT	138
	139	/**
	140	* lockref_mark_dead - mark lockref dead
	141	* @lockref: pointer to lockref structure
	142	*/
	143	void lockref_mark_dead(struct lockref *lockref)
	144	{
	145	assert_spin_locked(&lockref->lock);
	146	lockref->count = -128;
	147	}
e66cf161	148	EXPORT_SYMBOL(lockref_mark_dead);
e7d33bb5 LT	149
	150	/**
	151	* lockref_get_not_dead - Increments count unless the ref is dead
	152	* @lockref: pointer to lockref structure
	153	* Return: 1 if count updated successfully or 0 if lockref was dead
	154	*/
	155	int lockref_get_not_dead(struct lockref *lockref)
	156	{
	157	int retval;
	158
	159	CMPXCHG_LOOP(
	160	new.count++;
	161	if ((int)old.count < 0)
	162	return 0;
	163	,
	164	return 1;
	165	);
	166
	167	spin_lock(&lockref->lock);
	168	retval = 0;
	169	if ((int) lockref->count >= 0) {
	170	lockref->count++;
	171	retval = 1;
	172	}
	173	spin_unlock(&lockref->lock);
	174	return retval;
	175	}
	176	EXPORT_SYMBOL(lockref_get_not_dead);