[mirror_ubuntu-hirsute-kernel.git] / kernel / rcu / rcu_segcblist.h

/* SPDX-License-Identifier: GPL-2.0+ */
/*
 * RCU segmented callback lists, internal-to-rcu header file
 *
 * Copyright IBM Corporation, 2017
 *
 * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
 */

#include <linux/rcu_segcblist.h>

/* Return number of callbacks in the specified callback list. */
static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp)
{
	return READ_ONCE(rclp->len);
}

/*
 * Account for the fact that a previously dequeued callback turned out
 * to be marked as lazy.
 */
static inline void rcu_cblist_dequeued_lazy(struct rcu_cblist *rclp)
{
	rclp->len_lazy--;
}

void rcu_cblist_init(struct rcu_cblist *rclp);
void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp);
void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp,
			      struct rcu_cblist *srclp,
			      struct rcu_head *rhp);
struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp);

/*
 * Is the specified rcu_segcblist structure empty?
 *
 * But careful!  The fact that the ->head field is NULL does not
 * necessarily imply that there are no callbacks associated with
 * this structure.  When callbacks are being invoked, they are
 * removed as a group.  If callback invocation must be preempted,
 * the remaining callbacks will be added back to the list.  Either
 * way, the counts are updated later.
 *
 * So it is often the case that rcu_segcblist_n_cbs() should be used
 * instead.
 */
static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp)
{
	return !READ_ONCE(rsclp->head);
}

/* Return number of callbacks in segmented callback list. */
static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp)
{
#ifdef CONFIG_RCU_NOCB_CPU
	return atomic_long_read(&rsclp->len);
#else
	return READ_ONCE(rsclp->len);
#endif
}

/* Return number of lazy callbacks in segmented callback list. */
static inline long rcu_segcblist_n_lazy_cbs(struct rcu_segcblist *rsclp)
{
	return rsclp->len_lazy;
}

/* Return number of lazy callbacks in segmented callback list. */
static inline long rcu_segcblist_n_nonlazy_cbs(struct rcu_segcblist *rsclp)
{
	return rcu_segcblist_n_cbs(rsclp) - rsclp->len_lazy;
}

/*
 * Is the specified rcu_segcblist enabled, for example, not corresponding
 * to an offline CPU?
 */
static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp)
{
	return rsclp->enabled;
}

/* Is the specified rcu_segcblist offloaded?  */
static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp)
{
	return rsclp->offloaded;
}

/*
 * Are all segments following the specified segment of the specified
 * rcu_segcblist structure empty of callbacks?  (The specified
 * segment might well contain callbacks.)
 */
static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg)
{
	return !READ_ONCE(*READ_ONCE(rsclp->tails[seg]));
}

void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp);
void rcu_segcblist_init(struct rcu_segcblist *rsclp);
void rcu_segcblist_disable(struct rcu_segcblist *rsclp);
void rcu_segcblist_offload(struct rcu_segcblist *rsclp);
bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp);
bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp);
struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp);
struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp);
bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp);
void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
			   struct rcu_head *rhp, bool lazy);
bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
			   struct rcu_head *rhp, bool lazy);
void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp,
				 struct rcu_cblist *rclp);
void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
				    struct rcu_cblist *rclp);
void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,
				    struct rcu_cblist *rclp);
void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
				struct rcu_cblist *rclp);
void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,
				   struct rcu_cblist *rclp);
void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
				   struct rcu_cblist *rclp);
void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq);
bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq);
void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,
			 struct rcu_segcblist *src_rsclp);
Commit	Line	Data
eb7935e4	1	/* SPDX-License-Identifier: GPL-2.0+ */
45753c5f	2	/*
98059b98	3	* RCU segmented callback lists, internal-to-rcu header file
45753c5f	4	*
45753c5f IM	5	* Copyright IBM Corporation, 2017
45753c5f IM	6	*
eb7935e4	7	* Authors: Paul E. McKenney <paulmck@linux.ibm.com>
45753c5f IM	8	*/
	9
	10	#include <linux/rcu_segcblist.h>
	11
eda669a6 PM	12	/* Return number of callbacks in the specified callback list. */
	13	static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp)
	14	{
	15	return READ_ONCE(rclp->len);
	16	}
	17
45753c5f IM	18	/*
	19	* Account for the fact that a previously dequeued callback turned out
	20	* to be marked as lazy.
	21	*/
	22	static inline void rcu_cblist_dequeued_lazy(struct rcu_cblist *rclp)
	23	{
	24	rclp->len_lazy--;
	25	}
	26
98059b98	27	void rcu_cblist_init(struct rcu_cblist *rclp);
d1b222c6 PM	28	void rcu_cblist_enqueue(struct rcu_cblist rclp, struct rcu_head rhp);
	29	void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp,
	30	struct rcu_cblist *srclp,
	31	struct rcu_head *rhp);
98059b98	32	struct rcu_head rcu_cblist_dequeue(struct rcu_cblist rclp);
45753c5f IM	33
	34	/*
	35	* Is the specified rcu_segcblist structure empty?
	36	*
	37	* But careful! The fact that the ->head field is NULL does not
	38	* necessarily imply that there are no callbacks associated with
	39	* this structure. When callbacks are being invoked, they are
	40	* removed as a group. If callback invocation must be preempted,
	41	* the remaining callbacks will be added back to the list. Either
	42	* way, the counts are updated later.
	43	*
	44	* So it is often the case that rcu_segcblist_n_cbs() should be used
	45	* instead.
	46	*/
	47	static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp)
	48	{
e6060b41	49	return !READ_ONCE(rsclp->head);
45753c5f IM	50	}
	51
	52	/* Return number of callbacks in segmented callback list. */
	53	static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp)
	54	{
eda669a6 PM	55	#ifdef CONFIG_RCU_NOCB_CPU
	56	return atomic_long_read(&rsclp->len);
	57	#else
45753c5f	58	return READ_ONCE(rsclp->len);
eda669a6	59	#endif
45753c5f IM	60	}
	61
	62	/* Return number of lazy callbacks in segmented callback list. */
	63	static inline long rcu_segcblist_n_lazy_cbs(struct rcu_segcblist *rsclp)
	64	{
	65	return rsclp->len_lazy;
	66	}
	67
	68	/* Return number of lazy callbacks in segmented callback list. */
	69	static inline long rcu_segcblist_n_nonlazy_cbs(struct rcu_segcblist *rsclp)
	70	{
eda669a6	71	return rcu_segcblist_n_cbs(rsclp) - rsclp->len_lazy;
45753c5f IM	72	}
	73
	74	/*
	75	* Is the specified rcu_segcblist enabled, for example, not corresponding
e83e73f5	76	* to an offline CPU?
45753c5f IM	77	*/
	78	static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp)
	79	{
1bb5f9b9	80	return rsclp->enabled;
45753c5f IM	81	}
45753c5f IM	82
ce5215c1 PM	83	/* Is the specified rcu_segcblist offloaded? */
	84	static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp)
	85	{
	86	return rsclp->offloaded;
	87	}
	88
45753c5f IM	89	/*
	90	* Are all segments following the specified segment of the specified
	91	* rcu_segcblist structure empty of callbacks? (The specified
	92	* segment might well contain callbacks.)
	93	*/
	94	static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg)
	95	{
76c6927c	96	return !READ_ONCE(*READ_ONCE(rsclp->tails[seg]));
45753c5f IM	97	}
45753c5f IM	98
d1b222c6	99	void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp);
98059b98 PM	100	void rcu_segcblist_init(struct rcu_segcblist *rsclp);
98059b98 PM	101	void rcu_segcblist_disable(struct rcu_segcblist *rsclp);
ce5215c1	102	void rcu_segcblist_offload(struct rcu_segcblist *rsclp);
98059b98 PM	103	bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp);
98059b98 PM	104	bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp);
98059b98 PM	105	struct rcu_head rcu_segcblist_first_cb(struct rcu_segcblist rsclp);
98059b98 PM	106	struct rcu_head rcu_segcblist_first_pend_cb(struct rcu_segcblist rsclp);
5d6742b3	107	bool rcu_segcblist_nextgp(struct rcu_segcblist rsclp, unsigned long lp);
98059b98 PM	108	void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
	109	struct rcu_head *rhp, bool lazy);
	110	bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
	111	struct rcu_head *rhp, bool lazy);
	112	void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp,
	113	struct rcu_cblist *rclp);
	114	void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
	115	struct rcu_cblist *rclp);
	116	void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,
	117	struct rcu_cblist *rclp);
	118	void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
	119	struct rcu_cblist *rclp);
	120	void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,
	121	struct rcu_cblist *rclp);
	122	void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
	123	struct rcu_cblist *rclp);
	124	void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq);
	125	bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq);
f2dbe4a5 PM	126	void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,
f2dbe4a5 PM	127	struct rcu_segcblist *src_rsclp);