[mirror_ubuntu-eoan-kernel.git] / kernel / rcu / rcu_segcblist.h

/* SPDX-License-Identifier: GPL-2.0+ */
/*
 * RCU segmented callback lists, internal-to-rcu header file
 *
 * Copyright IBM Corporation, 2017
 *
 * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
 */

#include <linux/rcu_segcblist.h>

/*
 * Account for the fact that a previously dequeued callback turned out
 * to be marked as lazy.
 */
static inline void rcu_cblist_dequeued_lazy(struct rcu_cblist *rclp)
{
	rclp->len_lazy--;
}

void rcu_cblist_init(struct rcu_cblist *rclp);
struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp);

/*
 * Is the specified rcu_segcblist structure empty?
 *
 * But careful!  The fact that the ->head field is NULL does not
 * necessarily imply that there are no callbacks associated with
 * this structure.  When callbacks are being invoked, they are
 * removed as a group.  If callback invocation must be preempted,
 * the remaining callbacks will be added back to the list.  Either
 * way, the counts are updated later.
 *
 * So it is often the case that rcu_segcblist_n_cbs() should be used
 * instead.
 */
static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp)
{
	return !rsclp->head;
}

/* Return number of callbacks in segmented callback list. */
static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp)
{
	return READ_ONCE(rsclp->len);
}

/* Return number of lazy callbacks in segmented callback list. */
static inline long rcu_segcblist_n_lazy_cbs(struct rcu_segcblist *rsclp)
{
	return rsclp->len_lazy;
}

/* Return number of lazy callbacks in segmented callback list. */
static inline long rcu_segcblist_n_nonlazy_cbs(struct rcu_segcblist *rsclp)
{
	return rsclp->len - rsclp->len_lazy;
}

/*
 * Is the specified rcu_segcblist enabled, for example, not corresponding
 * to an offline or callback-offloaded CPU?
 */
static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp)
{
	return !!rsclp->tails[RCU_NEXT_TAIL];
}

/*
 * Are all segments following the specified segment of the specified
 * rcu_segcblist structure empty of callbacks?  (The specified
 * segment might well contain callbacks.)
 */
static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg)
{
	return !*rsclp->tails[seg];
}

/*
 * Interim function to return rcu_segcblist head pointer.  Longer term, the
 * rcu_segcblist will be used more pervasively, removing the need for this
 * function.
 */
static inline struct rcu_head *rcu_segcblist_head(struct rcu_segcblist *rsclp)
{
	return rsclp->head;
}

/*
 * Interim function to return rcu_segcblist head pointer.  Longer term, the
 * rcu_segcblist will be used more pervasively, removing the need for this
 * function.
 */
static inline struct rcu_head **rcu_segcblist_tail(struct rcu_segcblist *rsclp)
{
	WARN_ON_ONCE(rcu_segcblist_empty(rsclp));
	return rsclp->tails[RCU_NEXT_TAIL];
}

void rcu_segcblist_init(struct rcu_segcblist *rsclp);
void rcu_segcblist_disable(struct rcu_segcblist *rsclp);
bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp);
bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp);
struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp);
struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp);
void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
			   struct rcu_head *rhp, bool lazy);
bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
			   struct rcu_head *rhp, bool lazy);
void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp,
				 struct rcu_cblist *rclp);
void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
				    struct rcu_cblist *rclp);
void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,
				    struct rcu_cblist *rclp);
void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
				struct rcu_cblist *rclp);
void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,
				   struct rcu_cblist *rclp);
void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
				   struct rcu_cblist *rclp);
void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq);
bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq);
void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,
			 struct rcu_segcblist *src_rsclp);
Commit	Line	Data
eb7935e4	1	/* SPDX-License-Identifier: GPL-2.0+ */
45753c5f	2	/*
98059b98	3	* RCU segmented callback lists, internal-to-rcu header file
45753c5f	4	*
45753c5f IM	5	* Copyright IBM Corporation, 2017
45753c5f IM	6	*
eb7935e4	7	* Authors: Paul E. McKenney <paulmck@linux.ibm.com>
45753c5f IM	8	*/
	9
	10	#include <linux/rcu_segcblist.h>
	11
45753c5f IM	12	/*
	13	* Account for the fact that a previously dequeued callback turned out
	14	* to be marked as lazy.
	15	*/
	16	static inline void rcu_cblist_dequeued_lazy(struct rcu_cblist *rclp)
	17	{
	18	rclp->len_lazy--;
	19	}
	20
98059b98	21	void rcu_cblist_init(struct rcu_cblist *rclp);
98059b98	22	struct rcu_head rcu_cblist_dequeue(struct rcu_cblist rclp);
45753c5f IM	23
	24	/*
	25	* Is the specified rcu_segcblist structure empty?
	26	*
	27	* But careful! The fact that the ->head field is NULL does not
	28	* necessarily imply that there are no callbacks associated with
	29	* this structure. When callbacks are being invoked, they are
	30	* removed as a group. If callback invocation must be preempted,
	31	* the remaining callbacks will be added back to the list. Either
	32	* way, the counts are updated later.
	33	*
	34	* So it is often the case that rcu_segcblist_n_cbs() should be used
	35	* instead.
	36	*/
	37	static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp)
	38	{
	39	return !rsclp->head;
	40	}
	41
	42	/* Return number of callbacks in segmented callback list. */
	43	static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp)
	44	{
	45	return READ_ONCE(rsclp->len);
	46	}
	47
	48	/* Return number of lazy callbacks in segmented callback list. */
	49	static inline long rcu_segcblist_n_lazy_cbs(struct rcu_segcblist *rsclp)
	50	{
	51	return rsclp->len_lazy;
	52	}
	53
	54	/* Return number of lazy callbacks in segmented callback list. */
	55	static inline long rcu_segcblist_n_nonlazy_cbs(struct rcu_segcblist *rsclp)
	56	{
	57	return rsclp->len - rsclp->len_lazy;
	58	}
	59
	60	/*
	61	* Is the specified rcu_segcblist enabled, for example, not corresponding
	62	* to an offline or callback-offloaded CPU?
	63	*/
	64	static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp)
	65	{
	66	return !!rsclp->tails[RCU_NEXT_TAIL];
	67	}
	68
45753c5f IM	69	/*
	70	* Are all segments following the specified segment of the specified
	71	* rcu_segcblist structure empty of callbacks? (The specified
	72	* segment might well contain callbacks.)
	73	*/
	74	static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg)
	75	{
	76	return !*rsclp->tails[seg];
	77	}
	78
45753c5f IM	79	/*
	80	* Interim function to return rcu_segcblist head pointer. Longer term, the
	81	* rcu_segcblist will be used more pervasively, removing the need for this
	82	* function.
	83	*/
	84	static inline struct rcu_head rcu_segcblist_head(struct rcu_segcblist rsclp)
	85	{
	86	return rsclp->head;
	87	}
	88
	89	/*
	90	* Interim function to return rcu_segcblist head pointer. Longer term, the
	91	* rcu_segcblist will be used more pervasively, removing the need for this
	92	* function.
	93	*/
	94	static inline struct rcu_head *rcu_segcblist_tail(struct rcu_segcblist rsclp)
	95	{
	96	WARN_ON_ONCE(rcu_segcblist_empty(rsclp));
	97	return rsclp->tails[RCU_NEXT_TAIL];
	98	}
98059b98 PM	99
	100	void rcu_segcblist_init(struct rcu_segcblist *rsclp);
	101	void rcu_segcblist_disable(struct rcu_segcblist *rsclp);
98059b98 PM	102	bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp);
98059b98 PM	103	bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp);
98059b98 PM	104	struct rcu_head rcu_segcblist_first_cb(struct rcu_segcblist rsclp);
98059b98 PM	105	struct rcu_head rcu_segcblist_first_pend_cb(struct rcu_segcblist rsclp);
98059b98 PM	106	void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
	107	struct rcu_head *rhp, bool lazy);
	108	bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
	109	struct rcu_head *rhp, bool lazy);
	110	void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp,
	111	struct rcu_cblist *rclp);
	112	void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
	113	struct rcu_cblist *rclp);
	114	void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,
	115	struct rcu_cblist *rclp);
	116	void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
	117	struct rcu_cblist *rclp);
	118	void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,
	119	struct rcu_cblist *rclp);
	120	void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
	121	struct rcu_cblist *rclp);
	122	void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq);
	123	bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq);
f2dbe4a5 PM	124	void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,
f2dbe4a5 PM	125	struct rcu_segcblist *src_rsclp);