[mirror_ubuntu-bionic-kernel.git] / include / linux / radix-tree.h

/*
 * Copyright (C) 2001 Momchil Velikov
 * Portions Copyright (C) 2001 Christoph Hellwig
 * Copyright (C) 2006 Nick Piggin
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2, or (at
 * your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */
#ifndef _LINUX_RADIX_TREE_H
#define _LINUX_RADIX_TREE_H

#include <linux/preempt.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/rcupdate.h>

/*
 * An indirect pointer (root->rnode pointing to a radix_tree_node, rather
 * than a data item) is signalled by the low bit set in the root->rnode
 * pointer.
 *
 * In this case root->height is > 0, but the indirect pointer tests are
 * needed for RCU lookups (because root->height is unreliable). The only
 * time callers need worry about this is when doing a lookup_slot under
 * RCU.
 *
 * Indirect pointer in fact is also used to tag the last pointer of a node
 * when it is shrunk, before we rcu free the node. See shrink code for
 * details.
 */
#define RADIX_TREE_INDIRECT_PTR		1
/*
 * A common use of the radix tree is to store pointers to struct pages;
 * but shmem/tmpfs needs also to store swap entries in the same tree:
 * those are marked as exceptional entries to distinguish them.
 * EXCEPTIONAL_ENTRY tests the bit, EXCEPTIONAL_SHIFT shifts content past it.
 */
#define RADIX_TREE_EXCEPTIONAL_ENTRY	2
#define RADIX_TREE_EXCEPTIONAL_SHIFT	2

static inline int radix_tree_is_indirect_ptr(void *ptr)
{
	return (int)((unsigned long)ptr & RADIX_TREE_INDIRECT_PTR);
}

/*** radix-tree API starts here ***/

#define RADIX_TREE_MAX_TAGS 3

/* root tags are stored in gfp_mask, shifted by __GFP_BITS_SHIFT */
struct radix_tree_root {
	unsigned int		height;
	gfp_t			gfp_mask;
	struct radix_tree_node	__rcu *rnode;
};

#define RADIX_TREE_INIT(mask)	{					\
	.height = 0,							\
	.gfp_mask = (mask),						\
	.rnode = NULL,							\
}

#define RADIX_TREE(name, mask) \
	struct radix_tree_root name = RADIX_TREE_INIT(mask)

#define INIT_RADIX_TREE(root, mask)					\
do {									\
	(root)->height = 0;						\
	(root)->gfp_mask = (mask);					\
	(root)->rnode = NULL;						\
} while (0)

/**
 * Radix-tree synchronization
 *
 * The radix-tree API requires that users provide all synchronisation (with
 * specific exceptions, noted below).
 *
 * Synchronization of access to the data items being stored in the tree, and
 * management of their lifetimes must be completely managed by API users.
 *
 * For API usage, in general,
 * - any function _modifying_ the tree or tags (inserting or deleting
 *   items, setting or clearing tags) must exclude other modifications, and
 *   exclude any functions reading the tree.
 * - any function _reading_ the tree or tags (looking up items or tags,
 *   gang lookups) must exclude modifications to the tree, but may occur
 *   concurrently with other readers.
 *
 * The notable exceptions to this rule are the following functions:
 * radix_tree_lookup
 * radix_tree_lookup_slot
 * radix_tree_tag_get
 * radix_tree_gang_lookup
 * radix_tree_gang_lookup_slot
 * radix_tree_gang_lookup_tag
 * radix_tree_gang_lookup_tag_slot
 * radix_tree_tagged
 *
 * The first 7 functions are able to be called locklessly, using RCU. The
 * caller must ensure calls to these functions are made within rcu_read_lock()
 * regions. Other readers (lock-free or otherwise) and modifications may be
 * running concurrently.
 *
 * It is still required that the caller manage the synchronization and lifetimes
 * of the items. So if RCU lock-free lookups are used, typically this would mean
 * that the items have their own locks, or are amenable to lock-free access; and
 * that the items are freed by RCU (or only freed after having been deleted from
 * the radix tree *and* a synchronize_rcu() grace period).
 *
 * (Note, rcu_assign_pointer and rcu_dereference are not needed to control
 * access to data items when inserting into or looking up from the radix tree)
 *
 * Note that the value returned by radix_tree_tag_get() may not be relied upon
 * if only the RCU read lock is held.  Functions to set/clear tags and to
 * delete nodes running concurrently with it may affect its result such that
 * two consecutive reads in the same locked section may return different
 * values.  If reliability is required, modification functions must also be
 * excluded from concurrency.
 *
 * radix_tree_tagged is able to be called without locking or RCU.
 */

/**
 * radix_tree_deref_slot	- dereference a slot
 * @pslot:	pointer to slot, returned by radix_tree_lookup_slot
 * Returns:	item that was stored in that slot with any direct pointer flag
 *		removed.
 *
 * For use with radix_tree_lookup_slot().  Caller must hold tree at least read
 * locked across slot lookup and dereference. Not required if write lock is
 * held (ie. items cannot be concurrently inserted).
 *
 * radix_tree_deref_retry must be used to confirm validity of the pointer if
 * only the read lock is held.
 */
static inline void *radix_tree_deref_slot(void **pslot)
{
	return rcu_dereference(*pslot);
}

/**
 * radix_tree_deref_slot_protected	- dereference a slot without RCU lock but with tree lock held
 * @pslot:	pointer to slot, returned by radix_tree_lookup_slot
 * Returns:	item that was stored in that slot with any direct pointer flag
 *		removed.
 *
 * Similar to radix_tree_deref_slot but only used during migration when a pages
 * mapping is being moved. The caller does not hold the RCU read lock but it
 * must hold the tree lock to prevent parallel updates.
 */
static inline void *radix_tree_deref_slot_protected(void **pslot,
							spinlock_t *treelock)
{
	return rcu_dereference_protected(*pslot, lockdep_is_held(treelock));
}

/**
 * radix_tree_deref_retry	- check radix_tree_deref_slot
 * @arg:	pointer returned by radix_tree_deref_slot
 * Returns:	0 if retry is not required, otherwise retry is required
 *
 * radix_tree_deref_retry must be used with radix_tree_deref_slot.
 */
static inline int radix_tree_deref_retry(void *arg)
{
	return unlikely((unsigned long)arg & RADIX_TREE_INDIRECT_PTR);
}

/**
 * radix_tree_exceptional_entry	- radix_tree_deref_slot gave exceptional entry?
 * @arg:	value returned by radix_tree_deref_slot
 * Returns:	0 if well-aligned pointer, non-0 if exceptional entry.
 */
static inline int radix_tree_exceptional_entry(void *arg)
{
	/* Not unlikely because radix_tree_exception often tested first */
	return (unsigned long)arg & RADIX_TREE_EXCEPTIONAL_ENTRY;
}

/**
 * radix_tree_exception	- radix_tree_deref_slot returned either exception?
 * @arg:	value returned by radix_tree_deref_slot
 * Returns:	0 if well-aligned pointer, non-0 if either kind of exception.
 */
static inline int radix_tree_exception(void *arg)
{
	return unlikely((unsigned long)arg &
		(RADIX_TREE_INDIRECT_PTR | RADIX_TREE_EXCEPTIONAL_ENTRY));
}

/**
 * radix_tree_replace_slot	- replace item in a slot
 * @pslot:	pointer to slot, returned by radix_tree_lookup_slot
 * @item:	new item to store in the slot.
 *
 * For use with radix_tree_lookup_slot().  Caller must hold tree write locked
 * across slot lookup and replacement.
 */
static inline void radix_tree_replace_slot(void **pslot, void *item)
{
	BUG_ON(radix_tree_is_indirect_ptr(item));
	rcu_assign_pointer(*pslot, item);
}

int radix_tree_insert(struct radix_tree_root *, unsigned long, void *);
void *radix_tree_lookup(struct radix_tree_root *, unsigned long);
void **radix_tree_lookup_slot(struct radix_tree_root *, unsigned long);
void *radix_tree_delete(struct radix_tree_root *, unsigned long);
unsigned int
radix_tree_gang_lookup(struct radix_tree_root *root, void **results,
			unsigned long first_index, unsigned int max_items);
unsigned int radix_tree_gang_lookup_slot(struct radix_tree_root *root,
			void ***results, unsigned long *indices,
			unsigned long first_index, unsigned int max_items);
unsigned long radix_tree_next_hole(struct radix_tree_root *root,
				unsigned long index, unsigned long max_scan);
unsigned long radix_tree_prev_hole(struct radix_tree_root *root,
				unsigned long index, unsigned long max_scan);
int radix_tree_preload(gfp_t gfp_mask);
void radix_tree_init(void);
void *radix_tree_tag_set(struct radix_tree_root *root,
			unsigned long index, unsigned int tag);
void *radix_tree_tag_clear(struct radix_tree_root *root,
			unsigned long index, unsigned int tag);
int radix_tree_tag_get(struct radix_tree_root *root,
			unsigned long index, unsigned int tag);
unsigned int
radix_tree_gang_lookup_tag(struct radix_tree_root *root, void **results,
		unsigned long first_index, unsigned int max_items,
		unsigned int tag);
unsigned int
radix_tree_gang_lookup_tag_slot(struct radix_tree_root *root, void ***results,
		unsigned long first_index, unsigned int max_items,
		unsigned int tag);
unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root,
		unsigned long *first_indexp, unsigned long last_index,
		unsigned long nr_to_tag,
		unsigned int fromtag, unsigned int totag);
int radix_tree_tagged(struct radix_tree_root *root, unsigned int tag);
unsigned long radix_tree_locate_item(struct radix_tree_root *root, void *item);

static inline void radix_tree_preload_end(void)
{
	preempt_enable();
}

#endif /* _LINUX_RADIX_TREE_H */
Commit	Line	Data
1da177e4 LT	1	/*
	2	* Copyright (C) 2001 Momchil Velikov
	3	* Portions Copyright (C) 2001 Christoph Hellwig
7cf9c2c7	4	* Copyright (C) 2006 Nick Piggin
1da177e4 LT	5	*
	6	* This program is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU General Public License as
	8	* published by the Free Software Foundation; either version 2, or (at
	9	* your option) any later version.
	10	*
	11	* This program is distributed in the hope that it will be useful, but
	12	* WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	* General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU General Public License
	17	* along with this program; if not, write to the Free Software
	18	* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
	19	*/
	20	#ifndef _LINUX_RADIX_TREE_H
	21	#define _LINUX_RADIX_TREE_H
	22
	23	#include <linux/preempt.h>
	24	#include <linux/types.h>
7cf9c2c7 NP	25	#include <linux/kernel.h>
	26	#include <linux/rcupdate.h>
	27
	28	/*
c0bc9875 NP	29	* An indirect pointer (root->rnode pointing to a radix_tree_node, rather
	30	* than a data item) is signalled by the low bit set in the root->rnode
	31	* pointer.
7cf9c2c7	32	*
c0bc9875 NP	33	* In this case root->height is > 0, but the indirect pointer tests are
	34	* needed for RCU lookups (because root->height is unreliable). The only
	35	* time callers need worry about this is when doing a lookup_slot under
	36	* RCU.
27d20fdd NP	37	*
	38	* Indirect pointer in fact is also used to tag the last pointer of a node
	39	* when it is shrunk, before we rcu free the node. See shrink code for
	40	* details.
7cf9c2c7	41	*/
6328650b HD	42	#define RADIX_TREE_INDIRECT_PTR 1
	43	/*
	44	* A common use of the radix tree is to store pointers to struct pages;
	45	* but shmem/tmpfs needs also to store swap entries in the same tree:
	46	* those are marked as exceptional entries to distinguish them.
	47	* EXCEPTIONAL_ENTRY tests the bit, EXCEPTIONAL_SHIFT shifts content past it.
	48	*/
	49	#define RADIX_TREE_EXCEPTIONAL_ENTRY 2
	50	#define RADIX_TREE_EXCEPTIONAL_SHIFT 2
7cf9c2c7	51
c0bc9875	52	static inline int radix_tree_is_indirect_ptr(void *ptr)
7cf9c2c7	53	{
c0bc9875	54	return (int)((unsigned long)ptr & RADIX_TREE_INDIRECT_PTR);
7cf9c2c7 NP	55	}
	56
	57	/* radix-tree API starts here */
1da177e4	58
f446daae	59	#define RADIX_TREE_MAX_TAGS 3
612d6c19 NP	60
612d6c19 NP	61	/* root tags are stored in gfp_mask, shifted by __GFP_BITS_SHIFT */
1da177e4 LT	62	struct radix_tree_root {
1da177e4 LT	63	unsigned int height;
fd4f2df2	64	gfp_t gfp_mask;
a1115570	65	struct radix_tree_node __rcu *rnode;
1da177e4 LT	66	};
	67
	68	#define RADIX_TREE_INIT(mask) { \
	69	.height = 0, \
	70	.gfp_mask = (mask), \
	71	.rnode = NULL, \
	72	}
	73
	74	#define RADIX_TREE(name, mask) \
	75	struct radix_tree_root name = RADIX_TREE_INIT(mask)
	76
	77	#define INIT_RADIX_TREE(root, mask) \
	78	do { \
	79	(root)->height = 0; \
	80	(root)->gfp_mask = (mask); \
	81	(root)->rnode = NULL; \
	82	} while (0)
	83
7cf9c2c7 NP	84	/**
	85	* Radix-tree synchronization
	86	*
	87	* The radix-tree API requires that users provide all synchronisation (with
	88	* specific exceptions, noted below).
	89	*
	90	* Synchronization of access to the data items being stored in the tree, and
	91	* management of their lifetimes must be completely managed by API users.
	92	*
	93	* For API usage, in general,
59c51591	94	* - any function _modifying_ the tree or tags (inserting or deleting
eb8dc5e7	95	* items, setting or clearing tags) must exclude other modifications, and
7cf9c2c7	96	* exclude any functions reading the tree.
59c51591	97	* - any function _reading_ the tree or tags (looking up items or tags,
7cf9c2c7 NP	98	* gang lookups) must exclude modifications to the tree, but may occur
	99	* concurrently with other readers.
	100	*
	101	* The notable exceptions to this rule are the following functions:
	102	* radix_tree_lookup
47feff2c	103	* radix_tree_lookup_slot
7cf9c2c7 NP	104	* radix_tree_tag_get
7cf9c2c7 NP	105	* radix_tree_gang_lookup
47feff2c	106	* radix_tree_gang_lookup_slot
7cf9c2c7	107	* radix_tree_gang_lookup_tag
47feff2c	108	* radix_tree_gang_lookup_tag_slot
7cf9c2c7 NP	109	* radix_tree_tagged
7cf9c2c7 NP	110	*
47feff2c	111	* The first 7 functions are able to be called locklessly, using RCU. The
7cf9c2c7 NP	112	* caller must ensure calls to these functions are made within rcu_read_lock()
	113	* regions. Other readers (lock-free or otherwise) and modifications may be
	114	* running concurrently.
	115	*
	116	* It is still required that the caller manage the synchronization and lifetimes
	117	* of the items. So if RCU lock-free lookups are used, typically this would mean
	118	* that the items have their own locks, or are amenable to lock-free access; and
	119	* that the items are freed by RCU (or only freed after having been deleted from
	120	* the radix tree and a synchronize_rcu() grace period).
	121	*
	122	* (Note, rcu_assign_pointer and rcu_dereference are not needed to control
	123	* access to data items when inserting into or looking up from the radix tree)
	124	*
ce82653d DH	125	* Note that the value returned by radix_tree_tag_get() may not be relied upon
	126	* if only the RCU read lock is held. Functions to set/clear tags and to
	127	* delete nodes running concurrently with it may affect its result such that
	128	* two consecutive reads in the same locked section may return different
	129	* values. If reliability is required, modification functions must also be
	130	* excluded from concurrency.
	131	*
7cf9c2c7 NP	132	* radix_tree_tagged is able to be called without locking or RCU.
	133	*/
	134
	135	/**
	136	* radix_tree_deref_slot - dereference a slot
	137	* @pslot: pointer to slot, returned by radix_tree_lookup_slot
	138	* Returns: item that was stored in that slot with any direct pointer flag
	139	* removed.
	140	*
	141	* For use with radix_tree_lookup_slot(). Caller must hold tree at least read
27d20fdd NP	142	* locked across slot lookup and dereference. Not required if write lock is
	143	* held (ie. items cannot be concurrently inserted).
	144	*
	145	* radix_tree_deref_retry must be used to confirm validity of the pointer if
	146	* only the read lock is held.
7cf9c2c7 NP	147	*/
	148	static inline void radix_tree_deref_slot(void *pslot)
	149	{
27d20fdd	150	return rcu_dereference(*pslot);
7cf9c2c7	151	}
27d20fdd	152
29c1f677 MG	153	/**
	154	* radix_tree_deref_slot_protected - dereference a slot without RCU lock but with tree lock held
	155	* @pslot: pointer to slot, returned by radix_tree_lookup_slot
	156	* Returns: item that was stored in that slot with any direct pointer flag
	157	* removed.
	158	*
	159	* Similar to radix_tree_deref_slot but only used during migration when a pages
	160	* mapping is being moved. The caller does not hold the RCU read lock but it
	161	* must hold the tree lock to prevent parallel updates.
	162	*/
	163	static inline void radix_tree_deref_slot_protected(void *pslot,
	164	spinlock_t *treelock)
	165	{
	166	return rcu_dereference_protected(*pslot, lockdep_is_held(treelock));
	167	}
	168
27d20fdd NP	169	/**
	170	* radix_tree_deref_retry - check radix_tree_deref_slot
	171	* @arg: pointer returned by radix_tree_deref_slot
	172	* Returns: 0 if retry is not required, otherwise retry is required
	173	*
	174	* radix_tree_deref_retry must be used with radix_tree_deref_slot.
	175	*/
	176	static inline int radix_tree_deref_retry(void *arg)
	177	{
	178	return unlikely((unsigned long)arg & RADIX_TREE_INDIRECT_PTR);
	179	}
	180
6328650b HD	181	/**
	182	* radix_tree_exceptional_entry - radix_tree_deref_slot gave exceptional entry?
	183	* @arg: value returned by radix_tree_deref_slot
	184	* Returns: 0 if well-aligned pointer, non-0 if exceptional entry.
	185	*/
	186	static inline int radix_tree_exceptional_entry(void *arg)
	187	{
	188	/* Not unlikely because radix_tree_exception often tested first */
	189	return (unsigned long)arg & RADIX_TREE_EXCEPTIONAL_ENTRY;
	190	}
	191
	192	/**
	193	* radix_tree_exception - radix_tree_deref_slot returned either exception?
	194	* @arg: value returned by radix_tree_deref_slot
	195	* Returns: 0 if well-aligned pointer, non-0 if either kind of exception.
	196	*/
	197	static inline int radix_tree_exception(void *arg)
	198	{
	199	return unlikely((unsigned long)arg &
	200	(RADIX_TREE_INDIRECT_PTR \| RADIX_TREE_EXCEPTIONAL_ENTRY));
	201	}
	202
7cf9c2c7 NP	203	/**
	204	* radix_tree_replace_slot - replace item in a slot
	205	* @pslot: pointer to slot, returned by radix_tree_lookup_slot
	206	* @item: new item to store in the slot.
	207	*
	208	* For use with radix_tree_lookup_slot(). Caller must hold tree write locked
	209	* across slot lookup and replacement.
	210	*/
	211	static inline void radix_tree_replace_slot(void *pslot, void item)
	212	{
c0bc9875 NP	213	BUG_ON(radix_tree_is_indirect_ptr(item));
c0bc9875 NP	214	rcu_assign_pointer(*pslot, item);
7cf9c2c7 NP	215	}
7cf9c2c7 NP	216
1da177e4 LT	217	int radix_tree_insert(struct radix_tree_root , unsigned long, void );
1da177e4 LT	218	void radix_tree_lookup(struct radix_tree_root , unsigned long);
a4331366	219	void *radix_tree_lookup_slot(struct radix_tree_root , unsigned long);
1da177e4 LT	220	void radix_tree_delete(struct radix_tree_root , unsigned long);
	221	unsigned int
	222	radix_tree_gang_lookup(struct radix_tree_root root, void *results,
	223	unsigned long first_index, unsigned int max_items);
6328650b HD	224	unsigned int radix_tree_gang_lookup_slot(struct radix_tree_root *root,
6328650b HD	225	void **results, unsigned long indices,
47feff2c	226	unsigned long first_index, unsigned int max_items);
6df8ba4f FW	227	unsigned long radix_tree_next_hole(struct radix_tree_root *root,
6df8ba4f FW	228	unsigned long index, unsigned long max_scan);
dc566127 WF	229	unsigned long radix_tree_prev_hole(struct radix_tree_root *root,
dc566127 WF	230	unsigned long index, unsigned long max_scan);
dd0fc66f	231	int radix_tree_preload(gfp_t gfp_mask);
1da177e4 LT	232	void radix_tree_init(void);
1da177e4 LT	233	void radix_tree_tag_set(struct radix_tree_root root,
daff89f3	234	unsigned long index, unsigned int tag);
1da177e4	235	void radix_tree_tag_clear(struct radix_tree_root root,
daff89f3	236	unsigned long index, unsigned int tag);
1da177e4	237	int radix_tree_tag_get(struct radix_tree_root *root,
daff89f3	238	unsigned long index, unsigned int tag);
1da177e4 LT	239	unsigned int
1da177e4 LT	240	radix_tree_gang_lookup_tag(struct radix_tree_root root, void *results,
daff89f3 JC	241	unsigned long first_index, unsigned int max_items,
daff89f3 JC	242	unsigned int tag);
47feff2c NP	243	unsigned int
	244	radix_tree_gang_lookup_tag_slot(struct radix_tree_root root, void **results,
	245	unsigned long first_index, unsigned int max_items,
	246	unsigned int tag);
ebf8aa44 JK	247	unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root,
	248	unsigned long *first_indexp, unsigned long last_index,
	249	unsigned long nr_to_tag,
	250	unsigned int fromtag, unsigned int totag);
daff89f3	251	int radix_tree_tagged(struct radix_tree_root *root, unsigned int tag);
e504f3fd	252	unsigned long radix_tree_locate_item(struct radix_tree_root root, void item);
1da177e4 LT	253
	254	static inline void radix_tree_preload_end(void)
	255	{
	256	preempt_enable();
	257	}
	258
	259	#endif /* _LINUX_RADIX_TREE_H */