[mirror_ubuntu-bionic-kernel.git] / include / linux / nodemask.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_NODEMASK_H
#define __LINUX_NODEMASK_H

/*
 * Nodemasks provide a bitmap suitable for representing the
 * set of Node's in a system, one bit position per Node number.
 *
 * See detailed comments in the file linux/bitmap.h describing the
 * data type on which these nodemasks are based.
 *
 * For details of nodemask_parse_user(), see bitmap_parse_user() in
 * lib/bitmap.c.  For details of nodelist_parse(), see bitmap_parselist(),
 * also in bitmap.c.  For details of node_remap(), see bitmap_bitremap in
 * lib/bitmap.c.  For details of nodes_remap(), see bitmap_remap in
 * lib/bitmap.c.  For details of nodes_onto(), see bitmap_onto in
 * lib/bitmap.c.  For details of nodes_fold(), see bitmap_fold in
 * lib/bitmap.c.
 *
 * The available nodemask operations are:
 *
 * void node_set(node, mask)		turn on bit 'node' in mask
 * void node_clear(node, mask)		turn off bit 'node' in mask
 * void nodes_setall(mask)		set all bits
 * void nodes_clear(mask)		clear all bits
 * int node_isset(node, mask)		true iff bit 'node' set in mask
 * int node_test_and_set(node, mask)	test and set bit 'node' in mask
 *
 * void nodes_and(dst, src1, src2)	dst = src1 & src2  [intersection]
 * void nodes_or(dst, src1, src2)	dst = src1 | src2  [union]
 * void nodes_xor(dst, src1, src2)	dst = src1 ^ src2
 * void nodes_andnot(dst, src1, src2)	dst = src1 & ~src2
 * void nodes_complement(dst, src)	dst = ~src
 *
 * int nodes_equal(mask1, mask2)	Does mask1 == mask2?
 * int nodes_intersects(mask1, mask2)	Do mask1 and mask2 intersect?
 * int nodes_subset(mask1, mask2)	Is mask1 a subset of mask2?
 * int nodes_empty(mask)		Is mask empty (no bits sets)?
 * int nodes_full(mask)			Is mask full (all bits sets)?
 * int nodes_weight(mask)		Hamming weight - number of set bits
 *
 * void nodes_shift_right(dst, src, n)	Shift right
 * void nodes_shift_left(dst, src, n)	Shift left
 *
 * int first_node(mask)			Number lowest set bit, or MAX_NUMNODES
 * int next_node(node, mask)		Next node past 'node', or MAX_NUMNODES
 * int next_node_in(node, mask)		Next node past 'node', or wrap to first,
 *					or MAX_NUMNODES
 * int first_unset_node(mask)		First node not set in mask, or 
 *					MAX_NUMNODES
 *
 * nodemask_t nodemask_of_node(node)	Return nodemask with bit 'node' set
 * NODE_MASK_ALL			Initializer - all bits set
 * NODE_MASK_NONE			Initializer - no bits set
 * unsigned long *nodes_addr(mask)	Array of unsigned long's in mask
 *
 * int nodemask_parse_user(ubuf, ulen, mask)	Parse ascii string as nodemask
 * int nodelist_parse(buf, map)		Parse ascii string as nodelist
 * int node_remap(oldbit, old, new)	newbit = map(old, new)(oldbit)
 * void nodes_remap(dst, src, old, new)	*dst = map(old, new)(src)
 * void nodes_onto(dst, orig, relmap)	*dst = orig relative to relmap
 * void nodes_fold(dst, orig, sz)	dst bits = orig bits mod sz
 *
 * for_each_node_mask(node, mask)	for-loop node over mask
 *
 * int num_online_nodes()		Number of online Nodes
 * int num_possible_nodes()		Number of all possible Nodes
 *
 * int node_random(mask)		Random node with set bit in mask
 *
 * int node_online(node)		Is some node online?
 * int node_possible(node)		Is some node possible?
 *
 * node_set_online(node)		set bit 'node' in node_online_map
 * node_set_offline(node)		clear bit 'node' in node_online_map
 *
 * for_each_node(node)			for-loop node over node_possible_map
 * for_each_online_node(node)		for-loop node over node_online_map
 *
 * Subtlety:
 * 1) The 'type-checked' form of node_isset() causes gcc (3.3.2, anyway)
 *    to generate slightly worse code.  So use a simple one-line #define
 *    for node_isset(), instead of wrapping an inline inside a macro, the
 *    way we do the other calls.
 *
 * NODEMASK_SCRATCH
 * When doing above logical AND, OR, XOR, Remap operations the callers tend to
 * need temporary nodemask_t's on the stack. But if NODES_SHIFT is large,
 * nodemask_t's consume too much stack space.  NODEMASK_SCRATCH is a helper
 * for such situations. See below and CPUMASK_ALLOC also.
 */

#include <linux/kernel.h>
#include <linux/threads.h>
#include <linux/bitmap.h>
#include <linux/numa.h>

typedef struct { DECLARE_BITMAP(bits, MAX_NUMNODES); } nodemask_t;
extern nodemask_t _unused_nodemask_arg_;

/**
 * nodemask_pr_args - printf args to output a nodemask
 * @maskp: nodemask to be printed
 *
 * Can be used to provide arguments for '%*pb[l]' when printing a nodemask.
 */
#define nodemask_pr_args(maskp)		MAX_NUMNODES, (maskp)->bits

/*
 * The inline keyword gives the compiler room to decide to inline, or
 * not inline a function as it sees best.  However, as these functions
 * are called in both __init and non-__init functions, if they are not
 * inlined we will end up with a section mis-match error (of the type of
 * freeable items not being freed).  So we must use __always_inline here
 * to fix the problem.  If other functions in the future also end up in
 * this situation they will also need to be annotated as __always_inline
 */
#define node_set(node, dst) __node_set((node), &(dst))
static __always_inline void __node_set(int node, volatile nodemask_t *dstp)
{
	set_bit(node, dstp->bits);
}

#define node_clear(node, dst) __node_clear((node), &(dst))
static inline void __node_clear(int node, volatile nodemask_t *dstp)
{
	clear_bit(node, dstp->bits);
}

#define nodes_setall(dst) __nodes_setall(&(dst), MAX_NUMNODES)
static inline void __nodes_setall(nodemask_t *dstp, unsigned int nbits)
{
	bitmap_fill(dstp->bits, nbits);
}

#define nodes_clear(dst) __nodes_clear(&(dst), MAX_NUMNODES)
static inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits)
{
	bitmap_zero(dstp->bits, nbits);
}

/* No static inline type checking - see Subtlety (1) above. */
#define node_isset(node, nodemask) test_bit((node), (nodemask).bits)

#define node_test_and_set(node, nodemask) \
			__node_test_and_set((node), &(nodemask))
static inline int __node_test_and_set(int node, nodemask_t *addr)
{
	return test_and_set_bit(node, addr->bits);
}

#define nodes_and(dst, src1, src2) \
			__nodes_and(&(dst), &(src1), &(src2), MAX_NUMNODES)
static inline void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p,
					const nodemask_t *src2p, unsigned int nbits)
{
	bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits);
}

#define nodes_or(dst, src1, src2) \
			__nodes_or(&(dst), &(src1), &(src2), MAX_NUMNODES)
static inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p,
					const nodemask_t *src2p, unsigned int nbits)
{
	bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits);
}

#define nodes_xor(dst, src1, src2) \
			__nodes_xor(&(dst), &(src1), &(src2), MAX_NUMNODES)
static inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p,
					const nodemask_t *src2p, unsigned int nbits)
{
	bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits);
}

#define nodes_andnot(dst, src1, src2) \
			__nodes_andnot(&(dst), &(src1), &(src2), MAX_NUMNODES)
static inline void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p,
					const nodemask_t *src2p, unsigned int nbits)
{
	bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits);
}

#define nodes_complement(dst, src) \
			__nodes_complement(&(dst), &(src), MAX_NUMNODES)
static inline void __nodes_complement(nodemask_t *dstp,
					const nodemask_t *srcp, unsigned int nbits)
{
	bitmap_complement(dstp->bits, srcp->bits, nbits);
}

#define nodes_equal(src1, src2) \
			__nodes_equal(&(src1), &(src2), MAX_NUMNODES)
static inline int __nodes_equal(const nodemask_t *src1p,
					const nodemask_t *src2p, unsigned int nbits)
{
	return bitmap_equal(src1p->bits, src2p->bits, nbits);
}

#define nodes_intersects(src1, src2) \
			__nodes_intersects(&(src1), &(src2), MAX_NUMNODES)
static inline int __nodes_intersects(const nodemask_t *src1p,
					const nodemask_t *src2p, unsigned int nbits)
{
	return bitmap_intersects(src1p->bits, src2p->bits, nbits);
}

#define nodes_subset(src1, src2) \
			__nodes_subset(&(src1), &(src2), MAX_NUMNODES)
static inline int __nodes_subset(const nodemask_t *src1p,
					const nodemask_t *src2p, unsigned int nbits)
{
	return bitmap_subset(src1p->bits, src2p->bits, nbits);
}

#define nodes_empty(src) __nodes_empty(&(src), MAX_NUMNODES)
static inline int __nodes_empty(const nodemask_t *srcp, unsigned int nbits)
{
	return bitmap_empty(srcp->bits, nbits);
}

#define nodes_full(nodemask) __nodes_full(&(nodemask), MAX_NUMNODES)
static inline int __nodes_full(const nodemask_t *srcp, unsigned int nbits)
{
	return bitmap_full(srcp->bits, nbits);
}

#define nodes_weight(nodemask) __nodes_weight(&(nodemask), MAX_NUMNODES)
static inline int __nodes_weight(const nodemask_t *srcp, unsigned int nbits)
{
	return bitmap_weight(srcp->bits, nbits);
}

#define nodes_shift_right(dst, src, n) \
			__nodes_shift_right(&(dst), &(src), (n), MAX_NUMNODES)
static inline void __nodes_shift_right(nodemask_t *dstp,
					const nodemask_t *srcp, int n, int nbits)
{
	bitmap_shift_right(dstp->bits, srcp->bits, n, nbits);
}

#define nodes_shift_left(dst, src, n) \
			__nodes_shift_left(&(dst), &(src), (n), MAX_NUMNODES)
static inline void __nodes_shift_left(nodemask_t *dstp,
					const nodemask_t *srcp, int n, int nbits)
{
	bitmap_shift_left(dstp->bits, srcp->bits, n, nbits);
}

/* FIXME: better would be to fix all architectures to never return
          > MAX_NUMNODES, then the silly min_ts could be dropped. */

#define first_node(src) __first_node(&(src))
static inline int __first_node(const nodemask_t *srcp)
{
	return min_t(int, MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES));
}

#define next_node(n, src) __next_node((n), &(src))
static inline int __next_node(int n, const nodemask_t *srcp)
{
	return min_t(int,MAX_NUMNODES,find_next_bit(srcp->bits, MAX_NUMNODES, n+1));
}

/*
 * Find the next present node in src, starting after node n, wrapping around to
 * the first node in src if needed.  Returns MAX_NUMNODES if src is empty.
 */
#define next_node_in(n, src) __next_node_in((n), &(src))
int __next_node_in(int node, const nodemask_t *srcp);

static inline void init_nodemask_of_node(nodemask_t *mask, int node)
{
	nodes_clear(*mask);
	node_set(node, *mask);
}

#define nodemask_of_node(node)						\
({									\
	typeof(_unused_nodemask_arg_) m;				\
	if (sizeof(m) == sizeof(unsigned long)) {			\
		m.bits[0] = 1UL << (node);				\
	} else {							\
		init_nodemask_of_node(&m, (node));			\
	}								\
	m;								\
})

#define first_unset_node(mask) __first_unset_node(&(mask))
static inline int __first_unset_node(const nodemask_t *maskp)
{
	return min_t(int,MAX_NUMNODES,
			find_first_zero_bit(maskp->bits, MAX_NUMNODES));
}

#define NODE_MASK_LAST_WORD BITMAP_LAST_WORD_MASK(MAX_NUMNODES)

#if MAX_NUMNODES <= BITS_PER_LONG

#define NODE_MASK_ALL							\
((nodemask_t) { {							\
	[BITS_TO_LONGS(MAX_NUMNODES)-1] = NODE_MASK_LAST_WORD		\
} })

#else

#define NODE_MASK_ALL							\
((nodemask_t) { {							\
	[0 ... BITS_TO_LONGS(MAX_NUMNODES)-2] = ~0UL,			\
	[BITS_TO_LONGS(MAX_NUMNODES)-1] = NODE_MASK_LAST_WORD		\
} })

#endif

#define NODE_MASK_NONE							\
((nodemask_t) { {							\
	[0 ... BITS_TO_LONGS(MAX_NUMNODES)-1] =  0UL			\
} })

#define nodes_addr(src) ((src).bits)

#define nodemask_parse_user(ubuf, ulen, dst) \
		__nodemask_parse_user((ubuf), (ulen), &(dst), MAX_NUMNODES)
static inline int __nodemask_parse_user(const char __user *buf, int len,
					nodemask_t *dstp, int nbits)
{
	return bitmap_parse_user(buf, len, dstp->bits, nbits);
}

#define nodelist_parse(buf, dst) __nodelist_parse((buf), &(dst), MAX_NUMNODES)
static inline int __nodelist_parse(const char *buf, nodemask_t *dstp, int nbits)
{
	return bitmap_parselist(buf, dstp->bits, nbits);
}

#define node_remap(oldbit, old, new) \
		__node_remap((oldbit), &(old), &(new), MAX_NUMNODES)
static inline int __node_remap(int oldbit,
		const nodemask_t *oldp, const nodemask_t *newp, int nbits)
{
	return bitmap_bitremap(oldbit, oldp->bits, newp->bits, nbits);
}

#define nodes_remap(dst, src, old, new) \
		__nodes_remap(&(dst), &(src), &(old), &(new), MAX_NUMNODES)
static inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp,
		const nodemask_t *oldp, const nodemask_t *newp, int nbits)
{
	bitmap_remap(dstp->bits, srcp->bits, oldp->bits, newp->bits, nbits);
}

#define nodes_onto(dst, orig, relmap) \
		__nodes_onto(&(dst), &(orig), &(relmap), MAX_NUMNODES)
static inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp,
		const nodemask_t *relmapp, int nbits)
{
	bitmap_onto(dstp->bits, origp->bits, relmapp->bits, nbits);
}

#define nodes_fold(dst, orig, sz) \
		__nodes_fold(&(dst), &(orig), sz, MAX_NUMNODES)
static inline void __nodes_fold(nodemask_t *dstp, const nodemask_t *origp,
		int sz, int nbits)
{
	bitmap_fold(dstp->bits, origp->bits, sz, nbits);
}

#if MAX_NUMNODES > 1
#define for_each_node_mask(node, mask)			\
	for ((node) = first_node(mask);			\
		(node) < MAX_NUMNODES;			\
		(node) = next_node((node), (mask)))
#else /* MAX_NUMNODES == 1 */
#define for_each_node_mask(node, mask)			\
	if (!nodes_empty(mask))				\
		for ((node) = 0; (node) < 1; (node)++)
#endif /* MAX_NUMNODES */

/*
 * Bitmasks that are kept for all the nodes.
 */
enum node_states {
	N_POSSIBLE,		/* The node could become online at some point */
	N_ONLINE,		/* The node is online */
	N_NORMAL_MEMORY,	/* The node has regular memory */
#ifdef CONFIG_HIGHMEM
	N_HIGH_MEMORY,		/* The node has regular or high memory */
#else
	N_HIGH_MEMORY = N_NORMAL_MEMORY,
#endif
	N_MEMORY,		/* The node has memory(regular, high, movable) */
	N_CPU,		/* The node has one or more cpus */
	NR_NODE_STATES
};

/*
 * The following particular system nodemasks and operations
 * on them manage all possible and online nodes.
 */

extern nodemask_t node_states[NR_NODE_STATES];

#if MAX_NUMNODES > 1
static inline int node_state(int node, enum node_states state)
{
	return node_isset(node, node_states[state]);
}

static inline void node_set_state(int node, enum node_states state)
{
	__node_set(node, &node_states[state]);
}

static inline void node_clear_state(int node, enum node_states state)
{
	__node_clear(node, &node_states[state]);
}

static inline int num_node_state(enum node_states state)
{
	return nodes_weight(node_states[state]);
}

#define for_each_node_state(__node, __state) \
	for_each_node_mask((__node), node_states[__state])

#define first_online_node	first_node(node_states[N_ONLINE])
#define first_memory_node	first_node(node_states[N_MEMORY])
static inline int next_online_node(int nid)
{
	return next_node(nid, node_states[N_ONLINE]);
}
static inline int next_memory_node(int nid)
{
	return next_node(nid, node_states[N_MEMORY]);
}

extern int nr_node_ids;
extern int nr_online_nodes;

static inline void node_set_online(int nid)
{
	node_set_state(nid, N_ONLINE);
	nr_online_nodes = num_node_state(N_ONLINE);
}

static inline void node_set_offline(int nid)
{
	node_clear_state(nid, N_ONLINE);
	nr_online_nodes = num_node_state(N_ONLINE);
}

#else

static inline int node_state(int node, enum node_states state)
{
	return node == 0;
}

static inline void node_set_state(int node, enum node_states state)
{
}

static inline void node_clear_state(int node, enum node_states state)
{
}

static inline int num_node_state(enum node_states state)
{
	return 1;
}

#define for_each_node_state(node, __state) \
	for ( (node) = 0; (node) == 0; (node) = 1)

#define first_online_node	0
#define first_memory_node	0
#define next_online_node(nid)	(MAX_NUMNODES)
#define nr_node_ids		1
#define nr_online_nodes		1

#define node_set_online(node)	   node_set_state((node), N_ONLINE)
#define node_set_offline(node)	   node_clear_state((node), N_ONLINE)

#endif

#if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1)
extern int node_random(const nodemask_t *maskp);
#else
static inline int node_random(const nodemask_t *mask)
{
	return 0;
}
#endif

#define node_online_map 	node_states[N_ONLINE]
#define node_possible_map 	node_states[N_POSSIBLE]

#define num_online_nodes()	num_node_state(N_ONLINE)
#define num_possible_nodes()	num_node_state(N_POSSIBLE)
#define node_online(node)	node_state((node), N_ONLINE)
#define node_possible(node)	node_state((node), N_POSSIBLE)

#define for_each_node(node)	   for_each_node_state(node, N_POSSIBLE)
#define for_each_online_node(node) for_each_node_state(node, N_ONLINE)

/*
 * For nodemask scrach area.
 * NODEMASK_ALLOC(type, name) allocates an object with a specified type and
 * name.
 */
#if NODES_SHIFT > 8 /* nodemask_t > 256 bytes */
#define NODEMASK_ALLOC(type, name, gfp_flags)	\
			type *name = kmalloc(sizeof(*name), gfp_flags)
#define NODEMASK_FREE(m)			kfree(m)
#else
#define NODEMASK_ALLOC(type, name, gfp_flags)	type _##name, *name = &_##name
#define NODEMASK_FREE(m)			do {} while (0)
#endif

/* A example struture for using NODEMASK_ALLOC, used in mempolicy. */
struct nodemask_scratch {
	nodemask_t	mask1;
	nodemask_t	mask2;
};

#define NODEMASK_SCRATCH(x)						\
			NODEMASK_ALLOC(struct nodemask_scratch, x,	\
					GFP_KERNEL | __GFP_NORETRY)
#define NODEMASK_SCRATCH_FREE(x)	NODEMASK_FREE(x)


#endif /* __LINUX_NODEMASK_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
1da177e4 LT	2	#ifndef __LINUX_NODEMASK_H
	3	#define __LINUX_NODEMASK_H
	4
	5	/*
	6	* Nodemasks provide a bitmap suitable for representing the
	7	* set of Node's in a system, one bit position per Node number.
	8	*
	9	* See detailed comments in the file linux/bitmap.h describing the
	10	* data type on which these nodemasks are based.
	11	*
46385326 TH	12	* For details of nodemask_parse_user(), see bitmap_parse_user() in
	13	* lib/bitmap.c. For details of nodelist_parse(), see bitmap_parselist(),
	14	* also in bitmap.c. For details of node_remap(), see bitmap_bitremap in
	15	* lib/bitmap.c. For details of nodes_remap(), see bitmap_remap in
	16	* lib/bitmap.c. For details of nodes_onto(), see bitmap_onto in
	17	* lib/bitmap.c. For details of nodes_fold(), see bitmap_fold in
	18	* lib/bitmap.c.
1da177e4 LT	19	*
	20	* The available nodemask operations are:
	21	*
	22	* void node_set(node, mask) turn on bit 'node' in mask
	23	* void node_clear(node, mask) turn off bit 'node' in mask
	24	* void nodes_setall(mask) set all bits
	25	* void nodes_clear(mask) clear all bits
	26	* int node_isset(node, mask) true iff bit 'node' set in mask
	27	* int node_test_and_set(node, mask) test and set bit 'node' in mask
	28	*
	29	* void nodes_and(dst, src1, src2) dst = src1 & src2 [intersection]
	30	* void nodes_or(dst, src1, src2) dst = src1 \| src2 [union]
	31	* void nodes_xor(dst, src1, src2) dst = src1 ^ src2
	32	* void nodes_andnot(dst, src1, src2) dst = src1 & ~src2
	33	* void nodes_complement(dst, src) dst = ~src
	34	*
	35	* int nodes_equal(mask1, mask2) Does mask1 == mask2?
	36	* int nodes_intersects(mask1, mask2) Do mask1 and mask2 intersect?
	37	* int nodes_subset(mask1, mask2) Is mask1 a subset of mask2?
	38	* int nodes_empty(mask) Is mask empty (no bits sets)?
	39	* int nodes_full(mask) Is mask full (all bits sets)?
	40	* int nodes_weight(mask) Hamming weight - number of set bits
	41	*
	42	* void nodes_shift_right(dst, src, n) Shift right
	43	* void nodes_shift_left(dst, src, n) Shift left
	44	*
	45	* int first_node(mask) Number lowest set bit, or MAX_NUMNODES
	46	* int next_node(node, mask) Next node past 'node', or MAX_NUMNODES
0edaf86c AM	47	* int next_node_in(node, mask) Next node past 'node', or wrap to first,
0edaf86c AM	48	* or MAX_NUMNODES
1da177e4	49	* int first_unset_node(mask) First node not set in mask, or
0edaf86c	50	* MAX_NUMNODES
1da177e4 LT	51	*
	52	* nodemask_t nodemask_of_node(node) Return nodemask with bit 'node' set
	53	* NODE_MASK_ALL Initializer - all bits set
	54	* NODE_MASK_NONE Initializer - no bits set
	55	* unsigned long *nodes_addr(mask) Array of unsigned long's in mask
	56	*
01a3ee2b	57	* int nodemask_parse_user(ubuf, ulen, mask) Parse ascii string as nodemask
1da177e4	58	* int nodelist_parse(buf, map) Parse ascii string as nodelist
fb5eeeee	59	* int node_remap(oldbit, old, new) newbit = map(old, new)(oldbit)
7ea931c9 PJ	60	* void nodes_remap(dst, src, old, new) *dst = map(old, new)(src)
	61	* void nodes_onto(dst, orig, relmap) *dst = orig relative to relmap
	62	* void nodes_fold(dst, orig, sz) dst bits = orig bits mod sz
1da177e4 LT	63	*
	64	* for_each_node_mask(node, mask) for-loop node over mask
	65	*
	66	* int num_online_nodes() Number of online Nodes
	67	* int num_possible_nodes() Number of all possible Nodes
	68	*
778d3b0f MH	69	* int node_random(mask) Random node with set bit in mask
778d3b0f MH	70	*
1da177e4 LT	71	* int node_online(node) Is some node online?
	72	* int node_possible(node) Is some node possible?
	73	*
1da177e4 LT	74	* node_set_online(node) set bit 'node' in node_online_map
	75	* node_set_offline(node) clear bit 'node' in node_online_map
	76	*
	77	* for_each_node(node) for-loop node over node_possible_map
	78	* for_each_online_node(node) for-loop node over node_online_map
	79	*
	80	* Subtlety:
	81	* 1) The 'type-checked' form of node_isset() causes gcc (3.3.2, anyway)
	82	* to generate slightly worse code. So use a simple one-line #define
	83	* for node_isset(), instead of wrapping an inline inside a macro, the
	84	* way we do the other calls.
4bfc4495 KH	85	*
	86	* NODEMASK_SCRATCH
	87	* When doing above logical AND, OR, XOR, Remap operations the callers tend to
	88	* need temporary nodemask_t's on the stack. But if NODES_SHIFT is large,
	89	* nodemask_t's consume too much stack space. NODEMASK_SCRATCH is a helper
	90	* for such situations. See below and CPUMASK_ALLOC also.
1da177e4 LT	91	*/
	92
	93	#include <linux/kernel.h>
	94	#include <linux/threads.h>
	95	#include <linux/bitmap.h>
	96	#include <linux/numa.h>
1da177e4 LT	97
	98	typedef struct { DECLARE_BITMAP(bits, MAX_NUMNODES); } nodemask_t;
	99	extern nodemask_t _unused_nodemask_arg_;
	100
f1bbc032 TH	101	/**
	102	* nodemask_pr_args - printf args to output a nodemask
	103	* @maskp: nodemask to be printed
	104	*
	105	* Can be used to provide arguments for '%*pb[l]' when printing a nodemask.
	106	*/
	107	#define nodemask_pr_args(maskp) MAX_NUMNODES, (maskp)->bits
	108
323f54ed TR	109	/*
	110	* The inline keyword gives the compiler room to decide to inline, or
	111	* not inline a function as it sees best. However, as these functions
	112	* are called in both __init and non-__init functions, if they are not
	113	* inlined we will end up with a section mis-match error (of the type of
	114	* freeable items not being freed). So we must use __always_inline here
	115	* to fix the problem. If other functions in the future also end up in
	116	* this situation they will also need to be annotated as __always_inline
	117	*/
1da177e4	118	#define node_set(node, dst) __node_set((node), &(dst))
323f54ed	119	static __always_inline void __node_set(int node, volatile nodemask_t *dstp)
1da177e4 LT	120	{
	121	set_bit(node, dstp->bits);
	122	}
	123
	124	#define node_clear(node, dst) __node_clear((node), &(dst))
	125	static inline void __node_clear(int node, volatile nodemask_t *dstp)
	126	{
	127	clear_bit(node, dstp->bits);
	128	}
	129
	130	#define nodes_setall(dst) __nodes_setall(&(dst), MAX_NUMNODES)
33c4fa8c	131	static inline void __nodes_setall(nodemask_t *dstp, unsigned int nbits)
1da177e4 LT	132	{
	133	bitmap_fill(dstp->bits, nbits);
	134	}
	135
	136	#define nodes_clear(dst) __nodes_clear(&(dst), MAX_NUMNODES)
33c4fa8c	137	static inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits)
1da177e4 LT	138	{
	139	bitmap_zero(dstp->bits, nbits);
	140	}
	141
	142	/* No static inline type checking - see Subtlety (1) above. */
	143	#define node_isset(node, nodemask) test_bit((node), (nodemask).bits)
	144
	145	#define node_test_and_set(node, nodemask) \
	146	__node_test_and_set((node), &(nodemask))
	147	static inline int __node_test_and_set(int node, nodemask_t *addr)
	148	{
	149	return test_and_set_bit(node, addr->bits);
	150	}
	151
	152	#define nodes_and(dst, src1, src2) \
	153	__nodes_and(&(dst), &(src1), &(src2), MAX_NUMNODES)
	154	static inline void __nodes_and(nodemask_t dstp, const nodemask_t src1p,
33c4fa8c	155	const nodemask_t *src2p, unsigned int nbits)
1da177e4 LT	156	{
	157	bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits);
	158	}
	159
	160	#define nodes_or(dst, src1, src2) \
	161	__nodes_or(&(dst), &(src1), &(src2), MAX_NUMNODES)
	162	static inline void __nodes_or(nodemask_t dstp, const nodemask_t src1p,
33c4fa8c	163	const nodemask_t *src2p, unsigned int nbits)
1da177e4 LT	164	{
	165	bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits);
	166	}
	167
	168	#define nodes_xor(dst, src1, src2) \
	169	__nodes_xor(&(dst), &(src1), &(src2), MAX_NUMNODES)
	170	static inline void __nodes_xor(nodemask_t dstp, const nodemask_t src1p,
33c4fa8c	171	const nodemask_t *src2p, unsigned int nbits)
1da177e4 LT	172	{
	173	bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits);
	174	}
	175
	176	#define nodes_andnot(dst, src1, src2) \
	177	__nodes_andnot(&(dst), &(src1), &(src2), MAX_NUMNODES)
	178	static inline void __nodes_andnot(nodemask_t dstp, const nodemask_t src1p,
33c4fa8c	179	const nodemask_t *src2p, unsigned int nbits)
1da177e4 LT	180	{
	181	bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits);
	182	}
	183
	184	#define nodes_complement(dst, src) \
	185	__nodes_complement(&(dst), &(src), MAX_NUMNODES)
	186	static inline void __nodes_complement(nodemask_t *dstp,
33c4fa8c	187	const nodemask_t *srcp, unsigned int nbits)
1da177e4 LT	188	{
	189	bitmap_complement(dstp->bits, srcp->bits, nbits);
	190	}
	191
	192	#define nodes_equal(src1, src2) \
	193	__nodes_equal(&(src1), &(src2), MAX_NUMNODES)
	194	static inline int __nodes_equal(const nodemask_t *src1p,
33c4fa8c	195	const nodemask_t *src2p, unsigned int nbits)
1da177e4 LT	196	{
	197	return bitmap_equal(src1p->bits, src2p->bits, nbits);
	198	}
	199
	200	#define nodes_intersects(src1, src2) \
	201	__nodes_intersects(&(src1), &(src2), MAX_NUMNODES)
	202	static inline int __nodes_intersects(const nodemask_t *src1p,
33c4fa8c	203	const nodemask_t *src2p, unsigned int nbits)
1da177e4 LT	204	{
	205	return bitmap_intersects(src1p->bits, src2p->bits, nbits);
	206	}
	207
	208	#define nodes_subset(src1, src2) \
	209	__nodes_subset(&(src1), &(src2), MAX_NUMNODES)
	210	static inline int __nodes_subset(const nodemask_t *src1p,
33c4fa8c	211	const nodemask_t *src2p, unsigned int nbits)
1da177e4 LT	212	{
	213	return bitmap_subset(src1p->bits, src2p->bits, nbits);
	214	}
	215
	216	#define nodes_empty(src) __nodes_empty(&(src), MAX_NUMNODES)
33c4fa8c	217	static inline int __nodes_empty(const nodemask_t *srcp, unsigned int nbits)
1da177e4 LT	218	{
	219	return bitmap_empty(srcp->bits, nbits);
	220	}
	221
	222	#define nodes_full(nodemask) __nodes_full(&(nodemask), MAX_NUMNODES)
33c4fa8c	223	static inline int __nodes_full(const nodemask_t *srcp, unsigned int nbits)
1da177e4 LT	224	{
	225	return bitmap_full(srcp->bits, nbits);
	226	}
	227
	228	#define nodes_weight(nodemask) __nodes_weight(&(nodemask), MAX_NUMNODES)
33c4fa8c	229	static inline int __nodes_weight(const nodemask_t *srcp, unsigned int nbits)
1da177e4 LT	230	{
	231	return bitmap_weight(srcp->bits, nbits);
	232	}
	233
	234	#define nodes_shift_right(dst, src, n) \
	235	__nodes_shift_right(&(dst), &(src), (n), MAX_NUMNODES)
	236	static inline void __nodes_shift_right(nodemask_t *dstp,
	237	const nodemask_t *srcp, int n, int nbits)
	238	{
	239	bitmap_shift_right(dstp->bits, srcp->bits, n, nbits);
	240	}
	241
	242	#define nodes_shift_left(dst, src, n) \
	243	__nodes_shift_left(&(dst), &(src), (n), MAX_NUMNODES)
	244	static inline void __nodes_shift_left(nodemask_t *dstp,
	245	const nodemask_t *srcp, int n, int nbits)
	246	{
	247	bitmap_shift_left(dstp->bits, srcp->bits, n, nbits);
	248	}
	249
	250	/* FIXME: better would be to fix all architectures to never return
	251	> MAX_NUMNODES, then the silly min_ts could be dropped. */
	252
	253	#define first_node(src) __first_node(&(src))
	254	static inline int __first_node(const nodemask_t *srcp)
	255	{
	256	return min_t(int, MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES));
	257	}
	258
	259	#define next_node(n, src) __next_node((n), &(src))
	260	static inline int __next_node(int n, const nodemask_t *srcp)
	261	{
	262	return min_t(int,MAX_NUMNODES,find_next_bit(srcp->bits, MAX_NUMNODES, n+1));
	263	}
	264
0edaf86c AM	265	/*
	266	* Find the next present node in src, starting after node n, wrapping around to
	267	* the first node in src if needed. Returns MAX_NUMNODES if src is empty.
	268	*/
	269	#define next_node_in(n, src) __next_node_in((n), &(src))
	270	int __next_node_in(int node, const nodemask_t *srcp);
	271
c1e6c8d0 LS	272	static inline void init_nodemask_of_node(nodemask_t *mask, int node)
	273	{
	274	nodes_clear(*mask);
	275	node_set(node, *mask);
	276	}
	277
1da177e4 LT	278	#define nodemask_of_node(node) \
	279	({ \
	280	typeof(_unused_nodemask_arg_) m; \
	281	if (sizeof(m) == sizeof(unsigned long)) { \
c1e6c8d0	282	m.bits[0] = 1UL << (node); \
1da177e4	283	} else { \
c1e6c8d0	284	init_nodemask_of_node(&m, (node)); \
1da177e4 LT	285	} \
	286	m; \
	287	})
	288
	289	#define first_unset_node(mask) __first_unset_node(&(mask))
	290	static inline int __first_unset_node(const nodemask_t *maskp)
	291	{
	292	return min_t(int,MAX_NUMNODES,
	293	find_first_zero_bit(maskp->bits, MAX_NUMNODES));
	294	}
	295
	296	#define NODE_MASK_LAST_WORD BITMAP_LAST_WORD_MASK(MAX_NUMNODES)
	297
	298	#if MAX_NUMNODES <= BITS_PER_LONG
	299
	300	#define NODE_MASK_ALL \
	301	((nodemask_t) { { \
	302	[BITS_TO_LONGS(MAX_NUMNODES)-1] = NODE_MASK_LAST_WORD \
	303	} })
	304
	305	#else
	306
	307	#define NODE_MASK_ALL \
	308	((nodemask_t) { { \
	309	[0 ... BITS_TO_LONGS(MAX_NUMNODES)-2] = ~0UL, \
	310	[BITS_TO_LONGS(MAX_NUMNODES)-1] = NODE_MASK_LAST_WORD \
	311	} })
	312
	313	#endif
	314
	315	#define NODE_MASK_NONE \
	316	((nodemask_t) { { \
	317	[0 ... BITS_TO_LONGS(MAX_NUMNODES)-1] = 0UL \
	318	} })
	319
	320	#define nodes_addr(src) ((src).bits)
	321
01a3ee2b RC	322	#define nodemask_parse_user(ubuf, ulen, dst) \
	323	__nodemask_parse_user((ubuf), (ulen), &(dst), MAX_NUMNODES)
	324	static inline int __nodemask_parse_user(const char __user *buf, int len,
1da177e4 LT	325	nodemask_t *dstp, int nbits)
1da177e4 LT	326	{
01a3ee2b	327	return bitmap_parse_user(buf, len, dstp->bits, nbits);
1da177e4 LT	328	}
1da177e4 LT	329
1da177e4 LT	330	#define nodelist_parse(buf, dst) __nodelist_parse((buf), &(dst), MAX_NUMNODES)
	331	static inline int __nodelist_parse(const char buf, nodemask_t dstp, int nbits)
	332	{
	333	return bitmap_parselist(buf, dstp->bits, nbits);
	334	}
	335
fb5eeeee PJ	336	#define node_remap(oldbit, old, new) \
	337	__node_remap((oldbit), &(old), &(new), MAX_NUMNODES)
	338	static inline int __node_remap(int oldbit,
	339	const nodemask_t oldp, const nodemask_t newp, int nbits)
	340	{
	341	return bitmap_bitremap(oldbit, oldp->bits, newp->bits, nbits);
	342	}
	343
	344	#define nodes_remap(dst, src, old, new) \
	345	__nodes_remap(&(dst), &(src), &(old), &(new), MAX_NUMNODES)
	346	static inline void __nodes_remap(nodemask_t dstp, const nodemask_t srcp,
	347	const nodemask_t oldp, const nodemask_t newp, int nbits)
	348	{
	349	bitmap_remap(dstp->bits, srcp->bits, oldp->bits, newp->bits, nbits);
	350	}
	351
7ea931c9 PJ	352	#define nodes_onto(dst, orig, relmap) \
	353	__nodes_onto(&(dst), &(orig), &(relmap), MAX_NUMNODES)
	354	static inline void __nodes_onto(nodemask_t dstp, const nodemask_t origp,
	355	const nodemask_t *relmapp, int nbits)
	356	{
	357	bitmap_onto(dstp->bits, origp->bits, relmapp->bits, nbits);
	358	}
	359
	360	#define nodes_fold(dst, orig, sz) \
	361	__nodes_fold(&(dst), &(orig), sz, MAX_NUMNODES)
	362	static inline void __nodes_fold(nodemask_t dstp, const nodemask_t origp,
	363	int sz, int nbits)
	364	{
	365	bitmap_fold(dstp->bits, origp->bits, sz, nbits);
	366	}
	367
1da177e4 LT	368	#if MAX_NUMNODES > 1
	369	#define for_each_node_mask(node, mask) \
	370	for ((node) = first_node(mask); \
	371	(node) < MAX_NUMNODES; \
	372	(node) = next_node((node), (mask)))
	373	#else /* MAX_NUMNODES == 1 */
	374	#define for_each_node_mask(node, mask) \
	375	if (!nodes_empty(mask)) \
	376	for ((node) = 0; (node) < 1; (node)++)
	377	#endif /* MAX_NUMNODES */
	378
13808910 CL	379	/*
	380	* Bitmasks that are kept for all the nodes.
	381	*/
	382	enum node_states {
7ea1530a CL	383	N_POSSIBLE, /* The node could become online at some point */
	384	N_ONLINE, /* The node is online */
	385	N_NORMAL_MEMORY, /* The node has regular memory */
	386	#ifdef CONFIG_HIGHMEM
	387	N_HIGH_MEMORY, /* The node has regular or high memory */
	388	#else
	389	N_HIGH_MEMORY = N_NORMAL_MEMORY,
	390	#endif
20b2f52b	391	N_MEMORY, /* The node has memory(regular, high, movable) */
37c0708d	392	N_CPU, /* The node has one or more cpus */
13808910 CL	393	NR_NODE_STATES
	394	};
	395
1da177e4 LT	396	/*
	397	* The following particular system nodemasks and operations
	398	* on them manage all possible and online nodes.
	399	*/
	400
13808910	401	extern nodemask_t node_states[NR_NODE_STATES];
1da177e4 LT	402
1da177e4 LT	403	#if MAX_NUMNODES > 1
13808910 CL	404	static inline int node_state(int node, enum node_states state)
	405	{
	406	return node_isset(node, node_states[state]);
	407	}
	408
	409	static inline void node_set_state(int node, enum node_states state)
	410	{
	411	__node_set(node, &node_states[state]);
	412	}
	413
	414	static inline void node_clear_state(int node, enum node_states state)
	415	{
	416	__node_clear(node, &node_states[state]);
	417	}
	418
	419	static inline int num_node_state(enum node_states state)
	420	{
	421	return nodes_weight(node_states[state]);
	422	}
	423
	424	#define for_each_node_state(__node, __state) \
	425	for_each_node_mask((__node), node_states[__state])
	426
	427	#define first_online_node first_node(node_states[N_ONLINE])
8d060bf4 DR	428	#define first_memory_node first_node(node_states[N_MEMORY])
	429	static inline int next_online_node(int nid)
	430	{
	431	return next_node(nid, node_states[N_ONLINE]);
	432	}
	433	static inline int next_memory_node(int nid)
	434	{
	435	return next_node(nid, node_states[N_MEMORY]);
	436	}
13808910	437
74c7aa8b	438	extern int nr_node_ids;
62bc62a8 CL	439	extern int nr_online_nodes;
	440
	441	static inline void node_set_online(int nid)
	442	{
	443	node_set_state(nid, N_ONLINE);
	444	nr_online_nodes = num_node_state(N_ONLINE);
	445	}
	446
	447	static inline void node_set_offline(int nid)
	448	{
	449	node_clear_state(nid, N_ONLINE);
	450	nr_online_nodes = num_node_state(N_ONLINE);
	451	}
778d3b0f	452
1da177e4	453	#else
13808910 CL	454
	455	static inline int node_state(int node, enum node_states state)
	456	{
	457	return node == 0;
	458	}
	459
	460	static inline void node_set_state(int node, enum node_states state)
	461	{
	462	}
	463
	464	static inline void node_clear_state(int node, enum node_states state)
	465	{
	466	}
	467
	468	static inline int num_node_state(enum node_states state)
	469	{
	470	return 1;
	471	}
	472
	473	#define for_each_node_state(node, __state) \
	474	for ( (node) = 0; (node) == 0; (node) = 1)
	475
8357f869	476	#define first_online_node 0
8d060bf4	477	#define first_memory_node 0
8357f869	478	#define next_online_node(nid) (MAX_NUMNODES)
74c7aa8b	479	#define nr_node_ids 1
62bc62a8	480	#define nr_online_nodes 1
13808910	481
62bc62a8 CL	482	#define node_set_online(node) node_set_state((node), N_ONLINE)
62bc62a8 CL	483	#define node_set_offline(node) node_clear_state((node), N_ONLINE)
778d3b0f MH	484
	485	#endif
	486
	487	#if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1)
	488	extern int node_random(const nodemask_t *maskp);
	489	#else
	490	static inline int node_random(const nodemask_t *mask)
	491	{
	492	return 0;
	493	}
1da177e4 LT	494	#endif
1da177e4 LT	495
13808910 CL	496	#define node_online_map node_states[N_ONLINE]
	497	#define node_possible_map node_states[N_POSSIBLE]
	498
13808910 CL	499	#define num_online_nodes() num_node_state(N_ONLINE)
	500	#define num_possible_nodes() num_node_state(N_POSSIBLE)
	501	#define node_online(node) node_state((node), N_ONLINE)
	502	#define node_possible(node) node_state((node), N_POSSIBLE)
	503
13808910 CL	504	#define for_each_node(node) for_each_node_state(node, N_POSSIBLE)
13808910 CL	505	#define for_each_online_node(node) for_each_node_state(node, N_ONLINE)
1da177e4	506
4bfc4495	507	/*
bad44b5b DR	508	* For nodemask scrach area.
	509	* NODEMASK_ALLOC(type, name) allocates an object with a specified type and
	510	* name.
4bfc4495	511	*/
bad44b5b DR	512	#if NODES_SHIFT > 8 /* nodemask_t > 256 bytes */
	513	#define NODEMASK_ALLOC(type, name, gfp_flags) \
	514	type name = kmalloc(sizeof(name), gfp_flags)
	515	#define NODEMASK_FREE(m) kfree(m)
4bfc4495	516	#else
7baab93f	517	#define NODEMASK_ALLOC(type, name, gfp_flags) type _##name, *name = &_##name
bad44b5b	518	#define NODEMASK_FREE(m) do {} while (0)
4bfc4495 KH	519	#endif
	520
	521	/* A example struture for using NODEMASK_ALLOC, used in mempolicy. */
	522	struct nodemask_scratch {
	523	nodemask_t mask1;
	524	nodemask_t mask2;
	525	};
	526
bad44b5b DR	527	#define NODEMASK_SCRATCH(x) \
	528	NODEMASK_ALLOC(struct nodemask_scratch, x, \
	529	GFP_KERNEL \| __GFP_NORETRY)
4e7b8a6c	530	#define NODEMASK_SCRATCH_FREE(x) NODEMASK_FREE(x)
4bfc4495 KH	531
4bfc4495 KH	532
1da177e4	533	#endif /* __LINUX_NODEMASK_H */