]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - include/linux/nodemask.h
8021q: fix a potential memory leak
[mirror_ubuntu-bionic-kernel.git] / include / linux / nodemask.h
1 #ifndef __LINUX_NODEMASK_H
2 #define __LINUX_NODEMASK_H
3
4 /*
5 * Nodemasks provide a bitmap suitable for representing the
6 * set of Node's in a system, one bit position per Node number.
7 *
8 * See detailed comments in the file linux/bitmap.h describing the
9 * data type on which these nodemasks are based.
10 *
11 * For details of nodemask_scnprintf() and nodemask_parse_user(),
12 * see bitmap_scnprintf() and bitmap_parse_user() in lib/bitmap.c.
13 * For details of nodelist_scnprintf() and nodelist_parse(), see
14 * bitmap_scnlistprintf() and bitmap_parselist(), also in bitmap.c.
15 * For details of node_remap(), see bitmap_bitremap in lib/bitmap.c.
16 * For details of nodes_remap(), see bitmap_remap in lib/bitmap.c.
17 * For details of nodes_onto(), see bitmap_onto in lib/bitmap.c.
18 * For details of nodes_fold(), see bitmap_fold in lib/bitmap.c.
19 *
20 * The available nodemask operations are:
21 *
22 * void node_set(node, mask) turn on bit 'node' in mask
23 * void node_clear(node, mask) turn off bit 'node' in mask
24 * void nodes_setall(mask) set all bits
25 * void nodes_clear(mask) clear all bits
26 * int node_isset(node, mask) true iff bit 'node' set in mask
27 * int node_test_and_set(node, mask) test and set bit 'node' in mask
28 *
29 * void nodes_and(dst, src1, src2) dst = src1 & src2 [intersection]
30 * void nodes_or(dst, src1, src2) dst = src1 | src2 [union]
31 * void nodes_xor(dst, src1, src2) dst = src1 ^ src2
32 * void nodes_andnot(dst, src1, src2) dst = src1 & ~src2
33 * void nodes_complement(dst, src) dst = ~src
34 *
35 * int nodes_equal(mask1, mask2) Does mask1 == mask2?
36 * int nodes_intersects(mask1, mask2) Do mask1 and mask2 intersect?
37 * int nodes_subset(mask1, mask2) Is mask1 a subset of mask2?
38 * int nodes_empty(mask) Is mask empty (no bits sets)?
39 * int nodes_full(mask) Is mask full (all bits sets)?
40 * int nodes_weight(mask) Hamming weight - number of set bits
41 *
42 * void nodes_shift_right(dst, src, n) Shift right
43 * void nodes_shift_left(dst, src, n) Shift left
44 *
45 * int first_node(mask) Number lowest set bit, or MAX_NUMNODES
46 * int next_node(node, mask) Next node past 'node', or MAX_NUMNODES
47 * int first_unset_node(mask) First node not set in mask, or
48 * MAX_NUMNODES.
49 *
50 * nodemask_t nodemask_of_node(node) Return nodemask with bit 'node' set
51 * NODE_MASK_ALL Initializer - all bits set
52 * NODE_MASK_NONE Initializer - no bits set
53 * unsigned long *nodes_addr(mask) Array of unsigned long's in mask
54 *
55 * int nodemask_scnprintf(buf, len, mask) Format nodemask for printing
56 * int nodemask_parse_user(ubuf, ulen, mask) Parse ascii string as nodemask
57 * int nodelist_scnprintf(buf, len, mask) Format nodemask as list for printing
58 * int nodelist_parse(buf, map) Parse ascii string as nodelist
59 * int node_remap(oldbit, old, new) newbit = map(old, new)(oldbit)
60 * void nodes_remap(dst, src, old, new) *dst = map(old, new)(src)
61 * void nodes_onto(dst, orig, relmap) *dst = orig relative to relmap
62 * void nodes_fold(dst, orig, sz) dst bits = orig bits mod sz
63 *
64 * for_each_node_mask(node, mask) for-loop node over mask
65 *
66 * int num_online_nodes() Number of online Nodes
67 * int num_possible_nodes() Number of all possible Nodes
68 *
69 * int node_random(mask) Random node with set bit in mask
70 *
71 * int node_online(node) Is some node online?
72 * int node_possible(node) Is some node possible?
73 *
74 * node_set_online(node) set bit 'node' in node_online_map
75 * node_set_offline(node) clear bit 'node' in node_online_map
76 *
77 * for_each_node(node) for-loop node over node_possible_map
78 * for_each_online_node(node) for-loop node over node_online_map
79 *
80 * Subtlety:
81 * 1) The 'type-checked' form of node_isset() causes gcc (3.3.2, anyway)
82 * to generate slightly worse code. So use a simple one-line #define
83 * for node_isset(), instead of wrapping an inline inside a macro, the
84 * way we do the other calls.
85 *
86 * NODEMASK_SCRATCH
87 * When doing above logical AND, OR, XOR, Remap operations the callers tend to
88 * need temporary nodemask_t's on the stack. But if NODES_SHIFT is large,
89 * nodemask_t's consume too much stack space. NODEMASK_SCRATCH is a helper
90 * for such situations. See below and CPUMASK_ALLOC also.
91 */
92
93 #include <linux/kernel.h>
94 #include <linux/threads.h>
95 #include <linux/bitmap.h>
96 #include <linux/numa.h>
97
98 typedef struct { DECLARE_BITMAP(bits, MAX_NUMNODES); } nodemask_t;
99 extern nodemask_t _unused_nodemask_arg_;
100
101 /*
102 * The inline keyword gives the compiler room to decide to inline, or
103 * not inline a function as it sees best. However, as these functions
104 * are called in both __init and non-__init functions, if they are not
105 * inlined we will end up with a section mis-match error (of the type of
106 * freeable items not being freed). So we must use __always_inline here
107 * to fix the problem. If other functions in the future also end up in
108 * this situation they will also need to be annotated as __always_inline
109 */
110 #define node_set(node, dst) __node_set((node), &(dst))
111 static __always_inline void __node_set(int node, volatile nodemask_t *dstp)
112 {
113 set_bit(node, dstp->bits);
114 }
115
116 #define node_clear(node, dst) __node_clear((node), &(dst))
117 static inline void __node_clear(int node, volatile nodemask_t *dstp)
118 {
119 clear_bit(node, dstp->bits);
120 }
121
122 #define nodes_setall(dst) __nodes_setall(&(dst), MAX_NUMNODES)
123 static inline void __nodes_setall(nodemask_t *dstp, int nbits)
124 {
125 bitmap_fill(dstp->bits, nbits);
126 }
127
128 #define nodes_clear(dst) __nodes_clear(&(dst), MAX_NUMNODES)
129 static inline void __nodes_clear(nodemask_t *dstp, int nbits)
130 {
131 bitmap_zero(dstp->bits, nbits);
132 }
133
134 /* No static inline type checking - see Subtlety (1) above. */
135 #define node_isset(node, nodemask) test_bit((node), (nodemask).bits)
136
137 #define node_test_and_set(node, nodemask) \
138 __node_test_and_set((node), &(nodemask))
139 static inline int __node_test_and_set(int node, nodemask_t *addr)
140 {
141 return test_and_set_bit(node, addr->bits);
142 }
143
144 #define nodes_and(dst, src1, src2) \
145 __nodes_and(&(dst), &(src1), &(src2), MAX_NUMNODES)
146 static inline void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p,
147 const nodemask_t *src2p, int nbits)
148 {
149 bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits);
150 }
151
152 #define nodes_or(dst, src1, src2) \
153 __nodes_or(&(dst), &(src1), &(src2), MAX_NUMNODES)
154 static inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p,
155 const nodemask_t *src2p, int nbits)
156 {
157 bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits);
158 }
159
160 #define nodes_xor(dst, src1, src2) \
161 __nodes_xor(&(dst), &(src1), &(src2), MAX_NUMNODES)
162 static inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p,
163 const nodemask_t *src2p, int nbits)
164 {
165 bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits);
166 }
167
168 #define nodes_andnot(dst, src1, src2) \
169 __nodes_andnot(&(dst), &(src1), &(src2), MAX_NUMNODES)
170 static inline void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p,
171 const nodemask_t *src2p, int nbits)
172 {
173 bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits);
174 }
175
176 #define nodes_complement(dst, src) \
177 __nodes_complement(&(dst), &(src), MAX_NUMNODES)
178 static inline void __nodes_complement(nodemask_t *dstp,
179 const nodemask_t *srcp, int nbits)
180 {
181 bitmap_complement(dstp->bits, srcp->bits, nbits);
182 }
183
184 #define nodes_equal(src1, src2) \
185 __nodes_equal(&(src1), &(src2), MAX_NUMNODES)
186 static inline int __nodes_equal(const nodemask_t *src1p,
187 const nodemask_t *src2p, int nbits)
188 {
189 return bitmap_equal(src1p->bits, src2p->bits, nbits);
190 }
191
192 #define nodes_intersects(src1, src2) \
193 __nodes_intersects(&(src1), &(src2), MAX_NUMNODES)
194 static inline int __nodes_intersects(const nodemask_t *src1p,
195 const nodemask_t *src2p, int nbits)
196 {
197 return bitmap_intersects(src1p->bits, src2p->bits, nbits);
198 }
199
200 #define nodes_subset(src1, src2) \
201 __nodes_subset(&(src1), &(src2), MAX_NUMNODES)
202 static inline int __nodes_subset(const nodemask_t *src1p,
203 const nodemask_t *src2p, int nbits)
204 {
205 return bitmap_subset(src1p->bits, src2p->bits, nbits);
206 }
207
208 #define nodes_empty(src) __nodes_empty(&(src), MAX_NUMNODES)
209 static inline int __nodes_empty(const nodemask_t *srcp, int nbits)
210 {
211 return bitmap_empty(srcp->bits, nbits);
212 }
213
214 #define nodes_full(nodemask) __nodes_full(&(nodemask), MAX_NUMNODES)
215 static inline int __nodes_full(const nodemask_t *srcp, int nbits)
216 {
217 return bitmap_full(srcp->bits, nbits);
218 }
219
220 #define nodes_weight(nodemask) __nodes_weight(&(nodemask), MAX_NUMNODES)
221 static inline int __nodes_weight(const nodemask_t *srcp, int nbits)
222 {
223 return bitmap_weight(srcp->bits, nbits);
224 }
225
226 #define nodes_shift_right(dst, src, n) \
227 __nodes_shift_right(&(dst), &(src), (n), MAX_NUMNODES)
228 static inline void __nodes_shift_right(nodemask_t *dstp,
229 const nodemask_t *srcp, int n, int nbits)
230 {
231 bitmap_shift_right(dstp->bits, srcp->bits, n, nbits);
232 }
233
234 #define nodes_shift_left(dst, src, n) \
235 __nodes_shift_left(&(dst), &(src), (n), MAX_NUMNODES)
236 static inline void __nodes_shift_left(nodemask_t *dstp,
237 const nodemask_t *srcp, int n, int nbits)
238 {
239 bitmap_shift_left(dstp->bits, srcp->bits, n, nbits);
240 }
241
242 /* FIXME: better would be to fix all architectures to never return
243 > MAX_NUMNODES, then the silly min_ts could be dropped. */
244
245 #define first_node(src) __first_node(&(src))
246 static inline int __first_node(const nodemask_t *srcp)
247 {
248 return min_t(int, MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES));
249 }
250
251 #define next_node(n, src) __next_node((n), &(src))
252 static inline int __next_node(int n, const nodemask_t *srcp)
253 {
254 return min_t(int,MAX_NUMNODES,find_next_bit(srcp->bits, MAX_NUMNODES, n+1));
255 }
256
257 static inline void init_nodemask_of_node(nodemask_t *mask, int node)
258 {
259 nodes_clear(*mask);
260 node_set(node, *mask);
261 }
262
263 #define nodemask_of_node(node) \
264 ({ \
265 typeof(_unused_nodemask_arg_) m; \
266 if (sizeof(m) == sizeof(unsigned long)) { \
267 m.bits[0] = 1UL << (node); \
268 } else { \
269 init_nodemask_of_node(&m, (node)); \
270 } \
271 m; \
272 })
273
274 #define first_unset_node(mask) __first_unset_node(&(mask))
275 static inline int __first_unset_node(const nodemask_t *maskp)
276 {
277 return min_t(int,MAX_NUMNODES,
278 find_first_zero_bit(maskp->bits, MAX_NUMNODES));
279 }
280
281 #define NODE_MASK_LAST_WORD BITMAP_LAST_WORD_MASK(MAX_NUMNODES)
282
283 #if MAX_NUMNODES <= BITS_PER_LONG
284
285 #define NODE_MASK_ALL \
286 ((nodemask_t) { { \
287 [BITS_TO_LONGS(MAX_NUMNODES)-1] = NODE_MASK_LAST_WORD \
288 } })
289
290 #else
291
292 #define NODE_MASK_ALL \
293 ((nodemask_t) { { \
294 [0 ... BITS_TO_LONGS(MAX_NUMNODES)-2] = ~0UL, \
295 [BITS_TO_LONGS(MAX_NUMNODES)-1] = NODE_MASK_LAST_WORD \
296 } })
297
298 #endif
299
300 #define NODE_MASK_NONE \
301 ((nodemask_t) { { \
302 [0 ... BITS_TO_LONGS(MAX_NUMNODES)-1] = 0UL \
303 } })
304
305 #define nodes_addr(src) ((src).bits)
306
307 #define nodemask_scnprintf(buf, len, src) \
308 __nodemask_scnprintf((buf), (len), &(src), MAX_NUMNODES)
309 static inline int __nodemask_scnprintf(char *buf, int len,
310 const nodemask_t *srcp, int nbits)
311 {
312 return bitmap_scnprintf(buf, len, srcp->bits, nbits);
313 }
314
315 #define nodemask_parse_user(ubuf, ulen, dst) \
316 __nodemask_parse_user((ubuf), (ulen), &(dst), MAX_NUMNODES)
317 static inline int __nodemask_parse_user(const char __user *buf, int len,
318 nodemask_t *dstp, int nbits)
319 {
320 return bitmap_parse_user(buf, len, dstp->bits, nbits);
321 }
322
323 #define nodelist_scnprintf(buf, len, src) \
324 __nodelist_scnprintf((buf), (len), &(src), MAX_NUMNODES)
325 static inline int __nodelist_scnprintf(char *buf, int len,
326 const nodemask_t *srcp, int nbits)
327 {
328 return bitmap_scnlistprintf(buf, len, srcp->bits, nbits);
329 }
330
331 #define nodelist_parse(buf, dst) __nodelist_parse((buf), &(dst), MAX_NUMNODES)
332 static inline int __nodelist_parse(const char *buf, nodemask_t *dstp, int nbits)
333 {
334 return bitmap_parselist(buf, dstp->bits, nbits);
335 }
336
337 #define node_remap(oldbit, old, new) \
338 __node_remap((oldbit), &(old), &(new), MAX_NUMNODES)
339 static inline int __node_remap(int oldbit,
340 const nodemask_t *oldp, const nodemask_t *newp, int nbits)
341 {
342 return bitmap_bitremap(oldbit, oldp->bits, newp->bits, nbits);
343 }
344
345 #define nodes_remap(dst, src, old, new) \
346 __nodes_remap(&(dst), &(src), &(old), &(new), MAX_NUMNODES)
347 static inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp,
348 const nodemask_t *oldp, const nodemask_t *newp, int nbits)
349 {
350 bitmap_remap(dstp->bits, srcp->bits, oldp->bits, newp->bits, nbits);
351 }
352
353 #define nodes_onto(dst, orig, relmap) \
354 __nodes_onto(&(dst), &(orig), &(relmap), MAX_NUMNODES)
355 static inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp,
356 const nodemask_t *relmapp, int nbits)
357 {
358 bitmap_onto(dstp->bits, origp->bits, relmapp->bits, nbits);
359 }
360
361 #define nodes_fold(dst, orig, sz) \
362 __nodes_fold(&(dst), &(orig), sz, MAX_NUMNODES)
363 static inline void __nodes_fold(nodemask_t *dstp, const nodemask_t *origp,
364 int sz, int nbits)
365 {
366 bitmap_fold(dstp->bits, origp->bits, sz, nbits);
367 }
368
369 #if MAX_NUMNODES > 1
370 #define for_each_node_mask(node, mask) \
371 for ((node) = first_node(mask); \
372 (node) < MAX_NUMNODES; \
373 (node) = next_node((node), (mask)))
374 #else /* MAX_NUMNODES == 1 */
375 #define for_each_node_mask(node, mask) \
376 if (!nodes_empty(mask)) \
377 for ((node) = 0; (node) < 1; (node)++)
378 #endif /* MAX_NUMNODES */
379
380 /*
381 * Bitmasks that are kept for all the nodes.
382 */
383 enum node_states {
384 N_POSSIBLE, /* The node could become online at some point */
385 N_ONLINE, /* The node is online */
386 N_NORMAL_MEMORY, /* The node has regular memory */
387 #ifdef CONFIG_HIGHMEM
388 N_HIGH_MEMORY, /* The node has regular or high memory */
389 #else
390 N_HIGH_MEMORY = N_NORMAL_MEMORY,
391 #endif
392 #ifdef CONFIG_MOVABLE_NODE
393 N_MEMORY, /* The node has memory(regular, high, movable) */
394 #else
395 N_MEMORY = N_HIGH_MEMORY,
396 #endif
397 N_CPU, /* The node has one or more cpus */
398 NR_NODE_STATES
399 };
400
401 /*
402 * The following particular system nodemasks and operations
403 * on them manage all possible and online nodes.
404 */
405
406 extern nodemask_t node_states[NR_NODE_STATES];
407
408 #if MAX_NUMNODES > 1
409 static inline int node_state(int node, enum node_states state)
410 {
411 return node_isset(node, node_states[state]);
412 }
413
414 static inline void node_set_state(int node, enum node_states state)
415 {
416 __node_set(node, &node_states[state]);
417 }
418
419 static inline void node_clear_state(int node, enum node_states state)
420 {
421 __node_clear(node, &node_states[state]);
422 }
423
424 static inline int num_node_state(enum node_states state)
425 {
426 return nodes_weight(node_states[state]);
427 }
428
429 #define for_each_node_state(__node, __state) \
430 for_each_node_mask((__node), node_states[__state])
431
432 #define first_online_node first_node(node_states[N_ONLINE])
433 #define next_online_node(nid) next_node((nid), node_states[N_ONLINE])
434
435 extern int nr_node_ids;
436 extern int nr_online_nodes;
437
438 static inline void node_set_online(int nid)
439 {
440 node_set_state(nid, N_ONLINE);
441 nr_online_nodes = num_node_state(N_ONLINE);
442 }
443
444 static inline void node_set_offline(int nid)
445 {
446 node_clear_state(nid, N_ONLINE);
447 nr_online_nodes = num_node_state(N_ONLINE);
448 }
449
450 #else
451
452 static inline int node_state(int node, enum node_states state)
453 {
454 return node == 0;
455 }
456
457 static inline void node_set_state(int node, enum node_states state)
458 {
459 }
460
461 static inline void node_clear_state(int node, enum node_states state)
462 {
463 }
464
465 static inline int num_node_state(enum node_states state)
466 {
467 return 1;
468 }
469
470 #define for_each_node_state(node, __state) \
471 for ( (node) = 0; (node) == 0; (node) = 1)
472
473 #define first_online_node 0
474 #define next_online_node(nid) (MAX_NUMNODES)
475 #define nr_node_ids 1
476 #define nr_online_nodes 1
477
478 #define node_set_online(node) node_set_state((node), N_ONLINE)
479 #define node_set_offline(node) node_clear_state((node), N_ONLINE)
480
481 #endif
482
483 #if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1)
484 extern int node_random(const nodemask_t *maskp);
485 #else
486 static inline int node_random(const nodemask_t *mask)
487 {
488 return 0;
489 }
490 #endif
491
492 #define node_online_map node_states[N_ONLINE]
493 #define node_possible_map node_states[N_POSSIBLE]
494
495 #define num_online_nodes() num_node_state(N_ONLINE)
496 #define num_possible_nodes() num_node_state(N_POSSIBLE)
497 #define node_online(node) node_state((node), N_ONLINE)
498 #define node_possible(node) node_state((node), N_POSSIBLE)
499
500 #define for_each_node(node) for_each_node_state(node, N_POSSIBLE)
501 #define for_each_online_node(node) for_each_node_state(node, N_ONLINE)
502
503 /*
504 * For nodemask scrach area.
505 * NODEMASK_ALLOC(type, name) allocates an object with a specified type and
506 * name.
507 */
508 #if NODES_SHIFT > 8 /* nodemask_t > 256 bytes */
509 #define NODEMASK_ALLOC(type, name, gfp_flags) \
510 type *name = kmalloc(sizeof(*name), gfp_flags)
511 #define NODEMASK_FREE(m) kfree(m)
512 #else
513 #define NODEMASK_ALLOC(type, name, gfp_flags) type _##name, *name = &_##name
514 #define NODEMASK_FREE(m) do {} while (0)
515 #endif
516
517 /* A example struture for using NODEMASK_ALLOC, used in mempolicy. */
518 struct nodemask_scratch {
519 nodemask_t mask1;
520 nodemask_t mask2;
521 };
522
523 #define NODEMASK_SCRATCH(x) \
524 NODEMASK_ALLOC(struct nodemask_scratch, x, \
525 GFP_KERNEL | __GFP_NORETRY)
526 #define NODEMASK_SCRATCH_FREE(x) NODEMASK_FREE(x)
527
528
529 #endif /* __LINUX_NODEMASK_H */