]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
798ab48e KO |
2 | #ifndef __PERCPU_IDA_H__ |
3 | #define __PERCPU_IDA_H__ | |
4 | ||
5 | #include <linux/types.h> | |
6 | #include <linux/bitops.h> | |
7 | #include <linux/init.h> | |
6f6b5d1e | 8 | #include <linux/sched.h> |
798ab48e KO |
9 | #include <linux/spinlock_types.h> |
10 | #include <linux/wait.h> | |
11 | #include <linux/cpumask.h> | |
12 | ||
13 | struct percpu_ida_cpu; | |
14 | ||
15 | struct percpu_ida { | |
16 | /* | |
17 | * number of tags available to be allocated, as passed to | |
18 | * percpu_ida_init() | |
19 | */ | |
20 | unsigned nr_tags; | |
e26b53d0 SL |
21 | unsigned percpu_max_size; |
22 | unsigned percpu_batch_size; | |
798ab48e KO |
23 | |
24 | struct percpu_ida_cpu __percpu *tag_cpu; | |
25 | ||
26 | /* | |
27 | * Bitmap of cpus that (may) have tags on their percpu freelists: | |
28 | * steal_tags() uses this to decide when to steal tags, and which cpus | |
29 | * to try stealing from. | |
30 | * | |
31 | * It's ok for a freelist to be empty when its bit is set - steal_tags() | |
32 | * will just keep looking - but the bitmap _must_ be set whenever a | |
33 | * percpu freelist does have tags. | |
34 | */ | |
35 | cpumask_t cpus_have_tags; | |
36 | ||
37 | struct { | |
38 | spinlock_t lock; | |
39 | /* | |
40 | * When we go to steal tags from another cpu (see steal_tags()), | |
41 | * we want to pick a cpu at random. Cycling through them every | |
42 | * time we steal is a bit easier and more or less equivalent: | |
43 | */ | |
44 | unsigned cpu_last_stolen; | |
45 | ||
46 | /* For sleeping on allocation failure */ | |
47 | wait_queue_head_t wait; | |
48 | ||
49 | /* | |
50 | * Global freelist - it's a stack where nr_free points to the | |
51 | * top | |
52 | */ | |
53 | unsigned nr_free; | |
54 | unsigned *freelist; | |
55 | } ____cacheline_aligned_in_smp; | |
56 | }; | |
57 | ||
e26b53d0 SL |
58 | /* |
59 | * Number of tags we move between the percpu freelist and the global freelist at | |
60 | * a time | |
61 | */ | |
62 | #define IDA_DEFAULT_PCPU_BATCH_MOVE 32U | |
63 | /* Max size of percpu freelist, */ | |
64 | #define IDA_DEFAULT_PCPU_SIZE ((IDA_DEFAULT_PCPU_BATCH_MOVE * 3) / 2) | |
65 | ||
6f6b5d1e | 66 | int percpu_ida_alloc(struct percpu_ida *pool, int state); |
798ab48e KO |
67 | void percpu_ida_free(struct percpu_ida *pool, unsigned tag); |
68 | ||
69 | void percpu_ida_destroy(struct percpu_ida *pool); | |
e26b53d0 SL |
70 | int __percpu_ida_init(struct percpu_ida *pool, unsigned long nr_tags, |
71 | unsigned long max_size, unsigned long batch_size); | |
72 | static inline int percpu_ida_init(struct percpu_ida *pool, unsigned long nr_tags) | |
73 | { | |
74 | return __percpu_ida_init(pool, nr_tags, IDA_DEFAULT_PCPU_SIZE, | |
75 | IDA_DEFAULT_PCPU_BATCH_MOVE); | |
76 | } | |
798ab48e | 77 | |
7fc2ba17 SL |
78 | typedef int (*percpu_ida_cb)(unsigned, void *); |
79 | int percpu_ida_for_each_free(struct percpu_ida *pool, percpu_ida_cb fn, | |
80 | void *data); | |
81 | ||
1dddc01a | 82 | unsigned percpu_ida_free_tags(struct percpu_ida *pool, int cpu); |
798ab48e | 83 | #endif /* __PERCPU_IDA_H__ */ |