]>
Commit | Line | Data |
---|---|---|
5a0e3ad6 | 1 | #include <linux/slab.h> |
ccb46000 AM |
2 | #include <linux/kernel.h> |
3 | #include <linux/bitops.h> | |
4 | #include <linux/cpumask.h> | |
8bc3bcc9 | 5 | #include <linux/export.h> |
2d3854a3 | 6 | #include <linux/bootmem.h> |
ccb46000 | 7 | |
2d3854a3 RR |
8 | /** |
9 | * cpumask_next_and - get the next cpu in *src1p & *src2p | |
10 | * @n: the cpu prior to the place to search (ie. return will be > @n) | |
11 | * @src1p: the first cpumask pointer | |
12 | * @src2p: the second cpumask pointer | |
13 | * | |
14 | * Returns >= nr_cpu_ids if no further cpus set in both. | |
15 | */ | |
16 | int cpumask_next_and(int n, const struct cpumask *src1p, | |
17 | const struct cpumask *src2p) | |
18 | { | |
534b483a SS |
19 | struct cpumask tmp; |
20 | ||
21 | if (cpumask_and(&tmp, src1p, src2p)) | |
22 | return cpumask_next(n, &tmp); | |
23 | return nr_cpu_ids; | |
2d3854a3 RR |
24 | } |
25 | EXPORT_SYMBOL(cpumask_next_and); | |
26 | ||
27 | /** | |
28 | * cpumask_any_but - return a "random" in a cpumask, but not this one. | |
29 | * @mask: the cpumask to search | |
30 | * @cpu: the cpu to ignore. | |
31 | * | |
32 | * Often used to find any cpu but smp_processor_id() in a mask. | |
33 | * Returns >= nr_cpu_ids if no cpus set. | |
34 | */ | |
35 | int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) | |
36 | { | |
37 | unsigned int i; | |
38 | ||
984f2f37 | 39 | cpumask_check(cpu); |
2d3854a3 RR |
40 | for_each_cpu(i, mask) |
41 | if (i != cpu) | |
42 | break; | |
43 | return i; | |
44 | } | |
45 | ||
46 | /* These are not inline because of header tangles. */ | |
47 | #ifdef CONFIG_CPUMASK_OFFSTACK | |
ec26b805 MT |
48 | /** |
49 | * alloc_cpumask_var_node - allocate a struct cpumask on a given node | |
50 | * @mask: pointer to cpumask_var_t where the cpumask is returned | |
51 | * @flags: GFP_ flags | |
52 | * | |
53 | * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is | |
54 | * a nop returning a constant 1 (in <linux/cpumask.h>) | |
55 | * Returns TRUE if memory allocation succeeded, FALSE otherwise. | |
56 | * | |
57 | * In addition, mask will be NULL if this fails. Note that gcc is | |
58 | * usually smart enough to know that mask can never be NULL if | |
59 | * CONFIG_CPUMASK_OFFSTACK=n, so does code elimination in that case | |
60 | * too. | |
61 | */ | |
7b4967c5 | 62 | bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) |
2d3854a3 | 63 | { |
38c7fed2 YL |
64 | *mask = kmalloc_node(cpumask_size(), flags, node); |
65 | ||
2d3854a3 RR |
66 | #ifdef CONFIG_DEBUG_PER_CPU_MAPS |
67 | if (!*mask) { | |
68 | printk(KERN_ERR "=> alloc_cpumask_var: failed!\n"); | |
69 | dump_stack(); | |
70 | } | |
71 | #endif | |
2a530080 | 72 | |
2d3854a3 RR |
73 | return *mask != NULL; |
74 | } | |
7b4967c5 MT |
75 | EXPORT_SYMBOL(alloc_cpumask_var_node); |
76 | ||
0281b5dc YL |
77 | bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) |
78 | { | |
79 | return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node); | |
80 | } | |
81 | EXPORT_SYMBOL(zalloc_cpumask_var_node); | |
82 | ||
ec26b805 MT |
83 | /** |
84 | * alloc_cpumask_var - allocate a struct cpumask | |
85 | * @mask: pointer to cpumask_var_t where the cpumask is returned | |
86 | * @flags: GFP_ flags | |
87 | * | |
88 | * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is | |
89 | * a nop returning a constant 1 (in <linux/cpumask.h>). | |
90 | * | |
91 | * See alloc_cpumask_var_node. | |
92 | */ | |
7b4967c5 MT |
93 | bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) |
94 | { | |
37e7b5f1 | 95 | return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE); |
7b4967c5 | 96 | } |
2d3854a3 RR |
97 | EXPORT_SYMBOL(alloc_cpumask_var); |
98 | ||
0281b5dc YL |
99 | bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) |
100 | { | |
101 | return alloc_cpumask_var(mask, flags | __GFP_ZERO); | |
102 | } | |
103 | EXPORT_SYMBOL(zalloc_cpumask_var); | |
104 | ||
ec26b805 MT |
105 | /** |
106 | * alloc_bootmem_cpumask_var - allocate a struct cpumask from the bootmem arena. | |
107 | * @mask: pointer to cpumask_var_t where the cpumask is returned | |
108 | * | |
109 | * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is | |
e9690a6e | 110 | * a nop (in <linux/cpumask.h>). |
ec26b805 MT |
111 | * Either returns an allocated (zero-filled) cpumask, or causes the |
112 | * system to panic. | |
113 | */ | |
2d3854a3 RR |
114 | void __init alloc_bootmem_cpumask_var(cpumask_var_t *mask) |
115 | { | |
c1529500 | 116 | *mask = memblock_virt_alloc(cpumask_size(), 0); |
2d3854a3 RR |
117 | } |
118 | ||
ec26b805 MT |
119 | /** |
120 | * free_cpumask_var - frees memory allocated for a struct cpumask. | |
121 | * @mask: cpumask to free | |
122 | * | |
123 | * This is safe on a NULL mask. | |
124 | */ | |
2d3854a3 RR |
125 | void free_cpumask_var(cpumask_var_t mask) |
126 | { | |
127 | kfree(mask); | |
128 | } | |
129 | EXPORT_SYMBOL(free_cpumask_var); | |
cd83e42c | 130 | |
ec26b805 MT |
131 | /** |
132 | * free_bootmem_cpumask_var - frees result of alloc_bootmem_cpumask_var | |
133 | * @mask: cpumask to free | |
134 | */ | |
984f2f37 | 135 | void __init free_bootmem_cpumask_var(cpumask_var_t mask) |
cd83e42c | 136 | { |
c1529500 | 137 | memblock_free_early(__pa(mask), cpumask_size()); |
cd83e42c | 138 | } |
2d3854a3 | 139 | #endif |
da91309e AV |
140 | |
141 | /** | |
142 | * cpumask_set_cpu_local_first - set i'th cpu with local numa cpu's first | |
143 | * | |
144 | * @i: index number | |
145 | * @numa_node: local numa_node | |
146 | * @dstp: cpumask with the relevant cpu bit set according to the policy | |
147 | * | |
148 | * This function sets the cpumask according to a numa aware policy. | |
149 | * cpumask could be used as an affinity hint for the IRQ related to a | |
150 | * queue. When the policy is to spread queues across cores - local cores | |
151 | * first. | |
152 | * | |
153 | * Returns 0 on success, -ENOMEM for no memory, and -EAGAIN when failed to set | |
154 | * the cpu bit and need to re-call the function. | |
155 | */ | |
156 | int cpumask_set_cpu_local_first(int i, int numa_node, cpumask_t *dstp) | |
157 | { | |
158 | cpumask_var_t mask; | |
159 | int cpu; | |
160 | int ret = 0; | |
161 | ||
162 | if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) | |
163 | return -ENOMEM; | |
164 | ||
165 | i %= num_online_cpus(); | |
166 | ||
143b5ba2 | 167 | if (numa_node == -1 || !cpumask_of_node(numa_node)) { |
da91309e AV |
168 | /* Use all online cpu's for non numa aware system */ |
169 | cpumask_copy(mask, cpu_online_mask); | |
170 | } else { | |
171 | int n; | |
172 | ||
173 | cpumask_and(mask, | |
174 | cpumask_of_node(numa_node), cpu_online_mask); | |
175 | ||
176 | n = cpumask_weight(mask); | |
177 | if (i >= n) { | |
178 | i -= n; | |
179 | ||
180 | /* If index > number of local cpu's, mask out local | |
181 | * cpu's | |
182 | */ | |
183 | cpumask_andnot(mask, cpu_online_mask, mask); | |
184 | } | |
185 | } | |
186 | ||
187 | for_each_cpu(cpu, mask) { | |
188 | if (--i < 0) | |
189 | goto out; | |
190 | } | |
191 | ||
192 | ret = -EAGAIN; | |
193 | ||
194 | out: | |
195 | free_cpumask_var(mask); | |
196 | ||
197 | if (!ret) | |
198 | cpumask_set_cpu(cpu, dstp); | |
199 | ||
200 | return ret; | |
201 | } | |
202 | EXPORT_SYMBOL(cpumask_set_cpu_local_first); |