]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
9a0ef98e CH |
2 | /* |
3 | * Copyright (C) 2016 Thomas Gleixner. | |
4 | * Copyright (C) 2016-2017 Christoph Hellwig. | |
5 | */ | |
5e385a6e CH |
6 | #include <linux/interrupt.h> |
7 | #include <linux/kernel.h> | |
8 | #include <linux/slab.h> | |
9 | #include <linux/cpu.h> | |
10 | ||
34c3d981 TG |
11 | static void irq_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk, |
12 | int cpus_per_vec) | |
13 | { | |
14 | const struct cpumask *siblmsk; | |
15 | int cpu, sibl; | |
16 | ||
17 | for ( ; cpus_per_vec > 0; ) { | |
18 | cpu = cpumask_first(nmsk); | |
19 | ||
20 | /* Should not happen, but I'm too lazy to think about it */ | |
21 | if (cpu >= nr_cpu_ids) | |
22 | return; | |
23 | ||
24 | cpumask_clear_cpu(cpu, nmsk); | |
25 | cpumask_set_cpu(cpu, irqmsk); | |
26 | cpus_per_vec--; | |
27 | ||
28 | /* If the cpu has siblings, use them first */ | |
29 | siblmsk = topology_sibling_cpumask(cpu); | |
30 | for (sibl = -1; cpus_per_vec > 0; ) { | |
31 | sibl = cpumask_next(sibl, siblmsk); | |
32 | if (sibl >= nr_cpu_ids) | |
33 | break; | |
34 | if (!cpumask_test_and_clear_cpu(sibl, nmsk)) | |
35 | continue; | |
36 | cpumask_set_cpu(sibl, irqmsk); | |
37 | cpus_per_vec--; | |
38 | } | |
39 | } | |
40 | } | |
41 | ||
c0936d9d | 42 | static cpumask_var_t *alloc_node_to_cpumask(void) |
9a0ef98e CH |
43 | { |
44 | cpumask_var_t *masks; | |
45 | int node; | |
46 | ||
47 | masks = kcalloc(nr_node_ids, sizeof(cpumask_var_t), GFP_KERNEL); | |
48 | if (!masks) | |
49 | return NULL; | |
50 | ||
51 | for (node = 0; node < nr_node_ids; node++) { | |
52 | if (!zalloc_cpumask_var(&masks[node], GFP_KERNEL)) | |
53 | goto out_unwind; | |
54 | } | |
55 | ||
56 | return masks; | |
57 | ||
58 | out_unwind: | |
59 | while (--node >= 0) | |
60 | free_cpumask_var(masks[node]); | |
61 | kfree(masks); | |
62 | return NULL; | |
63 | } | |
64 | ||
c0936d9d | 65 | static void free_node_to_cpumask(cpumask_var_t *masks) |
9a0ef98e CH |
66 | { |
67 | int node; | |
68 | ||
69 | for (node = 0; node < nr_node_ids; node++) | |
70 | free_cpumask_var(masks[node]); | |
71 | kfree(masks); | |
72 | } | |
73 | ||
c0936d9d | 74 | static void build_node_to_cpumask(cpumask_var_t *masks) |
9a0ef98e CH |
75 | { |
76 | int cpu; | |
77 | ||
c189f43b | 78 | for_each_possible_cpu(cpu) |
9a0ef98e CH |
79 | cpumask_set_cpu(cpu, masks[cpu_to_node(cpu)]); |
80 | } | |
81 | ||
c0936d9d | 82 | static int get_nodes_in_cpumask(cpumask_var_t *node_to_cpumask, |
9a0ef98e | 83 | const struct cpumask *mask, nodemask_t *nodemsk) |
34c3d981 | 84 | { |
c0af5243 | 85 | int n, nodes = 0; |
34c3d981 TG |
86 | |
87 | /* Calculate the number of nodes in the supplied affinity mask */ | |
9a0ef98e | 88 | for_each_node(n) { |
c0936d9d | 89 | if (cpumask_intersects(mask, node_to_cpumask[n])) { |
34c3d981 TG |
90 | node_set(n, *nodemsk); |
91 | nodes++; | |
92 | } | |
93 | } | |
94 | return nodes; | |
95 | } | |
96 | ||
83a604e6 ML |
97 | static int irq_build_affinity_masks(const struct irq_affinity *affd, |
98 | int startvec, int numvecs, | |
c192c97f ML |
99 | cpumask_var_t *node_to_cpumask, |
100 | const struct cpumask *cpu_mask, | |
101 | struct cpumask *nmsk, | |
102 | struct cpumask *masks) | |
34c3d981 | 103 | { |
83a604e6 ML |
104 | int n, nodes, cpus_per_vec, extra_vecs, done = 0; |
105 | int last_affv = affd->pre_vectors + numvecs; | |
106 | int curvec = startvec; | |
34c3d981 | 107 | nodemask_t nodemsk = NODE_MASK_NONE; |
34c3d981 | 108 | |
2a4c050c ML |
109 | if (!cpumask_weight(cpu_mask)) |
110 | return 0; | |
111 | ||
c192c97f | 112 | nodes = get_nodes_in_cpumask(node_to_cpumask, cpu_mask, &nodemsk); |
34c3d981 TG |
113 | |
114 | /* | |
c0af5243 | 115 | * If the number of nodes in the mask is greater than or equal the |
34c3d981 TG |
116 | * number of vectors we just spread the vectors across the nodes. |
117 | */ | |
83a604e6 | 118 | if (numvecs <= nodes) { |
34c3d981 | 119 | for_each_node_mask(n, nodemsk) { |
ea8a829d | 120 | cpumask_or(masks + curvec, masks + curvec, node_to_cpumask[n]); |
83a604e6 ML |
121 | if (++curvec == last_affv) |
122 | curvec = affd->pre_vectors; | |
34c3d981 | 123 | } |
ea8a829d | 124 | done = numvecs; |
c192c97f | 125 | goto out; |
34c3d981 TG |
126 | } |
127 | ||
34c3d981 | 128 | for_each_node_mask(n, nodemsk) { |
7bf8222b KB |
129 | int ncpus, v, vecs_to_assign, vecs_per_node; |
130 | ||
131 | /* Spread the vectors per node */ | |
83a604e6 | 132 | vecs_per_node = (numvecs - (curvec - affd->pre_vectors)) / nodes; |
34c3d981 TG |
133 | |
134 | /* Get the cpus on this node which are in the mask */ | |
c192c97f | 135 | cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]); |
34c3d981 TG |
136 | |
137 | /* Calculate the number of cpus per vector */ | |
138 | ncpus = cpumask_weight(nmsk); | |
7bf8222b KB |
139 | vecs_to_assign = min(vecs_per_node, ncpus); |
140 | ||
141 | /* Account for rounding errors */ | |
3412386b | 142 | extra_vecs = ncpus - vecs_to_assign * (ncpus / vecs_to_assign); |
34c3d981 | 143 | |
bfe13077 CH |
144 | for (v = 0; curvec < last_affv && v < vecs_to_assign; |
145 | curvec++, v++) { | |
34c3d981 TG |
146 | cpus_per_vec = ncpus / vecs_to_assign; |
147 | ||
148 | /* Account for extra vectors to compensate rounding errors */ | |
149 | if (extra_vecs) { | |
150 | cpus_per_vec++; | |
7bf8222b | 151 | --extra_vecs; |
34c3d981 TG |
152 | } |
153 | irq_spread_init_one(masks + curvec, nmsk, cpus_per_vec); | |
154 | } | |
155 | ||
83a604e6 ML |
156 | done += v; |
157 | if (done >= numvecs) | |
34c3d981 | 158 | break; |
83a604e6 ML |
159 | if (curvec >= last_affv) |
160 | curvec = affd->pre_vectors; | |
7bf8222b | 161 | --nodes; |
34c3d981 TG |
162 | } |
163 | ||
c192c97f | 164 | out: |
83a604e6 | 165 | return done; |
c192c97f ML |
166 | } |
167 | ||
168 | /** | |
169 | * irq_create_affinity_masks - Create affinity masks for multiqueue spreading | |
170 | * @nvecs: The total number of vectors | |
171 | * @affd: Description of the affinity requirements | |
172 | * | |
173 | * Returns the masks pointer or NULL if allocation failed. | |
174 | */ | |
175 | struct cpumask * | |
176 | irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd) | |
177 | { | |
2a4c050c ML |
178 | int affvecs = nvecs - affd->pre_vectors - affd->post_vectors; |
179 | int curvec, usedvecs; | |
180 | cpumask_var_t nmsk, npresmsk, *node_to_cpumask; | |
c192c97f | 181 | struct cpumask *masks = NULL; |
c192c97f ML |
182 | |
183 | /* | |
184 | * If there aren't any vectors left after applying the pre/post | |
185 | * vectors don't bother with assigning affinity. | |
186 | */ | |
187 | if (nvecs == affd->pre_vectors + affd->post_vectors) | |
188 | return NULL; | |
189 | ||
190 | if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL)) | |
191 | return NULL; | |
192 | ||
2a4c050c ML |
193 | if (!zalloc_cpumask_var(&npresmsk, GFP_KERNEL)) |
194 | goto outcpumsk; | |
195 | ||
c192c97f ML |
196 | node_to_cpumask = alloc_node_to_cpumask(); |
197 | if (!node_to_cpumask) | |
2a4c050c | 198 | goto outnpresmsk; |
c192c97f ML |
199 | |
200 | masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL); | |
201 | if (!masks) | |
202 | goto outnodemsk; | |
203 | ||
204 | /* Fill out vectors at the beginning that don't need affinity */ | |
205 | for (curvec = 0; curvec < affd->pre_vectors; curvec++) | |
206 | cpumask_copy(masks + curvec, irq_default_affinity); | |
207 | ||
208 | /* Stabilize the cpumasks */ | |
209 | get_online_cpus(); | |
210 | build_node_to_cpumask(node_to_cpumask); | |
2a4c050c ML |
211 | |
212 | /* Spread on present CPUs starting from affd->pre_vectors */ | |
213 | usedvecs = irq_build_affinity_masks(affd, curvec, affvecs, | |
214 | node_to_cpumask, cpu_present_mask, | |
215 | nmsk, masks); | |
216 | ||
217 | /* | |
218 | * Spread on non present CPUs starting from the next vector to be | |
219 | * handled. If the spreading of present CPUs already exhausted the | |
220 | * vector space, assign the non present CPUs to the already spread | |
221 | * out vectors. | |
222 | */ | |
223 | if (usedvecs >= affvecs) | |
224 | curvec = affd->pre_vectors; | |
225 | else | |
226 | curvec = affd->pre_vectors + usedvecs; | |
227 | cpumask_andnot(npresmsk, cpu_possible_mask, cpu_present_mask); | |
228 | usedvecs += irq_build_affinity_masks(affd, curvec, affvecs, | |
229 | node_to_cpumask, npresmsk, | |
230 | nmsk, masks); | |
34c3d981 | 231 | put_online_cpus(); |
67c93c21 CH |
232 | |
233 | /* Fill out vectors at the end that don't need affinity */ | |
2a4c050c ML |
234 | if (usedvecs >= affvecs) |
235 | curvec = affd->pre_vectors + affvecs; | |
236 | else | |
237 | curvec = affd->pre_vectors + usedvecs; | |
67c93c21 | 238 | for (; curvec < nvecs; curvec++) |
b6e5d5b9 | 239 | cpumask_copy(masks + curvec, irq_default_affinity); |
2a4c050c | 240 | |
1efd89a3 | 241 | outnodemsk: |
c0936d9d | 242 | free_node_to_cpumask(node_to_cpumask); |
2a4c050c ML |
243 | outnpresmsk: |
244 | free_cpumask_var(npresmsk); | |
1efd89a3 | 245 | outcpumsk: |
34c3d981 TG |
246 | free_cpumask_var(nmsk); |
247 | return masks; | |
248 | } | |
249 | ||
250 | /** | |
212bd846 | 251 | * irq_calc_affinity_vectors - Calculate the optimal number of vectors |
6f9a22bc | 252 | * @minvec: The minimum number of vectors available |
212bd846 CH |
253 | * @maxvec: The maximum number of vectors available |
254 | * @affd: Description of the affinity requirements | |
34c3d981 | 255 | */ |
6f9a22bc | 256 | int irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity *affd) |
34c3d981 | 257 | { |
212bd846 CH |
258 | int resv = affd->pre_vectors + affd->post_vectors; |
259 | int vecs = maxvec - resv; | |
9a0ef98e | 260 | int ret; |
34c3d981 | 261 | |
6f9a22bc MH |
262 | if (resv > minvec) |
263 | return 0; | |
264 | ||
34c3d981 | 265 | get_online_cpus(); |
c189f43b | 266 | ret = min_t(int, cpumask_weight(cpu_possible_mask), vecs) + resv; |
34c3d981 | 267 | put_online_cpus(); |
9a0ef98e | 268 | return ret; |
34c3d981 | 269 | } |