]>
Commit | Line | Data |
---|---|---|
9a0ef98e CH |
1 | /* |
2 | * Copyright (C) 2016 Thomas Gleixner. | |
3 | * Copyright (C) 2016-2017 Christoph Hellwig. | |
4 | */ | |
5e385a6e CH |
5 | #include <linux/interrupt.h> |
6 | #include <linux/kernel.h> | |
7 | #include <linux/slab.h> | |
8 | #include <linux/cpu.h> | |
9 | ||
34c3d981 TG |
10 | static void irq_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk, |
11 | int cpus_per_vec) | |
12 | { | |
13 | const struct cpumask *siblmsk; | |
14 | int cpu, sibl; | |
15 | ||
16 | for ( ; cpus_per_vec > 0; ) { | |
17 | cpu = cpumask_first(nmsk); | |
18 | ||
19 | /* Should not happen, but I'm too lazy to think about it */ | |
20 | if (cpu >= nr_cpu_ids) | |
21 | return; | |
22 | ||
23 | cpumask_clear_cpu(cpu, nmsk); | |
24 | cpumask_set_cpu(cpu, irqmsk); | |
25 | cpus_per_vec--; | |
26 | ||
27 | /* If the cpu has siblings, use them first */ | |
28 | siblmsk = topology_sibling_cpumask(cpu); | |
29 | for (sibl = -1; cpus_per_vec > 0; ) { | |
30 | sibl = cpumask_next(sibl, siblmsk); | |
31 | if (sibl >= nr_cpu_ids) | |
32 | break; | |
33 | if (!cpumask_test_and_clear_cpu(sibl, nmsk)) | |
34 | continue; | |
35 | cpumask_set_cpu(sibl, irqmsk); | |
36 | cpus_per_vec--; | |
37 | } | |
38 | } | |
39 | } | |
40 | ||
9a0ef98e CH |
41 | static cpumask_var_t *alloc_node_to_present_cpumask(void) |
42 | { | |
43 | cpumask_var_t *masks; | |
44 | int node; | |
45 | ||
46 | masks = kcalloc(nr_node_ids, sizeof(cpumask_var_t), GFP_KERNEL); | |
47 | if (!masks) | |
48 | return NULL; | |
49 | ||
50 | for (node = 0; node < nr_node_ids; node++) { | |
51 | if (!zalloc_cpumask_var(&masks[node], GFP_KERNEL)) | |
52 | goto out_unwind; | |
53 | } | |
54 | ||
55 | return masks; | |
56 | ||
57 | out_unwind: | |
58 | while (--node >= 0) | |
59 | free_cpumask_var(masks[node]); | |
60 | kfree(masks); | |
61 | return NULL; | |
62 | } | |
63 | ||
64 | static void free_node_to_present_cpumask(cpumask_var_t *masks) | |
65 | { | |
66 | int node; | |
67 | ||
68 | for (node = 0; node < nr_node_ids; node++) | |
69 | free_cpumask_var(masks[node]); | |
70 | kfree(masks); | |
71 | } | |
72 | ||
73 | static void build_node_to_present_cpumask(cpumask_var_t *masks) | |
74 | { | |
75 | int cpu; | |
76 | ||
77 | for_each_present_cpu(cpu) | |
78 | cpumask_set_cpu(cpu, masks[cpu_to_node(cpu)]); | |
79 | } | |
80 | ||
81 | static int get_nodes_in_cpumask(cpumask_var_t *node_to_present_cpumask, | |
82 | const struct cpumask *mask, nodemask_t *nodemsk) | |
34c3d981 | 83 | { |
c0af5243 | 84 | int n, nodes = 0; |
34c3d981 TG |
85 | |
86 | /* Calculate the number of nodes in the supplied affinity mask */ | |
9a0ef98e CH |
87 | for_each_node(n) { |
88 | if (cpumask_intersects(mask, node_to_present_cpumask[n])) { | |
34c3d981 TG |
89 | node_set(n, *nodemsk); |
90 | nodes++; | |
91 | } | |
92 | } | |
93 | return nodes; | |
94 | } | |
95 | ||
96 | /** | |
97 | * irq_create_affinity_masks - Create affinity masks for multiqueue spreading | |
67c93c21 CH |
98 | * @nvecs: The total number of vectors |
99 | * @affd: Description of the affinity requirements | |
34c3d981 TG |
100 | * |
101 | * Returns the masks pointer or NULL if allocation failed. | |
102 | */ | |
67c93c21 CH |
103 | struct cpumask * |
104 | irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd) | |
34c3d981 | 105 | { |
7bf8222b | 106 | int n, nodes, cpus_per_vec, extra_vecs, curvec; |
67c93c21 | 107 | int affv = nvecs - affd->pre_vectors - affd->post_vectors; |
bfe13077 | 108 | int last_affv = affv + affd->pre_vectors; |
34c3d981 TG |
109 | nodemask_t nodemsk = NODE_MASK_NONE; |
110 | struct cpumask *masks; | |
9a0ef98e | 111 | cpumask_var_t nmsk, *node_to_present_cpumask; |
34c3d981 | 112 | |
6f9a22bc MH |
113 | /* |
114 | * If there aren't any vectors left after applying the pre/post | |
115 | * vectors don't bother with assigning affinity. | |
116 | */ | |
117 | if (!affv) | |
118 | return NULL; | |
119 | ||
34c3d981 TG |
120 | if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL)) |
121 | return NULL; | |
122 | ||
67c93c21 | 123 | masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL); |
34c3d981 TG |
124 | if (!masks) |
125 | goto out; | |
126 | ||
9a0ef98e CH |
127 | node_to_present_cpumask = alloc_node_to_present_cpumask(); |
128 | if (!node_to_present_cpumask) | |
129 | goto out; | |
130 | ||
67c93c21 CH |
131 | /* Fill out vectors at the beginning that don't need affinity */ |
132 | for (curvec = 0; curvec < affd->pre_vectors; curvec++) | |
b6e5d5b9 | 133 | cpumask_copy(masks + curvec, irq_default_affinity); |
67c93c21 | 134 | |
34c3d981 TG |
135 | /* Stabilize the cpumasks */ |
136 | get_online_cpus(); | |
9a0ef98e CH |
137 | build_node_to_present_cpumask(node_to_present_cpumask); |
138 | nodes = get_nodes_in_cpumask(node_to_present_cpumask, cpu_present_mask, | |
139 | &nodemsk); | |
34c3d981 TG |
140 | |
141 | /* | |
c0af5243 | 142 | * If the number of nodes in the mask is greater than or equal the |
34c3d981 TG |
143 | * number of vectors we just spread the vectors across the nodes. |
144 | */ | |
67c93c21 | 145 | if (affv <= nodes) { |
34c3d981 | 146 | for_each_node_mask(n, nodemsk) { |
9a0ef98e CH |
147 | cpumask_copy(masks + curvec, |
148 | node_to_present_cpumask[n]); | |
bfe13077 | 149 | if (++curvec == last_affv) |
34c3d981 TG |
150 | break; |
151 | } | |
67c93c21 | 152 | goto done; |
34c3d981 TG |
153 | } |
154 | ||
34c3d981 | 155 | for_each_node_mask(n, nodemsk) { |
7bf8222b KB |
156 | int ncpus, v, vecs_to_assign, vecs_per_node; |
157 | ||
158 | /* Spread the vectors per node */ | |
b72f8051 | 159 | vecs_per_node = (affv - (curvec - affd->pre_vectors)) / nodes; |
34c3d981 TG |
160 | |
161 | /* Get the cpus on this node which are in the mask */ | |
9a0ef98e | 162 | cpumask_and(nmsk, cpu_present_mask, node_to_present_cpumask[n]); |
34c3d981 TG |
163 | |
164 | /* Calculate the number of cpus per vector */ | |
165 | ncpus = cpumask_weight(nmsk); | |
7bf8222b KB |
166 | vecs_to_assign = min(vecs_per_node, ncpus); |
167 | ||
168 | /* Account for rounding errors */ | |
3412386b | 169 | extra_vecs = ncpus - vecs_to_assign * (ncpus / vecs_to_assign); |
34c3d981 | 170 | |
bfe13077 CH |
171 | for (v = 0; curvec < last_affv && v < vecs_to_assign; |
172 | curvec++, v++) { | |
34c3d981 TG |
173 | cpus_per_vec = ncpus / vecs_to_assign; |
174 | ||
175 | /* Account for extra vectors to compensate rounding errors */ | |
176 | if (extra_vecs) { | |
177 | cpus_per_vec++; | |
7bf8222b | 178 | --extra_vecs; |
34c3d981 TG |
179 | } |
180 | irq_spread_init_one(masks + curvec, nmsk, cpus_per_vec); | |
181 | } | |
182 | ||
bfe13077 | 183 | if (curvec >= last_affv) |
34c3d981 | 184 | break; |
7bf8222b | 185 | --nodes; |
34c3d981 TG |
186 | } |
187 | ||
67c93c21 | 188 | done: |
34c3d981 | 189 | put_online_cpus(); |
67c93c21 CH |
190 | |
191 | /* Fill out vectors at the end that don't need affinity */ | |
192 | for (; curvec < nvecs; curvec++) | |
b6e5d5b9 | 193 | cpumask_copy(masks + curvec, irq_default_affinity); |
9a0ef98e | 194 | free_node_to_present_cpumask(node_to_present_cpumask); |
34c3d981 TG |
195 | out: |
196 | free_cpumask_var(nmsk); | |
197 | return masks; | |
198 | } | |
199 | ||
200 | /** | |
212bd846 | 201 | * irq_calc_affinity_vectors - Calculate the optimal number of vectors |
6f9a22bc | 202 | * @minvec: The minimum number of vectors available |
212bd846 CH |
203 | * @maxvec: The maximum number of vectors available |
204 | * @affd: Description of the affinity requirements | |
34c3d981 | 205 | */ |
6f9a22bc | 206 | int irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity *affd) |
34c3d981 | 207 | { |
212bd846 CH |
208 | int resv = affd->pre_vectors + affd->post_vectors; |
209 | int vecs = maxvec - resv; | |
9a0ef98e | 210 | int ret; |
34c3d981 | 211 | |
6f9a22bc MH |
212 | if (resv > minvec) |
213 | return 0; | |
214 | ||
34c3d981 | 215 | get_online_cpus(); |
9a0ef98e | 216 | ret = min_t(int, cpumask_weight(cpu_present_mask), vecs) + resv; |
34c3d981 | 217 | put_online_cpus(); |
9a0ef98e | 218 | return ret; |
34c3d981 | 219 | } |