]>
git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - kernel/irq/affinity.c
2 * Copyright (C) 2016 Thomas Gleixner.
3 * Copyright (C) 2016-2017 Christoph Hellwig.
5 #include <linux/interrupt.h>
6 #include <linux/kernel.h>
7 #include <linux/slab.h>
10 static void irq_spread_init_one(struct cpumask
*irqmsk
, struct cpumask
*nmsk
,
13 const struct cpumask
*siblmsk
;
16 for ( ; cpus_per_vec
> 0; ) {
17 cpu
= cpumask_first(nmsk
);
19 /* Should not happen, but I'm too lazy to think about it */
20 if (cpu
>= nr_cpu_ids
)
23 cpumask_clear_cpu(cpu
, nmsk
);
24 cpumask_set_cpu(cpu
, irqmsk
);
27 /* If the cpu has siblings, use them first */
28 siblmsk
= topology_sibling_cpumask(cpu
);
29 for (sibl
= -1; cpus_per_vec
> 0; ) {
30 sibl
= cpumask_next(sibl
, siblmsk
);
31 if (sibl
>= nr_cpu_ids
)
33 if (!cpumask_test_and_clear_cpu(sibl
, nmsk
))
35 cpumask_set_cpu(sibl
, irqmsk
);
41 static cpumask_var_t
*alloc_node_to_present_cpumask(void)
46 masks
= kcalloc(nr_node_ids
, sizeof(cpumask_var_t
), GFP_KERNEL
);
50 for (node
= 0; node
< nr_node_ids
; node
++) {
51 if (!zalloc_cpumask_var(&masks
[node
], GFP_KERNEL
))
59 free_cpumask_var(masks
[node
]);
64 static void free_node_to_present_cpumask(cpumask_var_t
*masks
)
68 for (node
= 0; node
< nr_node_ids
; node
++)
69 free_cpumask_var(masks
[node
]);
73 static void build_node_to_present_cpumask(cpumask_var_t
*masks
)
77 for_each_present_cpu(cpu
)
78 cpumask_set_cpu(cpu
, masks
[cpu_to_node(cpu
)]);
81 static int get_nodes_in_cpumask(cpumask_var_t
*node_to_present_cpumask
,
82 const struct cpumask
*mask
, nodemask_t
*nodemsk
)
86 /* Calculate the number of nodes in the supplied affinity mask */
88 if (cpumask_intersects(mask
, node_to_present_cpumask
[n
])) {
89 node_set(n
, *nodemsk
);
97 * irq_create_affinity_masks - Create affinity masks for multiqueue spreading
98 * @nvecs: The total number of vectors
99 * @affd: Description of the affinity requirements
101 * Returns the masks pointer or NULL if allocation failed.
104 irq_create_affinity_masks(int nvecs
, const struct irq_affinity
*affd
)
106 int n
, nodes
, cpus_per_vec
, extra_vecs
, curvec
;
107 int affv
= nvecs
- affd
->pre_vectors
- affd
->post_vectors
;
108 int last_affv
= affv
+ affd
->pre_vectors
;
109 nodemask_t nodemsk
= NODE_MASK_NONE
;
110 struct cpumask
*masks
;
111 cpumask_var_t nmsk
, *node_to_present_cpumask
;
114 * If there aren't any vectors left after applying the pre/post
115 * vectors don't bother with assigning affinity.
120 if (!zalloc_cpumask_var(&nmsk
, GFP_KERNEL
))
123 masks
= kcalloc(nvecs
, sizeof(*masks
), GFP_KERNEL
);
127 node_to_present_cpumask
= alloc_node_to_present_cpumask();
128 if (!node_to_present_cpumask
)
131 /* Fill out vectors at the beginning that don't need affinity */
132 for (curvec
= 0; curvec
< affd
->pre_vectors
; curvec
++)
133 cpumask_copy(masks
+ curvec
, irq_default_affinity
);
135 /* Stabilize the cpumasks */
137 build_node_to_present_cpumask(node_to_present_cpumask
);
138 nodes
= get_nodes_in_cpumask(node_to_present_cpumask
, cpu_present_mask
,
142 * If the number of nodes in the mask is greater than or equal the
143 * number of vectors we just spread the vectors across the nodes.
146 for_each_node_mask(n
, nodemsk
) {
147 cpumask_copy(masks
+ curvec
,
148 node_to_present_cpumask
[n
]);
149 if (++curvec
== last_affv
)
155 for_each_node_mask(n
, nodemsk
) {
156 int ncpus
, v
, vecs_to_assign
, vecs_per_node
;
158 /* Spread the vectors per node */
159 vecs_per_node
= (affv
- (curvec
- affd
->pre_vectors
)) / nodes
;
161 /* Get the cpus on this node which are in the mask */
162 cpumask_and(nmsk
, cpu_present_mask
, node_to_present_cpumask
[n
]);
164 /* Calculate the number of cpus per vector */
165 ncpus
= cpumask_weight(nmsk
);
166 vecs_to_assign
= min(vecs_per_node
, ncpus
);
168 /* Account for rounding errors */
169 extra_vecs
= ncpus
- vecs_to_assign
* (ncpus
/ vecs_to_assign
);
171 for (v
= 0; curvec
< last_affv
&& v
< vecs_to_assign
;
173 cpus_per_vec
= ncpus
/ vecs_to_assign
;
175 /* Account for extra vectors to compensate rounding errors */
180 irq_spread_init_one(masks
+ curvec
, nmsk
, cpus_per_vec
);
183 if (curvec
>= last_affv
)
191 /* Fill out vectors at the end that don't need affinity */
192 for (; curvec
< nvecs
; curvec
++)
193 cpumask_copy(masks
+ curvec
, irq_default_affinity
);
194 free_node_to_present_cpumask(node_to_present_cpumask
);
196 free_cpumask_var(nmsk
);
201 * irq_calc_affinity_vectors - Calculate the optimal number of vectors
202 * @minvec: The minimum number of vectors available
203 * @maxvec: The maximum number of vectors available
204 * @affd: Description of the affinity requirements
206 int irq_calc_affinity_vectors(int minvec
, int maxvec
, const struct irq_affinity
*affd
)
208 int resv
= affd
->pre_vectors
+ affd
->post_vectors
;
209 int vecs
= maxvec
- resv
;
216 ret
= min_t(int, cpumask_weight(cpu_present_mask
), vecs
) + resv
;