]>
Commit | Line | Data |
---|---|---|
5e385a6e CH |
1 | |
2 | #include <linux/interrupt.h> | |
3 | #include <linux/kernel.h> | |
4 | #include <linux/slab.h> | |
5 | #include <linux/cpu.h> | |
6 | ||
34c3d981 TG |
7 | static void irq_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk, |
8 | int cpus_per_vec) | |
9 | { | |
10 | const struct cpumask *siblmsk; | |
11 | int cpu, sibl; | |
12 | ||
13 | for ( ; cpus_per_vec > 0; ) { | |
14 | cpu = cpumask_first(nmsk); | |
15 | ||
16 | /* Should not happen, but I'm too lazy to think about it */ | |
17 | if (cpu >= nr_cpu_ids) | |
18 | return; | |
19 | ||
20 | cpumask_clear_cpu(cpu, nmsk); | |
21 | cpumask_set_cpu(cpu, irqmsk); | |
22 | cpus_per_vec--; | |
23 | ||
24 | /* If the cpu has siblings, use them first */ | |
25 | siblmsk = topology_sibling_cpumask(cpu); | |
26 | for (sibl = -1; cpus_per_vec > 0; ) { | |
27 | sibl = cpumask_next(sibl, siblmsk); | |
28 | if (sibl >= nr_cpu_ids) | |
29 | break; | |
30 | if (!cpumask_test_and_clear_cpu(sibl, nmsk)) | |
31 | continue; | |
32 | cpumask_set_cpu(sibl, irqmsk); | |
33 | cpus_per_vec--; | |
34 | } | |
35 | } | |
36 | } | |
37 | ||
38 | static int get_nodes_in_cpumask(const struct cpumask *mask, nodemask_t *nodemsk) | |
39 | { | |
c0af5243 | 40 | int n, nodes = 0; |
34c3d981 TG |
41 | |
42 | /* Calculate the number of nodes in the supplied affinity mask */ | |
c0af5243 | 43 | for_each_online_node(n) { |
34c3d981 TG |
44 | if (cpumask_intersects(mask, cpumask_of_node(n))) { |
45 | node_set(n, *nodemsk); | |
46 | nodes++; | |
47 | } | |
48 | } | |
49 | return nodes; | |
50 | } | |
51 | ||
52 | /** | |
53 | * irq_create_affinity_masks - Create affinity masks for multiqueue spreading | |
67c93c21 CH |
54 | * @nvecs: The total number of vectors |
55 | * @affd: Description of the affinity requirements | |
34c3d981 TG |
56 | * |
57 | * Returns the masks pointer or NULL if allocation failed. | |
58 | */ | |
67c93c21 CH |
59 | struct cpumask * |
60 | irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd) | |
34c3d981 | 61 | { |
67c93c21 CH |
62 | int n, nodes, vecs_per_node, cpus_per_vec, extra_vecs, curvec; |
63 | int affv = nvecs - affd->pre_vectors - affd->post_vectors; | |
bfe13077 | 64 | int last_affv = affv + affd->pre_vectors; |
34c3d981 TG |
65 | nodemask_t nodemsk = NODE_MASK_NONE; |
66 | struct cpumask *masks; | |
67 | cpumask_var_t nmsk; | |
68 | ||
69 | if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL)) | |
70 | return NULL; | |
71 | ||
67c93c21 | 72 | masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL); |
34c3d981 TG |
73 | if (!masks) |
74 | goto out; | |
75 | ||
67c93c21 CH |
76 | /* Fill out vectors at the beginning that don't need affinity */ |
77 | for (curvec = 0; curvec < affd->pre_vectors; curvec++) | |
b6e5d5b9 | 78 | cpumask_copy(masks + curvec, irq_default_affinity); |
67c93c21 | 79 | |
34c3d981 TG |
80 | /* Stabilize the cpumasks */ |
81 | get_online_cpus(); | |
67c93c21 | 82 | nodes = get_nodes_in_cpumask(cpu_online_mask, &nodemsk); |
34c3d981 TG |
83 | |
84 | /* | |
c0af5243 | 85 | * If the number of nodes in the mask is greater than or equal the |
34c3d981 TG |
86 | * number of vectors we just spread the vectors across the nodes. |
87 | */ | |
67c93c21 | 88 | if (affv <= nodes) { |
34c3d981 TG |
89 | for_each_node_mask(n, nodemsk) { |
90 | cpumask_copy(masks + curvec, cpumask_of_node(n)); | |
bfe13077 | 91 | if (++curvec == last_affv) |
34c3d981 TG |
92 | break; |
93 | } | |
67c93c21 | 94 | goto done; |
34c3d981 TG |
95 | } |
96 | ||
97 | /* Spread the vectors per node */ | |
67c93c21 | 98 | vecs_per_node = affv / nodes; |
34c3d981 | 99 | /* Account for rounding errors */ |
67c93c21 | 100 | extra_vecs = affv - (nodes * vecs_per_node); |
34c3d981 TG |
101 | |
102 | for_each_node_mask(n, nodemsk) { | |
103 | int ncpus, v, vecs_to_assign = vecs_per_node; | |
104 | ||
105 | /* Get the cpus on this node which are in the mask */ | |
67c93c21 | 106 | cpumask_and(nmsk, cpu_online_mask, cpumask_of_node(n)); |
34c3d981 TG |
107 | |
108 | /* Calculate the number of cpus per vector */ | |
109 | ncpus = cpumask_weight(nmsk); | |
110 | ||
bfe13077 CH |
111 | for (v = 0; curvec < last_affv && v < vecs_to_assign; |
112 | curvec++, v++) { | |
34c3d981 TG |
113 | cpus_per_vec = ncpus / vecs_to_assign; |
114 | ||
115 | /* Account for extra vectors to compensate rounding errors */ | |
116 | if (extra_vecs) { | |
117 | cpus_per_vec++; | |
118 | if (!--extra_vecs) | |
119 | vecs_per_node++; | |
120 | } | |
121 | irq_spread_init_one(masks + curvec, nmsk, cpus_per_vec); | |
122 | } | |
123 | ||
bfe13077 | 124 | if (curvec >= last_affv) |
34c3d981 TG |
125 | break; |
126 | } | |
127 | ||
67c93c21 | 128 | done: |
34c3d981 | 129 | put_online_cpus(); |
67c93c21 CH |
130 | |
131 | /* Fill out vectors at the end that don't need affinity */ | |
132 | for (; curvec < nvecs; curvec++) | |
b6e5d5b9 | 133 | cpumask_copy(masks + curvec, irq_default_affinity); |
34c3d981 TG |
134 | out: |
135 | free_cpumask_var(nmsk); | |
136 | return masks; | |
137 | } | |
138 | ||
139 | /** | |
212bd846 CH |
140 | * irq_calc_affinity_vectors - Calculate the optimal number of vectors |
141 | * @maxvec: The maximum number of vectors available | |
142 | * @affd: Description of the affinity requirements | |
34c3d981 | 143 | */ |
212bd846 | 144 | int irq_calc_affinity_vectors(int maxvec, const struct irq_affinity *affd) |
34c3d981 | 145 | { |
212bd846 CH |
146 | int resv = affd->pre_vectors + affd->post_vectors; |
147 | int vecs = maxvec - resv; | |
148 | int cpus; | |
34c3d981 TG |
149 | |
150 | /* Stabilize the cpumasks */ | |
151 | get_online_cpus(); | |
212bd846 | 152 | cpus = cpumask_weight(cpu_online_mask); |
34c3d981 | 153 | put_online_cpus(); |
212bd846 CH |
154 | |
155 | return min(cpus, vecs) + resv; | |
34c3d981 | 156 | } |