]>
Commit | Line | Data |
---|---|---|
1eee9950 DHB |
1 | /* |
2 | * QEMU PowerPC pSeries Logical Partition NUMA associativity handling | |
3 | * | |
4 | * Copyright IBM Corp. 2020 | |
5 | * | |
6 | * Authors: | |
7 | * Daniel Henrique Barboza <danielhb413@gmail.com> | |
8 | * | |
9 | * This work is licensed under the terms of the GNU GPL, version 2 or later. | |
10 | * See the COPYING file in the top-level directory. | |
11 | */ | |
12 | ||
13 | #include "qemu/osdep.h" | |
14 | #include "qemu-common.h" | |
15 | #include "hw/ppc/spapr_numa.h" | |
dd7e1d7a | 16 | #include "hw/pci-host/spapr.h" |
1eee9950 DHB |
17 | #include "hw/ppc/fdt.h" |
18 | ||
dd7e1d7a DHB |
19 | /* Moved from hw/ppc/spapr_pci_nvlink2.c */ |
20 | #define SPAPR_GPU_NUMA_ID (cpu_to_be32(1)) | |
f1aa45ff | 21 | |
3b880445 DHB |
22 | static bool spapr_machine_using_legacy_numa(SpaprMachineState *spapr) |
23 | { | |
24 | MachineState *machine = MACHINE(spapr); | |
25 | SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(machine); | |
26 | ||
27 | return smc->pre_5_2_numa_associativity || | |
28 | machine->numa_state->num_nodes <= 1; | |
29 | } | |
30 | ||
ee6635b2 DHB |
31 | static bool spapr_numa_is_symmetrical(MachineState *ms) |
32 | { | |
33 | int src, dst; | |
34 | int nb_numa_nodes = ms->numa_state->num_nodes; | |
35 | NodeInfo *numa_info = ms->numa_state->nodes; | |
36 | ||
37 | for (src = 0; src < nb_numa_nodes; src++) { | |
38 | for (dst = src; dst < nb_numa_nodes; dst++) { | |
39 | if (numa_info[src].distance[dst] != | |
40 | numa_info[dst].distance[src]) { | |
41 | return false; | |
42 | } | |
43 | } | |
44 | } | |
45 | ||
46 | return true; | |
47 | } | |
48 | ||
66407069 DHB |
49 | /* |
50 | * NVLink2-connected GPU RAM needs to be placed on a separate NUMA node. | |
51 | * We assign a new numa ID per GPU in spapr_pci_collect_nvgpu() which is | |
52 | * called from vPHB reset handler so we initialize the counter here. | |
53 | * If no NUMA is configured from the QEMU side, we start from 1 as GPU RAM | |
54 | * must be equally distant from any other node. | |
55 | * The final value of spapr->gpu_numa_id is going to be written to | |
56 | * max-associativity-domains in spapr_build_fdt(). | |
57 | */ | |
58 | unsigned int spapr_numa_initial_nvgpu_numa_id(MachineState *machine) | |
59 | { | |
60 | return MAX(1, machine->numa_state->num_nodes); | |
61 | } | |
62 | ||
690fbe42 DHB |
63 | /* |
64 | * This function will translate the user distances into | |
65 | * what the kernel understand as possible values: 10 | |
66 | * (local distance), 20, 40, 80 and 160, and return the equivalent | |
67 | * NUMA level for each. Current heuristic is: | |
68 | * - local distance (10) returns numa_level = 0x4, meaning there is | |
69 | * no rounding for local distance | |
70 | * - distances between 11 and 30 inclusive -> rounded to 20, | |
71 | * numa_level = 0x3 | |
72 | * - distances between 31 and 60 inclusive -> rounded to 40, | |
73 | * numa_level = 0x2 | |
74 | * - distances between 61 and 120 inclusive -> rounded to 80, | |
75 | * numa_level = 0x1 | |
76 | * - everything above 120 returns numa_level = 0 to indicate that | |
77 | * there is no match. This will be calculated as disntace = 160 | |
78 | * by the kernel (as of v5.9) | |
79 | */ | |
80 | static uint8_t spapr_numa_get_numa_level(uint8_t distance) | |
81 | { | |
82 | if (distance == 10) { | |
83 | return 0x4; | |
84 | } else if (distance > 11 && distance <= 30) { | |
85 | return 0x3; | |
86 | } else if (distance > 31 && distance <= 60) { | |
87 | return 0x2; | |
88 | } else if (distance > 61 && distance <= 120) { | |
89 | return 0x1; | |
90 | } | |
91 | ||
92 | return 0; | |
93 | } | |
94 | ||
95 | static void spapr_numa_define_associativity_domains(SpaprMachineState *spapr) | |
96 | { | |
97 | MachineState *ms = MACHINE(spapr); | |
98 | NodeInfo *numa_info = ms->numa_state->nodes; | |
99 | int nb_numa_nodes = ms->numa_state->num_nodes; | |
100 | int src, dst, i; | |
101 | ||
102 | for (src = 0; src < nb_numa_nodes; src++) { | |
103 | for (dst = src; dst < nb_numa_nodes; dst++) { | |
104 | /* | |
105 | * This is how the associativity domain between A and B | |
106 | * is calculated: | |
107 | * | |
108 | * - get the distance D between them | |
109 | * - get the correspondent NUMA level 'n_level' for D | |
110 | * - all associativity arrays were initialized with their own | |
111 | * numa_ids, and we're calculating the distance in node_id | |
112 | * ascending order, starting from node id 0 (the first node | |
113 | * retrieved by numa_state). This will have a cascade effect in | |
114 | * the algorithm because the associativity domains that node 0 | |
115 | * defines will be carried over to other nodes, and node 1 | |
116 | * associativities will be carried over after taking node 0 | |
117 | * associativities into account, and so on. This happens because | |
118 | * we'll assign assoc_src as the associativity domain of dst | |
119 | * as well, for all NUMA levels beyond and including n_level. | |
120 | * | |
121 | * The PPC kernel expects the associativity domains of node 0 to | |
122 | * be always 0, and this algorithm will grant that by default. | |
123 | */ | |
124 | uint8_t distance = numa_info[src].distance[dst]; | |
125 | uint8_t n_level = spapr_numa_get_numa_level(distance); | |
126 | uint32_t assoc_src; | |
127 | ||
128 | /* | |
129 | * n_level = 0 means that the distance is greater than our last | |
130 | * rounded value (120). In this case there is no NUMA level match | |
131 | * between src and dst and we can skip the remaining of the loop. | |
132 | * | |
133 | * The Linux kernel will assume that the distance between src and | |
134 | * dst, in this case of no match, is 10 (local distance) doubled | |
135 | * for each NUMA it didn't match. We have MAX_DISTANCE_REF_POINTS | |
136 | * levels (4), so this gives us 10*2*2*2*2 = 160. | |
137 | * | |
138 | * This logic can be seen in the Linux kernel source code, as of | |
139 | * v5.9, in arch/powerpc/mm/numa.c, function __node_distance(). | |
140 | */ | |
141 | if (n_level == 0) { | |
142 | continue; | |
143 | } | |
144 | ||
145 | /* | |
146 | * We must assign all assoc_src to dst, starting from n_level | |
147 | * and going up to 0x1. | |
148 | */ | |
149 | for (i = n_level; i > 0; i--) { | |
150 | assoc_src = spapr->numa_assoc_array[src][i]; | |
151 | spapr->numa_assoc_array[dst][i] = assoc_src; | |
152 | } | |
153 | } | |
154 | } | |
155 | ||
156 | } | |
157 | ||
f1aa45ff DHB |
158 | void spapr_numa_associativity_init(SpaprMachineState *spapr, |
159 | MachineState *machine) | |
160 | { | |
dd7e1d7a | 161 | SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr); |
f1aa45ff | 162 | int nb_numa_nodes = machine->numa_state->num_nodes; |
dd7e1d7a | 163 | int i, j, max_nodes_with_gpus; |
690fbe42 | 164 | bool using_legacy_numa = spapr_machine_using_legacy_numa(spapr); |
f1aa45ff DHB |
165 | |
166 | /* | |
167 | * For all associativity arrays: first position is the size, | |
168 | * position MAX_DISTANCE_REF_POINTS is always the numa_id, | |
169 | * represented by the index 'i'. | |
170 | * | |
171 | * This will break on sparse NUMA setups, when/if QEMU starts | |
172 | * to support it, because there will be no more guarantee that | |
173 | * 'i' will be a valid node_id set by the user. | |
174 | */ | |
175 | for (i = 0; i < nb_numa_nodes; i++) { | |
176 | spapr->numa_assoc_array[i][0] = cpu_to_be32(MAX_DISTANCE_REF_POINTS); | |
177 | spapr->numa_assoc_array[i][MAX_DISTANCE_REF_POINTS] = cpu_to_be32(i); | |
690fbe42 DHB |
178 | |
179 | /* | |
180 | * Fill all associativity domains of non-zero NUMA nodes with | |
181 | * node_id. This is required because the default value (0) is | |
182 | * considered a match with associativity domains of node 0. | |
183 | */ | |
184 | if (!using_legacy_numa && i != 0) { | |
185 | for (j = 1; j < MAX_DISTANCE_REF_POINTS; j++) { | |
186 | spapr->numa_assoc_array[i][j] = cpu_to_be32(i); | |
187 | } | |
188 | } | |
f1aa45ff | 189 | } |
dd7e1d7a DHB |
190 | |
191 | /* | |
192 | * Initialize NVLink GPU associativity arrays. We know that | |
193 | * the first GPU will take the first available NUMA id, and | |
194 | * we'll have a maximum of NVGPU_MAX_NUM GPUs in the machine. | |
195 | * At this point we're not sure if there are GPUs or not, but | |
196 | * let's initialize the associativity arrays and allow NVLink | |
197 | * GPUs to be handled like regular NUMA nodes later on. | |
198 | */ | |
199 | max_nodes_with_gpus = nb_numa_nodes + NVGPU_MAX_NUM; | |
200 | ||
201 | for (i = nb_numa_nodes; i < max_nodes_with_gpus; i++) { | |
202 | spapr->numa_assoc_array[i][0] = cpu_to_be32(MAX_DISTANCE_REF_POINTS); | |
203 | ||
204 | for (j = 1; j < MAX_DISTANCE_REF_POINTS; j++) { | |
205 | uint32_t gpu_assoc = smc->pre_5_1_assoc_refpoints ? | |
206 | SPAPR_GPU_NUMA_ID : cpu_to_be32(i); | |
207 | spapr->numa_assoc_array[i][j] = gpu_assoc; | |
208 | } | |
209 | ||
210 | spapr->numa_assoc_array[i][MAX_DISTANCE_REF_POINTS] = cpu_to_be32(i); | |
211 | } | |
ee6635b2 DHB |
212 | |
213 | /* | |
214 | * Legacy NUMA guests (pseries-5.1 and older, or guests with only | |
215 | * 1 NUMA node) will not benefit from anything we're going to do | |
216 | * after this point. | |
217 | */ | |
690fbe42 | 218 | if (using_legacy_numa) { |
ee6635b2 DHB |
219 | return; |
220 | } | |
221 | ||
222 | if (!spapr_numa_is_symmetrical(machine)) { | |
223 | error_report("Asymmetrical NUMA topologies aren't supported " | |
224 | "in the pSeries machine"); | |
225 | exit(EXIT_FAILURE); | |
226 | } | |
227 | ||
690fbe42 | 228 | spapr_numa_define_associativity_domains(spapr); |
f1aa45ff DHB |
229 | } |
230 | ||
231 | void spapr_numa_write_associativity_dt(SpaprMachineState *spapr, void *fdt, | |
232 | int offset, int nodeid) | |
233 | { | |
234 | _FDT((fdt_setprop(fdt, offset, "ibm,associativity", | |
235 | spapr->numa_assoc_array[nodeid], | |
236 | sizeof(spapr->numa_assoc_array[nodeid])))); | |
8f86a408 DHB |
237 | } |
238 | ||
d370f9cf DHB |
239 | static uint32_t *spapr_numa_get_vcpu_assoc(SpaprMachineState *spapr, |
240 | PowerPCCPU *cpu) | |
8f86a408 | 241 | { |
d370f9cf | 242 | uint32_t *vcpu_assoc = g_new(uint32_t, VCPU_ASSOC_SIZE); |
8f86a408 | 243 | int index = spapr_get_vcpu_id(cpu); |
8f86a408 DHB |
244 | |
245 | /* | |
246 | * VCPUs have an extra 'cpu_id' value in ibm,associativity | |
247 | * compared to other resources. Increment the size at index | |
d370f9cf DHB |
248 | * 0, put cpu_id last, then copy the remaining associativity |
249 | * domains. | |
8f86a408 DHB |
250 | */ |
251 | vcpu_assoc[0] = cpu_to_be32(MAX_DISTANCE_REF_POINTS + 1); | |
d370f9cf DHB |
252 | vcpu_assoc[VCPU_ASSOC_SIZE - 1] = cpu_to_be32(index); |
253 | memcpy(vcpu_assoc + 1, spapr->numa_assoc_array[cpu->node_id] + 1, | |
254 | (VCPU_ASSOC_SIZE - 2) * sizeof(uint32_t)); | |
8f86a408 | 255 | |
d370f9cf DHB |
256 | return vcpu_assoc; |
257 | } | |
258 | ||
259 | int spapr_numa_fixup_cpu_dt(SpaprMachineState *spapr, void *fdt, | |
260 | int offset, PowerPCCPU *cpu) | |
261 | { | |
262 | g_autofree uint32_t *vcpu_assoc = NULL; | |
8f86a408 | 263 | |
d370f9cf | 264 | vcpu_assoc = spapr_numa_get_vcpu_assoc(spapr, cpu); |
8f86a408 DHB |
265 | |
266 | /* Advertise NUMA via ibm,associativity */ | |
d370f9cf DHB |
267 | return fdt_setprop(fdt, offset, "ibm,associativity", vcpu_assoc, |
268 | VCPU_ASSOC_SIZE * sizeof(uint32_t)); | |
f1aa45ff DHB |
269 | } |
270 | ||
0ee52012 DHB |
271 | |
272 | int spapr_numa_write_assoc_lookup_arrays(SpaprMachineState *spapr, void *fdt, | |
273 | int offset) | |
274 | { | |
275 | MachineState *machine = MACHINE(spapr); | |
276 | int nb_numa_nodes = machine->numa_state->num_nodes; | |
277 | int nr_nodes = nb_numa_nodes ? nb_numa_nodes : 1; | |
278 | uint32_t *int_buf, *cur_index, buf_len; | |
279 | int ret, i; | |
280 | ||
281 | /* ibm,associativity-lookup-arrays */ | |
282 | buf_len = (nr_nodes * MAX_DISTANCE_REF_POINTS + 2) * sizeof(uint32_t); | |
283 | cur_index = int_buf = g_malloc0(buf_len); | |
284 | int_buf[0] = cpu_to_be32(nr_nodes); | |
285 | /* Number of entries per associativity list */ | |
286 | int_buf[1] = cpu_to_be32(MAX_DISTANCE_REF_POINTS); | |
287 | cur_index += 2; | |
288 | for (i = 0; i < nr_nodes; i++) { | |
289 | /* | |
290 | * For the lookup-array we use the ibm,associativity array, | |
291 | * from numa_assoc_array. without the first element (size). | |
292 | */ | |
293 | uint32_t *associativity = spapr->numa_assoc_array[i]; | |
294 | memcpy(cur_index, ++associativity, | |
295 | sizeof(uint32_t) * MAX_DISTANCE_REF_POINTS); | |
296 | cur_index += MAX_DISTANCE_REF_POINTS; | |
297 | } | |
298 | ret = fdt_setprop(fdt, offset, "ibm,associativity-lookup-arrays", int_buf, | |
299 | (cur_index - int_buf) * sizeof(uint32_t)); | |
300 | g_free(int_buf); | |
301 | ||
302 | return ret; | |
303 | } | |
304 | ||
1eee9950 DHB |
305 | /* |
306 | * Helper that writes ibm,associativity-reference-points and | |
307 | * max-associativity-domains in the RTAS pointed by @rtas | |
308 | * in the DT @fdt. | |
309 | */ | |
310 | void spapr_numa_write_rtas_dt(SpaprMachineState *spapr, void *fdt, int rtas) | |
311 | { | |
491e884e | 312 | MachineState *ms = MACHINE(spapr); |
1eee9950 | 313 | SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr); |
b01fec36 DHB |
314 | uint32_t number_nvgpus_nodes = spapr->gpu_numa_id - |
315 | spapr_numa_initial_nvgpu_numa_id(ms); | |
1eee9950 DHB |
316 | uint32_t refpoints[] = { |
317 | cpu_to_be32(0x4), | |
491e884e | 318 | cpu_to_be32(0x3), |
1eee9950 | 319 | cpu_to_be32(0x2), |
491e884e | 320 | cpu_to_be32(0x1), |
1eee9950 DHB |
321 | }; |
322 | uint32_t nr_refpoints = ARRAY_SIZE(refpoints); | |
b01fec36 | 323 | uint32_t maxdomain = ms->numa_state->num_nodes + number_nvgpus_nodes; |
1eee9950 DHB |
324 | uint32_t maxdomains[] = { |
325 | cpu_to_be32(4), | |
491e884e DHB |
326 | cpu_to_be32(maxdomain), |
327 | cpu_to_be32(maxdomain), | |
328 | cpu_to_be32(maxdomain), | |
329 | cpu_to_be32(maxdomain) | |
1eee9950 DHB |
330 | }; |
331 | ||
491e884e DHB |
332 | if (spapr_machine_using_legacy_numa(spapr)) { |
333 | uint32_t legacy_refpoints[] = { | |
334 | cpu_to_be32(0x4), | |
335 | cpu_to_be32(0x4), | |
336 | cpu_to_be32(0x2), | |
337 | }; | |
338 | uint32_t legacy_maxdomain = spapr->gpu_numa_id > 1 ? 1 : 0; | |
339 | uint32_t legacy_maxdomains[] = { | |
340 | cpu_to_be32(4), | |
341 | cpu_to_be32(legacy_maxdomain), | |
342 | cpu_to_be32(legacy_maxdomain), | |
343 | cpu_to_be32(legacy_maxdomain), | |
344 | cpu_to_be32(spapr->gpu_numa_id), | |
345 | }; | |
346 | ||
347 | G_STATIC_ASSERT(sizeof(legacy_refpoints) <= sizeof(refpoints)); | |
348 | G_STATIC_ASSERT(sizeof(legacy_maxdomains) <= sizeof(maxdomains)); | |
349 | ||
350 | nr_refpoints = 3; | |
351 | ||
352 | memcpy(refpoints, legacy_refpoints, sizeof(legacy_refpoints)); | |
353 | memcpy(maxdomains, legacy_maxdomains, sizeof(legacy_maxdomains)); | |
354 | ||
355 | /* pseries-5.0 and older reference-points array is {0x4, 0x4} */ | |
356 | if (smc->pre_5_1_assoc_refpoints) { | |
357 | nr_refpoints = 2; | |
358 | } | |
1eee9950 DHB |
359 | } |
360 | ||
361 | _FDT(fdt_setprop(fdt, rtas, "ibm,associativity-reference-points", | |
362 | refpoints, nr_refpoints * sizeof(refpoints[0]))); | |
363 | ||
364 | _FDT(fdt_setprop(fdt, rtas, "ibm,max-associativity-domains", | |
365 | maxdomains, sizeof(maxdomains))); | |
366 | } | |
f8a13fc3 DHB |
367 | |
368 | static target_ulong h_home_node_associativity(PowerPCCPU *cpu, | |
369 | SpaprMachineState *spapr, | |
370 | target_ulong opcode, | |
371 | target_ulong *args) | |
372 | { | |
876ab8d8 | 373 | g_autofree uint32_t *vcpu_assoc = NULL; |
f8a13fc3 DHB |
374 | target_ulong flags = args[0]; |
375 | target_ulong procno = args[1]; | |
376 | PowerPCCPU *tcpu; | |
876ab8d8 | 377 | int idx, assoc_idx; |
f8a13fc3 DHB |
378 | |
379 | /* only support procno from H_REGISTER_VPA */ | |
380 | if (flags != 0x1) { | |
381 | return H_FUNCTION; | |
382 | } | |
383 | ||
384 | tcpu = spapr_find_cpu(procno); | |
385 | if (tcpu == NULL) { | |
386 | return H_P2; | |
387 | } | |
388 | ||
876ab8d8 DHB |
389 | /* |
390 | * Given that we want to be flexible with the sizes and indexes, | |
391 | * we must consider that there is a hard limit of how many | |
392 | * associativities domain we can fit in R4 up to R9, which would be | |
393 | * 12 associativity domains for vcpus. Assert and bail if that's | |
394 | * not the case. | |
395 | */ | |
396 | G_STATIC_ASSERT((VCPU_ASSOC_SIZE - 1) <= 12); | |
397 | ||
398 | vcpu_assoc = spapr_numa_get_vcpu_assoc(spapr, tcpu); | |
399 | /* assoc_idx starts at 1 to skip associativity size */ | |
400 | assoc_idx = 1; | |
f8a13fc3 | 401 | |
f8a13fc3 DHB |
402 | #define ASSOCIATIVITY(a, b) (((uint64_t)(a) << 32) | \ |
403 | ((uint64_t)(b) & 0xffffffff)) | |
876ab8d8 DHB |
404 | |
405 | for (idx = 0; idx < 6; idx++) { | |
406 | int32_t a, b; | |
407 | ||
408 | /* | |
409 | * vcpu_assoc[] will contain the associativity domains for tcpu, | |
410 | * including tcpu->node_id and procno, meaning that we don't | |
411 | * need to use these variables here. | |
412 | * | |
413 | * We'll read 2 values at a time to fill up the ASSOCIATIVITY() | |
414 | * macro. The ternary will fill the remaining registers with -1 | |
415 | * after we went through vcpu_assoc[]. | |
416 | */ | |
417 | a = assoc_idx < VCPU_ASSOC_SIZE ? | |
418 | be32_to_cpu(vcpu_assoc[assoc_idx++]) : -1; | |
419 | b = assoc_idx < VCPU_ASSOC_SIZE ? | |
420 | be32_to_cpu(vcpu_assoc[assoc_idx++]) : -1; | |
421 | ||
422 | args[idx] = ASSOCIATIVITY(a, b); | |
f8a13fc3 DHB |
423 | } |
424 | #undef ASSOCIATIVITY | |
425 | ||
426 | return H_SUCCESS; | |
427 | } | |
428 | ||
429 | static void spapr_numa_register_types(void) | |
430 | { | |
431 | /* Virtual Processor Home Node */ | |
432 | spapr_register_hypercall(H_HOME_NODE_ASSOCIATIVITY, | |
433 | h_home_node_associativity); | |
434 | } | |
435 | ||
436 | type_init(spapr_numa_register_types) |