2 * QEMU PowerPC pSeries Logical Partition NUMA associativity handling
4 * Copyright IBM Corp. 2020
7 * Daniel Henrique Barboza <danielhb413@gmail.com>
9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
10 * See the COPYING file in the top-level directory.
13 #include "qemu/osdep.h"
14 #include "qemu-common.h"
15 #include "hw/ppc/spapr_numa.h"
16 #include "hw/pci-host/spapr.h"
17 #include "hw/ppc/fdt.h"
19 /* Moved from hw/ppc/spapr_pci_nvlink2.c */
20 #define SPAPR_GPU_NUMA_ID (cpu_to_be32(1))
23 * Retrieves max_dist_ref_points of the current NUMA affinity.
25 static int get_max_dist_ref_points(SpaprMachineState
*spapr
)
27 return FORM1_DIST_REF_POINTS
;
31 * Retrieves numa_assoc_size of the current NUMA affinity.
33 static int get_numa_assoc_size(SpaprMachineState
*spapr
)
35 return FORM1_NUMA_ASSOC_SIZE
;
39 * Retrieves vcpu_assoc_size of the current NUMA affinity.
41 * vcpu_assoc_size is the size of ibm,associativity array
42 * for CPUs, which has an extra element (vcpu_id) in the end.
44 static int get_vcpu_assoc_size(SpaprMachineState
*spapr
)
46 return get_numa_assoc_size(spapr
) + 1;
49 static bool spapr_numa_is_symmetrical(MachineState
*ms
)
52 int nb_numa_nodes
= ms
->numa_state
->num_nodes
;
53 NodeInfo
*numa_info
= ms
->numa_state
->nodes
;
55 for (src
= 0; src
< nb_numa_nodes
; src
++) {
56 for (dst
= src
; dst
< nb_numa_nodes
; dst
++) {
57 if (numa_info
[src
].distance
[dst
] !=
58 numa_info
[dst
].distance
[src
]) {
68 * NVLink2-connected GPU RAM needs to be placed on a separate NUMA node.
69 * We assign a new numa ID per GPU in spapr_pci_collect_nvgpu() which is
70 * called from vPHB reset handler so we initialize the counter here.
71 * If no NUMA is configured from the QEMU side, we start from 1 as GPU RAM
72 * must be equally distant from any other node.
73 * The final value of spapr->gpu_numa_id is going to be written to
74 * max-associativity-domains in spapr_build_fdt().
76 unsigned int spapr_numa_initial_nvgpu_numa_id(MachineState
*machine
)
78 return MAX(1, machine
->numa_state
->num_nodes
);
82 * This function will translate the user distances into
83 * what the kernel understand as possible values: 10
84 * (local distance), 20, 40, 80 and 160, and return the equivalent
85 * NUMA level for each. Current heuristic is:
86 * - local distance (10) returns numa_level = 0x4, meaning there is
87 * no rounding for local distance
88 * - distances between 11 and 30 inclusive -> rounded to 20,
90 * - distances between 31 and 60 inclusive -> rounded to 40,
92 * - distances between 61 and 120 inclusive -> rounded to 80,
94 * - everything above 120 returns numa_level = 0 to indicate that
95 * there is no match. This will be calculated as disntace = 160
96 * by the kernel (as of v5.9)
98 static uint8_t spapr_numa_get_numa_level(uint8_t distance
)
100 if (distance
== 10) {
102 } else if (distance
> 11 && distance
<= 30) {
104 } else if (distance
> 31 && distance
<= 60) {
106 } else if (distance
> 61 && distance
<= 120) {
113 static void spapr_numa_define_FORM1_domains(SpaprMachineState
*spapr
)
115 MachineState
*ms
= MACHINE(spapr
);
116 NodeInfo
*numa_info
= ms
->numa_state
->nodes
;
117 int nb_numa_nodes
= ms
->numa_state
->num_nodes
;
121 * Fill all associativity domains of non-zero NUMA nodes with
122 * node_id. This is required because the default value (0) is
123 * considered a match with associativity domains of node 0.
125 for (i
= 1; i
< nb_numa_nodes
; i
++) {
126 for (j
= 1; j
< FORM1_DIST_REF_POINTS
; j
++) {
127 spapr
->numa_assoc_array
[i
][j
] = cpu_to_be32(i
);
131 for (src
= 0; src
< nb_numa_nodes
; src
++) {
132 for (dst
= src
; dst
< nb_numa_nodes
; dst
++) {
134 * This is how the associativity domain between A and B
137 * - get the distance D between them
138 * - get the correspondent NUMA level 'n_level' for D
139 * - all associativity arrays were initialized with their own
140 * numa_ids, and we're calculating the distance in node_id
141 * ascending order, starting from node id 0 (the first node
142 * retrieved by numa_state). This will have a cascade effect in
143 * the algorithm because the associativity domains that node 0
144 * defines will be carried over to other nodes, and node 1
145 * associativities will be carried over after taking node 0
146 * associativities into account, and so on. This happens because
147 * we'll assign assoc_src as the associativity domain of dst
148 * as well, for all NUMA levels beyond and including n_level.
150 * The PPC kernel expects the associativity domains of node 0 to
151 * be always 0, and this algorithm will grant that by default.
153 uint8_t distance
= numa_info
[src
].distance
[dst
];
154 uint8_t n_level
= spapr_numa_get_numa_level(distance
);
158 * n_level = 0 means that the distance is greater than our last
159 * rounded value (120). In this case there is no NUMA level match
160 * between src and dst and we can skip the remaining of the loop.
162 * The Linux kernel will assume that the distance between src and
163 * dst, in this case of no match, is 10 (local distance) doubled
164 * for each NUMA it didn't match. We have FORM1_DIST_REF_POINTS
165 * levels (4), so this gives us 10*2*2*2*2 = 160.
167 * This logic can be seen in the Linux kernel source code, as of
168 * v5.9, in arch/powerpc/mm/numa.c, function __node_distance().
175 * We must assign all assoc_src to dst, starting from n_level
176 * and going up to 0x1.
178 for (i
= n_level
; i
> 0; i
--) {
179 assoc_src
= spapr
->numa_assoc_array
[src
][i
];
180 spapr
->numa_assoc_array
[dst
][i
] = assoc_src
;
188 * Set NUMA machine state data based on FORM1 affinity semantics.
190 static void spapr_numa_FORM1_affinity_init(SpaprMachineState
*spapr
,
191 MachineState
*machine
)
193 SpaprMachineClass
*smc
= SPAPR_MACHINE_GET_CLASS(spapr
);
194 int nb_numa_nodes
= machine
->numa_state
->num_nodes
;
195 int i
, j
, max_nodes_with_gpus
;
198 * For all associativity arrays: first position is the size,
199 * position FORM1_DIST_REF_POINTS is always the numa_id,
200 * represented by the index 'i'.
202 * This will break on sparse NUMA setups, when/if QEMU starts
203 * to support it, because there will be no more guarantee that
204 * 'i' will be a valid node_id set by the user.
206 for (i
= 0; i
< nb_numa_nodes
; i
++) {
207 spapr
->numa_assoc_array
[i
][0] = cpu_to_be32(FORM1_DIST_REF_POINTS
);
208 spapr
->numa_assoc_array
[i
][FORM1_DIST_REF_POINTS
] = cpu_to_be32(i
);
212 * Initialize NVLink GPU associativity arrays. We know that
213 * the first GPU will take the first available NUMA id, and
214 * we'll have a maximum of NVGPU_MAX_NUM GPUs in the machine.
215 * At this point we're not sure if there are GPUs or not, but
216 * let's initialize the associativity arrays and allow NVLink
217 * GPUs to be handled like regular NUMA nodes later on.
219 max_nodes_with_gpus
= nb_numa_nodes
+ NVGPU_MAX_NUM
;
221 for (i
= nb_numa_nodes
; i
< max_nodes_with_gpus
; i
++) {
222 spapr
->numa_assoc_array
[i
][0] = cpu_to_be32(FORM1_DIST_REF_POINTS
);
224 for (j
= 1; j
< FORM1_DIST_REF_POINTS
; j
++) {
225 uint32_t gpu_assoc
= smc
->pre_5_1_assoc_refpoints
?
226 SPAPR_GPU_NUMA_ID
: cpu_to_be32(i
);
227 spapr
->numa_assoc_array
[i
][j
] = gpu_assoc
;
230 spapr
->numa_assoc_array
[i
][FORM1_DIST_REF_POINTS
] = cpu_to_be32(i
);
234 * Guests pseries-5.1 and older uses zeroed associativity domains,
235 * i.e. no domain definition based on NUMA distance input.
237 * Same thing with guests that have only one NUMA node.
239 if (smc
->pre_5_2_numa_associativity
||
240 machine
->numa_state
->num_nodes
<= 1) {
244 if (!spapr_numa_is_symmetrical(machine
)) {
245 error_report("Asymmetrical NUMA topologies aren't supported "
246 "in the pSeries machine");
250 spapr_numa_define_FORM1_domains(spapr
);
253 void spapr_numa_associativity_init(SpaprMachineState
*spapr
,
254 MachineState
*machine
)
256 spapr_numa_FORM1_affinity_init(spapr
, machine
);
259 void spapr_numa_write_associativity_dt(SpaprMachineState
*spapr
, void *fdt
,
260 int offset
, int nodeid
)
262 _FDT((fdt_setprop(fdt
, offset
, "ibm,associativity",
263 spapr
->numa_assoc_array
[nodeid
],
264 get_numa_assoc_size(spapr
) * sizeof(uint32_t))));
267 static uint32_t *spapr_numa_get_vcpu_assoc(SpaprMachineState
*spapr
,
270 int max_distance_ref_points
= get_max_dist_ref_points(spapr
);
271 int vcpu_assoc_size
= get_vcpu_assoc_size(spapr
);
272 uint32_t *vcpu_assoc
= g_new(uint32_t, vcpu_assoc_size
);
273 int index
= spapr_get_vcpu_id(cpu
);
276 * VCPUs have an extra 'cpu_id' value in ibm,associativity
277 * compared to other resources. Increment the size at index
278 * 0, put cpu_id last, then copy the remaining associativity
281 vcpu_assoc
[0] = cpu_to_be32(max_distance_ref_points
+ 1);
282 vcpu_assoc
[vcpu_assoc_size
- 1] = cpu_to_be32(index
);
283 memcpy(vcpu_assoc
+ 1, spapr
->numa_assoc_array
[cpu
->node_id
] + 1,
284 (vcpu_assoc_size
- 2) * sizeof(uint32_t));
289 int spapr_numa_fixup_cpu_dt(SpaprMachineState
*spapr
, void *fdt
,
290 int offset
, PowerPCCPU
*cpu
)
292 g_autofree
uint32_t *vcpu_assoc
= NULL
;
293 int vcpu_assoc_size
= get_vcpu_assoc_size(spapr
);
295 vcpu_assoc
= spapr_numa_get_vcpu_assoc(spapr
, cpu
);
297 /* Advertise NUMA via ibm,associativity */
298 return fdt_setprop(fdt
, offset
, "ibm,associativity", vcpu_assoc
,
299 vcpu_assoc_size
* sizeof(uint32_t));
303 int spapr_numa_write_assoc_lookup_arrays(SpaprMachineState
*spapr
, void *fdt
,
306 MachineState
*machine
= MACHINE(spapr
);
307 int max_distance_ref_points
= get_max_dist_ref_points(spapr
);
308 int nb_numa_nodes
= machine
->numa_state
->num_nodes
;
309 int nr_nodes
= nb_numa_nodes
? nb_numa_nodes
: 1;
310 uint32_t *int_buf
, *cur_index
, buf_len
;
313 /* ibm,associativity-lookup-arrays */
314 buf_len
= (nr_nodes
* max_distance_ref_points
+ 2) * sizeof(uint32_t);
315 cur_index
= int_buf
= g_malloc0(buf_len
);
316 int_buf
[0] = cpu_to_be32(nr_nodes
);
317 /* Number of entries per associativity list */
318 int_buf
[1] = cpu_to_be32(max_distance_ref_points
);
320 for (i
= 0; i
< nr_nodes
; i
++) {
322 * For the lookup-array we use the ibm,associativity array,
323 * from numa_assoc_array. without the first element (size).
325 uint32_t *associativity
= spapr
->numa_assoc_array
[i
];
326 memcpy(cur_index
, ++associativity
,
327 sizeof(uint32_t) * max_distance_ref_points
);
328 cur_index
+= max_distance_ref_points
;
330 ret
= fdt_setprop(fdt
, offset
, "ibm,associativity-lookup-arrays", int_buf
,
331 (cur_index
- int_buf
) * sizeof(uint32_t));
337 static void spapr_numa_FORM1_write_rtas_dt(SpaprMachineState
*spapr
,
340 MachineState
*ms
= MACHINE(spapr
);
341 SpaprMachineClass
*smc
= SPAPR_MACHINE_GET_CLASS(spapr
);
342 uint32_t number_nvgpus_nodes
= spapr
->gpu_numa_id
-
343 spapr_numa_initial_nvgpu_numa_id(ms
);
344 uint32_t refpoints
[] = {
350 uint32_t nr_refpoints
= ARRAY_SIZE(refpoints
);
351 uint32_t maxdomain
= ms
->numa_state
->num_nodes
+ number_nvgpus_nodes
;
352 uint32_t maxdomains
[] = {
354 cpu_to_be32(maxdomain
),
355 cpu_to_be32(maxdomain
),
356 cpu_to_be32(maxdomain
),
357 cpu_to_be32(maxdomain
)
360 if (smc
->pre_5_2_numa_associativity
||
361 ms
->numa_state
->num_nodes
<= 1) {
362 uint32_t legacy_refpoints
[] = {
367 uint32_t legacy_maxdomain
= spapr
->gpu_numa_id
> 1 ? 1 : 0;
368 uint32_t legacy_maxdomains
[] = {
370 cpu_to_be32(legacy_maxdomain
),
371 cpu_to_be32(legacy_maxdomain
),
372 cpu_to_be32(legacy_maxdomain
),
373 cpu_to_be32(spapr
->gpu_numa_id
),
376 G_STATIC_ASSERT(sizeof(legacy_refpoints
) <= sizeof(refpoints
));
377 G_STATIC_ASSERT(sizeof(legacy_maxdomains
) <= sizeof(maxdomains
));
381 memcpy(refpoints
, legacy_refpoints
, sizeof(legacy_refpoints
));
382 memcpy(maxdomains
, legacy_maxdomains
, sizeof(legacy_maxdomains
));
384 /* pseries-5.0 and older reference-points array is {0x4, 0x4} */
385 if (smc
->pre_5_1_assoc_refpoints
) {
390 _FDT(fdt_setprop(fdt
, rtas
, "ibm,associativity-reference-points",
391 refpoints
, nr_refpoints
* sizeof(refpoints
[0])));
393 _FDT(fdt_setprop(fdt
, rtas
, "ibm,max-associativity-domains",
394 maxdomains
, sizeof(maxdomains
)));
398 * Helper that writes ibm,associativity-reference-points and
399 * max-associativity-domains in the RTAS pointed by @rtas
402 void spapr_numa_write_rtas_dt(SpaprMachineState
*spapr
, void *fdt
, int rtas
)
404 spapr_numa_FORM1_write_rtas_dt(spapr
, fdt
, rtas
);
407 static target_ulong
h_home_node_associativity(PowerPCCPU
*cpu
,
408 SpaprMachineState
*spapr
,
412 g_autofree
uint32_t *vcpu_assoc
= NULL
;
413 target_ulong flags
= args
[0];
414 target_ulong procno
= args
[1];
417 int vcpu_assoc_size
= get_vcpu_assoc_size(spapr
);
419 /* only support procno from H_REGISTER_VPA */
424 tcpu
= spapr_find_cpu(procno
);
430 * Given that we want to be flexible with the sizes and indexes,
431 * we must consider that there is a hard limit of how many
432 * associativities domain we can fit in R4 up to R9, which would be
433 * 12 associativity domains for vcpus. Assert and bail if that's
436 g_assert((vcpu_assoc_size
- 1) <= 12);
438 vcpu_assoc
= spapr_numa_get_vcpu_assoc(spapr
, tcpu
);
439 /* assoc_idx starts at 1 to skip associativity size */
442 #define ASSOCIATIVITY(a, b) (((uint64_t)(a) << 32) | \
443 ((uint64_t)(b) & 0xffffffff))
445 for (idx
= 0; idx
< 6; idx
++) {
449 * vcpu_assoc[] will contain the associativity domains for tcpu,
450 * including tcpu->node_id and procno, meaning that we don't
451 * need to use these variables here.
453 * We'll read 2 values at a time to fill up the ASSOCIATIVITY()
454 * macro. The ternary will fill the remaining registers with -1
455 * after we went through vcpu_assoc[].
457 a
= assoc_idx
< vcpu_assoc_size
?
458 be32_to_cpu(vcpu_assoc
[assoc_idx
++]) : -1;
459 b
= assoc_idx
< vcpu_assoc_size
?
460 be32_to_cpu(vcpu_assoc
[assoc_idx
++]) : -1;
462 args
[idx
] = ASSOCIATIVITY(a
, b
);
469 static void spapr_numa_register_types(void)
471 /* Virtual Processor Home Node */
472 spapr_register_hypercall(H_HOME_NODE_ASSOCIATIVITY
,
473 h_home_node_associativity
);
476 type_init(spapr_numa_register_types
)