my $MAX_MEM = 4194304;
my $STATICMEM = 1024;
+sub get_numa_node_list {
+ my ($conf) = @_;
+ my @numa_map;
+ for (my $i = 0; $i < $MAX_NUMA; $i++) {
+ my $entry = $conf->{"numa$i"} or next;
+ my $numa = PVE::QemuServer::parse_numa($entry) or next;
+ push @numa_map, $i;
+ }
+ return @numa_map if @numa_map;
+ my $sockets = $conf->{sockets} || 1;
+ return (0..($sockets-1));
+}
+
+# only valid when numa nodes map to a single host node
+sub get_numa_guest_to_host_map {
+ my ($conf) = @_;
+ my $map = {};
+ for (my $i = 0; $i < $MAX_NUMA; $i++) {
+ my $entry = $conf->{"numa$i"} or next;
+ my $numa = PVE::QemuServer::parse_numa($entry) or next;
+ $map->{$i} = print_numa_hostnodes($numa->{hostnodes});
+ }
+ return $map if %$map;
+ my $sockets = $conf->{sockets} || 1;
+ return {map { $_ => $_ } (0..($sockets-1))};
+}
+
sub foreach_dimm{
my ($conf, $vmid, $memory, $sockets, $func) = @_;
return if $current_size == $memory;
+ my @numa_map = get_numa_node_list($conf);
+
for (my $j = 0; $j < 8; $j++) {
for (my $i = 0; $i < 32; $i++) {
my $name = "dimm${dimm_id}";
$dimm_id++;
- my $numanode = $i % $sockets;
+ my $numanode = $numa_map[$i % @numa_map];
$current_size += $dimm_size;
&$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
return $current_size if $current_size >= $memory;
return if $current_size == $memory;
+ my @numa_map = get_numa_node_list($conf);
+
for (my $j = 0; $j < 8; $j++) {
for (my $i = 0; $i < 32; $i++) {
my $name = "dimm${dimm_id}";
$dimm_id--;
- my $numanode = $i % $sockets;
+ my $numanode = $numa_map[(31-$i) % @numa_map];
$current_size -= $dimm_size;
&$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
return $current_size if $current_size <= $memory;
if($value > $memory) {
+ my $numa_hostmap = get_numa_guest_to_host_map($conf) if $conf->{hugepages};
+
foreach_dimm($conf, $vmid, $value, $sockets, sub {
my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
my $hugepages_size = hugepages_size($conf, $dimm_size);
my $path = hugepages_mount_path($hugepages_size);
- my $hugepages_topology->{$hugepages_size}->{$numanode} = hugepages_nr($dimm_size, $hugepages_size);
+ my $host_numanode = $numa_hostmap->{$numanode};
+ my $hugepages_topology->{$hugepages_size}->{$host_numanode} = hugepages_nr($dimm_size, $hugepages_size);
my $code = sub {
my $hugepages_host_topology = hugepages_host_topology();
# cpus
my $cpulists = $numa->{cpus};
die "missing NUMA node$i cpus\n" if !defined($cpulists);
- my $cpus = join(',', map {
+ my $cpus = join(',cpus=', map {
my ($start, $end) = @$_;
defined($end) ? "$start-$end" : $start
} @$cpulists);
# hostnodes
my $hostnodelists = $numa->{hostnodes};
if (defined($hostnodelists)) {
- my $hostnodes;
- foreach my $hostnoderange (@$hostnodelists) {
- my ($start, $end) = @$hostnoderange;
- $hostnodes .= ',' if $hostnodes;
- $hostnodes .= $start;
- $hostnodes .= "-$end" if defined($end);
- $end //= $start;
- for (my $i = $start; $i <= $end; ++$i ) {
- die "host NUMA node$i doesn't exist\n" if ! -d "/sys/devices/system/node/node$i/";
- }
- }
+
+ my $hostnodes = print_numa_hostnodes($hostnodelists);
# policy
my $policy = $numa->{policy};
}
+sub print_numa_hostnodes {
+ my ($hostnodelists) = @_;
+
+ my $hostnodes;
+ foreach my $hostnoderange (@$hostnodelists) {
+ my ($start, $end) = @$hostnoderange;
+ $hostnodes .= ',' if $hostnodes;
+ $hostnodes .= $start;
+ $hostnodes .= "-$end" if defined($end);
+ $end //= $start;
+ for (my $i = $start; $i <= $end; ++$i ) {
+ die "host NUMA node$i doesn't exist\n" if ! -d "/sys/devices/system/node/node$i/";
+ }
+ }
+ return $hostnodes;
+}
+
sub hugepages_mount {
my $mountdata = PVE::ProcFSTools::parse_proc_mounts();
$numa_custom_topology = 1;
my $numa_memory = $numa->{memory};
+ my $hostnodelists = $numa->{hostnodes};
+ my $hostnodes = print_numa_hostnodes($hostnodelists);
+ die "more than 1 hostnode value in numa node is not supported when hugepages are enabled" if $hostnodes !~ m/^(\d)$/;
my $hugepages_size = hugepages_size($conf, $numa_memory);
- $hugepages_topology->{$hugepages_size}->{$i} += hugepages_nr($numa_memory, $hugepages_size);
+ $hugepages_topology->{$hugepages_size}->{$hostnodes} += hugepages_nr($numa_memory, $hugepages_size);
}
}
if ($hotplug_features->{memory}) {
+ my $numa_hostmap = get_numa_guest_to_host_map($conf);
+
foreach_dimm($conf, undef, $memory, $sockets, sub {
my ($conf, undef, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
+ $numanode = $numa_hostmap->{$numanode};
+
my $hugepages_size = hugepages_size($conf, $dimm_size);
$hugepages_topology->{$hugepages_size}->{$numanode} += hugepages_nr($dimm_size, $hugepages_size);
});
}
+sub hugepages_default_nr_hugepages {
+ my ($size) = @_;
+
+ my $cmdline = PVE::Tools::file_read_firstline("/proc/cmdline");
+ my $args = PVE::Tools::split_args($cmdline);
+
+ my $parsed_size = 2; # default is 2M
+
+ foreach my $arg (@$args) {
+ if ($arg eq "hugepagesz=2M") {
+ $parsed_size = 2;
+ } elsif ($arg eq "hugepagesz=1G") {
+ $parsed_size = 1024;
+ } elsif ($arg =~ m/^hugepages=(\d+)?$/) {
+ if ($parsed_size == $size) {
+ return $1;
+ }
+ }
+ }
+
+ return 0;
+}
+
sub hugepages_pre_deallocate {
my ($hugepages_topology) = @_;
my $hugepages_size = $size * 1024;
my $path = "/sys/kernel/mm/hugepages/hugepages-${hugepages_size}kB/";
- my $hugepages_nr = PVE::Tools::file_read_firstline($path."nr_hugepages");
- PVE::ProcFSTools::write_proc_entry($path."nr_hugepages", 0);
+ my $hugepages_nr = hugepages_default_nr_hugepages($size);
+ PVE::ProcFSTools::write_proc_entry($path."nr_hugepages", $hugepages_nr);
}
}
projects / qemu-server.git / blobdiff