PVE/QemuServer/Memory.pm

   1 package PVE::QemuServer::Memory;
   2
   3 use strict;
   4 use warnings;
   5 use PVE::QemuServer;
   6
   7 my $MAX_NUMA = 8;
   8 my $MAX_MEM = 4194304;
   9 my $STATICMEM = 1024;
  10
  11 sub foreach_dimm{
  12     my ($conf, $vmid, $memory, $sockets, $func) = @_;
  13
  14     my $dimm_id = 0;
  15     my $current_size = 1024;
  16     my $dimm_size = 512;
  17     return if $current_size == $memory;
  18
  19     for (my $j = 0; $j < 8; $j++) {
  20         for (my $i = 0; $i < 32; $i++) {
  21             my $name = "dimm${dimm_id}";
  22             $dimm_id++;
  23             my $numanode = $i % $sockets;
  24             $current_size += $dimm_size;
  25             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
  26             return  $current_size if $current_size >= $memory;
  27         }
  28         $dimm_size *= 2;
  29     }
  30 }
  31
  32 sub foreach_reverse_dimm {
  33     my ($conf, $vmid, $memory, $sockets, $func) = @_;
  34
  35     my $dimm_id = 253;
  36     my $current_size = 4177920;
  37     my $dimm_size = 65536;
  38     return if $current_size == $memory;
  39
  40     for (my $j = 0; $j < 8; $j++) {
  41         for (my $i = 0; $i < 32; $i++) {
  42             my $name = "dimm${dimm_id}";
  43             $dimm_id--;
  44             my $numanode = $i % $sockets;
  45             $current_size -= $dimm_size;
  46             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
  47             return  $current_size if $current_size <= $memory;
  48         }
  49         $dimm_size /= 2;
  50     }
  51 }
  52
  53 sub qemu_memory_hotplug {
  54     my ($vmid, $conf, $defaults, $opt, $value) = @_;
  55
  56     return $value if !check_running($vmid);
  57
  58     my $memory = $conf->{memory} || $defaults->{memory};
  59     $value = $defaults->{memory} if !$value;
  60     return $value if $value == $memory;
  61
  62     my $static_memory = $STATICMEM;
  63     my $dimm_memory = $memory - $static_memory;
  64
  65     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
  66     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
  67
  68
  69     my $sockets = 1;
  70     $sockets = $conf->{sockets} if $conf->{sockets};
  71
  72     if($value > $memory) {
  73
  74         foreach_dimm($conf, $vmid, $value, $sockets, sub {
  75             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
  76
  77                 return if $current_size <= $conf->{memory};
  78
  79                 eval { PVE::QemuServer::vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
  80                 if (my $err = $@) {
  81                     eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); };
  82                     die $err;
  83                 }
  84
  85                 eval { PVE::QemuServer::vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
  86                 if (my $err = $@) {
  87                     eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); };
  88                     die $err;
  89                 }
  90                 #update conf after each succesful module hotplug
  91                 $conf->{memory} = $current_size;
  92                 PVE::QemuConfig->write_config($vmid, $conf);
  93         });
  94
  95     } else {
  96
  97         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
  98             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
  99
 100                 return if $current_size >= $conf->{memory};
 101                 print "try to unplug memory dimm $name\n";
 102
 103                 my $retry = 0;
 104                 while (1) {
 105                     eval { PVE::QemuServer::qemu_devicedel($vmid, $name) };
 106                     sleep 3;
 107                     my $dimm_list = qemu_dimm_list($vmid);
 108                     last if !$dimm_list->{$name};
 109                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
 110                     $retry++;
 111                 }
 112
 113                 #update conf after each succesful module unplug
 114                 $conf->{memory} = $current_size;
 115
 116                 eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); };
 117                 PVE::QemuConfig->write_config($vmid, $conf);
 118         });
 119     }
 120 }
 121
 122 sub qemu_dimm_list {
 123     my ($vmid) = @_;
 124
 125     my $dimmarray = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "query-memory-devices");
 126     my $dimms = {};
 127
 128     foreach my $dimm (@$dimmarray) {
 129
 130         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
 131         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
 132         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
 133         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
 134         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
 135     }
 136     return $dimms;
 137 }
 138
 139 sub config {
 140     my ($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd) = @_;
 141
 142     my $memory = $conf->{memory} || $defaults->{memory};
 143     my $static_memory = 0;
 144     my $dimm_memory = 0;
 145
 146     if ($hotplug_features->{memory}) {
 147         die "NUMA need to be enabled for memory hotplug\n" if !$conf->{numa};
 148         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
 149         $static_memory = $STATICMEM;
 150         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
 151         $dimm_memory = $memory - $static_memory;
 152         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
 153
 154     } else {
 155
 156         $static_memory = $memory;
 157         push @$cmd, '-m', $static_memory;
 158     }
 159
 160     if ($conf->{numa}) {
 161
 162         my $numa_totalmemory = undef;
 163         for (my $i = 0; $i < $MAX_NUMA; $i++) {
 164             next if !$conf->{"numa$i"};
 165             my $numa = PVE::QemuServer::parse_numa($conf->{"numa$i"});
 166             next if !$numa;
 167             # memory
 168             die "missing NUMA node$i memory value\n" if !$numa->{memory};
 169             my $numa_memory = $numa->{memory};
 170             $numa_totalmemory += $numa_memory;
 171             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
 172
 173             # cpus
 174             my $cpulists = $numa->{cpus};
 175             die "missing NUMA node$i cpus\n" if !defined($cpulists);
 176             my $cpus = join(',', map {
 177                 my ($start, $end) = @$_;
 178                 defined($end) ? "$start-$end" : $start
 179             } @$cpulists);
 180
 181             # hostnodes
 182             my $hostnodelists = $numa->{hostnodes};
 183             if (defined($hostnodelists)) {
 184                 my $hostnodes;
 185                 foreach my $hostnoderange (@$hostnodelists) {
 186                     my ($start, $end) = @$hostnoderange;
 187                     $hostnodes .= ',' if $hostnodes;
 188                     $hostnodes .= $start;
 189                     $hostnodes .= "-$end" if defined($end);
 190                     $end //= $start;
 191                     for (my $i = $start; $i <= $end; ++$i ) {
 192                         die "host NUMA node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
 193                     }
 194                 }
 195
 196                 # policy
 197                 my $policy = $numa->{policy};
 198                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
 199                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
 200             }
 201
 202             push @$cmd, '-object', $numa_object;
 203             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
 204         }
 205
 206         die "total memory for NUMA nodes must be equal to vm static memory\n"
 207             if $numa_totalmemory && $numa_totalmemory != $static_memory;
 208
 209         #if no custom tology, we split memory and cores across numa nodes
 210         if(!$numa_totalmemory) {
 211
 212             my $numa_memory = ($static_memory / $sockets) . "M";
 213
 214             for (my $i = 0; $i < $sockets; $i++)  {
 215
 216                 my $cpustart = ($cores * $i);
 217                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
 218                 my $cpus = $cpustart;
 219                 $cpus .= "-$cpuend" if $cpuend;
 220
 221                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
 222                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
 223             }
 224         }
 225     }
 226
 227     if ($hotplug_features->{memory}) {
 228         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
 229             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
 230             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
 231             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
 232
 233             #if dimm_memory is not aligned to dimm map
 234             if($current_size > $memory) {
 235                  $conf->{memory} = $current_size;
 236                  PVE::QemuConfig->write_config($vmid, $conf);
 237             }
 238         });
 239     }
 240 }
 241
 242
 243 1;
 244