PVE/QemuServer/Memory.pm

   1 package PVE::QemuServer::Memory;
   2
   3 use strict;
   4 use warnings;
   5 use PVE::QemuServer;
   6 use PVE::Exception qw(raise raise_param_exc);
   7
   8 my $MAX_NUMA = 8;
   9 my $MAX_MEM = 4194304;
  10 my $STATICMEM = 1024;
  11
  12 sub foreach_dimm{
  13     my ($conf, $vmid, $memory, $sockets, $func) = @_;
  14
  15     my $dimm_id = 0;
  16     my $current_size = 1024;
  17     my $dimm_size = 512;
  18     return if $current_size == $memory;
  19
  20     for (my $j = 0; $j < 8; $j++) {
  21         for (my $i = 0; $i < 32; $i++) {
  22             my $name = "dimm${dimm_id}";
  23             $dimm_id++;
  24             my $numanode = $i % $sockets;
  25             $current_size += $dimm_size;
  26             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
  27             return  $current_size if $current_size >= $memory;
  28         }
  29         $dimm_size *= 2;
  30     }
  31 }
  32
  33 sub foreach_reverse_dimm {
  34     my ($conf, $vmid, $memory, $sockets, $func) = @_;
  35
  36     my $dimm_id = 253;
  37     my $current_size = 4177920;
  38     my $dimm_size = 65536;
  39     return if $current_size == $memory;
  40
  41     for (my $j = 0; $j < 8; $j++) {
  42         for (my $i = 0; $i < 32; $i++) {
  43             my $name = "dimm${dimm_id}";
  44             $dimm_id--;
  45             my $numanode = $i % $sockets;
  46             $current_size -= $dimm_size;
  47             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
  48             return  $current_size if $current_size <= $memory;
  49         }
  50         $dimm_size /= 2;
  51     }
  52 }
  53
  54 sub qemu_memory_hotplug {
  55     my ($vmid, $conf, $defaults, $opt, $value) = @_;
  56
  57     return $value if !PVE::QemuServer::check_running($vmid);
  58
  59     my $memory = $conf->{memory} || $defaults->{memory};
  60     $value = $defaults->{memory} if !$value;
  61     return $value if $value == $memory;
  62
  63     my $static_memory = $STATICMEM;
  64     my $dimm_memory = $memory - $static_memory;
  65
  66     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
  67     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
  68
  69
  70     my $sockets = 1;
  71     $sockets = $conf->{sockets} if $conf->{sockets};
  72
  73     if($value > $memory) {
  74
  75         foreach_dimm($conf, $vmid, $value, $sockets, sub {
  76             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
  77
  78                 return if $current_size <= $conf->{memory};
  79
  80                 eval { PVE::QemuServer::vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
  81                 if (my $err = $@) {
  82                     eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); };
  83                     die $err;
  84                 }
  85
  86                 eval { PVE::QemuServer::vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
  87                 if (my $err = $@) {
  88                     eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); };
  89                     die $err;
  90                 }
  91                 #update conf after each succesful module hotplug
  92                 $conf->{memory} = $current_size;
  93                 PVE::QemuConfig->write_config($vmid, $conf);
  94         });
  95
  96     } else {
  97
  98         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
  99             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
 100
 101                 return if $current_size >= $conf->{memory};
 102                 print "try to unplug memory dimm $name\n";
 103
 104                 my $retry = 0;
 105                 while (1) {
 106                     eval { PVE::QemuServer::qemu_devicedel($vmid, $name) };
 107                     sleep 3;
 108                     my $dimm_list = qemu_dimm_list($vmid);
 109                     last if !$dimm_list->{$name};
 110                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
 111                     $retry++;
 112                 }
 113
 114                 #update conf after each succesful module unplug
 115                 $conf->{memory} = $current_size;
 116
 117                 eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); };
 118                 PVE::QemuConfig->write_config($vmid, $conf);
 119         });
 120     }
 121 }
 122
 123 sub qemu_dimm_list {
 124     my ($vmid) = @_;
 125
 126     my $dimmarray = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "query-memory-devices");
 127     my $dimms = {};
 128
 129     foreach my $dimm (@$dimmarray) {
 130
 131         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
 132         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
 133         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
 134         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
 135         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
 136     }
 137     return $dimms;
 138 }
 139
 140 sub config {
 141     my ($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd) = @_;
 142
 143     my $memory = $conf->{memory} || $defaults->{memory};
 144     my $static_memory = 0;
 145     my $dimm_memory = 0;
 146
 147     if ($hotplug_features->{memory}) {
 148         die "NUMA need to be enabled for memory hotplug\n" if !$conf->{numa};
 149         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
 150         $static_memory = $STATICMEM;
 151         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
 152         $dimm_memory = $memory - $static_memory;
 153         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
 154
 155     } else {
 156
 157         $static_memory = $memory;
 158         push @$cmd, '-m', $static_memory;
 159     }
 160
 161     if ($conf->{numa}) {
 162
 163         my $numa_totalmemory = undef;
 164         for (my $i = 0; $i < $MAX_NUMA; $i++) {
 165             next if !$conf->{"numa$i"};
 166             my $numa = PVE::QemuServer::parse_numa($conf->{"numa$i"});
 167             next if !$numa;
 168             # memory
 169             die "missing NUMA node$i memory value\n" if !$numa->{memory};
 170             my $numa_memory = $numa->{memory};
 171             $numa_totalmemory += $numa_memory;
 172             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
 173
 174             # cpus
 175             my $cpulists = $numa->{cpus};
 176             die "missing NUMA node$i cpus\n" if !defined($cpulists);
 177             my $cpus = join(',', map {
 178                 my ($start, $end) = @$_;
 179                 defined($end) ? "$start-$end" : $start
 180             } @$cpulists);
 181
 182             # hostnodes
 183             my $hostnodelists = $numa->{hostnodes};
 184             if (defined($hostnodelists)) {
 185                 my $hostnodes;
 186                 foreach my $hostnoderange (@$hostnodelists) {
 187                     my ($start, $end) = @$hostnoderange;
 188                     $hostnodes .= ',' if $hostnodes;
 189                     $hostnodes .= $start;
 190                     $hostnodes .= "-$end" if defined($end);
 191                     $end //= $start;
 192                     for (my $i = $start; $i <= $end; ++$i ) {
 193                         die "host NUMA node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
 194                     }
 195                 }
 196
 197                 # policy
 198                 my $policy = $numa->{policy};
 199                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
 200                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
 201             }
 202
 203             push @$cmd, '-object', $numa_object;
 204             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
 205         }
 206
 207         die "total memory for NUMA nodes must be equal to vm static memory\n"
 208             if $numa_totalmemory && $numa_totalmemory != $static_memory;
 209
 210         #if no custom tology, we split memory and cores across numa nodes
 211         if(!$numa_totalmemory) {
 212
 213             my $numa_memory = ($static_memory / $sockets) . "M";
 214
 215             for (my $i = 0; $i < $sockets; $i++)  {
 216
 217                 my $cpustart = ($cores * $i);
 218                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
 219                 my $cpus = $cpustart;
 220                 $cpus .= "-$cpuend" if $cpuend;
 221
 222                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
 223                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
 224             }
 225         }
 226     }
 227
 228     if ($hotplug_features->{memory}) {
 229         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
 230             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
 231             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
 232             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
 233
 234             #if dimm_memory is not aligned to dimm map
 235             if($current_size > $memory) {
 236                  $conf->{memory} = $current_size;
 237                  PVE::QemuConfig->write_config($vmid, $conf);
 238             }
 239         });
 240     }
 241 }
 242
 243
 244 1;
 245