From 4d3f29edd4587036d3d385c84b6ad5b1b962e706 Mon Sep 17 00:00:00 2001 From: Alexandre Derumier Date: Wed, 28 Jan 2015 06:47:24 +0100 Subject: [PATCH] memory hotplug patch v10 This patch allow to hotplug memory dimm modules though a new option : dimm_memory The dimm modules are generated from a map dimmid size dimm_memory dimm0 512 512 100.00 0 dimm1 512 1024 50.00 1 dimm2 512 1536 33.33 2 dimm3 512 2048 25.00 3 dimm4 512 2560 20.00 0 dimm5 512 3072 16.67 1 dimm6 512 3584 14.29 2 dimm7 512 4096 12.50 3 dimm8 512 4608 11.11 0 dimm9 512 5120 10.00 1 dimm10 512 5632 9.09 2 dimm11 512 6144 8.33 3 dimm12 512 6656 7.69 0 dimm13 512 7168 7.14 1 dimm14 512 7680 6.67 2 dimm15 512 8192 6.25 3 dimm16 512 8704 5.88 0 dimm17 512 9216 5.56 1 dimm18 512 9728 5.26 2 dimm19 512 10240 5.00 3 dimm20 512 10752 4.76 0 ... dimm241 65536 3260416 2.01 1 dimm242 65536 3325952 1.97 2 dimm243 65536 3391488 1.93 3 dimm244 65536 3457024 1.90 0 dimm245 65536 3522560 1.86 1 dimm246 65536 3588096 1.83 2 dimm247 65536 3653632 1.79 3 dimm248 65536 3719168 1.76 0 dimm249 65536 3784704 1.73 1 dimm250 65536 3850240 1.70 2 dimm251 65536 3915776 1.67 3 dimm252 65536 3981312 1.65 0 dimm253 65536 4046848 1.62 1 dimm254 65536 4112384 1.59 2 dimm255 65536 4177920 1.57 3 max dimm_memory size is 4TB, which is the current qemu limit If the dimm_memory value is not aligned on memory module, we align the dimm_memory on the next module. vmid.conf --------- memory: 1024 numa:1 hotplug: memmory when hotplug memory option is enabled, the minimum memory value must be 1GB, and also numa need to be enabled. we assign the first 1GB as static memory, splitted on each numa nodes. The remaining memory is assigned on hotpluggable dimm devices. The static memory need to be also 128MB aligned, to have other dimm devices aligned too. This 128MB alignment is a linux limitation, windows can align on 2MB size. Numa need to be aligned, as linux guest don't boot on some setup with multi sockets, and windows need numa to be able to hotplug memory hotplug ---- qm set -memory X (where X is bigger than current value) unplug (not yet implemented in qemu) ------ qm set -memory X (where X is lower than current value) linux guest ----------- -acpi hotplug module should be loaded in guest -need a recent kernel. (tested with 3.10) can be enable automaticaly, adding: /lib/udev/rules.d/80-hotplug-cpu-mem.rules SUBSYSTEM=="cpu", ACTION=="add", TEST=="online", ATTR{online}=="0", \ ATTR{online}="1" SUBSYSTEM=="memory", ACTION=="add", TEST=="state", ATTR{state}=="offline", \ ATTR{state}="online" windows guest ------------- tested with: - windows 2012 standard - windows 2008 enterprise/datacenter Signed-off-by: Alexandre Derumier --- PVE/QemuServer.pm | 128 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 123 insertions(+), 5 deletions(-) diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm index 5a6c278..7045c14 100644 --- a/PVE/QemuServer.pm +++ b/PVE/QemuServer.pm @@ -490,6 +490,8 @@ my $MAX_HOSTPCI_DEVICES = 4; my $MAX_SERIAL_PORTS = 4; my $MAX_PARALLEL_PORTS = 3; my $MAX_NUMA = 8; +my $MAX_MEM = 4194304; +my $STATICMEM = 1024; my $numadesc = { optional => 1, @@ -2625,6 +2627,7 @@ sub config_to_command { my $have_ovz = -f '/proc/vz/vestat'; my $q35 = machine_type_is_q35($conf); + my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1'); push @$cmd, '/usr/bin/kvm'; @@ -2883,8 +2886,24 @@ sub config_to_command { # push @$cmd, '-cpu', "$cpu,enforce"; push @$cmd, '-cpu', $cpu; - my $memory = $conf->{memory} || $defaults->{memory}; - push @$cmd, '-m', $memory; + my $memory = $conf->{memory} || $defaults->{memory}; + my $static_memory = 0; + my $dimm_memory = 0; + + if ($hotplug_features->{memory}) { + die "Numa need to be enabled for memory hotplug" if !$conf->{numa}; + die "Total memory is bigger than $MAX_MEM MB" if $memory > $MAX_MEM; + die "memory should be a multiple of 128!" if ($memory % 128 != 0); + $static_memory = $STATICMEM; + die "minimum memory must be $static_memory"."MB" if($memory < $static_memory); + $dimm_memory = $memory - $static_memory; + push @$cmd, '-m', "size=".$static_memory.",slots=255,maxmem=".$MAX_MEM."M"; + + } else { + + $static_memory = $memory; + push @$cmd, '-m', $static_memory; + } if ($conf->{numa}) { @@ -2934,13 +2953,13 @@ sub config_to_command { push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i"; } - die "total memory for NUMA nodes must be equal to vm memory\n" - if $numa_totalmemory && $numa_totalmemory != $memory; + die "total memory for NUMA nodes must be equal to vm static memory\n" + if $numa_totalmemory && $numa_totalmemory != $static_memory; #if no custom tology, we split memory and cores across numa nodes if(!$numa_totalmemory) { - my $numa_memory = ($memory / $sockets) . "M"; + my $numa_memory = ($static_memory / $sockets) . "M"; for (my $i = 0; $i < $sockets; $i++) { @@ -2955,6 +2974,29 @@ sub config_to_command { } } + if ($hotplug_features->{memory}) { + my $dimm_id = 0; + my $dimm_size = 512; + my $current_size = $static_memory; + for (my $j = 0; $j < 8; $j++) { + for (my $i = 0; $i < 32; $i++) { + my $name = "dimm${dimm_id}"; + $dimm_id++; + last if $current_size >= $memory; + my $numanode = $i % $sockets; + push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=$dimm_size"."M"; + push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode"; + $current_size += $dimm_size; + #if dimm_memory is not aligned to dimm map + if($current_size > $memory) { + $conf->{memory} = $current_size; + update_config_nolock($vmid, $conf, 1); + } + } + $dimm_size *= 2; + } + } + push @$cmd, '-S' if $conf->{freeze}; # set keyboard layout @@ -3311,6 +3353,22 @@ sub qemu_devicedel { my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid); } +sub qemu_objectadd { + my($vmid, $objectid, $qomtype) = @_; + + vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype); + + return 1; +} + +sub qemu_objectdel { + my($vmid, $objectid) = @_; + + vm_mon_cmd($vmid, "object-del", id => $objectid); + + return 1; +} + sub qemu_driveadd { my ($storecfg, $vmid, $device) = @_; @@ -3454,6 +3512,60 @@ sub qemu_cpu_hotplug { } } +sub qemu_memory_hotplug { + my ($vmid, $conf, $defaults, $opt, $value) = @_; + + return $value if !check_running($vmid); + + my $memory = $conf->{memory} || $defaults->{memory}; + $value = $defaults->{memory} if !$value; + return $value if $value == $memory; + + my $static_memory = $STATICMEM; + my $dimm_memory = $memory - $static_memory; + + die "memory can't be lower than $static_memory MB" if $value < $static_memory; + die "memory unplug is not yet available" if $value < $memory; + die "memory should be a multiple of 128!\n" if ($value % 128 != 0); + die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM; + + + my $sockets = 1; + $sockets = $conf->{sockets} if $conf->{sockets}; + + my $dimm_id = 0; + my $current_size = $static_memory; + my $dimm_size = 512; + for (my $j = 0; $j < 8; $j++) { + for (my $i = 0; $i < 32; $i++) { + my $name = "dimm${dimm_id}"; + $dimm_id++; + $current_size += $dimm_size; + next if $current_size <= $memory; + my $numanode = $i % $sockets; + + eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) }; + if (my $err = $@) { + eval { qemu_objectdel($vmid, "mem-$name"); }; + die $err; + } + + eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) }; + if (my $err = $@) { + eval { qemu_objectdel($vmid, "mem-$name"); }; + die $err; + } + #update conf after each succesful module hotplug + $conf->{$opt} = $current_size; + update_config_nolock($vmid, $conf, 1); + + return $current_size if $current_size >= $value; + } + $dimm_size *= 2; + } + +} + sub qemu_block_set_io_throttle { my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_; @@ -3701,6 +3813,9 @@ sub vmconfig_hotplug_pending { die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/; vm_deviceunplug($vmid, $conf, $opt); vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt})); + } elsif ($opt =~ m/^memory$/) { + die "skip\n" if !$hotplug_features->{memory}; + qemu_memory_hotplug($vmid, $conf, $defaults, $opt); } else { die "skip\n"; } @@ -3749,6 +3864,9 @@ sub vmconfig_hotplug_pending { # some changes can be done without hotplug vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk}, $vmid, $opt, $value, 1); + } elsif ($opt =~ m/^memory$/) { #dimms + die "skip\n" if !$hotplug_features->{memory}; + $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value); } else { die "skip\n"; # skip non-hot-pluggable options } -- 2.39.2