X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=PVE%2FQemuServer.pm;h=dea7b6c084c711b777be771e6b03692901813a55;hb=7196b757e7e0c5b252a5ea88cb288bc550f3fb7b;hp=75ddcdd9714b5d09db07c74d5089f0fe6934ded7;hpb=6f7086436010684c5f9b90e385fdcc6e3a1bbb22;p=qemu-server.git

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 75ddcdd..dea7b6c 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -174,7 +174,7 @@ my $confdesc = {
         optional => 1,
         type => 'boolean',
         description => "Allow hotplug for disk and network device",
-        default => 0,
+        default => 1,
     },
     reboot => {
 	optional => 1,
@@ -306,6 +306,12 @@ EODESC
 	minimum => 1,
 	default => 1,
     },
+    numa => {
+	optional => 1,
+	type => 'boolean',
+	description => "Enable/disable Numa.",
+	default => 0,
+    },
     maxcpus => {
 	optional => 1,
 	type => 'integer',
@@ -483,15 +489,29 @@ my $MAX_UNUSED_DISKS = 8;
 my $MAX_HOSTPCI_DEVICES = 4;
 my $MAX_SERIAL_PORTS = 4;
 my $MAX_PARALLEL_PORTS = 3;
+my $MAX_NUMA = 8;
+
+my $numadesc = {
+    optional => 1,
+    type => 'string', format => 'pve-qm-numanode',
+    typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
+    description => "numa topology",
+};
+PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
+
+for (my $i = 0; $i < $MAX_NUMA; $i++)  {
+    $confdesc->{"numa$i"} = $numadesc;
+}
 
 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
-		      'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3'];
+		      'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
+		      'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 
 my $netdesc = {
     optional => 1,
     type => 'string', format => 'pve-qm-net',
-    typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>][,tag=<vlanid>][,firewall=0|1]",
+    typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
     description => <<EODESCR,
 Specify network devices.
 
@@ -1217,11 +1237,8 @@ sub print_netdevice_full {
          $device = 'virtio-net-pci';
      };
 
-    # qemu > 0.15 always try to boot from network - we disable that by
-    # not loading the pxe rom file
-    my $extra = ($bootorder !~ m/n/) ? "romfile=," : '';
     my $pciaddr = print_pci_addr("$netid", $bridges);
-    my $tmpstr = "$device,${extra}mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
+    my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
 	#Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
 	my $vectors = $net->{queues} * 2 + 2;
@@ -1272,6 +1289,31 @@ sub drive_is_cdrom {
 
 }
 
+sub parse_numa {
+    my ($data) = @_;
+
+    my $res = {};
+
+    foreach my $kvp (split(/,/, $data)) {
+
+	if ($kvp =~ m/^memory=(\S+)$/) {
+	    $res->{memory} = $1;
+	} elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
+	    $res->{policy} = $1;
+	} elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
+	    $res->{cpus}->{start} = $1;
+	    $res->{cpus}->{end} = $3;
+	} elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
+	    $res->{hostnodes}->{start} = $1;
+	    $res->{hostnodes}->{end} = $3;
+	} else {
+	    return undef;
+	}
+    }
+
+    return $res;
+}
+
 sub parse_hostpci {
     my ($value) = @_;
 
@@ -1319,7 +1361,7 @@ sub parse_net {
 
     foreach my $kvp (split(/,/, $data)) {
 
-	if ($kvp =~ m/^(ne2k_pci|e1000|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
+	if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
 	    my $model = lc($1);
 	    my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
 	    $res->{model} = $model;
@@ -1332,8 +1374,10 @@ sub parse_net {
 	    $res->{rate} = $1;
         } elsif ($kvp =~ m/^tag=(\d+)$/) {
             $res->{tag} = $1;
-        } elsif ($kvp =~ m/^firewall=(\d+)$/) {
+        } elsif ($kvp =~ m/^firewall=([01])$/) {
 	    $res->{firewall} = $1;
+	} elsif ($kvp =~ m/^link_down=([01])$/) {
+	    $res->{link_down} = $1;
 	} else {
 	    return undef;
 	}
@@ -1353,7 +1397,8 @@ sub print_net {
     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
     $res .= ",rate=$net->{rate}" if $net->{rate};
     $res .= ",tag=$net->{tag}" if $net->{tag};
-    $res .= ",firewall=$net->{firewall}" if $net->{firewall};
+    $res .= ",firewall=1" if $net->{firewall};
+    $res .= ",link_down=1" if $net->{link_down};
 
     return $res;
 }
@@ -1389,6 +1434,91 @@ sub add_unused_volume {
     return $key;
 }
 
+sub vm_is_volid_owner {
+    my ($storecfg, $vmid, $volid) = @_;
+
+    if ($volid !~  m|^/|) {
+	my ($path, $owner);
+	eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
+	if ($owner && ($owner == $vmid)) {
+	    return 1;
+	}
+    }
+
+    return undef;
+}
+
+sub vmconfig_delete_pending_option {
+    my ($conf, $key) = @_;
+
+    delete $conf->{pending}->{$key};
+    my $pending_delete_hash = { $key => 1 };
+    foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
+	$pending_delete_hash->{$opt} = 1;
+    }
+    $conf->{pending}->{delete} = join(',', keys %$pending_delete_hash);
+}
+
+sub vmconfig_undelete_pending_option {
+    my ($conf, $key) = @_;
+
+    my $pending_delete_hash = {};
+    foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
+	$pending_delete_hash->{$opt} = 1;
+    }
+    delete $pending_delete_hash->{$key};
+
+    my @keylist = keys %$pending_delete_hash;
+    if (scalar(@keylist)) {
+	$conf->{pending}->{delete} = join(',', @keylist);
+    } else {
+	delete $conf->{pending}->{delete};
+    }
+}
+
+sub vmconfig_register_unused_drive {
+    my ($storecfg, $vmid, $conf, $drive) = @_;
+
+    if (!drive_is_cdrom($drive)) {
+	my $volid = $drive->{file};
+	if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
+	    add_unused_volume($conf, $volid, $vmid);
+	}
+    }
+}
+
+sub vmconfig_cleanup_pending {
+    my ($conf) = @_;
+
+    # remove pending changes when nothing changed
+    my $changes;
+    foreach my $opt (keys %{$conf->{pending}}) {
+	if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
+	    $changes = 1;
+	    delete $conf->{pending}->{$opt};
+	}
+    }
+
+    # remove delete if option is not set
+    my $pending_delete_hash = {};
+    foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
+	if (defined($conf->{$opt})) {
+	    $pending_delete_hash->{$opt} = 1;
+	} else {
+	    $changes = 1;
+	}
+    }
+
+    my @keylist = keys %$pending_delete_hash;
+    if (scalar(@keylist)) {
+	$conf->{pending}->{delete} = join(',', @keylist);
+    } else {
+	delete $conf->{pending}->{delete};
+    }
+
+    return $changes;
+}
+
 my $valid_smbios1_options = {
     manufacturer => '\S+',
     product => '\S+',
@@ -1452,6 +1582,17 @@ sub verify_bootdisk {
     die "invalid boot disk '$value'\n";
 }
 
+PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
+sub verify_numa {
+    my ($value, $noerr) = @_;
+
+    return $value if parse_numa($value);
+
+    return undef if $noerr;
+
+    die "unable to parse numa options\n";
+}
+
 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
 sub verify_net {
     my ($value, $noerr) = @_;
@@ -1782,6 +1923,7 @@ sub parse_vm_config {
     my $res = {
 	digest => Digest::SHA::sha1_hex($raw),
 	snapshots => {},
+	pending => {},
     };
 
     $filename =~ m|/qemu-server/(\d+)\.conf$|
@@ -1791,16 +1933,24 @@ sub parse_vm_config {
 
     my $conf = $res;
     my $descr = '';
+    my $section = '';
 
     my @lines = split(/\n/, $raw);
     foreach my $line (@lines) {
 	next if $line =~ m/^\s*$/;
 
-	if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
-	    my $snapname = $1;
+	if ($line =~ m/^\[PENDING\]\s*$/i) {
+	    $section = 'pending';
 	    $conf->{description} = $descr if $descr;
 	    $descr = '';
-	    $conf = $res->{snapshots}->{$snapname} = {};
+	    $conf = $res->{$section} = {};
+	    next;
+
+	} elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
+	    $section = $1;
+	    $conf->{description} = $descr if $descr;
+	    $descr = '';
+	    $conf = $res->{snapshots}->{$section} = {};
 	    next;
 	}
 
@@ -1817,6 +1967,13 @@ sub parse_vm_config {
 	    my $key = $1;
 	    my $value = $2;
 	    $conf->{$key} = $value;
+	} elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
+	    my $value = $1;
+	    if ($section eq 'pending') {
+		$conf->{delete} = $value; # we parse this later
+	    } else {
+		warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
+	    }
 	} elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
 	    my $key = $1;
 	    my $value = $2;
@@ -1879,12 +2036,18 @@ sub write_vm_config {
     my $used_volids = {};
 
     my $cleanup_config = sub {
-	my ($cref, $snapname) = @_;
+	my ($cref, $pending, $snapname) = @_;
 
 	foreach my $key (keys %$cref) {
 	    next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
-		$key eq 'snapstate';
+		$key eq 'snapstate' || $key eq 'pending';
 	    my $value = $cref->{$key};
+	    if ($key eq 'delete') {
+		die "propertry 'delete' is only allowed in [PENDING]\n"
+		    if !$pending;
+		# fixme: check syntax?
+		next;
+	    }
 	    eval { $value = check_type($key, $value); };
 	    die "unable to parse value of '$key' - $@" if $@;
 
@@ -1898,8 +2061,12 @@ sub write_vm_config {
     };
 
     &$cleanup_config($conf);
+
+    &$cleanup_config($conf->{pending}, 1);
+
     foreach my $snapname (keys %{$conf->{snapshots}}) {
-	&$cleanup_config($conf->{snapshots}->{$snapname}, $snapname);
+	die "internal error" if $snapname eq 'pending';
+	&$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
     }
 
     # remove 'unusedX' settings if we re-add a volume
@@ -1922,13 +2089,19 @@ sub write_vm_config {
 	}
 
 	foreach my $key (sort keys %$conf) {
-	    next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots';
+	    next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
 	    $raw .= "$key: $conf->{$key}\n";
 	}
 	return $raw;
     };
 
     my $raw = &$generate_raw_config($conf);
+
+    if (scalar(keys %{$conf->{pending}})){
+	$raw .= "\n[PENDING]\n";
+	$raw .= &$generate_raw_config($conf->{pending});
+    }
+
     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
 	$raw .= "\n[$snapname]\n";
 	$raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
@@ -2611,7 +2784,7 @@ sub config_to_command {
 	$i++;
     }
 
-    push @$cmd, '-boot', "menu=on";
+    push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
 
     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
 
@@ -2686,6 +2859,78 @@ sub config_to_command {
     # push @$cmd, '-cpu', "$cpu,enforce";
     push @$cmd, '-cpu', $cpu;
 
+    my $memory =  $conf->{memory} || $defaults->{memory};
+    push @$cmd, '-m', $memory;
+
+    if ($conf->{numa}) {
+
+	my $numa_totalmemory = undef;
+	for (my $i = 0; $i < $MAX_NUMA; $i++) {
+	    next if !$conf->{"numa$i"};
+	    my $numa = parse_numa($conf->{"numa$i"});
+	    next if !$numa;
+	    # memory
+	    die "missing numa node$i memory value\n" if !$numa->{memory};
+	    my $numa_memory = $numa->{memory};
+	    $numa_totalmemory += $numa_memory;
+	    my $numa_object = "memory-backend-ram,id=ram-node$i,size=$numa_memory"."M";
+
+	    # cpus
+	    my $cpus_start = $numa->{cpus}->{start};
+	    die "missing numa node$i cpus\n" if !defined($cpus_start);
+	    my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
+	    my $cpus = $cpus_start;
+	    if (defined($cpus_end)) {
+		$cpus .= "-$cpus_end";
+		die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
+	    }
+
+	    # hostnodes
+	    my $hostnodes_start = $numa->{hostnodes}->{start};
+	    if (defined($hostnodes_start)) {
+		my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
+		my $hostnodes = $hostnodes_start;
+		if (defined($hostnodes_end)) {
+		    $hostnodes .= "-$hostnodes_end";
+		    die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
+		}
+
+		my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
+		for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
+		    die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
+		}
+
+		# policy
+		my $policy = $numa->{policy};
+		die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
+		$numa_object .= ",host-nodes=$hostnodes,policy=$policy";
+	    }
+
+	    push @$cmd, '-object', $numa_object;
+	    push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
+	}
+
+	die "total memory for NUMA nodes must be equal to vm memory\n"
+	    if $numa_totalmemory && $numa_totalmemory != $memory;
+
+	#if no custom tology, we split memory and cores across numa nodes
+	if(!$numa_totalmemory) {
+
+	    my $numa_memory = ($memory / $sockets) . "M";
+
+	    for (my $i = 0; $i < $sockets; $i++)  {
+
+		my $cpustart = ($cores * $i);
+		my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
+		my $cpus = $cpustart;
+		$cpus .= "-$cpuend" if $cpuend;
+
+		push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
+		push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
+	    }
+	}
+    }
+
     push @$cmd, '-S' if $conf->{freeze};
 
     # set keyboard layout
@@ -2798,8 +3043,6 @@ sub config_to_command {
 	push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
     });
 
-    push @$cmd, '-m', $conf->{memory} || $defaults->{memory};
-
     for (my $i = 0; $i < $MAX_NETS; $i++) {
          next if !$conf->{"net$i"};
          my $d = parse_net($conf->{"net$i"});
@@ -2885,7 +3128,6 @@ sub vm_devices_list {
     my ($vmid) = @_;
 
     my $res = vm_mon_cmd($vmid, 'query-pci');
-
     my $devices = {};
     foreach my $pcibus (@$res) {
 	foreach my $device (@{$pcibus->{devices}}) {
@@ -2901,114 +3143,142 @@ sub vm_devices_list {
 	}
     }
 
+    my $resmice = vm_mon_cmd($vmid, 'query-mice');
+    foreach my $mice (@$resmice) {
+	if ($mice->{name} eq 'QEMU HID Tablet') {
+	    $devices->{tablet} = 1;
+	    last;
+	}
+    }
+
     return $devices;
 }
 
+sub hotplug_enabled {
+    my ($conf) = @_;
+
+    my $default = $confdesc->{'hotplug'}->{default};
+
+    return defined($conf->{hotplug}) ? $conf->{hotplug} : $default;
+}
+
 sub vm_deviceplug {
     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
 
-    return 1 if !check_running($vmid);
+    die "internal error" if !hotplug_enabled($conf);
 
     my $q35 = machine_type_is_q35($conf);
 
-    if ($deviceid eq 'tablet') {
-	qemu_deviceadd($vmid, print_tabletdevice_full($conf));
-	return 1;
-    }
-
-    return 1 if !$conf->{hotplug};
-
     my $devices_list = vm_devices_list($vmid);
     return 1 if defined($devices_list->{$deviceid});
 
-    qemu_bridgeadd($storecfg, $conf, $vmid, $deviceid); #add bridge if we need it for the device
+    qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
+
+    if ($deviceid eq 'tablet') {
+
+	qemu_deviceadd($vmid, print_tabletdevice_full($conf));
+
+    } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
 
-    if ($deviceid =~ m/^(virtio)(\d+)$/) {
-        return undef if !qemu_driveadd($storecfg, $vmid, $device);
+        qemu_driveadd($storecfg, $vmid, $device);
         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
+
         qemu_deviceadd($vmid, $devicefull);
-        if(!qemu_deviceaddverify($vmid, $deviceid)) {
-           qemu_drivedel($vmid, $deviceid);
-           return undef;
+	eval { qemu_deviceaddverify($vmid, $deviceid); };
+	if (my $err = $@) {
+	    eval { qemu_drivedel($vmid, $deviceid); };
+	    warn $@ if $@;
+	    die $err;
         }
-    }
 
-    if ($deviceid =~ m/^(scsihw)(\d+)$/) {
+    } elsif ($deviceid =~ m/^(scsihw)(\d+)$/) {
+
         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
         my $pciaddr = print_pci_addr($deviceid);
         my $devicefull = "$scsihw,id=$deviceid$pciaddr";
+
         qemu_deviceadd($vmid, $devicefull);
-        return undef if(!qemu_deviceaddverify($vmid, $deviceid));
-    }
+        qemu_deviceaddverify($vmid, $deviceid);
 
-    if ($deviceid =~ m/^(scsi)(\d+)$/) {
-        return undef if !qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
-        return undef if !qemu_driveadd($storecfg, $vmid, $device);
-        my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
-        if(!qemu_deviceadd($vmid, $devicefull)) {
-           qemu_drivedel($vmid, $deviceid);
-           return undef;
+    } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
+
+        qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
+        qemu_driveadd($storecfg, $vmid, $device);
+        
+	my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
+	eval { qemu_deviceadd($vmid, $devicefull); };
+	if (my $err = $@) {
+	    eval { qemu_drivedel($vmid, $deviceid); };
+	    warn $@ if $@;
+	    die $err;
         }
-    }
 
-    if ($deviceid =~ m/^(net)(\d+)$/) {
+    } elsif ($deviceid =~ m/^(net)(\d+)$/) {
+
         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
         qemu_deviceadd($vmid, $netdevicefull);
-        if(!qemu_deviceaddverify($vmid, $deviceid)) {
-           qemu_netdevdel($vmid, $deviceid);
-           return undef;
+        eval { qemu_deviceaddverify($vmid, $deviceid); };
+	if (my $err = $@) {
+	    eval { qemu_netdevdel($vmid, $deviceid); };
+	    warn $@ if $@;
+	    die $err;
         }
-    }
 
+    } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
 
-    if (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
 	my $bridgeid = $2;
 	my $pciaddr = print_pci_addr($deviceid);
 	my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
+	
 	qemu_deviceadd($vmid, $devicefull);
-	return undef if !qemu_deviceaddverify($vmid, $deviceid);
+	qemu_deviceaddverify($vmid, $deviceid);
+
+    } else {
+	die "can't hotplug device '$deviceid'\n";	
     }
 
     return 1;
 }
 
+# fixme: this should raise exceptions on error!
 sub vm_deviceunplug {
     my ($vmid, $conf, $deviceid) = @_;
 
-    return 1 if !check_running ($vmid);
-
-    if ($deviceid eq 'tablet') {
-	qemu_devicedel($vmid, $deviceid);
-	return 1;
-    }
-
-    return 1 if !$conf->{hotplug};
+    die "internal error" if !hotplug_enabled($conf);
 
     my $devices_list = vm_devices_list($vmid);
     return 1 if !defined($devices_list->{$deviceid});
 
     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
 
-    if ($deviceid =~ m/^(virtio)(\d+)$/) {
-        qemu_devicedel($vmid, $deviceid);
-        return undef if !qemu_devicedelverify($vmid, $deviceid);
-        return undef if !qemu_drivedel($vmid, $deviceid);
-    }
+    if ($deviceid eq 'tablet') {
 
-    if ($deviceid =~ m/^(lsi)(\d+)$/) {
-        return undef if !qemu_devicedel($vmid, $deviceid);
-    }
+	qemu_devicedel($vmid, $deviceid);
 
-    if ($deviceid =~ m/^(scsi)(\d+)$/) {
-        return undef if !qemu_devicedel($vmid, $deviceid);
-        return undef if !qemu_drivedel($vmid, $deviceid);
-    }
+    } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
 
-    if ($deviceid =~ m/^(net)(\d+)$/) {
         qemu_devicedel($vmid, $deviceid);
-        return undef if !qemu_devicedelverify($vmid, $deviceid);
-        return undef if !qemu_netdevdel($vmid, $deviceid);
+        qemu_devicedelverify($vmid, $deviceid);
+        qemu_drivedel($vmid, $deviceid);
+   
+    } elsif ($deviceid =~ m/^(lsi)(\d+)$/) {
+    
+	qemu_devicedel($vmid, $deviceid);
+    
+    } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
+
+        qemu_devicedel($vmid, $deviceid);
+        qemu_drivedel($vmid, $deviceid);
+    
+    } elsif ($deviceid =~ m/^(net)(\d+)$/) {
+
+        qemu_devicedel($vmid, $deviceid);
+        qemu_devicedelverify($vmid, $deviceid);
+        qemu_netdevdel($vmid, $deviceid);
+
+    } else {
+	die "can't unplug device '$deviceid'\n";
     }
 
     return 1;
@@ -3021,26 +3291,24 @@ sub qemu_deviceadd {
     my %options =  split(/[=,]/, $devicefull);
 
     vm_mon_cmd($vmid, "device_add" , %options);
-    return 1;
 }
 
 sub qemu_devicedel {
-    my($vmid, $deviceid) = @_;
+    my ($vmid, $deviceid) = @_;
+
     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
-    return 1;
 }
 
 sub qemu_driveadd {
-    my($storecfg, $vmid, $device) = @_;
+    my ($storecfg, $vmid, $device) = @_;
 
     my $drive = print_drive_full($storecfg, $vmid, $device);
     my $ret = vm_human_monitor_command($vmid, "drive_add auto $drive");
+
     # If the command succeeds qemu prints: "OK"
-    if ($ret !~ m/OK/s) {
-        syslog("err", "adding drive failed: $ret");
-        return undef;
-    }
-    return 1;
+    return 1 if $ret =~ m/OK/s;
+
+    die "adding drive failed: $ret\n";
 }
 
 sub qemu_drivedel {
@@ -3048,40 +3316,41 @@ sub qemu_drivedel {
 
     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
     $ret =~ s/^\s+//;
-    if ($ret =~ m/Device \'.*?\' not found/s) {
-        # NB: device not found errors mean the drive was auto-deleted and we ignore the error
-    }
-    elsif ($ret ne "") {
-      syslog("err", "deleting drive $deviceid failed : $ret");
-      return undef;
-    }
-    return 1;
+    
+    return 1 if $ret eq "";
+  
+    # NB: device not found errors mean the drive was auto-deleted and we ignore the error
+    return 1 if $ret =~ m/Device \'.*?\' not found/s; 
+    
+    die "deleting drive $deviceid failed : $ret\n";
 }
 
 sub qemu_deviceaddverify {
-    my ($vmid,$deviceid) = @_;
+    my ($vmid, $deviceid) = @_;
 
     for (my $i = 0; $i <= 5; $i++) {
          my $devices_list = vm_devices_list($vmid);
          return 1 if defined($devices_list->{$deviceid});
          sleep 1;
     }
-    syslog("err", "error on hotplug device $deviceid");
-    return undef;
+
+    die "error on hotplug device '$deviceid'\n";
 }
 
 
 sub qemu_devicedelverify {
-    my ($vmid,$deviceid) = @_;
+    my ($vmid, $deviceid) = @_;
+
+    # need to verify that the device is correctly removed as device_del 
+    # is async and empty return is not reliable
 
-    #need to verify the device is correctly remove as device_del is async and empty return is not reliable
     for (my $i = 0; $i <= 5; $i++) {
          my $devices_list = vm_devices_list($vmid);
          return 1 if !defined($devices_list->{$deviceid});
          sleep 1;
     }
-    syslog("err", "error on hot-unplugging device $deviceid");
-    return undef;
+
+    die "error on hot-unplugging device '$deviceid'\n";
 }
 
 sub qemu_findorcreatescsihw {
@@ -3093,31 +3362,43 @@ sub qemu_findorcreatescsihw {
     my $devices_list = vm_devices_list($vmid);
 
     if(!defined($devices_list->{$scsihwid})) {
-       return undef if !vm_deviceplug($storecfg, $conf, $vmid, $scsihwid);
+	vm_deviceplug($storecfg, $conf, $vmid, $scsihwid);
     }
+
     return 1;
 }
 
-sub qemu_bridgeadd {
+sub qemu_add_pci_bridge {
     my ($storecfg, $conf, $vmid, $device) = @_;
 
     my $bridges = {};
-    my $bridgeid = undef;
+
+    my $bridgeid;
+
     print_pci_addr($device, $bridges);
 
     while (my ($k, $v) = each %$bridges) {
 	$bridgeid = $k;
     }
-    return if !$bridgeid || $bridgeid < 1;
+    return 1 if !defined($bridgeid) || $bridgeid < 1;
+
     my $bridge = "pci.$bridgeid";
     my $devices_list = vm_devices_list($vmid);
 
-    if(!defined($devices_list->{$bridge})) {
-	return undef if !vm_deviceplug($storecfg, $conf, $vmid, $bridge);
+    if (!defined($devices_list->{$bridge})) {
+	vm_deviceplug($storecfg, $conf, $vmid, $bridge);
     }
+
     return 1;
 }
 
+sub qemu_set_link_status {
+    my ($vmid, $device, $up) = @_;
+
+    vm_mon_cmd($vmid, "set_link", name => $device, 
+	       up => $up ? JSON::true : JSON::false);
+}
+
 sub qemu_netdevadd {
     my ($vmid, $conf, $device, $deviceid) = @_;
 
@@ -3132,28 +3413,30 @@ sub qemu_netdevdel {
     my ($vmid, $deviceid) = @_;
 
     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
-    return 1;
 }
 
 sub qemu_cpu_hotplug {
     my ($vmid, $conf, $cores) = @_;
 
-    die "new cores config is not defined" if !$cores;
-    die "you can't add more cores than maxcpus"
-	if $conf->{maxcpus} && ($cores > $conf->{maxcpus});
-    return if !check_running($vmid);
+    my $sockets = $conf->{sockets} || 1;
+    die "cpu hotplug only works with one socket\n"
+	if $sockets > 1;
+
+    die "maxcpus is not defined\n"
+	if !$conf->{maxcpus};
 
-    my $currentcores = $conf->{cores} if $conf->{cores};
-    die "current cores is not defined" if !$currentcores;
-    die "maxcpus is not defined" if !$conf->{maxcpus};
-    raise_param_exc({ 'cores' => "online cpu unplug is not yet possible" })
-	if($cores < $currentcores);
+    die "you can't add more cores than maxcpus\n"
+	if $cores > $conf->{maxcpus};
+
+    my $currentcores = $conf->{cores} || 1;
+    die "online cpu unplug is not yet possible\n"
+	if $cores < $currentcores;
 
     my $currentrunningcores = vm_mon_cmd($vmid, "query-cpus");
-    raise_param_exc({ 'cores' => "cores number if running vm is different than configuration" })
-	if scalar (@{$currentrunningcores}) != $currentcores;
+    die "cores number if running vm is different than configuration\n"
+	if scalar(@{$currentrunningcores}) != $currentcores;
 
-    for(my $i = $currentcores; $i < $cores; $i++) {
+    for (my $i = $currentcores; $i < $cores; $i++) {
 	vm_mon_cmd($vmid, "cpu-add", id => int($i));
     }
 }
@@ -3313,18 +3596,6 @@ sub qemu_volume_snapshot_delete {
     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
 }
 
-sub qga_freezefs {
-    my ($vmid) = @_;
-
-    #need to impplement call to qemu-ga
-}
-
-sub qga_unfreezefs {
-    my ($vmid) = @_;
-
-    #need to impplement call to qemu-ga
-}
-
 sub set_migration_caps {
     my ($vmid) = @_;
 
@@ -3349,6 +3620,338 @@ sub set_migration_caps {
     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
 }
 
+my $fast_plug_option = {
+    'name' => 1,
+    'hotplug' => 1,
+    'onboot' => 1, 
+    'shares' => 1,
+    'startup' => 1,
+};
+
+# hotplug changes in [PENDING]
+# $selection hash can be used to only apply specified options, for
+# example: { cores => 1 } (only apply changed 'cores')
+# $errors ref is used to return error messages
+sub vmconfig_hotplug_pending {
+    my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
+
+    my $defaults = load_defaults();
+
+    # commit values which do not have any impact on running VM first
+    # Note: those option cannot raise errors, we we do not care about
+    # $selection and always apply them.
+
+    my $add_error = sub {
+	my ($opt, $msg) = @_;
+	$errors->{$opt} = "hotplug problem - $msg";
+    };
+
+    my $changes = 0;
+    foreach my $opt (keys %{$conf->{pending}}) { # add/change
+	if ($fast_plug_option->{$opt}) {
+	    $conf->{$opt} = $conf->{pending}->{$opt};
+	    delete $conf->{pending}->{$opt};
+	    $changes = 1;
+	}
+    }
+
+    if ($changes) {
+	update_config_nolock($vmid, $conf, 1);
+	$conf = load_config($vmid); # update/reload
+    }
+
+    my $hotplug = hotplug_enabled($conf);
+
+    my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
+    foreach my $opt (@delete) {
+	next if $selection && !$selection->{$opt};
+	eval {
+	    if ($opt eq 'tablet') {
+		die "skip\n" if !$hotplug;
+		if ($defaults->{tablet}) {
+		    vm_deviceplug($storecfg, $conf, $vmid, $opt);
+		} else {
+		    vm_deviceunplug($vmid, $conf, $opt);
+		}
+	    } elsif ($opt eq 'cores') {
+		die "skip\n" if !$hotplug;
+		qemu_cpu_hotplug($vmid, $conf, 1);
+            } elsif ($opt eq 'balloon') {
+		# enable balloon device is not hotpluggable
+		die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
+	    } elsif ($fast_plug_option->{$opt}) {
+		# do nothing
+	    } elsif ($opt =~ m/^net(\d+)$/) {
+		die "skip\n" if !$hotplug;
+		vm_deviceunplug($vmid, $conf, $opt);
+	    } elsif (valid_drivename($opt)) {
+		die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
+		vm_deviceunplug($vmid, $conf, $opt);
+		vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
+	    } else {
+		die "skip\n";
+	    }
+	};
+	if (my $err = $@) {
+	    &$add_error($opt, $err) if $err ne "skip\n";
+	} else {
+	    # save new config if hotplug was successful
+	    delete $conf->{$opt};
+	    vmconfig_undelete_pending_option($conf, $opt);
+	    update_config_nolock($vmid, $conf, 1);
+	    $conf = load_config($vmid); # update/reload
+	}
+    }
+
+    foreach my $opt (keys %{$conf->{pending}}) {
+	next if $selection && !$selection->{$opt};
+	my $value = $conf->{pending}->{$opt};
+	eval {
+	    if ($opt eq 'tablet') {
+		die "skip\n" if !$hotplug;
+		if ($value == 1) {
+		    vm_deviceplug($storecfg, $conf, $vmid, $opt);
+		} elsif ($value == 0) {
+		    vm_deviceunplug($vmid, $conf, $opt);
+		}
+	    } elsif ($opt eq 'cores') {
+		die "skip\n" if !$hotplug;
+		qemu_cpu_hotplug($vmid, $conf, $value);
+	    } elsif ($opt eq 'balloon') {
+		# enable/disable balloning device is not hotpluggable
+		my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
+		my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});		
+		die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
+
+		# allow manual ballooning if shares is set to zero
+		if (!(defined($conf->{shares}) && ($conf->{shares} == 0))) {
+		    my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
+		    vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
+		}
+	    } elsif ($opt =~ m/^net(\d+)$/) { 
+		# some changes can be done without hotplug
+		vmconfig_update_net($storecfg, $conf, $vmid, $opt, $value);
+	    } elsif (valid_drivename($opt)) {
+		# some changes can be done without hotplug
+		vmconfig_update_disk($storecfg, $conf, $vmid, $opt, $value, 1);
+	    } else {
+		die "skip\n";  # skip non-hot-pluggable options
+	    }
+	};
+	if (my $err = $@) {
+	    &$add_error($opt, $err) if $err ne "skip\n";
+	} else {
+	    # save new config if hotplug was successful
+	    $conf->{$opt} = $value;
+	    delete $conf->{pending}->{$opt};
+	    update_config_nolock($vmid, $conf, 1);
+	    $conf = load_config($vmid); # update/reload
+	}
+    }
+}
+
+sub vmconfig_apply_pending {
+    my ($vmid, $conf, $storecfg) = @_;
+
+    # cold plug
+
+    my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
+    foreach my $opt (@delete) { # delete
+	die "internal error" if $opt =~ m/^unused/;
+	$conf = load_config($vmid); # update/reload
+	if (!defined($conf->{$opt})) {
+	    vmconfig_undelete_pending_option($conf, $opt);
+	    update_config_nolock($vmid, $conf, 1);
+	} elsif (valid_drivename($opt)) {
+	    vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
+	    vmconfig_undelete_pending_option($conf, $opt);
+	    delete $conf->{$opt};
+	    update_config_nolock($vmid, $conf, 1);
+	} else {
+	    vmconfig_undelete_pending_option($conf, $opt);
+	    delete $conf->{$opt};
+	    update_config_nolock($vmid, $conf, 1);
+	}
+    }
+
+    $conf = load_config($vmid); # update/reload
+
+    foreach my $opt (keys %{$conf->{pending}}) { # add/change
+	$conf = load_config($vmid); # update/reload
+
+	if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
+	    # skip if nothing changed
+	} elsif (valid_drivename($opt)) {
+	    vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
+		if defined($conf->{$opt});
+	    $conf->{$opt} = $conf->{pending}->{$opt};
+	} else {
+	    $conf->{$opt} = $conf->{pending}->{$opt};
+	}
+
+	delete $conf->{pending}->{$opt};
+	update_config_nolock($vmid, $conf, 1);
+    }
+}
+
+my $safe_num_ne = sub {
+    my ($a, $b) = @_;
+
+    return 0 if !defined($a) && !defined($b);
+    return 1 if !defined($a);
+    return 1 if !defined($b);
+
+    return $a != $b;
+};
+
+my $safe_string_ne = sub {
+    my ($a, $b) = @_;
+
+    return 0 if !defined($a) && !defined($b);
+    return 1 if !defined($a);
+    return 1 if !defined($b);
+
+    return $a ne $b;
+};
+
+sub vmconfig_update_net {
+    my ($storecfg, $conf, $vmid, $opt, $value) = @_;
+
+    my $newnet = parse_net($value);
+
+    my $hotplug = hotplug_enabled($conf);
+
+    if ($conf->{$opt}) {
+	my $oldnet = parse_net($conf->{$opt});
+
+	if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
+	    &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
+	    &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
+	    !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
+
+            # for non online change, we try to hot-unplug
+	    die "skip\n" if !$hotplug;
+	    vm_deviceunplug($vmid, $conf, $opt);
+	} else {
+
+	    die "internal error" if $opt !~ m/net(\d+)/;
+	    my $iface = "tap${vmid}i$1";
+		
+	    if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
+		PVE::Network::tap_rate_limit($iface, $newnet->{rate});
+	    }
+
+	    if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
+		&$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
+		&$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
+		PVE::Network::tap_unplug($iface);
+		PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
+	    }
+
+	    if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
+		qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
+	    }
+
+	    return 1;
+	}
+    }
+    
+    if ($hotplug) {
+	vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
+    } else {
+	die "skip\n";
+    }
+}
+
+sub vmconfig_update_disk {
+    my ($storecfg, $conf, $vmid, $opt, $value, $force) = @_;
+
+    # fixme: do we need force?
+
+    my $drive = parse_drive($opt, $value);
+
+    my $hotplug = hotplug_enabled($conf);
+
+    if ($conf->{$opt}) {
+
+	if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
+
+	    my $media = $drive->{media} || 'disk';
+	    my $oldmedia = $old_drive->{media} || 'disk';
+	    die "unable to change media type\n" if $media ne $oldmedia;
+
+	    if (!drive_is_cdrom($old_drive)) {
+
+		if ($drive->{file} ne $old_drive->{file}) {  
+
+		    die "skip\n" if !$hotplug;
+
+		    # unplug and register as unused
+		    vm_deviceunplug($vmid, $conf, $opt);
+		    vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
+       
+		} else {
+		    # update existing disk
+
+		    # skip non hotpluggable value
+		    if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) || 
+			&$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
+			die "skip\n";
+		    }
+
+		    # apply throttle
+		    if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
+			&$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
+			&$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
+			&$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
+			&$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
+			&$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
+			&$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
+			&$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
+			&$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
+			&$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
+			&$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
+			&$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
+			
+			qemu_block_set_io_throttle($vmid,"drive-$opt",
+						   ($drive->{mbps} || 0)*1024*1024,
+						   ($drive->{mbps_rd} || 0)*1024*1024,
+						   ($drive->{mbps_wr} || 0)*1024*1024,
+						   $drive->{iops} || 0,
+						   $drive->{iops_rd} || 0,
+						   $drive->{iops_wr} || 0,
+						   ($drive->{mbps_max} || 0)*1024*1024,
+						   ($drive->{mbps_rd_max} || 0)*1024*1024,
+						   ($drive->{mbps_wr_max} || 0)*1024*1024,
+						   $drive->{iops_max} || 0,
+						   $drive->{iops_rd_max} || 0,
+						   $drive->{iops_wr_max} || 0);
+
+		    }
+		    
+		    return 1;
+	        }
+	    }
+	}
+    }
+
+    if (drive_is_cdrom($drive)) { # cdrom
+
+	if ($drive->{file} eq 'none') {
+	    vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
+	} else {
+	    my $path = get_iso_path($storecfg, $vmid, $drive->{file});
+	    vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
+	    vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
+	}
+
+    } else { 
+	die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;   
+	# hotplug new disks
+	vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
+    }
+}
+
 sub vm_start {
     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
 
@@ -3361,6 +3964,11 @@ sub vm_start {
 
 	die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
 
+	if (!$statefile && scalar(keys %{$conf->{pending}})) {
+	    vmconfig_apply_pending($vmid, $conf, $storecfg);
+	    $conf = load_config($vmid); # update/reload
+	}
+
 	my $defaults = load_defaults();
 
 	# set environment variable useful inside network script
@@ -3428,15 +4036,15 @@ sub vm_start {
 	if ($migratedfrom) {
 
 	    eval {
-		PVE::QemuServer::set_migration_caps($vmid);
+		set_migration_caps($vmid);
 	    };
 	    warn $@ if $@;
 
 	    if ($spice_port) {
 	        print "spice listens on port $spice_port\n";
 		if ($spice_ticket) {
-		    PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
-		    PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
+		    vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
+		    vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
 		}
 	    }
 
@@ -3450,6 +4058,12 @@ sub vm_start {
 			    property => "guest-stats-polling-interval",
 			    value => 2);
 	    }
+
+	    foreach my $opt (keys %$conf) {
+		next if $opt !~  m/^net\d+$/;
+		my $nicconf = parse_net($conf->{$opt});
+		qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
+	    }
 	}
     });
 }
@@ -3608,11 +4222,10 @@ sub vm_stop {
 	}
 
 	$timeout = 60 if !defined($timeout);
-	my $config = load_config($vmid);
-	
+
 	eval {
 	    if ($shutdown) {
-		if ($config->{agent}) {
+		if (defined($conf) && $conf->{agent}) {
 		    vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
 		} else {
 		    vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
@@ -4504,7 +5117,7 @@ sub restore_tar_archive {
     my $storecfg = cfs_read_file('storage.cfg');
 
     # destroy existing data - keep empty config
-    my $vmcfgfn = PVE::QemuServer::config_file($vmid);
+    my $vmcfgfn = config_file($vmid);
     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
 
     my $tocmd = "/usr/lib/qemu-server/qmextract";
@@ -4888,16 +5501,26 @@ my $savevm_wait = sub {
 };
 
 sub snapshot_create {
-    my ($vmid, $snapname, $save_vmstate, $freezefs, $comment) = @_;
+    my ($vmid, $snapname, $save_vmstate, $comment) = @_;
 
     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
 
-    $freezefs = $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
+    $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
 
-    my $drivehash = {};
+    my $config = load_config($vmid);
 
     my $running = check_running($vmid);
 
+    my $freezefs = $running && $config->{agent};
+    $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
+
+    my $drivehash = {};
+
+    if ($freezefs) {
+	eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
+	warn "guest-fsfreeze-freeze problems - $@" if $@;
+    }
+
     eval {
 	# create internal snapshots of all drives
 
@@ -4913,8 +5536,6 @@ sub snapshot_create {
  	    }
 	};
 
-	qga_freezefs($vmid) if $running && $freezefs;
-
 	foreach_drive($snap, sub {
 	    my ($ds, $drive) = @_;
 
@@ -4929,14 +5550,16 @@ sub snapshot_create {
     };
     my $err = $@;
 
-    eval { qga_unfreezefs($vmid) if $running && $freezefs; };
-    warn $@ if $@;
+    if ($running) {
+	eval { vm_mon_cmd($vmid, "savevm-end")  };
+	warn $@ if $@;
 
-    eval { vm_mon_cmd($vmid, "savevm-end") if $running; };
-    warn $@ if $@;
+	if ($freezefs) {
+	    eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
+	    warn "guest-fsfreeze-thaw problems - $@" if $@;
+	}
 
-    # savevm-end is async, we need to wait
-    if ($running) {
+	# savevm-end is async, we need to wait
 	for (;;) {
 	    my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
 	    if (!$stat->{bytes}) {
@@ -5206,7 +5829,7 @@ sub qemu_drive_mirror {
 		my $transferred = $stat->{offset} || 0;
 		my $remaining = $total - $transferred;
 		my $percent = sprintf "%.2f", ($transferred * 100 / $total);
-		
+
 		print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy\n";
 	    }
 
@@ -5310,7 +5933,7 @@ sub get_current_qemu_machine {
     my ($vmid) = @_;
 
     my $cmd = { execute => 'query-machines', arguments => {} };
-    my $res = PVE::QemuServer::vm_qmp_command($vmid, $cmd);
+    my $res = vm_qmp_command($vmid, $cmd);
 
     my ($current, $default);
     foreach my $e (@$res) {