X-Git-Url: https://git.proxmox.com/?p=qemu-server.git;a=blobdiff_plain;f=PVE%2FQemuServer.pm;h=f7d99e398b17a752ff5207b191e06b160d123c35;hp=ac9dfdead8bda95751714bbf587d55f049958ccf;hb=05a4c550f334c4a60ea978cc7fc9f1844ea5d06a;hpb=311e92935abb5c722be587bde5414125f47d04e7

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index ac9dfde..f7d99e3 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -3,45 +3,51 @@ package PVE::QemuServer;
 use strict;
 use warnings;
 
-use POSIX;
-use IO::Handle;
-use IO::Select;
-use IO::File;
-use IO::Dir;
-use IO::Socket::UNIX;
+use Cwd 'abs_path';
+use Digest::SHA;
+use Fcntl ':flock';
+use Fcntl;
 use File::Basename;
+use File::Copy qw(copy);
 use File::Path;
 use File::stat;
 use Getopt::Long;
-use Digest::SHA;
-use Fcntl ':flock';
-use Cwd 'abs_path';
+use IO::Dir;
+use IO::File;
+use IO::Handle;
+use IO::Select;
+use IO::Socket::UNIX;
 use IPC::Open3;
 use JSON;
-use Fcntl;
-use PVE::SafeSyslog;
-use Storable qw(dclone);
 use MIME::Base64;
-use PVE::Exception qw(raise raise_param_exc);
-use PVE::Storage;
-use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
-use PVE::JSONSchema qw(get_standard_option);
+use POSIX;
+use Storable qw(dclone);
+use Time::HiRes qw(gettimeofday);
+use URI::Escape;
+use UUID;
+
 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
+use PVE::DataCenterConfig;
+use PVE::Exception qw(raise raise_param_exc);
+use PVE::GuestHelpers;
 use PVE::INotify;
+use PVE::JSONSchema qw(get_standard_option);
 use PVE::ProcFSTools;
-use PVE::QemuConfig;
-use PVE::QMPClient;
 use PVE::RPCEnvironment;
-use PVE::GuestHelpers;
-use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
-use PVE::QemuServer::Memory;
-use PVE::QemuServer::USB qw(parse_usb_device);
-use PVE::QemuServer::Cloudinit;
+use PVE::Storage;
 use PVE::SysFSTools;
 use PVE::Systemd;
-use Time::HiRes qw(gettimeofday);
-use File::Copy qw(copy);
-use URI::Escape;
+use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
+
+use PVE::QMPClient;
+use PVE::QemuConfig;
+use PVE::QemuServer::Helpers qw(min_version);
+use PVE::QemuServer::Cloudinit;
+use PVE::QemuServer::Machine;
+use PVE::QemuServer::Memory;
+use PVE::QemuServer::Monitor qw(mon_cmd);
+use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
+use PVE::QemuServer::USB qw(parse_usb_device);
 
 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
 my $OVMF = {
@@ -55,8 +61,6 @@ my $OVMF = {
     ],
 };
 
-my $qemu_snap_storage = { rbd => 1 };
-
 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 
 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
@@ -89,7 +93,7 @@ PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 	description => "Specifies the Qemu machine type.",
 	type => 'string',
-	pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
+	pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 	maxLength => 40,
 	optional => 1,
 });
@@ -104,17 +108,11 @@ sub cgroups_write {
 
 }
 
-my $nodename = PVE::INotify::nodename();
-
-mkdir "/etc/pve/nodes/$nodename";
-my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
-mkdir $confdir;
-
-my $var_run_tmpdir = "/var/run/qemu-server";
-mkdir $var_run_tmpdir;
-
-my $lock_dir = "/var/lock/qemu-server";
-mkdir $lock_dir;
+my $nodename_cache;
+sub nodename {
+    $nodename_cache //= PVE::INotify::nodename();
+    return $nodename_cache;
+}
 
 my $cpu_vendor_list = {
     # Intel CPUs
@@ -249,6 +247,13 @@ my $agent_fmt = {
 	optional => 1,
 	default => 0
     },
+    type => {
+	description => "Select the agent type",
+	type => 'string',
+	default => 'virtio',
+	optional => 1,
+	enum => [qw(virtio isa)],
+    },
 };
 
 my $vga_fmt = {
@@ -430,7 +435,7 @@ win7;; Microsoft Windows 7
 win8;; Microsoft Windows 8/2012/2012r2
 win10;; Microsoft Windows 10/2016
 l24;; Linux 2.4 Kernel
-l26;; Linux 2.6/3.X Kernel
+l26;; Linux 2.6 - 5.X Kernel
 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 EODESC
     },
@@ -698,6 +703,11 @@ EODESCR
 	description => "Configure additional enhancements for SPICE.",
 	optional => 1
     },
+    tags => {
+	type => 'string', format => 'pve-tag-list',
+	description => 'Tags of the VM. This is only meta information.',
+	optional => 1,
+    },
 };
 
 my $cicustom_fmt = {
@@ -1334,7 +1344,7 @@ my $usbdesc = {
 };
 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 
-my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
+my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
 my $hostpci_fmt = {
     host => {
 	default_key => 1,
@@ -1551,29 +1561,7 @@ sub option_exists {
     return defined($confdesc->{$key});
 }
 
-sub nic_models {
-    return $nic_model_list;
-}
-
-sub os_list_description {
-
-    return {
-	other => 'Other',
-	wxp => 'Windows XP',
-	w2k => 'Windows 2000',
-	w2k3 =>, 'Windows 2003',
-	w2k8 => 'Windows 2008',
-	wvista => 'Windows Vista',
-	win7 => 'Windows 7',
-	win8 => 'Windows 8/2012',
-	win10 => 'Windows 10/2016',
-	l24 => 'Linux 2.4',
-	l26 => 'Linux 2.6',
-    };
-}
-
 my $cdrom_path;
-
 sub get_cdrom_path {
 
     return  $cdrom_path if $cdrom_path;
@@ -1767,7 +1755,7 @@ sub parse_drive {
 }
 
 sub print_drive {
-    my ($vmid, $drive) = @_;
+    my ($drive) = @_;
     my $data = { %$drive };
     delete $data->{$_} for qw(index interface);
     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
@@ -1834,20 +1822,14 @@ sub path_is_scsi {
     return $res;
 }
 
-sub machine_type_is_q35 {
-    my ($conf) = @_;
-
-    return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
-}
-
 sub print_tabletdevice_full {
     my ($conf, $arch) = @_;
 
-    my $q35 = machine_type_is_q35($conf);
+    my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
 
     # we use uhci for old VMs because tablet driver was buggy in older qemu
     my $usbbus;
-    if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
+    if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
 	$usbbus = 'ehci';
     } else {
 	$usbbus = 'uhci';
@@ -1896,7 +1878,10 @@ sub print_drivedevice_full {
 		 $path = PVE::Storage::path($storecfg, $drive->{file});
 	    }
 
-	    if($path =~ m/^iscsi\:\/\//){
+	    # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
+	    my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
+	    if ($path =~ m/^iscsi\:\/\// &&
+	       !min_version($version, 4, 1)) {
 		$devicetype = 'generic';
 	    }
 	}
@@ -2164,16 +2149,26 @@ my $vga_map = {
 };
 
 sub print_vga_device {
-    my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
+    my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
 
     my $type = $vga_map->{$vga->{type}};
     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
 	$type = 'virtio-gpu';
     }
     my $vgamem_mb = $vga->{memory};
+
+    my $max_outputs = '';
     if ($qxlnum) {
 	$type = $id ? 'qxl' : 'qxl-vga';
+
+	if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
+	    # set max outputs so linux can have up to 4 qxl displays with one device
+	    if (min_version($machine_version, 4, 1)) {
+		$max_outputs = ",max_outputs=4";
+	    }
+	}
     }
+
     die "no devicetype for $vga->{type}\n" if !$type;
 
     my $memory = "";
@@ -2194,7 +2189,7 @@ sub print_vga_device {
 	$memory = ",ram_size=67108864,vram_size=33554432";
     }
 
-    my $q35 = machine_type_is_q35($conf);
+    my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
     my $vgaid = "vga" . ($id // '');
     my $pciaddr;
 
@@ -2205,7 +2200,7 @@ sub print_vga_device {
 	$pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
     }
 
-    return "$type,id=${vgaid}${memory}${pciaddr}";
+    return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
 }
 
 sub drive_is_cloudinit {
@@ -2255,13 +2250,9 @@ sub parse_hostpci {
     my @idlist = split(/;/, $res->{host});
     delete $res->{host};
     foreach my $id (@idlist) {
-	if ($id =~ m/\./) { # full id 00:00.1
-	    push @{$res->{pciid}}, {
-		id => $id,
-	    };
-	} else { # partial id 00:00
-	    $res->{pciid} = PVE::SysFSTools::lspci($id);
-	}
+	my $devs = PVE::SysFSTools::lspci($id);
+	die "no PCI device found for '$id'\n" if !scalar(@$devs);
+	push @{$res->{pciid}}, @$devs;
     }
     return $res;
 }
@@ -2348,40 +2339,6 @@ sub vm_is_volid_owner {
     return undef;
 }
 
-sub split_flagged_list {
-    my $text = shift || '';
-    $text =~ s/[,;]/ /g;
-    $text =~ s/^\s+//;
-    return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
-}
-
-sub join_flagged_list {
-    my ($how, $lst) = @_;
-    join $how, map { $lst->{$_} . $_ } keys %$lst;
-}
-
-sub vmconfig_delete_pending_option {
-    my ($conf, $key, $force) = @_;
-
-    delete $conf->{pending}->{$key};
-    my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
-    $pending_delete_hash->{$key} = $force ? '!' : '';
-    $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
-}
-
-sub vmconfig_undelete_pending_option {
-    my ($conf, $key) = @_;
-
-    my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
-    delete $pending_delete_hash->{$key};
-
-    if (%$pending_delete_hash) {
-	$conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
-    } else {
-	delete $conf->{pending}->{delete};
-    }
-}
-
 sub vmconfig_register_unused_drive {
     my ($storecfg, $vmid, $conf, $drive) = @_;
 
@@ -2396,37 +2353,6 @@ sub vmconfig_register_unused_drive {
     }
 }
 
-sub vmconfig_cleanup_pending {
-    my ($conf) = @_;
-
-    # remove pending changes when nothing changed
-    my $changes;
-    foreach my $opt (keys %{$conf->{pending}}) {
-	if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
-	    $changes = 1;
-	    delete $conf->{pending}->{$opt};
-	}
-    }
-
-    my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
-    my $pending_delete_hash = {};
-    while (my ($opt, $force) = each %$current_delete_hash) {
-	if (defined($conf->{$opt})) {
-	    $pending_delete_hash->{$opt} = $force;
-	} else {
-	    $changes = 1;
-	}
-    }
-
-    if (%$pending_delete_hash) {
-	$conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
-    } else {
-	delete $conf->{pending}->{delete};
-    }
-
-    return $changes;
-}
-
 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
 my $smbios1_fmt = {
     uuid => {
@@ -2609,17 +2535,8 @@ sub check_type {
     }
 }
 
-sub touch_config {
-    my ($vmid) = @_;
-
-    my $conf = PVE::QemuConfig->config_file($vmid);
-    utime undef, undef, $conf;
-}
-
 sub destroy_vm {
-    my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
-
-    my $conffile = PVE::QemuConfig->config_file($vmid);
+    my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
 
     my $conf = PVE::QemuConfig->load_config($vmid);
 
@@ -2629,11 +2546,9 @@ sub destroy_vm {
 	# check if any base image is still used by a linked clone
 	foreach_drive($conf, sub {
 		my ($ds, $drive) = @_;
-
 		return if drive_is_cdrom($drive);
 
 		my $volid = $drive->{file};
-
 		return if !$volid || $volid =~ m|^/|;
 
 		die "base volume '$volid' is still in use by linked cloned\n"
@@ -2645,43 +2560,31 @@ sub destroy_vm {
     # only remove disks owned by this VM
     foreach_drive($conf, sub {
 	my ($ds, $drive) = @_;
-
  	return if drive_is_cdrom($drive, 1);
 
 	my $volid = $drive->{file};
-
 	return if !$volid || $volid =~ m|^/|;
 
 	my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
 	return if !$path || !$owner || ($owner != $vmid);
 
-	eval {
-	    PVE::Storage::vdisk_free($storecfg, $volid);
-	};
+	eval { PVE::Storage::vdisk_free($storecfg, $volid) };
 	warn "Could not remove disk '$volid', check manually: $@" if $@;
-
     });
 
-    if ($keep_empty_config) {
-	PVE::Tools::file_set_contents($conffile, "memory: 128\n");
-    } else {
-	unlink $conffile;
-    }
-
     # also remove unused disk
-    eval {
-	my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
-
-	eval {
-	    PVE::Storage::foreach_volid($dl, sub {
-		my ($volid, $sid, $volname, $d) = @_;
-		PVE::Storage::vdisk_free($storecfg, $volid);
-	    });
-	};
+    my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
+    PVE::Storage::foreach_volid($vmdisks, sub {
+	my ($volid, $sid, $volname, $d) = @_;
+	eval { PVE::Storage::vdisk_free($storecfg, $volid) };
 	warn $@ if $@;
+    });
 
-    };
-    warn $@ if $@;
+    if (defined $replacement_conf) {
+	PVE::QemuConfig->write_config($vmid, $replacement_conf);
+    } else {
+	PVE::QemuConfig->destroy_config($vmid);
+    }
 }
 
 sub parse_vm_config {
@@ -2764,7 +2667,7 @@ sub parse_vm_config {
 		    my $v = parse_drive($key, $value);
 		    if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
 			$v->{file} = $volid;
-			$value = print_drive($vmid, $v);
+			$value = print_drive($v);
 		    } else {
 			warn "vm $vmid - unable to parse value of '$key'\n";
 			next;
@@ -2837,7 +2740,7 @@ sub write_vm_config {
     &$cleanup_config($conf->{pending}, 1);
 
     foreach my $snapname (keys %{$conf->{snapshots}}) {
-	die "internal error" if $snapname eq 'pending';
+	die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
 	&$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
     }
 
@@ -2906,6 +2809,7 @@ sub config_list {
     my $res = {};
     return $res if !$vmlist || !$vmlist->{ids};
     my $ids = $vmlist->{ids};
+    my $nodename = nodename();
 
     foreach my $vmid (keys %$ids) {
 	my $d = $ids->{$vmid};
@@ -2964,7 +2868,7 @@ sub shared_nodes {
 
     my $nodelist = PVE::Cluster::get_nodelist();
     my $nodehash = { map { $_ => 1 } @$nodelist };
-    my $nodename = PVE::INotify::nodename();
+    my $nodename = nodename();
 
     foreach_drive($conf, sub {
 	my ($ds, $drive) = @_;
@@ -3031,70 +2935,19 @@ sub check_local_storage_availability {
     return $nodehash
 }
 
-sub check_cmdline {
-    my ($pidfile, $pid) = @_;
-
-    my $fh = IO::File->new("/proc/$pid/cmdline", "r");
-    if (defined($fh)) {
-	my $line = <$fh>;
-	$fh->close;
-	return undef if !$line;
-	my @param = split(/\0/, $line);
-
-	my $cmd = $param[0];
-	return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
-
-	for (my $i = 0; $i < scalar (@param); $i++) {
-	    my $p = $param[$i];
-	    next if !$p;
-	    if (($p eq '-pidfile') || ($p eq '--pidfile')) {
-		my $p = $param[$i+1];
-		return 1 if $p && ($p eq $pidfile);
-		return undef;
-	    }
-	}
-    }
-    return undef;
-}
-
+# Compat only, use assert_config_exists_on_node and vm_running_locally where possible
 sub check_running {
     my ($vmid, $nocheck, $node) = @_;
 
-    my $filename = PVE::QemuConfig->config_file($vmid, $node);
-
-    die "unable to find configuration file for VM $vmid - no such machine\n"
-	if !$nocheck && ! -f $filename;
-
-    my $pidfile = pidfile_name($vmid);
-
-    if (my $fd = IO::File->new("<$pidfile")) {
-	my $st = stat($fd);
-	my $line = <$fd>;
-	close($fd);
-
-	my $mtime = $st->mtime;
-	if ($mtime > time()) {
-	    warn "file '$filename' modified in future\n";
-	}
-
-	if ($line =~ m/^(\d+)$/) {
-	    my $pid = $1;
-	    if (check_cmdline($pidfile, $pid)) {
-		if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
-		    return $pid;
-		}
-	    }
-	}
-    }
-
-    return undef;
+    PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
+    return PVE::QemuServer::Helpers::vm_running_locally($vmid);
 }
 
 sub vzlist {
 
     my $vzlist = config_list();
 
-    my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
+    my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
 
     while (defined(my $de = $fd->read)) {
 	next if $de !~ m/^(\d+)\.pid$/;
@@ -3177,7 +3030,12 @@ our $vmstatus_return_properties = {
 	description => "The current config lock, if any.",
 	type => 'string',
 	optional => 1,
-    }
+    },
+    tags => {
+	description  => "The current configured tags, if any",
+	type => 'string',
+	optional => 1,
+    },
 };
 
 my $last_proc_pid_stat;
@@ -3202,8 +3060,7 @@ sub vmstatus {
     foreach my $vmid (keys %$list) {
 	next if $opt_vmid && ($vmid ne $opt_vmid);
 
-	my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
-	my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
+	my $conf = PVE::QemuConfig->load_config($vmid);
 
 	my $d = { vmid => $vmid };
 	$d->{pid} = $list->{$vmid}->{pid};
@@ -3249,6 +3106,7 @@ sub vmstatus {
 
 	$d->{serial} = 1 if conf_has_serial($conf);
 	$d->{lock} = $conf->{lock} if $conf->{lock};
+	$d->{tags} = $conf->{tags} if defined($conf->{tags});
 
 	$res->{$vmid} = $d;
     }
@@ -3488,28 +3346,33 @@ sub vga_conf_has_spice {
     return $1 || 1;
 }
 
-my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
-sub get_host_arch() {
-    $host_arch = (POSIX::uname())[4] if !$host_arch;
-    return $host_arch;
-}
-
 sub is_native($) {
     my ($arch) = @_;
     return get_host_arch() eq $arch;
 }
 
+sub get_vm_arch {
+    my ($conf) = @_;
+    return $conf->{arch} // get_host_arch();
+}
+
 my $default_machines = {
     x86_64 => 'pc',
     aarch64 => 'virt',
 };
 
-sub get_basic_machine_info {
-    my ($conf, $forcemachine) = @_;
+sub get_vm_machine {
+    my ($conf, $forcemachine, $arch, $add_pve_version) = @_;
+
+    my $machine = $forcemachine || $conf->{machine};
+
+    if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
+	$arch //= 'x86_64';
+	$machine ||= $default_machines->{$arch};
+	$machine .= "+pve$PVE::QemuServer::Machine::PVE_MACHINE_VERSION" if $add_pve_version;
+    }
 
-    my $arch = $conf->{arch} // get_host_arch();
-    my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
-    return ($arch, $machine);
+    return $machine;
 }
 
 sub get_ovmf_files($) {
@@ -3534,8 +3397,124 @@ sub get_command_for_arch($) {
     return $cmd;
 }
 
+# To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
+# to use in a QEMU command line (-cpu element), first array_intersect the result
+# of query_supported_ with query_understood_. This is necessary because:
+#
+# a) query_understood_ returns flags the host cannot use and
+# b) query_supported_ (rather the QMP call) doesn't actually return CPU
+#    flags, but CPU settings - with most of them being flags. Those settings
+#    (and some flags, curiously) cannot be specified as a "-cpu" argument.
+#
+# query_supported_ needs to start up to 2 temporary VMs and is therefore rather
+# expensive. If you need the value returned from this, you can get it much
+# cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
+# $accel being 'kvm' or 'tcg'.
+#
+# pvestatd calls this function on startup and whenever the QEMU/KVM version
+# changes, automatically populating pmxcfs.
+#
+# Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
+# since kvm and tcg machines support different flags
+#
+sub query_supported_cpu_flags {
+    my $flags = {};
+
+    my ($arch, $default_machine) = get_basic_machine_info();
+
+    # FIXME: Once this is merged, the code below should work for ARM as well:
+    # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
+    die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
+	$arch eq "aarch64";
+
+    my $kvm_supported = defined(kvm_version());
+    my $qemu_cmd = get_command_for_arch($arch);
+    my $fakevmid = -1;
+    my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
+
+    # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
+    my $query_supported_run_qemu = sub {
+	my ($kvm) = @_;
+
+	my $flags = {};
+	my $cmd = [
+	    $qemu_cmd,
+	    '-machine', $default_machine,
+	    '-display', 'none',
+	    '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
+	    '-mon', 'chardev=qmp,mode=control',
+	    '-pidfile', $pidfile,
+	    '-S', '-daemonize'
+	];
+
+	if (!$kvm) {
+	    push @$cmd, '-accel', 'tcg';
+	}
+
+	my $rc = run_command($cmd, noerr => 1, quiet => 0);
+	die "QEMU flag querying VM exited with code " . $rc if $rc;
+
+	eval {
+	    my $cmd_result = mon_cmd(
+		$fakevmid,
+		'query-cpu-model-expansion',
+		type => 'full',
+		model => { name => 'host' }
+	    );
+
+	    my $props = $cmd_result->{model}->{props};
+	    foreach my $prop (keys %$props) {
+		next if $props->{$prop} ne '1';
+		# QEMU returns some flags multiple times, with '_', '.' or '-'
+		# (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
+		# We only keep those with underscores, to match /proc/cpuinfo
+		$prop =~ s/\.|-/_/g;
+		$flags->{$prop} = 1;
+	    }
+	};
+	my $err = $@;
+
+	# force stop with 10 sec timeout and 'nocheck'
+	# always stop, even if QMP failed
+	vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
+
+	die $err if $err;
+
+	return [ sort keys %$flags ];
+    };
+
+    # We need to query QEMU twice, since KVM and TCG have different supported flags
+    PVE::QemuConfig->lock_config($fakevmid, sub {
+	$flags->{tcg} = eval { $query_supported_run_qemu->(0) };
+	warn "warning: failed querying supported tcg flags: $@\n" if $@;
+
+	if ($kvm_supported) {
+	    $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
+	    warn "warning: failed querying supported kvm flags: $@\n" if $@;
+	}
+    });
+
+    return $flags;
+}
+
+# Understood CPU flags are written to a file at 'pve-qemu' compile time
+my $understood_cpu_flag_dir = "/usr/share/kvm";
+sub query_understood_cpu_flags {
+    my $arch = get_host_arch();
+    my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
+
+    die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
+	if ! -e $filepath;
+
+    my $raw = file_get_contents($filepath);
+    $raw =~ s/^\s+|\s+$//g;
+    my @flags = split(/\s+/, $raw);
+
+    return \@flags;
+}
+
 sub get_cpu_options {
-    my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
+    my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
 
     my $cpuFlags = [];
     my $ostype = $conf->{ostype};
@@ -3559,20 +3538,19 @@ sub get_cpu_options {
 
     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
 
-    push @$cpuFlags , '-x2apic'
-	if $conf->{ostype} && $conf->{ostype} eq 'solaris';
+    push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
 
     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
 
     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
 
-    if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
+    if (min_version($machine_version, 2, 3) && $arch eq 'x86_64') {
 
 	push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
 	push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
     }
 
-    add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
+    add_hyperv_enlightenments($cpuFlags, $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
 
     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
 
@@ -3604,12 +3582,22 @@ sub config_to_command {
     my $ostype = $conf->{ostype};
     my $winversion = windows_version($ostype);
     my $kvm = $conf->{kvm};
+    my $nodename = nodename();
 
-    my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
+    my $arch = get_vm_arch($conf);
     my $kvm_binary = get_command_for_arch($arch);
     my $kvmver = kvm_user_version($kvm_binary);
+
+    my $add_pve_version = min_version($kvmver, 4, 1);
+
+    my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
+    my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
     $kvm //= 1 if is_native($arch);
 
+    $machine_version =~ m/(\d+)\.(\d+)/;
+    die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
+	if !PVE::QemuServer::min_version($kvmver, $1, $2);
+
     if ($kvm) {
 	die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
 	    if !defined kvm_version();
@@ -3623,9 +3611,7 @@ sub config_to_command {
 
     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
 
-    my $have_ovz = -f '/proc/vz/vestat';
-
-    my $q35 = machine_type_is_q35($conf);
+    my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
     my $use_old_bios_files = undef;
     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
@@ -3643,16 +3629,16 @@ sub config_to_command {
 
     my $use_virtio = 0;
 
-    my $qmpsocket = qmp_socket($vmid);
+    my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
     push @$cmd, '-mon', "chardev=qmp,mode=control";
 
-    if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
+    if (min_version($machine_version, 2, 12)) {
 	push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
 	push @$cmd, '-mon', "chardev=qmp-event,mode=control";
     }
 
-    push @$cmd, '-pidfile' , pidfile_name($vmid);
+    push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
 
     push @$cmd, '-daemonize';
 
@@ -3718,7 +3704,7 @@ sub config_to_command {
     # load q35 config
     if ($q35) {
 	# we use different pcie-port hardware for qemu >= 4.0 for passthrough
-	if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
+	if (min_version($machine_version, 4, 0)) {
 	    push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
 	} else {
 	    push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
@@ -3736,7 +3722,7 @@ sub config_to_command {
     if (!$vga->{type}) {
 	if ($arch eq 'aarch64') {
 	    $vga->{type} = 'virtio';
-	} elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
+	} elsif (min_version($machine_version, 2, 9)) {
 	    $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
 	} else {
 	    $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
@@ -3801,7 +3787,7 @@ sub config_to_command {
 	if ($d->{mdev} && scalar(@$pcidevices) == 1) {
 	    my $pci_id = $pcidevices->[0]->{id};
 	    my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
-	    $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
+	    $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
 	} elsif ($d->{mdev}) {
 	    warn "ignoring mediated device '$id' with multifunction device\n";
 	}
@@ -3833,7 +3819,7 @@ sub config_to_command {
 
     # usb devices
     my $usb_dev_features = {};
-    $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
+    $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
 
     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
     push @$devices, @usbdevices if @usbdevices;
@@ -3902,7 +3888,7 @@ sub config_to_command {
     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
 	if ($allowed_vcpus < $maxcpus);
 
-    if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
+    if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
 
 	push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
         for (my $i = 2; $i <= $vcpus; $i++)  {
@@ -3932,8 +3918,8 @@ sub config_to_command {
     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
 
     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
-	push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
-	my $socket = vnc_socket($vmid);
+	push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
+	my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
 	push @$cmd,  '-vnc', "unix:$socket,password";
     } else {
 	push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
@@ -3975,7 +3961,7 @@ sub config_to_command {
 	push @$rtcFlags, 'base=localtime';
     }
 
-    push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
+    push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
 
     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
 
@@ -3983,12 +3969,19 @@ sub config_to_command {
 
     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
 
-    if (parse_guest_agent($conf)->{enabled}) {
-	my $qgasocket = qmp_socket($vmid, 1);
-	my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
+    my $guest_agent = parse_guest_agent($conf);
+
+    if ($guest_agent->{enabled}) {
+	my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
 	push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
-	push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
-	push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
+
+	if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
+	    my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
+	    push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
+	    push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
+	} elsif ($guest_agent->{type} eq 'isa') {
+	    push @$devices, '-device', "isa-serial,chardev=qga0";
+	}
     }
 
     my $spice_port;
@@ -3997,7 +3990,7 @@ sub config_to_command {
 	if ($qxlnum > 1) {
 	    if ($winversion){
 		for(my $i = 1; $i < $qxlnum; $i++){
-		    push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
+		    push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
 		}
 	    } else {
 		# assume other OS works like Linux
@@ -4013,7 +4006,6 @@ sub config_to_command {
 
 	my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
 
-	my $nodename = PVE::INotify::nodename();
 	my $pfamily = PVE::Tools::get_host_address_family($nodename);
 	my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
 	die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
@@ -4162,7 +4154,7 @@ sub config_to_command {
 
     if (!$q35) {
 	# add pci bridges
-        if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
+        if (min_version($machine_version, 2, 3)) {
 	   $bridges->{1} = 1;
 	   $bridges->{2} = 1;
 	}
@@ -4185,8 +4177,9 @@ sub config_to_command {
 
     if (my $vmstate = $conf->{vmstate}) {
 	my $statepath = PVE::Storage::path($storecfg, $vmstate);
-	PVE::Storage::activate_volumes($storecfg, [$vmstate]);
+	push @$vollist, $vmstate;
 	push @$cmd, '-loadstate', $statepath;
+	print "activating and using '$vmstate' as vmstate\n";
     }
 
     # add custom args
@@ -4198,35 +4191,18 @@ sub config_to_command {
     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
 }
 
-sub vnc_socket {
-    my ($vmid) = @_;
-    return "${var_run_tmpdir}/$vmid.vnc";
-}
-
 sub spice_port {
     my ($vmid) = @_;
 
-    my $res = vm_mon_cmd($vmid, 'query-spice');
+    my $res = mon_cmd($vmid, 'query-spice');
 
     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
 }
 
-sub qmp_socket {
-    my ($vmid, $qga, $name) = @_;
-    my $sockettype = $qga ? 'qga' : 'qmp';
-    my $ext = $name ? '-'.$name : '';
-    return "${var_run_tmpdir}/$vmid$ext.$sockettype";
-}
-
-sub pidfile_name {
-    my ($vmid) = @_;
-    return "${var_run_tmpdir}/$vmid.pid";
-}
-
 sub vm_devices_list {
     my ($vmid) = @_;
 
-    my $res = vm_mon_cmd($vmid, 'query-pci');
+    my $res = mon_cmd($vmid, 'query-pci');
     my $devices_to_check = [];
     my $devices = {};
     foreach my $pcibus (@$res) {
@@ -4245,14 +4221,14 @@ sub vm_devices_list {
 	$devices_to_check = $to_check;
     }
 
-    my $resblock = vm_mon_cmd($vmid, 'query-block');
+    my $resblock = mon_cmd($vmid, 'query-block');
     foreach my $block (@$resblock) {
 	if($block->{device} =~ m/^drive-(\S+)/){
 		$devices->{$1} = 1;
 	}
     }
 
-    my $resmice = vm_mon_cmd($vmid, 'query-mice');
+    my $resmice = mon_cmd($vmid, 'query-mice');
     foreach my $mice (@$resmice) {
 	if ($mice->{name} eq 'QEMU HID Tablet') {
 	    $devices->{tablet} = 1;
@@ -4263,7 +4239,7 @@ sub vm_devices_list {
     # for usb devices there is no query-usb
     # but we can iterate over the entries in
     # qom-list path=/machine/peripheral
-    my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
+    my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
     foreach my $per (@$resperipheral) {
 	if ($per->{name} =~ m/^usb\d+$/) {
 	    $devices->{$per->{name}} = 1;
@@ -4276,7 +4252,7 @@ sub vm_devices_list {
 sub vm_deviceplug {
     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
 
-    my $q35 = machine_type_is_q35($conf);
+    my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
 
     my $devices_list = vm_devices_list($vmid);
     return 1 if defined($devices_list->{$deviceid});
@@ -4352,7 +4328,7 @@ sub vm_deviceplug {
 
 	return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
 
-	my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
+	my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
 	my $use_old_bios_files = undef;
 	($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
 
@@ -4444,13 +4420,13 @@ sub qemu_deviceadd {
     $devicefull = "driver=".$devicefull;
     my %options =  split(/[=,]/, $devicefull);
 
-    vm_mon_cmd($vmid, "device_add" , %options);
+    mon_cmd($vmid, "device_add" , %options);
 }
 
 sub qemu_devicedel {
     my ($vmid, $deviceid) = @_;
 
-    my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
+    my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
 }
 
 sub qemu_iothread_add {
@@ -4479,7 +4455,7 @@ sub qemu_iothread_del {
 sub qemu_objectadd {
     my($vmid, $objectid, $qomtype) = @_;
 
-    vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
+    mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
 
     return 1;
 }
@@ -4487,7 +4463,7 @@ sub qemu_objectadd {
 sub qemu_objectdel {
     my($vmid, $objectid) = @_;
 
-    vm_mon_cmd($vmid, "object-del", id => $objectid);
+    mon_cmd($vmid, "object-del", id => $objectid);
 
     return 1;
 }
@@ -4497,7 +4473,7 @@ sub qemu_driveadd {
 
     my $drive = print_drive_full($storecfg, $vmid, $device);
     $drive =~ s/\\/\\\\/g;
-    my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
+    my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
 
     # If the command succeeds qemu prints: "OK"
     return 1 if $ret =~ m/OK/s;
@@ -4508,7 +4484,7 @@ sub qemu_driveadd {
 sub qemu_drivedel {
     my($vmid, $deviceid) = @_;
 
-    my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
+    my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
     $ret =~ s/^\s+//;
 
     return 1 if $ret eq "";
@@ -4618,7 +4594,7 @@ sub qemu_add_pci_bridge {
 sub qemu_set_link_status {
     my ($vmid, $device, $up) = @_;
 
-    vm_mon_cmd($vmid, "set_link", name => $device,
+    mon_cmd($vmid, "set_link", name => $device,
 	       up => $up ? JSON::true : JSON::false);
 }
 
@@ -4628,14 +4604,14 @@ sub qemu_netdevadd {
     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
     my %options =  split(/[=,]/, $netdev);
 
-    vm_mon_cmd($vmid, "netdev_add",  %options);
+    mon_cmd($vmid, "netdev_add",  %options);
     return 1;
 }
 
 sub qemu_netdevdel {
     my ($vmid, $deviceid) = @_;
 
-    vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
+    mon_cmd($vmid, "netdev_del", id => $deviceid);
 }
 
 sub qemu_usb_hotplug {
@@ -4666,7 +4642,7 @@ sub qemu_usb_hotplug {
 sub qemu_cpu_hotplug {
     my ($vmid, $conf, $vcpus) = @_;
 
-    my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
+    my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
 
     my $sockets = 1;
     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
@@ -4683,14 +4659,14 @@ sub qemu_cpu_hotplug {
 
     if ($vcpus < $currentvcpus) {
 
-	if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
+	if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
 
 	    for (my $i = $currentvcpus; $i > $vcpus; $i--) {
 		qemu_devicedel($vmid, "cpu$i");
 		my $retry = 0;
 		my $currentrunningvcpus = undef;
 		while (1) {
-		    $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
+		    $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
 		    last if scalar(@{$currentrunningvcpus}) == $i-1;
 		    raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
 		    $retry++;
@@ -4707,11 +4683,11 @@ sub qemu_cpu_hotplug {
 	return;
     }
 
-    my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
+    my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
     die "vcpus in running vm does not match its configuration\n"
 	if scalar(@{$currentrunningvcpus}) != $currentvcpus;
 
-    if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
+    if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
 
 	for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
 	    my $cpustr = print_cpu_device($conf, $i);
@@ -4720,7 +4696,7 @@ sub qemu_cpu_hotplug {
 	    my $retry = 0;
 	    my $currentrunningvcpus = undef;
 	    while (1) {
-		$currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
+		$currentrunningvcpus = mon_cmd($vmid, "query-cpus");
 		last if scalar(@{$currentrunningvcpus}) == $i;
 		raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
 		sleep 1;
@@ -4733,7 +4709,7 @@ sub qemu_cpu_hotplug {
     } else {
 
 	for (my $i = $currentvcpus; $i < $vcpus; $i++) {
-	    vm_mon_cmd($vmid, "cpu-add", id => int($i));
+	    mon_cmd($vmid, "cpu-add", id => int($i));
 	}
     }
 }
@@ -4747,7 +4723,7 @@ sub qemu_block_set_io_throttle {
 
     return if !check_running($vmid) ;
 
-    vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
+    mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
 	bps => int($bps),
 	bps_rd => int($bps_rd),
 	bps_wr => int($bps_wr),
@@ -4812,7 +4788,7 @@ sub qemu_block_resize {
 
     return if !$running;
 
-    vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
+    mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
 
 }
 
@@ -4822,7 +4798,7 @@ sub qemu_volume_snapshot {
     my $running = check_running($vmid);
 
     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
-	vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
+	mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
     } else {
 	PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
     }
@@ -4844,7 +4820,7 @@ sub qemu_volume_snapshot_delete {
     }
 
     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
-	vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
+	mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
     } else {
 	PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
     }
@@ -4863,7 +4839,7 @@ sub set_migration_caps {
 	"compress" => 0
     };
 
-    my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
+    my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
 
     for my $supported_capability (@$supported_capabilities) {
 	push @$cap_ref, {
@@ -4872,7 +4848,7 @@ sub set_migration_caps {
 	};
     }
 
-    vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
+    mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
 }
 
 my $fast_plug_option = {
@@ -4885,6 +4861,7 @@ my $fast_plug_option = {
     'protection' => 1,
     'vmstatestorage' => 1,
     'hookscript' => 1,
+    'tags' => 1,
 };
 
 # hotplug changes in [PENDING]
@@ -4895,7 +4872,8 @@ sub vmconfig_hotplug_pending {
     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
 
     my $defaults = load_defaults();
-    my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
+    my $arch = get_vm_arch($conf);
+    my $machine_type = get_vm_machine($conf, undef, $arch);
 
     # commit values which do not have any impact on running VM first
     # Note: those option cannot raise errors, we we do not care about
@@ -4917,14 +4895,14 @@ sub vmconfig_hotplug_pending {
 
     if ($changes) {
 	PVE::QemuConfig->write_config($vmid, $conf);
-	$conf = PVE::QemuConfig->load_config($vmid); # update/reload
     }
 
     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
 
-    my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
-    while (my ($opt, $force) = each %$pending_delete_hash) {
+    my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
+    foreach my $opt (sort keys %$pending_delete_hash) {
 	next if $selection && !$selection->{$opt};
+	my $force = $pending_delete_hash->{$opt}->{force};
 	eval {
 	    if ($opt eq 'hotplug') {
 		die "skip\n" if ($conf->{hotplug} =~ /memory/);
@@ -4952,7 +4930,7 @@ sub vmconfig_hotplug_pending {
 		die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
 		# here we reset the ballooning value to memory
 		my $balloon = $conf->{memory} || $defaults->{memory};
-		vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
+		mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
 	    } elsif ($fast_plug_option->{$opt}) {
 		# do nothing
 	    } elsif ($opt =~ m/^net(\d+)$/) {
@@ -4976,18 +4954,17 @@ sub vmconfig_hotplug_pending {
 	if (my $err = $@) {
 	    &$add_error($opt, $err) if $err ne "skip\n";
 	} else {
-	    # save new config if hotplug was successful
 	    delete $conf->{$opt};
-	    vmconfig_undelete_pending_option($conf, $opt);
-	    PVE::QemuConfig->write_config($vmid, $conf);
-	    $conf = PVE::QemuConfig->load_config($vmid); # update/reload
+	    PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
 	}
     }
 
-    my $apply_pending_cloudinit;
+    my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
     $apply_pending_cloudinit = sub {
+	return if $apply_pending_cloudinit_done; # once is enough
+	$apply_pending_cloudinit_done = 1; # once is enough
+
 	my ($key, $value) = @_;
-	$apply_pending_cloudinit = sub {}; # once is enough
 
 	my @cloudinit_opts = keys %$confdesc_cloudinit;
 	foreach my $opt (keys %{$conf->{pending}}) {
@@ -5036,13 +5013,14 @@ sub vmconfig_hotplug_pending {
 		# allow manual ballooning if shares is set to zero
 		if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
 		    my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
-		    vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
+		    mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
 		}
 	    } elsif ($opt =~ m/^net(\d+)$/) {
 		# some changes can be done without hotplug
 		vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
 				    $vmid, $opt, $value, $arch, $machine_type);
 	    } elsif (is_valid_drivename($opt)) {
+		die "skip\n" if $opt eq 'efidisk0';
 		# some changes can be done without hotplug
 		my $drive = parse_drive($opt, $value);
 		if (drive_is_cloudinit($drive)) {
@@ -5065,13 +5043,12 @@ sub vmconfig_hotplug_pending {
 	if (my $err = $@) {
 	    &$add_error($opt, $err) if $err ne "skip\n";
 	} else {
-	    # save new config if hotplug was successful
 	    $conf->{$opt} = $value;
 	    delete $conf->{pending}->{$opt};
-	    PVE::QemuConfig->write_config($vmid, $conf);
-	    $conf = PVE::QemuConfig->load_config($vmid); # update/reload
 	}
     }
+
+    PVE::QemuConfig->write_config($vmid, $conf);
 }
 
 sub try_deallocate_drive {
@@ -5113,25 +5090,44 @@ sub vmconfig_delete_or_detach_drive {
     }
 }
 
+
+
 sub vmconfig_apply_pending {
-    my ($vmid, $conf, $storecfg) = @_;
+    my ($vmid, $conf, $storecfg, $errors) = @_;
+
+    my $add_apply_error = sub {
+	my ($opt, $msg) = @_;
+	my $err_msg = "unable to apply pending change $opt : $msg";
+	$errors->{$opt} = $err_msg;
+	warn $err_msg;
+    };
 
     # cold plug
 
-    my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
-    while (my ($opt, $force) = each %$pending_delete_hash) {
-	die "internal error" if $opt =~ m/^unused/;
-	$conf = PVE::QemuConfig->load_config($vmid); # update/reload
-	if (!defined($conf->{$opt})) {
-	    vmconfig_undelete_pending_option($conf, $opt);
-	    PVE::QemuConfig->write_config($vmid, $conf);
-	} elsif (is_valid_drivename($opt)) {
-	    vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
-	    vmconfig_undelete_pending_option($conf, $opt);
-	    delete $conf->{$opt};
-	    PVE::QemuConfig->write_config($vmid, $conf);
+    my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
+    foreach my $opt (sort keys %$pending_delete_hash) {
+	my $force = $pending_delete_hash->{$opt}->{force};
+	eval {
+	    die "internal error" if $opt =~ m/^unused/;
+	    $conf = PVE::QemuConfig->load_config($vmid); # update/reload
+	    if (!defined($conf->{$opt})) {
+		PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
+		PVE::QemuConfig->write_config($vmid, $conf);
+	    } elsif (is_valid_drivename($opt)) {
+		vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
+		PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
+		delete $conf->{$opt};
+		PVE::QemuConfig->write_config($vmid, $conf);
+	    } else {
+		PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
+		delete $conf->{$opt};
+		PVE::QemuConfig->write_config($vmid, $conf);
+	    }
+	};
+	if (my $err = $@) {
+	    $add_apply_error->($opt, $err);
 	} else {
-	    vmconfig_undelete_pending_option($conf, $opt);
+	    PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
 	    delete $conf->{$opt};
 	    PVE::QemuConfig->write_config($vmid, $conf);
 	}
@@ -5142,17 +5138,24 @@ sub vmconfig_apply_pending {
     foreach my $opt (keys %{$conf->{pending}}) { # add/change
 	$conf = PVE::QemuConfig->load_config($vmid); # update/reload
 
-	if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
-	    # skip if nothing changed
-	} elsif (is_valid_drivename($opt)) {
-	    vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
-		if defined($conf->{$opt});
-	    $conf->{$opt} = $conf->{pending}->{$opt};
+	eval {
+	    if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
+		# skip if nothing changed
+	    } elsif (is_valid_drivename($opt)) {
+		vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
+		    if defined($conf->{$opt});
+		$conf->{$opt} = $conf->{pending}->{$opt};
+	    } else {
+		$conf->{$opt} = $conf->{pending}->{$opt};
+	    }
+	};
+	if (my $err = $@) {
+	    $add_apply_error->($opt, $err);
 	} else {
-	    $conf->{$opt} = $conf->{pending}->{$opt};
+	    $conf->{$opt} = delete $conf->{pending}->{$opt};
+	    PVE::QemuConfig->cleanup_pending($conf);
 	}
 
-	delete $conf->{pending}->{$opt};
 	PVE::QemuConfig->write_config($vmid, $conf);
     }
 }
@@ -5309,14 +5312,14 @@ sub vmconfig_update_disk {
 	    } else { # cdrom
 
 		if ($drive->{file} eq 'none') {
-		    vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
+		    mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
 		    if (drive_is_cloudinit($old_drive)) {
 			vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
 		    }
 		} else {
 		    my $path = get_iso_path($storecfg, $vmid, $drive->{file});
-		    vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
-		    vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
+		    mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
+		    mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
 		}
 
 		return 1;
@@ -5403,7 +5406,7 @@ sub vm_start {
 		my $newdrive = $drive;
 		$newdrive->{format} = $format;
 		$newdrive->{file} = $newvolid;
-		my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
+		my $drivestr = print_drive($newdrive);
 		$local_volumes->{$opt} = $drivestr;
 		#pass drive to conf for command line
 		$conf->{$opt} = $drivestr;
@@ -5420,13 +5423,41 @@ sub vm_start {
 
 	my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
 
-	my $migrate_port = 0;
+	my $migration_ip;
+	my $get_migration_ip = sub {
+	    my ($cidr, $nodename) = @_;
+
+	    return $migration_ip if defined($migration_ip);
+
+	    if (!defined($cidr)) {
+		my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
+		$cidr = $dc_conf->{migration}->{network};
+	    }
+
+	    if (defined($cidr)) {
+		my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
+
+		die "could not get IP: no address configured on local " .
+		    "node for network '$cidr'\n" if scalar(@$ips) == 0;
+
+		die "could not get IP: multiple addresses configured on local " .
+		    "node for network '$cidr'\n" if scalar(@$ips) > 1;
+
+		$migration_ip = @$ips[0];
+	    }
+
+	    $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
+		if !defined($migration_ip);
+
+	    return $migration_ip;
+	};
+
 	my $migrate_uri;
 	if ($statefile) {
 	    if ($statefile eq 'tcp') {
 		my $localip = "localhost";
 		my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
-		my $nodename = PVE::INotify::nodename();
+		my $nodename = nodename();
 
 		if (!defined($migration_type)) {
 		    if (defined($datacenterconf->{migration}->{type})) {
@@ -5437,18 +5468,12 @@ sub vm_start {
 		}
 
 		if ($migration_type eq 'insecure') {
-		    my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
-		    if ($migrate_network_addr) {
-			$localip = $migrate_network_addr;
-		    } else {
-			$localip = PVE::Cluster::remote_node_ip($nodename, 1);
-		    }
-
+		    $localip = $get_migration_ip->($migration_network, $nodename);
 		    $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
 		}
 
 		my $pfamily = PVE::Tools::get_host_address_family($nodename);
-		$migrate_port = PVE::Tools::next_migrate_port($pfamily);
+		my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
 		$migrate_uri = "tcp:${localip}:${migrate_port}";
 		push @$cmd, '-incoming', $migrate_uri;
 		push @$cmd, '-S';
@@ -5465,8 +5490,12 @@ sub vm_start {
 		push @$cmd, '-incoming', $migrate_uri;
 		push @$cmd, '-S';
 
-	    } else {
+	    } elsif (-e $statefile) {
 		push @$cmd, '-loadstate', $statefile;
+	    } else {
+		my $statepath = PVE::Storage::path($storecfg, $statefile);
+		push @$vollist, $statefile;
+		push @$cmd, '-loadstate', $statepath;
 	    }
 	} elsif ($paused) {
 	    push @$cmd, '-S';
@@ -5480,7 +5509,7 @@ sub vm_start {
 	  foreach my $pcidevice (@$pcidevices) {
 		my $pciid = $pcidevice->{id};
 
-		my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
+		my $info = PVE::SysFSTools::pci_device_info("$pciid");
 		die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
 		die "no pci device info for device '$pciid'\n" if !$info;
 
@@ -5510,7 +5539,18 @@ sub vm_start {
 	                                          : $defaults->{cpuunits};
 
 	my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
-	my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
+	my %run_params = (
+	    timeout => $statefile ? undef : $start_timeout,
+	    umask => 0077,
+	    noerr => 1,
+	);
+
+	# when migrating, prefix QEMU output so other side can pick up any
+	# errors that might occur and show the user
+	if ($migratedfrom) {
+	    $run_params{quiet} = 1;
+	    $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
+	}
 
 	my %properties = (
 	    Slice => 'qemu.slice',
@@ -5526,7 +5566,9 @@ sub vm_start {
 	my $run_qemu = sub {
 	    PVE::Tools::run_fork sub {
 		PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
-		run_command($cmd, %run_params);
+
+		my $exitcode = run_command($cmd, %run_params);
+		die "QEMU exited with code $exitcode\n" if $exitcode;
 	    };
 	};
 
@@ -5562,26 +5604,25 @@ sub vm_start {
 	print "migration listens on $migrate_uri\n" if $migrate_uri;
 
 	if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
-	    eval { vm_mon_cmd_nocheck($vmid, "cont"); };
+	    eval { mon_cmd($vmid, "cont"); };
 	    warn $@ if $@;
 	}
 
 	#start nbd server for storage migration
 	if ($targetstorage) {
-	    my $nodename = PVE::INotify::nodename();
-	    my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
-	    my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
+	    my $nodename = nodename();
+	    my $localip = $get_migration_ip->($migration_network, $nodename);
 	    my $pfamily = PVE::Tools::get_host_address_family($nodename);
-	    $migrate_port = PVE::Tools::next_migrate_port($pfamily);
+	    my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
 
-	    vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
+	    mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
 
 	    $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
 
 	    foreach my $opt (sort keys %$local_volumes) {
 		my $volid = $local_volumes->{$opt};
-		vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
-		my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
+		mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
+		my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
 		print "storage migration listens on $migrate_storage_uri volume:$volid\n";
 	    }
 	}
@@ -5595,13 +5636,13 @@ sub vm_start {
 	    if ($spice_port) {
 	        print "spice listens on port $spice_port\n";
 		if ($spice_ticket) {
-		    vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
-		    vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
+		    mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
+		    mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
 		}
 	    }
 
 	} else {
-	    vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
+	    mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
 		if !$statefile && $conf->{balloon};
 
 	    foreach my $opt (keys %$conf) {
@@ -5611,16 +5652,18 @@ sub vm_start {
 	    }
 	}
 
-	vm_mon_cmd_nocheck($vmid, 'qom-set',
+	mon_cmd($vmid, 'qom-set',
 		    path => "machine/peripheral/balloon0",
 		    property => "guest-stats-polling-interval",
 		    value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
 
-	if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
+	if ($is_suspended) {
 	    print "Resumed VM, removing state\n";
+	    if (my $vmstate = $conf->{vmstate}) {
+		PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
+		PVE::Storage::vdisk_free($storecfg, $vmstate);
+	    }
 	    delete $conf->@{qw(lock vmstate runningmachine)};
-	    PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
-	    PVE::Storage::vdisk_free($storecfg, $vmstate);
 	    PVE::QemuConfig->write_config($vmid, $conf);
 	}
 
@@ -5628,69 +5671,19 @@ sub vm_start {
     });
 }
 
-sub vm_mon_cmd {
-    my ($vmid, $execute, %params) = @_;
-
-    my $cmd = { execute => $execute, arguments => \%params };
-    vm_qmp_command($vmid, $cmd);
-}
-
-sub vm_mon_cmd_nocheck {
-    my ($vmid, $execute, %params) = @_;
-
-    my $cmd = { execute => $execute, arguments => \%params };
-    vm_qmp_command($vmid, $cmd, 1);
-}
-
-sub vm_qmp_command {
-    my ($vmid, $cmd, $nocheck) = @_;
-
-    my $res;
-
-    my $timeout;
-    if ($cmd->{arguments}) {
-	$timeout = delete $cmd->{arguments}->{timeout};
-    }
-
-    eval {
-	die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
-	my $sname = qmp_socket($vmid);
-	if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
-	    my $qmpclient = PVE::QMPClient->new();
-
-	    $res = $qmpclient->cmd($vmid, $cmd, $timeout);
-	} else {
-	    die "unable to open monitor socket\n";
-	}
-    };
-    if (my $err = $@) {
-	syslog("err", "VM $vmid qmp command failed - $err");
-	die $err;
-    }
-
-    return $res;
-}
-
-sub vm_human_monitor_command {
-    my ($vmid, $cmdline) = @_;
-
-    my $cmd = {
-	execute => 'human-monitor-command',
-	arguments => { 'command-line' => $cmdline},
-    };
-
-    return vm_qmp_command($vmid, $cmd);
-}
-
 sub vm_commandline {
     my ($storecfg, $vmid, $snapname) = @_;
 
     my $conf = PVE::QemuConfig->load_config($vmid);
+    my $forcemachine;
 
     if ($snapname) {
 	my $snapshot = $conf->{snapshots}->{$snapname};
 	die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
 
+	# check for a 'runningmachine' in snapshot
+	$forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
+
 	$snapshot->{digest} = $conf->{digest}; # keep file digest for API
 
 	$conf = $snapshot;
@@ -5698,7 +5691,7 @@ sub vm_commandline {
 
     my $defaults = load_defaults();
 
-    my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
+    my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
 
     return PVE::Tools::cmd2string($cmd);
 }
@@ -5712,7 +5705,7 @@ sub vm_reset {
 
 	PVE::QemuConfig->check_lock($conf) if !$skiplock;
 
-	vm_mon_cmd($vmid, "system_reset");
+	mon_cmd($vmid, "system_reset");
     });
 }
 
@@ -5795,15 +5788,12 @@ sub _do_vm_stop {
     eval {
 	if ($shutdown) {
 	    if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
-		vm_qmp_command($vmid, {
-			execute => "guest-shutdown",
-			arguments => { timeout => $timeout }
-		    }, $nocheck);
+		mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
 	    } else {
-		vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
+		mon_cmd($vmid, "system_powerdown");
 	    }
 	} else {
-	    vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
+	    mon_cmd($vmid, "quit");
 	}
     };
     my $err = $@;
@@ -5880,18 +5870,26 @@ sub vm_reboot {
     my ($vmid, $timeout) = @_;
 
     PVE::QemuConfig->lock_config($vmid, sub {
+	eval {
 
-	# only reboot if running, as qmeventd starts it again on a stop event
-	return if !check_running($vmid);
+	    # only reboot if running, as qmeventd starts it again on a stop event
+	    return if !check_running($vmid);
 
-	create_reboot_request($vmid);
+	    create_reboot_request($vmid);
 
-	my $storecfg = PVE::Storage::config();
-	_do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
+	    my $storecfg = PVE::Storage::config();
+	    _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
 
+	};
+	if (my $err = $@) {
+	    # avoid that the next normal shutdown will be confused for a reboot
+	    clear_reboot_request($vmid);
+	    die $err;
+	}
    });
 }
 
+# note: if using the statestorage parameter, the caller has to check privileges
 sub vm_suspend {
     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
 
@@ -5915,11 +5913,22 @@ sub vm_suspend {
 	    $conf->{lock} = 'suspending';
 	    my $date = strftime("%Y-%m-%d", localtime(time()));
 	    $storecfg = PVE::Storage::config();
+	    if (!$statestorage) {
+		$statestorage = find_vmstate_storage($conf, $storecfg);
+		# check permissions for the storage
+		my $rpcenv = PVE::RPCEnvironment::get();
+		if ($rpcenv->{type} ne 'cli') {
+		    my $authuser = $rpcenv->get_user();
+		    $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
+		}
+	    }
+
+
 	    $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
 	    $path = PVE::Storage::path($storecfg, $vmstate);
 	    PVE::QemuConfig->write_config($vmid, $conf);
 	} else {
-	    vm_mon_cmd($vmid, "stop");
+	    mon_cmd($vmid, "stop");
 	}
     });
 
@@ -5928,9 +5937,9 @@ sub vm_suspend {
 	PVE::Storage::activate_volumes($storecfg, [$vmstate]);
 
 	eval {
-	    vm_mon_cmd($vmid, "savevm-start", statefile => $path);
+	    mon_cmd($vmid, "savevm-start", statefile => $path);
 	    for(;;) {
-		my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
+		my $state = mon_cmd($vmid, "query-savevm");
 		if (!$state->{status}) {
 		    die "savevm not active\n";
 		} elsif ($state->{status} eq 'active') {
@@ -5953,7 +5962,7 @@ sub vm_suspend {
 	    if ($err) {
 		# cleanup, but leave suspending lock, to indicate something went wrong
 		eval {
-		    vm_mon_cmd($vmid, "savevm-end");
+		    mon_cmd($vmid, "savevm-end");
 		    PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
 		    PVE::Storage::vdisk_free($storecfg, $vmstate);
 		    delete $conf->@{qw(vmstate runningmachine)};
@@ -5966,7 +5975,7 @@ sub vm_suspend {
 	    die "lock changed unexpectedly\n"
 		if !PVE::QemuConfig->has_lock($conf, 'suspending');
 
-	    vm_qmp_command($vmid, { execute => "quit" });
+	    mon_cmd($vmid, "quit");
 	    $conf->{lock} = 'suspended';
 	    PVE::QemuConfig->write_config($vmid, $conf);
 	});
@@ -5977,8 +5986,7 @@ sub vm_resume {
     my ($vmid, $skiplock, $nocheck) = @_;
 
     PVE::QemuConfig->lock_config($vmid, sub {
-	my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
-	my $res = $vm_mon_cmd->($vmid, 'query-status');
+	my $res = mon_cmd($vmid, 'query-status');
 	my $resume_cmd = 'cont';
 
 	if ($res->{status} && $res->{status} eq 'suspended') {
@@ -5993,7 +6001,7 @@ sub vm_resume {
 		if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
 	}
 
-	$vm_mon_cmd->($vmid, $resume_cmd);
+	mon_cmd($vmid, $resume_cmd);
     });
 }
 
@@ -6005,26 +6013,11 @@ sub vm_sendkey {
 	my $conf = PVE::QemuConfig->load_config($vmid);
 
 	# there is no qmp command, so we use the human monitor command
-	my $res = vm_human_monitor_command($vmid, "sendkey $key");
+	my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
 	die $res if $res ne '';
     });
 }
 
-sub vm_destroy {
-    my ($storecfg, $vmid, $skiplock) = @_;
-
-    PVE::QemuConfig->lock_config($vmid, sub {
-
-	my $conf = PVE::QemuConfig->load_config($vmid);
-
-	if (!check_running($vmid)) {
-	    destroy_vm($storecfg, $vmid, undef, $skiplock);
-	} else {
-	    die "VM $vmid is running - destroy failed\n";
-	}
-    });
-}
-
 # vzdump restore implementaion
 
 sub tar_archive_read_firstfile {
@@ -6146,7 +6139,7 @@ sub restore_update_config_line {
 	} elsif ($map->{$virtdev}) {
 	    delete $di->{format}; # format can change on restore
 	    $di->{file} = $map->{$virtdev};
-	    $value = print_drive($vmid, $di);
+	    $value = print_drive($di);
 	    print $outfd "$virtdev: $value\n";
 	} else {
 	    print $outfd $line;
@@ -6229,6 +6222,27 @@ sub is_volume_in_use {
 }
 
 sub update_disksize {
+    my ($drive, $volid_hash) = @_;
+
+    my $volid = $drive->{file};
+    return undef if !defined($volid);
+
+    my $oldsize = $drive->{size};
+    my $newsize = $volid_hash->{$volid}->{size};
+
+    if (defined($newsize) && defined($oldsize) && $newsize != $oldsize) {
+	$drive->{size} = $newsize;
+
+	my $old_fmt = PVE::JSONSchema::format_size($oldsize);
+	my $new_fmt = PVE::JSONSchema::format_size($newsize);
+
+	return wantarray ? ($drive, $old_fmt, $new_fmt) : $drive;
+    }
+
+    return undef;
+}
+
+sub update_disk_config {
     my ($vmid, $conf, $volid_hash) = @_;
 
     my $changes;
@@ -6250,6 +6264,7 @@ sub update_disksize {
 	    my $volid = $drive->{file};
 	    next if !$volid;
 
+	    # mark volid as "in-use" for next step
 	    $referenced->{$volid} = 1;
 	    if ($volid_hash->{$volid} &&
 		(my $path = $volid_hash->{$volid}->{path})) {
@@ -6259,12 +6274,11 @@ sub update_disksize {
 	    next if drive_is_cdrom($drive);
 	    next if !$volid_hash->{$volid};
 
-	    $drive->{size} = $volid_hash->{$volid}->{size};
-	    my $new = print_drive($vmid, $drive);
-	    if ($new ne $conf->{$opt}) {
+	    my ($updated, $old_size, $new_size) = update_disksize($drive, $volid_hash);
+	    if (defined($updated)) {
 		$changes = 1;
-		$conf->{$opt} = $new;
-		print "$prefix update disk '$opt' information.\n";
+		$conf->{$opt} = print_drive($updated);
+		print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
 	    }
 	}
     }
@@ -6275,7 +6289,7 @@ sub update_disksize {
 	my $volid = $conf->{$opt};
 	my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
 	if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
-	    print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
+	    print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
 	    $changes = 1;
 	    delete $conf->{$opt};
 	}
@@ -6292,7 +6306,7 @@ sub update_disksize {
 	next if $referencedpath->{$path};
 	$changes = 1;
 	my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
-	print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
+	print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
 	$referencedpath->{$path} = 1; # avoid to add more than once (aliases)
     }
 
@@ -6326,7 +6340,7 @@ sub rescan {
 	    $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
 	}
 
-	my $changes = update_disksize($vmid, $conf, $vm_volids);
+	my $changes = update_disk_config($vmid, $conf, $vm_volids);
 
 	PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
     };
@@ -6426,7 +6440,7 @@ sub restore_vma_archive {
     my $conffile = PVE::QemuConfig->config_file($vmid);
     my $tmpfn = "$conffile.$$.tmp";
 
-    # Note: $oldconf is undef if VM does not exists
+    # Note: $oldconf is undef if VM does not exist
     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
 
@@ -6692,9 +6706,11 @@ sub restore_tar_archive {
 
     my $storecfg = PVE::Storage::config();
 
-    # destroy existing data - keep empty config
+    # avoid zombie disks when restoring over an existing VM -> cleanup first
+    # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
+    # skiplock=1 because qmrestore has set the 'create' lock itself already
     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
-    destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
+    destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
 
     my $tocmd = "/usr/lib/qemu-server/qmextract";
 
@@ -6772,14 +6788,9 @@ sub restore_tar_archive {
 	$srcfd->close();
 	$outfd->close();
     };
-    my $err = $@;
-
-    if ($err) {
-
+    if (my $err = $@) {
 	unlink $tmpfn;
-
 	tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
-
 	die $err;
     }
 
@@ -6814,6 +6825,9 @@ sub foreach_storage_used_by_vm {
     }
 }
 
+my $qemu_snap_storage = {
+    rbd => 1,
+};
 sub do_snapshots_with_qemu {
     my ($storecfg, $volid) = @_;
 
@@ -6834,7 +6848,7 @@ sub do_snapshots_with_qemu {
 sub qga_check_running {
     my ($vmid, $nowarn) = @_;
 
-    eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
+    eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
     if ($@) {
 	warn "Qemu Guest Agent is not running - $@" if !$nowarn;
 	return 0;
@@ -6858,7 +6872,7 @@ sub template_create {
 
 	my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
 	$drive->{file} = $voliddst;
-	$conf->{$ds} = print_drive($vmid, $drive);
+	$conf->{$ds} = print_drive($drive);
 	PVE::QemuConfig->write_config($vmid, $conf);
     });
 }
@@ -6887,66 +6901,78 @@ sub qemu_img_convert {
     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
 
-    if ($src_storeid && $dst_storeid) {
+    die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
 
-	PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
+    my $cachemode;
+    my $src_path;
+    my $src_is_iscsi = 0;
+    my $src_format;
 
+    if ($src_storeid) {
+	PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
 	my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
-	my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
+	$src_format = qemu_img_format($src_scfg, $src_volname);
+	$src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
+	$src_is_iscsi = ($src_path =~ m|^iscsi://|);
+	$cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
+    } elsif (-f $src_volid) {
+	$src_path = $src_volid;
+	if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
+	    $src_format = $1;
+	}
+    }
 
-	my $src_format = qemu_img_format($src_scfg, $src_volname);
-	my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
+    die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
 
-	my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
-	my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
+    my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
+    my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
+    my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
+    my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
 
-	my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
-	my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
-
-	my $cmd = [];
-	push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
-	push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
-	push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
-	push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
+    my $cmd = [];
+    push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
+    push @$cmd, '-l', "snapshot.name=$snapname"
+	if $snapname && $src_format && $src_format eq "qcow2";
+    push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
+    push @$cmd, '-T', $cachemode if defined($cachemode);
+
+    if ($src_is_iscsi) {
+	push @$cmd, '--image-opts';
+	$src_path = convert_iscsi_path($src_path);
+    } elsif ($src_format) {
+	push @$cmd, '-f', $src_format;
+    }
+
+    if ($dst_is_iscsi) {
+	push @$cmd, '--target-image-opts';
+	$dst_path = convert_iscsi_path($dst_path);
+    } else {
+	push @$cmd, '-O', $dst_format;
+    }
 
-	if ($src_is_iscsi) {
-	    push @$cmd, '--image-opts';
-	    $src_path = convert_iscsi_path($src_path);
-	} else {
-	    push @$cmd, '-f', $src_format;
-	}
+    push @$cmd, $src_path;
 
-	if ($dst_is_iscsi) {
-	    push @$cmd, '--target-image-opts';
-	    $dst_path = convert_iscsi_path($dst_path);
-	} else {
-	    push @$cmd, '-O', $dst_format;
-	}
+    if (!$dst_is_iscsi && $is_zero_initialized) {
+	push @$cmd, "zeroinit:$dst_path";
+    } else {
+	push @$cmd, $dst_path;
+    }
 
-	push @$cmd, $src_path;
+    my $parser = sub {
+	my $line = shift;
+	if($line =~ m/\((\S+)\/100\%\)/){
+	    my $percent = $1;
+	    my $transferred = int($size * $percent / 100);
+	    my $remaining = $size - $transferred;
 
-	if (!$dst_is_iscsi && $is_zero_initialized) {
-	    push @$cmd, "zeroinit:$dst_path";
-	} else {
-	    push @$cmd, $dst_path;
+	    print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
 	}
 
-	my $parser = sub {
-	    my $line = shift;
-	    if($line =~ m/\((\S+)\/100\%\)/){
-		my $percent = $1;
-		my $transferred = int($size * $percent / 100);
-		my $remaining = $size - $transferred;
-
-		print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
-	    }
-
-	};
+    };
 
-	eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
-	my $err = $@;
-	die "copy failed: $err" if $err;
-    }
+    eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
+    my $err = $@;
+    die "copy failed: $err" if $err;
 }
 
 sub qemu_img_format {
@@ -6995,7 +7021,7 @@ sub qemu_drive_mirror {
     }
 
     # if a job already runs for this device we get an error, catch it for cleanup
-    eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
+    eval { mon_cmd($vmid, "drive-mirror", %$opts); };
     if (my $err = $@) {
 	eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
 	warn "$@\n" if $@;
@@ -7014,7 +7040,7 @@ sub qemu_drive_mirror_monitor {
 	while (1) {
 	    die "storage migration timed out\n" if $err_complete > 300;
 
-	    my $stats = vm_mon_cmd($vmid, "query-block-jobs");
+	    my $stats = mon_cmd($vmid, "query-block-jobs");
 
 	    my $running_mirror_jobs = {};
 	    foreach my $stat (@$stats) {
@@ -7057,7 +7083,7 @@ sub qemu_drive_mirror_monitor {
 		    my $agent_running = $qga && qga_check_running($vmid);
 		    if ($agent_running) {
 			print "freeze filesystem\n";
-			eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
+			eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
 		    } else {
 			print "suspend vm\n";
 			eval { PVE::QemuServer::vm_suspend($vmid, 1); };
@@ -7068,7 +7094,7 @@ sub qemu_drive_mirror_monitor {
 
 		    if ($agent_running) {
 			print "unfreeze filesystem\n";
-			eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
+			eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
 		    } else {
 			print "resume vm\n";
 			eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
@@ -7081,7 +7107,7 @@ sub qemu_drive_mirror_monitor {
 			# try to switch the disk if source and destination are on the same guest
 			print "$job: Completing block job...\n";
 
-			eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
+			eval { mon_cmd($vmid, "block-job-complete", device => $job) };
 			if ($@ =~ m/cannot be completed/) {
 			    print "$job: Block job cannot be completed, try again.\n";
 			    $err_complete++;
@@ -7109,12 +7135,12 @@ sub qemu_blockjobs_cancel {
 
     foreach my $job (keys %$jobs) {
 	print "$job: Cancelling block job\n";
-	eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
+	eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
 	$jobs->{$job}->{cancel} = 1;
     }
 
     while (1) {
-	my $stats = vm_mon_cmd($vmid, "query-block-jobs");
+	my $stats = mon_cmd($vmid, "query-block-jobs");
 
 	my $running_jobs = {};
 	foreach my $stat (@$stats) {
@@ -7155,11 +7181,21 @@ sub clone_disk {
 
 	print "create full clone of drive $drivename ($drive->{file})\n";
 	my $name = undef;
+	if (drive_is_cloudinit($drive)) {
+	    $name = "vm-$newvmid-cloudinit";
+	    $name .= ".$dst_format" if $dst_format ne 'raw';
+	    $snapname = undef;
+	    $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
+	}
 	$newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
 	push @$newvollist, $newvolid;
 
 	PVE::Storage::activate_volumes($storecfg, [$newvolid]);
 
+	if (drive_is_cloudinit($drive)) {
+	    goto no_data_clone;
+	}
+
 	my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
 	if (!$running || $snapname) {
 	    # TODO: handle bwlimits
@@ -7167,7 +7203,7 @@ sub clone_disk {
 	} else {
 
 	    my $kvmver = get_running_qemu_version ($vmid);
-	    if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
+	    if (!min_version($kvmver, 2, 7)) {
 		die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
 		    if $drive->{iothread};
 	    }
@@ -7176,6 +7212,7 @@ sub clone_disk {
 	}
     }
 
+no_data_clone:
     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
 
     my $disk = $drive;
@@ -7186,64 +7223,12 @@ sub clone_disk {
     return $disk;
 }
 
-# this only works if VM is running
-sub get_current_qemu_machine {
-    my ($vmid) = @_;
-
-    my $cmd = { execute => 'query-machines', arguments => {} };
-    my $res = vm_qmp_command($vmid, $cmd);
-
-    my ($current, $default);
-    foreach my $e (@$res) {
-	$default = $e->{name} if $e->{'is-default'};
-	$current = $e->{name} if $e->{'is-current'};
-    }
-
-    # fallback to the default machine if current is not supported by qemu
-    return $current || $default || 'pc';
-}
-
 sub get_running_qemu_version {
     my ($vmid) = @_;
-    my $cmd = { execute => 'query-version', arguments => {} };
-    my $res = vm_qmp_command($vmid, $cmd);
+    my $res = mon_cmd($vmid, "query-version");
     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
 }
 
-sub qemu_machine_feature_enabled {
-    my ($machine, $kvmver, $version_major, $version_minor) = @_;
-
-    my $current_major;
-    my $current_minor;
-
-    if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
-
-	$current_major = $3;
-	$current_minor = $4;
-
-    } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
-
-	$current_major = $1;
-	$current_minor = $2;
-    }
-
-    return 1 if $current_major > $version_major ||
-                ($current_major == $version_major &&
-                 $current_minor >= $version_minor);
-}
-
-sub qemu_machine_pxe {
-    my ($vmid, $conf) = @_;
-
-    my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
-
-    if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
-	$machine .= '.pxe';
-    }
-
-    return $machine;
-}
-
 sub qemu_use_old_bios_files {
     my ($machine_type) = @_;
 
@@ -7255,12 +7240,12 @@ sub qemu_use_old_bios_files {
         $machine_type = $1;
         $use_old_bios_files = 1;
     } else {
-	my $kvmver = kvm_user_version();
+	my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
         # load new efi bios files on migration. So this hack is required to allow
         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
-	$use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
+	$use_old_bios_files = !min_version($version, 2, 4);
     }
 
     return ($use_old_bios_files, $machine_type);
@@ -7272,23 +7257,21 @@ sub create_efidisk($$$$$) {
     my (undef, $ovmf_vars) = get_ovmf_files($arch);
     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
 
-    my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
+    my $vars_size_b = -s $ovmf_vars;
+    my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
     PVE::Storage::activate_volumes($storecfg, [$volid]);
 
-    my $path = PVE::Storage::path($storecfg, $volid);
-    eval {
-	run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
-    };
-    die "Copying EFI vars image failed: $@" if $@;
+    qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
+    my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
 
-    return ($volid, $vars_size);
+    return ($volid, $size/1024);
 }
 
 sub vm_iothreads_list {
     my ($vmid) = @_;
 
-    my $res = vm_mon_cmd($vmid, 'query-iothreads');
+    my $res = mon_cmd($vmid, 'query-iothreads');
 
     my $iothreads = {};
     foreach my $iothread (@$res) {
@@ -7318,7 +7301,7 @@ sub scsihw_infos {
 }
 
 sub add_hyperv_enlightenments {
-    my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
+    my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
 
     return if $winversion < 6;
     return if $bios && $bios eq 'ovmf' && $winversion < 8;
@@ -7328,7 +7311,7 @@ sub add_hyperv_enlightenments {
 	push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
     }
 
-    if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
+    if (min_version($machine_version, 2, 3)) {
 	push @$cpuFlags , 'hv_spinlocks=0x1fff';
 	push @$cpuFlags , 'hv_vapic';
 	push @$cpuFlags , 'hv_time';
@@ -7336,7 +7319,7 @@ sub add_hyperv_enlightenments {
 	push @$cpuFlags , 'hv_spinlocks=0xffff';
     }
 
-    if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
+    if (min_version($machine_version, 2, 6)) {
 	push @$cpuFlags , 'hv_reset';
 	push @$cpuFlags , 'hv_vpindex';
 	push @$cpuFlags , 'hv_runtime';
@@ -7345,12 +7328,12 @@ sub add_hyperv_enlightenments {
     if ($winversion >= 7) {
 	push @$cpuFlags , 'hv_relaxed';
 
-	if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
+	if (min_version($machine_version, 2, 12)) {
 	    push @$cpuFlags , 'hv_synic';
 	    push @$cpuFlags , 'hv_stimer';
 	}
 
-	if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
+	if (min_version($machine_version, 3, 1)) {
 	    push @$cpuFlags , 'hv_ipi';
 	}
     }
@@ -7407,6 +7390,30 @@ sub resolve_first_disk {
     return $firstdisk;
 }
 
+# NOTE: if this logic changes, please update docs & possibly gui logic
+sub find_vmstate_storage {
+    my ($conf, $storecfg) = @_;
+
+    # first, return storage from conf if set
+    return $conf->{vmstatestorage} if $conf->{vmstatestorage};
+
+    my ($target, $shared, $local);
+
+    foreach_storage_used_by_vm($conf, sub {
+	my ($sid) = @_;
+	my $scfg = PVE::Storage::storage_config($storecfg, $sid);
+	my $dst = $scfg->{shared} ? \$shared : \$local;
+	$$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
+    });
+
+    # second, use shared storage where VM has at least one disk
+    # third, use local storage where VM has at least one disk
+    # fall back to local storage
+    $target = $shared // $local // 'local';
+
+    return $target;
+}
+
 sub generate_uuid {
     my ($uuid, $uuid_str);
     UUID::generate($uuid);
@@ -7421,7 +7428,7 @@ sub generate_smbios1_uuid {
 sub nbd_stop {
     my ($vmid) = @_;
 
-    vm_mon_cmd($vmid, 'nbd-server-stop');
+    mon_cmd($vmid, 'nbd-server-stop');
 }
 
 sub create_reboot_request {
@@ -7516,4 +7523,22 @@ sub complete_storage {
     return $res;
 }
 
+sub complete_migration_storage {
+    my ($cmd, $param, $current_value, $all_args) = @_;
+
+    my $targetnode = @$all_args[1];
+
+    my $cfg = PVE::Storage::config();
+    my $ids = $cfg->{ids};
+
+    my $res = [];
+    foreach my $sid (keys %$ids) {
+	next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
+	next if !$ids->{$sid}->{content}->{images};
+	push @$res, $sid;
+    }
+
+    return $res;
+}
+
 1;