X-Git-Url: https://git.proxmox.com/?p=qemu-server.git;a=blobdiff_plain;f=PVE%2FQemuServer.pm;h=76146ca22f43d5ae31e60d9aa77d02d53f8625cd;hp=8c519b5f6ec8801a0d4ff1a38bc88c1b1d7d8232;hb=bdd1feef5bf74176fc4591a0fee4a530abb1e666;hpb=b0f96836dbb989d2175bc872cefbab4ced7180e7

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 8c519b5..76146ca 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -3,45 +3,51 @@ package PVE::QemuServer;
 use strict;
 use warnings;
 
-use POSIX;
-use IO::Handle;
-use IO::Select;
-use IO::File;
-use IO::Dir;
-use IO::Socket::UNIX;
+use Cwd 'abs_path';
+use Digest::SHA;
+use Fcntl ':flock';
+use Fcntl;
 use File::Basename;
+use File::Copy qw(copy);
 use File::Path;
 use File::stat;
 use Getopt::Long;
-use Digest::SHA;
-use Fcntl ':flock';
-use Cwd 'abs_path';
+use IO::Dir;
+use IO::File;
+use IO::Handle;
+use IO::Select;
+use IO::Socket::UNIX;
 use IPC::Open3;
 use JSON;
-use Fcntl;
-use PVE::SafeSyslog;
-use Storable qw(dclone);
 use MIME::Base64;
-use PVE::Exception qw(raise raise_param_exc);
-use PVE::Storage;
-use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
-use PVE::JSONSchema qw(get_standard_option);
+use POSIX;
+use Storable qw(dclone);
+use Time::HiRes qw(gettimeofday);
+use URI::Escape;
+use UUID;
+
 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
+use PVE::DataCenterConfig;
+use PVE::Exception qw(raise raise_param_exc);
+use PVE::GuestHelpers;
 use PVE::INotify;
+use PVE::JSONSchema qw(get_standard_option);
 use PVE::ProcFSTools;
-use PVE::QemuConfig;
-use PVE::QMPClient;
 use PVE::RPCEnvironment;
-use PVE::GuestHelpers;
-use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
-use PVE::QemuServer::Memory;
-use PVE::QemuServer::USB qw(parse_usb_device);
-use PVE::QemuServer::Cloudinit;
+use PVE::Storage;
 use PVE::SysFSTools;
 use PVE::Systemd;
-use Time::HiRes qw(gettimeofday);
-use File::Copy qw(copy);
-use URI::Escape;
+use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
+
+use PVE::QMPClient;
+use PVE::QemuConfig;
+use PVE::QemuServer::Helpers qw(min_version);
+use PVE::QemuServer::Cloudinit;
+use PVE::QemuServer::Machine;
+use PVE::QemuServer::Memory;
+use PVE::QemuServer::Monitor qw(mon_cmd);
+use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
+use PVE::QemuServer::USB qw(parse_usb_device);
 
 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
 my $OVMF = {
@@ -55,8 +61,6 @@ my $OVMF = {
     ],
 };
 
-my $qemu_snap_storage = { rbd => 1 };
-
 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 
 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
@@ -89,7 +93,7 @@ PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 	description => "Specifies the Qemu machine type.",
 	type => 'string',
-	pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
+	pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 	maxLength => 40,
 	optional => 1,
 });
@@ -106,16 +110,6 @@ sub cgroups_write {
 
 my $nodename = PVE::INotify::nodename();
 
-mkdir "/etc/pve/nodes/$nodename";
-my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
-mkdir $confdir;
-
-my $var_run_tmpdir = "/var/run/qemu-server";
-mkdir $var_run_tmpdir;
-
-my $lock_dir = "/var/lock/qemu-server";
-mkdir $lock_dir;
-
 my $cpu_vendor_list = {
     # Intel CPUs
     486 => 'GenuineIntel',
@@ -146,6 +140,9 @@ my $cpu_vendor_list = {
     'Skylake-Client-IBRS' => 'GenuineIntel',
     'Skylake-Server' => 'GenuineIntel',
     'Skylake-Server-IBRS' => 'GenuineIntel',
+    'Cascadelake-Server' => 'GenuineIntel',
+    KnightsMill => 'GenuineIntel',
+
 
     # AMD CPUs
     athlon => 'AuthenticAMD',
@@ -178,7 +175,8 @@ my @supported_cpu_flags = (
     'pdpe1gb',
     'md-clear',
     'hv-tlbflush',
-    'hv-evmcs'
+    'hv-evmcs',
+    'aes'
 );
 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 
@@ -245,6 +243,13 @@ my $agent_fmt = {
 	optional => 1,
 	default => 0
     },
+    type => {
+	description => "Select the agent type",
+	type => 'string',
+	default => 'virtio',
+	optional => 1,
+	enum => [qw(virtio isa)],
+    },
 };
 
 my $vga_fmt = {
@@ -295,6 +300,22 @@ my $audio_fmt = {
     },
 };
 
+my $spice_enhancements_fmt = {
+    foldersharing => {
+	type => 'boolean',
+	optional => 1,
+	default => '0',
+	description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
+    },
+    videostreaming =>  {
+	type => 'string',
+	enum => ['off', 'all', 'filter'],
+	default => 'off',
+	optional => 1,
+	description => "Enable video streaming. Uses compression for detected video streams."
+    },
+};
+
 my $confdesc = {
     onboot => {
 	optional => 1,
@@ -410,7 +431,7 @@ win7;; Microsoft Windows 7
 win8;; Microsoft Windows 8/2012/2012r2
 win10;; Microsoft Windows 10/2016
 l24;; Linux 2.4 Kernel
-l26;; Linux 2.6/3.X Kernel
+l26;; Linux 2.6 - 5.X Kernel
 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 EODESC
     },
@@ -672,6 +693,17 @@ EODESCR
 	description => "Configure a audio device, useful in combination with QXL/Spice.",
 	optional => 1
     },
+    spice_enhancements => {
+	type => 'string',
+	format => $spice_enhancements_fmt,
+	description => "Configure additional enhancements for SPICE.",
+	optional => 1
+    },
+    tags => {
+	type => 'string', format => 'pve-tag-list',
+	description => 'Tags of the VM. This is only meta information.',
+	optional => 1,
+    },
 };
 
 my $cicustom_fmt = {
@@ -768,7 +800,7 @@ my $MAX_SATA_DISKS = 6;
 my $MAX_USB_DEVICES = 5;
 my $MAX_NETS = 32;
 my $MAX_UNUSED_DISKS = 256;
-my $MAX_HOSTPCI_DEVICES = 4;
+my $MAX_HOSTPCI_DEVICES = 16;
 my $MAX_SERIAL_PORTS = 4;
 my $MAX_PARALLEL_PORTS = 3;
 my $MAX_NUMA = 8;
@@ -1296,7 +1328,7 @@ EODESCR
     usb3 => {
 	optional => 1,
 	type => 'boolean',
-	description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
+	description => "Specifies whether if given host option is a USB3 device or port.",
         default => 0,
     },
 };
@@ -1308,7 +1340,7 @@ my $usbdesc = {
 };
 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 
-my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
+my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
 my $hostpci_fmt = {
     host => {
 	default_key => 1,
@@ -1471,25 +1503,33 @@ sub kvm_version {
     return $kvm_api_version;
 }
 
-my $kvm_user_version;
+my $kvm_user_version = {};
+my $kvm_mtime = {};
 
 sub kvm_user_version {
+    my ($binary) = @_;
+
+    $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
+    my $st = stat($binary);
 
-    return $kvm_user_version if $kvm_user_version;
+    my $cachedmtime = $kvm_mtime->{$binary} // -1;
+    return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
+	$cachedmtime == $st->mtime;
 
-    $kvm_user_version = 'unknown';
+    $kvm_user_version->{$binary} = 'unknown';
+    $kvm_mtime->{$binary} = $st->mtime;
 
     my $code = sub {
 	my $line = shift;
 	if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
-	    $kvm_user_version = $2;
+	    $kvm_user_version->{$binary} = $2;
 	}
     };
 
-    eval { run_command("kvm -version", outfunc => $code); };
+    eval { run_command([$binary, '--version'], outfunc => $code); };
     warn $@ if $@;
 
-    return $kvm_user_version;
+    return $kvm_user_version->{$binary};
 
 }
 
@@ -1517,29 +1557,7 @@ sub option_exists {
     return defined($confdesc->{$key});
 }
 
-sub nic_models {
-    return $nic_model_list;
-}
-
-sub os_list_description {
-
-    return {
-	other => 'Other',
-	wxp => 'Windows XP',
-	w2k => 'Windows 2000',
-	w2k3 =>, 'Windows 2003',
-	w2k8 => 'Windows 2008',
-	wvista => 'Windows Vista',
-	win7 => 'Windows 7',
-	win8 => 'Windows 8/2012',
-	win10 => 'Windows 10/2016',
-	l24 => 'Linux 2.4',
-	l26 => 'Linux 2.6',
-    };
-}
-
 my $cdrom_path;
-
 sub get_cdrom_path {
 
     return  $cdrom_path if $cdrom_path;
@@ -1800,20 +1818,14 @@ sub path_is_scsi {
     return $res;
 }
 
-sub machine_type_is_q35 {
-    my ($conf) = @_;
-
-    return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
-}
-
 sub print_tabletdevice_full {
     my ($conf, $arch) = @_;
 
-    my $q35 = machine_type_is_q35($conf);
+    my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
 
     # we use uhci for old VMs because tablet driver was buggy in older qemu
     my $usbbus;
-    if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
+    if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
 	$usbbus = 'ehci';
     } else {
 	$usbbus = 'uhci';
@@ -1862,7 +1874,10 @@ sub print_drivedevice_full {
 		 $path = PVE::Storage::path($storecfg, $drive->{file});
 	    }
 
-	    if($path =~ m/^iscsi\:\/\//){
+	    # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
+	    my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
+	    if ($path =~ m/^iscsi\:\/\// &&
+	       !min_version($version, 4, 1)) {
 		$devicetype = 'generic';
 	    }
 	}
@@ -2130,16 +2145,26 @@ my $vga_map = {
 };
 
 sub print_vga_device {
-    my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
+    my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
 
     my $type = $vga_map->{$vga->{type}};
     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
 	$type = 'virtio-gpu';
     }
     my $vgamem_mb = $vga->{memory};
+
+    my $max_outputs = '';
     if ($qxlnum) {
 	$type = $id ? 'qxl' : 'qxl-vga';
+
+	if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
+	    # set max outputs so linux can have up to 4 qxl displays with one device
+	    if (min_version($machine_version, 4, 1)) {
+		$max_outputs = ",max_outputs=4";
+	    }
+	}
     }
+
     die "no devicetype for $vga->{type}\n" if !$type;
 
     my $memory = "";
@@ -2160,7 +2185,7 @@ sub print_vga_device {
 	$memory = ",ram_size=67108864,vram_size=33554432";
     }
 
-    my $q35 = machine_type_is_q35($conf);
+    my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
     my $vgaid = "vga" . ($id // '');
     my $pciaddr;
 
@@ -2171,7 +2196,7 @@ sub print_vga_device {
 	$pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
     }
 
-    return "$type,id=${vgaid}${memory}${pciaddr}";
+    return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
 }
 
 sub drive_is_cloudinit {
@@ -2314,40 +2339,6 @@ sub vm_is_volid_owner {
     return undef;
 }
 
-sub split_flagged_list {
-    my $text = shift || '';
-    $text =~ s/[,;]/ /g;
-    $text =~ s/^\s+//;
-    return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
-}
-
-sub join_flagged_list {
-    my ($how, $lst) = @_;
-    join $how, map { $lst->{$_} . $_ } keys %$lst;
-}
-
-sub vmconfig_delete_pending_option {
-    my ($conf, $key, $force) = @_;
-
-    delete $conf->{pending}->{$key};
-    my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
-    $pending_delete_hash->{$key} = $force ? '!' : '';
-    $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
-}
-
-sub vmconfig_undelete_pending_option {
-    my ($conf, $key) = @_;
-
-    my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
-    delete $pending_delete_hash->{$key};
-
-    if (%$pending_delete_hash) {
-	$conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
-    } else {
-	delete $conf->{pending}->{delete};
-    }
-}
-
 sub vmconfig_register_unused_drive {
     my ($storecfg, $vmid, $conf, $drive) = @_;
 
@@ -2362,37 +2353,6 @@ sub vmconfig_register_unused_drive {
     }
 }
 
-sub vmconfig_cleanup_pending {
-    my ($conf) = @_;
-
-    # remove pending changes when nothing changed
-    my $changes;
-    foreach my $opt (keys %{$conf->{pending}}) {
-	if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
-	    $changes = 1;
-	    delete $conf->{pending}->{$opt};
-	}
-    }
-
-    my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
-    my $pending_delete_hash = {};
-    while (my ($opt, $force) = each %$current_delete_hash) {
-	if (defined($conf->{$opt})) {
-	    $pending_delete_hash->{$opt} = $force;
-	} else {
-	    $changes = 1;
-	}
-    }
-
-    if (%$pending_delete_hash) {
-	$conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
-    } else {
-	delete $conf->{pending}->{delete};
-    }
-
-    return $changes;
-}
-
 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
 my $smbios1_fmt = {
     uuid => {
@@ -2575,17 +2535,8 @@ sub check_type {
     }
 }
 
-sub touch_config {
-    my ($vmid) = @_;
-
-    my $conf = PVE::QemuConfig->config_file($vmid);
-    utime undef, undef, $conf;
-}
-
 sub destroy_vm {
-    my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
-
-    my $conffile = PVE::QemuConfig->config_file($vmid);
+    my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
 
     my $conf = PVE::QemuConfig->load_config($vmid);
 
@@ -2595,11 +2546,9 @@ sub destroy_vm {
 	# check if any base image is still used by a linked clone
 	foreach_drive($conf, sub {
 		my ($ds, $drive) = @_;
-
 		return if drive_is_cdrom($drive);
 
 		my $volid = $drive->{file};
-
 		return if !$volid || $volid =~ m|^/|;
 
 		die "base volume '$volid' is still in use by linked cloned\n"
@@ -2611,43 +2560,31 @@ sub destroy_vm {
     # only remove disks owned by this VM
     foreach_drive($conf, sub {
 	my ($ds, $drive) = @_;
-
  	return if drive_is_cdrom($drive, 1);
 
 	my $volid = $drive->{file};
-
 	return if !$volid || $volid =~ m|^/|;
 
 	my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
 	return if !$path || !$owner || ($owner != $vmid);
 
-	eval {
-	    PVE::Storage::vdisk_free($storecfg, $volid);
-	};
+	eval { PVE::Storage::vdisk_free($storecfg, $volid) };
 	warn "Could not remove disk '$volid', check manually: $@" if $@;
-
     });
 
-    if ($keep_empty_config) {
-	PVE::Tools::file_set_contents($conffile, "memory: 128\n");
-    } else {
-	unlink $conffile;
-    }
-
     # also remove unused disk
-    eval {
-	my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
-
-	eval {
-	    PVE::Storage::foreach_volid($dl, sub {
-		my ($volid, $sid, $volname, $d) = @_;
-		PVE::Storage::vdisk_free($storecfg, $volid);
-	    });
-	};
+    my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
+    PVE::Storage::foreach_volid($vmdisks, sub {
+	my ($volid, $sid, $volname, $d) = @_;
+	eval { PVE::Storage::vdisk_free($storecfg, $volid) };
 	warn $@ if $@;
+    });
 
-    };
-    warn $@ if $@;
+    if (defined $replacement_conf) {
+	PVE::QemuConfig->write_config($vmid, $replacement_conf);
+    } else {
+	PVE::QemuConfig->destroy_config($vmid);
+    }
 }
 
 sub parse_vm_config {
@@ -2803,7 +2740,7 @@ sub write_vm_config {
     &$cleanup_config($conf->{pending}, 1);
 
     foreach my $snapname (keys %{$conf->{snapshots}}) {
-	die "internal error" if $snapname eq 'pending';
+	die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
 	&$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
     }
 
@@ -2894,7 +2831,7 @@ sub check_local_resources {
     push @loc_res, "ivshmem" if $conf->{ivshmem};
 
     foreach my $k (keys %$conf) {
-	next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
+	next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
 	# sockets are safe: they will recreated be on the target side post-migrate
 	next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
 	push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
@@ -2997,70 +2934,19 @@ sub check_local_storage_availability {
     return $nodehash
 }
 
-sub check_cmdline {
-    my ($pidfile, $pid) = @_;
-
-    my $fh = IO::File->new("/proc/$pid/cmdline", "r");
-    if (defined($fh)) {
-	my $line = <$fh>;
-	$fh->close;
-	return undef if !$line;
-	my @param = split(/\0/, $line);
-
-	my $cmd = $param[0];
-	return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
-
-	for (my $i = 0; $i < scalar (@param); $i++) {
-	    my $p = $param[$i];
-	    next if !$p;
-	    if (($p eq '-pidfile') || ($p eq '--pidfile')) {
-		my $p = $param[$i+1];
-		return 1 if $p && ($p eq $pidfile);
-		return undef;
-	    }
-	}
-    }
-    return undef;
-}
-
+# Compat only, use assert_config_exists_on_node and vm_running_locally where possible
 sub check_running {
     my ($vmid, $nocheck, $node) = @_;
 
-    my $filename = PVE::QemuConfig->config_file($vmid, $node);
-
-    die "unable to find configuration file for VM $vmid - no such machine\n"
-	if !$nocheck && ! -f $filename;
-
-    my $pidfile = pidfile_name($vmid);
-
-    if (my $fd = IO::File->new("<$pidfile")) {
-	my $st = stat($fd);
-	my $line = <$fd>;
-	close($fd);
-
-	my $mtime = $st->mtime;
-	if ($mtime > time()) {
-	    warn "file '$filename' modified in future\n";
-	}
-
-	if ($line =~ m/^(\d+)$/) {
-	    my $pid = $1;
-	    if (check_cmdline($pidfile, $pid)) {
-		if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
-		    return $pid;
-		}
-	    }
-	}
-    }
-
-    return undef;
+    PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
+    return PVE::QemuServer::Helpers::vm_running_locally($vmid);
 }
 
 sub vzlist {
 
     my $vzlist = config_list();
 
-    my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
+    my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
 
     while (defined(my $de = $fd->read)) {
 	next if $de !~ m/^(\d+)\.pid$/;
@@ -3143,7 +3029,12 @@ our $vmstatus_return_properties = {
 	description => "The current config lock, if any.",
 	type => 'string',
 	optional => 1,
-    }
+    },
+    tags => {
+	description  => "The current configured tags, if any",
+	type => 'string',
+	optional => 1,
+    },
 };
 
 my $last_proc_pid_stat;
@@ -3168,8 +3059,7 @@ sub vmstatus {
     foreach my $vmid (keys %$list) {
 	next if $opt_vmid && ($vmid ne $opt_vmid);
 
-	my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
-	my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
+	my $conf = PVE::QemuConfig->load_config($vmid);
 
 	my $d = { vmid => $vmid };
 	$d->{pid} = $list->{$vmid}->{pid};
@@ -3215,6 +3105,7 @@ sub vmstatus {
 
 	$d->{serial} = 1 if conf_has_serial($conf);
 	$d->{lock} = $conf->{lock} if $conf->{lock};
+	$d->{tags} = $conf->{tags} if defined($conf->{tags});
 
 	$res->{$vmid} = $d;
     }
@@ -3454,28 +3345,33 @@ sub vga_conf_has_spice {
     return $1 || 1;
 }
 
-my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
-sub get_host_arch() {
-    $host_arch = (POSIX::uname())[4] if !$host_arch;
-    return $host_arch;
-}
-
 sub is_native($) {
     my ($arch) = @_;
     return get_host_arch() eq $arch;
 }
 
+sub get_vm_arch {
+    my ($conf) = @_;
+    return $conf->{arch} // get_host_arch();
+}
+
 my $default_machines = {
     x86_64 => 'pc',
     aarch64 => 'virt',
 };
 
-sub get_basic_machine_info {
-    my ($conf, $forcemachine) = @_;
+sub get_vm_machine {
+    my ($conf, $forcemachine, $arch, $add_pve_version) = @_;
 
-    my $arch = $conf->{arch} // get_host_arch();
-    my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
-    return ($arch, $machine);
+    my $machine = $forcemachine || $conf->{machine};
+
+    if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
+	$arch //= 'x86_64';
+	$machine ||= $default_machines->{$arch};
+	$machine .= "+pve$PVE::QemuServer::Machine::PVE_MACHINE_VERSION" if $add_pve_version;
+    }
+
+    return $machine;
 }
 
 sub get_ovmf_files($) {
@@ -3501,7 +3397,7 @@ sub get_command_for_arch($) {
 }
 
 sub get_cpu_options {
-    my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
+    my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
 
     my $cpuFlags = [];
     my $ostype = $conf->{ostype};
@@ -3525,20 +3421,19 @@ sub get_cpu_options {
 
     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
 
-    push @$cpuFlags , '-x2apic'
-	if $conf->{ostype} && $conf->{ostype} eq 'solaris';
+    push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
 
     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
 
     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
 
-    if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
+    if (min_version($machine_version, 2, 3) && $arch eq 'x86_64') {
 
 	push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
 	push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
     }
 
-    add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
+    add_hyperv_enlightenments($cpuFlags, $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
 
     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
 
@@ -3566,13 +3461,19 @@ sub config_to_command {
     my $devices = [];
     my $pciaddr = '';
     my $bridges = {};
-    my $kvmver = kvm_user_version();
     my $vernum = 0; # unknown
     my $ostype = $conf->{ostype};
     my $winversion = windows_version($ostype);
     my $kvm = $conf->{kvm};
 
-    my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
+    my $arch = get_vm_arch($conf);
+    my $kvm_binary = get_command_for_arch($arch);
+    my $kvmver = kvm_user_version($kvm_binary);
+
+    my $add_pve_version = min_version($kvmver, 4, 1);
+
+    my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
+    my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
     $kvm //= 1 if is_native($arch);
 
     if ($kvm) {
@@ -3588,9 +3489,7 @@ sub config_to_command {
 
     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
 
-    my $have_ovz = -f '/proc/vz/vestat';
-
-    my $q35 = machine_type_is_q35($conf);
+    my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
     my $use_old_bios_files = undef;
     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
@@ -3598,7 +3497,7 @@ sub config_to_command {
     my $cpuunits = defined($conf->{cpuunits}) ?
             $conf->{cpuunits} : $defaults->{cpuunits};
 
-    push @$cmd, get_command_for_arch($arch);
+    push @$cmd, $kvm_binary;
 
     push @$cmd, '-id', $vmid;
 
@@ -3608,16 +3507,16 @@ sub config_to_command {
 
     my $use_virtio = 0;
 
-    my $qmpsocket = qmp_socket($vmid);
+    my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
     push @$cmd, '-mon', "chardev=qmp,mode=control";
 
-    if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
+    if (min_version($machine_version, 2, 12)) {
 	push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
 	push @$cmd, '-mon', "chardev=qmp-event,mode=control";
     }
 
-    push @$cmd, '-pidfile' , pidfile_name($vmid);
+    push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
 
     push @$cmd, '-daemonize';
 
@@ -3683,7 +3582,7 @@ sub config_to_command {
     # load q35 config
     if ($q35) {
 	# we use different pcie-port hardware for qemu >= 4.0 for passthrough
-	if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
+	if (min_version($machine_version, 4, 0)) {
 	    push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
 	} else {
 	    push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
@@ -3701,7 +3600,7 @@ sub config_to_command {
     if (!$vga->{type}) {
 	if ($arch eq 'aarch64') {
 	    $vga->{type} = 'virtio';
-	} elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
+	} elsif (min_version($machine_version, 2, 9)) {
 	    $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
 	} else {
 	    $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
@@ -3729,67 +3628,66 @@ sub config_to_command {
 
     # host pci devices
     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
-	my $d = parse_hostpci($conf->{"hostpci$i"});
+	my $id = "hostpci$i";
+	my $d = parse_hostpci($conf->{$id});
 	next if !$d;
 
-	my $pcie = $d->{pcie};
-	if ($pcie) {
+	if (my $pcie = $d->{pcie}) {
 	    die "q35 machine model is not enabled" if !$q35;
 	    # win7 wants to have the pcie devices directly on the pcie bus
 	    # instead of in the root port
 	    if ($winversion == 7) {
-		$pciaddr = print_pcie_addr("hostpci${i}bus0");
+		$pciaddr = print_pcie_addr("${id}bus0");
 	    } else {
-		$pciaddr = print_pcie_addr("hostpci$i");
+		# add more root ports if needed, 4 are present by default
+		# by pve-q35 cfgs, rest added here on demand.
+		if ($i > 3) {
+		    push @$devices, '-device', print_pcie_root_port($i);
+		}
+		$pciaddr = print_pcie_addr($id);
 	    }
 	} else {
-	    $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
+	    $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
 	}
 
-	my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
-	my $romfile = $d->{romfile};
-
 	my $xvga = '';
 	if ($d->{'x-vga'}) {
-	    $xvga = ',x-vga=on';
+	    $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
 	    $kvm_off = 1;
 	    $vga->{type} = 'none' if !defined($conf->{vga});
 	    $gpu_passthrough = 1;
-
-	    if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
-		$xvga = "";
-	    }
 	}
+
 	my $pcidevices = $d->{pciid};
 	my $multifunction = 1 if @$pcidevices > 1;
+
 	my $sysfspath;
 	if ($d->{mdev} && scalar(@$pcidevices) == 1) {
-	    my $id = $pcidevices->[0]->{id};
+	    my $pci_id = $pcidevices->[0]->{id};
 	    my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
-	    $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
+	    $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
 	} elsif ($d->{mdev}) {
-	    warn "ignoring mediated device with multifunction device\n";
+	    warn "ignoring mediated device '$id' with multifunction device\n";
 	}
 
 	my $j=0;
-        foreach my $pcidevice (@$pcidevices) {
-
-	    my $id = "hostpci$i";
-	    $id .= ".$j" if $multifunction;
-	    my $addr = $pciaddr;
-	    $addr .= ".$j" if $multifunction;
+	foreach my $pcidevice (@$pcidevices) {
 	    my $devicestr = "vfio-pci";
+
 	    if ($sysfspath) {
 		$devicestr .= ",sysfsdev=$sysfspath";
 	    } else {
 		$devicestr .= ",host=$pcidevice->{id}";
 	    }
-	    $devicestr .= ",id=$id$addr";
 
-	    if($j == 0){
-		$devicestr .= "$rombar$xvga";
+	    my $mf_addr = $multifunction ? ".$j" : '';
+	    $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
+
+	    if ($j == 0) {
+		$devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
+		$devicestr .= "$xvga";
 		$devicestr .= ",multifunction=on" if $multifunction;
-		$devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
+		$devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
 	    }
 
 	    push @$devices, '-device', $devicestr;
@@ -3798,7 +3696,10 @@ sub config_to_command {
     }
 
     # usb devices
-    my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
+    my $usb_dev_features = {};
+    $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
+
+    my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
     push @$devices, @usbdevices if @usbdevices;
     # serial devices
     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
@@ -3865,7 +3766,7 @@ sub config_to_command {
     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
 	if ($allowed_vcpus < $maxcpus);
 
-    if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
+    if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
 
 	push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
         for (my $i = 2; $i <= $vcpus; $i++)  {
@@ -3895,8 +3796,8 @@ sub config_to_command {
     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
 
     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
-	push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
-	my $socket = vnc_socket($vmid);
+	push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
+	my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
 	push @$cmd,  '-vnc', "unix:$socket,password";
     } else {
 	push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
@@ -3938,7 +3839,7 @@ sub config_to_command {
 	push @$rtcFlags, 'base=localtime';
     }
 
-    push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
+    push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
 
     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
 
@@ -3946,12 +3847,19 @@ sub config_to_command {
 
     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
 
-    if (parse_guest_agent($conf)->{enabled}) {
-	my $qgasocket = qmp_socket($vmid, 1);
-	my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
+    my $guest_agent = parse_guest_agent($conf);
+
+    if ($guest_agent->{enabled}) {
+	my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
 	push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
-	push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
-	push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
+
+	if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
+	    my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
+	    push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
+	    push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
+	} elsif ($guest_agent->{type} eq 'isa') {
+	    push @$devices, '-device', "isa-serial,chardev=qga0";
+	}
     }
 
     my $spice_port;
@@ -3960,7 +3868,7 @@ sub config_to_command {
 	if ($qxlnum > 1) {
 	    if ($winversion){
 		for(my $i = 1; $i < $qxlnum; $i++){
-		    push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
+		    push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
 		}
 	    } else {
 		# assume other OS works like Linux
@@ -3980,14 +3888,23 @@ sub config_to_command {
 	my $pfamily = PVE::Tools::get_host_address_family($nodename);
 	my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
 	die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
-	my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
-	$spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
-
-	push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
 
 	push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
 	push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
 	push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
+
+	my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
+	$spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
+
+	my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
+	if ($spice_enhancement->{foldersharing}) {
+	    push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
+	    push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
+	}
+
+	my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
+	$spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
+	push @$devices, '-spice', "$spice_opts";
     }
 
     # enable balloon by default, unless explicitly disabled
@@ -4116,14 +4033,14 @@ sub config_to_command {
 
     if (!$q35) {
 	# add pci bridges
-        if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
+        if (min_version($machine_version, 2, 3)) {
 	   $bridges->{1} = 1;
 	   $bridges->{2} = 1;
 	}
 
 	$bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
 
-	while (my ($k, $v) = each %$bridges) {
+	for my $k (sort {$b cmp $a} keys %$bridges) {
 	    $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
 	    unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
 	}
@@ -4139,8 +4056,9 @@ sub config_to_command {
 
     if (my $vmstate = $conf->{vmstate}) {
 	my $statepath = PVE::Storage::path($storecfg, $vmstate);
-	PVE::Storage::activate_volumes($storecfg, [$vmstate]);
+	push @$vollist, $vmstate;
 	push @$cmd, '-loadstate', $statepath;
+	print "activating and using '$vmstate' as vmstate\n";
     }
 
     # add custom args
@@ -4152,35 +4070,18 @@ sub config_to_command {
     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
 }
 
-sub vnc_socket {
-    my ($vmid) = @_;
-    return "${var_run_tmpdir}/$vmid.vnc";
-}
-
 sub spice_port {
     my ($vmid) = @_;
 
-    my $res = vm_mon_cmd($vmid, 'query-spice');
+    my $res = mon_cmd($vmid, 'query-spice');
 
     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
 }
 
-sub qmp_socket {
-    my ($vmid, $qga, $name) = @_;
-    my $sockettype = $qga ? 'qga' : 'qmp';
-    my $ext = $name ? '-'.$name : '';
-    return "${var_run_tmpdir}/$vmid$ext.$sockettype";
-}
-
-sub pidfile_name {
-    my ($vmid) = @_;
-    return "${var_run_tmpdir}/$vmid.pid";
-}
-
 sub vm_devices_list {
     my ($vmid) = @_;
 
-    my $res = vm_mon_cmd($vmid, 'query-pci');
+    my $res = mon_cmd($vmid, 'query-pci');
     my $devices_to_check = [];
     my $devices = {};
     foreach my $pcibus (@$res) {
@@ -4199,14 +4100,14 @@ sub vm_devices_list {
 	$devices_to_check = $to_check;
     }
 
-    my $resblock = vm_mon_cmd($vmid, 'query-block');
+    my $resblock = mon_cmd($vmid, 'query-block');
     foreach my $block (@$resblock) {
 	if($block->{device} =~ m/^drive-(\S+)/){
 		$devices->{$1} = 1;
 	}
     }
 
-    my $resmice = vm_mon_cmd($vmid, 'query-mice');
+    my $resmice = mon_cmd($vmid, 'query-mice');
     foreach my $mice (@$resmice) {
 	if ($mice->{name} eq 'QEMU HID Tablet') {
 	    $devices->{tablet} = 1;
@@ -4217,7 +4118,7 @@ sub vm_devices_list {
     # for usb devices there is no query-usb
     # but we can iterate over the entries in
     # qom-list path=/machine/peripheral
-    my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
+    my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
     foreach my $per (@$resperipheral) {
 	if ($per->{name} =~ m/^usb\d+$/) {
 	    $devices->{$per->{name}} = 1;
@@ -4230,7 +4131,7 @@ sub vm_devices_list {
 sub vm_deviceplug {
     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
 
-    my $q35 = machine_type_is_q35($conf);
+    my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
 
     my $devices_list = vm_devices_list($vmid);
     return 1 if defined($devices_list->{$deviceid});
@@ -4306,7 +4207,7 @@ sub vm_deviceplug {
 
 	return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
 
-	my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
+	my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
 	my $use_old_bios_files = undef;
 	($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
 
@@ -4398,13 +4299,13 @@ sub qemu_deviceadd {
     $devicefull = "driver=".$devicefull;
     my %options =  split(/[=,]/, $devicefull);
 
-    vm_mon_cmd($vmid, "device_add" , %options);
+    mon_cmd($vmid, "device_add" , %options);
 }
 
 sub qemu_devicedel {
     my ($vmid, $deviceid) = @_;
 
-    my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
+    my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
 }
 
 sub qemu_iothread_add {
@@ -4433,7 +4334,7 @@ sub qemu_iothread_del {
 sub qemu_objectadd {
     my($vmid, $objectid, $qomtype) = @_;
 
-    vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
+    mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
 
     return 1;
 }
@@ -4441,7 +4342,7 @@ sub qemu_objectadd {
 sub qemu_objectdel {
     my($vmid, $objectid) = @_;
 
-    vm_mon_cmd($vmid, "object-del", id => $objectid);
+    mon_cmd($vmid, "object-del", id => $objectid);
 
     return 1;
 }
@@ -4451,7 +4352,7 @@ sub qemu_driveadd {
 
     my $drive = print_drive_full($storecfg, $vmid, $device);
     $drive =~ s/\\/\\\\/g;
-    my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
+    my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
 
     # If the command succeeds qemu prints: "OK"
     return 1 if $ret =~ m/OK/s;
@@ -4462,7 +4363,7 @@ sub qemu_driveadd {
 sub qemu_drivedel {
     my($vmid, $deviceid) = @_;
 
-    my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
+    my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
     $ret =~ s/^\s+//;
 
     return 1 if $ret eq "";
@@ -4572,7 +4473,7 @@ sub qemu_add_pci_bridge {
 sub qemu_set_link_status {
     my ($vmid, $device, $up) = @_;
 
-    vm_mon_cmd($vmid, "set_link", name => $device,
+    mon_cmd($vmid, "set_link", name => $device,
 	       up => $up ? JSON::true : JSON::false);
 }
 
@@ -4582,14 +4483,14 @@ sub qemu_netdevadd {
     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
     my %options =  split(/[=,]/, $netdev);
 
-    vm_mon_cmd($vmid, "netdev_add",  %options);
+    mon_cmd($vmid, "netdev_add",  %options);
     return 1;
 }
 
 sub qemu_netdevdel {
     my ($vmid, $deviceid) = @_;
 
-    vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
+    mon_cmd($vmid, "netdev_del", id => $deviceid);
 }
 
 sub qemu_usb_hotplug {
@@ -4620,7 +4521,7 @@ sub qemu_usb_hotplug {
 sub qemu_cpu_hotplug {
     my ($vmid, $conf, $vcpus) = @_;
 
-    my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
+    my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
 
     my $sockets = 1;
     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
@@ -4637,14 +4538,14 @@ sub qemu_cpu_hotplug {
 
     if ($vcpus < $currentvcpus) {
 
-	if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
+	if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
 
 	    for (my $i = $currentvcpus; $i > $vcpus; $i--) {
 		qemu_devicedel($vmid, "cpu$i");
 		my $retry = 0;
 		my $currentrunningvcpus = undef;
 		while (1) {
-		    $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
+		    $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
 		    last if scalar(@{$currentrunningvcpus}) == $i-1;
 		    raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
 		    $retry++;
@@ -4661,11 +4562,11 @@ sub qemu_cpu_hotplug {
 	return;
     }
 
-    my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
+    my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
     die "vcpus in running vm does not match its configuration\n"
 	if scalar(@{$currentrunningvcpus}) != $currentvcpus;
 
-    if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
+    if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
 
 	for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
 	    my $cpustr = print_cpu_device($conf, $i);
@@ -4674,7 +4575,7 @@ sub qemu_cpu_hotplug {
 	    my $retry = 0;
 	    my $currentrunningvcpus = undef;
 	    while (1) {
-		$currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
+		$currentrunningvcpus = mon_cmd($vmid, "query-cpus");
 		last if scalar(@{$currentrunningvcpus}) == $i;
 		raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
 		sleep 1;
@@ -4687,7 +4588,7 @@ sub qemu_cpu_hotplug {
     } else {
 
 	for (my $i = $currentvcpus; $i < $vcpus; $i++) {
-	    vm_mon_cmd($vmid, "cpu-add", id => int($i));
+	    mon_cmd($vmid, "cpu-add", id => int($i));
 	}
     }
 }
@@ -4701,7 +4602,7 @@ sub qemu_block_set_io_throttle {
 
     return if !check_running($vmid) ;
 
-    vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
+    mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
 	bps => int($bps),
 	bps_rd => int($bps_rd),
 	bps_wr => int($bps_wr),
@@ -4766,7 +4667,7 @@ sub qemu_block_resize {
 
     return if !$running;
 
-    vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
+    mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
 
 }
 
@@ -4776,7 +4677,7 @@ sub qemu_volume_snapshot {
     my $running = check_running($vmid);
 
     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
-	vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
+	mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
     } else {
 	PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
     }
@@ -4798,7 +4699,7 @@ sub qemu_volume_snapshot_delete {
     }
 
     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
-	vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
+	mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
     } else {
 	PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
     }
@@ -4817,7 +4718,7 @@ sub set_migration_caps {
 	"compress" => 0
     };
 
-    my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
+    my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
 
     for my $supported_capability (@$supported_capabilities) {
 	push @$cap_ref, {
@@ -4826,7 +4727,7 @@ sub set_migration_caps {
 	};
     }
 
-    vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
+    mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
 }
 
 my $fast_plug_option = {
@@ -4839,6 +4740,7 @@ my $fast_plug_option = {
     'protection' => 1,
     'vmstatestorage' => 1,
     'hookscript' => 1,
+    'tags' => 1,
 };
 
 # hotplug changes in [PENDING]
@@ -4849,7 +4751,8 @@ sub vmconfig_hotplug_pending {
     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
 
     my $defaults = load_defaults();
-    my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
+    my $arch = get_vm_arch($conf);
+    my $machine_type = get_vm_machine($conf, undef, $arch);
 
     # commit values which do not have any impact on running VM first
     # Note: those option cannot raise errors, we we do not care about
@@ -4876,9 +4779,10 @@ sub vmconfig_hotplug_pending {
 
     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
 
-    my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
-    while (my ($opt, $force) = each %$pending_delete_hash) {
+    my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
+    foreach my $opt (sort keys %$pending_delete_hash) {
 	next if $selection && !$selection->{$opt};
+	my $force = $pending_delete_hash->{$opt}->{force};
 	eval {
 	    if ($opt eq 'hotplug') {
 		die "skip\n" if ($conf->{hotplug} =~ /memory/);
@@ -4906,7 +4810,7 @@ sub vmconfig_hotplug_pending {
 		die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
 		# here we reset the ballooning value to memory
 		my $balloon = $conf->{memory} || $defaults->{memory};
-		vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
+		mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
 	    } elsif ($fast_plug_option->{$opt}) {
 		# do nothing
 	    } elsif ($opt =~ m/^net(\d+)$/) {
@@ -4932,16 +4836,18 @@ sub vmconfig_hotplug_pending {
 	} else {
 	    # save new config if hotplug was successful
 	    delete $conf->{$opt};
-	    vmconfig_undelete_pending_option($conf, $opt);
+	    PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
 	    PVE::QemuConfig->write_config($vmid, $conf);
 	    $conf = PVE::QemuConfig->load_config($vmid); # update/reload
 	}
     }
 
-    my $apply_pending_cloudinit;
+    my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
     $apply_pending_cloudinit = sub {
+	return if $apply_pending_cloudinit_done; # once is enough
+	$apply_pending_cloudinit_done = 1; # once is enough
+
 	my ($key, $value) = @_;
-	$apply_pending_cloudinit = sub {}; # once is enough
 
 	my @cloudinit_opts = keys %$confdesc_cloudinit;
 	foreach my $opt (keys %{$conf->{pending}}) {
@@ -4990,13 +4896,14 @@ sub vmconfig_hotplug_pending {
 		# allow manual ballooning if shares is set to zero
 		if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
 		    my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
-		    vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
+		    mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
 		}
 	    } elsif ($opt =~ m/^net(\d+)$/) {
 		# some changes can be done without hotplug
 		vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
 				    $vmid, $opt, $value, $arch, $machine_type);
 	    } elsif (is_valid_drivename($opt)) {
+		die "skip\n" if $opt eq 'efidisk0';
 		# some changes can be done without hotplug
 		my $drive = parse_drive($opt, $value);
 		if (drive_is_cloudinit($drive)) {
@@ -5067,25 +4974,28 @@ sub vmconfig_delete_or_detach_drive {
     }
 }
 
+
+
 sub vmconfig_apply_pending {
     my ($vmid, $conf, $storecfg) = @_;
 
     # cold plug
 
-    my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
-    while (my ($opt, $force) = each %$pending_delete_hash) {
+    my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
+    foreach my $opt (sort keys %$pending_delete_hash) {
 	die "internal error" if $opt =~ m/^unused/;
+	my $force = $pending_delete_hash->{$opt}->{force};
 	$conf = PVE::QemuConfig->load_config($vmid); # update/reload
 	if (!defined($conf->{$opt})) {
-	    vmconfig_undelete_pending_option($conf, $opt);
+	    PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
 	    PVE::QemuConfig->write_config($vmid, $conf);
 	} elsif (is_valid_drivename($opt)) {
 	    vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
-	    vmconfig_undelete_pending_option($conf, $opt);
+	    PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
 	    delete $conf->{$opt};
 	    PVE::QemuConfig->write_config($vmid, $conf);
 	} else {
-	    vmconfig_undelete_pending_option($conf, $opt);
+	    PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
 	    delete $conf->{$opt};
 	    PVE::QemuConfig->write_config($vmid, $conf);
 	}
@@ -5263,14 +5173,14 @@ sub vmconfig_update_disk {
 	    } else { # cdrom
 
 		if ($drive->{file} eq 'none') {
-		    vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
+		    mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
 		    if (drive_is_cloudinit($old_drive)) {
 			vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
 		    }
 		} else {
 		    my $path = get_iso_path($storecfg, $vmid, $drive->{file});
-		    vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
-		    vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
+		    mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
+		    mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
 		}
 
 		return 1;
@@ -5300,6 +5210,10 @@ sub vm_start {
 
 	die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
 
+	# clean up leftover reboot request files
+	eval { clear_reboot_request($vmid); };
+	warn $@ if $@;
+
 	if (!$statefile && scalar(keys %{$conf->{pending}})) {
 	    vmconfig_apply_pending($vmid, $conf, $storecfg);
 	    $conf = PVE::QemuConfig->load_config($vmid); # update/reload
@@ -5370,7 +5284,35 @@ sub vm_start {
 
 	my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
 
-	my $migrate_port = 0;
+	my $migration_ip;
+	my $get_migration_ip = sub {
+	    my ($cidr, $nodename) = @_;
+
+	    return $migration_ip if defined($migration_ip);
+
+	    if (!defined($cidr)) {
+		my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
+		$cidr = $dc_conf->{migration}->{network};
+	    }
+
+	    if (defined($cidr)) {
+		my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
+
+		die "could not get IP: no address configured on local " .
+		    "node for network '$cidr'\n" if scalar(@$ips) == 0;
+
+		die "could not get IP: multiple addresses configured on local " .
+		    "node for network '$cidr'\n" if scalar(@$ips) > 1;
+
+		$migration_ip = @$ips[0];
+	    }
+
+	    $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
+		if !defined($migration_ip);
+
+	    return $migration_ip;
+	};
+
 	my $migrate_uri;
 	if ($statefile) {
 	    if ($statefile eq 'tcp') {
@@ -5387,18 +5329,12 @@ sub vm_start {
 		}
 
 		if ($migration_type eq 'insecure') {
-		    my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
-		    if ($migrate_network_addr) {
-			$localip = $migrate_network_addr;
-		    } else {
-			$localip = PVE::Cluster::remote_node_ip($nodename, 1);
-		    }
-
+		    $localip = $get_migration_ip->($migration_network, $nodename);
 		    $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
 		}
 
 		my $pfamily = PVE::Tools::get_host_address_family($nodename);
-		$migrate_port = PVE::Tools::next_migrate_port($pfamily);
+		my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
 		$migrate_uri = "tcp:${localip}:${migrate_port}";
 		push @$cmd, '-incoming', $migrate_uri;
 		push @$cmd, '-S';
@@ -5415,8 +5351,12 @@ sub vm_start {
 		push @$cmd, '-incoming', $migrate_uri;
 		push @$cmd, '-S';
 
-	    } else {
+	    } elsif (-e $statefile) {
 		push @$cmd, '-loadstate', $statefile;
+	    } else {
+		my $statepath = PVE::Storage::path($storecfg, $statefile);
+		push @$vollist, $statefile;
+		push @$cmd, '-loadstate', $statepath;
 	    }
 	} elsif ($paused) {
 	    push @$cmd, '-S';
@@ -5429,8 +5369,9 @@ sub vm_start {
 	  my $pcidevices = $d->{pciid};
 	  foreach my $pcidevice (@$pcidevices) {
 		my $pciid = $pcidevice->{id};
+		$pciid = "0000:$pciid" if $pciid !~ m/^[0-9a-f]{4}:/;
 
-		my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
+		my $info = PVE::SysFSTools::pci_device_info("$pciid");
 		die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
 		die "no pci device info for device '$pciid'\n" if !$info;
 
@@ -5512,26 +5453,25 @@ sub vm_start {
 	print "migration listens on $migrate_uri\n" if $migrate_uri;
 
 	if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
-	    eval { vm_mon_cmd_nocheck($vmid, "cont"); };
+	    eval { mon_cmd($vmid, "cont"); };
 	    warn $@ if $@;
 	}
 
 	#start nbd server for storage migration
 	if ($targetstorage) {
 	    my $nodename = PVE::INotify::nodename();
-	    my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
-	    my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
+	    my $localip = $get_migration_ip->($migration_network, $nodename);
 	    my $pfamily = PVE::Tools::get_host_address_family($nodename);
-	    $migrate_port = PVE::Tools::next_migrate_port($pfamily);
+	    my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
 
-	    vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
+	    mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
 
 	    $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
 
 	    foreach my $opt (sort keys %$local_volumes) {
 		my $volid = $local_volumes->{$opt};
-		vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
-		my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
+		mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
+		my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
 		print "storage migration listens on $migrate_storage_uri volume:$volid\n";
 	    }
 	}
@@ -5545,13 +5485,13 @@ sub vm_start {
 	    if ($spice_port) {
 	        print "spice listens on port $spice_port\n";
 		if ($spice_ticket) {
-		    vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
-		    vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
+		    mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
+		    mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
 		}
 	    }
 
 	} else {
-	    vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
+	    mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
 		if !$statefile && $conf->{balloon};
 
 	    foreach my $opt (keys %$conf) {
@@ -5561,16 +5501,18 @@ sub vm_start {
 	    }
 	}
 
-	vm_mon_cmd_nocheck($vmid, 'qom-set',
+	mon_cmd($vmid, 'qom-set',
 		    path => "machine/peripheral/balloon0",
 		    property => "guest-stats-polling-interval",
 		    value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
 
-	if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
+	if ($is_suspended) {
 	    print "Resumed VM, removing state\n";
+	    if (my $vmstate = $conf->{vmstate}) {
+		PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
+		PVE::Storage::vdisk_free($storecfg, $vmstate);
+	    }
 	    delete $conf->@{qw(lock vmstate runningmachine)};
-	    PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
-	    PVE::Storage::vdisk_free($storecfg, $vmstate);
 	    PVE::QemuConfig->write_config($vmid, $conf);
 	}
 
@@ -5578,69 +5520,19 @@ sub vm_start {
     });
 }
 
-sub vm_mon_cmd {
-    my ($vmid, $execute, %params) = @_;
-
-    my $cmd = { execute => $execute, arguments => \%params };
-    vm_qmp_command($vmid, $cmd);
-}
-
-sub vm_mon_cmd_nocheck {
-    my ($vmid, $execute, %params) = @_;
-
-    my $cmd = { execute => $execute, arguments => \%params };
-    vm_qmp_command($vmid, $cmd, 1);
-}
-
-sub vm_qmp_command {
-    my ($vmid, $cmd, $nocheck) = @_;
-
-    my $res;
-
-    my $timeout;
-    if ($cmd->{arguments}) {
-	$timeout = delete $cmd->{arguments}->{timeout};
-    }
-
-    eval {
-	die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
-	my $sname = qmp_socket($vmid);
-	if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
-	    my $qmpclient = PVE::QMPClient->new();
-
-	    $res = $qmpclient->cmd($vmid, $cmd, $timeout);
-	} else {
-	    die "unable to open monitor socket\n";
-	}
-    };
-    if (my $err = $@) {
-	syslog("err", "VM $vmid qmp command failed - $err");
-	die $err;
-    }
-
-    return $res;
-}
-
-sub vm_human_monitor_command {
-    my ($vmid, $cmdline) = @_;
-
-    my $cmd = {
-	execute => 'human-monitor-command',
-	arguments => { 'command-line' => $cmdline},
-    };
-
-    return vm_qmp_command($vmid, $cmd);
-}
-
 sub vm_commandline {
     my ($storecfg, $vmid, $snapname) = @_;
 
     my $conf = PVE::QemuConfig->load_config($vmid);
+    my $forcemachine;
 
     if ($snapname) {
 	my $snapshot = $conf->{snapshots}->{$snapname};
 	die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
 
+	# check for a 'runningmachine' in snapshot
+	$forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
+
 	$snapshot->{digest} = $conf->{digest}; # keep file digest for API
 
 	$conf = $snapshot;
@@ -5648,7 +5540,7 @@ sub vm_commandline {
 
     my $defaults = load_defaults();
 
-    my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
+    my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
 
     return PVE::Tools::cmd2string($cmd);
 }
@@ -5662,7 +5554,7 @@ sub vm_reset {
 
 	PVE::QemuConfig->check_lock($conf) if !$skiplock;
 
-	vm_mon_cmd($vmid, "system_reset");
+	mon_cmd($vmid, "system_reset");
     });
 }
 
@@ -5724,7 +5616,85 @@ sub vm_stop_cleanup {
     warn $@ if $@; # avoid errors - just warn
 }
 
-# Note: use $nockeck to skip tests if VM configuration file exists.
+# call only in locked context
+sub _do_vm_stop {
+    my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
+
+    my $pid = check_running($vmid, $nocheck);
+    return if !$pid;
+
+    my $conf;
+    if (!$nocheck) {
+	$conf = PVE::QemuConfig->load_config($vmid);
+	PVE::QemuConfig->check_lock($conf) if !$skiplock;
+	if (!defined($timeout) && $shutdown && $conf->{startup}) {
+	    my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
+	    $timeout = $opts->{down} if $opts->{down};
+	}
+	PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
+    }
+
+    eval {
+	if ($shutdown) {
+	    if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
+		mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
+	    } else {
+		mon_cmd($vmid, "system_powerdown");
+	    }
+	} else {
+	    mon_cmd($vmid, "quit");
+	}
+    };
+    my $err = $@;
+
+    if (!$err) {
+	$timeout = 60 if !defined($timeout);
+
+	my $count = 0;
+	while (($count < $timeout) && check_running($vmid, $nocheck)) {
+	    $count++;
+	    sleep 1;
+	}
+
+	if ($count >= $timeout) {
+	    if ($force) {
+		warn "VM still running - terminating now with SIGTERM\n";
+		kill 15, $pid;
+	    } else {
+		die "VM quit/powerdown failed - got timeout\n";
+	    }
+	} else {
+	    vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
+	    return;
+	}
+    } else {
+	if ($force) {
+	    warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
+	    kill 15, $pid;
+	} else {
+	    die "VM quit/powerdown failed\n";
+	}
+    }
+
+    # wait again
+    $timeout = 10;
+
+    my $count = 0;
+    while (($count < $timeout) && check_running($vmid, $nocheck)) {
+	$count++;
+	sleep 1;
+    }
+
+    if ($count >= $timeout) {
+	warn "VM still running - terminating now with SIGKILL\n";
+	kill 9, $pid;
+	sleep 1;
+    }
+
+    vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
+}
+
+# Note: use $nocheck to skip tests if VM configuration file exists.
 # We need that when migration VMs to other nodes (files already moved)
 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
 sub vm_stop {
@@ -5741,82 +5711,30 @@ sub vm_stop {
     }
 
     PVE::QemuConfig->lock_config($vmid, sub {
+	_do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
+   });
+}
 
-	my $pid = check_running($vmid, $nocheck);
-	return if !$pid;
-
-	my $conf;
-	if (!$nocheck) {
-	    $conf = PVE::QemuConfig->load_config($vmid);
-	    PVE::QemuConfig->check_lock($conf) if !$skiplock;
-	    if (!defined($timeout) && $shutdown && $conf->{startup}) {
-		my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
-		$timeout = $opts->{down} if $opts->{down};
-	    }
-	    PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
-	}
+sub vm_reboot {
+    my ($vmid, $timeout) = @_;
 
+    PVE::QemuConfig->lock_config($vmid, sub {
 	eval {
-	    if ($shutdown) {
-		if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
-		    vm_qmp_command($vmid, {
-			execute => "guest-shutdown",
-			arguments => { timeout => $timeout }
-		    }, $nocheck);
-		} else {
-		    vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
-		}
-	    } else {
-		vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
-	    }
-	};
-	my $err = $@;
 
-	if (!$err) {
-	    $timeout = 60 if !defined($timeout);
+	    # only reboot if running, as qmeventd starts it again on a stop event
+	    return if !check_running($vmid);
 
-	    my $count = 0;
-	    while (($count < $timeout) && check_running($vmid, $nocheck)) {
-		$count++;
-		sleep 1;
-	    }
+	    create_reboot_request($vmid);
 
-	    if ($count >= $timeout) {
-		if ($force) {
-		    warn "VM still running - terminating now with SIGTERM\n";
-		    kill 15, $pid;
-		} else {
-		    die "VM quit/powerdown failed - got timeout\n";
-		}
-	    } else {
-		vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
-		return;
-	    }
-	} else {
-	    if ($force) {
-		warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
-		kill 15, $pid;
-	    } else {
-		die "VM quit/powerdown failed\n";
-	    }
-	}
+	    my $storecfg = PVE::Storage::config();
+	    _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
 
-	# wait again
-	$timeout = 10;
-
-	my $count = 0;
-	while (($count < $timeout) && check_running($vmid, $nocheck)) {
-	    $count++;
-	    sleep 1;
-	}
-
-	if ($count >= $timeout) {
-	    warn "VM still running - terminating now with SIGKILL\n";
-	    kill 9, $pid;
-	    sleep 1;
+	};
+	if (my $err = $@) {
+	    # avoid that the next normal shutdown will be confused for a reboot
+	    clear_reboot_request($vmid);
+	    die $err;
 	}
-
-	vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
    });
 }
 
@@ -5847,7 +5765,7 @@ sub vm_suspend {
 	    $path = PVE::Storage::path($storecfg, $vmstate);
 	    PVE::QemuConfig->write_config($vmid, $conf);
 	} else {
-	    vm_mon_cmd($vmid, "stop");
+	    mon_cmd($vmid, "stop");
 	}
     });
 
@@ -5856,9 +5774,9 @@ sub vm_suspend {
 	PVE::Storage::activate_volumes($storecfg, [$vmstate]);
 
 	eval {
-	    vm_mon_cmd($vmid, "savevm-start", statefile => $path);
+	    mon_cmd($vmid, "savevm-start", statefile => $path);
 	    for(;;) {
-		my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
+		my $state = mon_cmd($vmid, "query-savevm");
 		if (!$state->{status}) {
 		    die "savevm not active\n";
 		} elsif ($state->{status} eq 'active') {
@@ -5881,7 +5799,7 @@ sub vm_suspend {
 	    if ($err) {
 		# cleanup, but leave suspending lock, to indicate something went wrong
 		eval {
-		    vm_mon_cmd($vmid, "savevm-end");
+		    mon_cmd($vmid, "savevm-end");
 		    PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
 		    PVE::Storage::vdisk_free($storecfg, $vmstate);
 		    delete $conf->@{qw(vmstate runningmachine)};
@@ -5894,7 +5812,7 @@ sub vm_suspend {
 	    die "lock changed unexpectedly\n"
 		if !PVE::QemuConfig->has_lock($conf, 'suspending');
 
-	    vm_qmp_command($vmid, { execute => "quit" });
+	    mon_cmd($vmid, "quit");
 	    $conf->{lock} = 'suspended';
 	    PVE::QemuConfig->write_config($vmid, $conf);
 	});
@@ -5905,8 +5823,7 @@ sub vm_resume {
     my ($vmid, $skiplock, $nocheck) = @_;
 
     PVE::QemuConfig->lock_config($vmid, sub {
-	my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
-	my $res = $vm_mon_cmd->($vmid, 'query-status');
+	my $res = mon_cmd($vmid, 'query-status');
 	my $resume_cmd = 'cont';
 
 	if ($res->{status} && $res->{status} eq 'suspended') {
@@ -5921,7 +5838,7 @@ sub vm_resume {
 		if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
 	}
 
-	$vm_mon_cmd->($vmid, $resume_cmd);
+	mon_cmd($vmid, $resume_cmd);
     });
 }
 
@@ -5933,26 +5850,11 @@ sub vm_sendkey {
 	my $conf = PVE::QemuConfig->load_config($vmid);
 
 	# there is no qmp command, so we use the human monitor command
-	my $res = vm_human_monitor_command($vmid, "sendkey $key");
+	my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
 	die $res if $res ne '';
     });
 }
 
-sub vm_destroy {
-    my ($storecfg, $vmid, $skiplock) = @_;
-
-    PVE::QemuConfig->lock_config($vmid, sub {
-
-	my $conf = PVE::QemuConfig->load_config($vmid);
-
-	if (!check_running($vmid)) {
-	    destroy_vm($storecfg, $vmid, undef, $skiplock);
-	} else {
-	    die "VM $vmid is running - destroy failed\n";
-	}
-    });
-}
-
 # vzdump restore implementaion
 
 sub tar_archive_read_firstfile {
@@ -6440,7 +6342,7 @@ sub restore_vma_archive {
 	    foreach_drive($oldconf, sub {
 		my ($ds, $drive) = @_;
 
-		return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
+		return if drive_is_cdrom($drive, 1);
 
 		my $volid = $drive->{file};
 		return if !$volid || $volid =~ m|^/|;
@@ -6620,9 +6522,11 @@ sub restore_tar_archive {
 
     my $storecfg = PVE::Storage::config();
 
-    # destroy existing data - keep empty config
+    # avoid zombie disks when restoring over an existing VM -> cleanup first
+    # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
+    # skiplock=1 because qmrestore has set the 'create' lock itself already
     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
-    destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
+    destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
 
     my $tocmd = "/usr/lib/qemu-server/qmextract";
 
@@ -6700,14 +6604,9 @@ sub restore_tar_archive {
 	$srcfd->close();
 	$outfd->close();
     };
-    my $err = $@;
-
-    if ($err) {
-
+    if (my $err = $@) {
 	unlink $tmpfn;
-
 	tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
-
 	die $err;
     }
 
@@ -6742,6 +6641,9 @@ sub foreach_storage_used_by_vm {
     }
 }
 
+my $qemu_snap_storage = {
+    rbd => 1,
+};
 sub do_snapshots_with_qemu {
     my ($storecfg, $volid) = @_;
 
@@ -6762,7 +6664,7 @@ sub do_snapshots_with_qemu {
 sub qga_check_running {
     my ($vmid, $nowarn) = @_;
 
-    eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
+    eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
     if ($@) {
 	warn "Qemu Guest Agent is not running - $@" if !$nowarn;
 	return 0;
@@ -6815,66 +6717,78 @@ sub qemu_img_convert {
     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
 
-    if ($src_storeid && $dst_storeid) {
+    die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
 
-	PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
+    my $cachemode;
+    my $src_path;
+    my $src_is_iscsi = 0;
+    my $src_format;
 
+    if ($src_storeid) {
+	PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
 	my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
-	my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
-
-	my $src_format = qemu_img_format($src_scfg, $src_volname);
-	my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
+	$src_format = qemu_img_format($src_scfg, $src_volname);
+	$src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
+	$src_is_iscsi = ($src_path =~ m|^iscsi://|);
+	$cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
+    } elsif (-f $src_volid) {
+	$src_path = $src_volid;
+	if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
+	    $src_format = $1;
+	}
+    }
 
-	my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
-	my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
+    die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
 
-	my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
-	my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
+    my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
+    my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
+    my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
+    my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
 
-	my $cmd = [];
-	push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
-	push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
-	push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
-	push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
+    my $cmd = [];
+    push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
+    push @$cmd, '-l', "snapshot.name=$snapname"
+	if $snapname && $src_format && $src_format eq "qcow2";
+    push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
+    push @$cmd, '-T', $cachemode if defined($cachemode);
+
+    if ($src_is_iscsi) {
+	push @$cmd, '--image-opts';
+	$src_path = convert_iscsi_path($src_path);
+    } elsif ($src_format) {
+	push @$cmd, '-f', $src_format;
+    }
+
+    if ($dst_is_iscsi) {
+	push @$cmd, '--target-image-opts';
+	$dst_path = convert_iscsi_path($dst_path);
+    } else {
+	push @$cmd, '-O', $dst_format;
+    }
 
-	if ($src_is_iscsi) {
-	    push @$cmd, '--image-opts';
-	    $src_path = convert_iscsi_path($src_path);
-	} else {
-	    push @$cmd, '-f', $src_format;
-	}
+    push @$cmd, $src_path;
 
-	if ($dst_is_iscsi) {
-	    push @$cmd, '--target-image-opts';
-	    $dst_path = convert_iscsi_path($dst_path);
-	} else {
-	    push @$cmd, '-O', $dst_format;
-	}
+    if (!$dst_is_iscsi && $is_zero_initialized) {
+	push @$cmd, "zeroinit:$dst_path";
+    } else {
+	push @$cmd, $dst_path;
+    }
 
-	push @$cmd, $src_path;
+    my $parser = sub {
+	my $line = shift;
+	if($line =~ m/\((\S+)\/100\%\)/){
+	    my $percent = $1;
+	    my $transferred = int($size * $percent / 100);
+	    my $remaining = $size - $transferred;
 
-	if (!$dst_is_iscsi && $is_zero_initialized) {
-	    push @$cmd, "zeroinit:$dst_path";
-	} else {
-	    push @$cmd, $dst_path;
+	    print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
 	}
 
-	my $parser = sub {
-	    my $line = shift;
-	    if($line =~ m/\((\S+)\/100\%\)/){
-		my $percent = $1;
-		my $transferred = int($size * $percent / 100);
-		my $remaining = $size - $transferred;
-
-		print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
-	    }
-
-	};
+    };
 
-	eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
-	my $err = $@;
-	die "copy failed: $err" if $err;
-    }
+    eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
+    my $err = $@;
+    die "copy failed: $err" if $err;
 }
 
 sub qemu_img_format {
@@ -6923,7 +6837,7 @@ sub qemu_drive_mirror {
     }
 
     # if a job already runs for this device we get an error, catch it for cleanup
-    eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
+    eval { mon_cmd($vmid, "drive-mirror", %$opts); };
     if (my $err = $@) {
 	eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
 	warn "$@\n" if $@;
@@ -6942,7 +6856,7 @@ sub qemu_drive_mirror_monitor {
 	while (1) {
 	    die "storage migration timed out\n" if $err_complete > 300;
 
-	    my $stats = vm_mon_cmd($vmid, "query-block-jobs");
+	    my $stats = mon_cmd($vmid, "query-block-jobs");
 
 	    my $running_mirror_jobs = {};
 	    foreach my $stat (@$stats) {
@@ -6985,7 +6899,7 @@ sub qemu_drive_mirror_monitor {
 		    my $agent_running = $qga && qga_check_running($vmid);
 		    if ($agent_running) {
 			print "freeze filesystem\n";
-			eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
+			eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
 		    } else {
 			print "suspend vm\n";
 			eval { PVE::QemuServer::vm_suspend($vmid, 1); };
@@ -6996,7 +6910,7 @@ sub qemu_drive_mirror_monitor {
 
 		    if ($agent_running) {
 			print "unfreeze filesystem\n";
-			eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
+			eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
 		    } else {
 			print "resume vm\n";
 			eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
@@ -7009,7 +6923,7 @@ sub qemu_drive_mirror_monitor {
 			# try to switch the disk if source and destination are on the same guest
 			print "$job: Completing block job...\n";
 
-			eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
+			eval { mon_cmd($vmid, "block-job-complete", device => $job) };
 			if ($@ =~ m/cannot be completed/) {
 			    print "$job: Block job cannot be completed, try again.\n";
 			    $err_complete++;
@@ -7037,12 +6951,12 @@ sub qemu_blockjobs_cancel {
 
     foreach my $job (keys %$jobs) {
 	print "$job: Cancelling block job\n";
-	eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
+	eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
 	$jobs->{$job}->{cancel} = 1;
     }
 
     while (1) {
-	my $stats = vm_mon_cmd($vmid, "query-block-jobs");
+	my $stats = mon_cmd($vmid, "query-block-jobs");
 
 	my $running_jobs = {};
 	foreach my $stat (@$stats) {
@@ -7085,17 +6999,19 @@ sub clone_disk {
 	my $name = undef;
 	if (drive_is_cloudinit($drive)) {
 	    $name = "vm-$newvmid-cloudinit";
+	    $name .= ".$dst_format" if $dst_format ne 'raw';
 	    $snapname = undef;
-	    # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
-	    if ($dst_format ne 'raw') {
-		$name .= ".$dst_format";
-	    }
+	    $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
 	}
 	$newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
 	push @$newvollist, $newvolid;
 
 	PVE::Storage::activate_volumes($storecfg, [$newvolid]);
 
+	if (drive_is_cloudinit($drive)) {
+	    goto no_data_clone;
+	}
+
 	my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
 	if (!$running || $snapname) {
 	    # TODO: handle bwlimits
@@ -7103,7 +7019,7 @@ sub clone_disk {
 	} else {
 
 	    my $kvmver = get_running_qemu_version ($vmid);
-	    if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
+	    if (!min_version($kvmver, 2, 7)) {
 		die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
 		    if $drive->{iothread};
 	    }
@@ -7112,6 +7028,7 @@ sub clone_disk {
 	}
     }
 
+no_data_clone:
     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
 
     my $disk = $drive;
@@ -7122,64 +7039,12 @@ sub clone_disk {
     return $disk;
 }
 
-# this only works if VM is running
-sub get_current_qemu_machine {
-    my ($vmid) = @_;
-
-    my $cmd = { execute => 'query-machines', arguments => {} };
-    my $res = vm_qmp_command($vmid, $cmd);
-
-    my ($current, $default);
-    foreach my $e (@$res) {
-	$default = $e->{name} if $e->{'is-default'};
-	$current = $e->{name} if $e->{'is-current'};
-    }
-
-    # fallback to the default machine if current is not supported by qemu
-    return $current || $default || 'pc';
-}
-
 sub get_running_qemu_version {
     my ($vmid) = @_;
-    my $cmd = { execute => 'query-version', arguments => {} };
-    my $res = vm_qmp_command($vmid, $cmd);
+    my $res = mon_cmd($vmid, "query-version");
     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
 }
 
-sub qemu_machine_feature_enabled {
-    my ($machine, $kvmver, $version_major, $version_minor) = @_;
-
-    my $current_major;
-    my $current_minor;
-
-    if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
-
-	$current_major = $3;
-	$current_minor = $4;
-
-    } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
-
-	$current_major = $1;
-	$current_minor = $2;
-    }
-
-    return 1 if $current_major > $version_major ||
-                ($current_major == $version_major &&
-                 $current_minor >= $version_minor);
-}
-
-sub qemu_machine_pxe {
-    my ($vmid, $conf, $machine) = @_;
-
-    $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
-
-    if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
-	$machine .= '.pxe';
-    }
-
-    return $machine;
-}
-
 sub qemu_use_old_bios_files {
     my ($machine_type) = @_;
 
@@ -7191,12 +7056,12 @@ sub qemu_use_old_bios_files {
         $machine_type = $1;
         $use_old_bios_files = 1;
     } else {
-	my $kvmver = kvm_user_version();
+	my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
         # load new efi bios files on migration. So this hack is required to allow
         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
-	$use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
+	$use_old_bios_files = !min_version($version, 2, 4);
     }
 
     return ($use_old_bios_files, $machine_type);
@@ -7208,23 +7073,21 @@ sub create_efidisk($$$$$) {
     my (undef, $ovmf_vars) = get_ovmf_files($arch);
     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
 
-    my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
+    my $vars_size_b = -s $ovmf_vars;
+    my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
     PVE::Storage::activate_volumes($storecfg, [$volid]);
 
-    my $path = PVE::Storage::path($storecfg, $volid);
-    eval {
-	run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
-    };
-    die "Copying EFI vars image failed: $@" if $@;
+    qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
+    my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
 
-    return ($volid, $vars_size);
+    return ($volid, $size/1024);
 }
 
 sub vm_iothreads_list {
     my ($vmid) = @_;
 
-    my $res = vm_mon_cmd($vmid, 'query-iothreads');
+    my $res = mon_cmd($vmid, 'query-iothreads');
 
     my $iothreads = {};
     foreach my $iothread (@$res) {
@@ -7254,7 +7117,7 @@ sub scsihw_infos {
 }
 
 sub add_hyperv_enlightenments {
-    my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
+    my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
 
     return if $winversion < 6;
     return if $bios && $bios eq 'ovmf' && $winversion < 8;
@@ -7264,7 +7127,7 @@ sub add_hyperv_enlightenments {
 	push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
     }
 
-    if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
+    if (min_version($machine_version, 2, 3)) {
 	push @$cpuFlags , 'hv_spinlocks=0x1fff';
 	push @$cpuFlags , 'hv_vapic';
 	push @$cpuFlags , 'hv_time';
@@ -7272,7 +7135,7 @@ sub add_hyperv_enlightenments {
 	push @$cpuFlags , 'hv_spinlocks=0xffff';
     }
 
-    if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
+    if (min_version($machine_version, 2, 6)) {
 	push @$cpuFlags , 'hv_reset';
 	push @$cpuFlags , 'hv_vpindex';
 	push @$cpuFlags , 'hv_runtime';
@@ -7281,12 +7144,12 @@ sub add_hyperv_enlightenments {
     if ($winversion >= 7) {
 	push @$cpuFlags , 'hv_relaxed';
 
-	if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
+	if (min_version($machine_version, 2, 12)) {
 	    push @$cpuFlags , 'hv_synic';
 	    push @$cpuFlags , 'hv_stimer';
 	}
 
-	if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
+	if (min_version($machine_version, 3, 1)) {
 	    push @$cpuFlags , 'hv_ipi';
 	}
     }
@@ -7357,7 +7220,26 @@ sub generate_smbios1_uuid {
 sub nbd_stop {
     my ($vmid) = @_;
 
-    vm_mon_cmd($vmid, 'nbd-server-stop');
+    mon_cmd($vmid, 'nbd-server-stop');
+}
+
+sub create_reboot_request {
+    my ($vmid) = @_;
+    open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
+	or die "failed to create reboot trigger file: $!\n";
+    close($fh);
+}
+
+sub clear_reboot_request {
+    my ($vmid) = @_;
+    my $path = "/run/qemu-server/$vmid.reboot";
+    my $res = 0;
+
+    $res = unlink($path);
+    die "could not remove reboot request for $vmid: $!"
+	if !$res && $! != POSIX::ENOENT;
+
+    return $res;
 }
 
 # bash completion helper
@@ -7433,4 +7315,22 @@ sub complete_storage {
     return $res;
 }
 
+sub complete_migration_storage {
+    my ($cmd, $param, $current_value, $all_args) = @_;
+
+    my $targetnode = @$all_args[1];
+
+    my $cfg = PVE::Storage::config();
+    my $ids = $cfg->{ids};
+
+    my $res = [];
+    foreach my $sid (keys %$ids) {
+	next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
+	next if !$ids->{$sid}->{content}->{images};
+	push @$res, $sid;
+    }
+
+    return $res;
+}
+
 1;