X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=PVE%2FQemuServer.pm;h=8c40ea211ea40cbb42e5d33b55f67a16dcb39a78;hb=045749f2fc;hp=a054199c81331b810b25bc38b6eda42352e0427f;hpb=fb4d1ba27e192b9611d1fc97ad464160b61832f2;p=qemu-server.git

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index a054199..8c40ea2 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -24,24 +24,28 @@ use POSIX;
 use Storable qw(dclone);
 use Time::HiRes qw(gettimeofday);
 use URI::Escape;
+use UUID;
 
 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
+use PVE::DataCenterConfig;
 use PVE::Exception qw(raise raise_param_exc);
 use PVE::GuestHelpers;
 use PVE::INotify;
 use PVE::JSONSchema qw(get_standard_option);
 use PVE::ProcFSTools;
 use PVE::RPCEnvironment;
-use PVE::SafeSyslog;
 use PVE::Storage;
 use PVE::SysFSTools;
 use PVE::Systemd;
-use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
+use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
 
 use PVE::QMPClient;
 use PVE::QemuConfig;
+use PVE::QemuServer::Helpers qw(min_version);
 use PVE::QemuServer::Cloudinit;
+use PVE::QemuServer::Machine;
 use PVE::QemuServer::Memory;
+use PVE::QemuServer::Monitor qw(mon_cmd);
 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
 use PVE::QemuServer::USB qw(parse_usb_device);
 
@@ -57,8 +61,6 @@ my $OVMF = {
     ],
 };
 
-my $qemu_snap_storage = { rbd => 1 };
-
 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 
 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
@@ -91,7 +93,7 @@ PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 	description => "Specifies the Qemu machine type.",
 	type => 'string',
-	pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
+	pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\.pxe)?|virt(?:-\d+(\.\d+)+)?)',
 	maxLength => 40,
 	optional => 1,
 });
@@ -108,16 +110,6 @@ sub cgroups_write {
 
 my $nodename = PVE::INotify::nodename();
 
-mkdir "/etc/pve/nodes/$nodename";
-my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
-mkdir $confdir;
-
-my $var_run_tmpdir = "/var/run/qemu-server";
-mkdir $var_run_tmpdir;
-
-my $lock_dir = "/var/lock/qemu-server";
-mkdir $lock_dir;
-
 my $cpu_vendor_list = {
     # Intel CPUs
     486 => 'GenuineIntel',
@@ -251,6 +243,13 @@ my $agent_fmt = {
 	optional => 1,
 	default => 0
     },
+    type => {
+	description => "Select the agent type",
+	type => 'string',
+	default => 'virtio',
+	optional => 1,
+	enum => [qw(virtio isa)],
+    },
 };
 
 my $vga_fmt = {
@@ -432,7 +431,7 @@ win7;; Microsoft Windows 7
 win8;; Microsoft Windows 8/2012/2012r2
 win10;; Microsoft Windows 10/2016
 l24;; Linux 2.4 Kernel
-l26;; Linux 2.6/3.X Kernel
+l26;; Linux 2.6 - 5.X Kernel
 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 EODESC
     },
@@ -700,6 +699,11 @@ EODESCR
 	description => "Configure additional enhancements for SPICE.",
 	optional => 1
     },
+    tags => {
+	type => 'string', format => 'pve-tag-list',
+	description => 'Tags of the VM. This is only meta information.',
+	optional => 1,
+    },
 };
 
 my $cicustom_fmt = {
@@ -1336,7 +1340,7 @@ my $usbdesc = {
 };
 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 
-my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
+my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
 my $hostpci_fmt = {
     host => {
 	default_key => 1,
@@ -1553,29 +1557,7 @@ sub option_exists {
     return defined($confdesc->{$key});
 }
 
-sub nic_models {
-    return $nic_model_list;
-}
-
-sub os_list_description {
-
-    return {
-	other => 'Other',
-	wxp => 'Windows XP',
-	w2k => 'Windows 2000',
-	w2k3 =>, 'Windows 2003',
-	w2k8 => 'Windows 2008',
-	wvista => 'Windows Vista',
-	win7 => 'Windows 7',
-	win8 => 'Windows 8/2012',
-	win10 => 'Windows 10/2016',
-	l24 => 'Linux 2.4',
-	l26 => 'Linux 2.6',
-    };
-}
-
 my $cdrom_path;
-
 sub get_cdrom_path {
 
     return  $cdrom_path if $cdrom_path;
@@ -1836,20 +1818,14 @@ sub path_is_scsi {
     return $res;
 }
 
-sub machine_type_is_q35 {
-    my ($conf) = @_;
-
-    return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
-}
-
 sub print_tabletdevice_full {
     my ($conf, $arch) = @_;
 
-    my $q35 = machine_type_is_q35($conf);
+    my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
 
     # we use uhci for old VMs because tablet driver was buggy in older qemu
     my $usbbus;
-    if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
+    if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
 	$usbbus = 'ehci';
     } else {
 	$usbbus = 'uhci';
@@ -1898,7 +1874,10 @@ sub print_drivedevice_full {
 		 $path = PVE::Storage::path($storecfg, $drive->{file});
 	    }
 
-	    if($path =~ m/^iscsi\:\/\//){
+	    # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
+	    my $version = PVE::QemuServer::Machine::extract_version($machine_type) // kvm_user_version();
+	    if ($path =~ m/^iscsi\:\/\// &&
+	       !min_version($version, 4, 1)) {
 		$devicetype = 'generic';
 	    }
 	}
@@ -2166,16 +2145,26 @@ my $vga_map = {
 };
 
 sub print_vga_device {
-    my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
+    my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
 
     my $type = $vga_map->{$vga->{type}};
     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
 	$type = 'virtio-gpu';
     }
     my $vgamem_mb = $vga->{memory};
+
+    my $max_outputs = '';
     if ($qxlnum) {
 	$type = $id ? 'qxl' : 'qxl-vga';
+
+	if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
+	    # set max outputs so linux can have up to 4 qxl displays with one device
+	    if (min_version($machine_version, 4, 1)) {
+		$max_outputs = ",max_outputs=4";
+	    }
+	}
     }
+
     die "no devicetype for $vga->{type}\n" if !$type;
 
     my $memory = "";
@@ -2196,7 +2185,7 @@ sub print_vga_device {
 	$memory = ",ram_size=67108864,vram_size=33554432";
     }
 
-    my $q35 = machine_type_is_q35($conf);
+    my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
     my $vgaid = "vga" . ($id // '');
     my $pciaddr;
 
@@ -2207,7 +2196,7 @@ sub print_vga_device {
 	$pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
     }
 
-    return "$type,id=${vgaid}${memory}${pciaddr}";
+    return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
 }
 
 sub drive_is_cloudinit {
@@ -2546,15 +2535,8 @@ sub check_type {
     }
 }
 
-sub touch_config {
-    my ($vmid) = @_;
-
-    my $conf = PVE::QemuConfig->config_file($vmid);
-    utime undef, undef, $conf;
-}
-
 sub destroy_vm {
-    my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
+    my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
 
     my $conf = PVE::QemuConfig->load_config($vmid);
 
@@ -2564,11 +2546,9 @@ sub destroy_vm {
 	# check if any base image is still used by a linked clone
 	foreach_drive($conf, sub {
 		my ($ds, $drive) = @_;
-
 		return if drive_is_cdrom($drive);
 
 		my $volid = $drive->{file};
-
 		return if !$volid || $volid =~ m|^/|;
 
 		die "base volume '$volid' is still in use by linked cloned\n"
@@ -2580,43 +2560,31 @@ sub destroy_vm {
     # only remove disks owned by this VM
     foreach_drive($conf, sub {
 	my ($ds, $drive) = @_;
-
  	return if drive_is_cdrom($drive, 1);
 
 	my $volid = $drive->{file};
-
 	return if !$volid || $volid =~ m|^/|;
 
 	my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
 	return if !$path || !$owner || ($owner != $vmid);
 
-	eval {
-	    PVE::Storage::vdisk_free($storecfg, $volid);
-	};
+	eval { PVE::Storage::vdisk_free($storecfg, $volid) };
 	warn "Could not remove disk '$volid', check manually: $@" if $@;
+    });
 
+    # also remove unused disk
+    my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
+    PVE::Storage::foreach_volid($vmdisks, sub {
+	my ($volid, $sid, $volname, $d) = @_;
+	eval { PVE::Storage::vdisk_free($storecfg, $volid) };
+	warn $@ if $@;
     });
 
-    if ($keep_empty_config) {
-	PVE::QemuConfig->write_config($vmid, { memory => 128 });
+    if (defined $replacement_conf) {
+	PVE::QemuConfig->write_config($vmid, $replacement_conf);
     } else {
 	PVE::QemuConfig->destroy_config($vmid);
     }
-
-    # also remove unused disk
-    eval {
-	my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
-
-	eval {
-	    PVE::Storage::foreach_volid($dl, sub {
-		my ($volid, $sid, $volname, $d) = @_;
-		PVE::Storage::vdisk_free($storecfg, $volid);
-	    });
-	};
-	warn $@ if $@;
-
-    };
-    warn $@ if $@;
 }
 
 sub parse_vm_config {
@@ -2966,70 +2934,19 @@ sub check_local_storage_availability {
     return $nodehash
 }
 
-sub check_cmdline {
-    my ($pidfile, $pid) = @_;
-
-    my $fh = IO::File->new("/proc/$pid/cmdline", "r");
-    if (defined($fh)) {
-	my $line = <$fh>;
-	$fh->close;
-	return undef if !$line;
-	my @param = split(/\0/, $line);
-
-	my $cmd = $param[0];
-	return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
-
-	for (my $i = 0; $i < scalar (@param); $i++) {
-	    my $p = $param[$i];
-	    next if !$p;
-	    if (($p eq '-pidfile') || ($p eq '--pidfile')) {
-		my $p = $param[$i+1];
-		return 1 if $p && ($p eq $pidfile);
-		return undef;
-	    }
-	}
-    }
-    return undef;
-}
-
+# Compat only, use assert_config_exists_on_node and vm_running_locally where possible
 sub check_running {
     my ($vmid, $nocheck, $node) = @_;
 
-    my $filename = PVE::QemuConfig->config_file($vmid, $node);
-
-    die "unable to find configuration file for VM $vmid - no such machine\n"
-	if !$nocheck && ! -f $filename;
-
-    my $pidfile = pidfile_name($vmid);
-
-    if (my $fd = IO::File->new("<$pidfile")) {
-	my $st = stat($fd);
-	my $line = <$fd>;
-	close($fd);
-
-	my $mtime = $st->mtime;
-	if ($mtime > time()) {
-	    warn "file '$filename' modified in future\n";
-	}
-
-	if ($line =~ m/^(\d+)$/) {
-	    my $pid = $1;
-	    if (check_cmdline($pidfile, $pid)) {
-		if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
-		    return $pid;
-		}
-	    }
-	}
-    }
-
-    return undef;
+    PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
+    return PVE::QemuServer::Helpers::vm_running_locally($vmid);
 }
 
 sub vzlist {
 
     my $vzlist = config_list();
 
-    my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
+    my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
 
     while (defined(my $de = $fd->read)) {
 	next if $de !~ m/^(\d+)\.pid$/;
@@ -3112,7 +3029,12 @@ our $vmstatus_return_properties = {
 	description => "The current config lock, if any.",
 	type => 'string',
 	optional => 1,
-    }
+    },
+    tags => {
+	description  => "The current configured tags, if any",
+	type => 'string',
+	optional => 1,
+    },
 };
 
 my $last_proc_pid_stat;
@@ -3137,8 +3059,7 @@ sub vmstatus {
     foreach my $vmid (keys %$list) {
 	next if $opt_vmid && ($vmid ne $opt_vmid);
 
-	my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
-	my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
+	my $conf = PVE::QemuConfig->load_config($vmid);
 
 	my $d = { vmid => $vmid };
 	$d->{pid} = $list->{$vmid}->{pid};
@@ -3184,6 +3105,7 @@ sub vmstatus {
 
 	$d->{serial} = 1 if conf_has_serial($conf);
 	$d->{lock} = $conf->{lock} if $conf->{lock};
+	$d->{tags} = $conf->{tags} if defined($conf->{tags});
 
 	$res->{$vmid} = $d;
     }
@@ -3423,28 +3345,32 @@ sub vga_conf_has_spice {
     return $1 || 1;
 }
 
-my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
-sub get_host_arch() {
-    $host_arch = (POSIX::uname())[4] if !$host_arch;
-    return $host_arch;
-}
-
 sub is_native($) {
     my ($arch) = @_;
     return get_host_arch() eq $arch;
 }
 
+sub get_vm_arch {
+    my ($conf) = @_;
+    return $conf->{arch} // get_host_arch();
+}
+
 my $default_machines = {
     x86_64 => 'pc',
     aarch64 => 'virt',
 };
 
-sub get_basic_machine_info {
-    my ($conf, $forcemachine) = @_;
+sub get_vm_machine {
+    my ($conf, $forcemachine, $arch) = @_;
+
+    my $machine = $forcemachine || $conf->{machine};
 
-    my $arch = $conf->{arch} // get_host_arch();
-    my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
-    return ($arch, $machine);
+    if (!$machine) {
+	$arch //= 'x86_64';
+	$machine ||= $default_machines->{$arch};
+    }
+
+    return $machine;
 }
 
 sub get_ovmf_files($) {
@@ -3470,7 +3396,7 @@ sub get_command_for_arch($) {
 }
 
 sub get_cpu_options {
-    my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
+    my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
 
     my $cpuFlags = [];
     my $ostype = $conf->{ostype};
@@ -3494,20 +3420,19 @@ sub get_cpu_options {
 
     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
 
-    push @$cpuFlags , '-x2apic'
-	if $conf->{ostype} && $conf->{ostype} eq 'solaris';
+    push @$cpuFlags , '-x2apic' if $ostype && $ostype eq 'solaris';
 
     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
 
     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
 
-    if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
+    if (min_version($machine_version, 2, 3) && $arch eq 'x86_64') {
 
 	push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
 	push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
     }
 
-    add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
+    add_hyperv_enlightenments($cpuFlags, $winversion, $machine_version, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
 
     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
 
@@ -3540,9 +3465,12 @@ sub config_to_command {
     my $winversion = windows_version($ostype);
     my $kvm = $conf->{kvm};
 
-    my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
+    my $arch = get_vm_arch($conf);
     my $kvm_binary = get_command_for_arch($arch);
     my $kvmver = kvm_user_version($kvm_binary);
+
+    my $machine_type = get_vm_machine($conf, $forcemachine, $arch);
+    my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type) // $kvmver;
     $kvm //= 1 if is_native($arch);
 
     if ($kvm) {
@@ -3558,7 +3486,7 @@ sub config_to_command {
 
     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
 
-    my $q35 = machine_type_is_q35($conf);
+    my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
     my $use_old_bios_files = undef;
     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
@@ -3576,16 +3504,16 @@ sub config_to_command {
 
     my $use_virtio = 0;
 
-    my $qmpsocket = qmp_socket($vmid);
+    my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
     push @$cmd, '-mon', "chardev=qmp,mode=control";
 
-    if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
+    if (min_version($machine_version, 2, 12)) {
 	push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
 	push @$cmd, '-mon', "chardev=qmp-event,mode=control";
     }
 
-    push @$cmd, '-pidfile' , pidfile_name($vmid);
+    push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
 
     push @$cmd, '-daemonize';
 
@@ -3651,7 +3579,7 @@ sub config_to_command {
     # load q35 config
     if ($q35) {
 	# we use different pcie-port hardware for qemu >= 4.0 for passthrough
-	if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
+	if (min_version($machine_version, 4, 0)) {
 	    push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
 	} else {
 	    push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
@@ -3669,7 +3597,7 @@ sub config_to_command {
     if (!$vga->{type}) {
 	if ($arch eq 'aarch64') {
 	    $vga->{type} = 'virtio';
-	} elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
+	} elsif (min_version($machine_version, 2, 9)) {
 	    $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
 	} else {
 	    $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
@@ -3734,7 +3662,7 @@ sub config_to_command {
 	if ($d->{mdev} && scalar(@$pcidevices) == 1) {
 	    my $pci_id = $pcidevices->[0]->{id};
 	    my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
-	    $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
+	    $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
 	} elsif ($d->{mdev}) {
 	    warn "ignoring mediated device '$id' with multifunction device\n";
 	}
@@ -3766,7 +3694,7 @@ sub config_to_command {
 
     # usb devices
     my $usb_dev_features = {};
-    $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
+    $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
 
     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
     push @$devices, @usbdevices if @usbdevices;
@@ -3835,7 +3763,7 @@ sub config_to_command {
     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
 	if ($allowed_vcpus < $maxcpus);
 
-    if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
+    if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
 
 	push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
         for (my $i = 2; $i <= $vcpus; $i++)  {
@@ -3865,8 +3793,8 @@ sub config_to_command {
     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
 
     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
-	push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
-	my $socket = vnc_socket($vmid);
+	push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
+	my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
 	push @$cmd,  '-vnc', "unix:$socket,password";
     } else {
 	push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
@@ -3908,7 +3836,7 @@ sub config_to_command {
 	push @$rtcFlags, 'base=localtime';
     }
 
-    push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
+    push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
 
     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
 
@@ -3916,12 +3844,19 @@ sub config_to_command {
 
     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
 
-    if (parse_guest_agent($conf)->{enabled}) {
-	my $qgasocket = qmp_socket($vmid, 1);
-	my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
+    my $guest_agent = parse_guest_agent($conf);
+
+    if ($guest_agent->{enabled}) {
+	my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
 	push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
-	push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
-	push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
+
+	if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
+	    my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
+	    push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
+	    push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
+	} elsif ($guest_agent->{type} eq 'isa') {
+	    push @$devices, '-device', "isa-serial,chardev=qga0";
+	}
     }
 
     my $spice_port;
@@ -3930,7 +3865,7 @@ sub config_to_command {
 	if ($qxlnum > 1) {
 	    if ($winversion){
 		for(my $i = 1; $i < $qxlnum; $i++){
-		    push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
+		    push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
 		}
 	    } else {
 		# assume other OS works like Linux
@@ -4095,7 +4030,7 @@ sub config_to_command {
 
     if (!$q35) {
 	# add pci bridges
-        if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
+        if (min_version($machine_version, 2, 3)) {
 	   $bridges->{1} = 1;
 	   $bridges->{2} = 1;
 	}
@@ -4118,7 +4053,7 @@ sub config_to_command {
 
     if (my $vmstate = $conf->{vmstate}) {
 	my $statepath = PVE::Storage::path($storecfg, $vmstate);
-	push @$vollist, $statepath;
+	push @$vollist, $vmstate;
 	push @$cmd, '-loadstate', $statepath;
     }
 
@@ -4131,35 +4066,18 @@ sub config_to_command {
     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
 }
 
-sub vnc_socket {
-    my ($vmid) = @_;
-    return "${var_run_tmpdir}/$vmid.vnc";
-}
-
 sub spice_port {
     my ($vmid) = @_;
 
-    my $res = vm_mon_cmd($vmid, 'query-spice');
+    my $res = mon_cmd($vmid, 'query-spice');
 
     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
 }
 
-sub qmp_socket {
-    my ($vmid, $qga, $name) = @_;
-    my $sockettype = $qga ? 'qga' : 'qmp';
-    my $ext = $name ? '-'.$name : '';
-    return "${var_run_tmpdir}/$vmid$ext.$sockettype";
-}
-
-sub pidfile_name {
-    my ($vmid) = @_;
-    return "${var_run_tmpdir}/$vmid.pid";
-}
-
 sub vm_devices_list {
     my ($vmid) = @_;
 
-    my $res = vm_mon_cmd($vmid, 'query-pci');
+    my $res = mon_cmd($vmid, 'query-pci');
     my $devices_to_check = [];
     my $devices = {};
     foreach my $pcibus (@$res) {
@@ -4178,14 +4096,14 @@ sub vm_devices_list {
 	$devices_to_check = $to_check;
     }
 
-    my $resblock = vm_mon_cmd($vmid, 'query-block');
+    my $resblock = mon_cmd($vmid, 'query-block');
     foreach my $block (@$resblock) {
 	if($block->{device} =~ m/^drive-(\S+)/){
 		$devices->{$1} = 1;
 	}
     }
 
-    my $resmice = vm_mon_cmd($vmid, 'query-mice');
+    my $resmice = mon_cmd($vmid, 'query-mice');
     foreach my $mice (@$resmice) {
 	if ($mice->{name} eq 'QEMU HID Tablet') {
 	    $devices->{tablet} = 1;
@@ -4196,7 +4114,7 @@ sub vm_devices_list {
     # for usb devices there is no query-usb
     # but we can iterate over the entries in
     # qom-list path=/machine/peripheral
-    my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
+    my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
     foreach my $per (@$resperipheral) {
 	if ($per->{name} =~ m/^usb\d+$/) {
 	    $devices->{$per->{name}} = 1;
@@ -4209,7 +4127,7 @@ sub vm_devices_list {
 sub vm_deviceplug {
     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
 
-    my $q35 = machine_type_is_q35($conf);
+    my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
 
     my $devices_list = vm_devices_list($vmid);
     return 1 if defined($devices_list->{$deviceid});
@@ -4285,7 +4203,7 @@ sub vm_deviceplug {
 
 	return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
 
-	my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
+	my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
 	my $use_old_bios_files = undef;
 	($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
 
@@ -4377,13 +4295,13 @@ sub qemu_deviceadd {
     $devicefull = "driver=".$devicefull;
     my %options =  split(/[=,]/, $devicefull);
 
-    vm_mon_cmd($vmid, "device_add" , %options);
+    mon_cmd($vmid, "device_add" , %options);
 }
 
 sub qemu_devicedel {
     my ($vmid, $deviceid) = @_;
 
-    my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
+    my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
 }
 
 sub qemu_iothread_add {
@@ -4412,7 +4330,7 @@ sub qemu_iothread_del {
 sub qemu_objectadd {
     my($vmid, $objectid, $qomtype) = @_;
 
-    vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
+    mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
 
     return 1;
 }
@@ -4420,7 +4338,7 @@ sub qemu_objectadd {
 sub qemu_objectdel {
     my($vmid, $objectid) = @_;
 
-    vm_mon_cmd($vmid, "object-del", id => $objectid);
+    mon_cmd($vmid, "object-del", id => $objectid);
 
     return 1;
 }
@@ -4430,7 +4348,7 @@ sub qemu_driveadd {
 
     my $drive = print_drive_full($storecfg, $vmid, $device);
     $drive =~ s/\\/\\\\/g;
-    my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
+    my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
 
     # If the command succeeds qemu prints: "OK"
     return 1 if $ret =~ m/OK/s;
@@ -4441,7 +4359,7 @@ sub qemu_driveadd {
 sub qemu_drivedel {
     my($vmid, $deviceid) = @_;
 
-    my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
+    my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
     $ret =~ s/^\s+//;
 
     return 1 if $ret eq "";
@@ -4551,7 +4469,7 @@ sub qemu_add_pci_bridge {
 sub qemu_set_link_status {
     my ($vmid, $device, $up) = @_;
 
-    vm_mon_cmd($vmid, "set_link", name => $device,
+    mon_cmd($vmid, "set_link", name => $device,
 	       up => $up ? JSON::true : JSON::false);
 }
 
@@ -4561,14 +4479,14 @@ sub qemu_netdevadd {
     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
     my %options =  split(/[=,]/, $netdev);
 
-    vm_mon_cmd($vmid, "netdev_add",  %options);
+    mon_cmd($vmid, "netdev_add",  %options);
     return 1;
 }
 
 sub qemu_netdevdel {
     my ($vmid, $deviceid) = @_;
 
-    vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
+    mon_cmd($vmid, "netdev_del", id => $deviceid);
 }
 
 sub qemu_usb_hotplug {
@@ -4599,7 +4517,7 @@ sub qemu_usb_hotplug {
 sub qemu_cpu_hotplug {
     my ($vmid, $conf, $vcpus) = @_;
 
-    my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
+    my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
 
     my $sockets = 1;
     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
@@ -4616,14 +4534,14 @@ sub qemu_cpu_hotplug {
 
     if ($vcpus < $currentvcpus) {
 
-	if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
+	if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
 
 	    for (my $i = $currentvcpus; $i > $vcpus; $i--) {
 		qemu_devicedel($vmid, "cpu$i");
 		my $retry = 0;
 		my $currentrunningvcpus = undef;
 		while (1) {
-		    $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
+		    $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
 		    last if scalar(@{$currentrunningvcpus}) == $i-1;
 		    raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
 		    $retry++;
@@ -4640,11 +4558,11 @@ sub qemu_cpu_hotplug {
 	return;
     }
 
-    my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
+    my $currentrunningvcpus = mon_cmd($vmid, "query-cpus");
     die "vcpus in running vm does not match its configuration\n"
 	if scalar(@{$currentrunningvcpus}) != $currentvcpus;
 
-    if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
+    if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
 
 	for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
 	    my $cpustr = print_cpu_device($conf, $i);
@@ -4653,7 +4571,7 @@ sub qemu_cpu_hotplug {
 	    my $retry = 0;
 	    my $currentrunningvcpus = undef;
 	    while (1) {
-		$currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
+		$currentrunningvcpus = mon_cmd($vmid, "query-cpus");
 		last if scalar(@{$currentrunningvcpus}) == $i;
 		raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
 		sleep 1;
@@ -4666,7 +4584,7 @@ sub qemu_cpu_hotplug {
     } else {
 
 	for (my $i = $currentvcpus; $i < $vcpus; $i++) {
-	    vm_mon_cmd($vmid, "cpu-add", id => int($i));
+	    mon_cmd($vmid, "cpu-add", id => int($i));
 	}
     }
 }
@@ -4680,7 +4598,7 @@ sub qemu_block_set_io_throttle {
 
     return if !check_running($vmid) ;
 
-    vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
+    mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
 	bps => int($bps),
 	bps_rd => int($bps_rd),
 	bps_wr => int($bps_wr),
@@ -4745,7 +4663,7 @@ sub qemu_block_resize {
 
     return if !$running;
 
-    vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
+    mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
 
 }
 
@@ -4755,7 +4673,7 @@ sub qemu_volume_snapshot {
     my $running = check_running($vmid);
 
     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
-	vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
+	mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
     } else {
 	PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
     }
@@ -4777,7 +4695,7 @@ sub qemu_volume_snapshot_delete {
     }
 
     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
-	vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
+	mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
     } else {
 	PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
     }
@@ -4796,7 +4714,7 @@ sub set_migration_caps {
 	"compress" => 0
     };
 
-    my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
+    my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
 
     for my $supported_capability (@$supported_capabilities) {
 	push @$cap_ref, {
@@ -4805,7 +4723,7 @@ sub set_migration_caps {
 	};
     }
 
-    vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
+    mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
 }
 
 my $fast_plug_option = {
@@ -4818,6 +4736,7 @@ my $fast_plug_option = {
     'protection' => 1,
     'vmstatestorage' => 1,
     'hookscript' => 1,
+    'tags' => 1,
 };
 
 # hotplug changes in [PENDING]
@@ -4828,7 +4747,8 @@ sub vmconfig_hotplug_pending {
     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
 
     my $defaults = load_defaults();
-    my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
+    my $arch = get_vm_arch($conf);
+    my $machine_type = get_vm_machine($conf, undef, $arch);
 
     # commit values which do not have any impact on running VM first
     # Note: those option cannot raise errors, we we do not care about
@@ -4856,9 +4776,9 @@ sub vmconfig_hotplug_pending {
     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
 
     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
-    foreach my $opt (keys %$pending_delete_hash) {
-	my $force = $pending_delete_hash->{$opt}->{force};
+    foreach my $opt (sort keys %$pending_delete_hash) {
 	next if $selection && !$selection->{$opt};
+	my $force = $pending_delete_hash->{$opt}->{force};
 	eval {
 	    if ($opt eq 'hotplug') {
 		die "skip\n" if ($conf->{hotplug} =~ /memory/);
@@ -4886,7 +4806,7 @@ sub vmconfig_hotplug_pending {
 		die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
 		# here we reset the ballooning value to memory
 		my $balloon = $conf->{memory} || $defaults->{memory};
-		vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
+		mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
 	    } elsif ($fast_plug_option->{$opt}) {
 		# do nothing
 	    } elsif ($opt =~ m/^net(\d+)$/) {
@@ -4918,10 +4838,12 @@ sub vmconfig_hotplug_pending {
 	}
     }
 
-    my $apply_pending_cloudinit;
+    my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
     $apply_pending_cloudinit = sub {
+	return if $apply_pending_cloudinit_done; # once is enough
+	$apply_pending_cloudinit_done = 1; # once is enough
+
 	my ($key, $value) = @_;
-	$apply_pending_cloudinit = sub {}; # once is enough
 
 	my @cloudinit_opts = keys %$confdesc_cloudinit;
 	foreach my $opt (keys %{$conf->{pending}}) {
@@ -4970,7 +4892,7 @@ sub vmconfig_hotplug_pending {
 		# allow manual ballooning if shares is set to zero
 		if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
 		    my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
-		    vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
+		    mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
 		}
 	    } elsif ($opt =~ m/^net(\d+)$/) {
 		# some changes can be done without hotplug
@@ -5055,7 +4977,7 @@ sub vmconfig_apply_pending {
     # cold plug
 
     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
-    foreach my $opt (keys %$pending_delete_hash) {
+    foreach my $opt (sort keys %$pending_delete_hash) {
 	die "internal error" if $opt =~ m/^unused/;
 	my $force = $pending_delete_hash->{$opt}->{force};
 	$conf = PVE::QemuConfig->load_config($vmid); # update/reload
@@ -5246,14 +5168,14 @@ sub vmconfig_update_disk {
 	    } else { # cdrom
 
 		if ($drive->{file} eq 'none') {
-		    vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
+		    mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
 		    if (drive_is_cloudinit($old_drive)) {
 			vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
 		    }
 		} else {
 		    my $path = get_iso_path($storecfg, $vmid, $drive->{file});
-		    vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
-		    vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
+		    mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
+		    mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
 		}
 
 		return 1;
@@ -5357,6 +5279,35 @@ sub vm_start {
 
 	my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
 
+	my $migration_ip;
+	my $get_migration_ip = sub {
+	    my ($cidr, $nodename) = @_;
+
+	    return $migration_ip if defined($migration_ip);
+
+	    if (!defined($cidr)) {
+		my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
+		$cidr = $dc_conf->{migration}->{network};
+	    }
+
+	    if (defined($cidr)) {
+		my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
+
+		die "could not get IP: no address configured on local " .
+		    "node for network '$cidr'\n" if scalar(@$ips) == 0;
+
+		die "could not get IP: multiple addresses configured on local " .
+		    "node for network '$cidr'\n" if scalar(@$ips) > 1;
+
+		$migration_ip = @$ips[0];
+	    }
+
+	    $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
+		if !defined($migration_ip);
+
+	    return $migration_ip;
+	};
+
 	my $migrate_uri;
 	if ($statefile) {
 	    if ($statefile eq 'tcp') {
@@ -5373,13 +5324,7 @@ sub vm_start {
 		}
 
 		if ($migration_type eq 'insecure') {
-		    my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
-		    if ($migrate_network_addr) {
-			$localip = $migrate_network_addr;
-		    } else {
-			$localip = PVE::Cluster::remote_node_ip($nodename, 1);
-		    }
-
+		    $localip = $get_migration_ip->($migration_network, $nodename);
 		    $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
 		}
 
@@ -5419,8 +5364,9 @@ sub vm_start {
 	  my $pcidevices = $d->{pciid};
 	  foreach my $pcidevice (@$pcidevices) {
 		my $pciid = $pcidevice->{id};
+		$pciid = "0000:$pciid" if $pciid !~ m/^[0-9a-f]{4}:/;
 
-		my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
+		my $info = PVE::SysFSTools::pci_device_info("$pciid");
 		die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
 		die "no pci device info for device '$pciid'\n" if !$info;
 
@@ -5502,25 +5448,24 @@ sub vm_start {
 	print "migration listens on $migrate_uri\n" if $migrate_uri;
 
 	if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
-	    eval { vm_mon_cmd_nocheck($vmid, "cont"); };
+	    eval { mon_cmd($vmid, "cont"); };
 	    warn $@ if $@;
 	}
 
 	#start nbd server for storage migration
 	if ($targetstorage) {
 	    my $nodename = PVE::INotify::nodename();
-	    my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
-	    my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
+	    my $localip = $get_migration_ip->($migration_network, $nodename);
 	    my $pfamily = PVE::Tools::get_host_address_family($nodename);
 	    my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
 
-	    vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
+	    mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
 
 	    $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
 
 	    foreach my $opt (sort keys %$local_volumes) {
 		my $volid = $local_volumes->{$opt};
-		vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
+		mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
 		my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
 		print "storage migration listens on $migrate_storage_uri volume:$volid\n";
 	    }
@@ -5535,13 +5480,13 @@ sub vm_start {
 	    if ($spice_port) {
 	        print "spice listens on port $spice_port\n";
 		if ($spice_ticket) {
-		    vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
-		    vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
+		    mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
+		    mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
 		}
 	    }
 
 	} else {
-	    vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
+	    mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
 		if !$statefile && $conf->{balloon};
 
 	    foreach my $opt (keys %$conf) {
@@ -5551,7 +5496,7 @@ sub vm_start {
 	    }
 	}
 
-	vm_mon_cmd_nocheck($vmid, 'qom-set',
+	mon_cmd($vmid, 'qom-set',
 		    path => "machine/peripheral/balloon0",
 		    property => "guest-stats-polling-interval",
 		    value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
@@ -5568,69 +5513,19 @@ sub vm_start {
     });
 }
 
-sub vm_mon_cmd {
-    my ($vmid, $execute, %params) = @_;
-
-    my $cmd = { execute => $execute, arguments => \%params };
-    vm_qmp_command($vmid, $cmd);
-}
-
-sub vm_mon_cmd_nocheck {
-    my ($vmid, $execute, %params) = @_;
-
-    my $cmd = { execute => $execute, arguments => \%params };
-    vm_qmp_command($vmid, $cmd, 1);
-}
-
-sub vm_qmp_command {
-    my ($vmid, $cmd, $nocheck) = @_;
-
-    my $res;
-
-    my $timeout;
-    if ($cmd->{arguments}) {
-	$timeout = delete $cmd->{arguments}->{timeout};
-    }
-
-    eval {
-	die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
-	my $sname = qmp_socket($vmid);
-	if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
-	    my $qmpclient = PVE::QMPClient->new();
-
-	    $res = $qmpclient->cmd($vmid, $cmd, $timeout);
-	} else {
-	    die "unable to open monitor socket\n";
-	}
-    };
-    if (my $err = $@) {
-	syslog("err", "VM $vmid qmp command failed - $err");
-	die $err;
-    }
-
-    return $res;
-}
-
-sub vm_human_monitor_command {
-    my ($vmid, $cmdline) = @_;
-
-    my $cmd = {
-	execute => 'human-monitor-command',
-	arguments => { 'command-line' => $cmdline},
-    };
-
-    return vm_qmp_command($vmid, $cmd);
-}
-
 sub vm_commandline {
     my ($storecfg, $vmid, $snapname) = @_;
 
     my $conf = PVE::QemuConfig->load_config($vmid);
+    my $forcemachine;
 
     if ($snapname) {
 	my $snapshot = $conf->{snapshots}->{$snapname};
 	die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
 
+	# check for a 'runningmachine' in snapshot
+	$forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
+
 	$snapshot->{digest} = $conf->{digest}; # keep file digest for API
 
 	$conf = $snapshot;
@@ -5638,7 +5533,7 @@ sub vm_commandline {
 
     my $defaults = load_defaults();
 
-    my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
+    my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
 
     return PVE::Tools::cmd2string($cmd);
 }
@@ -5652,7 +5547,7 @@ sub vm_reset {
 
 	PVE::QemuConfig->check_lock($conf) if !$skiplock;
 
-	vm_mon_cmd($vmid, "system_reset");
+	mon_cmd($vmid, "system_reset");
     });
 }
 
@@ -5735,15 +5630,12 @@ sub _do_vm_stop {
     eval {
 	if ($shutdown) {
 	    if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
-		vm_qmp_command($vmid, {
-			execute => "guest-shutdown",
-			arguments => { timeout => $timeout }
-		    }, $nocheck);
+		mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
 	    } else {
-		vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
+		mon_cmd($vmid, "system_powerdown");
 	    }
 	} else {
-	    vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
+	    mon_cmd($vmid, "quit");
 	}
     };
     my $err = $@;
@@ -5820,15 +5712,22 @@ sub vm_reboot {
     my ($vmid, $timeout) = @_;
 
     PVE::QemuConfig->lock_config($vmid, sub {
+	eval {
 
-	# only reboot if running, as qmeventd starts it again on a stop event
-	return if !check_running($vmid);
+	    # only reboot if running, as qmeventd starts it again on a stop event
+	    return if !check_running($vmid);
 
-	create_reboot_request($vmid);
+	    create_reboot_request($vmid);
 
-	my $storecfg = PVE::Storage::config();
-	_do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
+	    my $storecfg = PVE::Storage::config();
+	    _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
 
+	};
+	if (my $err = $@) {
+	    # avoid that the next normal shutdown will be confused for a reboot
+	    clear_reboot_request($vmid);
+	    die $err;
+	}
    });
 }
 
@@ -5859,7 +5758,7 @@ sub vm_suspend {
 	    $path = PVE::Storage::path($storecfg, $vmstate);
 	    PVE::QemuConfig->write_config($vmid, $conf);
 	} else {
-	    vm_mon_cmd($vmid, "stop");
+	    mon_cmd($vmid, "stop");
 	}
     });
 
@@ -5868,9 +5767,9 @@ sub vm_suspend {
 	PVE::Storage::activate_volumes($storecfg, [$vmstate]);
 
 	eval {
-	    vm_mon_cmd($vmid, "savevm-start", statefile => $path);
+	    mon_cmd($vmid, "savevm-start", statefile => $path);
 	    for(;;) {
-		my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
+		my $state = mon_cmd($vmid, "query-savevm");
 		if (!$state->{status}) {
 		    die "savevm not active\n";
 		} elsif ($state->{status} eq 'active') {
@@ -5893,7 +5792,7 @@ sub vm_suspend {
 	    if ($err) {
 		# cleanup, but leave suspending lock, to indicate something went wrong
 		eval {
-		    vm_mon_cmd($vmid, "savevm-end");
+		    mon_cmd($vmid, "savevm-end");
 		    PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
 		    PVE::Storage::vdisk_free($storecfg, $vmstate);
 		    delete $conf->@{qw(vmstate runningmachine)};
@@ -5906,7 +5805,7 @@ sub vm_suspend {
 	    die "lock changed unexpectedly\n"
 		if !PVE::QemuConfig->has_lock($conf, 'suspending');
 
-	    vm_qmp_command($vmid, { execute => "quit" });
+	    mon_cmd($vmid, "quit");
 	    $conf->{lock} = 'suspended';
 	    PVE::QemuConfig->write_config($vmid, $conf);
 	});
@@ -5917,8 +5816,7 @@ sub vm_resume {
     my ($vmid, $skiplock, $nocheck) = @_;
 
     PVE::QemuConfig->lock_config($vmid, sub {
-	my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
-	my $res = $vm_mon_cmd->($vmid, 'query-status');
+	my $res = mon_cmd($vmid, 'query-status');
 	my $resume_cmd = 'cont';
 
 	if ($res->{status} && $res->{status} eq 'suspended') {
@@ -5933,7 +5831,7 @@ sub vm_resume {
 		if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
 	}
 
-	$vm_mon_cmd->($vmid, $resume_cmd);
+	mon_cmd($vmid, $resume_cmd);
     });
 }
 
@@ -5945,26 +5843,11 @@ sub vm_sendkey {
 	my $conf = PVE::QemuConfig->load_config($vmid);
 
 	# there is no qmp command, so we use the human monitor command
-	my $res = vm_human_monitor_command($vmid, "sendkey $key");
+	my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
 	die $res if $res ne '';
     });
 }
 
-sub vm_destroy {
-    my ($storecfg, $vmid, $skiplock) = @_;
-
-    PVE::QemuConfig->lock_config($vmid, sub {
-
-	my $conf = PVE::QemuConfig->load_config($vmid);
-
-	if (!check_running($vmid)) {
-	    destroy_vm($storecfg, $vmid, undef, $skiplock);
-	} else {
-	    die "VM $vmid is running - destroy failed\n";
-	}
-    });
-}
-
 # vzdump restore implementaion
 
 sub tar_archive_read_firstfile {
@@ -6632,9 +6515,11 @@ sub restore_tar_archive {
 
     my $storecfg = PVE::Storage::config();
 
-    # destroy existing data - keep empty config
+    # avoid zombie disks when restoring over an existing VM -> cleanup first
+    # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
+    # skiplock=1 because qmrestore has set the 'create' lock itself already
     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
-    destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
+    destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
 
     my $tocmd = "/usr/lib/qemu-server/qmextract";
 
@@ -6712,14 +6597,9 @@ sub restore_tar_archive {
 	$srcfd->close();
 	$outfd->close();
     };
-    my $err = $@;
-
-    if ($err) {
-
+    if (my $err = $@) {
 	unlink $tmpfn;
-
 	tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
-
 	die $err;
     }
 
@@ -6754,6 +6634,9 @@ sub foreach_storage_used_by_vm {
     }
 }
 
+my $qemu_snap_storage = {
+    rbd => 1,
+};
 sub do_snapshots_with_qemu {
     my ($storecfg, $volid) = @_;
 
@@ -6774,7 +6657,7 @@ sub do_snapshots_with_qemu {
 sub qga_check_running {
     my ($vmid, $nowarn) = @_;
 
-    eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
+    eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
     if ($@) {
 	warn "Qemu Guest Agent is not running - $@" if !$nowarn;
 	return 0;
@@ -6946,7 +6829,7 @@ sub qemu_drive_mirror {
     }
 
     # if a job already runs for this device we get an error, catch it for cleanup
-    eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
+    eval { mon_cmd($vmid, "drive-mirror", %$opts); };
     if (my $err = $@) {
 	eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
 	warn "$@\n" if $@;
@@ -6965,7 +6848,7 @@ sub qemu_drive_mirror_monitor {
 	while (1) {
 	    die "storage migration timed out\n" if $err_complete > 300;
 
-	    my $stats = vm_mon_cmd($vmid, "query-block-jobs");
+	    my $stats = mon_cmd($vmid, "query-block-jobs");
 
 	    my $running_mirror_jobs = {};
 	    foreach my $stat (@$stats) {
@@ -7008,7 +6891,7 @@ sub qemu_drive_mirror_monitor {
 		    my $agent_running = $qga && qga_check_running($vmid);
 		    if ($agent_running) {
 			print "freeze filesystem\n";
-			eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
+			eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
 		    } else {
 			print "suspend vm\n";
 			eval { PVE::QemuServer::vm_suspend($vmid, 1); };
@@ -7019,7 +6902,7 @@ sub qemu_drive_mirror_monitor {
 
 		    if ($agent_running) {
 			print "unfreeze filesystem\n";
-			eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
+			eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
 		    } else {
 			print "resume vm\n";
 			eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
@@ -7032,7 +6915,7 @@ sub qemu_drive_mirror_monitor {
 			# try to switch the disk if source and destination are on the same guest
 			print "$job: Completing block job...\n";
 
-			eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
+			eval { mon_cmd($vmid, "block-job-complete", device => $job) };
 			if ($@ =~ m/cannot be completed/) {
 			    print "$job: Block job cannot be completed, try again.\n";
 			    $err_complete++;
@@ -7060,12 +6943,12 @@ sub qemu_blockjobs_cancel {
 
     foreach my $job (keys %$jobs) {
 	print "$job: Cancelling block job\n";
-	eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
+	eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
 	$jobs->{$job}->{cancel} = 1;
     }
 
     while (1) {
-	my $stats = vm_mon_cmd($vmid, "query-block-jobs");
+	my $stats = mon_cmd($vmid, "query-block-jobs");
 
 	my $running_jobs = {};
 	foreach my $stat (@$stats) {
@@ -7106,11 +6989,21 @@ sub clone_disk {
 
 	print "create full clone of drive $drivename ($drive->{file})\n";
 	my $name = undef;
+	if (drive_is_cloudinit($drive)) {
+	    $name = "vm-$newvmid-cloudinit";
+	    $name .= ".$dst_format" if $dst_format ne 'raw';
+	    $snapname = undef;
+	    $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
+	}
 	$newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
 	push @$newvollist, $newvolid;
 
 	PVE::Storage::activate_volumes($storecfg, [$newvolid]);
 
+	if (drive_is_cloudinit($drive)) {
+	    goto no_data_clone;
+	}
+
 	my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
 	if (!$running || $snapname) {
 	    # TODO: handle bwlimits
@@ -7118,7 +7011,7 @@ sub clone_disk {
 	} else {
 
 	    my $kvmver = get_running_qemu_version ($vmid);
-	    if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
+	    if (!min_version($kvmver, 2, 7)) {
 		die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
 		    if $drive->{iothread};
 	    }
@@ -7127,6 +7020,7 @@ sub clone_disk {
 	}
     }
 
+no_data_clone:
     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
 
     my $disk = $drive;
@@ -7137,64 +7031,12 @@ sub clone_disk {
     return $disk;
 }
 
-# this only works if VM is running
-sub get_current_qemu_machine {
-    my ($vmid) = @_;
-
-    my $cmd = { execute => 'query-machines', arguments => {} };
-    my $res = vm_qmp_command($vmid, $cmd);
-
-    my ($current, $default);
-    foreach my $e (@$res) {
-	$default = $e->{name} if $e->{'is-default'};
-	$current = $e->{name} if $e->{'is-current'};
-    }
-
-    # fallback to the default machine if current is not supported by qemu
-    return $current || $default || 'pc';
-}
-
 sub get_running_qemu_version {
     my ($vmid) = @_;
-    my $cmd = { execute => 'query-version', arguments => {} };
-    my $res = vm_qmp_command($vmid, $cmd);
+    my $res = mon_cmd($vmid, "query-version");
     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
 }
 
-sub qemu_machine_feature_enabled {
-    my ($machine, $kvmver, $version_major, $version_minor) = @_;
-
-    my $current_major;
-    my $current_minor;
-
-    if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
-
-	$current_major = $3;
-	$current_minor = $4;
-
-    } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
-
-	$current_major = $1;
-	$current_minor = $2;
-    }
-
-    return 1 if $current_major > $version_major ||
-                ($current_major == $version_major &&
-                 $current_minor >= $version_minor);
-}
-
-sub qemu_machine_pxe {
-    my ($vmid, $conf) = @_;
-
-    my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
-
-    if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
-	$machine .= '.pxe';
-    }
-
-    return $machine;
-}
-
 sub qemu_use_old_bios_files {
     my ($machine_type) = @_;
 
@@ -7206,12 +7048,12 @@ sub qemu_use_old_bios_files {
         $machine_type = $1;
         $use_old_bios_files = 1;
     } else {
-	my $kvmver = kvm_user_version();
+	my $version = PVE::QemuServer::Machine::extract_version($machine_type) // kvm_user_version();
         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
         # load new efi bios files on migration. So this hack is required to allow
         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
-	$use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
+	$use_old_bios_files = !min_version($version, 2, 4);
     }
 
     return ($use_old_bios_files, $machine_type);
@@ -7236,7 +7078,7 @@ sub create_efidisk($$$$$) {
 sub vm_iothreads_list {
     my ($vmid) = @_;
 
-    my $res = vm_mon_cmd($vmid, 'query-iothreads');
+    my $res = mon_cmd($vmid, 'query-iothreads');
 
     my $iothreads = {};
     foreach my $iothread (@$res) {
@@ -7266,7 +7108,7 @@ sub scsihw_infos {
 }
 
 sub add_hyperv_enlightenments {
-    my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
+    my ($cpuFlags, $winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
 
     return if $winversion < 6;
     return if $bios && $bios eq 'ovmf' && $winversion < 8;
@@ -7276,7 +7118,7 @@ sub add_hyperv_enlightenments {
 	push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
     }
 
-    if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
+    if (min_version($machine_version, 2, 3)) {
 	push @$cpuFlags , 'hv_spinlocks=0x1fff';
 	push @$cpuFlags , 'hv_vapic';
 	push @$cpuFlags , 'hv_time';
@@ -7284,7 +7126,7 @@ sub add_hyperv_enlightenments {
 	push @$cpuFlags , 'hv_spinlocks=0xffff';
     }
 
-    if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
+    if (min_version($machine_version, 2, 6)) {
 	push @$cpuFlags , 'hv_reset';
 	push @$cpuFlags , 'hv_vpindex';
 	push @$cpuFlags , 'hv_runtime';
@@ -7293,12 +7135,12 @@ sub add_hyperv_enlightenments {
     if ($winversion >= 7) {
 	push @$cpuFlags , 'hv_relaxed';
 
-	if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
+	if (min_version($machine_version, 2, 12)) {
 	    push @$cpuFlags , 'hv_synic';
 	    push @$cpuFlags , 'hv_stimer';
 	}
 
-	if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
+	if (min_version($machine_version, 3, 1)) {
 	    push @$cpuFlags , 'hv_ipi';
 	}
     }
@@ -7369,7 +7211,7 @@ sub generate_smbios1_uuid {
 sub nbd_stop {
     my ($vmid) = @_;
 
-    vm_mon_cmd($vmid, 'nbd-server-stop');
+    mon_cmd($vmid, 'nbd-server-stop');
 }
 
 sub create_reboot_request {
@@ -7464,4 +7306,22 @@ sub complete_storage {
     return $res;
 }
 
+sub complete_migration_storage {
+    my ($cmd, $param, $current_value, $all_args) = @_;
+
+    my $targetnode = @$all_args[1];
+
+    my $cfg = PVE::Storage::config();
+    my $ids = $cfg->{ids};
+
+    my $res = [];
+    foreach my $sid (keys %$ids) {
+	next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
+	next if !$ids->{$sid}->{content}->{images};
+	push @$res, $sid;
+    }
+
+    return $res;
+}
+
 1;