X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=PVE%2FQemuServer.pm;h=48657bf82a63a060d84a8c57d1c27d1514b89414;hb=5da072fb8418906fd58524a7a87d677d1f13cff9;hp=d8c987456baed795915e3058a350f4d9877e2843;hpb=dc62a7fafc08596f866f2b33b7d40231bb506761;p=qemu-server.git

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index d8c98745..48657bf8 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -3,44 +3,47 @@ package PVE::QemuServer;
 use strict;
 use warnings;
 
-use POSIX;
-use IO::Handle;
-use IO::Select;
-use IO::File;
-use IO::Dir;
-use IO::Socket::UNIX;
+use Cwd 'abs_path';
+use Digest::SHA;
+use Fcntl ':flock';
+use Fcntl;
 use File::Basename;
+use File::Copy qw(copy);
 use File::Path;
 use File::stat;
 use Getopt::Long;
-use Digest::SHA;
-use Fcntl ':flock';
-use Cwd 'abs_path';
+use IO::Dir;
+use IO::File;
+use IO::Handle;
+use IO::Select;
+use IO::Socket::UNIX;
 use IPC::Open3;
 use JSON;
-use Fcntl;
-use PVE::SafeSyslog;
+use MIME::Base64;
+use POSIX;
 use Storable qw(dclone);
-use PVE::Exception qw(raise raise_param_exc);
-use PVE::Storage;
-use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
-use PVE::JSONSchema qw(get_standard_option);
+use Time::HiRes qw(gettimeofday);
+use URI::Escape;
+
 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
+use PVE::Exception qw(raise raise_param_exc);
+use PVE::GuestHelpers;
 use PVE::INotify;
+use PVE::JSONSchema qw(get_standard_option);
 use PVE::ProcFSTools;
-use PVE::QemuConfig;
-use PVE::QMPClient;
 use PVE::RPCEnvironment;
-use PVE::GuestHelpers;
-use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
-use PVE::QemuServer::Memory;
-use PVE::QemuServer::USB qw(parse_usb_device);
-use PVE::QemuServer::Cloudinit;
+use PVE::SafeSyslog;
+use PVE::Storage;
 use PVE::SysFSTools;
 use PVE::Systemd;
-use Time::HiRes qw(gettimeofday);
-use File::Copy qw(copy);
-use URI::Escape;
+use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
+
+use PVE::QMPClient;
+use PVE::QemuConfig;
+use PVE::QemuServer::Cloudinit;
+use PVE::QemuServer::Memory;
+use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
+use PVE::QemuServer::USB qw(parse_usb_device);
 
 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
 my $OVMF = {
@@ -145,6 +148,9 @@ my $cpu_vendor_list = {
     'Skylake-Client-IBRS' => 'GenuineIntel',
     'Skylake-Server' => 'GenuineIntel',
     'Skylake-Server-IBRS' => 'GenuineIntel',
+    'Cascadelake-Server' => 'GenuineIntel',
+    KnightsMill => 'GenuineIntel',
+
 
     # AMD CPUs
     athlon => 'AuthenticAMD',
@@ -166,7 +172,21 @@ my $cpu_vendor_list = {
     max => 'default',
 };
 
-my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
+my @supported_cpu_flags = (
+    'pcid',
+    'spec-ctrl',
+    'ibpb',
+    'ssbd',
+    'virt-ssbd',
+    'amd-ssbd',
+    'amd-no-ssb',
+    'pdpe1gb',
+    'md-clear',
+    'hv-tlbflush',
+    'hv-evmcs',
+    'aes'
+);
+my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 
 my $cpu_fmt = {
     cputype => {
@@ -192,7 +212,7 @@ my $cpu_fmt = {
     flags => {
 	description => "List of additional CPU flags separated by ';'."
 		     . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
-		     . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
+		     . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 	format_description => '+FLAG[;-FLAG...]',
 	type => 'string',
 	pattern => qr/$cpu_flag(;$cpu_flag)*/,
@@ -266,6 +286,37 @@ my $ivshmem_fmt = {
     },
 };
 
+my $audio_fmt = {
+    device => {
+	type => 'string',
+	enum => [qw(ich9-intel-hda intel-hda AC97)],
+	description =>  "Configure an audio device."
+    },
+    driver =>  {
+	type => 'string',
+	enum => ['spice'],
+	default => 'spice',
+	optional => 1,
+	description => "Driver backend for the audio device."
+    },
+};
+
+my $spice_enhancements_fmt = {
+    foldersharing => {
+	type => 'boolean',
+	optional => 1,
+	default => '0',
+	description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
+    },
+    videostreaming =>  {
+	type => 'string',
+	enum => ['off', 'all', 'filter'],
+	default => 'off',
+	optional => 1,
+	description => "Enable video streaming. Uses compression for detected video streams."
+    },
+};
+
 my $confdesc = {
     onboot => {
 	optional => 1,
@@ -591,7 +642,7 @@ EODESCR
     smbios1 => {
 	description => "Specify SMBIOS type 1 fields.",
 	type => 'string', format => 'pve-qm-smbios1',
-	maxLength => 256,
+	maxLength => 512,
 	optional => 1,
     },
     protection => {
@@ -636,7 +687,19 @@ EODESCR
 	format => $ivshmem_fmt,
 	description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 	optional => 1,
-    }
+    },
+    audio0 => {
+	type => 'string',
+	format => $audio_fmt,
+	description => "Configure a audio device, useful in combination with QXL/Spice.",
+	optional => 1
+    },
+    spice_enhancements => {
+	type => 'string',
+	format => $spice_enhancements_fmt,
+	description => "Configure additional enhancements for SPICE.",
+	optional => 1
+    },
 };
 
 my $cicustom_fmt = {
@@ -733,7 +796,7 @@ my $MAX_SATA_DISKS = 6;
 my $MAX_USB_DEVICES = 5;
 my $MAX_NETS = 32;
 my $MAX_UNUSED_DISKS = 256;
-my $MAX_HOSTPCI_DEVICES = 4;
+my $MAX_HOSTPCI_DEVICES = 16;
 my $MAX_SERIAL_PORTS = 4;
 my $MAX_PARALLEL_PORTS = 3;
 my $MAX_NUMA = 8;
@@ -1261,7 +1324,7 @@ EODESCR
     usb3 => {
 	optional => 1,
 	type => 'boolean',
-	description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
+	description => "Specifies whether if given host option is a USB3 device or port.",
         default => 0,
     },
 };
@@ -1436,25 +1499,33 @@ sub kvm_version {
     return $kvm_api_version;
 }
 
-my $kvm_user_version;
+my $kvm_user_version = {};
+my $kvm_mtime = {};
 
 sub kvm_user_version {
+    my ($binary) = @_;
 
-    return $kvm_user_version if $kvm_user_version;
+    $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
+    my $st = stat($binary);
 
-    $kvm_user_version = 'unknown';
+    my $cachedmtime = $kvm_mtime->{$binary} // -1;
+    return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
+	$cachedmtime == $st->mtime;
+
+    $kvm_user_version->{$binary} = 'unknown';
+    $kvm_mtime->{$binary} = $st->mtime;
 
     my $code = sub {
 	my $line = shift;
 	if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
-	    $kvm_user_version = $2;
+	    $kvm_user_version->{$binary} = $2;
 	}
     };
 
-    eval { run_command("kvm -version", outfunc => $code); };
+    eval { run_command([$binary, '--version'], outfunc => $code); };
     warn $@ if $@;
 
-    return $kvm_user_version;
+    return $kvm_user_version->{$binary};
 
 }
 
@@ -2358,7 +2429,7 @@ sub vmconfig_cleanup_pending {
     return $changes;
 }
 
-# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
+# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
 my $smbios1_fmt = {
     uuid => {
 	type => 'string',
@@ -2369,46 +2440,51 @@ my $smbios1_fmt = {
     },
     version => {
 	type => 'string',
-	pattern => '\S+',
-	format_description => 'string',
+	pattern => '[A-Za-z0-9+\/]+={0,2}',
+	format_description => 'Base64 encoded string',
         description => "Set SMBIOS1 version.",
 	optional => 1,
     },
     serial => {
 	type => 'string',
-	pattern => '\S+',
-	format_description => 'string',
+	pattern => '[A-Za-z0-9+\/]+={0,2}',
+	format_description => 'Base64 encoded string',
         description => "Set SMBIOS1 serial number.",
 	optional => 1,
     },
     manufacturer => {
 	type => 'string',
-	pattern => '\S+',
-	format_description => 'string',
+	pattern => '[A-Za-z0-9+\/]+={0,2}',
+	format_description => 'Base64 encoded string',
         description => "Set SMBIOS1 manufacturer.",
 	optional => 1,
     },
     product => {
 	type => 'string',
-	pattern => '\S+',
-	format_description => 'string',
+	pattern => '[A-Za-z0-9+\/]+={0,2}',
+	format_description => 'Base64 encoded string',
         description => "Set SMBIOS1 product ID.",
 	optional => 1,
     },
     sku => {
 	type => 'string',
-	pattern => '\S+',
-	format_description => 'string',
+	pattern => '[A-Za-z0-9+\/]+={0,2}',
+	format_description => 'Base64 encoded string',
         description => "Set SMBIOS1 SKU string.",
 	optional => 1,
     },
     family => {
 	type => 'string',
-	pattern => '\S+',
-	format_description => 'string',
+	pattern => '[A-Za-z0-9+\/]+={0,2}',
+	format_description => 'Base64 encoded string',
         description => "Set SMBIOS1 family string.",
 	optional => 1,
     },
+    base64 => {
+	type => 'boolean',
+	description => 'Flag to indicate that the SMBIOS values are base64 encoded',
+	optional => 1,
+    },
 };
 
 sub parse_smbios1 {
@@ -2545,8 +2621,6 @@ sub touch_config {
 sub destroy_vm {
     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
 
-    my $conffile = PVE::QemuConfig->config_file($vmid);
-
     my $conf = PVE::QemuConfig->load_config($vmid);
 
     PVE::QemuConfig->check_lock($conf) if !$skiplock;
@@ -2589,9 +2663,9 @@ sub destroy_vm {
     });
 
     if ($keep_empty_config) {
-	PVE::Tools::file_set_contents($conffile, "memory: 128\n");
+	PVE::QemuConfig->write_config($vmid, "memory: 128\n");
     } else {
-	unlink $conffile;
+	PVE::QemuConfig->destroy_config($vmid);
     }
 
     # also remove unused disk
@@ -2854,7 +2928,7 @@ sub check_local_resources {
     push @loc_res, "ivshmem" if $conf->{ivshmem};
 
     foreach my $k (keys %$conf) {
-	next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
+	next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
 	# sockets are safe: they will recreated be on the target side post-migrate
 	next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
 	push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
@@ -2918,6 +2992,45 @@ sub shared_nodes {
     return $nodehash
 }
 
+sub check_local_storage_availability {
+    my ($conf, $storecfg) = @_;
+
+    my $nodelist = PVE::Cluster::get_nodelist();
+    my $nodehash = { map { $_ => {} } @$nodelist };
+
+    foreach_drive($conf, sub {
+	my ($ds, $drive) = @_;
+
+	my $volid = $drive->{file};
+	return if !$volid;
+
+	my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
+	if ($storeid) {
+	    my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+
+	    if ($scfg->{disable}) {
+		foreach my $node (keys %$nodehash) {
+		    $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
+		}
+	    } elsif (my $avail = $scfg->{nodes}) {
+		foreach my $node (keys %$nodehash) {
+		    if (!$avail->{$node}) {
+			$nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
+		    }
+		}
+	    }
+	}
+    });
+
+    foreach my $node (values %$nodehash) {
+	if (my $unavail = $node->{unavailable_storages}) {
+	    $node->{unavailable_storages} = [ sort keys %$unavail ];
+	}
+    }
+
+    return $nodehash
+}
+
 sub check_cmdline {
     my ($pidfile, $pid) = @_;
 
@@ -3295,7 +3408,7 @@ sub foreach_volid {
     my $volhash = {};
 
     my $test_volid = sub {
-	my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
+	my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
 
 	return if !$volid;
 
@@ -3313,11 +3426,12 @@ sub foreach_volid {
 
 	$volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
 	    if defined($snapname);
+	$volhash->{$volid}->{size} = $size if $size;
     };
 
     foreach_drive($conf, sub {
 	my ($ds, $drive) = @_;
-	$test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
+	$test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
     });
 
     foreach my $snapname (keys %{$conf->{snapshots}}) {
@@ -3346,6 +3460,24 @@ sub conf_has_serial {
     return 0;
 }
 
+sub conf_has_audio {
+    my ($conf, $id) = @_;
+
+    $id //= 0;
+    my $audio = $conf->{"audio$id"};
+    return undef if !defined($audio);
+
+    my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
+    my $audiodriver = $audioproperties->{driver} // 'spice';
+
+    return {
+	dev => $audioproperties->{device},
+	dev_id => "audiodev$id",
+	backend => $audiodriver,
+	backend_id => "$audiodriver-backend${id}",
+    };
+}
+
 sub vga_conf_has_spice {
     my ($vga) = @_;
 
@@ -3468,13 +3600,14 @@ sub config_to_command {
     my $devices = [];
     my $pciaddr = '';
     my $bridges = {};
-    my $kvmver = kvm_user_version();
     my $vernum = 0; # unknown
     my $ostype = $conf->{ostype};
     my $winversion = windows_version($ostype);
     my $kvm = $conf->{kvm};
 
     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
+    my $kvm_binary = get_command_for_arch($arch);
+    my $kvmver = kvm_user_version($kvm_binary);
     $kvm //= 1 if is_native($arch);
 
     if ($kvm) {
@@ -3490,8 +3623,6 @@ sub config_to_command {
 
     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
 
-    my $have_ovz = -f '/proc/vz/vestat';
-
     my $q35 = machine_type_is_q35($conf);
     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
     my $use_old_bios_files = undef;
@@ -3500,7 +3631,7 @@ sub config_to_command {
     my $cpuunits = defined($conf->{cpuunits}) ?
             $conf->{cpuunits} : $defaults->{cpuunits};
 
-    push @$cmd, get_command_for_arch($arch);
+    push @$cmd, $kvm_binary;
 
     push @$cmd, '-id', $vmid;
 
@@ -3524,7 +3655,26 @@ sub config_to_command {
     push @$cmd, '-daemonize';
 
     if ($conf->{smbios1}) {
-	push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
+	my $smbios_conf = parse_smbios1($conf->{smbios1});
+	if ($smbios_conf->{base64}) {
+	    # Do not pass base64 flag to qemu
+	    delete $smbios_conf->{base64};
+	    my $smbios_string = "";
+	    foreach my $key (keys %$smbios_conf) {
+		my $value;
+		if ($key eq "uuid") {
+		    $value = $smbios_conf->{uuid}
+		} else {
+		    $value = decode_base64($smbios_conf->{$key});
+		}
+		# qemu accepts any binary data, only commas need escaping by double comma
+		$value =~ s/,/,,/g;
+		$smbios_string .= "," . $key . "=" . $value if $value;
+	    }
+	    push @$cmd, '-smbios', "type=1" . $smbios_string;
+	} else {
+	    push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
+	}
     }
 
     if ($conf->{vmgenid}) {
@@ -3563,6 +3713,15 @@ sub config_to_command {
 	push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
     }
 
+    # load q35 config
+    if ($q35) {
+	# we use different pcie-port hardware for qemu >= 4.0 for passthrough
+	if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
+	    push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
+	} else {
+	    push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
+	}
+    }
 
     # add usb controllers
     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
@@ -3603,67 +3762,66 @@ sub config_to_command {
 
     # host pci devices
     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
-	my $d = parse_hostpci($conf->{"hostpci$i"});
+	my $id = "hostpci$i";
+	my $d = parse_hostpci($conf->{$id});
 	next if !$d;
 
-	my $pcie = $d->{pcie};
-	if ($pcie) {
+	if (my $pcie = $d->{pcie}) {
 	    die "q35 machine model is not enabled" if !$q35;
 	    # win7 wants to have the pcie devices directly on the pcie bus
 	    # instead of in the root port
 	    if ($winversion == 7) {
-		$pciaddr = print_pcie_addr("hostpci${i}bus0");
+		$pciaddr = print_pcie_addr("${id}bus0");
 	    } else {
-		$pciaddr = print_pcie_addr("hostpci$i");
+		# add more root ports if needed, 4 are present by default
+		# by pve-q35 cfgs, rest added here on demand.
+		if ($i > 3) {
+		    push @$devices, '-device', print_pcie_root_port($i);
+		}
+		$pciaddr = print_pcie_addr($id);
 	    }
 	} else {
-	    $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
+	    $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
 	}
 
-	my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
-	my $romfile = $d->{romfile};
-
 	my $xvga = '';
 	if ($d->{'x-vga'}) {
-	    $xvga = ',x-vga=on';
+	    $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
 	    $kvm_off = 1;
 	    $vga->{type} = 'none' if !defined($conf->{vga});
 	    $gpu_passthrough = 1;
-
-	    if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
-		$xvga = "";
-	    }
 	}
+
 	my $pcidevices = $d->{pciid};
 	my $multifunction = 1 if @$pcidevices > 1;
+
 	my $sysfspath;
 	if ($d->{mdev} && scalar(@$pcidevices) == 1) {
-	    my $id = $pcidevices->[0]->{id};
+	    my $pci_id = $pcidevices->[0]->{id};
 	    my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
-	    $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
+	    $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
 	} elsif ($d->{mdev}) {
-	    warn "ignoring mediated device with multifunction device\n";
+	    warn "ignoring mediated device '$id' with multifunction device\n";
 	}
 
 	my $j=0;
-        foreach my $pcidevice (@$pcidevices) {
-
-	    my $id = "hostpci$i";
-	    $id .= ".$j" if $multifunction;
-	    my $addr = $pciaddr;
-	    $addr .= ".$j" if $multifunction;
+	foreach my $pcidevice (@$pcidevices) {
 	    my $devicestr = "vfio-pci";
+
 	    if ($sysfspath) {
 		$devicestr .= ",sysfsdev=$sysfspath";
 	    } else {
 		$devicestr .= ",host=$pcidevice->{id}";
 	    }
-	    $devicestr .= ",id=$id$addr";
 
-	    if($j == 0){
-		$devicestr .= "$rombar$xvga";
+	    my $mf_addr = $multifunction ? ".$j" : '';
+	    $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
+
+	    if ($j == 0) {
+		$devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
+		$devicestr .= "$xvga";
 		$devicestr .= ",multifunction=on" if $multifunction;
-		$devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
+		$devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
 	    }
 
 	    push @$devices, '-device', $devicestr;
@@ -3672,7 +3830,10 @@ sub config_to_command {
     }
 
     # usb devices
-    my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
+    my $usb_dev_features = {};
+    $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
+
+    my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
     push @$devices, @usbdevices if @usbdevices;
     # serial devices
     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
@@ -3706,6 +3867,23 @@ sub config_to_command {
 	}
     }
 
+    if (my $audio = conf_has_audio($conf)) {
+
+	my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
+
+	my $id = $audio->{dev_id};
+	if ($audio->{dev} eq 'AC97') {
+	    push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
+	} elsif ($audio->{dev} =~ /intel\-hda$/) {
+	    push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
+	    push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
+	    push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
+	} else {
+	    die "unkown audio device '$audio->{dev}', implement me!";
+	}
+
+	push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
+    }
 
     my $sockets = 1;
     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
@@ -3789,7 +3967,7 @@ sub config_to_command {
 	push @$machineFlags, "type=${machine_type}";
     }
 
-    if ($conf->{startdate}) {
+    if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
 	push @$rtcFlags, "base=$conf->{startdate}";
     } elsif ($useLocaltime) {
 	push @$rtcFlags, 'base=localtime';
@@ -3803,11 +3981,6 @@ sub config_to_command {
 
     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
 
-    # enable sound
-    #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
-    #push @$cmd, '-soundhw', 'es1370';
-    #push @$cmd, '-soundhw', $soundhw if $soundhw;
-
     if (parse_guest_agent($conf)->{enabled}) {
 	my $qgasocket = qmp_socket($vmid, 1);
 	my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
@@ -3842,14 +4015,23 @@ sub config_to_command {
 	my $pfamily = PVE::Tools::get_host_address_family($nodename);
 	my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
 	die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
-	my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
-	$spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
-
-	push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
 
 	push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
 	push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
 	push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
+
+	my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
+	$spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
+
+	my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
+	if ($spice_enhancement->{foldersharing}) {
+	    push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
+	    push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
+	}
+
+	my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
+	$spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
+	push @$devices, '-spice', "$spice_opts";
     }
 
     # enable balloon by default, unless explicitly disabled
@@ -3985,7 +4167,7 @@ sub config_to_command {
 
 	$bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
 
-	while (my ($k, $v) = each %$bridges) {
+	for my $k (sort {$b cmp $a} keys %$bridges) {
 	    $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
 	    unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
 	}
@@ -4001,7 +4183,7 @@ sub config_to_command {
 
     if (my $vmstate = $conf->{vmstate}) {
 	my $statepath = PVE::Storage::path($storecfg, $vmstate);
-	PVE::Storage::activate_volumes($storecfg, [$vmstate]);
+	push @$vollist, $statepath;
 	push @$cmd, '-loadstate', $statepath;
     }
 
@@ -5162,6 +5344,10 @@ sub vm_start {
 
 	die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
 
+	# clean up leftover reboot request files
+	eval { clear_reboot_request($vmid); };
+	warn $@ if $@;
+
 	if (!$statefile && scalar(keys %{$conf->{pending}})) {
 	    vmconfig_apply_pending($vmid, $conf, $storecfg);
 	    $conf = PVE::QemuConfig->load_config($vmid); # update/reload
@@ -5232,7 +5418,6 @@ sub vm_start {
 
 	my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
 
-	my $migrate_port = 0;
 	my $migrate_uri;
 	if ($statefile) {
 	    if ($statefile eq 'tcp') {
@@ -5260,7 +5445,7 @@ sub vm_start {
 		}
 
 		my $pfamily = PVE::Tools::get_host_address_family($nodename);
-		$migrate_port = PVE::Tools::next_migrate_port($pfamily);
+		my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
 		$migrate_uri = "tcp:${localip}:${migrate_port}";
 		push @$cmd, '-incoming', $migrate_uri;
 		push @$cmd, '-S';
@@ -5277,8 +5462,12 @@ sub vm_start {
 		push @$cmd, '-incoming', $migrate_uri;
 		push @$cmd, '-S';
 
-	    } else {
+	    } elsif (-e $statefile) {
 		push @$cmd, '-loadstate', $statefile;
+	    } else {
+		my $statepath = PVE::Storage::path($storecfg, $statefile);
+		push @$vollist, $statepath;
+		push @$cmd, '-loadstate', $statepath;
 	    }
 	} elsif ($paused) {
 	    push @$cmd, '-S';
@@ -5310,12 +5499,13 @@ sub vm_start {
 
 	PVE::Storage::activate_volumes($storecfg, $vollist);
 
-	if (-d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
-	    eval {
-		run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
-		    outfunc => sub {}, errfunc => sub {});
-	    };
-	}
+	eval {
+	    run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
+		outfunc => sub {}, errfunc => sub {});
+	};
+	# Issues with the above 'stop' not being fully completed are extremely rare, a very low
+	# timeout should be more than enough here...
+	PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
 
 	my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
 	                                          : $defaults->{cpuunits};
@@ -5383,16 +5573,16 @@ sub vm_start {
 	    my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
 	    my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
 	    my $pfamily = PVE::Tools::get_host_address_family($nodename);
-	    $migrate_port = PVE::Tools::next_migrate_port($pfamily);
+	    my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
 
-	    vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
+	    vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
 
 	    $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
 
 	    foreach my $opt (sort keys %$local_volumes) {
 		my $volid = $local_volumes->{$opt};
 		vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
-		my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
+		my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
 		print "storage migration listens on $migrate_storage_uri volume:$volid\n";
 	    }
 	}
@@ -5459,9 +5649,8 @@ sub vm_qmp_command {
     my $res;
 
     my $timeout;
-    if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
-	$timeout = $cmd->{arguments}->{timeout};
-	delete $cmd->{arguments}->{timeout};
+    if ($cmd->{arguments}) {
+	$timeout = delete $cmd->{arguments}->{timeout};
     }
 
     eval {
@@ -5486,8 +5675,6 @@ sub vm_qmp_command {
 sub vm_human_monitor_command {
     my ($vmid, $cmdline) = @_;
 
-    my $res;
-
     my $cmd = {
 	execute => 'human-monitor-command',
 	arguments => { 'command-line' => $cmdline},
@@ -5588,7 +5775,88 @@ sub vm_stop_cleanup {
     warn $@ if $@; # avoid errors - just warn
 }
 
-# Note: use $nockeck to skip tests if VM configuration file exists.
+# call only in locked context
+sub _do_vm_stop {
+    my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
+
+    my $pid = check_running($vmid, $nocheck);
+    return if !$pid;
+
+    my $conf;
+    if (!$nocheck) {
+	$conf = PVE::QemuConfig->load_config($vmid);
+	PVE::QemuConfig->check_lock($conf) if !$skiplock;
+	if (!defined($timeout) && $shutdown && $conf->{startup}) {
+	    my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
+	    $timeout = $opts->{down} if $opts->{down};
+	}
+	PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
+    }
+
+    eval {
+	if ($shutdown) {
+	    if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
+		vm_qmp_command($vmid, {
+			execute => "guest-shutdown",
+			arguments => { timeout => $timeout }
+		    }, $nocheck);
+	    } else {
+		vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
+	    }
+	} else {
+	    vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
+	}
+    };
+    my $err = $@;
+
+    if (!$err) {
+	$timeout = 60 if !defined($timeout);
+
+	my $count = 0;
+	while (($count < $timeout) && check_running($vmid, $nocheck)) {
+	    $count++;
+	    sleep 1;
+	}
+
+	if ($count >= $timeout) {
+	    if ($force) {
+		warn "VM still running - terminating now with SIGTERM\n";
+		kill 15, $pid;
+	    } else {
+		die "VM quit/powerdown failed - got timeout\n";
+	    }
+	} else {
+	    vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
+	    return;
+	}
+    } else {
+	if ($force) {
+	    warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
+	    kill 15, $pid;
+	} else {
+	    die "VM quit/powerdown failed\n";
+	}
+    }
+
+    # wait again
+    $timeout = 10;
+
+    my $count = 0;
+    while (($count < $timeout) && check_running($vmid, $nocheck)) {
+	$count++;
+	sleep 1;
+    }
+
+    if ($count >= $timeout) {
+	warn "VM still running - terminating now with SIGKILL\n";
+	kill 9, $pid;
+	sleep 1;
+    }
+
+    vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
+}
+
+# Note: use $nocheck to skip tests if VM configuration file exists.
 # We need that when migration VMs to other nodes (files already moved)
 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
 sub vm_stop {
@@ -5605,79 +5873,23 @@ sub vm_stop {
     }
 
     PVE::QemuConfig->lock_config($vmid, sub {
+	_do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
+   });
+}
 
-	my $pid = check_running($vmid, $nocheck);
-	return if !$pid;
-
-	my $conf;
-	if (!$nocheck) {
-	    $conf = PVE::QemuConfig->load_config($vmid);
-	    PVE::QemuConfig->check_lock($conf) if !$skiplock;
-	    if (!defined($timeout) && $shutdown && $conf->{startup}) {
-		my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
-		$timeout = $opts->{down} if $opts->{down};
-	    }
-	    PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
-	}
-
-	$timeout = 60 if !defined($timeout);
-
-	eval {
-	    if ($shutdown) {
-		if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
-		    vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
-		} else {
-		    vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
-		}
-	    } else {
-		vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
-	    }
-	};
-	my $err = $@;
-
-	if (!$err) {
-	    my $count = 0;
-	    while (($count < $timeout) && check_running($vmid, $nocheck)) {
-		$count++;
-		sleep 1;
-	    }
+sub vm_reboot {
+    my ($vmid, $timeout) = @_;
 
-	    if ($count >= $timeout) {
-		if ($force) {
-		    warn "VM still running - terminating now with SIGTERM\n";
-		    kill 15, $pid;
-		} else {
-		    die "VM quit/powerdown failed - got timeout\n";
-		}
-	    } else {
-		vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
-		return;
-	    }
-	} else {
-	    if ($force) {
-		warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
-		kill 15, $pid;
-	    } else {
-		die "VM quit/powerdown failed\n";
-	    }
-	}
+    PVE::QemuConfig->lock_config($vmid, sub {
 
-	# wait again
-	$timeout = 10;
+	# only reboot if running, as qmeventd starts it again on a stop event
+	return if !check_running($vmid);
 
-	my $count = 0;
-	while (($count < $timeout) && check_running($vmid, $nocheck)) {
-	    $count++;
-	    sleep 1;
-	}
+	create_reboot_request($vmid);
 
-	if ($count >= $timeout) {
-	    warn "VM still running - terminating now with SIGKILL\n";
-	    kill 9, $pid;
-	    sleep 1;
-	}
+	my $storecfg = PVE::Storage::config();
+	_do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
 
-	vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
    });
 }
 
@@ -5794,7 +6006,8 @@ sub vm_sendkey {
 	my $conf = PVE::QemuConfig->load_config($vmid);
 
 	# there is no qmp command, so we use the human monitor command
-	vm_human_monitor_command($vmid, "sendkey $key");
+	my $res = vm_human_monitor_command($vmid, "sendkey $key");
+	die $res if $res ne '';
     });
 }
 
@@ -6300,7 +6513,7 @@ sub restore_vma_archive {
 	    foreach_drive($oldconf, sub {
 		my ($ds, $drive) = @_;
 
-		return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
+		return if drive_is_cdrom($drive, 1);
 
 		my $volid = $drive->{file};
 		return if !$volid || $volid =~ m|^/|;
@@ -6675,66 +6888,77 @@ sub qemu_img_convert {
     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
 
-    if ($src_storeid && $dst_storeid) {
+    die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
 
-	PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
+    my $cachemode;
+    my $src_path;
+    my $src_is_iscsi = 0;
+    my $src_format = 'raw';
 
+    if ($src_storeid) {
+	PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
 	my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
-	my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
+	$src_format = qemu_img_format($src_scfg, $src_volname);
+	$src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
+	$src_is_iscsi = ($src_path =~ m|^iscsi://|);
+	$cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
+    } elsif (-f $src_volid) {
+	$src_path = $src_volid;
+	if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
+	    $src_format = $1;
+	}
+    }
 
-	my $src_format = qemu_img_format($src_scfg, $src_volname);
-	my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
+    die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
 
-	my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
-	my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
+    my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
+    my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
+    my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
+    my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
 
-	my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
-	my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
+    my $cmd = [];
+    push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
+    push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
+    push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
+    push @$cmd, '-T', $cachemode if defined($cachemode);
+
+    if ($src_is_iscsi) {
+	push @$cmd, '--image-opts';
+	$src_path = convert_iscsi_path($src_path);
+    } else {
+	push @$cmd, '-f', $src_format;
+    }
 
-	my $cmd = [];
-	push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
-	push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
-	push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
-	push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
+    if ($dst_is_iscsi) {
+	push @$cmd, '--target-image-opts';
+	$dst_path = convert_iscsi_path($dst_path);
+    } else {
+	push @$cmd, '-O', $dst_format;
+    }
 
-	if ($src_is_iscsi) {
-	    push @$cmd, '--image-opts';
-	    $src_path = convert_iscsi_path($src_path);
-	} else {
-	    push @$cmd, '-f', $src_format;
-	}
+    push @$cmd, $src_path;
 
-	if ($dst_is_iscsi) {
-	    push @$cmd, '--target-image-opts';
-	    $dst_path = convert_iscsi_path($dst_path);
-	} else {
-	    push @$cmd, '-O', $dst_format;
-	}
+    if (!$dst_is_iscsi && $is_zero_initialized) {
+	push @$cmd, "zeroinit:$dst_path";
+    } else {
+	push @$cmd, $dst_path;
+    }
 
-	push @$cmd, $src_path;
+    my $parser = sub {
+	my $line = shift;
+	if($line =~ m/\((\S+)\/100\%\)/){
+	    my $percent = $1;
+	    my $transferred = int($size * $percent / 100);
+	    my $remaining = $size - $transferred;
 
-	if (!$dst_is_iscsi && $is_zero_initialized) {
-	    push @$cmd, "zeroinit:$dst_path";
-	} else {
-	    push @$cmd, $dst_path;
+	    print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
 	}
 
-	my $parser = sub {
-	    my $line = shift;
-	    if($line =~ m/\((\S+)\/100\%\)/){
-		my $percent = $1;
-		my $transferred = int($size * $percent / 100);
-		my $remaining = $size - $transferred;
-
-		print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
-	    }
-
-	};
+    };
 
-	eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
-	my $err = $@;
-	die "copy failed: $err" if $err;
-    }
+    eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
+    my $err = $@;
+    die "copy failed: $err" if $err;
 }
 
 sub qemu_img_format {
@@ -6943,14 +7167,6 @@ sub clone_disk {
 
 	print "create full clone of drive $drivename ($drive->{file})\n";
 	my $name = undef;
-	if (drive_is_cloudinit($drive)) {
-	    $name = "vm-$newvmid-cloudinit";
-	    $snapname = undef;
-	    # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
-	    if ($dst_format ne 'raw') {
-		$name .= ".$dst_format";
-	    }
-	}
 	$newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
 	push @$newvollist, $newvolid;
 
@@ -7029,9 +7245,9 @@ sub qemu_machine_feature_enabled {
 }
 
 sub qemu_machine_pxe {
-    my ($vmid, $conf, $machine) = @_;
+    my ($vmid, $conf) = @_;
 
-    $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
+    my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
 
     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
 	$machine .= '.pxe';
@@ -7068,15 +7284,12 @@ sub create_efidisk($$$$$) {
     my (undef, $ovmf_vars) = get_ovmf_files($arch);
     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
 
-    my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
+    my $vars_size_b = -s $ovmf_vars;
+    my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
     PVE::Storage::activate_volumes($storecfg, [$volid]);
 
-    my $path = PVE::Storage::path($storecfg, $volid);
-    eval {
-	run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
-    };
-    die "Copying EFI vars image failed: $@" if $@;
+    qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
 
     return ($volid, $vars_size);
 }
@@ -7145,6 +7358,10 @@ sub add_hyperv_enlightenments {
 	    push @$cpuFlags , 'hv_synic';
 	    push @$cpuFlags , 'hv_stimer';
 	}
+
+	if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
+	    push @$cpuFlags , 'hv_ipi';
+	}
     }
 }
 
@@ -7216,6 +7433,25 @@ sub nbd_stop {
     vm_mon_cmd($vmid, 'nbd-server-stop');
 }
 
+sub create_reboot_request {
+    my ($vmid) = @_;
+    open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
+	or die "failed to create reboot trigger file: $!\n";
+    close($fh);
+}
+
+sub clear_reboot_request {
+    my ($vmid) = @_;
+    my $path = "/run/qemu-server/$vmid.reboot";
+    my $res = 0;
+
+    $res = unlink($path);
+    die "could not remove reboot request for $vmid: $!"
+	if !$res && $! != POSIX::ENOENT;
+
+    return $res;
+}
+
 # bash completion helper
 
 sub complete_backup_archives {