X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=PVE%2FQemuServer.pm;h=33bf966737ab2748f68c97fccf42181f3d57a6f0;hb=a9ce7583cf0e1979255978216d6860b10c49a1f9;hp=bc3bb1d34781d91c23bb35337342261042064e89;hpb=bfc0bb81798fe0781b0eb4796bc5c1c86ef42388;p=qemu-server.git

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index bc3bb1d..33bf966 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -2,6 +2,7 @@ package PVE::QemuServer;
 
 use strict;
 use warnings;
+
 use POSIX;
 use IO::Handle;
 use IO::Select;
@@ -20,6 +21,7 @@ use JSON;
 use Fcntl;
 use PVE::SafeSyslog;
 use Storable qw(dclone);
+use MIME::Base64;
 use PVE::Exception qw(raise raise_param_exc);
 use PVE::Storage;
 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
@@ -30,7 +32,8 @@ use PVE::ProcFSTools;
 use PVE::QemuConfig;
 use PVE::QMPClient;
 use PVE::RPCEnvironment;
-use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
+use PVE::GuestHelpers;
+use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
 use PVE::QemuServer::Memory;
 use PVE::QemuServer::USB qw(parse_usb_device);
 use PVE::QemuServer::Cloudinit;
@@ -52,7 +55,7 @@ my $OVMF = {
     ],
 };
 
-my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
+my $qemu_snap_storage = { rbd => 1 };
 
 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 
@@ -76,12 +79,6 @@ PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
     optional => 1,
 });
 
-PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
-    description => "The name of the snapshot.",
-    type => 'string', format => 'pve-configid',
-    maxLength => 40,
-});
-
 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
     type => 'string',
     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
@@ -170,7 +167,21 @@ my $cpu_vendor_list = {
     max => 'default',
 };
 
-my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
+my @supported_cpu_flags = (
+    'pcid',
+    'spec-ctrl',
+    'ibpb',
+    'ssbd',
+    'virt-ssbd',
+    'amd-ssbd',
+    'amd-no-ssb',
+    'pdpe1gb',
+    'md-clear',
+    'hv-tlbflush',
+    'hv-evmcs',
+    'aes'
+);
+my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 
 my $cpu_fmt = {
     cputype => {
@@ -196,7 +207,7 @@ my $cpu_fmt = {
     flags => {
 	description => "List of additional CPU flags separated by ';'."
 		     . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
-		     . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
+		     . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 	format_description => '+FLAG[;-FLAG...]',
 	type => 'string',
 	pattern => qr/$cpu_flag(;$cpu_flag)*/,
@@ -255,6 +266,52 @@ my $vga_fmt = {
     },
 };
 
+my $ivshmem_fmt = {
+    size => {
+	type => 'integer',
+	minimum => 1,
+	description => "The size of the file in MB.",
+    },
+    name => {
+	type => 'string',
+	pattern => '[a-zA-Z0-9\-]+',
+	optional => 1,
+	format_description => 'string',
+	description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
+    },
+};
+
+my $audio_fmt = {
+    device => {
+	type => 'string',
+	enum => [qw(ich9-intel-hda intel-hda AC97)],
+	description =>  "Configure an audio device."
+    },
+    driver =>  {
+	type => 'string',
+	enum => ['spice'],
+	default => 'spice',
+	optional => 1,
+	description => "Driver backend for the audio device."
+    },
+};
+
+my $spice_enhancements_fmt = {
+    foldersharing => {
+	type => 'boolean',
+	optional => 1,
+	default => '0',
+	description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
+    },
+    videostreaming =>  {
+	type => 'string',
+	enum => ['off', 'all', 'filter'],
+	default => 'off',
+	optional => 1,
+	description => "Enable video streaming. Uses compression for detected video streams."
+    },
+};
+
 my $confdesc = {
     onboot => {
 	optional => 1,
@@ -284,7 +341,7 @@ my $confdesc = {
 	optional => 1,
 	type => 'string',
 	description => "Lock/unlock the VM.",
-	enum => [qw(backup clone create migrate rollback snapshot snapshot-delete)],
+	enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
     },
     cpulimit => {
 	optional => 1,
@@ -580,7 +637,7 @@ EODESCR
     smbios1 => {
 	description => "Specify SMBIOS type 1 fields.",
 	type => 'string', format => 'pve-qm-smbios1',
-	maxLength => 256,
+	maxLength => 512,
 	optional => 1,
     },
     protection => {
@@ -614,7 +671,56 @@ EODESCR
 	default => "1 (autogenerated)",
 	optional => 1,
     },
+    hookscript => {
+	type => 'string',
+	format => 'pve-volume-id',
+	optional => 1,
+	description => "Script that will be executed during various steps in the vms lifetime.",
+    },
+    ivshmem => {
+	type => 'string',
+	format => $ivshmem_fmt,
+	description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
+	optional => 1,
+    },
+    audio0 => {
+	type => 'string',
+	format => $audio_fmt,
+	description => "Configure a audio device, useful in combination with QXL/Spice.",
+	optional => 1
+    },
+    spice_enhancements => {
+	type => 'string',
+	format => $spice_enhancements_fmt,
+	description => "Configure additional enhancements for SPICE.",
+	optional => 1
+    },
+};
+
+my $cicustom_fmt = {
+    meta => {
+	type => 'string',
+	optional => 1,
+	description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
+	format => 'pve-volume-id',
+	format_description => 'volume',
+    },
+    network => {
+	type => 'string',
+	optional => 1,
+	description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
+	format => 'pve-volume-id',
+	format_description => 'volume',
+    },
+    user => {
+	type => 'string',
+	optional => 1,
+	description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
+	format => 'pve-volume-id',
+	format_description => 'volume',
+    },
 };
+PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 
 my $confdesc_cloudinit = {
     citype => {
@@ -633,6 +739,12 @@ my $confdesc_cloudinit = {
 	type => 'string',
 	description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
     },
+    cicustom => {
+	optional => 1,
+	type => 'string',
+	description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
+	format => 'pve-qm-cicustom',
+    },
     searchdomain => {
 	optional => 1,
 	type => 'string',
@@ -679,7 +791,7 @@ my $MAX_SATA_DISKS = 6;
 my $MAX_USB_DEVICES = 5;
 my $MAX_NETS = 32;
 my $MAX_UNUSED_DISKS = 256;
-my $MAX_HOSTPCI_DEVICES = 4;
+my $MAX_HOSTPCI_DEVICES = 16;
 my $MAX_SERIAL_PORTS = 4;
 my $MAX_PARALLEL_PORTS = 3;
 my $MAX_NUMA = 8;
@@ -743,13 +855,9 @@ The DHCP server assign addresses to the guest starting from 10.0.2.15.
 __EOD__
 
 my $net_fmt = {
-    macaddr => {
-	type => 'string',
-	pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
+    macaddr  => get_standard_option('mac-addr', {
 	description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
-	format_description => "XX:XX:XX:XX:XX:XX",
-	optional => 1,
-    },
+    }),
     model => {
 	type => 'string',
 	description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
@@ -1045,6 +1153,16 @@ my %ssd_fmt = (
     },
 );
 
+my %wwn_fmt = (
+    wwn => {
+	type => 'string',
+	pattern => qr/^(0x)[0-9a-fA-F]{16}/,
+	format_description => 'wwn',
+	description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
+	optional => 1,
+    },
+);
+
 my $add_throttle_desc = sub {
     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
     my $d = {
@@ -1093,6 +1211,7 @@ my $ide_fmt = {
     %drivedesc_base,
     %model_fmt,
     %ssd_fmt,
+    %wwn_fmt,
 };
 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 
@@ -1109,6 +1228,7 @@ my $scsi_fmt = {
     %queues_fmt,
     %scsiblock_fmt,
     %ssd_fmt,
+    %wwn_fmt,
 };
 my $scsidesc = {
     optional => 1,
@@ -1120,6 +1240,7 @@ PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 my $sata_fmt = {
     %drivedesc_base,
     %ssd_fmt,
+    %wwn_fmt,
 };
 my $satadesc = {
     optional => 1,
@@ -1146,6 +1267,7 @@ my $alldrive_fmt = {
     %queues_fmt,
     %scsiblock_fmt,
     %ssd_fmt,
+    %wwn_fmt,
 };
 
 my $efidisk_fmt = {
@@ -1217,7 +1339,7 @@ my $hostpci_fmt = {
 	pattern => qr/$PCIRE(;$PCIRE)*/,
 	format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 	description => <<EODESCR,
-Host PCI device pass through. The PCI ID of a host's PCI device or a list 
+Host PCI device pass through. The PCI ID of a host's PCI device or a list
 of PCI virtual functions of the host. HOSTPCIID syntax is:
 
 'bus:dev.func' (hexadecimal numbers)
@@ -1271,7 +1393,7 @@ my $hostpcidesc = {
 	verbose_description =>  <<EODESCR,
 Map host PCI devices into guest.
 
-NOTE: This option allows direct access to host hardware. So it is no longer 
+NOTE: This option allows direct access to host hardware. So it is no longer
 possible to migrate such machines - use with special care.
 
 CAUTION: Experimental! User reported problems with this option.
@@ -1372,25 +1494,33 @@ sub kvm_version {
     return $kvm_api_version;
 }
 
-my $kvm_user_version;
+my $kvm_user_version = {};
+my $kvm_mtime = {};
 
 sub kvm_user_version {
+    my ($binary) = @_;
+
+    $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
+    my $st = stat($binary);
 
-    return $kvm_user_version if $kvm_user_version;
+    my $cachedmtime = $kvm_mtime->{$binary} // -1;
+    return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
+	$cachedmtime == $st->mtime;
 
-    $kvm_user_version = 'unknown';
+    $kvm_user_version->{$binary} = 'unknown';
+    $kvm_mtime->{$binary} = $st->mtime;
 
     my $code = sub {
 	my $line = shift;
 	if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
-	    $kvm_user_version = $2;
+	    $kvm_user_version->{$binary} = $2;
 	}
     };
 
-    eval { run_command("kvm -version", outfunc => $code); };
+    eval { run_command([$binary, '--version'], outfunc => $code); };
     warn $@ if $@;
 
-    return $kvm_user_version;
+    return $kvm_user_version->{$binary};
 
 }
 
@@ -1777,6 +1907,7 @@ sub print_drivedevice_full {
 	if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
 	    $device .= ",rotation_rate=1";
 	}
+	$device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
 
     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
 	my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
@@ -1801,6 +1932,7 @@ sub print_drivedevice_full {
 		$device .= ",rotation_rate=1";
 	    }
 	}
+	$device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
     } elsif ($drive->{interface} eq 'usb') {
 	die "implement me";
 	#  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
@@ -1839,7 +1971,7 @@ sub print_drive_full {
     my $path;
     my $volid = $drive->{file};
     my $format;
-    
+
     if (drive_is_cdrom($drive)) {
 	$path = get_iso_path($storecfg, $vmid, $volid);
     } else {
@@ -2292,7 +2424,7 @@ sub vmconfig_cleanup_pending {
     return $changes;
 }
 
-# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
+# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
 my $smbios1_fmt = {
     uuid => {
 	type => 'string',
@@ -2303,46 +2435,51 @@ my $smbios1_fmt = {
     },
     version => {
 	type => 'string',
-	pattern => '\S+',
-	format_description => 'string',
+	pattern => '[A-Za-z0-9+\/]+={0,2}',
+	format_description => 'Base64 encoded string',
         description => "Set SMBIOS1 version.",
 	optional => 1,
     },
     serial => {
 	type => 'string',
-	pattern => '\S+',
-	format_description => 'string',
+	pattern => '[A-Za-z0-9+\/]+={0,2}',
+	format_description => 'Base64 encoded string',
         description => "Set SMBIOS1 serial number.",
 	optional => 1,
     },
     manufacturer => {
 	type => 'string',
-	pattern => '\S+',
-	format_description => 'string',
+	pattern => '[A-Za-z0-9+\/]+={0,2}',
+	format_description => 'Base64 encoded string',
         description => "Set SMBIOS1 manufacturer.",
 	optional => 1,
     },
     product => {
 	type => 'string',
-	pattern => '\S+',
-	format_description => 'string',
+	pattern => '[A-Za-z0-9+\/]+={0,2}',
+	format_description => 'Base64 encoded string',
         description => "Set SMBIOS1 product ID.",
 	optional => 1,
     },
     sku => {
 	type => 'string',
-	pattern => '\S+',
-	format_description => 'string',
+	pattern => '[A-Za-z0-9+\/]+={0,2}',
+	format_description => 'Base64 encoded string',
         description => "Set SMBIOS1 SKU string.",
 	optional => 1,
     },
     family => {
 	type => 'string',
-	pattern => '\S+',
-	format_description => 'string',
+	pattern => '[A-Za-z0-9+\/]+={0,2}',
+	format_description => 'Base64 encoded string',
         description => "Set SMBIOS1 family string.",
 	optional => 1,
     },
+    base64 => {
+	type => 'boolean',
+	description => 'Flag to indicate that the SMBIOS values are base64 encoded',
+	optional => 1,
+    },
 };
 
 sub parse_smbios1 {
@@ -2780,21 +2917,23 @@ sub config_list {
 sub check_local_resources {
     my ($conf, $noerr) = @_;
 
-    my $loc_res = 0;
+    my @loc_res = ();
+
+    push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
+    push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
 
-    $loc_res = 1 if $conf->{hostusb}; # old syntax
-    $loc_res = 1 if $conf->{hostpci}; # old syntax
+    push @loc_res, "ivshmem" if $conf->{ivshmem};
 
     foreach my $k (keys %$conf) {
-	next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
+	next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
 	# sockets are safe: they will recreated be on the target side post-migrate
 	next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
-	$loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
+	push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
     }
 
-    die "VM uses local resources\n" if $loc_res && !$noerr;
+    die "VM uses local resources\n" if scalar @loc_res && !$noerr;
 
-    return $loc_res;
+    return \@loc_res;
 }
 
 # check if used storages are available on all nodes (use by migrate)
@@ -2850,6 +2989,45 @@ sub shared_nodes {
     return $nodehash
 }
 
+sub check_local_storage_availability {
+    my ($conf, $storecfg) = @_;
+
+    my $nodelist = PVE::Cluster::get_nodelist();
+    my $nodehash = { map { $_ => {} } @$nodelist };
+
+    foreach_drive($conf, sub {
+	my ($ds, $drive) = @_;
+
+	my $volid = $drive->{file};
+	return if !$volid;
+
+	my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
+	if ($storeid) {
+	    my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+
+	    if ($scfg->{disable}) {
+		foreach my $node (keys %$nodehash) {
+		    $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
+		}
+	    } elsif (my $avail = $scfg->{nodes}) {
+		foreach my $node (keys %$nodehash) {
+		    if (!$avail->{$node}) {
+			$nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
+		    }
+		}
+	    }
+	}
+    });
+
+    foreach my $node (values %$nodehash) {
+	if (my $unavail = $node->{unavailable_storages}) {
+	    $node->{unavailable_storages} = [ sort keys %$unavail ];
+	}
+    }
+
+    return $nodehash
+}
+
 sub check_cmdline {
     my ($pidfile, $pid) = @_;
 
@@ -2992,6 +3170,11 @@ our $vmstatus_return_properties = {
 	type => 'number',
 	optional => 1,
     },
+    lock => {
+	description => "The current config lock, if any.",
+	type => 'string',
+	optional => 1,
+    }
 };
 
 my $last_proc_pid_stat;
@@ -3062,6 +3245,7 @@ sub vmstatus {
         $d->{template} = PVE::QemuConfig->is_template($conf);
 
 	$d->{serial} = 1 if conf_has_serial($conf);
+	$d->{lock} = $conf->{lock} if $conf->{lock};
 
 	$res->{$vmid} = $d;
     }
@@ -3221,7 +3405,7 @@ sub foreach_volid {
     my $volhash = {};
 
     my $test_volid = sub {
-	my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
+	my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
 
 	return if !$volid;
 
@@ -3239,11 +3423,12 @@ sub foreach_volid {
 
 	$volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
 	    if defined($snapname);
+	$volhash->{$volid}->{size} = $size if $size;
     };
 
     foreach_drive($conf, sub {
 	my ($ds, $drive) = @_;
-	$test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
+	$test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
     });
 
     foreach my $snapname (keys %{$conf->{snapshots}}) {
@@ -3272,6 +3457,24 @@ sub conf_has_serial {
     return 0;
 }
 
+sub conf_has_audio {
+    my ($conf, $id) = @_;
+
+    $id //= 0;
+    my $audio = $conf->{"audio$id"};
+    return undef if !defined($audio);
+
+    my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
+    my $audiodriver = $audioproperties->{driver} // 'spice';
+
+    return {
+	dev => $audioproperties->{device},
+	dev_id => "audiodev$id",
+	backend => $audiodriver,
+	backend_id => "$audiodriver-backend${id}",
+    };
+}
+
 sub vga_conf_has_spice {
     my ($vga) = @_;
 
@@ -3394,13 +3597,14 @@ sub config_to_command {
     my $devices = [];
     my $pciaddr = '';
     my $bridges = {};
-    my $kvmver = kvm_user_version();
     my $vernum = 0; # unknown
     my $ostype = $conf->{ostype};
     my $winversion = windows_version($ostype);
     my $kvm = $conf->{kvm};
 
     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
+    my $kvm_binary = get_command_for_arch($arch);
+    my $kvmver = kvm_user_version($kvm_binary);
     $kvm //= 1 if is_native($arch);
 
     if ($kvm) {
@@ -3426,7 +3630,7 @@ sub config_to_command {
     my $cpuunits = defined($conf->{cpuunits}) ?
             $conf->{cpuunits} : $defaults->{cpuunits};
 
-    push @$cmd, get_command_for_arch($arch);
+    push @$cmd, $kvm_binary;
 
     push @$cmd, '-id', $vmid;
 
@@ -3450,7 +3654,26 @@ sub config_to_command {
     push @$cmd, '-daemonize';
 
     if ($conf->{smbios1}) {
-	push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
+	my $smbios_conf = parse_smbios1($conf->{smbios1});
+	if ($smbios_conf->{base64}) {
+	    # Do not pass base64 flag to qemu
+	    delete $smbios_conf->{base64};
+	    my $smbios_string = "";
+	    foreach my $key (keys %$smbios_conf) {
+		my $value;
+		if ($key eq "uuid") {
+		    $value = $smbios_conf->{uuid}
+		} else {
+		    $value = decode_base64($smbios_conf->{$key});
+		}
+		# qemu accepts any binary data, only commas need escaping by double comma
+		$value =~ s/,/,,/g;
+		$smbios_string .= "," . $key . "=" . $value if $value;
+	    }
+	    push @$cmd, '-smbios', "type=1" . $smbios_string;
+	} else {
+	    push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
+	}
     }
 
     if ($conf->{vmgenid}) {
@@ -3489,6 +3712,15 @@ sub config_to_command {
 	push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
     }
 
+    # load q35 config
+    if ($q35) {
+	# we use different pcie-port hardware for qemu >= 4.0 for passthrough
+	if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
+	    push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
+	} else {
+	    push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
+	}
+    }
 
     # add usb controllers
     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
@@ -3529,67 +3761,66 @@ sub config_to_command {
 
     # host pci devices
     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
-	my $d = parse_hostpci($conf->{"hostpci$i"});
+	my $id = "hostpci$i";
+	my $d = parse_hostpci($conf->{$id});
 	next if !$d;
 
-	my $pcie = $d->{pcie};
-	if($pcie){
+	if (my $pcie = $d->{pcie}) {
 	    die "q35 machine model is not enabled" if !$q35;
 	    # win7 wants to have the pcie devices directly on the pcie bus
 	    # instead of in the root port
 	    if ($winversion == 7) {
-		$pciaddr = print_pcie_addr("hostpci${i}bus0");
+		$pciaddr = print_pcie_addr("${id}bus0");
 	    } else {
-		$pciaddr = print_pcie_addr("hostpci$i");
+		# add more root ports if needed, 4 are present by default
+		# by pve-q35 cfgs, rest added here on demand.
+		if ($i > 3) {
+		    push @$devices, '-device', print_pcie_root_port($i);
+		}
+		$pciaddr = print_pcie_addr($id);
 	    }
-	}else{
-	    $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
+	} else {
+	    $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
 	}
 
-	my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
-	my $romfile = $d->{romfile};
-
 	my $xvga = '';
 	if ($d->{'x-vga'}) {
-	    $xvga = ',x-vga=on';
+	    $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
 	    $kvm_off = 1;
 	    $vga->{type} = 'none' if !defined($conf->{vga});
 	    $gpu_passthrough = 1;
-
-	    if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
-		$xvga = "";
-	    }
 	}
+
 	my $pcidevices = $d->{pciid};
 	my $multifunction = 1 if @$pcidevices > 1;
+
 	my $sysfspath;
 	if ($d->{mdev} && scalar(@$pcidevices) == 1) {
-	    my $id = $pcidevices->[0]->{id};
+	    my $pci_id = $pcidevices->[0]->{id};
 	    my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
-	    $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
+	    $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
 	} elsif ($d->{mdev}) {
-	    warn "ignoring mediated device with multifunction device\n";
+	    warn "ignoring mediated device '$id' with multifunction device\n";
 	}
 
 	my $j=0;
-        foreach my $pcidevice (@$pcidevices) {
-
-	    my $id = "hostpci$i";
-	    $id .= ".$j" if $multifunction;
-	    my $addr = $pciaddr;
-	    $addr .= ".$j" if $multifunction;
+	foreach my $pcidevice (@$pcidevices) {
 	    my $devicestr = "vfio-pci";
+
 	    if ($sysfspath) {
 		$devicestr .= ",sysfsdev=$sysfspath";
 	    } else {
 		$devicestr .= ",host=$pcidevice->{id}";
 	    }
-	    $devicestr .= ",id=$id$addr";
 
-	    if($j == 0){
-		$devicestr .= "$rombar$xvga";
+	    my $mf_addr = $multifunction ? ".$j" : '';
+	    $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
+
+	    if ($j == 0) {
+		$devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
+		$devicestr .= "$xvga";
 		$devicestr .= ",multifunction=on" if $multifunction;
-		$devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
+		$devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
 	    }
 
 	    push @$devices, '-device', $devicestr;
@@ -3598,7 +3829,10 @@ sub config_to_command {
     }
 
     # usb devices
-    my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
+    my $usb_dev_features = {};
+    $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 1);
+
+    my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
     push @$devices, @usbdevices if @usbdevices;
     # serial devices
     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
@@ -3632,6 +3866,23 @@ sub config_to_command {
 	}
     }
 
+    if (my $audio = conf_has_audio($conf)) {
+
+	my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
+
+	my $id = $audio->{dev_id};
+	if ($audio->{dev} eq 'AC97') {
+	    push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
+	} elsif ($audio->{dev} =~ /intel\-hda$/) {
+	    push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
+	    push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
+	    push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
+	} else {
+	    die "unkown audio device '$audio->{dev}', implement me!";
+	}
+
+	push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
+    }
 
     my $sockets = 1;
     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
@@ -3680,7 +3931,7 @@ sub config_to_command {
     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
 	push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
 	my $socket = vnc_socket($vmid);
-	push @$cmd,  '-vnc', "unix:$socket,x509,password";
+	push @$cmd,  '-vnc', "unix:$socket,password";
     } else {
 	push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
 	push @$cmd, '-nographic';
@@ -3715,7 +3966,7 @@ sub config_to_command {
 	push @$machineFlags, "type=${machine_type}";
     }
 
-    if ($conf->{startdate}) {
+    if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
 	push @$rtcFlags, "base=$conf->{startdate}";
     } elsif ($useLocaltime) {
 	push @$rtcFlags, 'base=localtime';
@@ -3724,16 +3975,11 @@ sub config_to_command {
     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
 
     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
-    
+
     push @$cmd, '-S' if $conf->{freeze};
 
     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
 
-    # enable sound
-    #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
-    #push @$cmd, '-soundhw', 'es1370';
-    #push @$cmd, '-soundhw', $soundhw if $soundhw;
-
     if (parse_guest_agent($conf)->{enabled}) {
 	my $qgasocket = qmp_socket($vmid, 1);
 	my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
@@ -3771,11 +4017,20 @@ sub config_to_command {
 	my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
 	$spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
 
-	push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
+	my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
+	if ($spice_enhancement->{foldersharing}) {
+	    push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
+	    push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
+	}
+
+	my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
+	$spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
+	push @$devices, '-spice', "$spice_opts";
 
 	push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
 	push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
 	push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
+
     }
 
     # enable balloon by default, unless explicitly disabled
@@ -3848,7 +4103,7 @@ sub config_to_command {
 	    my $queues = '';
 	    if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
 		$queues = ",num_queues=$drive->{queues}";
-	    } 
+	    }
 
 	    push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
 	    $scsicontroller->{$controller}=1;
@@ -3885,6 +4140,23 @@ sub config_to_command {
          push @$devices, '-device', $netdevicefull;
     }
 
+    if ($conf->{ivshmem}) {
+	my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
+
+	my $bus;
+	if ($q35) {
+	    $bus = print_pcie_addr("ivshmem");
+	} else {
+	    $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
+	}
+
+	my $ivshmem_name = $ivshmem->{name} // $vmid;
+	my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
+
+	push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
+	push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
+    }
+
     if (!$q35) {
 	# add pci bridges
         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
@@ -3908,6 +4180,12 @@ sub config_to_command {
     push @$cmd, '-global', join(',', @$globalFlags)
 	if scalar(@$globalFlags);
 
+    if (my $vmstate = $conf->{vmstate}) {
+	my $statepath = PVE::Storage::path($storecfg, $vmstate);
+	PVE::Storage::activate_volumes($storecfg, [$vmstate]);
+	push @$cmd, '-loadstate', $statepath;
+    }
+
     # add custom args
     if ($conf->{args}) {
 	my $aa = PVE::Tools::split_args($conf->{args});
@@ -4069,20 +4347,23 @@ sub vm_deviceplug {
 
     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
 
-        return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
+	return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
 
-        my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf); 
-        my $use_old_bios_files = undef;
-        ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
+	my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
+	my $use_old_bios_files = undef;
+	($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
 
-        my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
-        qemu_deviceadd($vmid, $netdevicefull);
-        eval { qemu_deviceaddverify($vmid, $deviceid); };
+	my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
+	qemu_deviceadd($vmid, $netdevicefull);
+	eval {
+	    qemu_deviceaddverify($vmid, $deviceid);
+	    qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
+	};
 	if (my $err = $@) {
 	    eval { qemu_netdevdel($vmid, $deviceid); };
 	    warn $@ if $@;
 	    die $err;
-        }
+	}
 
     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
 
@@ -4181,7 +4462,11 @@ sub qemu_iothread_add {
 sub qemu_iothread_del {
     my($conf, $vmid, $deviceid) = @_;
 
-    my $device = parse_drive($deviceid, $conf->{$deviceid});
+    my $confid = $deviceid;
+    if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
+	$confid = 'scsi' . $1;
+    }
+    my $device = parse_drive($confid, $conf->{$confid});
     if ($device->{iothread}) {
 	my $iothreads = vm_iothreads_list($vmid);
 	qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
@@ -4596,6 +4881,7 @@ my $fast_plug_option = {
     'description' => 1,
     'protection' => 1,
     'vmstatestorage' => 1,
+    'hookscript' => 1,
 };
 
 # hotplug changes in [PENDING]
@@ -5050,10 +5336,17 @@ sub vm_start {
 
 	die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
 
-	PVE::QemuConfig->check_lock($conf) if !$skiplock;
+	my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
+
+	PVE::QemuConfig->check_lock($conf)
+	    if !($skiplock || $is_suspended);
 
 	die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
 
+	# clean up leftover reboot request files
+	eval { clear_reboot_request($vmid); };
+	warn $@ if $@;
+
 	if (!$statefile && scalar(keys %{$conf->{pending}})) {
 	    vmconfig_apply_pending($vmid, $conf, $storecfg);
 	    $conf = PVE::QemuConfig->load_config($vmid); # update/reload
@@ -5114,6 +5407,14 @@ sub vm_start {
 	    }
 	}
 
+	PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
+
+	if ($is_suspended) {
+	    # enforce machine type on suspended vm to ensure HW compatibility
+	    $forcemachine = $conf->{runningmachine};
+	    print "Resuming suspended VM\n";
+	}
+
 	my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
 
 	my $migrate_port = 0;
@@ -5194,17 +5495,18 @@ sub vm_start {
 
 	PVE::Storage::activate_volumes($storecfg, $vollist);
 
-	if (!check_running($vmid, 1)) {
-	    eval {
-		run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
-		    outfunc => sub {}, errfunc => sub {});
-	    };
-	}
+	eval {
+	    run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
+		outfunc => sub {}, errfunc => sub {});
+	};
+	# Issues with the above 'stop' not being fully completed are extremely rare, a very low
+	# timeout should be more than enough here...
+	PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
 
 	my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
 	                                          : $defaults->{cpuunits};
 
-	my $start_timeout = $conf->{hugepages} ? 300 : 30;
+	my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
 	my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
 
 	my %properties = (
@@ -5311,6 +5613,15 @@ sub vm_start {
 		    property => "guest-stats-polling-interval",
 		    value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
 
+	if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
+	    print "Resumed VM, removing state\n";
+	    delete $conf->@{qw(lock vmstate runningmachine)};
+	    PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
+	    PVE::Storage::vdisk_free($storecfg, $vmstate);
+	    PVE::QemuConfig->write_config($vmid, $conf);
+	}
+
+	PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
     });
 }
 
@@ -5334,9 +5645,8 @@ sub vm_qmp_command {
     my $res;
 
     my $timeout;
-    if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
-	$timeout = $cmd->{arguments}->{timeout};
-	delete $cmd->{arguments}->{timeout};
+    if ($cmd->{arguments}) {
+	$timeout = delete $cmd->{arguments}->{timeout};
     }
 
     eval {
@@ -5361,8 +5671,6 @@ sub vm_qmp_command {
 sub vm_human_monitor_command {
     my ($vmid, $cmdline) = @_;
 
-    my $res;
-
     my $cmd = {
 	execute => 'human-monitor-command',
 	arguments => { 'command-line' => $cmdline},
@@ -5372,10 +5680,19 @@ sub vm_human_monitor_command {
 }
 
 sub vm_commandline {
-    my ($storecfg, $vmid) = @_;
+    my ($storecfg, $vmid, $snapname) = @_;
 
     my $conf = PVE::QemuConfig->load_config($vmid);
 
+    if ($snapname) {
+	my $snapshot = $conf->{snapshots}->{$snapname};
+	die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
+
+	$snapshot->{digest} = $conf->{digest}; # keep file digest for API
+
+	$conf = $snapshot;
+    }
+
     my $defaults = load_defaults();
 
     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
@@ -5428,6 +5745,15 @@ sub vm_stop_cleanup {
 	    unlink "/var/run/qemu-server/${vmid}.$ext";
 	}
 
+	if ($conf->{ivshmem}) {
+	    my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
+	    # just delete it for now, VMs which have this already open do not
+	    # are affected, but new VMs will get a separated one. If this
+	    # becomes an issue we either add some sort of ref-counting or just
+	    # add a "don't delete on stop" flag to the ivshmem format.
+	    unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
+	}
+
 	foreach my $key (keys %$conf) {
 	    next if $key !~ m/^hostpci(\d+)$/;
 	    my $hostpciindex = $1;
@@ -5445,7 +5771,88 @@ sub vm_stop_cleanup {
     warn $@ if $@; # avoid errors - just warn
 }
 
-# Note: use $nockeck to skip tests if VM configuration file exists.
+# call only in locked context
+sub _do_vm_stop {
+    my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
+
+    my $pid = check_running($vmid, $nocheck);
+    return if !$pid;
+
+    my $conf;
+    if (!$nocheck) {
+	$conf = PVE::QemuConfig->load_config($vmid);
+	PVE::QemuConfig->check_lock($conf) if !$skiplock;
+	if (!defined($timeout) && $shutdown && $conf->{startup}) {
+	    my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
+	    $timeout = $opts->{down} if $opts->{down};
+	}
+	PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
+    }
+
+    eval {
+	if ($shutdown) {
+	    if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
+		vm_qmp_command($vmid, {
+			execute => "guest-shutdown",
+			arguments => { timeout => $timeout }
+		    }, $nocheck);
+	    } else {
+		vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
+	    }
+	} else {
+	    vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
+	}
+    };
+    my $err = $@;
+
+    if (!$err) {
+	$timeout = 60 if !defined($timeout);
+
+	my $count = 0;
+	while (($count < $timeout) && check_running($vmid, $nocheck)) {
+	    $count++;
+	    sleep 1;
+	}
+
+	if ($count >= $timeout) {
+	    if ($force) {
+		warn "VM still running - terminating now with SIGTERM\n";
+		kill 15, $pid;
+	    } else {
+		die "VM quit/powerdown failed - got timeout\n";
+	    }
+	} else {
+	    vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
+	    return;
+	}
+    } else {
+	if ($force) {
+	    warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
+	    kill 15, $pid;
+	} else {
+	    die "VM quit/powerdown failed\n";
+	}
+    }
+
+    # wait again
+    $timeout = 10;
+
+    my $count = 0;
+    while (($count < $timeout) && check_running($vmid, $nocheck)) {
+	$count++;
+	sleep 1;
+    }
+
+    if ($count >= $timeout) {
+	warn "VM still running - terminating now with SIGKILL\n";
+	kill 9, $pid;
+	sleep 1;
+    }
+
+    vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
+}
+
+# Note: use $nocheck to skip tests if VM configuration file exists.
 # We need that when migration VMs to other nodes (files already moved)
 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
 sub vm_stop {
@@ -5462,101 +5869,113 @@ sub vm_stop {
     }
 
     PVE::QemuConfig->lock_config($vmid, sub {
+	_do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
+   });
+}
 
-	my $pid = check_running($vmid, $nocheck);
-	return if !$pid;
-
-	my $conf;
-	if (!$nocheck) {
-	    $conf = PVE::QemuConfig->load_config($vmid);
-	    PVE::QemuConfig->check_lock($conf) if !$skiplock;
-	    if (!defined($timeout) && $shutdown && $conf->{startup}) {
-		my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
-		$timeout = $opts->{down} if $opts->{down};
-	    }
-	}
-
-	$timeout = 60 if !defined($timeout);
-
-	eval {
-	    if ($shutdown) {
-		if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
-		    vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
-		} else {
-		    vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
-		}
-	    } else {
-		vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
-	    }
-	};
-	my $err = $@;
-
-	if (!$err) {
-	    my $count = 0;
-	    while (($count < $timeout) && check_running($vmid, $nocheck)) {
-		$count++;
-		sleep 1;
-	    }
+sub vm_reboot {
+    my ($vmid, $timeout) = @_;
 
-	    if ($count >= $timeout) {
-		if ($force) {
-		    warn "VM still running - terminating now with SIGTERM\n";
-		    kill 15, $pid;
-		} else {
-		    die "VM quit/powerdown failed - got timeout\n";
-		}
-	    } else {
-		vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
-		return;
-	    }
-	} else {
-	    if ($force) {
-		warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
-		kill 15, $pid;
-	    } else {
-		die "VM quit/powerdown failed\n";
-	    }
-	}
+    PVE::QemuConfig->lock_config($vmid, sub {
 
-	# wait again
-	$timeout = 10;
+	# only reboot if running, as qmeventd starts it again on a stop event
+	return if !check_running($vmid);
 
-	my $count = 0;
-	while (($count < $timeout) && check_running($vmid, $nocheck)) {
-	    $count++;
-	    sleep 1;
-	}
+	create_reboot_request($vmid);
 
-	if ($count >= $timeout) {
-	    warn "VM still running - terminating now with SIGKILL\n";
-	    kill 9, $pid;
-	    sleep 1;
-	}
+	my $storecfg = PVE::Storage::config();
+	_do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
 
-	vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
    });
 }
 
 sub vm_suspend {
-    my ($vmid, $skiplock) = @_;
+    my ($vmid, $skiplock, $includestate, $statestorage) = @_;
+
+    my $conf;
+    my $path;
+    my $storecfg;
+    my $vmstate;
 
     PVE::QemuConfig->lock_config($vmid, sub {
 
-	my $conf = PVE::QemuConfig->load_config($vmid);
+	$conf = PVE::QemuConfig->load_config($vmid);
 
+	my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
 	PVE::QemuConfig->check_lock($conf)
-	    if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
+	    if !($skiplock || $is_backing_up);
 
-	vm_mon_cmd($vmid, "stop");
+	die "cannot suspend to disk during backup\n"
+	    if $is_backing_up && $includestate;
+
+	if ($includestate) {
+	    $conf->{lock} = 'suspending';
+	    my $date = strftime("%Y-%m-%d", localtime(time()));
+	    $storecfg = PVE::Storage::config();
+	    $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
+	    $path = PVE::Storage::path($storecfg, $vmstate);
+	    PVE::QemuConfig->write_config($vmid, $conf);
+	} else {
+	    vm_mon_cmd($vmid, "stop");
+	}
     });
+
+    if ($includestate) {
+	# save vm state
+	PVE::Storage::activate_volumes($storecfg, [$vmstate]);
+
+	eval {
+	    vm_mon_cmd($vmid, "savevm-start", statefile => $path);
+	    for(;;) {
+		my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
+		if (!$state->{status}) {
+		    die "savevm not active\n";
+		} elsif ($state->{status} eq 'active') {
+		    sleep(1);
+		    next;
+		} elsif ($state->{status} eq 'completed') {
+		    print "State saved, quitting\n";
+		    last;
+		} elsif ($state->{status} eq 'failed' && $state->{error}) {
+		    die "query-savevm failed with error '$state->{error}'\n"
+		} else {
+		    die "query-savevm returned status '$state->{status}'\n";
+		}
+	    }
+	};
+	my $err = $@;
+
+	PVE::QemuConfig->lock_config($vmid, sub {
+	    $conf = PVE::QemuConfig->load_config($vmid);
+	    if ($err) {
+		# cleanup, but leave suspending lock, to indicate something went wrong
+		eval {
+		    vm_mon_cmd($vmid, "savevm-end");
+		    PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
+		    PVE::Storage::vdisk_free($storecfg, $vmstate);
+		    delete $conf->@{qw(vmstate runningmachine)};
+		    PVE::QemuConfig->write_config($vmid, $conf);
+		};
+		warn $@ if $@;
+		die $err;
+	    }
+
+	    die "lock changed unexpectedly\n"
+		if !PVE::QemuConfig->has_lock($conf, 'suspending');
+
+	    vm_qmp_command($vmid, { execute => "quit" });
+	    $conf->{lock} = 'suspended';
+	    PVE::QemuConfig->write_config($vmid, $conf);
+	});
+    }
 }
 
 sub vm_resume {
     my ($vmid, $skiplock, $nocheck) = @_;
 
     PVE::QemuConfig->lock_config($vmid, sub {
-
-	my $res = vm_mon_cmd($vmid, 'query-status');
+	my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
+	my $res = $vm_mon_cmd->($vmid, 'query-status');
 	my $resume_cmd = 'cont';
 
 	if ($res->{status} && $res->{status} eq 'suspended') {
@@ -5569,12 +5988,9 @@ sub vm_resume {
 
 	    PVE::QemuConfig->check_lock($conf)
 		if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
-
-	    vm_mon_cmd($vmid, $resume_cmd);
-
-	} else {
-	    vm_mon_cmd_nocheck($vmid, $resume_cmd);
 	}
+
+	$vm_mon_cmd->($vmid, $resume_cmd);
     });
 }
 
@@ -5586,7 +6002,8 @@ sub vm_sendkey {
 	my $conf = PVE::QemuConfig->load_config($vmid);
 
 	# there is no qmp command, so we use the human monitor command
-	vm_human_monitor_command($vmid, "sendkey $key");
+	my $res = vm_human_monitor_command($vmid, "sendkey $key");
+	die $res if $res ne '';
     });
 }
 
@@ -5693,7 +6110,6 @@ sub restore_update_config_line {
     return if $line =~ m/^lock:/;
     return if $line =~ m/^unused\d+:/;
     return if $line =~ m/^parent:/;
-    return if $line =~ m/^template:/; # restored VM is never a template
 
     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
@@ -6054,6 +6470,24 @@ sub restore_vma_archive {
 		$storage_limits{$storeid} = $bwlimit;
 
 		$virtdev_hash->{$virtdev} = $devinfo->{$devname};
+	    } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
+		my $virtdev = $1;
+		my $drive = parse_drive($virtdev, $2);
+		if (drive_is_cloudinit($drive)) {
+		    my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
+		    my $scfg = PVE::Storage::storage_config($cfg, $storeid);
+		    my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
+
+		    my $d = {
+			format => $format,
+			storeid => $opts->{storage} // $storeid,
+			size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
+			file => $drive->{file}, # to make drive_is_cloudinit check possible
+			name => "vm-$vmid-cloudinit",
+			is_cloudinit => 1,
+		    };
+		    $virtdev_hash->{$virtdev} = $d;
+		}
 	    }
 	}
 
@@ -6075,10 +6509,9 @@ sub restore_vma_archive {
 	    foreach_drive($oldconf, sub {
 		my ($ds, $drive) = @_;
 
-		return if drive_is_cdrom($drive);
+		return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
 
 		my $volid = $drive->{file};
-
 		return if !$volid || $volid =~ m|^/|;
 
 		my ($path, $owner) = PVE::Storage::path($cfg, $volid);
@@ -6094,8 +6527,7 @@ sub restore_vma_archive {
 		}
 	    });
 
-	    # delete vmstate files
-	    # since after the restore we have no snapshots anymore
+	    # delete vmstate files, after the restore we have no snapshots anymore
 	    foreach my $snapname (keys %{$oldconf->{snapshots}}) {
 		my $snap = $oldconf->{snapshots}->{$snapname};
 		if ($snap->{vmstate}) {
@@ -6124,22 +6556,30 @@ sub restore_vma_archive {
 	    my $supported = grep { $_ eq $d->{format} } @$validFormats;
 	    $d->{format} = $defFormat if !$supported;
 
-	    my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
-						  $d->{format}, undef, $alloc_size);
+	    my $name;
+	    if ($d->{is_cloudinit}) {
+		$name = $d->{name};
+		$name .= ".$d->{format}" if $d->{format} ne 'raw';
+	    }
+
+	    my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
 	    print STDERR "new volume ID is '$volid'\n";
 	    $d->{volid} = $volid;
-	    my $path = PVE::Storage::path($cfg, $volid);
 
-	    PVE::Storage::activate_volumes($cfg,[$volid]);
+	    PVE::Storage::activate_volumes($cfg, [$volid]);
 
 	    my $write_zeros = 1;
 	    if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
 		$write_zeros = 0;
 	    }
 
-	    print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
+	    if (!$d->{is_cloudinit}) {
+		my $path = PVE::Storage::path($cfg, $volid);
+
+		print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
 
-	    print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
+		print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
+	    }
 	    $map->{$virtdev} = $volid;
 	}
 
@@ -6375,9 +6815,9 @@ sub do_snapshots_with_qemu {
     my ($storecfg, $volid) = @_;
 
     my $storage_name = PVE::Storage::parse_volume_id($volid);
+    my $scfg = $storecfg->{ids}->{$storage_name};
 
-    if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}} 
-	&& !$storecfg->{ids}->{$storage_name}->{krbd}){
+    if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
 	return 1;
     }
 
@@ -6420,6 +6860,23 @@ sub template_create {
     });
 }
 
+sub convert_iscsi_path {
+    my ($path) = @_;
+
+    if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
+	my $portal = $1;
+	my $target = $2;
+	my $lun = $3;
+
+	my $initiator_name = get_initiator_name();
+
+	return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
+	       "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
+    }
+
+    die "cannot convert iscsi path '$path', unkown format\n";
+}
+
 sub qemu_img_convert {
     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
 
@@ -6440,13 +6897,32 @@ sub qemu_img_convert {
 	my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
 	my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
 
+	my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
+	my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
+
 	my $cmd = [];
 	push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
 	push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
 	push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
 	push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
-	push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
-	if ($is_zero_initialized) {
+
+	if ($src_is_iscsi) {
+	    push @$cmd, '--image-opts';
+	    $src_path = convert_iscsi_path($src_path);
+	} else {
+	    push @$cmd, '-f', $src_format;
+	}
+
+	if ($dst_is_iscsi) {
+	    push @$cmd, '--target-image-opts';
+	    $dst_path = convert_iscsi_path($dst_path);
+	} else {
+	    push @$cmd, '-O', $dst_format;
+	}
+
+	push @$cmd, $src_path;
+
+	if (!$dst_is_iscsi && $is_zero_initialized) {
 	    push @$cmd, "zeroinit:$dst_path";
 	} else {
 	    push @$cmd, $dst_path;
@@ -6481,7 +6957,7 @@ sub qemu_img_format {
 }
 
 sub qemu_drive_mirror {
-    my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
+    my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
 
     $jobs = {} if !$jobs;
 
@@ -6508,13 +6984,19 @@ sub qemu_drive_mirror {
     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
     $opts->{format} = $format if $format;
 
-    print "drive mirror is starting for drive-$drive\n";
-
-    eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
+    if (defined($bwlimit)) {
+	$opts->{speed} = $bwlimit * 1024;
+	print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
+    } else {
+	print "drive mirror is starting for drive-$drive\n";
+    }
 
+    # if a job already runs for this device we get an error, catch it for cleanup
+    eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
     if (my $err = $@) {
 	eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
-	die "mirroring error: $err";
+	warn "$@\n" if $@;
+	die "mirroring error: $err\n";
     }
 
     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
@@ -6652,7 +7134,7 @@ sub qemu_blockjobs_cancel {
 
 sub clone_disk {
     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
-	$newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
+	$newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
 
     my $newvolid;
 
@@ -6673,11 +7155,9 @@ sub clone_disk {
 	if (drive_is_cloudinit($drive)) {
 	    $name = "vm-$newvmid-cloudinit";
 	    $snapname = undef;
-	    # cloudinit only supports raw and qcow2 atm:
-	    if ($dst_format eq 'qcow2') {
-		$name .= '.qcow2';
-	    } elsif ($dst_format ne 'raw') {
-		die "clone: unhandled format for cloudinit image\n";
+	    # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
+	    if ($dst_format ne 'raw') {
+		$name .= ".$dst_format";
 	    }
 	}
 	$newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
@@ -6687,6 +7167,7 @@ sub clone_disk {
 
 	my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
 	if (!$running || $snapname) {
+	    # TODO: handle bwlimits
 	    qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
 	} else {
 
@@ -6696,7 +7177,7 @@ sub clone_disk {
 		    if $drive->{iothread};
 	    }
 
-	    qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
+	    qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
 	}
     }
 
@@ -6757,9 +7238,9 @@ sub qemu_machine_feature_enabled {
 }
 
 sub qemu_machine_pxe {
-    my ($vmid, $conf, $machine) = @_;
+    my ($vmid, $conf) = @_;
 
-    $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
+    my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
 
     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
 	$machine .= '.pxe';
@@ -6873,6 +7354,10 @@ sub add_hyperv_enlightenments {
 	    push @$cpuFlags , 'hv_synic';
 	    push @$cpuFlags , 'hv_stimer';
 	}
+
+	if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
+	    push @$cpuFlags , 'hv_ipi';
+	}
     }
 }
 
@@ -6944,6 +7429,25 @@ sub nbd_stop {
     vm_mon_cmd($vmid, 'nbd-server-stop');
 }
 
+sub create_reboot_request {
+    my ($vmid) = @_;
+    open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
+	or die "failed to create reboot trigger file: $!\n";
+    close($fh);
+}
+
+sub clear_reboot_request {
+    my ($vmid) = @_;
+    my $path = "/run/qemu-server/$vmid.reboot";
+    my $res = 0;
+
+    $res = unlink($path);
+    die "could not remove reboot request for $vmid: $!"
+	if !$res && $! != POSIX::ENOENT;
+
+    return $res;
+}
+
 # bash completion helper
 
 sub complete_backup_archives {