X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=PVE%2FQemuServer.pm;h=3be7e240276bc138b8d6cd50a2726ca860b23117;hb=41af2dfc2592a33bc0d643fe3fdf2702d2ed4e3b;hp=55602f50f7818bf4d949cd05aabca6a58eac796e;hpb=0c498cca366faebdd61b5a2af26764e630ac9855;p=qemu-server.git

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 55602f5..3be7e24 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -26,29 +26,29 @@ use Time::HiRes qw(gettimeofday);
 use URI::Escape;
 use UUID;
 
-use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
+use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
 use PVE::DataCenterConfig;
 use PVE::Exception qw(raise raise_param_exc);
 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
 use PVE::INotify;
-use PVE::JSONSchema qw(get_standard_option);
+use PVE::JSONSchema qw(get_standard_option parse_property_string);
 use PVE::ProcFSTools;
 use PVE::RPCEnvironment;
 use PVE::Storage;
 use PVE::SysFSTools;
 use PVE::Systemd;
-use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
+use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
 
 use PVE::QMPClient;
 use PVE::QemuConfig;
 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
 use PVE::QemuServer::Cloudinit;
 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
-use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive foreach_drive foreach_volid);
+use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
 use PVE::QemuServer::Machine;
 use PVE::QemuServer::Memory;
 use PVE::QemuServer::Monitor qw(mon_cmd);
-use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
+use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
 use PVE::QemuServer::USB qw(parse_usb_device);
 
 my $have_sdn;
@@ -97,6 +97,28 @@ PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 	optional => 1,
 });
 
+
+sub map_storage {
+    my ($map, $source) = @_;
+
+    return $source if !defined($map);
+
+    return $map->{entries}->{$source}
+	if $map->{entries} && defined($map->{entries}->{$source});
+
+    return $map->{default} if $map->{default};
+
+    # identity (fallback)
+    return $source;
+}
+
+PVE::JSONSchema::register_standard_option('pve-targetstorage', {
+    description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
+    type => 'string',
+    format => 'storagepair-list',
+    optional => 1,
+});
+
 #no warnings 'redefine';
 
 sub cgroups_write {
@@ -375,15 +397,14 @@ EODESC
     },
     boot => {
 	optional => 1,
-	type => 'string',
-	description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
-	pattern => '[acdn]{1,4}',
-	default => 'cdn',
+	type => 'string', format => 'pve-qm-boot',
+	description => "Specify guest boot order. Use with 'order=', usage with"
+		     . " no key or 'legacy=' is deprecated.",
     },
     bootdisk => {
 	optional => 1,
 	type => 'string', format => 'pve-qm-bootdisk',
-	description => "Enable booting from specified disk.",
+	description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 	pattern => '(ide|sata|scsi|virtio)\d+',
     },
     smp => {
@@ -419,6 +440,13 @@ EODESC
 	description => "Enable/disable hugepages memory.",
 	enum => [qw(any 2 1024)],
     },
+    keephugepages => {
+	optional => 1,
+	type => 'boolean',
+	default => 0,
+	description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
+	    ." after VM shutdown and can be used for subsequent starts.",
+    },
     vcpus => {
 	optional => 1,
 	type => 'integer',
@@ -453,7 +481,8 @@ EODESC
     localtime => {
 	optional => 1,
 	type => 'boolean',
-	description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
+	description => "Set the real time clock to local time. This is enabled by default if ostype"
+	    ." indicates a Microsoft OS.",
     },
     freeze => {
 	optional => 1,
@@ -464,29 +493,28 @@ EODESC
 	optional => 1,
 	type => 'string', format => $vga_fmt,
 	description => "Configure the VGA hardware.",
-	verbose_description => "Configure the VGA Hardware. If you want to use ".
-	    "high resolution modes (>= 1280x1024x16) you may need to increase " .
-	    "the vga memory option. Since QEMU 2.9 the default VGA display type " .
-	    "is 'std' for all OS types besides some Windows versions (XP and " .
-	    "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
-	    "display server. For win* OS you can select how many independent " .
-	    "displays you want, Linux guests can add displays them self.\n".
-	    "You can also run without any graphic card, using a serial device as terminal.",
+	verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
+	    ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
+	    ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
+	    ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
+	    ." display server. For win* OS you can select how many independent displays you want,"
+	    ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
+	    ." using a serial device as terminal.",
     },
     watchdog => {
 	optional => 1,
 	type => 'string', format => 'pve-qm-watchdog',
 	description => "Create a virtual hardware watchdog device.",
-	verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
-	    " (by a guest action), the watchdog must be periodically polled " .
-	    "by an agent inside the guest or else the watchdog will reset " .
-	    "the guest (or execute the respective action specified)",
+	verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
+	    ." action), the watchdog must be periodically polled by an agent inside the guest or"
+	    ." else the watchdog will reset the guest (or execute the respective action specified)",
     },
     startdate => {
 	optional => 1,
 	type => 'string',
 	typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
-	description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
+	description => "Set the initial date of the real time clock. Valid format for date are:"
+	    ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 	pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 	default => 'now',
     },
@@ -514,12 +542,11 @@ EODESCR
 	type => 'boolean',
 	default => 1,
 	description => "Enable/disable the USB tablet device.",
-	verbose_description => "Enable/disable the USB tablet device. This device is " .
-	    "usually needed to allow absolute mouse positioning with VNC. " .
-	    "Else the mouse runs out of sync with normal VNC clients. " .
-	    "If you're running lots of console-only guests on one host, " .
-	    "you may consider disabling this to save some context switches. " .
-	    "This is turned off by default if you use spice (-vga=qxl).",
+	verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
+	    ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
+	    ." normal VNC clients. If you're running lots of console-only guests on one host, you"
+	    ." may consider disabling this to save some context switches. This is turned off by"
+	    ." default if you use spice (`qm set <vmid> --vga qxl`).",
     },
     migrate_speed => {
 	optional => 1,
@@ -560,15 +587,25 @@ EODESCR
     vmstate => {
 	optional => 1,
 	type => 'string', format => 'pve-volume-id',
-	description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
+	description => "Reference to a volume which stores the VM state. This is used internally"
+	    ." for snapshots.",
     },
     vmstatestorage => get_standard_option('pve-storage-id', {
 	description => "Default storage for VM state volumes/files.",
 	optional => 1,
     }),
     runningmachine => get_standard_option('pve-qemu-machine', {
-	description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
+	description => "Specifies the QEMU machine type of the running vm. This is used internally"
+	    ." for snapshots.",
     }),
+    runningcpu => {
+	description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
+	    ." internally for snapshots.",
+	optional => 1,
+	type => 'string',
+	pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
+	format_description => 'QEMU -cpu parameter'
+    },
     machine => get_standard_option('pve-qemu-machine'),
     arch => {
 	description => "Virtual processor architecture. Defaults to the host.",
@@ -585,7 +622,8 @@ EODESCR
     protection => {
 	optional => 1,
 	type => 'boolean',
-	description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
+	description => "Sets the protection flag of the VM. This will disable the remove VM and"
+	    ." remove disk operations.",
 	default => 0,
     },
     bios => {
@@ -599,17 +637,16 @@ EODESCR
 	type => 'string',
 	pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 	format_description => 'UUID',
-	description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
-	verbose_description => "The VM generation ID (vmgenid) device exposes a".
-	    " 128-bit integer value identifier to the guest OS. This allows to".
-	    " notify the guest operating system when the virtual machine is".
-	    " executed with a different configuration (e.g. snapshot execution".
-	    " or creation from a template). The guest operating system notices".
-	    " the change, and is then able to react as appropriate by marking".
-	    " its copies of distributed databases as dirty, re-initializing its".
-	    " random number generator, etc.\n".
-	    "Note that auto-creation only works when done throug API/CLI create".
-	    " or update methods, but not when manually editing the config file.",
+	description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
+	    ." to disable explicitly.",
+	verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
+	    ." value identifier to the guest OS. This allows to notify the guest operating system"
+	    ." when the virtual machine is executed with a different configuration (e.g. snapshot"
+	    ." execution or creation from a template). The guest operating system notices the"
+	    ." change, and is then able to react as appropriate by marking its copies of"
+	    ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
+	    ."Note that auto-creation only works when done through API/CLI create or update methods"
+	    .", but not when manually editing the config file.",
 	default => "1 (autogenerated)",
 	optional => 1,
     },
@@ -622,7 +659,8 @@ EODESCR
     ivshmem => {
 	type => 'string',
 	format => $ivshmem_fmt,
-	description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
+	description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
+	    ." the host.",
 	optional => 1,
     },
     audio0 => {
@@ -654,21 +692,24 @@ my $cicustom_fmt = {
     meta => {
 	type => 'string',
 	optional => 1,
-	description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
+	description => 'Specify a custom file containing all meta data passed to the VM via"
+	    ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 	format => 'pve-volume-id',
 	format_description => 'volume',
     },
     network => {
 	type => 'string',
 	optional => 1,
-	description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
+	description => 'Specify a custom file containing all network data passed to the VM via'
+	    .' cloud-init.',
 	format => 'pve-volume-id',
 	format_description => 'volume',
     },
     user => {
 	type => 'string',
 	optional => 1,
-	description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
+	description => 'Specify a custom file containing all user data passed to the VM via'
+	    .' cloud-init.',
 	format => 'pve-volume-id',
 	format_description => 'volume',
     },
@@ -679,34 +720,44 @@ my $confdesc_cloudinit = {
     citype => {
 	optional => 1,
 	type => 'string',
-	description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
+	description => 'Specifies the cloud-init configuration format. The default depends on the'
+	    .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
+	    .' and `configdrive2` for windows.',
 	enum => ['configdrive2', 'nocloud'],
     },
     ciuser => {
 	optional => 1,
 	type => 'string',
-	description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
+	description => "cloud-init: User name to change ssh keys and password for instead of the"
+	    ." image's configured default user.",
     },
     cipassword => {
 	optional => 1,
 	type => 'string',
-	description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
+	description => 'cloud-init: Password to assign the user. Using this is generally not'
+	    .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
+	    .' support hashed passwords.',
     },
     cicustom => {
 	optional => 1,
 	type => 'string',
-	description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
+	description => 'cloud-init: Specify custom files to replace the automatically generated'
+	    .' ones at start.',
 	format => 'pve-qm-cicustom',
     },
     searchdomain => {
 	optional => 1,
 	type => 'string',
-	description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
+	description => "cloud-init: Sets DNS search domains for a container. Create will'
+	    .' automatically use the setting from the host if neither searchdomain nor nameserver'
+	    .' are set.",
     },
     nameserver => {
 	optional => 1,
 	type => 'string', format => 'address-list',
-	description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
+	description => "cloud-init: Sets DNS server IP address for a container. Create will'
+	    .' automatically use the setting from the host if neither searchdomain nor nameserver'
+	    .' are set.",
     },
     sshkeys => {
 	optional => 1,
@@ -739,7 +790,6 @@ while (my ($k, $v) = each %$confdesc) {
 
 my $MAX_USB_DEVICES = 5;
 my $MAX_NETS = 32;
-my $MAX_HOSTPCI_DEVICES = 16;
 my $MAX_SERIAL_PORTS = 4;
 my $MAX_PARALLEL_PORTS = 3;
 my $MAX_NUMA = 8;
@@ -804,11 +854,14 @@ __EOD__
 
 my $net_fmt = {
     macaddr  => get_standard_option('mac-addr', {
-	description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
+	description => "MAC address. That address must be unique withing your network. This is"
+	    ." automatically generated if not specified.",
     }),
     model => {
 	type => 'string',
-	description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
+	description => "Network Card Model. The 'virtio' model provides the best performance with"
+	    ." very low CPU overhead. If your guest does not support this driver, it is usually"
+	    ." best to use 'e1000'.",
         enum => $nic_model_list,
         default_key => 1,
     },
@@ -817,6 +870,7 @@ my $net_fmt = {
 	type => 'string',
 	description => $net_fmt_bridge_descr,
 	format_description => 'bridge',
+	pattern => '[-_.\w\d]+',
 	optional => 1,
     },
     queues => {
@@ -854,6 +908,12 @@ my $net_fmt = {
 	description => 'Whether this interface should be disconnected (like pulling the plug).',
 	optional => 1,
     },
+    mtu => {
+	type => 'integer',
+	minimum => 1, maximum => 65520,
+	description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
+	optional => 1,
+    },
 };
 
 my $netdesc = {
@@ -907,10 +967,12 @@ cloud-init: Specify IP addresses and gateways for the corresponding interface.
 
 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 
-The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
+The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
+gateway should be provided.
 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 
-If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
+If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
+dhcp on IPv4.
 EODESCR
 };
 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
@@ -935,7 +997,7 @@ sub verify_volume_id_or_qm_path {
     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
     if ($@) {
-	return undef if $noerr;
+	return if $noerr;
 	die $@;
     }
     return $volid;
@@ -955,7 +1017,8 @@ The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 
 You can use the 'lsusb -t' command to list existing usb devices.
 
-NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
+NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
+machines - use with special care.
 
 The value 'spice' can be used to add a usb redirection devices for spice.
 EODESCR
@@ -975,76 +1038,6 @@ my $usbdesc = {
 };
 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 
-my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
-my $hostpci_fmt = {
-    host => {
-	default_key => 1,
-	type => 'string',
-	pattern => qr/$PCIRE(;$PCIRE)*/,
-	format_description => 'HOSTPCIID[;HOSTPCIID2...]',
-	description => <<EODESCR,
-Host PCI device pass through. The PCI ID of a host's PCI device or a list
-of PCI virtual functions of the host. HOSTPCIID syntax is:
-
-'bus:dev.func' (hexadecimal numbers)
-
-You can us the 'lspci' command to list existing PCI devices.
-EODESCR
-    },
-    rombar => {
-	type => 'boolean',
-        description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
-	optional => 1,
-	default => 1,
-    },
-    romfile => {
-        type => 'string',
-        pattern => '[^,;]+',
-        format_description => 'string',
-        description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
-        optional => 1,
-    },
-    pcie => {
-	type => 'boolean',
-        description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
-	optional => 1,
-	default => 0,
-    },
-    'x-vga' => {
-	type => 'boolean',
-        description =>  "Enable vfio-vga device support.",
-	optional => 1,
-	default => 0,
-    },
-    'mdev' => {
-	type => 'string',
-        format_description => 'string',
-	pattern => '[^/\.:]+',
-	optional => 1,
-	description => <<EODESCR
-The type of mediated device to use.
-An instance of this type will be created on startup of the VM and
-will be cleaned up when the VM stops.
-EODESCR
-    }
-};
-PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
-
-my $hostpcidesc = {
-        optional => 1,
-        type => 'string', format => 'pve-qm-hostpci',
-        description => "Map host PCI devices into guest.",
-	verbose_description =>  <<EODESCR,
-Map host PCI devices into guest.
-
-NOTE: This option allows direct access to host hardware. So it is no longer
-possible to migrate such machines - use with special care.
-
-CAUTION: Experimental! User reported problems with this option.
-EODESCR
-};
-PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
-
 my $serialdesc = {
 	optional => 1,
 	type => 'string',
@@ -1055,7 +1048,8 @@ Create a serial device inside the VM (n is 0 to 3), and pass through a
 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
 host side (use 'qm terminal' to open a terminal connection).
 
-NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
+NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
+use with special care.
 
 CAUTION: Experimental! User reported problems with this option.
 EODESCR
@@ -1069,7 +1063,8 @@ my $paralleldesc= {
 	verbose_description =>  <<EODESCR,
 Map host parallel devices (n is 0 to 2).
 
-NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
+NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
+machines - use with special care.
 
 CAUTION: Experimental! User reported problems with this option.
 EODESCR
@@ -1083,8 +1078,8 @@ for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
     $confdesc->{"serial$i"} = $serialdesc;
 }
 
-for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
-    $confdesc->{"hostpci$i"} = $hostpcidesc;
+for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
+    $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
 }
 
 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
@@ -1095,13 +1090,75 @@ for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
     $confdesc->{"usb$i"} = $usbdesc;
 }
 
+my $boot_fmt = {
+    legacy => {
+	optional => 1,
+	default_key => 1,
+	type => 'string',
+	description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
+		     . " Deprecated, use 'order=' instead.",
+	pattern => '[acdn]{1,4}',
+	format_description => "[acdn]{1,4}",
+
+	# note: this is also the fallback if boot: is not given at all
+	default => 'cdn',
+    },
+    order => {
+	optional => 1,
+	type => 'string',
+	format => 'pve-qm-bootdev-list',
+	format_description => "device[;device...]",
+	description => <<EODESC,
+The guest will attempt to boot from devices in the order they appear here.
+
+Disks, optical drives and passed-through storage USB devices will be directly
+booted from, NICs will load PXE, and PCIe devices will either behave like disks
+(e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
+
+Note that only devices in this list will be marked as bootable and thus loaded
+by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
+(e.g. software-raid), you need to specify all of them here.
+
+Overrides the deprecated 'legacy=[acdn]*' value when given.
+EODESC
+    },
+};
+PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
+
+PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
+sub verify_bootdev {
+    my ($dev, $noerr) = @_;
+
+    return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
+
+    my $check = sub {
+	my ($base) = @_;
+	return 0 if $dev !~ m/^$base\d+$/;
+	return 0 if !$confdesc->{$dev};
+	return 1;
+    };
+
+    return $dev if $check->("net");
+    return $dev if $check->("usb");
+    return $dev if $check->("hostpci");
+
+    return if $noerr;
+    die "invalid boot device '$dev'\n";
+}
+
+sub print_bootorder {
+    my ($devs) = @_;
+    my $data = { order => join(';', @$devs) };
+    return PVE::JSONSchema::print_property_string($data, $boot_fmt);
+}
+
 my $kvm_api_version = 0;
 
 sub kvm_version {
     return $kvm_api_version if $kvm_api_version;
 
     open my $fh, '<', '/dev/kvm'
-	or return undef;
+	or return;
 
     # 0xae00 => KVM_GET_API_VERSION
     $kvm_api_version = ioctl($fh, 0xae00, 0);
@@ -1138,6 +1195,11 @@ sub kvm_user_version {
     return $kvm_user_version->{$binary};
 
 }
+my sub extract_version {
+    my ($machine_type, $version) = @_;
+    $version = kvm_user_version() if !defined($version);
+    PVE::QemuServer::Machine::extract_version($machine_type, $version)
+}
 
 sub kernel_has_vhost_net {
     return -c '/dev/vhost-net';
@@ -1179,7 +1241,7 @@ sub filename_to_volume_id {
      if (!($file eq 'none' || $file eq 'cdrom' ||
 	  $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 
-	return undef if $file =~ m|/|;
+	return if $file =~ m|/|;
 
 	if ($media && $media eq 'cdrom') {
 	    $file = "local:iso/$file";
@@ -1220,7 +1282,8 @@ sub cleanup_drive_path {
 	($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 	($drive->{file} !~ m/^\d+$/)) {
 	my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
-	raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
+	raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
+	    if !$vtype;
 	$drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 	verify_media_type($opt, $vtype, $drive->{media});
 	$drive->{file} = $volid;
@@ -1254,7 +1317,7 @@ sub pve_verify_hotplug_features {
 
     return $value if parse_hotplug_features($value);
 
-    return undef if $noerr;
+    return if $noerr;
 
     die "unable to parse hotplug option\n";
 }
@@ -1269,12 +1332,12 @@ sub scsi_inquiry {
     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
     if (!$ret) {
 	die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
-	return undef;
+	return;
     }
     my $version = unpack("I", $versionbuf);
     if ($version < 30000) {
 	die "scsi generic interface too old\n"  if !$noerr;
-	return undef;
+	return;
     }
 
     my $buf = "\x00" x 36;
@@ -1291,13 +1354,13 @@ sub scsi_inquiry {
     $ret = ioctl($fh, $SG_IO, $packet);
     if (!$ret) {
 	die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
-	return undef;
+	return;
     }
 
     my @res = unpack($sg_io_hdr_t, $packet);
     if ($res[17] || $res[18]) {
 	die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
-	return undef;
+	return;
     }
 
     my $res = {};
@@ -1313,7 +1376,7 @@ sub scsi_inquiry {
 sub path_is_scsi {
     my ($path) = @_;
 
-    my $fh = IO::File->new("+<$path") || return undef;
+    my $fh = IO::File->new("+<$path") || return;
     my $res = scsi_inquiry($fh, 1);
     close($fh);
 
@@ -1339,7 +1402,7 @@ sub print_tabletdevice_full {
 sub print_keyboarddevice_full {
     my ($conf, $arch, $machine) = @_;
 
-    return undef if $arch ne 'aarch64';
+    return if $arch ne 'aarch64';
 
     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
 }
@@ -1350,10 +1413,11 @@ sub print_drivedevice_full {
     my $device = '';
     my $maxdev = 0;
 
+    my $drive_id = "$drive->{interface}$drive->{index}";
     if ($drive->{interface} eq 'virtio') {
-	my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
-	$device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
-	$device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
+	my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
+	$device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
+	$device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
     } elsif ($drive->{interface} eq 'scsi') {
 
 	my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
@@ -1377,7 +1441,7 @@ sub print_drivedevice_full {
 	    }
 
 	    # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
-	    my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
+	    my $version = extract_version($machine_type, kvm_user_version());
 	    if ($path =~ m/^iscsi\:\/\// &&
 	       !min_version($version, 4, 1)) {
 		$devicetype = 'generic';
@@ -1385,10 +1449,12 @@ sub print_drivedevice_full {
 	}
 
 	if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
-	    $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
+	    $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
 	} else {
-	    $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
+	    $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
+	        .",lun=$drive->{index}";
 	}
+	$device .= ",drive=drive-$drive_id,id=$drive_id";
 
 	if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
 	    $device .= ",rotation_rate=1";
@@ -1407,7 +1473,7 @@ sub print_drivedevice_full {
 	} else {
 	    $device .= ",bus=ahci$controller.$unit";
 	}
-	$device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
+	$device .= ",drive=drive-$drive_id,id=$drive_id";
 
 	if ($devicetype eq 'hd') {
 	    if (my $model = $drive->{model}) {
@@ -1440,7 +1506,7 @@ sub print_drivedevice_full {
 sub get_initiator_name {
     my $initiator;
 
-    my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
+    my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
     while (defined(my $line = <$fh>)) {
 	next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
 	$initiator = $1;
@@ -1547,8 +1613,6 @@ sub print_drive_commandline_full {
 sub print_netdevice_full {
     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
 
-    my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
-
     my $device = $net->{model};
     if ($net->{model} eq 'virtio') {
          $device = 'virtio-net-pci';
@@ -1557,12 +1621,29 @@ sub print_netdevice_full {
     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
-	#Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
+	# Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
+	# and out of each queue plus one config interrupt and control vector queue
 	my $vectors = $net->{queues} * 2 + 2;
 	$tmpstr .= ",vectors=$vectors,mq=on";
     }
     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
 
+    if (my $mtu = $net->{mtu}) {
+	if ($net->{model} eq 'virtio' && $net->{bridge}) {
+	    my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
+	    if ($mtu == 1) {
+		 $mtu = $bridge_mtu;
+	    } elsif ($mtu < 576) {
+		die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
+	    } elsif ($mtu > $bridge_mtu) {
+		die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
+	    }
+	    $tmpstr .= ",host_mtu=$mtu";
+	} else {
+	    warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
+	}
+    }
+
     if ($use_old_bios_files) {
 	my $romfile;
 	if ($device eq 'virtio-net-pci') {
@@ -1609,7 +1690,8 @@ sub print_netdev_full {
     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
 
     if ($net->{bridge}) {
-        $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
+	$netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
+	    .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
     } else {
         $netdev = "type=user,id=$netid,hostname=$vmname";
     }
@@ -1667,6 +1749,11 @@ sub print_vga_device {
 	$memory = ",ram_size=67108864,vram_size=33554432";
     }
 
+    my $edidoff = "";
+    if ($type eq 'VGA' && windows_version($conf->{ostype})) {
+	$edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
+    }
+
     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
     my $vgaid = "vga" . ($id // '');
     my $pciaddr;
@@ -1678,7 +1765,7 @@ sub print_vga_device {
 	$pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
     }
 
-    return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
+    return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
 }
 
 sub parse_number_sets {
@@ -1698,37 +1785,20 @@ sub parse_number_sets {
 sub parse_numa {
     my ($data) = @_;
 
-    my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
+    my $res = parse_property_string($numa_fmt, $data);
     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
     return $res;
 }
 
-sub parse_hostpci {
-    my ($value) = @_;
-
-    return undef if !$value;
-
-    my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
-
-    my @idlist = split(/;/, $res->{host});
-    delete $res->{host};
-    foreach my $id (@idlist) {
-	my $devs = PVE::SysFSTools::lspci($id);
-	die "no PCI device found for '$id'\n" if !scalar(@$devs);
-	push @{$res->{pciid}}, @$devs;
-    }
-    return $res;
-}
-
 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
 sub parse_net {
     my ($data) = @_;
 
-    my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
+    my $res = eval { parse_property_string($net_fmt, $data) };
     if ($@) {
 	warn $@;
-	return undef;
+	return;
     }
     if (!defined($res->{macaddr})) {
 	my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
@@ -1741,28 +1811,28 @@ sub parse_net {
 sub parse_ipconfig {
     my ($data) = @_;
 
-    my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
+    my $res = eval { parse_property_string($ipconfig_fmt, $data) };
     if ($@) {
 	warn $@;
-	return undef;
+	return;
     }
 
     if ($res->{gw} && !$res->{ip}) {
 	warn 'gateway specified without specifying an IP address';
-	return undef;
+	return;
     }
     if ($res->{gw6} && !$res->{ip6}) {
 	warn 'IPv6 gateway specified without specifying an IPv6 address';
-	return undef;
+	return;
     }
     if ($res->{gw} && $res->{ip} eq 'dhcp') {
 	warn 'gateway specified together with DHCP';
-	return undef;
+	return;
     }
     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
 	# gw6 + auto/dhcp
 	warn "IPv6 gateway specified together with $res->{ip6} address";
-	return undef;
+	return;
     }
 
     if (!$res->{ip} && !$res->{ip6}) {
@@ -1800,7 +1870,7 @@ sub vm_is_volid_owner {
 	}
     }
 
-    return undef;
+    return;
 }
 
 sub vmconfig_register_unused_drive {
@@ -1878,7 +1948,7 @@ my $smbios1_fmt = {
 sub parse_smbios1 {
     my ($data) = @_;
 
-    my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
+    my $res = eval { parse_property_string($smbios1_fmt, $data) };
     warn $@ if $@;
     return $res;
 }
@@ -1893,9 +1963,9 @@ PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
 sub parse_watchdog {
     my ($value) = @_;
 
-    return undef if !$value;
+    return if !$value;
 
-    my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
+    my $res = eval { parse_property_string($watchdog_fmt, $value) };
     warn $@ if $@;
     return $res;
 }
@@ -1905,7 +1975,7 @@ sub parse_guest_agent {
 
     return {} if !defined($value->{agent});
 
-    my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
+    my $res = eval { parse_property_string($agent_fmt, $value->{agent}) };
     warn $@ if $@;
 
     # if the agent is disabled ignore the other potentially set properties
@@ -1917,7 +1987,7 @@ sub parse_vga {
     my ($value) = @_;
 
     return {} if !$value;
-    my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
+    my $res = eval { parse_property_string($vga_fmt, $value) };
     warn $@ if $@;
     return $res;
 }
@@ -1925,9 +1995,9 @@ sub parse_vga {
 sub parse_rng {
     my ($value) = @_;
 
-    return undef if !$value;
+    return if !$value;
 
-    my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
+    my $res = eval { parse_property_string($rng_fmt, $value) };
     warn $@ if $@;
     return $res;
 }
@@ -1938,7 +2008,7 @@ sub verify_usb_device {
 
     return $value if parse_usb_device($value);
 
-    return undef if $noerr;
+    return if $noerr;
 
     die "unable to parse usb device\n";
 }
@@ -1948,7 +2018,8 @@ sub json_config_properties {
     my $prop = shift;
 
     foreach my $opt (keys %$confdesc) {
-	next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
+	next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
+	    $opt eq 'runningmachine' || $opt eq 'runningcpu';
 	$prop->{$opt} = $confdesc->{$opt};
     }
 
@@ -2007,7 +2078,7 @@ sub destroy_vm {
 
     if ($conf->{template}) {
 	# check if any base image is still used by a linked clone
-	foreach_drive($conf, sub {
+	PVE::QemuConfig->foreach_volume($conf, sub {
 		my ($ds, $drive) = @_;
 		return if drive_is_cdrom($drive);
 
@@ -2021,7 +2092,7 @@ sub destroy_vm {
     }
 
     # only remove disks owned by this VM
-    foreach_drive($conf, sub {
+    PVE::QemuConfig->foreach_volume($conf, sub {
 	my ($ds, $drive) = @_;
  	return if drive_is_cdrom($drive, 1);
 
@@ -2053,7 +2124,7 @@ sub destroy_vm {
 sub parse_vm_config {
     my ($filename, $raw) = @_;
 
-    return undef if !defined($raw);
+    return if !defined($raw);
 
     my $res = {
 	digest => Digest::SHA::sha1_hex($raw),
@@ -2232,7 +2303,7 @@ sub write_vm_config {
 	}
 
 	foreach my $key (sort keys %$conf) {
-	    next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
+	    next if $key =~ /^(digest|description|pending|snapshots)$/;
 	    $raw .= "$key: $conf->{$key}\n";
 	}
 	return $raw;
@@ -2310,7 +2381,7 @@ sub check_local_resources {
 sub check_storage_availability {
     my ($storecfg, $conf, $node) = @_;
 
-    foreach_drive($conf, sub {
+    PVE::QemuConfig->foreach_volume($conf, sub {
 	my ($ds, $drive) = @_;
 
 	my $volid = $drive->{file};
@@ -2333,7 +2404,7 @@ sub shared_nodes {
     my $nodehash = { map { $_ => 1 } @$nodelist };
     my $nodename = nodename();
 
-    foreach_drive($conf, sub {
+    PVE::QemuConfig->foreach_volume($conf, sub {
 	my ($ds, $drive) = @_;
 
 	my $volid = $drive->{file};
@@ -2365,7 +2436,7 @@ sub check_local_storage_availability {
     my $nodelist = PVE::Cluster::get_nodelist();
     my $nodehash = { map { $_ => {} } @$nodelist };
 
-    foreach_drive($conf, sub {
+    PVE::QemuConfig->foreach_volume($conf, sub {
 	my ($ds, $drive) = @_;
 
 	my $volid = $drive->{file};
@@ -2707,9 +2778,9 @@ sub conf_has_audio {
 
     $id //= 0;
     my $audio = $conf->{"audio$id"};
-    return undef if !defined($audio);
+    return if !defined($audio);
 
-    my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
+    my $audioproperties = parse_property_string($audio_fmt, $audio);
     my $audiodriver = $audioproperties->{driver} // 'spice';
 
     return {
@@ -2720,6 +2791,32 @@ sub conf_has_audio {
     };
 }
 
+sub audio_devs {
+    my ($audio, $audiopciaddr, $machine_version) = @_;
+
+    my $devs = [];
+
+    my $id = $audio->{dev_id};
+    my $audiodev = "";
+    if (min_version($machine_version, 4, 2)) {
+	$audiodev = ",audiodev=$audio->{backend_id}";
+    }
+
+    if ($audio->{dev} eq 'AC97') {
+	push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
+    } elsif ($audio->{dev} =~ /intel\-hda$/) {
+	push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
+	push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
+	push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
+    } else {
+	die "unkown audio device '$audio->{dev}', implement me!";
+    }
+
+    push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
+
+    return $devs;
+}
+
 sub vga_conf_has_spice {
     my ($vga) = @_;
 
@@ -2911,7 +3008,7 @@ sub query_understood_cpu_flags {
 }
 
 sub config_to_command {
-    my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
+    my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
 
     my $cmd = [];
     my $globalFlags = [];
@@ -2937,18 +3034,22 @@ sub config_to_command {
     my $add_pve_version = min_version($kvmver, 4, 1);
 
     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
-    my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
+    my $machine_version = extract_version($machine_type, $kvmver);
     $kvm //= 1 if is_native($arch);
 
     $machine_version =~ m/(\d+)\.(\d+)/;
     my ($machine_major, $machine_minor) = ($1, $2);
-    die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
-	if !PVE::QemuServer::min_version($kvmver, $machine_major, $machine_minor);
 
-    if (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
+    if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
+	warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
+    } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
+	die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
+	    ." please upgrade node '$nodename'\n"
+    } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
 	my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
-	die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
-	  . " is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
+	die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
+	    ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
+	    ." node '$nodename'\n";
     }
 
     # if a specific +pve version is required for a feature, use $version_guard
@@ -2964,9 +3065,9 @@ sub config_to_command {
 	return 1;
     };
 
-    if ($kvm) {
-	die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
-	    if !defined kvm_version();
+    if ($kvm && !defined kvm_version()) {
+	die "KVM virtualisation configured, but not available. Either disable in VM configuration"
+	    ." or enable in BIOS.\n";
     }
 
     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
@@ -3023,12 +3124,11 @@ sub config_to_command {
 	}
     }
 
-    my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
-	die "uefi base image not found\n" if ! -f $ovmf_code;
+	my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
+	die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
 
-	my $path;
-	my $format;
+	my ($path, $format);
 	if (my $efidisk = $conf->{efidisk0}) {
 	    my $d = parse_drive('efidisk0', $efidisk);
 	    my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
@@ -3076,7 +3176,8 @@ sub config_to_command {
     }
 
     # add usb controllers
-    my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
+    my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
+        $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
     push @$devices, @usbcontrollers if @usbcontrollers;
     my $vga = parse_vga($conf->{vga});
 
@@ -3109,84 +3210,20 @@ sub config_to_command {
 	push @$devices, '-device', $kbd if defined($kbd);
     }
 
-    my $kvm_off = 0;
-    my $gpu_passthrough;
-
-    # host pci devices
-    for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
-	my $id = "hostpci$i";
-	my $d = parse_hostpci($conf->{$id});
-	next if !$d;
-
-	if (my $pcie = $d->{pcie}) {
-	    die "q35 machine model is not enabled" if !$q35;
-	    # win7 wants to have the pcie devices directly on the pcie bus
-	    # instead of in the root port
-	    if ($winversion == 7) {
-		$pciaddr = print_pcie_addr("${id}bus0");
-	    } else {
-		# add more root ports if needed, 4 are present by default
-		# by pve-q35 cfgs, rest added here on demand.
-		if ($i > 3) {
-		    push @$devices, '-device', print_pcie_root_port($i);
-		}
-		$pciaddr = print_pcie_addr($id);
-	    }
-	} else {
-	    $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
-	}
-
-	my $xvga = '';
-	if ($d->{'x-vga'}) {
-	    $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
-	    $kvm_off = 1;
-	    $vga->{type} = 'none' if !defined($conf->{vga});
-	    $gpu_passthrough = 1;
-	}
-
-	my $pcidevices = $d->{pciid};
-	my $multifunction = 1 if @$pcidevices > 1;
-
-	my $sysfspath;
-	if ($d->{mdev} && scalar(@$pcidevices) == 1) {
-	    my $pci_id = $pcidevices->[0]->{id};
-	    my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
-	    $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
-	} elsif ($d->{mdev}) {
-	    warn "ignoring mediated device '$id' with multifunction device\n";
-	}
-
-	my $j=0;
-	foreach my $pcidevice (@$pcidevices) {
-	    my $devicestr = "vfio-pci";
+    my $bootorder = device_bootorder($conf);
 
-	    if ($sysfspath) {
-		$devicestr .= ",sysfsdev=$sysfspath";
-	    } else {
-		$devicestr .= ",host=$pcidevice->{id}";
-	    }
-
-	    my $mf_addr = $multifunction ? ".$j" : '';
-	    $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
-
-	    if ($j == 0) {
-		$devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
-		$devicestr .= "$xvga";
-		$devicestr .= ",multifunction=on" if $multifunction;
-		$devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
-	    }
-
-	    push @$devices, '-device', $devicestr;
-	    $j++;
-	}
-    }
+    # host pci device passthrough
+    my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
+	$vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
 
     # usb devices
     my $usb_dev_features = {};
     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
 
-    my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
+    my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
+        $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
     push @$devices, @usbdevices if @usbdevices;
+
     # serial devices
     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 	if (my $path = $conf->{"serial$i"}) {
@@ -3219,22 +3256,10 @@ sub config_to_command {
 	}
     }
 
-    if (my $audio = conf_has_audio($conf)) {
-
+    if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
 	my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
-
-	my $id = $audio->{dev_id};
-	if ($audio->{dev} eq 'AC97') {
-	    push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
-	} elsif ($audio->{dev} =~ /intel\-hda$/) {
-	    push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
-	    push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
-	    push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
-	} else {
-	    die "unkown audio device '$audio->{dev}', implement me!";
-	}
-
-	push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
+	my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
+	push @$devices, @$audio_devs;
     }
 
     my $sockets = 1;
@@ -3266,15 +3291,6 @@ sub config_to_command {
     }
     push @$cmd, '-nodefaults';
 
-    my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
-
-    my $bootindex_hash = {};
-    my $i = 1;
-    foreach my $o (split(//, $bootorder)) {
-	$bootindex_hash->{$o} = $i*100;
-	$i++;
-    }
-
     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
 
     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
@@ -3282,7 +3298,8 @@ sub config_to_command {
     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
 
     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
-	push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
+	push @$devices, '-device', print_vga_device(
+	    $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
 	my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
 	push @$cmd,  '-vnc', "unix:$socket,password";
     } else {
@@ -3292,7 +3309,6 @@ sub config_to_command {
 
     # time drift fix
     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
-
     my $useLocaltime = $conf->{localtime};
 
     if ($winversion >= 5) { # windows
@@ -3311,13 +3327,17 @@ sub config_to_command {
 
     push @$rtcFlags, 'driftfix=slew' if $tdf;
 
-    if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
+    if ($conf->{startdate} && $conf->{startdate} ne 'now') {
 	push @$rtcFlags, "base=$conf->{startdate}";
     } elsif ($useLocaltime) {
 	push @$rtcFlags, 'base=localtime';
     }
 
-    push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
+    if ($forcecpu) {
+	push @$cmd, '-cpu', $forcecpu;
+    } else {
+	push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
+    }
 
     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
 
@@ -3340,22 +3360,18 @@ sub config_to_command {
 	}
     }
 
-    my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
-    if ($rng && &$version_guard(4, 1, 2)) {
+    my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
+    if ($rng && $version_guard->(4, 1, 2)) {
+	check_rng_source($rng->{source});
+
 	my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
 	my $period = $rng->{period} // $rng_fmt->{period}->{default};
-
 	my $limiter_str = "";
 	if ($max_bytes) {
 	    $limiter_str = ",max-bytes=$max_bytes,period=$period";
 	}
 
-	# mostly relevant for /dev/hwrng, but doesn't hurt to check others too
-	die "cannot create VirtIO RNG device: source file '$rng->{source}' doesn't exist\n"
-	    if ! -e $rng->{source};
-
 	my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
-
 	push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
 	push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
     }
@@ -3365,8 +3381,9 @@ sub config_to_command {
     if ($qxlnum) {
 	if ($qxlnum > 1) {
 	    if ($winversion){
-		for(my $i = 1; $i < $qxlnum; $i++){
-		    push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
+		for (my $i = 1; $i < $qxlnum; $i++){
+		    push @$devices, '-device', print_vga_device(
+		        $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
 		}
 	    } else {
 		# assume other OS works like Linux
@@ -3393,14 +3410,17 @@ sub config_to_command {
 	my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
 	$spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
 
-	my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
+	my $spice_enhancement_str = $conf->{spice_enhancements} // '';
+	my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
 	if ($spice_enhancement->{foldersharing}) {
 	    push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
 	    push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
 	}
 
 	my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
-	$spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
+	$spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
+	    if $spice_enhancement->{videostreaming};
+
 	push @$devices, '-spice', "$spice_opts";
     }
 
@@ -3428,7 +3448,7 @@ sub config_to_command {
 	push @$devices, '-iscsi', "initiator-name=$initiator";
     }
 
-    foreach_drive($conf, sub {
+    PVE::QemuConfig->foreach_volume($conf, sub {
 	my ($ds, $drive) = @_;
 
 	if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
@@ -3440,23 +3460,13 @@ sub config_to_command {
 
 	$use_virtio = 1 if $ds =~ m/^virtio/;
 
-	if (drive_is_cdrom ($drive)) {
-	    if ($bootindex_hash->{d}) {
-		$drive->{bootindex} = $bootindex_hash->{d};
-		$bootindex_hash->{d} += 1;
-	    }
-	} else {
-	    if ($bootindex_hash->{c}) {
-		$drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
-		$bootindex_hash->{c} += 1;
-	    }
-	}
+	$drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
 
-	if($drive->{interface} eq 'virtio'){
+	if ($drive->{interface} eq 'virtio'){
            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
 	}
 
-        if ($drive->{interface} eq 'scsi') {
+	if ($drive->{interface} eq 'scsi') {
 
 	    my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
 
@@ -3479,43 +3489,49 @@ sub config_to_command {
 		$queues = ",num_queues=$drive->{queues}";
 	    }
 
-	    push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
+	    push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
+		if !$scsicontroller->{$controller};
 	    $scsicontroller->{$controller}=1;
-        }
+	}
 
         if ($drive->{interface} eq 'sata') {
-           my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
-           $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
-           push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
-           $ahcicontroller->{$controller}=1;
+	    my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
+	    $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
+	    push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
+		if !$ahcicontroller->{$controller};
+	    $ahcicontroller->{$controller}=1;
         }
 
 	my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
+	$drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
+
 	push @$devices, '-drive',$drive_cmd;
-	push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
+	push @$devices, '-device', print_drivedevice_full(
+	    $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
     });
 
     for (my $i = 0; $i < $MAX_NETS; $i++) {
-         next if !$conf->{"net$i"};
-         my $d = parse_net($conf->{"net$i"});
-         next if !$d;
+	my $netname = "net$i";
+
+	next if !$conf->{$netname};
+	my $d = parse_net($conf->{$netname});
+	next if !$d;
 
-         $use_virtio = 1 if $d->{model} eq 'virtio';
+	$use_virtio = 1 if $d->{model} eq 'virtio';
 
-         if ($bootindex_hash->{n}) {
-            $d->{bootindex} = $bootindex_hash->{n};
-            $bootindex_hash->{n} += 1;
-         }
+	$d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
 
-         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
-         push @$devices, '-netdev', $netdevfull;
+	my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
+	push @$devices, '-netdev', $netdevfull;
 
-         my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
-         push @$devices, '-device', $netdevicefull;
+	my $netdevicefull = print_netdevice_full(
+	    $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
+
+	push @$devices, '-device', $netdevicefull;
     }
 
     if ($conf->{ivshmem}) {
-	my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
+	my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
 
 	my $bus;
 	if ($q35) {
@@ -3528,7 +3544,8 @@ sub config_to_command {
 	my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
 
 	push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
-	push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
+	push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
+	    .",size=$ivshmem->{size}M";
     }
 
     # pci.4 is nested in pci.1
@@ -3547,7 +3564,13 @@ sub config_to_command {
 
     for my $k (sort {$b cmp $a} keys %$bridges) {
 	next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
-	$pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
+
+	my $k_name = $k;
+	if ($k == 2 && $legacy_igd) {
+	    $k_name = "$k-igd";
+	}
+	$pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
+
 	my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
 	if ($q35) {
 	    # add after -readconfig pve-q35.cfg
@@ -3569,12 +3592,9 @@ sub config_to_command {
     push @$machineFlags, "type=${machine_type_min}";
 
     push @$cmd, @$devices;
-    push @$cmd, '-rtc', join(',', @$rtcFlags)
-	if scalar(@$rtcFlags);
-    push @$cmd, '-machine', join(',', @$machineFlags)
-	if scalar(@$machineFlags);
-    push @$cmd, '-global', join(',', @$globalFlags)
-	if scalar(@$globalFlags);
+    push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
+    push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
+    push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
 
     if (my $vmstate = $conf->{vmstate}) {
 	my $statepath = PVE::Storage::path($storecfg, $vmstate);
@@ -3592,6 +3612,23 @@ sub config_to_command {
     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
 }
 
+sub check_rng_source {
+    my ($source) = @_;
+
+    # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
+    die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
+	if ! -e $source;
+
+    my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
+    if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
+	# Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
+	# be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
+	die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
+	    ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
+	    ." to the host.\n";
+    }
+}
+
 sub spice_port {
     my ($vmid) = @_;
 
@@ -3658,7 +3695,8 @@ sub vm_deviceplug {
     my $devices_list = vm_devices_list($vmid);
     return 1 if defined($devices_list->{$deviceid});
 
-    qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
+    # add PCI bridge if we need it for the device
+    qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
 
     if ($deviceid eq 'tablet') {
 
@@ -3727,13 +3765,14 @@ sub vm_deviceplug {
 
     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
 
-	return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
+	return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
 
 	my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
 	my $use_old_bios_files = undef;
 	($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
 
-	my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
+	my $netdevicefull = print_netdevice_full(
+	    $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
 	qemu_deviceadd($vmid, $netdevicefull);
 	eval {
 	    qemu_deviceaddverify($vmid, $deviceid);
@@ -3768,7 +3807,8 @@ sub vm_deviceunplug {
     my $devices_list = vm_devices_list($vmid);
     return 1 if !defined($devices_list->{$deviceid});
 
-    die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
+    my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
+    die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
 
     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
 
@@ -4005,6 +4045,14 @@ sub qemu_netdevadd {
     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
     my %options =  split(/[=,]/, $netdev);
 
+    if (defined(my $vhost = $options{vhost})) {
+	$options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
+    }
+
+    if (defined(my $queues = $options{queues})) {
+	$options{queues} = $queues + 0;
+    }
+
     mon_cmd($vmid, "netdev_add",  %options);
     return 1;
 }
@@ -4147,39 +4195,6 @@ sub qemu_block_set_io_throttle {
 
 }
 
-# old code, only used to shutdown old VM after update
-sub __read_avail {
-    my ($fh, $timeout) = @_;
-
-    my $sel = new IO::Select;
-    $sel->add($fh);
-
-    my $res = '';
-    my $buf;
-
-    my @ready;
-    while (scalar (@ready = $sel->can_read($timeout))) {
-	my $count;
-	if ($count = $fh->sysread($buf, 8192)) {
-	    if ($buf =~ /^(.*)\(qemu\) $/s) {
-		$res .= $1;
-		last;
-	    } else {
-		$res .= $buf;
-	    }
-	} else {
-	    if (!defined($count)) {
-		die "$!\n";
-	    }
-	    last;
-	}
-    }
-
-    die "monitor read timeout\n" if !scalar(@ready);
-
-    return $res;
-}
-
 sub qemu_block_resize {
     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
 
@@ -4217,7 +4232,7 @@ sub qemu_volume_snapshot_delete {
 
 	$running = undef;
 	my $conf = PVE::QemuConfig->load_config($vmid);
-	foreach_drive($conf, sub {
+	PVE::QemuConfig->foreach_volume($conf, sub {
 	    my ($ds, $drive) = @_;
 	    $running = 1 if $drive->{file} eq $volid;
 	});
@@ -4255,6 +4270,59 @@ sub set_migration_caps {
     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
 }
 
+sub foreach_volid {
+    my ($conf, $func, @param) = @_;
+
+    my $volhash = {};
+
+    my $test_volid = sub {
+	my ($key, $drive, $snapname) = @_;
+
+	my $volid = $drive->{file};
+	return if !$volid;
+
+	$volhash->{$volid}->{cdrom} //= 1;
+	$volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
+
+	my $replicate = $drive->{replicate} // 1;
+	$volhash->{$volid}->{replicate} //= 0;
+	$volhash->{$volid}->{replicate} = 1 if $replicate;
+
+	$volhash->{$volid}->{shared} //= 0;
+	$volhash->{$volid}->{shared} = 1 if $drive->{shared};
+
+	$volhash->{$volid}->{referenced_in_config} //= 0;
+	$volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
+
+	$volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
+	    if defined($snapname);
+
+	my $size = $drive->{size};
+	$volhash->{$volid}->{size} //= $size if $size;
+
+	$volhash->{$volid}->{is_vmstate} //= 0;
+	$volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
+
+	$volhash->{$volid}->{is_unused} //= 0;
+	$volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
+    };
+
+    my $include_opts = {
+	extra_keys => ['vmstate'],
+	include_unused => 1,
+    };
+
+    PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
+    foreach my $snapname (keys %{$conf->{snapshots}}) {
+	my $snap = $conf->{snapshots}->{$snapname};
+	PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
+    }
+
+    foreach my $volid (keys %$volhash) {
+	&$func($volid, $volhash->{$volid}, @param);
+    }
+}
+
 my $fast_plug_option = {
     'lock' => 1,
     'name' => 1,
@@ -4402,7 +4470,7 @@ sub vmconfig_hotplug_pending {
 		# since we cannot reliably hot unplug usb devices
 		# we are disabling it
 		die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
-		my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
+		my $d = eval { parse_property_string($usbdesc->{format}, $value) };
 		die "skip\n" if !$d;
 		qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
 	    } elsif ($opt eq 'vcpus') {
@@ -4475,7 +4543,7 @@ sub try_deallocate_drive {
 	}
     }
 
-    return undef;
+    return;
 }
 
 sub vmconfig_delete_or_detach_drive {
@@ -4604,102 +4672,100 @@ sub vmconfig_update_disk {
 
     my $drive = parse_drive($opt, $value);
 
-    if ($conf->{$opt}) {
+    if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
+	my $media = $drive->{media} || 'disk';
+	my $oldmedia = $old_drive->{media} || 'disk';
+	die "unable to change media type\n" if $media ne $oldmedia;
 
-	if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
+	if (!drive_is_cdrom($old_drive)) {
 
-	    my $media = $drive->{media} || 'disk';
-	    my $oldmedia = $old_drive->{media} || 'disk';
-	    die "unable to change media type\n" if $media ne $oldmedia;
+	    if ($drive->{file} ne $old_drive->{file}) {
 
-	    if (!drive_is_cdrom($old_drive)) {
+		die "skip\n" if !$hotplug;
 
-		if ($drive->{file} ne $old_drive->{file}) {
-
-		    die "skip\n" if !$hotplug;
-
-		    # unplug and register as unused
-		    vm_deviceunplug($vmid, $conf, $opt);
-		    vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
+		# unplug and register as unused
+		vm_deviceunplug($vmid, $conf, $opt);
+		vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
 
-		} else {
-		    # update existing disk
-
-		    # skip non hotpluggable value
-		    if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
-			safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
-			safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
-			safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
-			safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
-			die "skip\n";
-		    }
+	    } else {
+		# update existing disk
+
+		# skip non hotpluggable value
+		if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
+		    safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
+		    safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
+		    safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
+		    safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
+		    die "skip\n";
+		}
 
-		    # apply throttle
-		    if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
-			safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
-			safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
-			safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
-			safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
-			safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
-			safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
-			safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
-			safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
-			safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
-			safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
-			safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
-			safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
-			safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
-			safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
-			safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
-			safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
-			safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
-
-			qemu_block_set_io_throttle($vmid,"drive-$opt",
-						   ($drive->{mbps} || 0)*1024*1024,
-						   ($drive->{mbps_rd} || 0)*1024*1024,
-						   ($drive->{mbps_wr} || 0)*1024*1024,
-						   $drive->{iops} || 0,
-						   $drive->{iops_rd} || 0,
-						   $drive->{iops_wr} || 0,
-						   ($drive->{mbps_max} || 0)*1024*1024,
-						   ($drive->{mbps_rd_max} || 0)*1024*1024,
-						   ($drive->{mbps_wr_max} || 0)*1024*1024,
-						   $drive->{iops_max} || 0,
-						   $drive->{iops_rd_max} || 0,
-						   $drive->{iops_wr_max} || 0,
-						   $drive->{bps_max_length} || 1,
-						   $drive->{bps_rd_max_length} || 1,
-						   $drive->{bps_wr_max_length} || 1,
-						   $drive->{iops_max_length} || 1,
-						   $drive->{iops_rd_max_length} || 1,
-						   $drive->{iops_wr_max_length} || 1);
+		# apply throttle
+		if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
+		    safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
+		    safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
+		    safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
+		    safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
+		    safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
+		    safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
+		    safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
+		    safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
+		    safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
+		    safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
+		    safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
+		    safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
+		    safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
+		    safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
+		    safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
+		    safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
+		    safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
+
+		    qemu_block_set_io_throttle(
+			$vmid,"drive-$opt",
+			($drive->{mbps} || 0)*1024*1024,
+			($drive->{mbps_rd} || 0)*1024*1024,
+			($drive->{mbps_wr} || 0)*1024*1024,
+			 $drive->{iops} || 0,
+			 $drive->{iops_rd} || 0,
+			 $drive->{iops_wr} || 0,
+			($drive->{mbps_max} || 0)*1024*1024,
+			($drive->{mbps_rd_max} || 0)*1024*1024,
+			($drive->{mbps_wr_max} || 0)*1024*1024,
+			 $drive->{iops_max} || 0,
+			 $drive->{iops_rd_max} || 0,
+			 $drive->{iops_wr_max} || 0,
+			 $drive->{bps_max_length} || 1,
+			 $drive->{bps_rd_max_length} || 1,
+			 $drive->{bps_wr_max_length} || 1,
+			 $drive->{iops_max_length} || 1,
+			 $drive->{iops_rd_max_length} || 1,
+			 $drive->{iops_wr_max_length} || 1,
+		    );
 
-		    }
+		}
 
-		    return 1;
-	        }
+		return 1;
+	    }
 
-	    } else { # cdrom
+	} else { # cdrom
 
-		if ($drive->{file} eq 'none') {
-		    mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
-		    if (drive_is_cloudinit($old_drive)) {
-			vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
-		    }
-		} else {
-		    my $path = get_iso_path($storecfg, $vmid, $drive->{file});
+	    if ($drive->{file} eq 'none') {
+		mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
+		if (drive_is_cloudinit($old_drive)) {
+		    vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
+		}
+	    } else {
+		my $path = get_iso_path($storecfg, $vmid, $drive->{file});
 
-		    # force eject if locked
-		    mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
+		# force eject if locked
+		mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
 
-		    if ($path) {
-			mon_cmd($vmid, "blockdev-change-medium",
-			    id => "$opt", filename => "$path");
-		    }
+		if ($path) {
+		    mon_cmd($vmid, "blockdev-change-medium",
+			id => "$opt", filename => "$path");
 		}
-
-		return 1;
 	    }
+
+	    return 1;
 	}
     }
 
@@ -4709,382 +4775,448 @@ sub vmconfig_update_disk {
     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
 }
 
-# params:
-#   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
-#   skiplock => 0/1, skip checking for config lock
-#   forcemachine => to force Qemu machine (rollback/migration)
-#   timeout => in seconds
-#   paused => start VM in paused state (backup)
-# migrate_opts:
-#   migratedfrom => source node
-#   spice_ticket => used for spice migration, passed via tunnel/stdin
-#   network => CIDR of migration network
-#   type => secure/insecure - tunnel over encrypted connection or plain-text
-#   targetstorage = storageid/'1' - target storage for disks migrated over NBD
-#   nbd_proto_version => int, 0 for TCP, 1 for UNIX
-#   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
-sub vm_start {
-    my ($storecfg, $vmid, $params, $migrate_opts) = @_;
+# called in locked context by incoming migration
+sub vm_migrate_get_nbd_disks {
+    my ($storecfg, $conf, $replicated_volumes) = @_;
 
-    PVE::QemuConfig->lock_config($vmid, sub {
-	my $statefile = $params->{statefile};
+    my $local_volumes = {};
+    PVE::QemuConfig->foreach_volume($conf, sub {
+	my ($ds, $drive) = @_;
 
-	my $migratedfrom = $migrate_opts->{migratedfrom};
-	my $migration_type = $migrate_opts->{type};
-	my $targetstorage = $migrate_opts->{targetstorage};
+	return if drive_is_cdrom($drive);
 
-	my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
+	my $volid = $drive->{file};
 
-	die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
+	return if !$volid;
 
-	my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
+	my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 
-	PVE::QemuConfig->check_lock($conf)
-	    if !($params->{skiplock} || $is_suspended);
+	my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+	return if $scfg->{shared};
 
-	die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
+	# replicated disks re-use existing state via bitmap
+	my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
+	$local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
+    });
+    return $local_volumes;
+}
 
-	# clean up leftover reboot request files
-	eval { clear_reboot_request($vmid); };
-	warn $@ if $@;
+# called in locked context by incoming migration
+sub vm_migrate_alloc_nbd_disks {
+    my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
 
-	if (!$statefile && scalar(keys %{$conf->{pending}})) {
-	    vmconfig_apply_pending($vmid, $conf, $storecfg);
-	    $conf = PVE::QemuConfig->load_config($vmid); # update/reload
-	}
+    my $format = undef;
 
-	PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
+    my $nbd = {};
+    foreach my $opt (sort keys %$source_volumes) {
+	my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
 
-	my $defaults = load_defaults();
+	if ($use_existing) {
+	    $nbd->{$opt}->{drivestr} = print_drive($drive);
+	    $nbd->{$opt}->{volid} = $volid;
+	    $nbd->{$opt}->{replicated} = 1;
+	    next;
+	}
 
-	# set environment variable useful inside network script
-	$ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
+	# If a remote storage is specified and the format of the original
+	# volume is not available there, fall back to the default format.
+	# Otherwise use the same format as the original.
+	if (!$storagemap->{identity}) {
+	    $storeid = map_storage($storagemap, $storeid);
+	    my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
+	    my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+	    my $fileFormat = qemu_img_format($scfg, $volname);
+	    $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
+	} else {
+	    my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+	    $format = qemu_img_format($scfg, $volname);
+	}
 
-	my $local_volumes = {};
+	my $size = $drive->{size} / 1024;
+	my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
+	my $newdrive = $drive;
+	$newdrive->{format} = $format;
+	$newdrive->{file} = $newvolid;
+	my $drivestr = print_drive($newdrive);
+	$nbd->{$opt}->{drivestr} = $drivestr;
+	$nbd->{$opt}->{volid} = $newvolid;
+    }
 
-	if ($targetstorage) {
-	    foreach_drive($conf, sub {
-		my ($ds, $drive) = @_;
+    return $nbd;
+}
 
-		return if drive_is_cdrom($drive);
+# see vm_start_nolock for parameters, additionally:
+# migrate_opts:
+#   storagemap = parsed storage map for allocating NBD disks
+sub vm_start {
+    my ($storecfg, $vmid, $params, $migrate_opts) = @_;
 
-		my $volid = $drive->{file};
+    return PVE::QemuConfig->lock_config($vmid, sub {
+	my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
 
-		return if !$volid;
+	die "you can't start a vm if it's a template\n"
+	    if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
 
-		my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
+	my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
 
-		my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
-		return if $scfg->{shared};
-		$local_volumes->{$ds} = [$volid, $storeid, $volname];
-	    });
+	PVE::QemuConfig->check_lock($conf)
+	    if !($params->{skiplock} || $has_suspended_lock);
 
-	    my $format = undef;
+	$params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
 
-	    foreach my $opt (sort keys %$local_volumes) {
+	die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
 
-		my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
-		if ($migrate_opts->{replicated_volumes}->{$volid}) {
-		    # re-use existing, replicated volume with bitmap on source side
-		    $local_volumes->{$opt} = $conf->{${opt}};
-		    print "re-using replicated volume: $opt - $volid\n";
-		    next;
-		}
-		my $drive = parse_drive($opt, $conf->{$opt});
-
-		# If a remote storage is specified and the format of the original
-		# volume is not available there, fall back to the default format.
-		# Otherwise use the same format as the original.
-		if ($targetstorage && $targetstorage ne "1") {
-		    $storeid = $targetstorage;
-		    my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
-		    my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
-		    my $fileFormat = qemu_img_format($scfg, $volname);
-		    $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
-		} else {
-		    my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
-		    $format = qemu_img_format($scfg, $volname);
-		}
+	if (my $storagemap = $migrate_opts->{storagemap}) {
+	    my $replicated = $migrate_opts->{replicated_volumes};
+	    my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
+	    $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
 
-		my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
-		my $newdrive = $drive;
-		$newdrive->{format} = $format;
-		$newdrive->{file} = $newvolid;
-		my $drivestr = print_drive($newdrive);
-		$local_volumes->{$opt} = $drivestr;
-		#pass drive to conf for command line
-		$conf->{$opt} = $drivestr;
+	    foreach my $opt (keys %{$migrate_opts->{nbd}}) {
+		$conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
 	    }
 	}
 
-	PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
+	return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
+    });
+}
 
-	my $forcemachine = $params->{forcemachine};
-	if ($is_suspended) {
-	    # enforce machine type on suspended vm to ensure HW compatibility
-	    $forcemachine = $conf->{runningmachine};
-	    print "Resuming suspended VM\n";
-	}
 
-	my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
+# params:
+#   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
+#   skiplock => 0/1, skip checking for config lock
+#   skiptemplate => 0/1, skip checking whether VM is template
+#   forcemachine => to force Qemu machine (rollback/migration)
+#   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
+#   timeout => in seconds
+#   paused => start VM in paused state (backup)
+#   resume => resume from hibernation
+# migrate_opts:
+#   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
+#   migratedfrom => source node
+#   spice_ticket => used for spice migration, passed via tunnel/stdin
+#   network => CIDR of migration network
+#   type => secure/insecure - tunnel over encrypted connection or plain-text
+#   nbd_proto_version => int, 0 for TCP, 1 for UNIX
+#   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
+sub vm_start_nolock {
+    my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
+
+    my $statefile = $params->{statefile};
+    my $resume = $params->{resume};
 
-	my $migration_ip;
-	my $get_migration_ip = sub {
-	    my ($nodename) = @_;
+    my $migratedfrom = $migrate_opts->{migratedfrom};
+    my $migration_type = $migrate_opts->{type};
 
-	    return $migration_ip if defined($migration_ip);
+    my $res = {};
 
-	    my $cidr = $migrate_opts->{network};
+    # clean up leftover reboot request files
+    eval { clear_reboot_request($vmid); };
+    warn $@ if $@;
 
-	    if (!defined($cidr)) {
-		my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
-		$cidr = $dc_conf->{migration}->{network};
-	    }
+    if (!$statefile && scalar(keys %{$conf->{pending}})) {
+	vmconfig_apply_pending($vmid, $conf, $storecfg);
+	$conf = PVE::QemuConfig->load_config($vmid); # update/reload
+    }
 
-	    if (defined($cidr)) {
-		my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
+    PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
 
-		die "could not get IP: no address configured on local " .
-		    "node for network '$cidr'\n" if scalar(@$ips) == 0;
+    my $defaults = load_defaults();
 
-		die "could not get IP: multiple addresses configured on local " .
-		    "node for network '$cidr'\n" if scalar(@$ips) > 1;
+    # set environment variable useful inside network script
+    $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
 
-		$migration_ip = @$ips[0];
-	    }
+    PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
 
-	    $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
-		if !defined($migration_ip);
+    my $forcemachine = $params->{forcemachine};
+    my $forcecpu = $params->{forcecpu};
+    if ($resume) {
+	# enforce machine and CPU type on suspended vm to ensure HW compatibility
+	$forcemachine = $conf->{runningmachine};
+	$forcecpu = $conf->{runningcpu};
+	print "Resuming suspended VM\n";
+    }
 
-	    return $migration_ip;
-	};
+    my ($cmd, $vollist, $spice_port) =
+	config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
 
-	my $migrate_uri;
-	if ($statefile) {
-	    if ($statefile eq 'tcp') {
-		my $localip = "localhost";
-		my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
-		my $nodename = nodename();
+    my $migration_ip;
+    my $get_migration_ip = sub {
+	my ($nodename) = @_;
 
-		if (!defined($migration_type)) {
-		    if (defined($datacenterconf->{migration}->{type})) {
-			$migration_type = $datacenterconf->{migration}->{type};
-		    } else {
-			$migration_type = 'secure';
-		    }
-		}
+	return $migration_ip if defined($migration_ip);
 
-		if ($migration_type eq 'insecure') {
-		    $localip = $get_migration_ip->($nodename);
-		    $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
-		}
+	my $cidr = $migrate_opts->{network};
 
-		my $pfamily = PVE::Tools::get_host_address_family($nodename);
-		my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
-		$migrate_uri = "tcp:${localip}:${migrate_port}";
-		push @$cmd, '-incoming', $migrate_uri;
-		push @$cmd, '-S';
+	if (!defined($cidr)) {
+	    my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
+	    $cidr = $dc_conf->{migration}->{network};
+	}
 
-	    } elsif ($statefile eq 'unix') {
-		# should be default for secure migrations as a ssh TCP forward
-		# tunnel is not deterministic reliable ready and fails regurarly
-		# to set up in time, so use UNIX socket forwards
-		my $socket_addr = "/run/qemu-server/$vmid.migrate";
-		unlink $socket_addr;
+	if (defined($cidr)) {
+	    my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
 
-		$migrate_uri = "unix:$socket_addr";
+	    die "could not get IP: no address configured on local " .
+		"node for network '$cidr'\n" if scalar(@$ips) == 0;
 
-		push @$cmd, '-incoming', $migrate_uri;
-		push @$cmd, '-S';
+	    die "could not get IP: multiple addresses configured on local " .
+		"node for network '$cidr'\n" if scalar(@$ips) > 1;
 
-	    } elsif (-e $statefile) {
-		push @$cmd, '-loadstate', $statefile;
-	    } else {
-		my $statepath = PVE::Storage::path($storecfg, $statefile);
-		push @$vollist, $statefile;
-		push @$cmd, '-loadstate', $statepath;
-	    }
-	} elsif ($params->{paused}) {
-	    push @$cmd, '-S';
+	    $migration_ip = @$ips[0];
 	}
 
-	# host pci devices
-        for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
-          my $d = parse_hostpci($conf->{"hostpci$i"});
-          next if !$d;
-	  my $pcidevices = $d->{pciid};
-	  foreach my $pcidevice (@$pcidevices) {
-		my $pciid = $pcidevice->{id};
+	$migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
+	    if !defined($migration_ip);
+
+	return $migration_ip;
+    };
 
-		my $info = PVE::SysFSTools::pci_device_info("$pciid");
-		die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
-		die "no pci device info for device '$pciid'\n" if !$info;
+    my $migrate_uri;
+    if ($statefile) {
+	if ($statefile eq 'tcp') {
+	    my $localip = "localhost";
+	    my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
+	    my $nodename = nodename();
 
-		if ($d->{mdev}) {
-		    my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
-		    PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
+	    if (!defined($migration_type)) {
+		if (defined($datacenterconf->{migration}->{type})) {
+		    $migration_type = $datacenterconf->{migration}->{type};
 		} else {
-		    die "can't unbind/bind pci group to vfio '$pciid'\n"
-			if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
-		    die "can't reset pci device '$pciid'\n"
-			if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
+		    $migration_type = 'secure';
 		}
-	  }
-        }
+	    }
 
-	PVE::Storage::activate_volumes($storecfg, $vollist);
+	    if ($migration_type eq 'insecure') {
+		$localip = $get_migration_ip->($nodename);
+		$localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
+	    }
 
-	eval {
-	    run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
-		outfunc => sub {}, errfunc => sub {});
-	};
-	# Issues with the above 'stop' not being fully completed are extremely rare, a very low
-	# timeout should be more than enough here...
-	PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
-
-	my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
-	                                          : $defaults->{cpuunits};
-
-	my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $is_suspended);
-	my %run_params = (
-	    timeout => $statefile ? undef : $start_timeout,
-	    umask => 0077,
-	    noerr => 1,
-	);
+	    my $pfamily = PVE::Tools::get_host_address_family($nodename);
+	    my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
+	    $migrate_uri = "tcp:${localip}:${migrate_port}";
+	    push @$cmd, '-incoming', $migrate_uri;
+	    push @$cmd, '-S';
 
-	# when migrating, prefix QEMU output so other side can pick up any
-	# errors that might occur and show the user
-	if ($migratedfrom) {
-	    $run_params{quiet} = 1;
-	    $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
-	}
+	} elsif ($statefile eq 'unix') {
+	    # should be default for secure migrations as a ssh TCP forward
+	    # tunnel is not deterministic reliable ready and fails regurarly
+	    # to set up in time, so use UNIX socket forwards
+	    my $socket_addr = "/run/qemu-server/$vmid.migrate";
+	    unlink $socket_addr;
 
-	my %properties = (
-	    Slice => 'qemu.slice',
-	    KillMode => 'none',
-	    CPUShares => $cpuunits
-	);
+	    $migrate_uri = "unix:$socket_addr";
+
+	    push @$cmd, '-incoming', $migrate_uri;
+	    push @$cmd, '-S';
 
-	if (my $cpulimit = $conf->{cpulimit}) {
-	    $properties{CPUQuota} = int($cpulimit * 100);
+	} elsif (-e $statefile) {
+	    push @$cmd, '-loadstate', $statefile;
+	} else {
+	    my $statepath = PVE::Storage::path($storecfg, $statefile);
+	    push @$vollist, $statefile;
+	    push @$cmd, '-loadstate', $statepath;
 	}
-	$properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
+    } elsif ($params->{paused}) {
+	push @$cmd, '-S';
+    }
+
+    # host pci devices
+    for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
+      my $d = parse_hostpci($conf->{"hostpci$i"});
+      next if !$d;
+      my $pcidevices = $d->{pciid};
+      foreach my $pcidevice (@$pcidevices) {
+	    my $pciid = $pcidevice->{id};
+
+	    my $info = PVE::SysFSTools::pci_device_info("$pciid");
+	    die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
+	    die "no pci device info for device '$pciid'\n" if !$info;
+
+	    if ($d->{mdev}) {
+		my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
+		PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
+	    } else {
+		die "can't unbind/bind pci group to vfio '$pciid'\n"
+		    if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
+		die "can't reset pci device '$pciid'\n"
+		    if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
+	    }
+      }
+    }
 
-	my $run_qemu = sub {
-	    PVE::Tools::run_fork sub {
-		PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
+    PVE::Storage::activate_volumes($storecfg, $vollist);
 
-		my $exitcode = run_command($cmd, %run_params);
-		die "QEMU exited with code $exitcode\n" if $exitcode;
-	    };
-	};
+    eval {
+	run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
+	    outfunc => sub {}, errfunc => sub {});
+    };
+    # Issues with the above 'stop' not being fully completed are extremely rare, a very low
+    # timeout should be more than enough here...
+    PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
+
+    my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
+	                                      : $defaults->{cpuunits};
+
+    my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
+    my %run_params = (
+	timeout => $statefile ? undef : $start_timeout,
+	umask => 0077,
+	noerr => 1,
+    );
 
-	if ($conf->{hugepages}) {
+    # when migrating, prefix QEMU output so other side can pick up any
+    # errors that might occur and show the user
+    if ($migratedfrom) {
+	$run_params{quiet} = 1;
+	$run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
+    }
 
-	    my $code = sub {
-		my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
-		my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
+    my %properties = (
+	Slice => 'qemu.slice',
+	KillMode => 'none',
+	CPUShares => $cpuunits
+    );
 
-		PVE::QemuServer::Memory::hugepages_mount();
-		PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
+    if (my $cpulimit = $conf->{cpulimit}) {
+	$properties{CPUQuota} = int($cpulimit * 100);
+    }
+    $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
 
-		eval { $run_qemu->() };
-		if (my $err = $@) {
-		    PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
-		    die $err;
-		}
+    my $run_qemu = sub {
+	PVE::Tools::run_fork sub {
+	    PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
 
-		PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
-	    };
-	    eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
+	    my $exitcode = run_command($cmd, %run_params);
+	    die "QEMU exited with code $exitcode\n" if $exitcode;
+	};
+    };
+
+    if ($conf->{hugepages}) {
+
+	my $code = sub {
+	    my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
+	    my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
+
+	    PVE::QemuServer::Memory::hugepages_mount();
+	    PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
 
-	} else {
 	    eval { $run_qemu->() };
-	}
+	    if (my $err = $@) {
+		PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
+		    if !$conf->{keephugepages};
+		die $err;
+	    }
 
-	if (my $err = $@) {
-	    # deactivate volumes if start fails
-	    eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
-	    die "start failed: $err";
-	}
+	    PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
+		if !$conf->{keephugepages};
+	};
+	eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
 
-	print "migration listens on $migrate_uri\n" if $migrate_uri;
+    } else {
+	eval { $run_qemu->() };
+    }
 
-	if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
-	    eval { mon_cmd($vmid, "cont"); };
-	    warn $@ if $@;
-	}
+    if (my $err = $@) {
+	# deactivate volumes if start fails
+	eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
+	die "start failed: $err";
+    }
 
-	#start nbd server for storage migration
-	if ($targetstorage) {
-	    my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
+    print "migration listens on $migrate_uri\n" if $migrate_uri;
+    $res->{migrate_uri} = $migrate_uri;
 
-	    my $migrate_storage_uri;
-	    # nbd_protocol_version > 0 for unix socket support
-	    if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
-		my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
-		mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
-		$migrate_storage_uri = "nbd:unix:$socket_path";
-	    } else {
-		my $nodename = nodename();
-		my $localip = $get_migration_ip->($nodename);
-		my $pfamily = PVE::Tools::get_host_address_family($nodename);
-		my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
+    if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
+	eval { mon_cmd($vmid, "cont"); };
+	warn $@ if $@;
+    }
 
-		mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
-		$localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
-		$migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
-	    }
+    #start nbd server for storage migration
+    if (my $nbd = $migrate_opts->{nbd}) {
+	my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
 
-	    foreach my $opt (sort keys %$local_volumes) {
-		my $drivestr = $local_volumes->{$opt};
-		mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
-		print "storage migration listens on $migrate_storage_uri:exportname=drive-$opt volume:$drivestr\n";
-	    }
+	my $migrate_storage_uri;
+	# nbd_protocol_version > 0 for unix socket support
+	if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
+	    my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
+	    mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
+	    $migrate_storage_uri = "nbd:unix:$socket_path";
+	} else {
+	    my $nodename = nodename();
+	    my $localip = $get_migration_ip->($nodename);
+	    my $pfamily = PVE::Tools::get_host_address_family($nodename);
+	    my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
+
+	    mon_cmd($vmid, "nbd-server-start", addr => {
+		type => 'inet',
+		data => {
+		    host => "${localip}",
+		    port => "${storage_migrate_port}",
+		},
+	    });
+	    $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
+	    $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
 	}
 
-	if ($migratedfrom) {
-	    eval {
-		set_migration_caps($vmid);
-	    };
-	    warn $@ if $@;
+	$res->{migrate_storage_uri} = $migrate_storage_uri;
 
-	    if ($spice_port) {
-	        print "spice listens on port $spice_port\n";
-		if ($migrate_opts->{spice_ticket}) {
-		    mon_cmd($vmid, "set_password", protocol => 'spice', password => $migrate_opts->{spice_ticket});
-		    mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
-		}
-	    }
+	foreach my $opt (sort keys %$nbd) {
+	    my $drivestr = $nbd->{$opt}->{drivestr};
+	    my $volid = $nbd->{$opt}->{volid};
+	    mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
+	    my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
+	    print "storage migration listens on $nbd_uri volume:$drivestr\n";
+	    print "re-using replicated volume: $opt - $volid\n"
+		if $nbd->{$opt}->{replicated};
 
-	} else {
-	    mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
-		if !$statefile && $conf->{balloon};
+	    $res->{drives}->{$opt} = $nbd->{$opt};
+	    $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
+	}
+    }
+
+    if ($migratedfrom) {
+	eval {
+	    set_migration_caps($vmid);
+	};
+	warn $@ if $@;
 
-	    foreach my $opt (keys %$conf) {
-		next if $opt !~  m/^net\d+$/;
-		my $nicconf = parse_net($conf->{$opt});
-		qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
+	if ($spice_port) {
+	    print "spice listens on port $spice_port\n";
+	    $res->{spice_port} = $spice_port;
+	    if ($migrate_opts->{spice_ticket}) {
+		mon_cmd($vmid, "set_password", protocol => 'spice', password =>
+		    $migrate_opts->{spice_ticket});
+		mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
 	    }
 	}
 
-	mon_cmd($vmid, 'qom-set',
-		    path => "machine/peripheral/balloon0",
-		    property => "guest-stats-polling-interval",
-		    value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
+    } else {
+	mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
+	    if !$statefile && $conf->{balloon};
 
-	if ($is_suspended) {
-	    print "Resumed VM, removing state\n";
-	    if (my $vmstate = $conf->{vmstate}) {
-		PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
-		PVE::Storage::vdisk_free($storecfg, $vmstate);
-	    }
-	    delete $conf->@{qw(lock vmstate runningmachine)};
-	    PVE::QemuConfig->write_config($vmid, $conf);
+	foreach my $opt (keys %$conf) {
+	    next if $opt !~  m/^net\d+$/;
+	    my $nicconf = parse_net($conf->{$opt});
+	    qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
 	}
+    }
 
-	PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
-    });
+    mon_cmd($vmid, 'qom-set',
+		path => "machine/peripheral/balloon0",
+		property => "guest-stats-polling-interval",
+		value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
+
+    if ($resume) {
+	print "Resumed VM, removing state\n";
+	if (my $vmstate = $conf->{vmstate}) {
+	    PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
+	    PVE::Storage::vdisk_free($storecfg, $vmstate);
+	}
+	delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
+	PVE::QemuConfig->write_config($vmid, $conf);
+    }
+
+    PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
+
+    return $res;
 }
 
 sub vm_commandline {
@@ -5092,13 +5224,15 @@ sub vm_commandline {
 
     my $conf = PVE::QemuConfig->load_config($vmid);
     my $forcemachine;
+    my $forcecpu;
 
     if ($snapname) {
 	my $snapshot = $conf->{snapshots}->{$snapname};
 	die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
 
-	# check for a 'runningmachine' in snapshot
-	$forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
+	# check for machine or CPU overrides in snapshot
+	$forcemachine = $snapshot->{runningmachine};
+	$forcecpu = $snapshot->{runningcpu};
 
 	$snapshot->{digest} = $conf->{digest}; # keep file digest for API
 
@@ -5107,7 +5241,8 @@ sub vm_commandline {
 
     my $defaults = load_defaults();
 
-    my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
+    my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
+	$forcemachine, $forcecpu);
 
     return PVE::Tools::cmd2string($cmd);
 }
@@ -5158,7 +5293,7 @@ sub vm_stop_cleanup {
 	}
 
 	if ($conf->{ivshmem}) {
-	    my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
+	    my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
 	    # just delete it for now, VMs which have this already open do not
 	    # are affected, but new VMs will get a separated one. If this
 	    # becomes an issue we either add some sort of ref-counting or just
@@ -5235,7 +5370,11 @@ sub _do_vm_stop {
 	    return;
 	}
     } else {
-	if ($force) {
+       if (!check_running($vmid, $nocheck)) {
+	    warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
+	    return;
+       }
+       if ($force) {
 	    warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
 	    kill 15, $pid;
 	} else {
@@ -5340,7 +5479,8 @@ sub vm_suspend {
 	    }
 
 
-	    $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
+	    $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
+	        $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
 	    $path = PVE::Storage::path($storecfg, $vmstate);
 	    PVE::QemuConfig->write_config($vmid, $conf);
 	} else {
@@ -5381,7 +5521,7 @@ sub vm_suspend {
 		    mon_cmd($vmid, "savevm-end");
 		    PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
 		    PVE::Storage::vdisk_free($storecfg, $vmstate);
-		    delete $conf->@{qw(vmstate runningmachine)};
+		    delete $conf->@{qw(vmstate runningmachine runningcpu)};
 		    PVE::QemuConfig->write_config($vmid, $conf);
 		};
 		warn $@ if $@;
@@ -5483,28 +5623,12 @@ sub tar_restore_cleanup {
 sub restore_file_archive {
     my ($archive, $vmid, $user, $opts) = @_;
 
-    my $format = $opts->{format};
-    my $comp;
-
-    if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
-	$format = 'tar' if !$format;
-	$comp = 'gzip';
-    } elsif ($archive =~ m/\.tar$/) {
-	$format = 'tar' if !$format;
-    } elsif ($archive =~ m/.tar.lzo$/) {
-	$format = 'tar' if !$format;
-	$comp = 'lzop';
-    } elsif ($archive =~ m/\.vma$/) {
-	$format = 'vma' if !$format;
-    } elsif ($archive =~ m/\.vma\.gz$/) {
-	$format = 'vma' if !$format;
-	$comp = 'gzip';
-    } elsif ($archive =~ m/\.vma\.lzo$/) {
-	$format = 'vma' if !$format;
-	$comp = 'lzop';
-    } else {
-	$format = 'vma' if !$format; # default
-    }
+    return restore_vma_archive($archive, $vmid, $user, $opts)
+	if $archive eq '-';
+
+    my $info = PVE::Storage::archive_info($archive);
+    my $format = $opts->{format} // $info->{format};
+    my $comp = $info->{compression};
 
     # try to detect archive format
     if ($format eq 'tar') {
@@ -5518,7 +5642,7 @@ sub restore_file_archive {
 my $restore_cleanup_oldconf = sub {
     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
 
-    foreach_drive($oldconf, sub {
+    PVE::QemuConfig->foreach_volume($oldconf, sub {
 	my ($ds, $drive) = @_;
 
 	return if drive_is_cdrom($drive, 1);
@@ -5595,12 +5719,13 @@ my $parse_backup_hints = sub {
 	    my $drive = parse_drive($virtdev, $2);
 	    if (drive_is_cloudinit($drive)) {
 		my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
+		$storeid = $options->{storage} if defined ($options->{storage});
 		my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
 		my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
 
 		$virtdev_hash->{$virtdev} = {
 		    format => $format,
-		    storeid => $options->{storage} // $storeid,
+		    storeid => $storeid,
 		    size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
 		    is_cloudinit => 1,
 		};
@@ -5638,7 +5763,8 @@ my $restore_allocate_devices = sub {
 	    $name .= ".$d->{format}" if $d->{format} ne 'raw';
 	}
 
-	my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
+	my $volid = PVE::Storage::vdisk_alloc(
+	    $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
 
 	print STDERR "new volume ID is '$volid'\n";
 	$d->{volid} = $volid;
@@ -5767,7 +5893,7 @@ sub update_disk_config {
     my ($vmid, $conf, $volid_hash) = @_;
 
     my $changes;
-    my $prefix = "VM $vmid:";
+    my $prefix = "VM $vmid";
 
     # used and unused disks
     my $referenced = {};
@@ -5784,22 +5910,22 @@ sub update_disk_config {
 
 	my $volid = $drive->{file};
 	return if !$volid;
+	my $volume = $volid_hash->{$volid};
 
 	# mark volid as "in-use" for next step
 	$referenced->{$volid} = 1;
-	if ($volid_hash->{$volid} &&
-	    (my $path = $volid_hash->{$volid}->{path})) {
+	if ($volume && (my $path = $volume->{path})) {
 	    $referencedpath->{$path} = 1;
 	}
 
 	return if drive_is_cdrom($drive);
-	return if !$volid_hash->{$volid};
+	return if !$volume;
 
-	my ($updated, $old_size, $new_size) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash);
+	my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
 	if (defined($updated)) {
 	    $changes = 1;
 	    $conf->{$opt} = print_drive($updated);
-	    print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
+	    print "$prefix ($opt): $msg\n";
 	}
     });
 
@@ -5810,7 +5936,8 @@ sub update_disk_config {
 	my $volid = $drive->{file};
 	return if !$volid;
 
-	my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
+	my $path;
+	$path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
 	if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
 	    print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
 	    $changes = 1;
@@ -5898,8 +6025,11 @@ sub restore_proxmox_backup_archive {
     my $datastore = $scfg->{datastore};
     my $username = $scfg->{username} // 'root@pam';
     my $fingerprint = $scfg->{fingerprint};
+    my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
 
     my $repo = "$username\@$server:$datastore";
+
+    # This is only used for `pbs-restore`!
     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
     local $ENV{PBS_PASSWORD} = $password;
     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
@@ -5961,7 +6091,9 @@ sub restore_proxmox_backup_archive {
 	    }
 	}
 
-	my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
+	my $is_qemu_server_backup = scalar(
+	    grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
+	);
 	if (!$is_qemu_server_backup) {
 	    die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
 	}
@@ -5980,7 +6112,7 @@ sub restore_proxmox_backup_archive {
 	}
 
 	my $fh = IO::File->new($cfgfn, "r") ||
-	    "unable to read qemu-server.conf - $!\n";
+	    die "unable to read qemu-server.conf - $!\n";
 
 	my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
 
@@ -6002,6 +6134,7 @@ sub restore_proxmox_backup_archive {
 
 	    my $path = PVE::Storage::path($storecfg, $volid);
 
+	    # This is the ONLY user of the PBS_ env vars set on top of this function!
 	    my $pbs_restore_cmd = [
 		'/usr/bin/pbs-restore',
 		'--repository', $repo,
@@ -6011,6 +6144,9 @@ sub restore_proxmox_backup_archive {
 		'--verbose',
 		];
 
+	    push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
+	    push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
+
 	    if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
 		push @$pbs_restore_cmd, '--skip-zero';
 	    }
@@ -6022,8 +6158,7 @@ sub restore_proxmox_backup_archive {
 
 	$fh->seek(0, 0) || die "seek failed - $!\n";
 
-	my $outfd = new IO::File ($tmpfn, "w") ||
-	    die "unable to write config for VM $vmid\n";
+	my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
 
 	my $cookie = { netcount => 0 };
 	while (defined(my $line = <$fh>)) {
@@ -6091,14 +6226,9 @@ sub restore_vma_archive {
     }
 
     if ($comp) {
-	my $cmd;
-	if ($comp eq 'gzip') {
-	    $cmd = ['zcat', $readfrom];
-	} elsif ($comp eq 'lzop') {
-	    $cmd = ['lzop', '-d', '-c', $readfrom];
-	} else {
-	    die "unknown compression method '$comp'\n";
-	}
+	my $info = PVE::Storage::decompressor_info('vma', $comp);
+	my $cmd = $info->{decompressor};
+	push @$cmd, $readfrom;
 	$add_pipe->($cmd);
     }
 
@@ -6142,7 +6272,7 @@ sub restore_vma_archive {
 
 	# we can read the config - that is already extracted
 	my $fh = IO::File->new($cfgfn, "r") ||
-	    "unable to read qemu-server.conf - $!\n";
+	    die "unable to read qemu-server.conf - $!\n";
 
 	my $fwcfgfn = "$tmpdir/qemu-server.fw";
 	if (-f $fwcfgfn) {
@@ -6201,8 +6331,7 @@ sub restore_vma_archive {
 
 	$fh->seek(0, 0) || die "seek failed - $!\n";
 
-	my $outfd = new IO::File ($tmpfn, "w") ||
-	    die "unable to write config for VM $vmid\n";
+	my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
 
 	my $cookie = { netcount => 0 };
 	while (defined(my $line = <$fh>)) {
@@ -6278,7 +6407,7 @@ sub restore_tar_archive {
 
     if ($archive ne '-') {
 	my $firstfile = tar_archive_read_firstfile($archive);
-	die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
+	die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
 	    if $firstfile ne 'qemu-server.conf';
     }
 
@@ -6352,11 +6481,9 @@ sub restore_tar_archive {
 
 	my $confsrc = "$tmpdir/qemu-server.conf";
 
-	my $srcfd = new IO::File($confsrc, "r") ||
-	    die "unable to open file '$confsrc'\n";
+	my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
 
-	my $outfd = new IO::File ($tmpfn, "w") ||
-	    die "unable to write config for VM $vmid\n";
+	my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
 
 	my $cookie = { netcount => 0 };
 	while (defined (my $line = <$srcfd>)) {
@@ -6388,7 +6515,7 @@ sub foreach_storage_used_by_vm {
 
     my $sidhash = {};
 
-    foreach_drive($conf, sub {
+    PVE::QemuConfig->foreach_volume($conf, sub {
 	my ($ds, $drive) = @_;
 	return if drive_is_cdrom($drive);
 
@@ -6411,6 +6538,7 @@ sub do_snapshots_with_qemu {
 
     my $storage_name = PVE::Storage::parse_volume_id($volid);
     my $scfg = $storecfg->{ids}->{$storage_name};
+    die "could not find storage '$storage_name'\n" if !defined($scfg);
 
     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
 	return 1;
@@ -6420,7 +6548,7 @@ sub do_snapshots_with_qemu {
 	return 1;
     }
 
-    return undef;
+    return;
 }
 
 sub qga_check_running {
@@ -6439,7 +6567,7 @@ sub template_create {
 
     my $storecfg = PVE::Storage::config();
 
-    foreach_drive($conf, sub {
+    PVE::QemuConfig->foreach_volume($conf, sub {
 	my ($ds, $drive) = @_;
 
 	return if drive_is_cdrom($drive);
@@ -6775,10 +6903,10 @@ sub clone_disk {
 	$storeid = $storage if $storage;
 
 	my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
-	my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
 
 	print "create full clone of drive $drivename ($drive->{file})\n";
 	my $name = undef;
+	my $size = undef;
 	if (drive_is_cloudinit($drive)) {
 	    $name = "vm-$newvmid-cloudinit";
 	    $name .= ".$dst_format" if $dst_format ne 'raw';
@@ -6786,8 +6914,11 @@ sub clone_disk {
 	    $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
 	} elsif ($drivename eq 'efidisk0') {
 	    $size = get_efivars_size($conf);
+	} else {
+	    ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
 	}
-	$newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
+	$size /= 1024;
+	$newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, $size);
 	push @$newvollist, $newvolid;
 
 	PVE::Storage::activate_volumes($storecfg, [$newvolid]);
@@ -6805,7 +6936,8 @@ sub clone_disk {
 		# that is given by the OVMF_VARS.fd
 		my $src_path = PVE::Storage::path($storecfg, $drive->{file});
 		my $dst_path = PVE::Storage::path($storecfg, $newvolid);
-		run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", "if=$src_path", "of=$dst_path"]);
+		run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size",
+		    "if=$src_path", "of=$dst_path"]);
 	    } else {
 		qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
 	    }
@@ -6817,7 +6949,8 @@ sub clone_disk {
 		    if $drive->{iothread};
 	    }
 
-	    qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
+	    qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
+	        $completion, $qga, $bwlimit);
 	}
     }
 
@@ -6849,7 +6982,7 @@ sub qemu_use_old_bios_files {
         $machine_type = $1;
         $use_old_bios_files = 1;
     } else {
-	my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
+	my $version = extract_version($machine_type, kvm_user_version());
         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
         # load new efi bios files on migration. So this hack is required to allow
         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
@@ -6924,7 +7057,9 @@ sub scsihw_infos {
     }
 
     my $controller = int($drive->{index} / $maxdev);
-    my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
+    my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
+	? "virtioscsi"
+	: "scsihw";
 
     return ($maxdev, $controller, $controller_prefix);
 }
@@ -7027,6 +7162,94 @@ sub clear_reboot_request {
     return $res;
 }
 
+sub bootorder_from_legacy {
+    my ($conf, $bootcfg) = @_;
+
+    my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
+    my $bootindex_hash = {};
+    my $i = 1;
+    foreach my $o (split(//, $boot)) {
+	$bootindex_hash->{$o} = $i*100;
+	$i++;
+    }
+
+    my $bootorder = {};
+
+    PVE::QemuConfig->foreach_volume($conf, sub {
+	my ($ds, $drive) = @_;
+
+	if (drive_is_cdrom ($drive, 1)) {
+	    if ($bootindex_hash->{d}) {
+		$bootorder->{$ds} = $bootindex_hash->{d};
+		$bootindex_hash->{d} += 1;
+	    }
+	} elsif ($bootindex_hash->{c}) {
+	    $bootorder->{$ds} = $bootindex_hash->{c}
+		if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
+	    $bootindex_hash->{c} += 1;
+	}
+    });
+
+    if ($bootindex_hash->{n}) {
+	for (my $i = 0; $i < $MAX_NETS; $i++) {
+	    my $netname = "net$i";
+	    next if !$conf->{$netname};
+	    $bootorder->{$netname} = $bootindex_hash->{n};
+	    $bootindex_hash->{n} += 1;
+	}
+    }
+
+    return $bootorder;
+}
+
+# Generate default device list for 'boot: order=' property. Matches legacy
+# default boot order, but with explicit device names. This is important, since
+# the fallback for when neither 'order' nor the old format is specified relies
+# on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
+sub get_default_bootdevices {
+    my ($conf) = @_;
+
+    my @ret = ();
+
+    # harddisk
+    my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
+    push @ret, $first if $first;
+
+    # cdrom
+    $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
+    push @ret, $first if $first;
+
+    # network
+    for (my $i = 0; $i < $MAX_NETS; $i++) {
+	my $netname = "net$i";
+	next if !$conf->{$netname};
+	push @ret, $netname;
+	last;
+    }
+
+    return \@ret;
+}
+
+sub device_bootorder {
+    my ($conf) = @_;
+
+    return bootorder_from_legacy($conf) if !defined($conf->{boot});
+
+    my $boot = parse_property_string($boot_fmt, $conf->{boot});
+
+    my $bootorder = {};
+    if (!defined($boot) || $boot->{legacy}) {
+	$bootorder = bootorder_from_legacy($conf, $boot);
+    } elsif ($boot->{order}) {
+	my $i = 100; # start at 100 to allow user to insert devices before us with -args
+	for my $dev (PVE::Tools::split_list($boot->{order})) {
+	    $bootorder->{$dev} = $i++;
+	}
+    }
+
+    return $bootorder;
+}
+
 # bash completion helper
 
 sub complete_backup_archives {
@@ -7045,7 +7268,7 @@ sub complete_backup_archives {
     my $res = [];
     foreach my $id (keys %$data) {
 	foreach my $item (@{$data->{$id}}) {
-	    next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
+	    next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
 	    push @$res, $item->{volid} if defined($item->{volid});
 	}
     }