add new 'boot' property format and introduce legacy conversion helpers

[qemu-server.git] / PVE / QemuServer.pm

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 9c9debfd79fa89bc8a74c851572056aa275a1023..cfac03a92858236763b8f9637efd04af2f94f8b8 100644 (file)
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -26,29 +26,29 @@ use Time::HiRes qw(gettimeofday);
 use URI::Escape;
 use UUID;
 
 use URI::Escape;
 use UUID;
 

-use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
+use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);

 use PVE::DataCenterConfig;
 use PVE::Exception qw(raise raise_param_exc);
 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
 use PVE::INotify;
 use PVE::DataCenterConfig;
 use PVE::Exception qw(raise raise_param_exc);
 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
 use PVE::INotify;

-use PVE::JSONSchema qw(get_standard_option);
+use PVE::JSONSchema qw(get_standard_option parse_property_string);

 use PVE::ProcFSTools;
 use PVE::RPCEnvironment;
 use PVE::Storage;
 use PVE::SysFSTools;
 use PVE::Systemd;
 use PVE::ProcFSTools;
 use PVE::RPCEnvironment;
 use PVE::Storage;
 use PVE::SysFSTools;
 use PVE::Systemd;

-use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
+use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);

 
 use PVE::QMPClient;
 use PVE::QemuConfig;
 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
 use PVE::QemuServer::Cloudinit;
 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
 
 use PVE::QMPClient;
 use PVE::QemuConfig;
 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
 use PVE::QemuServer::Cloudinit;
 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);

-use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive foreach_drive foreach_volid);
+use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);

 use PVE::QemuServer::Machine;
 use PVE::QemuServer::Memory;
 use PVE::QemuServer::Monitor qw(mon_cmd);
 use PVE::QemuServer::Machine;
 use PVE::QemuServer::Memory;
 use PVE::QemuServer::Monitor qw(mon_cmd);

-use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
+use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);

 use PVE::QemuServer::USB qw(parse_usb_device);
 
 my $have_sdn;
 use PVE::QemuServer::USB qw(parse_usb_device);
 
 my $have_sdn;


@@ -97,6 +97,28 @@ PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
        optional => 1,
 });
 
        optional => 1,
 });
 

+
+sub map_storage {
+    my ($map, $source) = @_;
+
+    return $source if !defined($map);
+
+    return $map->{entries}->{$source}
+       if $map->{entries} && defined($map->{entries}->{$source});
+
+    return $map->{default} if $map->{default};
+
+    # identity (fallback)
+    return $source;
+}
+
+PVE::JSONSchema::register_standard_option('pve-targetstorage', {
+    description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
+    type => 'string',
+    format => 'storagepair-list',
+    optional => 1,
+});
+

 #no warnings 'redefine';
 
 sub cgroups_write {
 #no warnings 'redefine';
 
 sub cgroups_write {


@@ -419,6 +441,13 @@ EODESC
        description => "Enable/disable hugepages memory.",
        enum => [qw(any 2 1024)],
     },
        description => "Enable/disable hugepages memory.",
        enum => [qw(any 2 1024)],
     },

+    keephugepages => {
+       optional => 1,
+       type => 'boolean',
+       default => 0,
+       description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
+           ." after VM shutdown and can be used for subsequent starts.",
+    },

     vcpus => {
        optional => 1,
        type => 'integer',
     vcpus => {
        optional => 1,
        type => 'integer',


@@ -453,7 +482,8 @@ EODESC
     localtime => {
        optional => 1,
        type => 'boolean',
     localtime => {
        optional => 1,
        type => 'boolean',

-       description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
+       description => "Set the real time clock to local time. This is enabled by default if ostype"
+           ." indicates a Microsoft OS.",

     },
     freeze => {
        optional => 1,
     },
     freeze => {
        optional => 1,


@@ -464,29 +494,28 @@ EODESC
        optional => 1,
        type => 'string', format => $vga_fmt,
        description => "Configure the VGA hardware.",
        optional => 1,
        type => 'string', format => $vga_fmt,
        description => "Configure the VGA hardware.",

-       verbose_description => "Configure the VGA Hardware. If you want to use ".
-           "high resolution modes (>= 1280x1024x16) you may need to increase " .
-           "the vga memory option. Since QEMU 2.9 the default VGA display type " .
-           "is 'std' for all OS types besides some Windows versions (XP and " .
-           "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
-           "display server. For win* OS you can select how many independent " .
-           "displays you want, Linux guests can add displays them self.\n".
-           "You can also run without any graphic card, using a serial device as terminal.",
+       verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
+           ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
+           ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
+           ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
+           ." display server. For win* OS you can select how many independent displays you want,"
+           ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
+           ." using a serial device as terminal.",

     },
     watchdog => {
        optional => 1,
        type => 'string', format => 'pve-qm-watchdog',
        description => "Create a virtual hardware watchdog device.",
     },
     watchdog => {
        optional => 1,
        type => 'string', format => 'pve-qm-watchdog',
        description => "Create a virtual hardware watchdog device.",

-       verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
-           " (by a guest action), the watchdog must be periodically polled " .
-           "by an agent inside the guest or else the watchdog will reset " .
-           "the guest (or execute the respective action specified)",
+       verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
+           ." action), the watchdog must be periodically polled by an agent inside the guest or"
+           ." else the watchdog will reset the guest (or execute the respective action specified)",

     },
     startdate => {
        optional => 1,
        type => 'string',
        typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
     },
     startdate => {
        optional => 1,
        type => 'string',
        typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",

-       description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
+       description => "Set the initial date of the real time clock. Valid format for date are:"
+           ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",

        pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
        default => 'now',
     },
        pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
        default => 'now',
     },


@@ -514,12 +543,11 @@ EODESCR
        type => 'boolean',
        default => 1,
        description => "Enable/disable the USB tablet device.",
        type => 'boolean',
        default => 1,
        description => "Enable/disable the USB tablet device.",

-       verbose_description => "Enable/disable the USB tablet device. This device is " .
-           "usually needed to allow absolute mouse positioning with VNC. " .
-           "Else the mouse runs out of sync with normal VNC clients. " .
-           "If you're running lots of console-only guests on one host, " .
-           "you may consider disabling this to save some context switches. " .
-           "This is turned off by default if you use spice (-vga=qxl).",
+       verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
+           ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
+           ." normal VNC clients. If you're running lots of console-only guests on one host, you"
+           ." may consider disabling this to save some context switches. This is turned off by"
+           ." default if you use spice (`qm set <vmid> --vga qxl`).",

     },
     migrate_speed => {
        optional => 1,
     },
     migrate_speed => {
        optional => 1,


@@ -560,15 +588,25 @@ EODESCR
     vmstate => {
        optional => 1,
        type => 'string', format => 'pve-volume-id',
     vmstate => {
        optional => 1,
        type => 'string', format => 'pve-volume-id',

-       description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
+       description => "Reference to a volume which stores the VM state. This is used internally"
+           ." for snapshots.",

     },
     vmstatestorage => get_standard_option('pve-storage-id', {
        description => "Default storage for VM state volumes/files.",
        optional => 1,
     }),
     runningmachine => get_standard_option('pve-qemu-machine', {
     },
     vmstatestorage => get_standard_option('pve-storage-id', {
        description => "Default storage for VM state volumes/files.",
        optional => 1,
     }),
     runningmachine => get_standard_option('pve-qemu-machine', {

-       description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
+       description => "Specifies the QEMU machine type of the running vm. This is used internally"
+           ." for snapshots.",

     }),
     }),

+    runningcpu => {
+       description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
+           ." internally for snapshots.",
+       optional => 1,
+       type => 'string',
+       pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
+       format_description => 'QEMU -cpu parameter'
+    },

     machine => get_standard_option('pve-qemu-machine'),
     arch => {
        description => "Virtual processor architecture. Defaults to the host.",
     machine => get_standard_option('pve-qemu-machine'),
     arch => {
        description => "Virtual processor architecture. Defaults to the host.",


@@ -585,7 +623,8 @@ EODESCR
     protection => {
        optional => 1,
        type => 'boolean',
     protection => {
        optional => 1,
        type => 'boolean',

-       description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
+       description => "Sets the protection flag of the VM. This will disable the remove VM and"
+           ." remove disk operations.",

        default => 0,
     },
     bios => {
        default => 0,
     },
     bios => {


@@ -599,17 +638,16 @@ EODESCR
        type => 'string',
        pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
        format_description => 'UUID',
        type => 'string',
        pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
        format_description => 'UUID',

-       description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
-       verbose_description => "The VM generation ID (vmgenid) device exposes a".
-           " 128-bit integer value identifier to the guest OS. This allows to".
-           " notify the guest operating system when the virtual machine is".
-           " executed with a different configuration (e.g. snapshot execution".
-           " or creation from a template). The guest operating system notices".
-           " the change, and is then able to react as appropriate by marking".
-           " its copies of distributed databases as dirty, re-initializing its".
-           " random number generator, etc.\n".
-           "Note that auto-creation only works when done throug API/CLI create".
-           " or update methods, but not when manually editing the config file.",
+       description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
+           ." to disable explicitly.",
+       verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
+           ." value identifier to the guest OS. This allows to notify the guest operating system"
+           ." when the virtual machine is executed with a different configuration (e.g. snapshot"
+           ." execution or creation from a template). The guest operating system notices the"
+           ." change, and is then able to react as appropriate by marking its copies of"
+           ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
+           ."Note that auto-creation only works when done through API/CLI create or update methods"
+           .", but not when manually editing the config file.",

        default => "1 (autogenerated)",
        optional => 1,
     },
        default => "1 (autogenerated)",
        optional => 1,
     },


@@ -622,7 +660,8 @@ EODESCR
     ivshmem => {
        type => 'string',
        format => $ivshmem_fmt,
     ivshmem => {
        type => 'string',
        format => $ivshmem_fmt,

-       description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
+       description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
+           ." the host.",

        optional => 1,
     },
     audio0 => {
        optional => 1,
     },
     audio0 => {


@@ -654,21 +693,24 @@ my $cicustom_fmt = {
     meta => {
        type => 'string',
        optional => 1,
     meta => {
        type => 'string',
        optional => 1,

-       description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
+       description => 'Specify a custom file containing all meta data passed to the VM via"
+           ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',

        format => 'pve-volume-id',
        format_description => 'volume',
     },
     network => {
        type => 'string',
        optional => 1,
        format => 'pve-volume-id',
        format_description => 'volume',
     },
     network => {
        type => 'string',
        optional => 1,

-       description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
+       description => 'Specify a custom file containing all network data passed to the VM via'
+           .' cloud-init.',

        format => 'pve-volume-id',
        format_description => 'volume',
     },
     user => {
        type => 'string',
        optional => 1,
        format => 'pve-volume-id',
        format_description => 'volume',
     },
     user => {
        type => 'string',
        optional => 1,

-       description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
+       description => 'Specify a custom file containing all user data passed to the VM via'
+           .' cloud-init.',

        format => 'pve-volume-id',
        format_description => 'volume',
     },
        format => 'pve-volume-id',
        format_description => 'volume',
     },


@@ -679,34 +721,44 @@ my $confdesc_cloudinit = {
     citype => {
        optional => 1,
        type => 'string',
     citype => {
        optional => 1,
        type => 'string',

-       description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
+       description => 'Specifies the cloud-init configuration format. The default depends on the'
+           .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
+           .' and `configdrive2` for windows.',

        enum => ['configdrive2', 'nocloud'],
     },
     ciuser => {
        optional => 1,
        type => 'string',
        enum => ['configdrive2', 'nocloud'],
     },
     ciuser => {
        optional => 1,
        type => 'string',

-       description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
+       description => "cloud-init: User name to change ssh keys and password for instead of the"
+           ." image's configured default user.",

     },
     cipassword => {
        optional => 1,
        type => 'string',
     },
     cipassword => {
        optional => 1,
        type => 'string',

-       description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
+       description => 'cloud-init: Password to assign the user. Using this is generally not'
+           .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
+           .' support hashed passwords.',

     },
     cicustom => {
        optional => 1,
        type => 'string',
     },
     cicustom => {
        optional => 1,
        type => 'string',

-       description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
+       description => 'cloud-init: Specify custom files to replace the automatically generated'
+           .' ones at start.',

        format => 'pve-qm-cicustom',
     },
     searchdomain => {
        optional => 1,
        type => 'string',
        format => 'pve-qm-cicustom',
     },
     searchdomain => {
        optional => 1,
        type => 'string',

-       description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
+       description => "cloud-init: Sets DNS search domains for a container. Create will'
+           .' automatically use the setting from the host if neither searchdomain nor nameserver'
+           .' are set.",

     },
     nameserver => {
        optional => 1,
        type => 'string', format => 'address-list',
     },
     nameserver => {
        optional => 1,
        type => 'string', format => 'address-list',

-       description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
+       description => "cloud-init: Sets DNS server IP address for a container. Create will'
+           .' automatically use the setting from the host if neither searchdomain nor nameserver'
+           .' are set.",

     },
     sshkeys => {
        optional => 1,
     },
     sshkeys => {
        optional => 1,


@@ -739,7 +791,6 @@ while (my ($k, $v) = each %$confdesc) {
 
 my $MAX_USB_DEVICES = 5;
 my $MAX_NETS = 32;
 
 my $MAX_USB_DEVICES = 5;
 my $MAX_NETS = 32;

-my $MAX_HOSTPCI_DEVICES = 16;

 my $MAX_SERIAL_PORTS = 4;
 my $MAX_PARALLEL_PORTS = 3;
 my $MAX_NUMA = 8;
 my $MAX_SERIAL_PORTS = 4;
 my $MAX_PARALLEL_PORTS = 3;
 my $MAX_NUMA = 8;


@@ -804,11 +855,14 @@ __EOD__
 
 my $net_fmt = {
     macaddr  => get_standard_option('mac-addr', {
 
 my $net_fmt = {
     macaddr  => get_standard_option('mac-addr', {

-       description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
+       description => "MAC address. That address must be unique withing your network. This is"
+           ." automatically generated if not specified.",

     }),
     model => {
        type => 'string',
     }),
     model => {
        type => 'string',

-       description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
+       description => "Network Card Model. The 'virtio' model provides the best performance with"
+           ." very low CPU overhead. If your guest does not support this driver, it is usually"
+           ." best to use 'e1000'.",

         enum => $nic_model_list,
         default_key => 1,
     },
         enum => $nic_model_list,
         default_key => 1,
     },


@@ -817,6 +871,7 @@ my $net_fmt = {
        type => 'string',
        description => $net_fmt_bridge_descr,
        format_description => 'bridge',
        type => 'string',
        description => $net_fmt_bridge_descr,
        format_description => 'bridge',

+       pattern => '[-_.\w\d]+',

        optional => 1,
     },
     queues => {
        optional => 1,
     },
     queues => {


@@ -854,6 +909,12 @@ my $net_fmt = {
        description => 'Whether this interface should be disconnected (like pulling the plug).',
        optional => 1,
     },
        description => 'Whether this interface should be disconnected (like pulling the plug).',
        optional => 1,
     },

+    mtu => {
+       type => 'integer',
+       minimum => 1, maximum => 65520,
+       description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
+       optional => 1,
+    },

 };
 
 my $netdesc = {
 };
 
 my $netdesc = {


@@ -907,10 +968,12 @@ cloud-init: Specify IP addresses and gateways for the corresponding interface.
 
 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 
 
 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 

-The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
+The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
+gateway should be provided.

 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 
 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 

-If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
+If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
+dhcp on IPv4.

 EODESCR
 };
 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 EODESCR
 };
 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);


@@ -955,7 +1018,8 @@ The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 
 You can use the 'lsusb -t' command to list existing usb devices.
 
 
 You can use the 'lsusb -t' command to list existing usb devices.
 

-NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
+NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
+machines - use with special care.

 
 The value 'spice' can be used to add a usb redirection devices for spice.
 EODESCR
 
 The value 'spice' can be used to add a usb redirection devices for spice.
 EODESCR


@@ -975,76 +1039,6 @@ my $usbdesc = {
 };
 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 
 };
 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 

-my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
-my $hostpci_fmt = {
-    host => {
-       default_key => 1,
-       type => 'string',
-       pattern => qr/$PCIRE(;$PCIRE)*/,
-       format_description => 'HOSTPCIID[;HOSTPCIID2...]',
-       description => <<EODESCR,
-Host PCI device pass through. The PCI ID of a host's PCI device or a list
-of PCI virtual functions of the host. HOSTPCIID syntax is:
-
-'bus:dev.func' (hexadecimal numbers)
-
-You can us the 'lspci' command to list existing PCI devices.
-EODESCR
-    },
-    rombar => {
-       type => 'boolean',
-        description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
-       optional => 1,
-       default => 1,
-    },
-    romfile => {
-        type => 'string',
-        pattern => '[^,;]+',
-        format_description => 'string',
-        description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
-        optional => 1,
-    },
-    pcie => {
-       type => 'boolean',
-        description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
-       optional => 1,
-       default => 0,
-    },
-    'x-vga' => {
-       type => 'boolean',
-        description =>  "Enable vfio-vga device support.",
-       optional => 1,
-       default => 0,
-    },
-    'mdev' => {
-       type => 'string',
-        format_description => 'string',
-       pattern => '[^/\.:]+',
-       optional => 1,
-       description => <<EODESCR
-The type of mediated device to use.
-An instance of this type will be created on startup of the VM and
-will be cleaned up when the VM stops.
-EODESCR
-    }
-};
-PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
-
-my $hostpcidesc = {
-        optional => 1,
-        type => 'string', format => 'pve-qm-hostpci',
-        description => "Map host PCI devices into guest.",
-       verbose_description =>  <<EODESCR,
-Map host PCI devices into guest.
-
-NOTE: This option allows direct access to host hardware. So it is no longer
-possible to migrate such machines - use with special care.
-
-CAUTION: Experimental! User reported problems with this option.
-EODESCR
-};
-PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
-

 my $serialdesc = {
        optional => 1,
        type => 'string',
 my $serialdesc = {
        optional => 1,
        type => 'string',


@@ -1055,7 +1049,8 @@ Create a serial device inside the VM (n is 0 to 3), and pass through a
 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
 host side (use 'qm terminal' to open a terminal connection).
 
 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
 host side (use 'qm terminal' to open a terminal connection).
 

-NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
+NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
+use with special care.

 
 CAUTION: Experimental! User reported problems with this option.
 EODESCR
 
 CAUTION: Experimental! User reported problems with this option.
 EODESCR


@@ -1069,7 +1064,8 @@ my $paralleldesc= {
        verbose_description =>  <<EODESCR,
 Map host parallel devices (n is 0 to 2).
 
        verbose_description =>  <<EODESCR,
 Map host parallel devices (n is 0 to 2).
 

-NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
+NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
+machines - use with special care.

 
 CAUTION: Experimental! User reported problems with this option.
 EODESCR
 
 CAUTION: Experimental! User reported problems with this option.
 EODESCR


@@ -1083,8 +1079,8 @@ for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
     $confdesc->{"serial$i"} = $serialdesc;
 }
 
     $confdesc->{"serial$i"} = $serialdesc;
 }
 

-for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
-    $confdesc->{"hostpci$i"} = $hostpcidesc;
+for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
+    $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;

 }
 
 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
 }
 
 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {


@@ -1095,6 +1091,68 @@ for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
     $confdesc->{"usb$i"} = $usbdesc;
 }
 
     $confdesc->{"usb$i"} = $usbdesc;
 }
 

+my $boot_fmt = {
+    legacy => {
+       optional => 1,
+       default_key => 1,
+       type => 'string',
+       description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
+                    . " Deprecated, use 'order=' instead.",
+       pattern => '[acdn]{1,4}',
+       format_description => "[acdn]{1,4}",
+
+       # note: this is also the fallback if boot: is not given at all
+       default => 'cdn',
+    },
+    order => {
+       optional => 1,
+       type => 'string',
+       format => 'pve-qm-bootdev-list',
+       format_description => "device[;device...]",
+       description => <<EODESC,
+The guest will attempt to boot from devices in the order they appear here.
+
+Disks, optical drives and passed-through storage USB devices will be directly
+booted from, NICs will load PXE, and PCIe devices will either behave like disks
+(e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
+
+Note that only devices in this list will be marked as bootable and thus loaded
+by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
+(e.g. software-raid), you need to specify all of them here.
+
+Overrides the deprecated 'legacy=[acdn]*' value when given.
+EODESC
+    },
+};
+PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
+
+PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
+sub verify_bootdev {
+    my ($dev, $noerr) = @_;
+
+    return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
+
+    my $check = sub {
+       my ($base) = @_;
+       return 0 if $dev !~ m/^$base\d+$/;
+       return 0 if !$confdesc->{$dev};
+       return 1;
+    };
+
+    return $dev if $check->("net");
+    return $dev if $check->("usb");
+    return $dev if $check->("hostpci");
+
+    return undef if $noerr;
+    die "invalid boot device '$dev'\n";
+}
+
+sub print_bootorder {
+    my ($devs) = @_;
+    my $data = { order => join(';', @$devs) };
+    return PVE::JSONSchema::print_property_string($data, $boot_fmt);
+}
+

 my $kvm_api_version = 0;
 
 sub kvm_version {
 my $kvm_api_version = 0;
 
 sub kvm_version {


@@ -1138,6 +1196,11 @@ sub kvm_user_version {
     return $kvm_user_version->{$binary};
 
 }
     return $kvm_user_version->{$binary};
 
 }

+my sub extract_version {
+    my ($machine_type, $version) = @_;
+    $version = kvm_user_version() if !defined($version);
+    PVE::QemuServer::Machine::extract_version($machine_type, $version)
+}

 
 sub kernel_has_vhost_net {
     return -c '/dev/vhost-net';
 
 sub kernel_has_vhost_net {
     return -c '/dev/vhost-net';


@@ -1220,7 +1283,8 @@ sub cleanup_drive_path {
        ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
        ($drive->{file} !~ m/^\d+$/)) {
        my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
        ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
        ($drive->{file} !~ m/^\d+$/)) {
        my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});

-       raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
+       raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
+           if !$vtype;

        $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
        verify_media_type($opt, $vtype, $drive->{media});
        $drive->{file} = $volid;
        $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
        verify_media_type($opt, $vtype, $drive->{media});
        $drive->{file} = $volid;


@@ -1350,10 +1414,11 @@ sub print_drivedevice_full {
     my $device = '';
     my $maxdev = 0;
 
     my $device = '';
     my $maxdev = 0;
 

+    my $drive_id = "$drive->{interface}$drive->{index}";

     if ($drive->{interface} eq 'virtio') {
     if ($drive->{interface} eq 'virtio') {

-       my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
-       $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
-       $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
+       my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
+       $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
+       $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};

     } elsif ($drive->{interface} eq 'scsi') {
 
        my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
     } elsif ($drive->{interface} eq 'scsi') {
 
        my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);


@@ -1377,7 +1442,7 @@ sub print_drivedevice_full {
            }
 
            # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
            }
 
            # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)

-           my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
+           my $version = extract_version($machine_type, kvm_user_version());

            if ($path =~ m/^iscsi\:\/\// &&
               !min_version($version, 4, 1)) {
                $devicetype = 'generic';
            if ($path =~ m/^iscsi\:\/\// &&
               !min_version($version, 4, 1)) {
                $devicetype = 'generic';


@@ -1385,10 +1450,12 @@ sub print_drivedevice_full {
        }
 
        if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
        }
 
        if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){

-           $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
+           $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";

        } else {
        } else {

-           $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
+           $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
+               .",lun=$drive->{index}";

        }
        }

+       $device .= ",drive=drive-$drive_id,id=$drive_id";

 
        if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
            $device .= ",rotation_rate=1";
 
        if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
            $device .= ",rotation_rate=1";


@@ -1407,7 +1474,7 @@ sub print_drivedevice_full {
        } else {
            $device .= ",bus=ahci$controller.$unit";
        }
        } else {
            $device .= ",bus=ahci$controller.$unit";
        }

-       $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
+       $device .= ",drive=drive-$drive_id,id=$drive_id";

 
        if ($devicetype eq 'hd') {
            if (my $model = $drive->{model}) {
 
        if ($devicetype eq 'hd') {
            if (my $model = $drive->{model}) {


@@ -1557,12 +1624,29 @@ sub print_netdevice_full {
     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){

-       #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
+       # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
+       # and out of each queue plus one config interrupt and control vector queue

        my $vectors = $net->{queues} * 2 + 2;
        $tmpstr .= ",vectors=$vectors,mq=on";
     }
     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
 
        my $vectors = $net->{queues} * 2 + 2;
        $tmpstr .= ",vectors=$vectors,mq=on";
     }
     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
 

+    if (my $mtu = $net->{mtu}) {
+       if ($net->{model} eq 'virtio' && $net->{bridge}) {
+           my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
+           if ($mtu == 1) {
+                $mtu = $bridge_mtu;
+           } elsif ($mtu < 576) {
+               die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
+           } elsif ($mtu > $bridge_mtu) {
+               die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
+           }
+           $tmpstr .= ",host_mtu=$mtu";
+       } else {
+           warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
+       }
+    }
+

     if ($use_old_bios_files) {
        my $romfile;
        if ($device eq 'virtio-net-pci') {
     if ($use_old_bios_files) {
        my $romfile;
        if ($device eq 'virtio-net-pci') {


@@ -1609,7 +1693,8 @@ sub print_netdev_full {
     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
 
     if ($net->{bridge}) {
     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
 
     if ($net->{bridge}) {

-        $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
+       $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
+           .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";

     } else {
         $netdev = "type=user,id=$netid,hostname=$vmname";
     }
     } else {
         $netdev = "type=user,id=$netid,hostname=$vmname";
     }


@@ -1667,6 +1752,11 @@ sub print_vga_device {
        $memory = ",ram_size=67108864,vram_size=33554432";
     }
 
        $memory = ",ram_size=67108864,vram_size=33554432";
     }
 

+    my $edidoff = "";
+    if ($type eq 'VGA' && windows_version($conf->{ostype})) {
+       $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
+    }
+

     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
     my $vgaid = "vga" . ($id // '');
     my $pciaddr;
     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
     my $vgaid = "vga" . ($id // '');
     my $pciaddr;


@@ -1678,7 +1768,7 @@ sub print_vga_device {
        $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
     }
 
        $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
     }
 

-    return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
+    return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";

 }
 
 sub parse_number_sets {
 }
 
 sub parse_number_sets {


@@ -1698,34 +1788,17 @@ sub parse_number_sets {
 sub parse_numa {
     my ($data) = @_;
 
 sub parse_numa {
     my ($data) = @_;
 

-    my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
+    my $res = parse_property_string($numa_fmt, $data);

     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
     return $res;
 }
 
     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
     return $res;
 }
 

-sub parse_hostpci {
-    my ($value) = @_;
-
-    return undef if !$value;
-
-    my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
-
-    my @idlist = split(/;/, $res->{host});
-    delete $res->{host};
-    foreach my $id (@idlist) {
-       my $devs = PVE::SysFSTools::lspci($id);
-       die "no PCI device found for '$id'\n" if !scalar(@$devs);
-       push @{$res->{pciid}}, @$devs;
-    }
-    return $res;
-}
-

 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
 sub parse_net {
     my ($data) = @_;
 
 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
 sub parse_net {
     my ($data) = @_;
 

-    my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
+    my $res = eval { parse_property_string($net_fmt, $data) };

     if ($@) {
        warn $@;
        return undef;
     if ($@) {
        warn $@;
        return undef;


@@ -1741,7 +1814,7 @@ sub parse_net {
 sub parse_ipconfig {
     my ($data) = @_;
 
 sub parse_ipconfig {
     my ($data) = @_;
 

-    my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
+    my $res = eval { parse_property_string($ipconfig_fmt, $data) };

     if ($@) {
        warn $@;
        return undef;
     if ($@) {
        warn $@;
        return undef;


@@ -1878,7 +1951,7 @@ my $smbios1_fmt = {
 sub parse_smbios1 {
     my ($data) = @_;
 
 sub parse_smbios1 {
     my ($data) = @_;
 

-    my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
+    my $res = eval { parse_property_string($smbios1_fmt, $data) };

     warn $@ if $@;
     return $res;
 }
     warn $@ if $@;
     return $res;
 }


@@ -1895,7 +1968,7 @@ sub parse_watchdog {
 
     return undef if !$value;
 
 
     return undef if !$value;
 

-    my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
+    my $res = eval { parse_property_string($watchdog_fmt, $value) };

     warn $@ if $@;
     return $res;
 }
     warn $@ if $@;
     return $res;
 }


@@ -1905,7 +1978,7 @@ sub parse_guest_agent {
 
     return {} if !defined($value->{agent});
 
 
     return {} if !defined($value->{agent});
 

-    my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
+    my $res = eval { parse_property_string($agent_fmt, $value->{agent}) };

     warn $@ if $@;
 
     # if the agent is disabled ignore the other potentially set properties
     warn $@ if $@;
 
     # if the agent is disabled ignore the other potentially set properties


@@ -1917,7 +1990,7 @@ sub parse_vga {
     my ($value) = @_;
 
     return {} if !$value;
     my ($value) = @_;
 
     return {} if !$value;

-    my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
+    my $res = eval { parse_property_string($vga_fmt, $value) };

     warn $@ if $@;
     return $res;
 }
     warn $@ if $@;
     return $res;
 }


@@ -1927,7 +2000,7 @@ sub parse_rng {
 
     return undef if !$value;
 
 
     return undef if !$value;
 

-    my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
+    my $res = eval { parse_property_string($rng_fmt, $value) };

     warn $@ if $@;
     return $res;
 }
     warn $@ if $@;
     return $res;
 }


@@ -1948,7 +2021,8 @@ sub json_config_properties {
     my $prop = shift;
 
     foreach my $opt (keys %$confdesc) {
     my $prop = shift;
 
     foreach my $opt (keys %$confdesc) {

-       next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
+       next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
+           $opt eq 'runningmachine' || $opt eq 'runningcpu';

        $prop->{$opt} = $confdesc->{$opt};
     }
 
        $prop->{$opt} = $confdesc->{$opt};
     }
 


@@ -2007,7 +2081,7 @@ sub destroy_vm {
 
     if ($conf->{template}) {
        # check if any base image is still used by a linked clone
 
     if ($conf->{template}) {
        # check if any base image is still used by a linked clone

-       foreach_drive($conf, sub {
+       PVE::QemuConfig->foreach_volume($conf, sub {

                my ($ds, $drive) = @_;
                return if drive_is_cdrom($drive);
 
                my ($ds, $drive) = @_;
                return if drive_is_cdrom($drive);
 


@@ -2021,7 +2095,7 @@ sub destroy_vm {
     }
 
     # only remove disks owned by this VM
     }
 
     # only remove disks owned by this VM

-    foreach_drive($conf, sub {
+    PVE::QemuConfig->foreach_volume($conf, sub {

        my ($ds, $drive) = @_;
        return if drive_is_cdrom($drive, 1);
 
        my ($ds, $drive) = @_;
        return if drive_is_cdrom($drive, 1);
 


@@ -2232,7 +2306,7 @@ sub write_vm_config {
        }
 
        foreach my $key (sort keys %$conf) {
        }
 
        foreach my $key (sort keys %$conf) {

-           next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
+           next if $key =~ /^(digest|description|pending|snapshots)$/;

            $raw .= "$key: $conf->{$key}\n";
        }
        return $raw;
            $raw .= "$key: $conf->{$key}\n";
        }
        return $raw;


@@ -2310,7 +2384,7 @@ sub check_local_resources {
 sub check_storage_availability {
     my ($storecfg, $conf, $node) = @_;
 
 sub check_storage_availability {
     my ($storecfg, $conf, $node) = @_;
 

-    foreach_drive($conf, sub {
+    PVE::QemuConfig->foreach_volume($conf, sub {

        my ($ds, $drive) = @_;
 
        my $volid = $drive->{file};
        my ($ds, $drive) = @_;
 
        my $volid = $drive->{file};


@@ -2333,7 +2407,7 @@ sub shared_nodes {
     my $nodehash = { map { $_ => 1 } @$nodelist };
     my $nodename = nodename();
 
     my $nodehash = { map { $_ => 1 } @$nodelist };
     my $nodename = nodename();
 

-    foreach_drive($conf, sub {
+    PVE::QemuConfig->foreach_volume($conf, sub {

        my ($ds, $drive) = @_;
 
        my $volid = $drive->{file};
        my ($ds, $drive) = @_;
 
        my $volid = $drive->{file};


@@ -2365,7 +2439,7 @@ sub check_local_storage_availability {
     my $nodelist = PVE::Cluster::get_nodelist();
     my $nodehash = { map { $_ => {} } @$nodelist };
 
     my $nodelist = PVE::Cluster::get_nodelist();
     my $nodehash = { map { $_ => {} } @$nodelist };
 

-    foreach_drive($conf, sub {
+    PVE::QemuConfig->foreach_volume($conf, sub {

        my ($ds, $drive) = @_;
 
        my $volid = $drive->{file};
        my ($ds, $drive) = @_;
 
        my $volid = $drive->{file};


@@ -2709,7 +2783,7 @@ sub conf_has_audio {
     my $audio = $conf->{"audio$id"};
     return undef if !defined($audio);
 
     my $audio = $conf->{"audio$id"};
     return undef if !defined($audio);
 

-    my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
+    my $audioproperties = parse_property_string($audio_fmt, $audio);

     my $audiodriver = $audioproperties->{driver} // 'spice';
 
     return {
     my $audiodriver = $audioproperties->{driver} // 'spice';
 
     return {


@@ -2720,6 +2794,32 @@ sub conf_has_audio {
     };
 }
 
     };
 }
 

+sub audio_devs {
+    my ($audio, $audiopciaddr, $machine_version) = @_;
+
+    my $devs = [];
+
+    my $id = $audio->{dev_id};
+    my $audiodev = "";
+    if (min_version($machine_version, 4, 2)) {
+       $audiodev = ",audiodev=$audio->{backend_id}";
+    }
+
+    if ($audio->{dev} eq 'AC97') {
+       push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
+    } elsif ($audio->{dev} =~ /intel\-hda$/) {
+       push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
+       push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
+       push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
+    } else {
+       die "unkown audio device '$audio->{dev}', implement me!";
+    }
+
+    push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
+
+    return $devs;
+}
+

 sub vga_conf_has_spice {
     my ($vga) = @_;
 
 sub vga_conf_has_spice {
     my ($vga) = @_;
 


@@ -2911,7 +3011,7 @@ sub query_understood_cpu_flags {
 }
 
 sub config_to_command {
 }
 
 sub config_to_command {

-    my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
+    my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;

 
     my $cmd = [];
     my $globalFlags = [];
 
     my $cmd = [];
     my $globalFlags = [];


@@ -2937,18 +3037,22 @@ sub config_to_command {
     my $add_pve_version = min_version($kvmver, 4, 1);
 
     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
     my $add_pve_version = min_version($kvmver, 4, 1);
 
     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);

-    my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
+    my $machine_version = extract_version($machine_type, $kvmver);

     $kvm //= 1 if is_native($arch);
 
     $machine_version =~ m/(\d+)\.(\d+)/;
     my ($machine_major, $machine_minor) = ($1, $2);
     $kvm //= 1 if is_native($arch);
 
     $machine_version =~ m/(\d+)\.(\d+)/;
     my ($machine_major, $machine_minor) = ($1, $2);

-    die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
-       if !PVE::QemuServer::min_version($kvmver, $machine_major, $machine_minor);

 
 

-    if (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
+    if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
+       warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
+    } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
+       die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
+           ." please upgrade node '$nodename'\n"
+    } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {

        my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
        my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);

-       die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
-         . " is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
+       die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
+           ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
+           ." node '$nodename'\n";

     }
 
     # if a specific +pve version is required for a feature, use $version_guard
     }
 
     # if a specific +pve version is required for a feature, use $version_guard


@@ -2964,9 +3068,9 @@ sub config_to_command {
        return 1;
     };
 
        return 1;
     };
 

-    if ($kvm) {
-       die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
-           if !defined kvm_version();
+    if ($kvm && !defined kvm_version()) {
+       die "KVM virtualisation configured, but not available. Either disable in VM configuration"
+           ." or enable in BIOS.\n";

     }
 
     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
     }
 
     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);


@@ -3023,12 +3127,11 @@ sub config_to_command {
        }
     }
 
        }
     }
 

-    my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);

     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {

-       die "uefi base image not found\n" if ! -f $ovmf_code;
+       my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
+       die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;

 
 

-       my $path;
-       my $format;
+       my ($path, $format);

        if (my $efidisk = $conf->{efidisk0}) {
            my $d = parse_drive('efidisk0', $efidisk);
            my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
        if (my $efidisk = $conf->{efidisk0}) {
            my $d = parse_drive('efidisk0', $efidisk);
            my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);


@@ -3051,8 +3154,14 @@ sub config_to_command {
            $format = 'raw';
        }
 
            $format = 'raw';
        }
 

+       my $size_str = "";
+
+       if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
+           $size_str = ",size=" . (-s $ovmf_vars);
+       }
+

        push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
        push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";

-       push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
+       push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";

     }
 
     # load q35 config
     }
 
     # load q35 config


@@ -3070,7 +3179,8 @@ sub config_to_command {
     }
 
     # add usb controllers
     }
 
     # add usb controllers

-    my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
+    my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
+        $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);

     push @$devices, @usbcontrollers if @usbcontrollers;
     my $vga = parse_vga($conf->{vga});
 
     push @$devices, @usbcontrollers if @usbcontrollers;
     my $vga = parse_vga($conf->{vga});
 


@@ -3103,83 +3213,16 @@ sub config_to_command {
        push @$devices, '-device', $kbd if defined($kbd);
     }
 
        push @$devices, '-device', $kbd if defined($kbd);
     }
 

-    my $kvm_off = 0;
-    my $gpu_passthrough;
-
-    # host pci devices
-    for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
-       my $id = "hostpci$i";
-       my $d = parse_hostpci($conf->{$id});
-       next if !$d;
-
-       if (my $pcie = $d->{pcie}) {
-           die "q35 machine model is not enabled" if !$q35;
-           # win7 wants to have the pcie devices directly on the pcie bus
-           # instead of in the root port
-           if ($winversion == 7) {
-               $pciaddr = print_pcie_addr("${id}bus0");
-           } else {
-               # add more root ports if needed, 4 are present by default
-               # by pve-q35 cfgs, rest added here on demand.
-               if ($i > 3) {
-                   push @$devices, '-device', print_pcie_root_port($i);
-               }
-               $pciaddr = print_pcie_addr($id);
-           }
-       } else {
-           $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
-       }
-
-       my $xvga = '';
-       if ($d->{'x-vga'}) {
-           $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
-           $kvm_off = 1;
-           $vga->{type} = 'none' if !defined($conf->{vga});
-           $gpu_passthrough = 1;
-       }
-
-       my $pcidevices = $d->{pciid};
-       my $multifunction = 1 if @$pcidevices > 1;
-
-       my $sysfspath;
-       if ($d->{mdev} && scalar(@$pcidevices) == 1) {
-           my $pci_id = $pcidevices->[0]->{id};
-           my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
-           $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
-       } elsif ($d->{mdev}) {
-           warn "ignoring mediated device '$id' with multifunction device\n";
-       }
-
-       my $j=0;
-       foreach my $pcidevice (@$pcidevices) {
-           my $devicestr = "vfio-pci";
-
-           if ($sysfspath) {
-               $devicestr .= ",sysfsdev=$sysfspath";
-           } else {
-               $devicestr .= ",host=$pcidevice->{id}";
-           }
-
-           my $mf_addr = $multifunction ? ".$j" : '';
-           $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
-
-           if ($j == 0) {
-               $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
-               $devicestr .= "$xvga";
-               $devicestr .= ",multifunction=on" if $multifunction;
-               $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
-           }
-
-           push @$devices, '-device', $devicestr;
-           $j++;
-       }
-    }
+    # host pci device passthrough
+    my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
+       $vmid, $conf, $devices, $winversion, $q35, $bridges, $arch, $machine_type);

 
     # usb devices
     my $usb_dev_features = {};
     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
 
 
     # usb devices
     my $usb_dev_features = {};
     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
 

-    my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
+    my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
+        $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);

     push @$devices, @usbdevices if @usbdevices;
     # serial devices
     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
     push @$devices, @usbdevices if @usbdevices;
     # serial devices
     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {


@@ -3213,22 +3256,10 @@ sub config_to_command {
        }
     }
 
        }
     }
 

-    if (my $audio = conf_has_audio($conf)) {
-
+    if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {

        my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
        my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);

-
-       my $id = $audio->{dev_id};
-       if ($audio->{dev} eq 'AC97') {
-           push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
-       } elsif ($audio->{dev} =~ /intel\-hda$/) {
-           push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
-           push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
-           push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
-       } else {
-           die "unkown audio device '$audio->{dev}', implement me!";
-       }
-
-       push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
+       my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
+       push @$devices, @$audio_devs;

     }
 
     my $sockets = 1;
     }
 
     my $sockets = 1;


@@ -3276,7 +3307,8 @@ sub config_to_command {
     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
 
     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
 
     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){

-       push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
+       push @$devices, '-device', print_vga_device(
+           $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);

        my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
        push @$cmd,  '-vnc', "unix:$socket,password";
     } else {
        my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
        push @$cmd,  '-vnc', "unix:$socket,password";
     } else {


@@ -3286,7 +3318,6 @@ sub config_to_command {
 
     # time drift fix
     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
 
     # time drift fix
     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};

-

     my $useLocaltime = $conf->{localtime};
 
     if ($winversion >= 5) { # windows
     my $useLocaltime = $conf->{localtime};
 
     if ($winversion >= 5) { # windows


@@ -3305,13 +3336,17 @@ sub config_to_command {
 
     push @$rtcFlags, 'driftfix=slew' if $tdf;
 
 
     push @$rtcFlags, 'driftfix=slew' if $tdf;
 

-    if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
+    if ($conf->{startdate} && $conf->{startdate} ne 'now') {

        push @$rtcFlags, "base=$conf->{startdate}";
     } elsif ($useLocaltime) {
        push @$rtcFlags, 'base=localtime';
     }
 
        push @$rtcFlags, "base=$conf->{startdate}";
     } elsif ($useLocaltime) {
        push @$rtcFlags, 'base=localtime';
     }
 

-    push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
+    if ($forcecpu) {
+       push @$cmd, '-cpu', $forcecpu;
+    } else {
+       push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
+    }

 
     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
 
 
     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
 


@@ -3336,20 +3371,16 @@ sub config_to_command {
 
     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
     if ($rng && &$version_guard(4, 1, 2)) {
 
     my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
     if ($rng && &$version_guard(4, 1, 2)) {

+       check_rng_source($rng->{source});
+

        my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
        my $period = $rng->{period} // $rng_fmt->{period}->{default};
        my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
        my $period = $rng->{period} // $rng_fmt->{period}->{default};

-

        my $limiter_str = "";
        if ($max_bytes) {
            $limiter_str = ",max-bytes=$max_bytes,period=$period";
        }
 
        my $limiter_str = "";
        if ($max_bytes) {
            $limiter_str = ",max-bytes=$max_bytes,period=$period";
        }
 

-       # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
-       die "cannot create VirtIO RNG device: source file '$rng->{source}' doesn't exist\n"
-           if ! -e $rng->{source};
-

        my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
        my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);

-

        push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
        push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
     }
        push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
        push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
     }


@@ -3359,8 +3390,9 @@ sub config_to_command {
     if ($qxlnum) {
        if ($qxlnum > 1) {
            if ($winversion){
     if ($qxlnum) {
        if ($qxlnum > 1) {
            if ($winversion){

-               for(my $i = 1; $i < $qxlnum; $i++){
-                   push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
+               for (my $i = 1; $i < $qxlnum; $i++){
+                   push @$devices, '-device', print_vga_device(
+                       $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);

                }
            } else {
                # assume other OS works like Linux
                }
            } else {
                # assume other OS works like Linux


@@ -3387,14 +3419,17 @@ sub config_to_command {
        my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
        $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
 
        my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
        $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
 

-       my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
+       my $spice_enhancement_str = $conf->{spice_enhancements} // '';
+       my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);

        if ($spice_enhancement->{foldersharing}) {
            push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
            push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
        }
 
        my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
        if ($spice_enhancement->{foldersharing}) {
            push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
            push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
        }
 
        my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";

-       $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
+       $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
+           if $spice_enhancement->{videostreaming};
+

        push @$devices, '-spice', "$spice_opts";
     }
 
        push @$devices, '-spice', "$spice_opts";
     }
 


@@ -3422,7 +3457,7 @@ sub config_to_command {
        push @$devices, '-iscsi', "initiator-name=$initiator";
     }
 
        push @$devices, '-iscsi', "initiator-name=$initiator";
     }
 

-    foreach_drive($conf, sub {
+    PVE::QemuConfig->foreach_volume($conf, sub {

        my ($ds, $drive) = @_;
 
        if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
        my ($ds, $drive) = @_;
 
        if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {


@@ -3446,11 +3481,11 @@ sub config_to_command {
            }
        }
 
            }
        }
 

-       if($drive->{interface} eq 'virtio'){
+       if ($drive->{interface} eq 'virtio'){

            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
        }
 
            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
        }
 

-        if ($drive->{interface} eq 'scsi') {
+       if ($drive->{interface} eq 'scsi') {

 
            my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
 
 
            my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
 


@@ -3473,43 +3508,50 @@ sub config_to_command {
                $queues = ",num_queues=$drive->{queues}";
            }
 
                $queues = ",num_queues=$drive->{queues}";
            }
 

-           push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
+           push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
+               if !$scsicontroller->{$controller};

            $scsicontroller->{$controller}=1;
            $scsicontroller->{$controller}=1;

-        }
+       }

 
         if ($drive->{interface} eq 'sata') {
 
         if ($drive->{interface} eq 'sata') {

-           my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
-           $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
-           push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
-           $ahcicontroller->{$controller}=1;
+           my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
+           $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
+           push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
+               if !$ahcicontroller->{$controller};
+           $ahcicontroller->{$controller}=1;

         }
 
        my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
         }
 
        my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);

+       $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
+

        push @$devices, '-drive',$drive_cmd;
        push @$devices, '-drive',$drive_cmd;

-       push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
+       push @$devices, '-device', print_drivedevice_full(
+           $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);

     });
 
     for (my $i = 0; $i < $MAX_NETS; $i++) {
     });
 
     for (my $i = 0; $i < $MAX_NETS; $i++) {

-         next if !$conf->{"net$i"};
-         my $d = parse_net($conf->{"net$i"});
-         next if !$d;
+       next if !$conf->{"net$i"};
+       my $d = parse_net($conf->{"net$i"});
+       next if !$d;
+
+       $use_virtio = 1 if $d->{model} eq 'virtio';

 
 

-         $use_virtio = 1 if $d->{model} eq 'virtio';
+       if ($bootindex_hash->{n}) {
+           $d->{bootindex} = $bootindex_hash->{n};
+           $bootindex_hash->{n} += 1;
+       }

 
 

-         if ($bootindex_hash->{n}) {
-            $d->{bootindex} = $bootindex_hash->{n};
-            $bootindex_hash->{n} += 1;
-         }
+       my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
+       push @$devices, '-netdev', $netdevfull;

 
 

-         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
-         push @$devices, '-netdev', $netdevfull;
+       my $netdevicefull = print_netdevice_full(
+           $vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);

 
 

-         my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
-         push @$devices, '-device', $netdevicefull;
+       push @$devices, '-device', $netdevicefull;

     }
 
     if ($conf->{ivshmem}) {
     }
 
     if ($conf->{ivshmem}) {

-       my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
+       my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});

 
        my $bus;
        if ($q35) {
 
        my $bus;
        if ($q35) {


@@ -3522,7 +3564,8 @@ sub config_to_command {
        my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
 
        push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
        my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
 
        push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";

-       push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
+       push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
+           .",size=$ivshmem->{size}M";

     }
 
     # pci.4 is nested in pci.1
     }
 
     # pci.4 is nested in pci.1


@@ -3541,7 +3584,13 @@ sub config_to_command {
 
     for my $k (sort {$b cmp $a} keys %$bridges) {
        next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
 
     for my $k (sort {$b cmp $a} keys %$bridges) {
        next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3

-       $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
+
+       my $k_name = $k;
+       if ($k == 2 && $legacy_igd) {
+           $k_name = "$k-igd";
+       }
+       $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
+

        my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
        if ($q35) {
            # add after -readconfig pve-q35.cfg
        my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
        if ($q35) {
            # add after -readconfig pve-q35.cfg


@@ -3563,12 +3612,9 @@ sub config_to_command {
     push @$machineFlags, "type=${machine_type_min}";
 
     push @$cmd, @$devices;
     push @$machineFlags, "type=${machine_type_min}";
 
     push @$cmd, @$devices;

-    push @$cmd, '-rtc', join(',', @$rtcFlags)
-       if scalar(@$rtcFlags);
-    push @$cmd, '-machine', join(',', @$machineFlags)
-       if scalar(@$machineFlags);
-    push @$cmd, '-global', join(',', @$globalFlags)
-       if scalar(@$globalFlags);
+    push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
+    push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
+    push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);

 
     if (my $vmstate = $conf->{vmstate}) {
        my $statepath = PVE::Storage::path($storecfg, $vmstate);
 
     if (my $vmstate = $conf->{vmstate}) {
        my $statepath = PVE::Storage::path($storecfg, $vmstate);


@@ -3586,6 +3632,23 @@ sub config_to_command {
     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
 }
 
     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
 }
 

+sub check_rng_source {
+    my ($source) = @_;
+
+    # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
+    die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
+       if ! -e $source;
+
+    my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
+    if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
+       # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
+       # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
+       die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
+           ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
+           ." to the host.\n";
+    }
+}
+

 sub spice_port {
     my ($vmid) = @_;
 
 sub spice_port {
     my ($vmid) = @_;
 


@@ -3652,7 +3715,8 @@ sub vm_deviceplug {
     my $devices_list = vm_devices_list($vmid);
     return 1 if defined($devices_list->{$deviceid});
 
     my $devices_list = vm_devices_list($vmid);
     return 1 if defined($devices_list->{$deviceid});
 

-    qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
+    # add PCI bridge if we need it for the device
+    qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);

 
     if ($deviceid eq 'tablet') {
 
 
     if ($deviceid eq 'tablet') {
 


@@ -3727,7 +3791,8 @@ sub vm_deviceplug {
        my $use_old_bios_files = undef;
        ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
 
        my $use_old_bios_files = undef;
        ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
 

-       my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
+       my $netdevicefull = print_netdevice_full(
+           $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);

        qemu_deviceadd($vmid, $netdevicefull);
        eval {
            qemu_deviceaddverify($vmid, $deviceid);
        qemu_deviceadd($vmid, $netdevicefull);
        eval {
            qemu_deviceaddverify($vmid, $deviceid);


@@ -3999,6 +4064,14 @@ sub qemu_netdevadd {
     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
     my %options =  split(/[=,]/, $netdev);
 
     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
     my %options =  split(/[=,]/, $netdev);
 

+    if (defined(my $vhost = $options{vhost})) {
+       $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
+    }
+
+    if (defined(my $queues = $options{queues})) {
+       $options{queues} = $queues + 0;
+    }
+

     mon_cmd($vmid, "netdev_add",  %options);
     return 1;
 }
     mon_cmd($vmid, "netdev_add",  %options);
     return 1;
 }


@@ -4211,7 +4284,7 @@ sub qemu_volume_snapshot_delete {
 
        $running = undef;
        my $conf = PVE::QemuConfig->load_config($vmid);
 
        $running = undef;
        my $conf = PVE::QemuConfig->load_config($vmid);

-       foreach_drive($conf, sub {
+       PVE::QemuConfig->foreach_volume($conf, sub {

            my ($ds, $drive) = @_;
            $running = 1 if $drive->{file} eq $volid;
        });
            my ($ds, $drive) = @_;
            $running = 1 if $drive->{file} eq $volid;
        });


@@ -4249,6 +4322,59 @@ sub set_migration_caps {
     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
 }
 
     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
 }
 

+sub foreach_volid {
+    my ($conf, $func, @param) = @_;
+
+    my $volhash = {};
+
+    my $test_volid = sub {
+       my ($key, $drive, $snapname) = @_;
+
+       my $volid = $drive->{file};
+       return if !$volid;
+
+       $volhash->{$volid}->{cdrom} //= 1;
+       $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
+
+       my $replicate = $drive->{replicate} // 1;
+       $volhash->{$volid}->{replicate} //= 0;
+       $volhash->{$volid}->{replicate} = 1 if $replicate;
+
+       $volhash->{$volid}->{shared} //= 0;
+       $volhash->{$volid}->{shared} = 1 if $drive->{shared};
+
+       $volhash->{$volid}->{referenced_in_config} //= 0;
+       $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
+
+       $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
+           if defined($snapname);
+
+       my $size = $drive->{size};
+       $volhash->{$volid}->{size} //= $size if $size;
+
+       $volhash->{$volid}->{is_vmstate} //= 0;
+       $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
+
+       $volhash->{$volid}->{is_unused} //= 0;
+       $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
+    };
+
+    my $include_opts = {
+       extra_keys => ['vmstate'],
+       include_unused => 1,
+    };
+
+    PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
+    foreach my $snapname (keys %{$conf->{snapshots}}) {
+       my $snap = $conf->{snapshots}->{$snapname};
+       PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
+    }
+
+    foreach my $volid (keys %$volhash) {
+       &$func($volid, $volhash->{$volid}, @param);
+    }
+}
+

 my $fast_plug_option = {
     'lock' => 1,
     'name' => 1,
 my $fast_plug_option = {
     'lock' => 1,
     'name' => 1,


@@ -4396,7 +4522,7 @@ sub vmconfig_hotplug_pending {
                # since we cannot reliably hot unplug usb devices
                # we are disabling it
                die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
                # since we cannot reliably hot unplug usb devices
                # we are disabling it
                die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;

-               my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
+               my $d = eval { parse_property_string($usbdesc->{format}, $value) };

                die "skip\n" if !$d;
                qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
            } elsif ($opt eq 'vcpus') {
                die "skip\n" if !$d;
                qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
            } elsif ($opt eq 'vcpus') {


@@ -4598,102 +4724,100 @@ sub vmconfig_update_disk {
 
     my $drive = parse_drive($opt, $value);
 
 
     my $drive = parse_drive($opt, $value);
 

-    if ($conf->{$opt}) {
-
-       if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
+    if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
+       my $media = $drive->{media} || 'disk';
+       my $oldmedia = $old_drive->{media} || 'disk';
+       die "unable to change media type\n" if $media ne $oldmedia;

 
 

-           my $media = $drive->{media} || 'disk';
-           my $oldmedia = $old_drive->{media} || 'disk';
-           die "unable to change media type\n" if $media ne $oldmedia;
+       if (!drive_is_cdrom($old_drive)) {

 
 

-           if (!drive_is_cdrom($old_drive)) {
+           if ($drive->{file} ne $old_drive->{file}) {

 
 

-               if ($drive->{file} ne $old_drive->{file}) {
+               die "skip\n" if !$hotplug;

 
 

-                   die "skip\n" if !$hotplug;
-
-                   # unplug and register as unused
-                   vm_deviceunplug($vmid, $conf, $opt);
-                   vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
+               # unplug and register as unused
+               vm_deviceunplug($vmid, $conf, $opt);
+               vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)

 
 

-               } else {
-                   # update existing disk
-
-                   # skip non hotpluggable value
-                   if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
-                       safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
-                       safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
-                       safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
-                       safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
-                       die "skip\n";
-                   }
+           } else {
+               # update existing disk
+
+               # skip non hotpluggable value
+               if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
+                   safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
+                   safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
+                   safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
+                   safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
+                   die "skip\n";
+               }

 
 

-                   # apply throttle
-                   if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
-                       safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
-                       safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
-                       safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
-                       safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
-                       safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
-                       safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
-                       safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
-                       safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
-                       safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
-                       safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
-                       safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
-                       safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
-                       safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
-                       safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
-                       safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
-                       safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
-                       safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
-
-                       qemu_block_set_io_throttle($vmid,"drive-$opt",
-                                                  ($drive->{mbps} || 0)*1024*1024,
-                                                  ($drive->{mbps_rd} || 0)*1024*1024,
-                                                  ($drive->{mbps_wr} || 0)*1024*1024,
-                                                  $drive->{iops} || 0,
-                                                  $drive->{iops_rd} || 0,
-                                                  $drive->{iops_wr} || 0,
-                                                  ($drive->{mbps_max} || 0)*1024*1024,
-                                                  ($drive->{mbps_rd_max} || 0)*1024*1024,
-                                                  ($drive->{mbps_wr_max} || 0)*1024*1024,
-                                                  $drive->{iops_max} || 0,
-                                                  $drive->{iops_rd_max} || 0,
-                                                  $drive->{iops_wr_max} || 0,
-                                                  $drive->{bps_max_length} || 1,
-                                                  $drive->{bps_rd_max_length} || 1,
-                                                  $drive->{bps_wr_max_length} || 1,
-                                                  $drive->{iops_max_length} || 1,
-                                                  $drive->{iops_rd_max_length} || 1,
-                                                  $drive->{iops_wr_max_length} || 1);
+               # apply throttle
+               if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
+                   safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
+                   safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
+                   safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
+                   safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
+                   safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
+                   safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
+                   safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
+                   safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
+                   safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
+                   safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
+                   safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
+                   safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
+                   safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
+                   safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
+                   safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
+                   safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
+                   safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
+
+                   qemu_block_set_io_throttle(
+                       $vmid,"drive-$opt",
+                       ($drive->{mbps} || 0)*1024*1024,
+                       ($drive->{mbps_rd} || 0)*1024*1024,
+                       ($drive->{mbps_wr} || 0)*1024*1024,
+                        $drive->{iops} || 0,
+                        $drive->{iops_rd} || 0,
+                        $drive->{iops_wr} || 0,
+                       ($drive->{mbps_max} || 0)*1024*1024,
+                       ($drive->{mbps_rd_max} || 0)*1024*1024,
+                       ($drive->{mbps_wr_max} || 0)*1024*1024,
+                        $drive->{iops_max} || 0,
+                        $drive->{iops_rd_max} || 0,
+                        $drive->{iops_wr_max} || 0,
+                        $drive->{bps_max_length} || 1,
+                        $drive->{bps_rd_max_length} || 1,
+                        $drive->{bps_wr_max_length} || 1,
+                        $drive->{iops_max_length} || 1,
+                        $drive->{iops_rd_max_length} || 1,
+                        $drive->{iops_wr_max_length} || 1,
+                   );

 
 

-                   }
+               }

 
 

-                   return 1;
-               }
+               return 1;
+           }

 
 

-           } else { # cdrom
+       } else { # cdrom

 
 

-               if ($drive->{file} eq 'none') {
-                   mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
-                   if (drive_is_cloudinit($old_drive)) {
-                       vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
-                   }
-               } else {
-                   my $path = get_iso_path($storecfg, $vmid, $drive->{file});
+           if ($drive->{file} eq 'none') {
+               mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
+               if (drive_is_cloudinit($old_drive)) {
+                   vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
+               }
+           } else {
+               my $path = get_iso_path($storecfg, $vmid, $drive->{file});

 
 

-                   # force eject if locked
-                   mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
+               # force eject if locked
+               mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");

 
 

-                   if ($path) {
-                       mon_cmd($vmid, "blockdev-change-medium",
-                           id => "$opt", filename => "$path");
-                   }
+               if ($path) {
+                   mon_cmd($vmid, "blockdev-change-medium",
+                       id => "$opt", filename => "$path");

                }
                }

-
-               return 1;

            }
            }

+
+           return 1;

        }
     }
 
        }
     }
 


@@ -4703,361 +4827,448 @@ sub vmconfig_update_disk {
     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
 }
 
     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
 }
 

-sub vm_start {
-    my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
-       $forcemachine, $spice_ticket, $migration_network, $migration_type,
-       $targetstorage, $timeout, $nbd_protocol_version, $replicated_volumes) = @_;
+# called in locked context by incoming migration
+sub vm_migrate_get_nbd_disks {
+    my ($storecfg, $conf, $replicated_volumes) = @_;

 
 

-    PVE::QemuConfig->lock_config($vmid, sub {
-       my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
+    my $local_volumes = {};
+    PVE::QemuConfig->foreach_volume($conf, sub {
+       my ($ds, $drive) = @_;

 
 

-       die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
+       return if drive_is_cdrom($drive);

 
 

-       my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
+       my $volid = $drive->{file};

 
 

-       PVE::QemuConfig->check_lock($conf)
-           if !($skiplock || $is_suspended);
+       return if !$volid;

 
 

-       die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
+       my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);

 
 

-       # clean up leftover reboot request files
-       eval { clear_reboot_request($vmid); };
-       warn $@ if $@;
+       my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+       return if $scfg->{shared};

 
 

-       if (!$statefile && scalar(keys %{$conf->{pending}})) {
-           vmconfig_apply_pending($vmid, $conf, $storecfg);
-           $conf = PVE::QemuConfig->load_config($vmid); # update/reload
-       }
+       # replicated disks re-use existing state via bitmap
+       my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
+       $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
+    });
+    return $local_volumes;
+}

 
 

-       PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
+# called in locked context by incoming migration
+sub vm_migrate_alloc_nbd_disks {
+    my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;

 
 

-       my $defaults = load_defaults();
+    my $format = undef;

 
 

-       # set environment variable useful inside network script
-       $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
+    my $nbd = {};
+    foreach my $opt (sort keys %$source_volumes) {
+       my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};

 
 

-       my $local_volumes = {};
+       if ($use_existing) {
+           $nbd->{$opt}->{drivestr} = print_drive($drive);
+           $nbd->{$opt}->{volid} = $volid;
+           $nbd->{$opt}->{replicated} = 1;
+           next;
+       }

 
 

-       if ($targetstorage) {
-           foreach_drive($conf, sub {
-               my ($ds, $drive) = @_;
+       # If a remote storage is specified and the format of the original
+       # volume is not available there, fall back to the default format.
+       # Otherwise use the same format as the original.
+       if (!$storagemap->{identity}) {
+           $storeid = map_storage($storagemap, $storeid);
+           my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
+           my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+           my $fileFormat = qemu_img_format($scfg, $volname);
+           $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
+       } else {
+           my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+           $format = qemu_img_format($scfg, $volname);
+       }

 
 

-               return if drive_is_cdrom($drive);
+       my $size = $drive->{size} / 1024;
+       my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
+       my $newdrive = $drive;
+       $newdrive->{format} = $format;
+       $newdrive->{file} = $newvolid;
+       my $drivestr = print_drive($newdrive);
+       $nbd->{$opt}->{drivestr} = $drivestr;
+       $nbd->{$opt}->{volid} = $newvolid;
+    }

 
 

-               my $volid = $drive->{file};
+    return $nbd;
+}

 
 

-               return if !$volid;
+# see vm_start_nolock for parameters, additionally:
+# migrate_opts:
+#   storagemap = parsed storage map for allocating NBD disks
+sub vm_start {
+    my ($storecfg, $vmid, $params, $migrate_opts) = @_;

 
 

-               my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
+    return PVE::QemuConfig->lock_config($vmid, sub {
+       my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});

 
 

-               my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
-               return if $scfg->{shared};
-               $local_volumes->{$ds} = [$volid, $storeid, $volname];
-           });
+       die "you can't start a vm if it's a template\n"
+           if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);

 
 

-           my $format = undef;
+       my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');

 
 

-           foreach my $opt (sort keys %$local_volumes) {
+       PVE::QemuConfig->check_lock($conf)
+           if !($params->{skiplock} || $has_suspended_lock);

 
 

-               my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
-               if ($replicated_volumes->{$volid}) {
-                   # re-use existing, replicated volume with bitmap on source side
-                   $local_volumes->{$opt} = $conf->{${opt}};
-                   print "re-using replicated volume: $opt - $volid\n";
-                   next;
-               }
-               my $drive = parse_drive($opt, $conf->{$opt});
-
-               # If a remote storage is specified and the format of the original
-               # volume is not available there, fall back to the default format.
-               # Otherwise use the same format as the original.
-               if ($targetstorage && $targetstorage ne "1") {
-                   $storeid = $targetstorage;
-                   my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
-                   my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
-                   my $fileFormat = qemu_img_format($scfg, $volname);
-                   $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
-               } else {
-                   my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
-                   $format = qemu_img_format($scfg, $volname);
-               }
+       $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
+
+       die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});

 
 

-               my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
-               my $newdrive = $drive;
-               $newdrive->{format} = $format;
-               $newdrive->{file} = $newvolid;
-               my $drivestr = print_drive($newdrive);
-               $local_volumes->{$opt} = $drivestr;
-               #pass drive to conf for command line
-               $conf->{$opt} = $drivestr;
+       if (my $storagemap = $migrate_opts->{storagemap}) {
+           my $replicated = $migrate_opts->{replicated_volumes};
+           my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
+           $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
+
+           foreach my $opt (keys %{$migrate_opts->{nbd}}) {
+               $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};

            }
        }
 
            }
        }
 

-       PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
+       return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
+    });
+}

 
 

-       if ($is_suspended) {
-           # enforce machine type on suspended vm to ensure HW compatibility
-           $forcemachine = $conf->{runningmachine};
-           print "Resuming suspended VM\n";
-       }

 
 

-       my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
+# params:
+#   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
+#   skiplock => 0/1, skip checking for config lock
+#   skiptemplate => 0/1, skip checking whether VM is template
+#   forcemachine => to force Qemu machine (rollback/migration)
+#   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
+#   timeout => in seconds
+#   paused => start VM in paused state (backup)
+#   resume => resume from hibernation
+# migrate_opts:
+#   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
+#   migratedfrom => source node
+#   spice_ticket => used for spice migration, passed via tunnel/stdin
+#   network => CIDR of migration network
+#   type => secure/insecure - tunnel over encrypted connection or plain-text
+#   nbd_proto_version => int, 0 for TCP, 1 for UNIX
+#   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
+sub vm_start_nolock {
+    my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
+
+    my $statefile = $params->{statefile};
+    my $resume = $params->{resume};
+
+    my $migratedfrom = $migrate_opts->{migratedfrom};
+    my $migration_type = $migrate_opts->{type};

 
 

-       my $migration_ip;
-       my $get_migration_ip = sub {
-           my ($cidr, $nodename) = @_;
+    my $res = {};

 
 

-           return $migration_ip if defined($migration_ip);
+    # clean up leftover reboot request files
+    eval { clear_reboot_request($vmid); };
+    warn $@ if $@;

 
 

-           if (!defined($cidr)) {
-               my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
-               $cidr = $dc_conf->{migration}->{network};
-           }
+    if (!$statefile && scalar(keys %{$conf->{pending}})) {
+       vmconfig_apply_pending($vmid, $conf, $storecfg);
+       $conf = PVE::QemuConfig->load_config($vmid); # update/reload
+    }

 
 

-           if (defined($cidr)) {
-               my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
+    PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);

 
 

-               die "could not get IP: no address configured on local " .
-                   "node for network '$cidr'\n" if scalar(@$ips) == 0;
+    my $defaults = load_defaults();

 
 

-               die "could not get IP: multiple addresses configured on local " .
-                   "node for network '$cidr'\n" if scalar(@$ips) > 1;
+    # set environment variable useful inside network script
+    $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;

 
 

-               $migration_ip = @$ips[0];
-           }
+    PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);

 
 

-           $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
-               if !defined($migration_ip);
+    my $forcemachine = $params->{forcemachine};
+    my $forcecpu = $params->{forcecpu};
+    if ($resume) {
+       # enforce machine and CPU type on suspended vm to ensure HW compatibility
+       $forcemachine = $conf->{runningmachine};
+       $forcecpu = $conf->{runningcpu};
+       print "Resuming suspended VM\n";
+    }

 
 

-           return $migration_ip;
-       };
+    my ($cmd, $vollist, $spice_port) =
+       config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);

 
 

-       my $migrate_uri;
-       if ($statefile) {
-           if ($statefile eq 'tcp') {
-               my $localip = "localhost";
-               my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
-               my $nodename = nodename();
+    my $migration_ip;
+    my $get_migration_ip = sub {
+       my ($nodename) = @_;

 
 

-               if (!defined($migration_type)) {
-                   if (defined($datacenterconf->{migration}->{type})) {
-                       $migration_type = $datacenterconf->{migration}->{type};
-                   } else {
-                       $migration_type = 'secure';
-                   }
-               }
+       return $migration_ip if defined($migration_ip);

 
 

-               if ($migration_type eq 'insecure') {
-                   $localip = $get_migration_ip->($migration_network, $nodename);
-                   $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
-               }
+       my $cidr = $migrate_opts->{network};

 
 

-               my $pfamily = PVE::Tools::get_host_address_family($nodename);
-               my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
-               $migrate_uri = "tcp:${localip}:${migrate_port}";
-               push @$cmd, '-incoming', $migrate_uri;
-               push @$cmd, '-S';
+       if (!defined($cidr)) {
+           my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
+           $cidr = $dc_conf->{migration}->{network};
+       }

 
 

-           } elsif ($statefile eq 'unix') {
-               # should be default for secure migrations as a ssh TCP forward
-               # tunnel is not deterministic reliable ready and fails regurarly
-               # to set up in time, so use UNIX socket forwards
-               my $socket_addr = "/run/qemu-server/$vmid.migrate";
-               unlink $socket_addr;
+       if (defined($cidr)) {
+           my $ips = PVE::Network::get_local_ip_from_cidr($cidr);

 
 

-               $migrate_uri = "unix:$socket_addr";
+           die "could not get IP: no address configured on local " .
+               "node for network '$cidr'\n" if scalar(@$ips) == 0;

 
 

-               push @$cmd, '-incoming', $migrate_uri;
-               push @$cmd, '-S';
+           die "could not get IP: multiple addresses configured on local " .
+               "node for network '$cidr'\n" if scalar(@$ips) > 1;

 
 

-           } elsif (-e $statefile) {
-               push @$cmd, '-loadstate', $statefile;
-           } else {
-               my $statepath = PVE::Storage::path($storecfg, $statefile);
-               push @$vollist, $statefile;
-               push @$cmd, '-loadstate', $statepath;
-           }
-       } elsif ($paused) {
-           push @$cmd, '-S';
+           $migration_ip = @$ips[0];

        }
 
        }
 

-       # host pci devices
-        for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
-          my $d = parse_hostpci($conf->{"hostpci$i"});
-          next if !$d;
-         my $pcidevices = $d->{pciid};
-         foreach my $pcidevice (@$pcidevices) {
-               my $pciid = $pcidevice->{id};
+       $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
+           if !defined($migration_ip);

 
 

-               my $info = PVE::SysFSTools::pci_device_info("$pciid");
-               die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
-               die "no pci device info for device '$pciid'\n" if !$info;
+       return $migration_ip;
+    };
+
+    my $migrate_uri;
+    if ($statefile) {
+       if ($statefile eq 'tcp') {
+           my $localip = "localhost";
+           my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
+           my $nodename = nodename();

 
 

-               if ($d->{mdev}) {
-                   my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
-                   PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
+           if (!defined($migration_type)) {
+               if (defined($datacenterconf->{migration}->{type})) {
+                   $migration_type = $datacenterconf->{migration}->{type};

                } else {
                } else {

-                   die "can't unbind/bind pci group to vfio '$pciid'\n"
-                       if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
-                   die "can't reset pci device '$pciid'\n"
-                       if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
+                   $migration_type = 'secure';

                }
                }

-         }
-        }
+           }

 
 

-       PVE::Storage::activate_volumes($storecfg, $vollist);
+           if ($migration_type eq 'insecure') {
+               $localip = $get_migration_ip->($nodename);
+               $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
+           }

 
 

-       eval {
-           run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
-               outfunc => sub {}, errfunc => sub {});
-       };
-       # Issues with the above 'stop' not being fully completed are extremely rare, a very low
-       # timeout should be more than enough here...
-       PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
-
-       my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
-                                                 : $defaults->{cpuunits};
-
-       my $start_timeout = $timeout // config_aware_timeout($conf, $is_suspended);
-       my %run_params = (
-           timeout => $statefile ? undef : $start_timeout,
-           umask => 0077,
-           noerr => 1,
-       );
+           my $pfamily = PVE::Tools::get_host_address_family($nodename);
+           my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
+           $migrate_uri = "tcp:${localip}:${migrate_port}";
+           push @$cmd, '-incoming', $migrate_uri;
+           push @$cmd, '-S';

 
 

-       # when migrating, prefix QEMU output so other side can pick up any
-       # errors that might occur and show the user
-       if ($migratedfrom) {
-           $run_params{quiet} = 1;
-           $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
-       }
+       } elsif ($statefile eq 'unix') {
+           # should be default for secure migrations as a ssh TCP forward
+           # tunnel is not deterministic reliable ready and fails regurarly
+           # to set up in time, so use UNIX socket forwards
+           my $socket_addr = "/run/qemu-server/$vmid.migrate";
+           unlink $socket_addr;

 
 

-       my %properties = (
-           Slice => 'qemu.slice',
-           KillMode => 'none',
-           CPUShares => $cpuunits
-       );
+           $migrate_uri = "unix:$socket_addr";
+
+           push @$cmd, '-incoming', $migrate_uri;
+           push @$cmd, '-S';

 
 

-       if (my $cpulimit = $conf->{cpulimit}) {
-           $properties{CPUQuota} = int($cpulimit * 100);
+       } elsif (-e $statefile) {
+           push @$cmd, '-loadstate', $statefile;
+       } else {
+           my $statepath = PVE::Storage::path($storecfg, $statefile);
+           push @$vollist, $statefile;
+           push @$cmd, '-loadstate', $statepath;

        }
        }

-       $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
+    } elsif ($params->{paused}) {
+       push @$cmd, '-S';
+    }

 
 

-       my $run_qemu = sub {
-           PVE::Tools::run_fork sub {
-               PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
+    # host pci devices
+    for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
+      my $d = parse_hostpci($conf->{"hostpci$i"});
+      next if !$d;
+      my $pcidevices = $d->{pciid};
+      foreach my $pcidevice (@$pcidevices) {
+           my $pciid = $pcidevice->{id};
+
+           my $info = PVE::SysFSTools::pci_device_info("$pciid");
+           die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
+           die "no pci device info for device '$pciid'\n" if !$info;
+
+           if ($d->{mdev}) {
+               my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
+               PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
+           } else {
+               die "can't unbind/bind pci group to vfio '$pciid'\n"
+                   if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
+               die "can't reset pci device '$pciid'\n"
+                   if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
+           }
+      }
+    }

 
 

-               my $exitcode = run_command($cmd, %run_params);
-               die "QEMU exited with code $exitcode\n" if $exitcode;
-           };
-       };
+    PVE::Storage::activate_volumes($storecfg, $vollist);

 
 

-       if ($conf->{hugepages}) {
+    eval {
+       run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
+           outfunc => sub {}, errfunc => sub {});
+    };
+    # Issues with the above 'stop' not being fully completed are extremely rare, a very low
+    # timeout should be more than enough here...
+    PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
+
+    my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
+                                             : $defaults->{cpuunits};
+
+    my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
+    my %run_params = (
+       timeout => $statefile ? undef : $start_timeout,
+       umask => 0077,
+       noerr => 1,
+    );

 
 

-           my $code = sub {
-               my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
-               my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
+    # when migrating, prefix QEMU output so other side can pick up any
+    # errors that might occur and show the user
+    if ($migratedfrom) {
+       $run_params{quiet} = 1;
+       $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
+    }

 
 

-               PVE::QemuServer::Memory::hugepages_mount();
-               PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
+    my %properties = (
+       Slice => 'qemu.slice',
+       KillMode => 'none',
+       CPUShares => $cpuunits
+    );

 
 

-               eval { $run_qemu->() };
-               if (my $err = $@) {
-                   PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
-                   die $err;
-               }
+    if (my $cpulimit = $conf->{cpulimit}) {
+       $properties{CPUQuota} = int($cpulimit * 100);
+    }
+    $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick

 
 

-               PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
-           };
-           eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
+    my $run_qemu = sub {
+       PVE::Tools::run_fork sub {
+           PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
+
+           my $exitcode = run_command($cmd, %run_params);
+           die "QEMU exited with code $exitcode\n" if $exitcode;
+       };
+    };
+
+    if ($conf->{hugepages}) {
+
+       my $code = sub {
+           my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
+           my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
+
+           PVE::QemuServer::Memory::hugepages_mount();
+           PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);

 
 

-       } else {

            eval { $run_qemu->() };
            eval { $run_qemu->() };

-       }
+           if (my $err = $@) {
+               PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
+                   if !$conf->{keephugepages};
+               die $err;
+           }

 
 

-       if (my $err = $@) {
-           # deactivate volumes if start fails
-           eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
-           die "start failed: $err";
-       }
+           PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
+               if !$conf->{keephugepages};
+       };
+       eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };

 
 

-       print "migration listens on $migrate_uri\n" if $migrate_uri;
+    } else {
+       eval { $run_qemu->() };
+    }

 
 

-       if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
-           eval { mon_cmd($vmid, "cont"); };
-           warn $@ if $@;
-       }
+    if (my $err = $@) {
+       # deactivate volumes if start fails
+       eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
+       die "start failed: $err";
+    }

 
 

-       #start nbd server for storage migration
-       if ($targetstorage) {
-           $nbd_protocol_version //= 0;
+    print "migration listens on $migrate_uri\n" if $migrate_uri;
+    $res->{migrate_uri} = $migrate_uri;

 
 

-           my $migrate_storage_uri;
-           # nbd_protocol_version > 0 for unix socket support
-           if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
-               my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
-               mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
-               $migrate_storage_uri = "nbd:unix:$socket_path";
-           } else {
-               my $nodename = nodename();
-               my $localip = $get_migration_ip->($migration_network, $nodename);
-               my $pfamily = PVE::Tools::get_host_address_family($nodename);
-               my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
+    if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
+       eval { mon_cmd($vmid, "cont"); };
+       warn $@ if $@;
+    }

 
 

-               mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
-               $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
-               $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
-           }
+    #start nbd server for storage migration
+    if (my $nbd = $migrate_opts->{nbd}) {
+       my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;

 
 

-           foreach my $opt (sort keys %$local_volumes) {
-               my $drivestr = $local_volumes->{$opt};
-               mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
-               print "storage migration listens on $migrate_storage_uri:exportname=drive-$opt volume:$drivestr\n";
-           }
+       my $migrate_storage_uri;
+       # nbd_protocol_version > 0 for unix socket support
+       if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
+           my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
+           mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
+           $migrate_storage_uri = "nbd:unix:$socket_path";
+       } else {
+           my $nodename = nodename();
+           my $localip = $get_migration_ip->($nodename);
+           my $pfamily = PVE::Tools::get_host_address_family($nodename);
+           my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
+
+           mon_cmd($vmid, "nbd-server-start", addr => {
+               type => 'inet',
+               data => {
+                   host => "${localip}",
+                   port => "${storage_migrate_port}",
+               },
+           });
+           $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
+           $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";

        }
 
        }
 

-       if ($migratedfrom) {
-           eval {
-               set_migration_caps($vmid);
-           };
-           warn $@ if $@;
+       $res->{migrate_storage_uri} = $migrate_storage_uri;

 
 

-           if ($spice_port) {
-               print "spice listens on port $spice_port\n";
-               if ($spice_ticket) {
-                   mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
-                   mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
-               }
-           }
+       foreach my $opt (sort keys %$nbd) {
+           my $drivestr = $nbd->{$opt}->{drivestr};
+           my $volid = $nbd->{$opt}->{volid};
+           mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
+           my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
+           print "storage migration listens on $nbd_uri volume:$drivestr\n";
+           print "re-using replicated volume: $opt - $volid\n"
+               if $nbd->{$opt}->{replicated};

 
 

-       } else {
-           mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
-               if !$statefile && $conf->{balloon};
+           $res->{drives}->{$opt} = $nbd->{$opt};
+           $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
+       }
+    }

 
 

-           foreach my $opt (keys %$conf) {
-               next if $opt !~  m/^net\d+$/;
-               my $nicconf = parse_net($conf->{$opt});
-               qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
+    if ($migratedfrom) {
+       eval {
+           set_migration_caps($vmid);
+       };
+       warn $@ if $@;
+
+       if ($spice_port) {
+           print "spice listens on port $spice_port\n";
+           $res->{spice_port} = $spice_port;
+           if ($migrate_opts->{spice_ticket}) {
+               mon_cmd($vmid, "set_password", protocol => 'spice', password =>
+                   $migrate_opts->{spice_ticket});
+               mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");

            }
        }
 
            }
        }
 

-       mon_cmd($vmid, 'qom-set',
-                   path => "machine/peripheral/balloon0",
-                   property => "guest-stats-polling-interval",
-                   value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
+    } else {
+       mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
+           if !$statefile && $conf->{balloon};

 
 

-       if ($is_suspended) {
-           print "Resumed VM, removing state\n";
-           if (my $vmstate = $conf->{vmstate}) {
-               PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
-               PVE::Storage::vdisk_free($storecfg, $vmstate);
-           }
-           delete $conf->@{qw(lock vmstate runningmachine)};
-           PVE::QemuConfig->write_config($vmid, $conf);
+       foreach my $opt (keys %$conf) {
+           next if $opt !~  m/^net\d+$/;
+           my $nicconf = parse_net($conf->{$opt});
+           qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};

        }
        }

+    }

 
 

-       PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
-    });
+    mon_cmd($vmid, 'qom-set',
+               path => "machine/peripheral/balloon0",
+               property => "guest-stats-polling-interval",
+               value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
+
+    if ($resume) {
+       print "Resumed VM, removing state\n";
+       if (my $vmstate = $conf->{vmstate}) {
+           PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
+           PVE::Storage::vdisk_free($storecfg, $vmstate);
+       }
+       delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
+       PVE::QemuConfig->write_config($vmid, $conf);
+    }
+
+    PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
+
+    return $res;

 }
 
 sub vm_commandline {
 }
 
 sub vm_commandline {


@@ -5065,13 +5276,15 @@ sub vm_commandline {
 
     my $conf = PVE::QemuConfig->load_config($vmid);
     my $forcemachine;
 
     my $conf = PVE::QemuConfig->load_config($vmid);
     my $forcemachine;

+    my $forcecpu;

 
     if ($snapname) {
        my $snapshot = $conf->{snapshots}->{$snapname};
        die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
 
 
     if ($snapname) {
        my $snapshot = $conf->{snapshots}->{$snapname};
        die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
 

-       # check for a 'runningmachine' in snapshot
-       $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
+       # check for machine or CPU overrides in snapshot
+       $forcemachine = $snapshot->{runningmachine};
+       $forcecpu = $snapshot->{runningcpu};

 
        $snapshot->{digest} = $conf->{digest}; # keep file digest for API
 
 
        $snapshot->{digest} = $conf->{digest}; # keep file digest for API
 


@@ -5080,7 +5293,8 @@ sub vm_commandline {
 
     my $defaults = load_defaults();
 
 
     my $defaults = load_defaults();
 

-    my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
+    my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
+       $forcemachine, $forcecpu);

 
     return PVE::Tools::cmd2string($cmd);
 }
 
     return PVE::Tools::cmd2string($cmd);
 }


@@ -5131,7 +5345,7 @@ sub vm_stop_cleanup {
        }
 
        if ($conf->{ivshmem}) {
        }
 
        if ($conf->{ivshmem}) {

-           my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
+           my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});

            # just delete it for now, VMs which have this already open do not
            # are affected, but new VMs will get a separated one. If this
            # becomes an issue we either add some sort of ref-counting or just
            # just delete it for now, VMs which have this already open do not
            # are affected, but new VMs will get a separated one. If this
            # becomes an issue we either add some sort of ref-counting or just


@@ -5208,7 +5422,11 @@ sub _do_vm_stop {
            return;
        }
     } else {
            return;
        }
     } else {

-       if ($force) {
+       if (!check_running($vmid, $nocheck)) {
+           warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
+           return;
+       }
+       if ($force) {

            warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
            kill 15, $pid;
        } else {
            warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
            kill 15, $pid;
        } else {


@@ -5313,7 +5531,8 @@ sub vm_suspend {
            }
 
 
            }
 
 

-           $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
+           $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
+               $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);

            $path = PVE::Storage::path($storecfg, $vmstate);
            PVE::QemuConfig->write_config($vmid, $conf);
        } else {
            $path = PVE::Storage::path($storecfg, $vmstate);
            PVE::QemuConfig->write_config($vmid, $conf);
        } else {


@@ -5354,7 +5573,7 @@ sub vm_suspend {
                    mon_cmd($vmid, "savevm-end");
                    PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
                    PVE::Storage::vdisk_free($storecfg, $vmstate);
                    mon_cmd($vmid, "savevm-end");
                    PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
                    PVE::Storage::vdisk_free($storecfg, $vmstate);

-                   delete $conf->@{qw(vmstate runningmachine)};
+                   delete $conf->@{qw(vmstate runningmachine runningcpu)};

                    PVE::QemuConfig->write_config($vmid, $conf);
                };
                warn $@ if $@;
                    PVE::QemuConfig->write_config($vmid, $conf);
                };
                warn $@ if $@;


@@ -5456,28 +5675,12 @@ sub tar_restore_cleanup {
 sub restore_file_archive {
     my ($archive, $vmid, $user, $opts) = @_;
 
 sub restore_file_archive {
     my ($archive, $vmid, $user, $opts) = @_;
 

-    my $format = $opts->{format};
-    my $comp;
-
-    if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
-       $format = 'tar' if !$format;
-       $comp = 'gzip';
-    } elsif ($archive =~ m/\.tar$/) {
-       $format = 'tar' if !$format;
-    } elsif ($archive =~ m/.tar.lzo$/) {
-       $format = 'tar' if !$format;
-       $comp = 'lzop';
-    } elsif ($archive =~ m/\.vma$/) {
-       $format = 'vma' if !$format;
-    } elsif ($archive =~ m/\.vma\.gz$/) {
-       $format = 'vma' if !$format;
-       $comp = 'gzip';
-    } elsif ($archive =~ m/\.vma\.lzo$/) {
-       $format = 'vma' if !$format;
-       $comp = 'lzop';
-    } else {
-       $format = 'vma' if !$format; # default
-    }
+    return restore_vma_archive($archive, $vmid, $user, $opts)
+       if $archive eq '-';
+
+    my $info = PVE::Storage::archive_info($archive);
+    my $format = $opts->{format} // $info->{format};
+    my $comp = $info->{compression};

 
     # try to detect archive format
     if ($format eq 'tar') {
 
     # try to detect archive format
     if ($format eq 'tar') {


@@ -5491,7 +5694,7 @@ sub restore_file_archive {
 my $restore_cleanup_oldconf = sub {
     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
 
 my $restore_cleanup_oldconf = sub {
     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
 

-    foreach_drive($oldconf, sub {
+    PVE::QemuConfig->foreach_volume($oldconf, sub {

        my ($ds, $drive) = @_;
 
        return if drive_is_cdrom($drive, 1);
        my ($ds, $drive) = @_;
 
        return if drive_is_cdrom($drive, 1);


@@ -5568,12 +5771,13 @@ my $parse_backup_hints = sub {
            my $drive = parse_drive($virtdev, $2);
            if (drive_is_cloudinit($drive)) {
                my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
            my $drive = parse_drive($virtdev, $2);
            if (drive_is_cloudinit($drive)) {
                my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});

+               $storeid = $options->{storage} if defined ($options->{storage});

                my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
                my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
 
                $virtdev_hash->{$virtdev} = {
                    format => $format,
                my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
                my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
 
                $virtdev_hash->{$virtdev} = {
                    format => $format,

-                   storeid => $options->{storage} // $storeid,
+                   storeid => $storeid,

                    size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
                    is_cloudinit => 1,
                };
                    size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
                    is_cloudinit => 1,
                };


@@ -5611,7 +5815,8 @@ my $restore_allocate_devices = sub {
            $name .= ".$d->{format}" if $d->{format} ne 'raw';
        }
 
            $name .= ".$d->{format}" if $d->{format} ne 'raw';
        }
 

-       my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
+       my $volid = PVE::Storage::vdisk_alloc(
+           $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);

 
        print STDERR "new volume ID is '$volid'\n";
        $d->{volid} = $volid;
 
        print STDERR "new volume ID is '$volid'\n";
        $d->{volid} = $volid;


@@ -5740,7 +5945,7 @@ sub update_disk_config {
     my ($vmid, $conf, $volid_hash) = @_;
 
     my $changes;
     my ($vmid, $conf, $volid_hash) = @_;
 
     my $changes;

-    my $prefix = "VM $vmid:";
+    my $prefix = "VM $vmid";

 
     # used and unused disks
     my $referenced = {};
 
     # used and unused disks
     my $referenced = {};


@@ -5752,35 +5957,37 @@ sub update_disk_config {
     my $referencedpath = {};
 
     # update size info
     my $referencedpath = {};
 
     # update size info

-    foreach my $opt (keys %$conf) {
-       if (is_valid_drivename($opt)) {
-           my $drive = parse_drive($opt, $conf->{$opt});
-           my $volid = $drive->{file};
-           next if !$volid;
-
-           # mark volid as "in-use" for next step
-           $referenced->{$volid} = 1;
-           if ($volid_hash->{$volid} &&
-               (my $path = $volid_hash->{$volid}->{path})) {
-               $referencedpath->{$path} = 1;
-           }
+    PVE::QemuConfig->foreach_volume($conf, sub {
+       my ($opt, $drive) = @_;
+
+       my $volid = $drive->{file};
+       return if !$volid;
+       my $volume = $volid_hash->{$volid};

 
 

-           next if drive_is_cdrom($drive);
-           next if !$volid_hash->{$volid};
+       # mark volid as "in-use" for next step
+       $referenced->{$volid} = 1;
+       if ($volume && (my $path = $volume->{path})) {
+           $referencedpath->{$path} = 1;
+       }

 
 

-           my ($updated, $old_size, $new_size) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash);
-           if (defined($updated)) {
-               $changes = 1;
-               $conf->{$opt} = print_drive($updated);
-               print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
-           }
+       return if drive_is_cdrom($drive);
+       return if !$volume;
+
+       my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
+       if (defined($updated)) {
+           $changes = 1;
+           $conf->{$opt} = print_drive($updated);
+           print "$prefix ($opt): $msg\n";

        }
        }

-    }
+    });

 
     # remove 'unusedX' entry if volume is used
 
     # remove 'unusedX' entry if volume is used

-    foreach my $opt (keys %$conf) {
-       next if $opt !~ m/^unused\d+$/;
-       my $volid = $conf->{$opt};
+    PVE::QemuConfig->foreach_unused_volume($conf, sub {
+       my ($opt, $drive) = @_;
+
+       my $volid = $drive->{file};
+       return if !$volid;
+

        my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
        if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
            print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
        my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
        if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
            print "$prefix remove entry '$opt', its volume '$volid' is in use\n";


@@ -5790,7 +5997,7 @@ sub update_disk_config {
 
        $referenced->{$volid} = 1;
        $referencedpath->{$path} = 1 if $path;
 
        $referenced->{$volid} = 1;
        $referencedpath->{$path} = 1 if $path;

-    }
+    });

 
     foreach my $volid (sort keys %$volid_hash) {
        next if $volid =~ m/vm-$vmid-state-/;
 
     foreach my $volid (sort keys %$volid_hash) {
        next if $volid =~ m/vm-$vmid-state-/;


@@ -5869,8 +6076,11 @@ sub restore_proxmox_backup_archive {
     my $datastore = $scfg->{datastore};
     my $username = $scfg->{username} // 'root@pam';
     my $fingerprint = $scfg->{fingerprint};
     my $datastore = $scfg->{datastore};
     my $username = $scfg->{username} // 'root@pam';
     my $fingerprint = $scfg->{fingerprint};

+    my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);

 
     my $repo = "$username\@$server:$datastore";
 
     my $repo = "$username\@$server:$datastore";

+
+    # This is only used for `pbs-restore`!

     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
     local $ENV{PBS_PASSWORD} = $password;
     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
     local $ENV{PBS_PASSWORD} = $password;
     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);


@@ -5932,7 +6142,9 @@ sub restore_proxmox_backup_archive {
            }
        }
 
            }
        }
 

-       my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
+       my $is_qemu_server_backup = scalar(
+           grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
+       );

        if (!$is_qemu_server_backup) {
            die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
        }
        if (!$is_qemu_server_backup) {
            die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
        }


@@ -5951,7 +6163,7 @@ sub restore_proxmox_backup_archive {
        }
 
        my $fh = IO::File->new($cfgfn, "r") ||
        }
 
        my $fh = IO::File->new($cfgfn, "r") ||

-           "unable to read qemu-server.conf - $!\n";
+           die "unable to read qemu-server.conf - $!\n";

 
        my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
 
 
        my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
 


@@ -5973,6 +6185,7 @@ sub restore_proxmox_backup_archive {
 
            my $path = PVE::Storage::path($storecfg, $volid);
 
 
            my $path = PVE::Storage::path($storecfg, $volid);
 

+           # This is the ONLY user of the PBS_ env vars set on top of this function!

            my $pbs_restore_cmd = [
                '/usr/bin/pbs-restore',
                '--repository', $repo,
            my $pbs_restore_cmd = [
                '/usr/bin/pbs-restore',
                '--repository', $repo,


@@ -5982,6 +6195,9 @@ sub restore_proxmox_backup_archive {
                '--verbose',
                ];
 
                '--verbose',
                ];
 

+           push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
+           push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
+

            if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
                push @$pbs_restore_cmd, '--skip-zero';
            }
            if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
                push @$pbs_restore_cmd, '--skip-zero';
            }


@@ -6062,14 +6278,9 @@ sub restore_vma_archive {
     }
 
     if ($comp) {
     }
 
     if ($comp) {

-       my $cmd;
-       if ($comp eq 'gzip') {
-           $cmd = ['zcat', $readfrom];
-       } elsif ($comp eq 'lzop') {
-           $cmd = ['lzop', '-d', '-c', $readfrom];
-       } else {
-           die "unknown compression method '$comp'\n";
-       }
+       my $info = PVE::Storage::decompressor_info('vma', $comp);
+       my $cmd = $info->{decompressor};
+       push @$cmd, $readfrom;

        $add_pipe->($cmd);
     }
 
        $add_pipe->($cmd);
     }
 


@@ -6113,7 +6324,7 @@ sub restore_vma_archive {
 
        # we can read the config - that is already extracted
        my $fh = IO::File->new($cfgfn, "r") ||
 
        # we can read the config - that is already extracted
        my $fh = IO::File->new($cfgfn, "r") ||

-           "unable to read qemu-server.conf - $!\n";
+           die "unable to read qemu-server.conf - $!\n";

 
        my $fwcfgfn = "$tmpdir/qemu-server.fw";
        if (-f $fwcfgfn) {
 
        my $fwcfgfn = "$tmpdir/qemu-server.fw";
        if (-f $fwcfgfn) {


@@ -6359,7 +6570,7 @@ sub foreach_storage_used_by_vm {
 
     my $sidhash = {};
 
 
     my $sidhash = {};
 

-    foreach_drive($conf, sub {
+    PVE::QemuConfig->foreach_volume($conf, sub {

        my ($ds, $drive) = @_;
        return if drive_is_cdrom($drive);
 
        my ($ds, $drive) = @_;
        return if drive_is_cdrom($drive);
 


@@ -6410,7 +6621,7 @@ sub template_create {
 
     my $storecfg = PVE::Storage::config();
 
 
     my $storecfg = PVE::Storage::config();
 

-    foreach_drive($conf, sub {
+    PVE::QemuConfig->foreach_volume($conf, sub {

        my ($ds, $drive) = @_;
 
        return if drive_is_cdrom($drive);
        my ($ds, $drive) = @_;
 
        return if drive_is_cdrom($drive);


@@ -6732,7 +6943,7 @@ sub qemu_blockjobs_cancel {
 
 sub clone_disk {
     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
 
 sub clone_disk {
     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,

-       $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
+       $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;

 
     my $newvolid;
 
 
     my $newvolid;
 


@@ -6755,8 +6966,11 @@ sub clone_disk {
            $name .= ".$dst_format" if $dst_format ne 'raw';
            $snapname = undef;
            $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
            $name .= ".$dst_format" if $dst_format ne 'raw';
            $snapname = undef;
            $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;

+       } elsif ($drivename eq 'efidisk0') {
+           $size = get_efivars_size($conf);

        }
        }

-       $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
+       $size /= 1024;
+       $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, $size);

        push @$newvollist, $newvolid;
 
        PVE::Storage::activate_volumes($storecfg, [$newvolid]);
        push @$newvollist, $newvolid;
 
        PVE::Storage::activate_volumes($storecfg, [$newvolid]);


@@ -6768,7 +6982,17 @@ sub clone_disk {
        my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
        if (!$running || $snapname) {
            # TODO: handle bwlimits
        my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
        if (!$running || $snapname) {
            # TODO: handle bwlimits

-           qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
+           if ($drivename eq 'efidisk0') {
+               # the relevant data on the efidisk may be smaller than the source
+               # e.g. on RBD/ZFS, so we use dd to copy only the amount
+               # that is given by the OVMF_VARS.fd
+               my $src_path = PVE::Storage::path($storecfg, $drive->{file});
+               my $dst_path = PVE::Storage::path($storecfg, $newvolid);
+               run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size",
+                   "if=$src_path", "of=$dst_path"]);
+           } else {
+               qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
+           }

        } else {
 
            my $kvmver = get_running_qemu_version ($vmid);
        } else {
 
            my $kvmver = get_running_qemu_version ($vmid);


@@ -6777,7 +7001,8 @@ sub clone_disk {
                    if $drive->{iothread};
            }
 
                    if $drive->{iothread};
            }
 

-           qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $completion, $qga, $bwlimit);
+           qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
+               $completion, $qga, $bwlimit);

        }
     }
 
        }
     }
 


@@ -6809,7 +7034,7 @@ sub qemu_use_old_bios_files {
         $machine_type = $1;
         $use_old_bios_files = 1;
     } else {
         $machine_type = $1;
         $use_old_bios_files = 1;
     } else {

-       my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
+       my $version = extract_version($machine_type, kvm_user_version());

         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
         # load new efi bios files on migration. So this hack is required to allow
         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
         # load new efi bios files on migration. So this hack is required to allow
         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when


@@ -6820,6 +7045,26 @@ sub qemu_use_old_bios_files {
     return ($use_old_bios_files, $machine_type);
 }
 
     return ($use_old_bios_files, $machine_type);
 }
 

+sub get_efivars_size {
+    my ($conf) = @_;
+    my $arch = get_vm_arch($conf);
+    my (undef, $ovmf_vars) = get_ovmf_files($arch);
+    die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
+    return -s $ovmf_vars;
+}
+
+sub update_efidisk_size {
+    my ($conf) = @_;
+
+    return if !defined($conf->{efidisk0});
+
+    my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
+    $disk->{size} = get_efivars_size($conf);
+    $conf->{efidisk0} = print_drive($disk);
+
+    return;
+}
+

 sub create_efidisk($$$$$) {
     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
 
 sub create_efidisk($$$$$) {
     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
 


@@ -6864,7 +7109,9 @@ sub scsihw_infos {
     }
 
     my $controller = int($drive->{index} / $maxdev);
     }
 
     my $controller = int($drive->{index} / $maxdev);

-    my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
+    my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
+       ? "virtioscsi"
+       : "scsihw";

 
     return ($maxdev, $controller, $controller_prefix);
 }
 
     return ($maxdev, $controller, $controller_prefix);
 }


@@ -6967,6 +7214,74 @@ sub clear_reboot_request {
     return $res;
 }
 
     return $res;
 }
 

+sub bootorder_from_legacy {
+    my ($conf, $bootcfg) = @_;
+
+    my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
+    my $bootindex_hash = {};
+    my $i = 1;
+    foreach my $o (split(//, $boot)) {
+       $bootindex_hash->{$o} = $i*100;
+       $i++;
+    }
+
+    my $bootorder = {};
+
+    PVE::QemuConfig->foreach_volume($conf, sub {
+       my ($ds, $drive) = @_;
+
+       if (drive_is_cdrom ($drive, 1)) {
+           if ($bootindex_hash->{d}) {
+               $bootorder->{$ds} = $bootindex_hash->{d};
+               $bootindex_hash->{d} += 1;
+           }
+       } elsif ($bootindex_hash->{c}) {
+           $bootorder->{$ds} = $bootindex_hash->{c}
+               if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
+           $bootindex_hash->{c} += 1;
+       }
+    });
+
+    if ($bootindex_hash->{n}) {
+       for (my $i = 0; $i < $MAX_NETS; $i++) {
+           my $netname = "net$i";
+           next if !$conf->{$netname};
+           $bootorder->{$netname} = $bootindex_hash->{n};
+           $bootindex_hash->{n} += 1;
+       }
+    }
+
+    return $bootorder;
+}
+
+# Generate default device list for 'boot: order=' property. Matches legacy
+# default boot order, but with explicit device names. This is important, since
+# the fallback for when neither 'order' nor the old format is specified relies
+# on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
+sub get_default_bootdevices {
+    my ($conf) = @_;
+
+    my @ret = ();
+
+    # harddisk
+    my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
+    push @ret, $first if $first;
+
+    # cdrom
+    $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
+    push @ret, $first if $first;
+
+    # network
+    for (my $i = 0; $i < $MAX_NETS; $i++) {
+       my $netname = "net$i";
+       next if !$conf->{$netname};
+       push @ret, $netname;
+       last;
+    }
+
+    return \@ret;
+}
+

 # bash completion helper
 
 sub complete_backup_archives {
 # bash completion helper
 
 sub complete_backup_archives {


@@ -6985,7 +7300,7 @@ sub complete_backup_archives {
     my $res = [];
     foreach my $id (keys %$data) {
        foreach my $item (@{$data->{$id}}) {
     my $res = [];
     foreach my $id (keys %$data) {
        foreach my $item (@{$data->{$id}}) {

-           next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
+           next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;

            push @$res, $item->{volid} if defined($item->{volid});
        }
     }
            push @$res, $item->{volid} if defined($item->{volid});
        }
     }