1 package PVE
::QemuServer
;
11 use File
::Copy
qw(copy);
24 use Storable
qw(dclone);
25 use Time
::HiRes
qw(gettimeofday usleep);
29 use PVE
::Cluster
qw(cfs_register_file cfs_read_file cfs_write_file);
31 use PVE
::DataCenterConfig
;
32 use PVE
::Exception
qw(raise raise_param_exc);
33 use PVE
::Format
qw(render_duration render_bytes);
34 use PVE
::GuestHelpers
qw(safe_string_ne safe_num_ne safe_boolean_ne);
36 use PVE
::JSONSchema
qw(get_standard_option parse_property_string);
39 use PVE
::RPCEnvironment
;
43 use PVE
::Tools
qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
47 use PVE
::QemuServer
::Helpers
qw(min_version config_aware_timeout);
48 use PVE
::QemuServer
::Cloudinit
;
49 use PVE
::QemuServer
::CGroup
;
50 use PVE
::QemuServer
::CPUConfig
qw(print_cpu_device get_cpu_options);
51 use PVE
::QemuServer
::Drive
qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
52 use PVE
::QemuServer
::Machine
;
53 use PVE
::QemuServer
::Memory
;
54 use PVE
::QemuServer
::Monitor
qw(mon_cmd);
55 use PVE
::QemuServer
::PCI
qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
56 use PVE
::QemuServer
::USB
qw(parse_usb_device);
60 require PVE
::Network
::SDN
::Zones
;
64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
68 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
69 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
72 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
73 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
76 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
77 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
80 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
81 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
84 "$EDK2_FW_BASE/OVMF_CODE.fd",
85 "$EDK2_FW_BASE/OVMF_VARS.fd",
90 "$EDK2_FW_BASE/AAVMF_CODE.fd",
91 "$EDK2_FW_BASE/AAVMF_VARS.fd",
96 my $cpuinfo = PVE
::ProcFSTools
::read_cpuinfo
();
98 # Note about locking: we use flock on the config file protect against concurent actions.
99 # Aditionaly, we have a 'lock' setting in the config file. This can be set to 'migrate',
100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
101 # But you can ignore this kind of lock with the --skiplock flag.
103 cfs_register_file
('/qemu-server/',
107 PVE
::JSONSchema
::register_standard_option
('pve-qm-stateuri', {
108 description
=> "Some command save/restore state from this location.",
114 PVE
::JSONSchema
::register_standard_option
('pve-qemu-machine', {
115 description
=> "Specifies the Qemu machine type.",
117 pattern
=> '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
122 PVE
::JSONSchema
::register_standard_option
('pve-targetstorage', {
123 description
=> "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
125 format
=> 'storage-pair-list',
129 #no warnings 'redefine';
133 $nodename_cache //= PVE
::INotify
::nodename
();
134 return $nodename_cache;
141 enum
=> [qw(i6300esb ib700)],
142 description
=> "Watchdog type to emulate.",
143 default => 'i6300esb',
148 enum
=> [qw(reset shutdown poweroff pause debug none)],
149 description
=> "The action to perform if after activation the guest fails to poll the watchdog in time.",
153 PVE
::JSONSchema
::register_format
('pve-qm-watchdog', $watchdog_fmt);
157 description
=> "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
162 fstrim_cloned_disks
=> {
163 description
=> "Run fstrim after moving a disk or migrating the VM.",
169 description
=> "Select the agent type",
173 enum
=> [qw(virtio isa)],
179 description
=> "Select the VGA type.",
184 enum
=> [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
187 description
=> "Sets the VGA memory (in MiB). Has no effect with serial display.",
199 description
=> "The size of the file in MB.",
203 pattern
=> '[a-zA-Z0-9\-]+',
205 format_description
=> 'string',
206 description
=> "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
213 enum
=> [qw(ich9-intel-hda intel-hda AC97)],
214 description
=> "Configure an audio device."
218 enum
=> ['spice', 'none'],
221 description
=> "Driver backend for the audio device."
225 my $spice_enhancements_fmt = {
230 description
=> "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
234 enum
=> ['off', 'all', 'filter'],
237 description
=> "Enable video streaming. Uses compression for detected video streams."
244 enum
=> ['/dev/urandom', '/dev/random', '/dev/hwrng'],
246 description
=> "The file on the host to gather entropy from. In most cases '/dev/urandom'"
247 ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
248 ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
249 ." still seeded from real entropy, and the bytes provided will most likely be mixed"
250 ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
251 ." a hardware RNG from the host.",
255 description
=> "Maximum bytes of entropy allowed to get injected into the guest every"
256 ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
257 ." `0` to disable limiting (potentially dangerous!).",
260 # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
261 # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
262 # reading from /dev/urandom
267 description
=> "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
268 ." the guest to retrieve another 'max_bytes' of entropy.",
274 my $meta_info_fmt = {
277 description
=> "The guest creation timestamp as UNIX epoch time",
283 description
=> "The QEMU (machine) version from the time this VM was created.",
284 pattern
=> '\d+(\.\d+)+',
293 description
=> "Specifies whether a VM will be started during system bootup.",
299 description
=> "Automatic restart after crash (currently ignored).",
304 type
=> 'string', format
=> 'pve-hotplug-features',
305 description
=> "Selectively enable hotplug features. This is a comma separated list of"
306 ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
307 ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
308 default => 'network,disk,usb',
313 description
=> "Allow reboot. If set to '0' the VM exit on reboot.",
319 description
=> "Lock/unlock the VM.",
320 enum
=> [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
325 description
=> "Limit of CPU usage.",
326 verbose_description
=> "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
327 ." total of '2' CPU time. Value '0' indicates no CPU limit.",
335 description
=> "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
336 verbose_description
=> "CPU weight for a VM. Argument is used in the kernel fair scheduler."
337 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
338 ." weights of all the other running VMs.",
341 default => 'cgroup v1: 1024, cgroup v2: 100',
346 description
=> "Amount of RAM for the VM in MB. This is the maximum available memory when"
347 ." you use the balloon device.",
354 description
=> "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
360 description
=> "Amount of memory shares for auto-ballooning. The larger the number is, the"
361 ." more memory this VM gets. Number is relative to weights of all other running VMs."
362 ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
370 description
=> "Keyboard layout for VNC server. The default is read from the"
371 ."'/etc/pve/datacenter.cfg' configuration file. It should not be necessary to set it.",
372 enum
=> PVE
::Tools
::kvmkeymaplist
(),
377 type
=> 'string', format
=> 'dns-name',
378 description
=> "Set a name for the VM. Only used on the configuration web interface.",
383 description
=> "SCSI controller model",
384 enum
=> [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
390 description
=> "Description for the VM. Shown in the web-interface VM's summary."
391 ." This is saved as comment inside the configuration file.",
392 maxLength
=> 1024 * 8,
397 enum
=> [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
398 description
=> "Specify guest operating system.",
399 verbose_description
=> <<EODESC,
400 Specify guest operating system. This is used to enable special
401 optimization/features for specific operating systems:
404 other;; unspecified OS
405 wxp;; Microsoft Windows XP
406 w2k;; Microsoft Windows 2000
407 w2k3;; Microsoft Windows 2003
408 w2k8;; Microsoft Windows 2008
409 wvista;; Microsoft Windows Vista
410 win7;; Microsoft Windows 7
411 win8;; Microsoft Windows 8/2012/2012r2
412 win10;; Microsoft Windows 10/2016/2019
413 win11;; Microsoft Windows 11/2022
414 l24;; Linux 2.4 Kernel
415 l26;; Linux 2.6 - 5.X Kernel
416 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
421 type
=> 'string', format
=> 'pve-qm-boot',
422 description
=> "Specify guest boot order. Use the 'order=' sub-property as usage with no"
423 ." key or 'legacy=' is deprecated.",
427 type
=> 'string', format
=> 'pve-qm-bootdisk',
428 description
=> "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
429 pattern
=> '(ide|sata|scsi|virtio)\d+',
434 description
=> "The number of CPUs. Please use option -sockets instead.",
441 description
=> "The number of CPU sockets.",
448 description
=> "The number of cores per socket.",
455 description
=> "Enable/disable NUMA.",
461 description
=> "Enable/disable hugepages memory.",
462 enum
=> [qw(any 2 1024)],
468 description
=> "Use together with hugepages. If enabled, hugepages will not not be deleted"
469 ." after VM shutdown and can be used for subsequent starts.",
474 description
=> "Number of hotplugged vcpus.",
481 description
=> "Enable/disable ACPI.",
486 description
=> "Enable/disable communication with the Qemu Guest Agent and its properties.",
488 format
=> $agent_fmt,
493 description
=> "Enable/disable KVM hardware virtualization.",
499 description
=> "Enable/disable time drift fix.",
505 description
=> "Set the real time clock (RTC) to local time. This is enabled by default if"
506 ." the `ostype` indicates a Microsoft Windows OS.",
511 description
=> "Freeze CPU at startup (use 'c' monitor command to start execution).",
515 type
=> 'string', format
=> $vga_fmt,
516 description
=> "Configure the VGA hardware.",
517 verbose_description
=> "Configure the VGA Hardware. If you want to use high resolution"
518 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
519 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
520 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
521 ." display server. For win* OS you can select how many independent displays you want,"
522 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
523 ." using a serial device as terminal.",
527 type
=> 'string', format
=> 'pve-qm-watchdog',
528 description
=> "Create a virtual hardware watchdog device.",
529 verbose_description
=> "Create a virtual hardware watchdog device. Once enabled (by a guest"
530 ." action), the watchdog must be periodically polled by an agent inside the guest or"
531 ." else the watchdog will reset the guest (or execute the respective action specified)",
536 typetext
=> "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
537 description
=> "Set the initial date of the real time clock. Valid format for date are:"
538 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
539 pattern
=> '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
542 startup
=> get_standard_option
('pve-startup-order'),
546 description
=> "Enable/disable Template.",
552 description
=> "Arbitrary arguments passed to kvm.",
553 verbose_description
=> <<EODESCR,
554 Arbitrary arguments passed to kvm, for example:
556 args: -no-reboot -no-hpet
558 NOTE: this option is for experts only.
565 description
=> "Enable/disable the USB tablet device.",
566 verbose_description
=> "Enable/disable the USB tablet device. This device is usually needed"
567 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
568 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
569 ." may consider disabling this to save some context switches. This is turned off by"
570 ." default if you use spice (`qm set <vmid> --vga qxl`).",
575 description
=> "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
579 migrate_downtime
=> {
582 description
=> "Set maximum tolerated downtime (in seconds) for migrations.",
588 type
=> 'string', format
=> 'pve-qm-ide',
589 typetext
=> '<volume>',
590 description
=> "This is an alias for option -ide2",
594 description
=> "Emulated CPU type.",
596 format
=> 'pve-vm-cpu-conf',
598 parent
=> get_standard_option
('pve-snapshot-name', {
600 description
=> "Parent snapshot name. This is used internally, and should not be modified.",
604 description
=> "Timestamp for snapshots.",
610 type
=> 'string', format
=> 'pve-volume-id',
611 description
=> "Reference to a volume which stores the VM state. This is used internally"
614 vmstatestorage
=> get_standard_option
('pve-storage-id', {
615 description
=> "Default storage for VM state volumes/files.",
618 runningmachine
=> get_standard_option
('pve-qemu-machine', {
619 description
=> "Specifies the QEMU machine type of the running vm. This is used internally"
623 description
=> "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
624 ." internally for snapshots.",
627 pattern
=> $PVE::QemuServer
::CPUConfig
::qemu_cmdline_cpu_re
,
628 format_description
=> 'QEMU -cpu parameter'
630 machine
=> get_standard_option
('pve-qemu-machine'),
632 description
=> "Virtual processor architecture. Defaults to the host.",
635 enum
=> [qw(x86_64 aarch64)],
638 description
=> "Specify SMBIOS type 1 fields.",
639 type
=> 'string', format
=> 'pve-qm-smbios1',
646 description
=> "Sets the protection flag of the VM. This will disable the remove VM and"
647 ." remove disk operations.",
653 enum
=> [ qw(seabios ovmf) ],
654 description
=> "Select BIOS implementation.",
655 default => 'seabios',
659 pattern
=> '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
660 format_description
=> 'UUID',
661 description
=> "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
662 ." to disable explicitly.",
663 verbose_description
=> "The VM generation ID (vmgenid) device exposes a 128-bit integer"
664 ." value identifier to the guest OS. This allows to notify the guest operating system"
665 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
666 ." execution or creation from a template). The guest operating system notices the"
667 ." change, and is then able to react as appropriate by marking its copies of"
668 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
669 ."Note that auto-creation only works when done through API/CLI create or update methods"
670 .", but not when manually editing the config file.",
671 default => "1 (autogenerated)",
676 format
=> 'pve-volume-id',
678 description
=> "Script that will be executed during various steps in the vms lifetime.",
682 format
=> $ivshmem_fmt,
683 description
=> "Inter-VM shared memory. Useful for direct communication between VMs, or to"
689 format
=> $audio_fmt,
690 description
=> "Configure a audio device, useful in combination with QXL/Spice.",
693 spice_enhancements
=> {
695 format
=> $spice_enhancements_fmt,
696 description
=> "Configure additional enhancements for SPICE.",
700 type
=> 'string', format
=> 'pve-tag-list',
701 description
=> 'Tags of the VM. This is only meta information.',
707 description
=> "Configure a VirtIO-based Random Number Generator.",
712 format
=> $meta_info_fmt,
713 description
=> "Some (read-only) meta-information about this guest.",
722 description
=> 'Specify a custom file containing all meta data passed to the VM via"
723 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
724 format
=> 'pve-volume-id',
725 format_description
=> 'volume',
730 description
=> 'Specify a custom file containing all network data passed to the VM via'
732 format
=> 'pve-volume-id',
733 format_description
=> 'volume',
738 description
=> 'Specify a custom file containing all user data passed to the VM via'
740 format
=> 'pve-volume-id',
741 format_description
=> 'volume',
746 description
=> 'Specify a custom file containing all vendor data passed to the VM via'
748 format
=> 'pve-volume-id',
749 format_description
=> 'volume',
752 PVE
::JSONSchema
::register_format
('pve-qm-cicustom', $cicustom_fmt);
754 my $confdesc_cloudinit = {
758 description
=> 'Specifies the cloud-init configuration format. The default depends on the'
759 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
760 .' and `configdrive2` for windows.',
761 enum
=> ['configdrive2', 'nocloud', 'opennebula'],
766 description
=> "cloud-init: User name to change ssh keys and password for instead of the"
767 ." image's configured default user.",
772 description
=> 'cloud-init: Password to assign the user. Using this is generally not'
773 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
774 .' support hashed passwords.',
779 description
=> 'cloud-init: Specify custom files to replace the automatically generated'
781 format
=> 'pve-qm-cicustom',
786 description
=> "cloud-init: Sets DNS search domains for a container. Create will'
787 .' automatically use the setting from the host if neither searchdomain nor nameserver'
792 type
=> 'string', format
=> 'address-list',
793 description
=> "cloud-init: Sets DNS server IP address for a container. Create will'
794 .' automatically use the setting from the host if neither searchdomain nor nameserver'
800 format
=> 'urlencoded',
801 description
=> "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
805 # what about other qemu settings ?
807 #machine => 'string',
820 ##soundhw => 'string',
822 while (my ($k, $v) = each %$confdesc) {
823 PVE
::JSONSchema
::register_standard_option
("pve-qm-$k", $v);
826 my $MAX_USB_DEVICES = 5;
828 my $MAX_SERIAL_PORTS = 4;
829 my $MAX_PARALLEL_PORTS = 3;
835 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
836 description
=> "CPUs accessing this NUMA node.",
837 format_description
=> "id[-id];...",
841 description
=> "Amount of memory this NUMA node provides.",
846 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
847 description
=> "Host NUMA nodes to use.",
848 format_description
=> "id[-id];...",
853 enum
=> [qw(preferred bind interleave)],
854 description
=> "NUMA allocation policy.",
858 PVE
::JSONSchema
::register_format
('pve-qm-numanode', $numa_fmt);
861 type
=> 'string', format
=> $numa_fmt,
862 description
=> "NUMA topology.",
864 PVE
::JSONSchema
::register_standard_option
("pve-qm-numanode", $numadesc);
866 for (my $i = 0; $i < $MAX_NUMA; $i++) {
867 $confdesc->{"numa$i"} = $numadesc;
870 my $nic_model_list = [
886 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
888 my $net_fmt_bridge_descr = <<__EOD__;
889 Bridge to attach the network device to. The Proxmox VE standard bridge
892 If you do not specify a bridge, we create a kvm user (NATed) network
893 device, which provides DHCP and DNS services. The following addresses
900 The DHCP server assign addresses to the guest starting from 10.0.2.15.
904 macaddr
=> get_standard_option
('mac-addr', {
905 description
=> "MAC address. That address must be unique withing your network. This is"
906 ." automatically generated if not specified.",
910 description
=> "Network Card Model. The 'virtio' model provides the best performance with"
911 ." very low CPU overhead. If your guest does not support this driver, it is usually"
912 ." best to use 'e1000'.",
913 enum
=> $nic_model_list,
916 (map { $_ => { keyAlias
=> 'model', alias
=> 'macaddr' }} @$nic_model_list),
917 bridge
=> get_standard_option
('pve-bridge-id', {
918 description
=> $net_fmt_bridge_descr,
923 minimum
=> 0, maximum
=> 16,
924 description
=> 'Number of packet queues to be used on the device.',
930 description
=> "Rate limit in mbps (megabytes per second) as floating point number.",
935 minimum
=> 1, maximum
=> 4094,
936 description
=> 'VLAN tag to apply to packets on this interface.',
941 pattern
=> qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
942 description
=> 'VLAN trunks to pass through this interface.',
943 format_description
=> 'vlanid[;vlanid...]',
948 description
=> 'Whether this interface should be protected by the firewall.',
953 description
=> 'Whether this interface should be disconnected (like pulling the plug).',
958 minimum
=> 1, maximum
=> 65520,
959 description
=> "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
966 type
=> 'string', format
=> $net_fmt,
967 description
=> "Specify network devices.",
970 PVE
::JSONSchema
::register_standard_option
("pve-qm-net", $netdesc);
975 format
=> 'pve-ipv4-config',
976 format_description
=> 'IPv4Format/CIDR',
977 description
=> 'IPv4 address in CIDR format.',
984 format_description
=> 'GatewayIPv4',
985 description
=> 'Default gateway for IPv4 traffic.',
991 format
=> 'pve-ipv6-config',
992 format_description
=> 'IPv6Format/CIDR',
993 description
=> 'IPv6 address in CIDR format.',
1000 format_description
=> 'GatewayIPv6',
1001 description
=> 'Default gateway for IPv6 traffic.',
1006 PVE
::JSONSchema
::register_format
('pve-qm-ipconfig', $ipconfig_fmt);
1007 my $ipconfigdesc = {
1009 type
=> 'string', format
=> 'pve-qm-ipconfig',
1010 description
=> <<'EODESCR',
1011 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1013 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1015 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1016 gateway should be provided.
1017 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1018 cloud-init 19.4 or newer.
1020 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1024 PVE
::JSONSchema
::register_standard_option
("pve-qm-ipconfig", $netdesc);
1026 for (my $i = 0; $i < $MAX_NETS; $i++) {
1027 $confdesc->{"net$i"} = $netdesc;
1028 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1031 foreach my $key (keys %$confdesc_cloudinit) {
1032 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1035 PVE
::JSONSchema
::register_format
('pve-volume-id-or-qm-path', \
&verify_volume_id_or_qm_path
);
1036 sub verify_volume_id_or_qm_path
{
1037 my ($volid, $noerr) = @_;
1039 return $volid if $volid eq 'none' || $volid eq 'cdrom';
1041 return verify_volume_id_or_absolute_path
($volid, $noerr);
1044 PVE
::JSONSchema
::register_format
('pve-volume-id-or-absolute-path', \
&verify_volume_id_or_absolute_path
);
1045 sub verify_volume_id_or_absolute_path
{
1046 my ($volid, $noerr) = @_;
1048 return $volid if $volid =~ m
|^/|;
1050 $volid = eval { PVE
::JSONSchema
::check_format
('pve-volume-id', $volid, '') };
1061 type
=> 'string', format
=> 'pve-qm-usb-device',
1062 format_description
=> 'HOSTUSBDEVICE|spice',
1063 description
=> <<EODESCR,
1064 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1066 'bus-port(.port)*' (decimal numbers) or
1067 'vendor_id:product_id' (hexadeciaml numbers) or
1070 You can use the 'lsusb -t' command to list existing usb devices.
1072 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1073 machines - use with special care.
1075 The value 'spice' can be used to add a usb redirection devices for spice.
1081 description
=> "Specifies whether if given host option is a USB3 device or port.",
1088 type
=> 'string', format
=> $usb_fmt,
1089 description
=> "Configure an USB device (n is 0 to 4).",
1091 PVE
::JSONSchema
::register_standard_option
("pve-qm-usb", $usbdesc);
1096 pattern
=> '(/dev/.+|socket)',
1097 description
=> "Create a serial device inside the VM (n is 0 to 3)",
1098 verbose_description
=> <<EODESCR,
1099 Create a serial device inside the VM (n is 0 to 3), and pass through a
1100 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1101 host side (use 'qm terminal' to open a terminal connection).
1103 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1104 use with special care.
1106 CAUTION: Experimental! User reported problems with this option.
1113 pattern
=> '/dev/parport\d+|/dev/usb/lp\d+',
1114 description
=> "Map host parallel devices (n is 0 to 2).",
1115 verbose_description
=> <<EODESCR,
1116 Map host parallel devices (n is 0 to 2).
1118 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1119 machines - use with special care.
1121 CAUTION: Experimental! User reported problems with this option.
1125 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1126 $confdesc->{"parallel$i"} = $paralleldesc;
1129 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1130 $confdesc->{"serial$i"} = $serialdesc;
1133 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
1134 $confdesc->{"hostpci$i"} = $PVE::QemuServer
::PCI
::hostpcidesc
;
1137 for my $key (keys %{$PVE::QemuServer
::Drive
::drivedesc_hash
}) {
1138 $confdesc->{$key} = $PVE::QemuServer
::Drive
::drivedesc_hash-
>{$key};
1141 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1142 $confdesc->{"usb$i"} = $usbdesc;
1150 description
=> "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1151 . " Deprecated, use 'order=' instead.",
1152 pattern
=> '[acdn]{1,4}',
1153 format_description
=> "[acdn]{1,4}",
1155 # note: this is also the fallback if boot: is not given at all
1161 format
=> 'pve-qm-bootdev-list',
1162 format_description
=> "device[;device...]",
1163 description
=> <<EODESC,
1164 The guest will attempt to boot from devices in the order they appear here.
1166 Disks, optical drives and passed-through storage USB devices will be directly
1167 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1168 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1170 Note that only devices in this list will be marked as bootable and thus loaded
1171 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1172 (e.g. software-raid), you need to specify all of them here.
1174 Overrides the deprecated 'legacy=[acdn]*' value when given.
1178 PVE
::JSONSchema
::register_format
('pve-qm-boot', $boot_fmt);
1180 PVE
::JSONSchema
::register_format
('pve-qm-bootdev', \
&verify_bootdev
);
1181 sub verify_bootdev
{
1182 my ($dev, $noerr) = @_;
1184 my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1185 return $dev if PVE
::QemuServer
::Drive
::is_valid_drivename
($dev) && !$special;
1189 return 0 if $dev !~ m/^$base\d+$/;
1190 return 0 if !$confdesc->{$dev};
1194 return $dev if $check->("net");
1195 return $dev if $check->("usb");
1196 return $dev if $check->("hostpci");
1199 die "invalid boot device '$dev'\n";
1202 sub print_bootorder
{
1204 return "" if !@$devs;
1205 my $data = { order
=> join(';', @$devs) };
1206 return PVE
::JSONSchema
::print_property_string
($data, $boot_fmt);
1209 my $kvm_api_version = 0;
1212 return $kvm_api_version if $kvm_api_version;
1214 open my $fh, '<', '/dev/kvm' or return;
1216 # 0xae00 => KVM_GET_API_VERSION
1217 $kvm_api_version = ioctl($fh, 0xae00, 0);
1220 return $kvm_api_version;
1223 my $kvm_user_version = {};
1226 sub kvm_user_version
{
1229 $binary //= get_command_for_arch
(get_host_arch
()); # get the native arch by default
1230 my $st = stat($binary);
1232 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1233 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1234 $cachedmtime == $st->mtime;
1236 $kvm_user_version->{$binary} = 'unknown';
1237 $kvm_mtime->{$binary} = $st->mtime;
1241 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1242 $kvm_user_version->{$binary} = $2;
1246 eval { run_command
([$binary, '--version'], outfunc
=> $code); };
1249 return $kvm_user_version->{$binary};
1252 my sub extract_version
{
1253 my ($machine_type, $version) = @_;
1254 $version = kvm_user_version
() if !defined($version);
1255 return PVE
::QemuServer
::Machine
::extract_version
($machine_type, $version)
1258 sub kernel_has_vhost_net
{
1259 return -c
'/dev/vhost-net';
1264 return defined($confdesc->{$key});
1268 sub get_cdrom_path
{
1270 return $cdrom_path if $cdrom_path;
1272 return $cdrom_path = "/dev/cdrom" if -l
"/dev/cdrom";
1273 return $cdrom_path = "/dev/cdrom1" if -l
"/dev/cdrom1";
1274 return $cdrom_path = "/dev/cdrom2" if -l
"/dev/cdrom2";
1278 my ($storecfg, $vmid, $cdrom) = @_;
1280 if ($cdrom eq 'cdrom') {
1281 return get_cdrom_path
();
1282 } elsif ($cdrom eq 'none') {
1284 } elsif ($cdrom =~ m
|^/|) {
1287 return PVE
::Storage
::path
($storecfg, $cdrom);
1291 # try to convert old style file names to volume IDs
1292 sub filename_to_volume_id
{
1293 my ($vmid, $file, $media) = @_;
1295 if (!($file eq 'none' || $file eq 'cdrom' ||
1296 $file =~ m
|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1298 return if $file =~ m
|/|;
1300 if ($media && $media eq 'cdrom') {
1301 $file = "local:iso/$file";
1303 $file = "local:$vmid/$file";
1310 sub verify_media_type
{
1311 my ($opt, $vtype, $media) = @_;
1316 if ($media eq 'disk') {
1318 } elsif ($media eq 'cdrom') {
1321 die "internal error";
1324 return if ($vtype eq $etype);
1326 raise_param_exc
({ $opt => "unexpected media type ($vtype != $etype)" });
1329 sub cleanup_drive_path
{
1330 my ($opt, $storecfg, $drive) = @_;
1332 # try to convert filesystem paths to volume IDs
1334 if (($drive->{file
} !~ m/^(cdrom|none)$/) &&
1335 ($drive->{file
} !~ m
|^/dev/.+|) &&
1336 ($drive->{file
} !~ m/^([^:]+):(.+)$/) &&
1337 ($drive->{file
} !~ m/^\d+$/)) {
1338 my ($vtype, $volid) = PVE
::Storage
::path_to_volume_id
($storecfg, $drive->{file
});
1339 raise_param_exc
({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1341 $drive->{media
} = 'cdrom' if !$drive->{media
} && $vtype eq 'iso';
1342 verify_media_type
($opt, $vtype, $drive->{media
});
1343 $drive->{file
} = $volid;
1346 $drive->{media
} = 'cdrom' if !$drive->{media
} && $drive->{file
} =~ m/^(cdrom|none)$/;
1349 sub parse_hotplug_features
{
1354 return $res if $data eq '0';
1356 $data = $confdesc->{hotplug
}->{default} if $data eq '1';
1358 foreach my $feature (PVE
::Tools
::split_list
($data)) {
1359 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1362 die "invalid hotplug feature '$feature'\n";
1368 PVE
::JSONSchema
::register_format
('pve-hotplug-features', \
&pve_verify_hotplug_features
);
1369 sub pve_verify_hotplug_features
{
1370 my ($value, $noerr) = @_;
1372 return $value if parse_hotplug_features
($value);
1376 die "unable to parse hotplug option\n";
1380 my($fh, $noerr) = @_;
1383 my $SG_GET_VERSION_NUM = 0x2282;
1385 my $versionbuf = "\x00" x
8;
1386 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1388 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1391 my $version = unpack("I", $versionbuf);
1392 if ($version < 30000) {
1393 die "scsi generic interface too old\n" if !$noerr;
1397 my $buf = "\x00" x
36;
1398 my $sensebuf = "\x00" x
8;
1399 my $cmd = pack("C x3 C x1", 0x12, 36);
1401 # see /usr/include/scsi/sg.h
1402 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1405 $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1408 $ret = ioctl($fh, $SG_IO, $packet);
1410 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1414 my @res = unpack($sg_io_hdr_t, $packet);
1415 if ($res[17] || $res[18]) {
1416 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1421 $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1423 $res->{removable
} = $res->{removable
} & 128 ?
1 : 0;
1424 $res->{type
} &= 0x1F;
1432 my $fh = IO
::File-
>new("+<$path") || return;
1433 my $res = scsi_inquiry
($fh, 1);
1439 sub print_tabletdevice_full
{
1440 my ($conf, $arch) = @_;
1442 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1444 # we use uhci for old VMs because tablet driver was buggy in older qemu
1446 if (PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf) || $arch eq 'aarch64') {
1452 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1455 sub print_keyboarddevice_full
{
1456 my ($conf, $arch) = @_;
1458 return if $arch ne 'aarch64';
1460 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1463 my sub get_drive_id
{
1465 return "$drive->{interface}$drive->{index}";
1468 sub print_drivedevice_full
{
1469 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1474 my $drive_id = get_drive_id
($drive);
1475 if ($drive->{interface
} eq 'virtio') {
1476 my $pciaddr = print_pci_addr
("$drive_id", $bridges, $arch, $machine_type);
1477 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1478 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread
};
1479 } elsif ($drive->{interface
} eq 'scsi') {
1481 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
1482 my $unit = $drive->{index} % $maxdev;
1483 my $devicetype = 'hd';
1485 if (drive_is_cdrom
($drive)) {
1488 if ($drive->{file
} =~ m
|^/|) {
1489 $path = $drive->{file
};
1490 if (my $info = path_is_scsi
($path)) {
1491 if ($info->{type
} == 0 && $drive->{scsiblock
}) {
1492 $devicetype = 'block';
1493 } elsif ($info->{type
} == 1) { # tape
1494 $devicetype = 'generic';
1498 $path = PVE
::Storage
::path
($storecfg, $drive->{file
});
1501 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1502 my $version = extract_version
($machine_type, kvm_user_version
());
1503 if ($path =~ m/^iscsi\:\/\
// &&
1504 !min_version
($version, 4, 1)) {
1505 $devicetype = 'generic';
1509 if (!$conf->{scsihw
} || $conf->{scsihw
} =~ m/^lsi/ || $conf->{scsihw
} eq 'pvscsi') {
1510 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1512 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1513 .",lun=$drive->{index}";
1515 $device .= ",drive=drive-$drive_id,id=$drive_id";
1517 if ($drive->{ssd
} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1518 $device .= ",rotation_rate=1";
1520 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1522 } elsif ($drive->{interface
} eq 'ide' || $drive->{interface
} eq 'sata') {
1523 my $maxdev = ($drive->{interface
} eq 'sata') ?
$PVE::QemuServer
::Drive
::MAX_SATA_DISKS
: 2;
1524 my $controller = int($drive->{index} / $maxdev);
1525 my $unit = $drive->{index} % $maxdev;
1526 my $devicetype = ($drive->{media
} && $drive->{media
} eq 'cdrom') ?
"cd" : "hd";
1528 $device = "ide-$devicetype";
1529 if ($drive->{interface
} eq 'ide') {
1530 $device .= ",bus=ide.$controller,unit=$unit";
1532 $device .= ",bus=ahci$controller.$unit";
1534 $device .= ",drive=drive-$drive_id,id=$drive_id";
1536 if ($devicetype eq 'hd') {
1537 if (my $model = $drive->{model
}) {
1538 $model = URI
::Escape
::uri_unescape
($model);
1539 $device .= ",model=$model";
1541 if ($drive->{ssd
}) {
1542 $device .= ",rotation_rate=1";
1545 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn
};
1546 } elsif ($drive->{interface
} eq 'usb') {
1548 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1550 die "unsupported interface type";
1553 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex
};
1555 if (my $serial = $drive->{serial
}) {
1556 $serial = URI
::Escape
::uri_unescape
($serial);
1557 $device .= ",serial=$serial";
1564 sub get_initiator_name
{
1567 my $fh = IO
::File-
>new('/etc/iscsi/initiatorname.iscsi') || return;
1568 while (defined(my $line = <$fh>)) {
1569 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1578 sub print_drive_commandline_full
{
1579 my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1582 my $volid = $drive->{file
};
1583 my $format = $drive->{format
};
1584 my $drive_id = get_drive_id
($drive);
1586 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
1587 my $scfg = $storeid ? PVE
::Storage
::storage_config
($storecfg, $storeid) : undef;
1589 if (drive_is_cdrom
($drive)) {
1590 $path = get_iso_path
($storecfg, $vmid, $volid);
1591 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1594 $path = PVE
::Storage
::path
($storecfg, $volid);
1595 $format //= qemu_img_format
($scfg, $volname);
1602 my $is_rbd = $path =~ m/^rbd:/;
1605 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1606 foreach my $o (@qemu_drive_options) {
1607 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1610 # snapshot only accepts on|off
1611 if (defined($drive->{snapshot
})) {
1612 my $v = $drive->{snapshot
} ?
'on' : 'off';
1613 $opts .= ",snapshot=$v";
1616 if (defined($drive->{ro
})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1617 $opts .= ",readonly=" . ($drive->{ro
} ?
'on' : 'off');
1620 foreach my $type (['', '-total'], [_rd
=> '-read'], [_wr
=> '-write']) {
1621 my ($dir, $qmpname) = @$type;
1622 if (my $v = $drive->{"mbps$dir"}) {
1623 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1625 if (my $v = $drive->{"mbps${dir}_max"}) {
1626 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1628 if (my $v = $drive->{"bps${dir}_max_length"}) {
1629 $opts .= ",throttling.bps$qmpname-max-length=$v";
1631 if (my $v = $drive->{"iops${dir}"}) {
1632 $opts .= ",throttling.iops$qmpname=$v";
1634 if (my $v = $drive->{"iops${dir}_max"}) {
1635 $opts .= ",throttling.iops$qmpname-max=$v";
1637 if (my $v = $drive->{"iops${dir}_max_length"}) {
1638 $opts .= ",throttling.iops$qmpname-max-length=$v";
1643 $format = "rbd" if $is_rbd;
1644 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1646 $opts .= ",format=alloc-track,file.driver=$format";
1648 $opts .= ",format=$format";
1651 my $cache_direct = 0;
1653 if (my $cache = $drive->{cache
}) {
1654 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1655 } elsif (!drive_is_cdrom
($drive) && !($scfg && $scfg->{type
} eq 'btrfs' && !$scfg->{nocow
})) {
1656 $opts .= ",cache=none";
1660 # io_uring with cache mode writeback or writethrough on krbd will hang...
1661 my $rbd_no_io_uring = $scfg && $scfg->{type
} eq 'rbd' && $scfg->{krbd
} && !$cache_direct;
1663 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1664 # sometimes, just plain disable...
1665 my $lvm_no_io_uring = $scfg && $scfg->{type
} eq 'lvm';
1667 if (!$drive->{aio
}) {
1668 if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1669 # io_uring supports all cache modes
1670 $opts .= ",aio=io_uring";
1672 # aio native works only with O_DIRECT
1674 $opts .= ",aio=native";
1676 $opts .= ",aio=threads";
1681 if (!drive_is_cdrom
($drive)) {
1683 if (defined($drive->{detect_zeroes
}) && !$drive->{detect_zeroes
}) {
1684 $detectzeroes = 'off';
1685 } elsif ($drive->{discard
}) {
1686 $detectzeroes = $drive->{discard
} eq 'on' ?
'unmap' : 'on';
1688 # This used to be our default with discard not being specified:
1689 $detectzeroes = 'on';
1692 # note: 'detect-zeroes' works per blockdev and we want it to persist
1693 # after the alloc-track is removed, so put it on 'file' directly
1694 my $dz_param = $pbs_name ?
"file.detect-zeroes" : "detect-zeroes";
1695 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1699 $opts .= ",backing=$pbs_name";
1700 $opts .= ",auto-remove=on";
1703 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1704 my $file_param = "file";
1706 # non-rbd drivers require the underlying file to be a seperate block
1707 # node, so add a second .file indirection
1708 $file_param .= ".file" if !$is_rbd;
1709 $file_param .= ".filename";
1711 my $pathinfo = $path ?
"$file_param=$path," : '';
1713 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1716 sub print_pbs_blockdev
{
1717 my ($pbs_conf, $pbs_name) = @_;
1718 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1719 $blockdev .= ",repository=$pbs_conf->{repository}";
1720 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1721 $blockdev .= ",archive=$pbs_conf->{archive}";
1722 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile
};
1726 sub print_netdevice_full
{
1727 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1729 my $device = $net->{model
};
1730 if ($net->{model
} eq 'virtio') {
1731 $device = 'virtio-net-pci';
1734 my $pciaddr = print_pci_addr
("$netid", $bridges, $arch, $machine_type);
1735 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1736 if ($net->{queues
} && $net->{queues
} > 1 && $net->{model
} eq 'virtio'){
1737 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1738 # and out of each queue plus one config interrupt and control vector queue
1739 my $vectors = $net->{queues
} * 2 + 2;
1740 $tmpstr .= ",vectors=$vectors,mq=on";
1742 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex
} ;
1744 if (my $mtu = $net->{mtu
}) {
1745 if ($net->{model
} eq 'virtio' && $net->{bridge
}) {
1746 my $bridge_mtu = PVE
::Network
::read_bridge_mtu
($net->{bridge
});
1749 } elsif ($mtu < 576) {
1750 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1751 } elsif ($mtu > $bridge_mtu) {
1752 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1754 $tmpstr .= ",host_mtu=$mtu";
1756 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1760 if ($use_old_bios_files) {
1762 if ($device eq 'virtio-net-pci') {
1763 $romfile = 'pxe-virtio.rom';
1764 } elsif ($device eq 'e1000') {
1765 $romfile = 'pxe-e1000.rom';
1766 } elsif ($device eq 'e1000e') {
1767 $romfile = 'pxe-e1000e.rom';
1768 } elsif ($device eq 'ne2k') {
1769 $romfile = 'pxe-ne2k_pci.rom';
1770 } elsif ($device eq 'pcnet') {
1771 $romfile = 'pxe-pcnet.rom';
1772 } elsif ($device eq 'rtl8139') {
1773 $romfile = 'pxe-rtl8139.rom';
1775 $tmpstr .= ",romfile=$romfile" if $romfile;
1781 sub print_netdev_full
{
1782 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1785 if ($netid =~ m/^net(\d+)$/) {
1789 die "got strange net id '$i'\n" if $i >= ${MAX_NETS
};
1791 my $ifname = "tap${vmid}i$i";
1793 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1794 die "interface name '$ifname' is too long (max 15 character)\n"
1795 if length($ifname) >= 16;
1797 my $vhostparam = '';
1798 if (is_native
($arch)) {
1799 $vhostparam = ',vhost=on' if kernel_has_vhost_net
() && $net->{model
} eq 'virtio';
1802 my $vmname = $conf->{name
} || "vm$vmid";
1805 my $script = $hotplug ?
"pve-bridge-hotplug" : "pve-bridge";
1807 if ($net->{bridge
}) {
1808 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1809 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1811 $netdev = "type=user,id=$netid,hostname=$vmname";
1814 $netdev .= ",queues=$net->{queues}" if ($net->{queues
} && $net->{model
} eq 'virtio');
1820 'cirrus' => 'cirrus-vga',
1822 'vmware' => 'vmware-svga',
1823 'virtio' => 'virtio-vga',
1826 sub print_vga_device
{
1827 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1829 my $type = $vga_map->{$vga->{type
}};
1830 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1831 $type = 'virtio-gpu';
1833 my $vgamem_mb = $vga->{memory
};
1835 my $max_outputs = '';
1837 $type = $id ?
'qxl' : 'qxl-vga';
1839 if (!$conf->{ostype
} || $conf->{ostype
} =~ m/^(?:l\d\d)|(?:other)$/) {
1840 # set max outputs so linux can have up to 4 qxl displays with one device
1841 if (min_version
($machine_version, 4, 1)) {
1842 $max_outputs = ",max_outputs=4";
1847 die "no devicetype for $vga->{type}\n" if !$type;
1851 if ($vga->{type
} eq 'virtio') {
1852 my $bytes = PVE
::Tools
::convert_size
($vgamem_mb, "mb" => "b");
1853 $memory = ",max_hostmem=$bytes";
1855 # from https://www.spice-space.org/multiple-monitors.html
1856 $memory = ",vgamem_mb=$vga->{memory}";
1857 my $ram = $vgamem_mb * 4;
1858 my $vram = $vgamem_mb * 2;
1859 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1861 $memory = ",vgamem_mb=$vga->{memory}";
1863 } elsif ($qxlnum && $id) {
1864 $memory = ",ram_size=67108864,vram_size=33554432";
1868 if ($type eq 'VGA' && windows_version
($conf->{ostype
})) {
1869 $edidoff=",edid=off" if (!defined($conf->{bios
}) || $conf->{bios
} ne 'ovmf');
1872 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
1873 my $vgaid = "vga" . ($id // '');
1875 if ($q35 && $vgaid eq 'vga') {
1876 # the first display uses pcie.0 bus on q35 machines
1877 $pciaddr = print_pcie_addr
($vgaid, $bridges, $arch, $machine);
1879 $pciaddr = print_pci_addr
($vgaid, $bridges, $arch, $machine);
1882 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1885 sub parse_number_sets
{
1888 foreach my $part (split(/;/, $set)) {
1889 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1890 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1891 push @$res, [ $1, $2 ];
1893 die "invalid range: $part\n";
1902 my $res = parse_property_string
($numa_fmt, $data);
1903 $res->{cpus
} = parse_number_sets
($res->{cpus
}) if defined($res->{cpus
});
1904 $res->{hostnodes
} = parse_number_sets
($res->{hostnodes
}) if defined($res->{hostnodes
});
1908 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1912 my $res = eval { parse_property_string
($net_fmt, $data) };
1917 if (!defined($res->{macaddr
})) {
1918 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
1919 $res->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
});
1924 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1925 sub parse_ipconfig
{
1928 my $res = eval { parse_property_string
($ipconfig_fmt, $data) };
1934 if ($res->{gw
} && !$res->{ip
}) {
1935 warn 'gateway specified without specifying an IP address';
1938 if ($res->{gw6
} && !$res->{ip6
}) {
1939 warn 'IPv6 gateway specified without specifying an IPv6 address';
1942 if ($res->{gw
} && $res->{ip
} eq 'dhcp') {
1943 warn 'gateway specified together with DHCP';
1946 if ($res->{gw6
} && $res->{ip6
} !~ /^$IPV6RE/) {
1948 warn "IPv6 gateway specified together with $res->{ip6} address";
1952 if (!$res->{ip
} && !$res->{ip6
}) {
1953 return { ip
=> 'dhcp', ip6
=> 'dhcp' };
1962 return PVE
::JSONSchema
::print_property_string
($net, $net_fmt);
1965 sub add_random_macs
{
1966 my ($settings) = @_;
1968 foreach my $opt (keys %$settings) {
1969 next if $opt !~ m/^net(\d+)$/;
1970 my $net = parse_net
($settings->{$opt});
1972 $settings->{$opt} = print_net
($net);
1976 sub vm_is_volid_owner
{
1977 my ($storecfg, $vmid, $volid) = @_;
1979 if ($volid !~ m
|^/|) {
1981 eval { ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid); };
1982 if ($owner && ($owner == $vmid)) {
1990 sub vmconfig_register_unused_drive
{
1991 my ($storecfg, $vmid, $conf, $drive) = @_;
1993 if (drive_is_cloudinit
($drive)) {
1994 eval { PVE
::Storage
::vdisk_free
($storecfg, $drive->{file
}) };
1996 } elsif (!drive_is_cdrom
($drive)) {
1997 my $volid = $drive->{file
};
1998 if (vm_is_volid_owner
($storecfg, $vmid, $volid)) {
1999 PVE
::QemuConfig-
>add_unused_volume($conf, $volid, $vmid);
2004 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2008 pattern
=> '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2009 format_description
=> 'UUID',
2010 description
=> "Set SMBIOS1 UUID.",
2015 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2016 format_description
=> 'Base64 encoded string',
2017 description
=> "Set SMBIOS1 version.",
2022 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2023 format_description
=> 'Base64 encoded string',
2024 description
=> "Set SMBIOS1 serial number.",
2029 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2030 format_description
=> 'Base64 encoded string',
2031 description
=> "Set SMBIOS1 manufacturer.",
2036 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2037 format_description
=> 'Base64 encoded string',
2038 description
=> "Set SMBIOS1 product ID.",
2043 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2044 format_description
=> 'Base64 encoded string',
2045 description
=> "Set SMBIOS1 SKU string.",
2050 pattern
=> '[A-Za-z0-9+\/]+={0,2}',
2051 format_description
=> 'Base64 encoded string',
2052 description
=> "Set SMBIOS1 family string.",
2057 description
=> 'Flag to indicate that the SMBIOS values are base64 encoded',
2065 my $res = eval { parse_property_string
($smbios1_fmt, $data) };
2072 return PVE
::JSONSchema
::print_property_string
($smbios1, $smbios1_fmt);
2075 PVE
::JSONSchema
::register_format
('pve-qm-smbios1', $smbios1_fmt);
2077 sub parse_watchdog
{
2082 my $res = eval { parse_property_string
($watchdog_fmt, $value) };
2087 sub parse_guest_agent
{
2090 return {} if !defined($conf->{agent
});
2092 my $res = eval { parse_property_string
($agent_fmt, $conf->{agent
}) };
2095 # if the agent is disabled ignore the other potentially set properties
2096 return {} if !$res->{enabled
};
2101 my ($conf, $key) = @_;
2102 return undef if !defined($conf->{agent
});
2104 my $agent = parse_guest_agent
($conf);
2105 return $agent->{$key};
2111 return {} if !$value;
2112 my $res = eval { parse_property_string
($vga_fmt, $value) };
2122 my $res = eval { parse_property_string
($rng_fmt, $value) };
2127 sub parse_meta_info
{
2132 my $res = eval { parse_property_string
($meta_info_fmt, $value) };
2137 sub new_meta_info_string
{
2138 my () = @_; # for now do not allow to override any value
2140 return PVE
::JSONSchema
::print_property_string
(
2142 'creation-qemu' => kvm_user_version
(),
2143 ctime
=> "". int(time()),
2149 sub qemu_created_version_fixups
{
2150 my ($conf, $forcemachine, $kvmver) = @_;
2152 my $meta = parse_meta_info
($conf->{meta
}) // {};
2153 my $forced_vers = PVE
::QemuServer
::Machine
::extract_version
($forcemachine);
2155 # check if we need to apply some handling for VMs that always use the latest machine version but
2156 # had a machine version transition happen that affected HW such that, e.g., an OS config change
2157 # would be required (we do not want to pin machine version for non-windows OS type)
2159 (!defined($conf->{machine
}) || $conf->{machine
} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2160 && (!defined($meta->{'creation-qemu'}) || !min_version
($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2161 && (!$forced_vers || min_version
($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2162 && min_version
($kvmver, 6, 1) # only need to apply the change since 6.1
2164 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
2165 if ($q35 && $conf->{ostype
} && $conf->{ostype
} eq 'l26') {
2166 # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2167 # and thus with the predictable interface naming of systemd
2168 return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2174 PVE
::JSONSchema
::register_format
('pve-qm-usb-device', \
&verify_usb_device
);
2175 sub verify_usb_device
{
2176 my ($value, $noerr) = @_;
2178 return $value if parse_usb_device
($value);
2182 die "unable to parse usb device\n";
2185 # add JSON properties for create and set function
2186 sub json_config_properties
{
2187 my ($prop, $with_disk_alloc) = @_;
2189 my $skip_json_config_opts = {
2193 runningmachine
=> 1,
2198 foreach my $opt (keys %$confdesc) {
2199 next if $skip_json_config_opts->{$opt};
2201 if ($with_disk_alloc && is_valid_drivename
($opt)) {
2202 $prop->{$opt} = $PVE::QemuServer
::Drive
::drivedesc_hash_with_alloc-
>{$opt};
2204 $prop->{$opt} = $confdesc->{$opt};
2211 # Properties that we can read from an OVF file
2212 sub json_ovf_properties
{
2215 for my $device (PVE
::QemuServer
::Drive
::valid_drive_names
()) {
2216 $prop->{$device} = {
2218 format
=> 'pve-volume-id-or-absolute-path',
2219 description
=> "Disk image that gets imported to $device",
2226 description
=> "The number of CPU cores.",
2231 description
=> "Amount of RAM for the VM in MB.",
2236 description
=> "Name of the VM.",
2243 # return copy of $confdesc_cloudinit to generate documentation
2244 sub cloudinit_config_properties
{
2246 return dclone
($confdesc_cloudinit);
2250 my ($key, $value) = @_;
2252 die "unknown setting '$key'\n" if !$confdesc->{$key};
2254 my $type = $confdesc->{$key}->{type
};
2256 if (!defined($value)) {
2257 die "got undefined value\n";
2260 if ($value =~ m/[\n\r]/) {
2261 die "property contains a line feed\n";
2264 if ($type eq 'boolean') {
2265 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2266 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2267 die "type check ('boolean') failed - got '$value'\n";
2268 } elsif ($type eq 'integer') {
2269 return int($1) if $value =~ m/^(\d+)$/;
2270 die "type check ('integer') failed - got '$value'\n";
2271 } elsif ($type eq 'number') {
2272 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2273 die "type check ('number') failed - got '$value'\n";
2274 } elsif ($type eq 'string') {
2275 if (my $fmt = $confdesc->{$key}->{format
}) {
2276 PVE
::JSONSchema
::check_format
($fmt, $value);
2279 $value =~ s/^\"(.*)\"$/$1/;
2282 die "internal error"
2287 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2289 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2291 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
2293 if ($conf->{template
}) {
2294 # check if any base image is still used by a linked clone
2295 PVE
::QemuConfig-
>foreach_volume_full($conf, { include_unused
=> 1 }, sub {
2296 my ($ds, $drive) = @_;
2297 return if drive_is_cdrom
($drive);
2299 my $volid = $drive->{file
};
2300 return if !$volid || $volid =~ m
|^/|;
2302 die "base volume '$volid' is still in use by linked cloned\n"
2303 if PVE
::Storage
::volume_is_base_and_used
($storecfg, $volid);
2309 my $remove_owned_drive = sub {
2310 my ($ds, $drive) = @_;
2311 return if drive_is_cdrom
($drive, 1);
2313 my $volid = $drive->{file
};
2314 return if !$volid || $volid =~ m
|^/|;
2315 return if $volids->{$volid};
2317 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
2318 return if !$path || !$owner || ($owner != $vmid);
2320 $volids->{$volid} = 1;
2321 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2322 warn "Could not remove disk '$volid', check manually: $@" if $@;
2325 # only remove disks owned by this VM (referenced in the config)
2326 my $include_opts = {
2327 include_unused
=> 1,
2328 extra_keys
=> ['vmstate'],
2330 PVE
::QemuConfig-
>foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2332 for my $snap (values %{$conf->{snapshots
}}) {
2333 next if !defined($snap->{vmstate
});
2334 my $drive = PVE
::QemuConfig-
>parse_volume('vmstate', $snap->{vmstate
}, 1);
2335 next if !defined($drive);
2336 $remove_owned_drive->('vmstate', $drive);
2339 PVE
::QemuConfig-
>foreach_volume_full($conf->{pending
}, $include_opts, $remove_owned_drive);
2341 if ($purge_unreferenced) { # also remove unreferenced disk
2342 my $vmdisks = PVE
::Storage
::vdisk_list
($storecfg, undef, $vmid, undef, 'images');
2343 PVE
::Storage
::foreach_volid
($vmdisks, sub {
2344 my ($volid, $sid, $volname, $d) = @_;
2345 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid) };
2350 if (defined $replacement_conf) {
2351 PVE
::QemuConfig-
>write_config($vmid, $replacement_conf);
2353 PVE
::QemuConfig-
>destroy_config($vmid);
2357 sub parse_vm_config
{
2358 my ($filename, $raw, $strict) = @_;
2360 return if !defined($raw);
2363 digest
=> Digest
::SHA
::sha1_hex
($raw),
2368 my $handle_error = sub {
2378 $filename =~ m
|/qemu-server/(\d
+)\
.conf
$|
2379 || die "got strange filename '$filename'";
2387 my @lines = split(/\n/, $raw);
2388 foreach my $line (@lines) {
2389 next if $line =~ m/^\s*$/;
2391 if ($line =~ m/^\[PENDING\]\s*$/i) {
2392 $section = 'pending';
2393 if (defined($descr)) {
2395 $conf->{description
} = $descr;
2398 $conf = $res->{$section} = {};
2401 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2403 if (defined($descr)) {
2405 $conf->{description
} = $descr;
2408 $conf = $res->{snapshots
}->{$section} = {};
2412 if ($line =~ m/^\#(.*)\s*$/) {
2413 $descr = '' if !defined($descr);
2414 $descr .= PVE
::Tools
::decode_text
($1) . "\n";
2418 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2419 $descr = '' if !defined($descr);
2420 $descr .= PVE
::Tools
::decode_text
($2);
2421 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2422 $conf->{snapstate
} = $1;
2423 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2426 $conf->{$key} = $value;
2427 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2429 if ($section eq 'pending') {
2430 $conf->{delete} = $value; # we parse this later
2432 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2434 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2437 eval { $value = check_type
($key, $value); };
2439 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2441 $key = 'ide2' if $key eq 'cdrom';
2442 my $fmt = $confdesc->{$key}->{format
};
2443 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2444 my $v = parse_drive
($key, $value);
2445 if (my $volid = filename_to_volume_id
($vmid, $v->{file
}, $v->{media
})) {
2446 $v->{file
} = $volid;
2447 $value = print_drive
($v);
2449 $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2454 $conf->{$key} = $value;
2457 $handle_error->("vm $vmid - unable to parse config: $line\n");
2461 if (defined($descr)) {
2463 $conf->{description
} = $descr;
2465 delete $res->{snapstate
}; # just to be sure
2470 sub write_vm_config
{
2471 my ($filename, $conf) = @_;
2473 delete $conf->{snapstate
}; # just to be sure
2475 if ($conf->{cdrom
}) {
2476 die "option ide2 conflicts with cdrom\n" if $conf->{ide2
};
2477 $conf->{ide2
} = $conf->{cdrom
};
2478 delete $conf->{cdrom
};
2481 # we do not use 'smp' any longer
2482 if ($conf->{sockets
}) {
2483 delete $conf->{smp
};
2484 } elsif ($conf->{smp
}) {
2485 $conf->{sockets
} = $conf->{smp
};
2486 delete $conf->{cores
};
2487 delete $conf->{smp
};
2490 my $used_volids = {};
2492 my $cleanup_config = sub {
2493 my ($cref, $pending, $snapname) = @_;
2495 foreach my $key (keys %$cref) {
2496 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2497 $key eq 'snapstate' || $key eq 'pending';
2498 my $value = $cref->{$key};
2499 if ($key eq 'delete') {
2500 die "propertry 'delete' is only allowed in [PENDING]\n"
2502 # fixme: check syntax?
2505 eval { $value = check_type
($key, $value); };
2506 die "unable to parse value of '$key' - $@" if $@;
2508 $cref->{$key} = $value;
2510 if (!$snapname && is_valid_drivename
($key)) {
2511 my $drive = parse_drive
($key, $value);
2512 $used_volids->{$drive->{file
}} = 1 if $drive && $drive->{file
};
2517 &$cleanup_config($conf);
2519 &$cleanup_config($conf->{pending
}, 1);
2521 foreach my $snapname (keys %{$conf->{snapshots
}}) {
2522 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2523 &$cleanup_config($conf->{snapshots
}->{$snapname}, undef, $snapname);
2526 # remove 'unusedX' settings if we re-add a volume
2527 foreach my $key (keys %$conf) {
2528 my $value = $conf->{$key};
2529 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2530 delete $conf->{$key};
2534 my $generate_raw_config = sub {
2535 my ($conf, $pending) = @_;
2539 # add description as comment to top of file
2540 if (defined(my $descr = $conf->{description
})) {
2542 foreach my $cl (split(/\n/, $descr)) {
2543 $raw .= '#' . PVE
::Tools
::encode_text
($cl) . "\n";
2546 $raw .= "#\n" if $pending;
2550 foreach my $key (sort keys %$conf) {
2551 next if $key =~ /^(digest|description|pending|snapshots)$/;
2552 $raw .= "$key: $conf->{$key}\n";
2557 my $raw = &$generate_raw_config($conf);
2559 if (scalar(keys %{$conf->{pending
}})){
2560 $raw .= "\n[PENDING]\n";
2561 $raw .= &$generate_raw_config($conf->{pending
}, 1);
2564 foreach my $snapname (sort keys %{$conf->{snapshots
}}) {
2565 $raw .= "\n[$snapname]\n";
2566 $raw .= &$generate_raw_config($conf->{snapshots
}->{$snapname});
2576 # we use static defaults from our JSON schema configuration
2577 foreach my $key (keys %$confdesc) {
2578 if (defined(my $default = $confdesc->{$key}->{default})) {
2579 $res->{$key} = $default;
2587 my $vmlist = PVE
::Cluster
::get_vmlist
();
2589 return $res if !$vmlist || !$vmlist->{ids
};
2590 my $ids = $vmlist->{ids
};
2591 my $nodename = nodename
();
2593 foreach my $vmid (keys %$ids) {
2594 my $d = $ids->{$vmid};
2595 next if !$d->{node
} || $d->{node
} ne $nodename;
2596 next if !$d->{type
} || $d->{type
} ne 'qemu';
2597 $res->{$vmid}->{exists} = 1;
2602 # test if VM uses local resources (to prevent migration)
2603 sub check_local_resources
{
2604 my ($conf, $noerr) = @_;
2608 push @loc_res, "hostusb" if $conf->{hostusb
}; # old syntax
2609 push @loc_res, "hostpci" if $conf->{hostpci
}; # old syntax
2611 push @loc_res, "ivshmem" if $conf->{ivshmem
};
2613 foreach my $k (keys %$conf) {
2614 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2615 # sockets are safe: they will recreated be on the target side post-migrate
2616 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2617 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2620 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2625 # check if used storages are available on all nodes (use by migrate)
2626 sub check_storage_availability
{
2627 my ($storecfg, $conf, $node) = @_;
2629 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2630 my ($ds, $drive) = @_;
2632 my $volid = $drive->{file
};
2635 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2638 # check if storage is available on both nodes
2639 my $scfg = PVE
::Storage
::storage_check_enabled
($storecfg, $sid);
2640 PVE
::Storage
::storage_check_enabled
($storecfg, $sid, $node);
2642 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $volid);
2644 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2645 if !$scfg->{content
}->{$vtype};
2649 # list nodes where all VM images are available (used by has_feature API)
2651 my ($conf, $storecfg) = @_;
2653 my $nodelist = PVE
::Cluster
::get_nodelist
();
2654 my $nodehash = { map { $_ => 1 } @$nodelist };
2655 my $nodename = nodename
();
2657 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2658 my ($ds, $drive) = @_;
2660 my $volid = $drive->{file
};
2663 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2665 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2666 if ($scfg->{disable
}) {
2668 } elsif (my $avail = $scfg->{nodes
}) {
2669 foreach my $node (keys %$nodehash) {
2670 delete $nodehash->{$node} if !$avail->{$node};
2672 } elsif (!$scfg->{shared
}) {
2673 foreach my $node (keys %$nodehash) {
2674 delete $nodehash->{$node} if $node ne $nodename
2683 sub check_local_storage_availability
{
2684 my ($conf, $storecfg) = @_;
2686 my $nodelist = PVE
::Cluster
::get_nodelist
();
2687 my $nodehash = { map { $_ => {} } @$nodelist };
2689 PVE
::QemuConfig-
>foreach_volume($conf, sub {
2690 my ($ds, $drive) = @_;
2692 my $volid = $drive->{file
};
2695 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
2697 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
2699 if ($scfg->{disable
}) {
2700 foreach my $node (keys %$nodehash) {
2701 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2703 } elsif (my $avail = $scfg->{nodes
}) {
2704 foreach my $node (keys %$nodehash) {
2705 if (!$avail->{$node}) {
2706 $nodehash->{$node}->{unavailable_storages
}->{$storeid} = 1;
2713 foreach my $node (values %$nodehash) {
2714 if (my $unavail = $node->{unavailable_storages
}) {
2715 $node->{unavailable_storages
} = [ sort keys %$unavail ];
2722 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2724 my ($vmid, $nocheck, $node) = @_;
2726 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid, $node) if !$nocheck;
2727 return PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
2732 my $vzlist = config_list
();
2734 my $fd = IO
::Dir-
>new($PVE::QemuServer
::Helpers
::var_run_tmpdir
) || return $vzlist;
2736 while (defined(my $de = $fd->read)) {
2737 next if $de !~ m/^(\d+)\.pid$/;
2739 next if !defined($vzlist->{$vmid});
2740 if (my $pid = check_running
($vmid)) {
2741 $vzlist->{$vmid}->{pid
} = $pid;
2748 our $vmstatus_return_properties = {
2749 vmid
=> get_standard_option
('pve-vmid'),
2751 description
=> "Qemu process status.",
2753 enum
=> ['stopped', 'running'],
2756 description
=> "Maximum memory in bytes.",
2759 renderer
=> 'bytes',
2762 description
=> "Root disk size in bytes.",
2765 renderer
=> 'bytes',
2768 description
=> "VM name.",
2773 description
=> "Qemu QMP agent status.",
2778 description
=> "PID of running qemu process.",
2783 description
=> "Uptime.",
2786 renderer
=> 'duration',
2789 description
=> "Maximum usable CPUs.",
2794 description
=> "The current config lock, if any.",
2799 description
=> "The current configured tags, if any",
2803 'running-machine' => {
2804 description
=> "The currently running machine type (if running).",
2809 description
=> "The currently running QEMU version (if running).",
2815 my $last_proc_pid_stat;
2817 # get VM status information
2818 # This must be fast and should not block ($full == false)
2819 # We only query KVM using QMP if $full == true (this can be slow)
2821 my ($opt_vmid, $full) = @_;
2825 my $storecfg = PVE
::Storage
::config
();
2827 my $list = vzlist
();
2828 my $defaults = load_defaults
();
2830 my ($uptime) = PVE
::ProcFSTools
::read_proc_uptime
(1);
2832 my $cpucount = $cpuinfo->{cpus
} || 1;
2834 foreach my $vmid (keys %$list) {
2835 next if $opt_vmid && ($vmid ne $opt_vmid);
2837 my $conf = PVE
::QemuConfig-
>load_config($vmid);
2839 my $d = { vmid
=> int($vmid) };
2840 $d->{pid
} = int($list->{$vmid}->{pid
}) if $list->{$vmid}->{pid
};
2842 # fixme: better status?
2843 $d->{status
} = $list->{$vmid}->{pid
} ?
'running' : 'stopped';
2845 my $size = PVE
::QemuServer
::Drive
::bootdisk_size
($storecfg, $conf);
2846 if (defined($size)) {
2847 $d->{disk
} = 0; # no info available
2848 $d->{maxdisk
} = $size;
2854 $d->{cpus
} = ($conf->{sockets
} || $defaults->{sockets
})
2855 * ($conf->{cores
} || $defaults->{cores
});
2856 $d->{cpus
} = $cpucount if $d->{cpus
} > $cpucount;
2857 $d->{cpus
} = $conf->{vcpus
} if $conf->{vcpus
};
2859 $d->{name
} = $conf->{name
} || "VM $vmid";
2860 $d->{maxmem
} = $conf->{memory
} ?
$conf->{memory
}*(1024*1024)
2861 : $defaults->{memory
}*(1024*1024);
2863 if ($conf->{balloon
}) {
2864 $d->{balloon_min
} = $conf->{balloon
}*(1024*1024);
2865 $d->{shares
} = defined($conf->{shares
}) ?
$conf->{shares
}
2866 : $defaults->{shares
};
2877 $d->{diskwrite
} = 0;
2879 $d->{template
} = 1 if PVE
::QemuConfig-
>is_template($conf);
2881 $d->{serial
} = 1 if conf_has_serial
($conf);
2882 $d->{lock} = $conf->{lock} if $conf->{lock};
2883 $d->{tags
} = $conf->{tags
} if defined($conf->{tags
});
2888 my $netdev = PVE
::ProcFSTools
::read_proc_net_dev
();
2889 foreach my $dev (keys %$netdev) {
2890 next if $dev !~ m/^tap([1-9]\d*)i/;
2892 my $d = $res->{$vmid};
2895 $d->{netout
} += $netdev->{$dev}->{receive
};
2896 $d->{netin
} += $netdev->{$dev}->{transmit
};
2899 $d->{nics
}->{$dev}->{netout
} = int($netdev->{$dev}->{receive
});
2900 $d->{nics
}->{$dev}->{netin
} = int($netdev->{$dev}->{transmit
});
2905 my $ctime = gettimeofday
;
2907 foreach my $vmid (keys %$list) {
2909 my $d = $res->{$vmid};
2910 my $pid = $d->{pid
};
2913 my $pstat = PVE
::ProcFSTools
::read_proc_pid_stat
($pid);
2914 next if !$pstat; # not running
2916 my $used = $pstat->{utime} + $pstat->{stime
};
2918 $d->{uptime
} = int(($uptime - $pstat->{starttime
})/$cpuinfo->{user_hz
});
2920 if ($pstat->{vsize
}) {
2921 $d->{mem
} = int(($pstat->{rss
}/$pstat->{vsize
})*$d->{maxmem
});
2924 my $old = $last_proc_pid_stat->{$pid};
2926 $last_proc_pid_stat->{$pid} = {
2934 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz
};
2936 if ($dtime > 1000) {
2937 my $dutime = $used - $old->{used
};
2939 $d->{cpu
} = (($dutime/$dtime)* $cpucount) / $d->{cpus
};
2940 $last_proc_pid_stat->{$pid} = {
2946 $d->{cpu
} = $old->{cpu
};
2950 return $res if !$full;
2952 my $qmpclient = PVE
::QMPClient-
>new();
2954 my $ballooncb = sub {
2955 my ($vmid, $resp) = @_;
2957 my $info = $resp->{'return'};
2958 return if !$info->{max_mem
};
2960 my $d = $res->{$vmid};
2962 # use memory assigned to VM
2963 $d->{maxmem
} = $info->{max_mem
};
2964 $d->{balloon
} = $info->{actual
};
2966 if (defined($info->{total_mem
}) && defined($info->{free_mem
})) {
2967 $d->{mem
} = $info->{total_mem
} - $info->{free_mem
};
2968 $d->{freemem
} = $info->{free_mem
};
2971 $d->{ballooninfo
} = $info;
2974 my $blockstatscb = sub {
2975 my ($vmid, $resp) = @_;
2976 my $data = $resp->{'return'} || [];
2977 my $totalrdbytes = 0;
2978 my $totalwrbytes = 0;
2980 for my $blockstat (@$data) {
2981 $totalrdbytes = $totalrdbytes + $blockstat->{stats
}->{rd_bytes
};
2982 $totalwrbytes = $totalwrbytes + $blockstat->{stats
}->{wr_bytes
};
2984 $blockstat->{device
} =~ s/drive-//;
2985 $res->{$vmid}->{blockstat
}->{$blockstat->{device
}} = $blockstat->{stats
};
2987 $res->{$vmid}->{diskread
} = $totalrdbytes;
2988 $res->{$vmid}->{diskwrite
} = $totalwrbytes;
2991 my $machinecb = sub {
2992 my ($vmid, $resp) = @_;
2993 my $data = $resp->{'return'} || [];
2995 $res->{$vmid}->{'running-machine'} =
2996 PVE
::QemuServer
::Machine
::current_from_query_machines
($data);
2999 my $versioncb = sub {
3000 my ($vmid, $resp) = @_;
3001 my $data = $resp->{'return'} // {};
3002 my $version = 'unknown';
3004 if (my $v = $data->{qemu
}) {
3005 $version = $v->{major
} . "." . $v->{minor
} . "." . $v->{micro
};
3008 $res->{$vmid}->{'running-qemu'} = $version;
3011 my $statuscb = sub {
3012 my ($vmid, $resp) = @_;
3014 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3015 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3016 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3017 # this fails if ballon driver is not loaded, so this must be
3018 # the last commnand (following command are aborted if this fails).
3019 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3021 my $status = 'unknown';
3022 if (!defined($status = $resp->{'return'}->{status
})) {
3023 warn "unable to get VM status\n";
3027 $res->{$vmid}->{qmpstatus
} = $resp->{'return'}->{status
};
3030 foreach my $vmid (keys %$list) {
3031 next if $opt_vmid && ($vmid ne $opt_vmid);
3032 next if !$res->{$vmid}->{pid
}; # not running
3033 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3036 $qmpclient->queue_execute(undef, 2);
3038 foreach my $vmid (keys %$list) {
3039 next if $opt_vmid && ($vmid ne $opt_vmid);
3040 next if !$res->{$vmid}->{pid
}; #not running
3042 # we can't use the $qmpclient since it might have already aborted on
3043 # 'query-balloon', but this might also fail for older versions...
3044 my $qemu_support = eval { mon_cmd
($vmid, "query-proxmox-support") };
3045 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3048 foreach my $vmid (keys %$list) {
3049 next if $opt_vmid && ($vmid ne $opt_vmid);
3050 $res->{$vmid}->{qmpstatus
} = $res->{$vmid}->{status
} if !$res->{$vmid}->{qmpstatus
};
3056 sub conf_has_serial
{
3059 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3060 if ($conf->{"serial$i"}) {
3068 sub conf_has_audio
{
3069 my ($conf, $id) = @_;
3072 my $audio = $conf->{"audio$id"};
3073 return if !defined($audio);
3075 my $audioproperties = parse_property_string
($audio_fmt, $audio);
3076 my $audiodriver = $audioproperties->{driver
} // 'spice';
3079 dev
=> $audioproperties->{device
},
3080 dev_id
=> "audiodev$id",
3081 backend
=> $audiodriver,
3082 backend_id
=> "$audiodriver-backend${id}",
3087 my ($audio, $audiopciaddr, $machine_version) = @_;
3091 my $id = $audio->{dev_id
};
3093 if (min_version
($machine_version, 4, 2)) {
3094 $audiodev = ",audiodev=$audio->{backend_id}";
3097 if ($audio->{dev
} eq 'AC97') {
3098 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3099 } elsif ($audio->{dev
} =~ /intel\-hda$/) {
3100 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3101 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3102 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3104 die "unkown audio device '$audio->{dev}', implement me!";
3107 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3115 socket => "/var/run/qemu-server/$vmid.swtpm",
3116 pid
=> "/var/run/qemu-server/$vmid.swtpm.pid",
3120 sub add_tpm_device
{
3121 my ($vmid, $devices, $conf) = @_;
3123 return if !$conf->{tpmstate0
};
3125 my $paths = get_tpm_paths
($vmid);
3127 push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3128 push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3129 push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3133 my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3135 return if !$tpmdrive;
3138 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
3139 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
3141 $state = PVE
::Storage
::map_volume
($storecfg, $tpm->{file
});
3143 $state = $tpm->{file
};
3146 my $paths = get_tpm_paths
($vmid);
3148 # during migration, we will get state from remote
3151 # run swtpm_setup to create a new TPM state if it doesn't exist yet
3158 "--create-platform-cert",
3161 "/etc/swtpm_setup.conf", # do not use XDG configs
3163 "0", # force creation as root, error if not possible
3164 "--not-overwrite", # ignore existing state, do not modify
3167 push @$setup_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3168 # TPM 2.0 supports ECC crypto, use if possible
3169 push @$setup_cmd, "--ecc" if $tpm->{version
} eq 'v2.0';
3171 run_command
($setup_cmd, outfunc
=> sub {
3172 print "swtpm_setup: $1\n";
3176 my $emulator_cmd = [
3180 "backend-uri=file://$state,mode=0600",
3182 "type=unixio,path=$paths->{socket},mode=0600",
3184 "file=$paths->{pid}",
3185 "--terminate", # terminate on QEMU disconnect
3188 push @$emulator_cmd, "--tpm2" if $tpm->{version
} eq 'v2.0';
3189 run_command
($emulator_cmd, outfunc
=> sub { print $1; });
3191 my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3192 while (! -e
$paths->{pid
}) {
3193 die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3197 # return untainted PID of swtpm daemon so it can be killed on error
3198 file_read_firstline
($paths->{pid
}) =~ m/(\d+)/;
3202 sub vga_conf_has_spice
{
3205 my $vgaconf = parse_vga
($vga);
3206 my $vgatype = $vgaconf->{type
};
3207 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3214 return get_host_arch
() eq $arch;
3219 return $conf->{arch
} // get_host_arch
();
3222 my $default_machines = {
3227 sub get_installed_machine_version
{
3228 my ($kvmversion) = @_;
3229 $kvmversion = kvm_user_version
() if !defined($kvmversion);
3230 $kvmversion =~ m/^(\d+\.\d+)/;
3234 sub windows_get_pinned_machine_version
{
3235 my ($machine, $base_version, $kvmversion) = @_;
3237 my $pin_version = $base_version;
3238 if (!defined($base_version) ||
3239 !PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($base_version, $kvmversion)
3241 $pin_version = get_installed_machine_version
($kvmversion);
3243 if (!$machine || $machine eq 'pc') {
3244 $machine = "pc-i440fx-$pin_version";
3245 } elsif ($machine eq 'q35') {
3246 $machine = "pc-q35-$pin_version";
3247 } elsif ($machine eq 'virt') {
3248 $machine = "virt-$pin_version";
3250 warn "unknown machine type '$machine', not touching that!\n";
3256 sub get_vm_machine
{
3257 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3259 my $machine = $forcemachine || $conf->{machine
};
3261 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3262 $kvmversion //= kvm_user_version
();
3263 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3264 # layout which confuses windows quite a bit and may result in various regressions..
3265 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3266 if (windows_version
($conf->{ostype
})) {
3267 $machine = windows_get_pinned_machine_version
($machine, '5.1', $kvmversion);
3270 $machine ||= $default_machines->{$arch};
3271 if ($add_pve_version) {
3272 my $pvever = PVE
::QemuServer
::Machine
::get_pve_version
($kvmversion);
3273 $machine .= "+pve$pvever";
3277 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3278 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3279 $machine = $1 if $is_pxe;
3281 # for version-pinned machines that do not include a pve-version (e.g.
3282 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3283 $machine .= '+pve0';
3285 $machine .= '.pxe' if $is_pxe;
3291 sub get_ovmf_files
($$$) {
3292 my ($arch, $efidisk, $smm) = @_;
3294 my $types = $OVMF->{$arch}
3295 or die "no OVMF images known for architecture '$arch'\n";
3297 my $type = 'default';
3298 if (defined($efidisk->{efitype
}) && $efidisk->{efitype
} eq '4m') {
3299 $type = $smm ?
"4m" : "4m-no-smm";
3300 $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3303 return $types->{$type}->@*;
3307 aarch64
=> '/usr/bin/qemu-system-aarch64',
3308 x86_64
=> '/usr/bin/qemu-system-x86_64',
3310 sub get_command_for_arch
($) {
3312 return '/usr/bin/kvm' if is_native
($arch);
3314 my $cmd = $Arch2Qemu->{$arch}
3315 or die "don't know how to emulate architecture '$arch'\n";
3319 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3320 # to use in a QEMU command line (-cpu element), first array_intersect the result
3321 # of query_supported_ with query_understood_. This is necessary because:
3323 # a) query_understood_ returns flags the host cannot use and
3324 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3325 # flags, but CPU settings - with most of them being flags. Those settings
3326 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
3328 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3329 # expensive. If you need the value returned from this, you can get it much
3330 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3331 # $accel being 'kvm' or 'tcg'.
3333 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3334 # changes, automatically populating pmxcfs.
3336 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3337 # since kvm and tcg machines support different flags
3339 sub query_supported_cpu_flags
{
3342 $arch //= get_host_arch
();
3343 my $default_machine = $default_machines->{$arch};
3347 # FIXME: Once this is merged, the code below should work for ARM as well:
3348 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3349 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3352 my $kvm_supported = defined(kvm_version
());
3353 my $qemu_cmd = get_command_for_arch
($arch);
3355 my $pidfile = PVE
::QemuServer
::Helpers
::pidfile_name
($fakevmid);
3357 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3358 my $query_supported_run_qemu = sub {
3364 '-machine', $default_machine,
3366 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3367 '-mon', 'chardev=qmp,mode=control',
3368 '-pidfile', $pidfile,
3373 push @$cmd, '-accel', 'tcg';
3376 my $rc = run_command
($cmd, noerr
=> 1, quiet
=> 0);
3377 die "QEMU flag querying VM exited with code " . $rc if $rc;
3380 my $cmd_result = mon_cmd
(
3382 'query-cpu-model-expansion',
3384 model
=> { name
=> 'host' }
3387 my $props = $cmd_result->{model
}->{props
};
3388 foreach my $prop (keys %$props) {
3389 next if $props->{$prop} ne '1';
3390 # QEMU returns some flags multiple times, with '_', '.' or '-'
3391 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3392 # We only keep those with underscores, to match /proc/cpuinfo
3393 $prop =~ s/\.|-/_/g;
3394 $flags->{$prop} = 1;
3399 # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3400 vm_stop
(undef, $fakevmid, 1, 1, 10, 0, 1);
3404 return [ sort keys %$flags ];
3407 # We need to query QEMU twice, since KVM and TCG have different supported flags
3408 PVE
::QemuConfig-
>lock_config($fakevmid, sub {
3409 $flags->{tcg
} = eval { $query_supported_run_qemu->(0) };
3410 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3412 if ($kvm_supported) {
3413 $flags->{kvm
} = eval { $query_supported_run_qemu->(1) };
3414 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3421 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3422 my $understood_cpu_flag_dir = "/usr/share/kvm";
3423 sub query_understood_cpu_flags
{
3424 my $arch = get_host_arch
();
3425 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3427 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3430 my $raw = file_get_contents
($filepath);
3431 $raw =~ s/^\s+|\s+$//g;
3432 my @flags = split(/\s+/, $raw);
3437 my sub get_cpuunits
{
3439 my $is_cgroupv2 = PVE
::CGroup
::cgroup_mode
() == 2;
3441 my $cpuunits = $conf->{cpuunits
};
3442 return $is_cgroupv2 ?
100 : 1024 if !defined($cpuunits);
3445 $cpuunits = 10000 if $cpuunits >= 10000; # v1 can be higher, so clamp v2 there
3447 $cpuunits = 2 if $cpuunits < 2; # v2 can be lower, so clamp v1 there
3452 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3453 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3454 my sub should_disable_smm
{
3455 my ($conf, $vga) = @_;
3457 return (!defined($conf->{bios
}) || $conf->{bios
} eq 'seabios') &&
3458 $vga->{type
} && $vga->{type
} =~ m/^(serial\d+|none)$/;
3461 sub config_to_command
{
3462 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3466 my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3469 my $ostype = $conf->{ostype
};
3470 my $winversion = windows_version
($ostype);
3471 my $kvm = $conf->{kvm
};
3472 my $nodename = nodename
();
3474 my $arch = get_vm_arch
($conf);
3475 my $kvm_binary = get_command_for_arch
($arch);
3476 my $kvmver = kvm_user_version
($kvm_binary);
3478 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3479 $kvmver //= "undefined";
3480 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3483 my $add_pve_version = min_version
($kvmver, 4, 1);
3485 my $machine_type = get_vm_machine
($conf, $forcemachine, $arch, $add_pve_version);
3486 my $machine_version = extract_version
($machine_type, $kvmver);
3487 $kvm //= 1 if is_native
($arch);
3489 $machine_version =~ m/(\d+)\.(\d+)/;
3490 my ($machine_major, $machine_minor) = ($1, $2);
3492 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3493 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3494 } elsif (!min_version
($kvmver, $machine_major, $machine_minor)) {
3495 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3496 ." please upgrade node '$nodename'\n"
3497 } elsif (!PVE
::QemuServer
::Machine
::can_run_pve_machine_version
($machine_version, $kvmver)) {
3498 my $max_pve_version = PVE
::QemuServer
::Machine
::get_pve_version
($machine_version);
3499 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3500 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3501 ." node '$nodename'\n";
3504 # if a specific +pve version is required for a feature, use $version_guard
3505 # instead of min_version to allow machines to be run with the minimum
3507 my $required_pve_version = 0;
3508 my $version_guard = sub {
3509 my ($major, $minor, $pve) = @_;
3510 return 0 if !min_version
($machine_version, $major, $minor, $pve);
3511 my $max_pve = PVE
::QemuServer
::Machine
::get_pve_version
("$major.$minor");
3512 return 1 if min_version
($machine_version, $major, $minor, $max_pve+1);
3513 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3517 if ($kvm && !defined kvm_version
()) {
3518 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3519 ." or enable in BIOS.\n";
3522 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
3523 my $hotplug_features = parse_hotplug_features
(defined($conf->{hotplug
}) ?
$conf->{hotplug
} : '1');
3524 my $use_old_bios_files = undef;
3525 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
3527 push @$cmd, $kvm_binary;
3529 push @$cmd, '-id', $vmid;
3531 my $vmname = $conf->{name
} || "vm$vmid";
3533 push @$cmd, '-name', $vmname;
3535 push @$cmd, '-no-shutdown';
3539 my $qmpsocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid);
3540 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3541 push @$cmd, '-mon', "chardev=qmp,mode=control";
3543 if (min_version
($machine_version, 2, 12)) {
3544 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3545 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3548 push @$cmd, '-pidfile' , PVE
::QemuServer
::Helpers
::pidfile_name
($vmid);
3550 push @$cmd, '-daemonize';
3552 if ($conf->{smbios1
}) {
3553 my $smbios_conf = parse_smbios1
($conf->{smbios1
});
3554 if ($smbios_conf->{base64
}) {
3555 # Do not pass base64 flag to qemu
3556 delete $smbios_conf->{base64
};
3557 my $smbios_string = "";
3558 foreach my $key (keys %$smbios_conf) {
3560 if ($key eq "uuid") {
3561 $value = $smbios_conf->{uuid
}
3563 $value = decode_base64
($smbios_conf->{$key});
3565 # qemu accepts any binary data, only commas need escaping by double comma
3567 $smbios_string .= "," . $key . "=" . $value if $value;
3569 push @$cmd, '-smbios', "type=1" . $smbios_string;
3571 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3575 if ($conf->{bios
} && $conf->{bios
} eq 'ovmf') {
3577 if (my $efidisk = $conf->{efidisk0
}) {
3578 $d = parse_drive
('efidisk0', $efidisk);
3581 my ($ovmf_code, $ovmf_vars) = get_ovmf_files
($arch, $d, $q35);
3582 die "uefi base image '$ovmf_code' not found\n" if ! -f
$ovmf_code;
3584 my ($path, $format);
3585 my $read_only_str = '';
3587 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($d->{file
}, 1);
3588 $format = $d->{format
};
3590 $path = PVE
::Storage
::path
($storecfg, $d->{file
});
3591 if (!defined($format)) {
3592 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
3593 $format = qemu_img_format
($scfg, $volname);
3597 die "efidisk format must be specified\n"
3598 if !defined($format);
3601 $read_only_str = ',readonly=on' if drive_is_read_only
($conf, $d);
3603 warn "no efidisk configured! Using temporary efivars disk.\n";
3604 $path = "/tmp/$vmid-ovmf.fd";
3605 PVE
::Tools
::file_copy
($ovmf_vars, $path, -s
$ovmf_vars);
3611 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3612 $size_str = ",size=" . (-s
$ovmf_vars);
3615 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3617 if ($path =~ m/^rbd:/) {
3618 $cache = ',cache=writeback';
3619 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3622 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3623 push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3626 if ($q35) { # tell QEMU to load q35 config early
3627 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3628 if (min_version
($machine_version, 4, 0)) {
3629 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3631 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3635 if (defined(my $fixups = qemu_created_version_fixups
($conf, $forcemachine, $kvmver))) {
3636 push @$cmd, $fixups->@*;
3639 if ($conf->{vmgenid
}) {
3640 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid
};
3643 # add usb controllers
3644 my @usbcontrollers = PVE
::QemuServer
::USB
::get_usb_controllers
(
3645 $conf, $bridges, $arch, $machine_type, $usbdesc->{format
}, $MAX_USB_DEVICES);
3646 push @$devices, @usbcontrollers if @usbcontrollers;
3647 my $vga = parse_vga
($conf->{vga
});
3649 my $qxlnum = vga_conf_has_spice
($conf->{vga
});
3650 $vga->{type
} = 'qxl' if $qxlnum;
3652 if (!$vga->{type
}) {
3653 if ($arch eq 'aarch64') {
3654 $vga->{type
} = 'virtio';
3655 } elsif (min_version
($machine_version, 2, 9)) {
3656 $vga->{type
} = (!$winversion || $winversion >= 6) ?
'std' : 'cirrus';
3658 $vga->{type
} = ($winversion >= 6) ?
'std' : 'cirrus';
3662 # enable absolute mouse coordinates (needed by vnc)
3663 my $tablet = $conf->{tablet
};
3664 if (!defined($tablet)) {
3665 $tablet = $defaults->{tablet
};
3666 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3667 $tablet = 0 if $vga->{type
} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3671 push @$devices, '-device', print_tabletdevice_full
($conf, $arch) if $tablet;
3672 my $kbd = print_keyboarddevice_full
($conf, $arch);
3673 push @$devices, '-device', $kbd if defined($kbd);
3676 my $bootorder = device_bootorder
($conf);
3678 # host pci device passthrough
3679 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE
::QemuServer
::PCI
::print_hostpci_devices
(
3680 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3683 my $usb_dev_features = {};
3684 $usb_dev_features->{spice_usb3
} = 1 if min_version
($machine_version, 4, 0);
3686 my @usbdevices = PVE
::QemuServer
::USB
::get_usb_devices
(
3687 $conf, $usbdesc->{format
}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3688 push @$devices, @usbdevices if @usbdevices;
3691 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3692 my $path = $conf->{"serial$i"} or next;
3693 if ($path eq 'socket') {
3694 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3695 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3696 # On aarch64, serial0 is the UART device. Qemu only allows
3697 # connecting UART devices via the '-serial' command line, as
3698 # the device has a fixed slot on the hardware...
3699 if ($arch eq 'aarch64' && $i == 0) {
3700 push @$devices, '-serial', "chardev:serial$i";
3702 push @$devices, '-device', "isa-serial,chardev=serial$i";
3705 die "no such serial device\n" if ! -c
$path;
3706 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3707 push @$devices, '-device', "isa-serial,chardev=serial$i";
3712 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3713 if (my $path = $conf->{"parallel$i"}) {
3714 die "no such parallel device\n" if ! -c
$path;
3715 my $devtype = $path =~ m!^/dev/usb/lp! ?
'tty' : 'parport';
3716 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3717 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3721 if (min_version
($machine_version, 4, 0) && (my $audio = conf_has_audio
($conf))) {
3722 my $audiopciaddr = print_pci_addr
("audio0", $bridges, $arch, $machine_type);
3723 my $audio_devs = audio_devs
($audio, $audiopciaddr, $machine_version);
3724 push @$devices, @$audio_devs;
3727 add_tpm_device
($vmid, $devices, $conf);
3730 $sockets = $conf->{smp
} if $conf->{smp
}; # old style - no longer iused
3731 $sockets = $conf->{sockets
} if $conf->{sockets
};
3733 my $cores = $conf->{cores
} || 1;
3735 my $maxcpus = $sockets * $cores;
3737 my $vcpus = $conf->{vcpus
} ?
$conf->{vcpus
} : $maxcpus;
3739 my $allowed_vcpus = $cpuinfo->{cpus
};
3741 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3743 if ($hotplug_features->{cpu
} && min_version
($machine_version, 2, 7)) {
3744 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3745 for (my $i = 2; $i <= $vcpus; $i++) {
3746 my $cpustr = print_cpu_device
($conf,$i);
3747 push @$cmd, '-device', $cpustr;
3752 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3754 push @$cmd, '-nodefaults';
3756 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3758 push @$cmd, '-no-acpi' if defined($conf->{acpi
}) && $conf->{acpi
} == 0;
3760 push @$cmd, '-no-reboot' if defined($conf->{reboot
}) && $conf->{reboot
} == 0;
3762 if ($vga->{type
} && $vga->{type
} !~ m/^serial\d+$/ && $vga->{type
} ne 'none'){
3763 push @$devices, '-device', print_vga_device
(
3764 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3765 my $socket = PVE
::QemuServer
::Helpers
::vnc_socket
($vmid);
3766 push @$cmd, '-vnc', "unix:$socket,password=on";
3768 push @$cmd, '-vga', 'none' if $vga->{type
} eq 'none';
3769 push @$cmd, '-nographic';
3773 my $tdf = defined($conf->{tdf
}) ?
$conf->{tdf
} : $defaults->{tdf
};
3774 my $useLocaltime = $conf->{localtime};
3776 if ($winversion >= 5) { # windows
3777 $useLocaltime = 1 if !defined($conf->{localtime});
3779 # use time drift fix when acpi is enabled
3780 if (!(defined($conf->{acpi
}) && $conf->{acpi
} == 0)) {
3781 $tdf = 1 if !defined($conf->{tdf
});
3785 if ($winversion >= 6) {
3786 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3787 push @$cmd, '-no-hpet';
3790 push @$rtcFlags, 'driftfix=slew' if $tdf;
3792 if ($conf->{startdate
} && $conf->{startdate
} ne 'now') {
3793 push @$rtcFlags, "base=$conf->{startdate}";
3794 } elsif ($useLocaltime) {
3795 push @$rtcFlags, 'base=localtime';
3799 push @$cmd, '-cpu', $forcecpu;
3801 push @$cmd, get_cpu_options
($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3804 PVE
::QemuServer
::Memory
::config
($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3806 push @$cmd, '-S' if $conf->{freeze
};
3808 push @$cmd, '-k', $conf->{keyboard
} if defined($conf->{keyboard
});
3810 my $guest_agent = parse_guest_agent
($conf);
3812 if ($guest_agent->{enabled
}) {
3813 my $qgasocket = PVE
::QemuServer
::Helpers
::qmp_socket
($vmid, 1);
3814 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3816 if (!$guest_agent->{type
} || $guest_agent->{type
} eq 'virtio') {
3817 my $pciaddr = print_pci_addr
("qga0", $bridges, $arch, $machine_type);
3818 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3819 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3820 } elsif ($guest_agent->{type
} eq 'isa') {
3821 push @$devices, '-device', "isa-serial,chardev=qga0";
3825 my $rng = $conf->{rng0
} ? parse_rng
($conf->{rng0
}) : undef;
3826 if ($rng && $version_guard->(4, 1, 2)) {
3827 check_rng_source
($rng->{source
});
3829 my $max_bytes = $rng->{max_bytes
} // $rng_fmt->{max_bytes
}->{default};
3830 my $period = $rng->{period
} // $rng_fmt->{period
}->{default};
3831 my $limiter_str = "";
3833 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3836 my $rng_addr = print_pci_addr
("rng0", $bridges, $arch, $machine_type);
3837 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3838 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3846 for (my $i = 1; $i < $qxlnum; $i++){
3847 push @$devices, '-device', print_vga_device
(
3848 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3851 # assume other OS works like Linux
3852 my ($ram, $vram) = ("134217728", "67108864");
3853 if ($vga->{memory
}) {
3854 $ram = PVE
::Tools
::convert_size
($qxlnum*4*$vga->{memory
}, 'mb' => 'b');
3855 $vram = PVE
::Tools
::convert_size
($qxlnum*2*$vga->{memory
}, 'mb' => 'b');
3857 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3858 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3862 my $pciaddr = print_pci_addr
("spice", $bridges, $arch, $machine_type);
3864 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
3865 my @nodeaddrs = PVE
::Tools
::getaddrinfo_all
('localhost', family
=> $pfamily);
3866 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3868 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3869 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3870 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3872 my $localhost = PVE
::Network
::addr_to_ip
($nodeaddrs[0]->{addr
});
3873 $spice_port = PVE
::Tools
::next_spice_port
($pfamily, $localhost);
3875 my $spice_enhancement_str = $conf->{spice_enhancements
} // '';
3876 my $spice_enhancement = parse_property_string
($spice_enhancements_fmt, $spice_enhancement_str);
3877 if ($spice_enhancement->{foldersharing
}) {
3878 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3879 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3882 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3883 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3884 if $spice_enhancement->{videostreaming
};
3886 push @$devices, '-spice', "$spice_opts";
3889 # enable balloon by default, unless explicitly disabled
3890 if (!defined($conf->{balloon
}) || $conf->{balloon
}) {
3891 my $pciaddr = print_pci_addr
("balloon0", $bridges, $arch, $machine_type);
3892 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3895 if ($conf->{watchdog
}) {
3896 my $wdopts = parse_watchdog
($conf->{watchdog
});
3897 my $pciaddr = print_pci_addr
("watchdog", $bridges, $arch, $machine_type);
3898 my $watchdog = $wdopts->{model
} || 'i6300esb';
3899 push @$devices, '-device', "$watchdog$pciaddr";
3900 push @$devices, '-watchdog-action', $wdopts->{action
} if $wdopts->{action
};
3904 my $scsicontroller = {};
3905 my $ahcicontroller = {};
3906 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : $defaults->{scsihw
};
3908 # Add iscsi initiator name if available
3909 if (my $initiator = get_initiator_name
()) {
3910 push @$devices, '-iscsi', "initiator-name=$initiator";
3913 PVE
::QemuConfig-
>foreach_volume($conf, sub {
3914 my ($ds, $drive) = @_;
3916 if (PVE
::Storage
::parse_volume_id
($drive->{file
}, 1)) {
3917 check_volume_storage_type
($storecfg, $drive->{file
});
3918 push @$vollist, $drive->{file
};
3921 # ignore efidisk here, already added in bios/fw handling code above
3922 return if $drive->{interface
} eq 'efidisk';
3924 return if $drive->{interface
} eq 'tpmstate';
3926 $use_virtio = 1 if $ds =~ m/^virtio/;
3928 $drive->{bootindex
} = $bootorder->{$ds} if $bootorder->{$ds};
3930 if ($drive->{interface
} eq 'virtio'){
3931 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread
};
3934 if ($drive->{interface
} eq 'scsi') {
3936 my ($maxdev, $controller, $controller_prefix) = scsihw_infos
($conf, $drive);
3938 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3939 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3941 my $pciaddr = print_pci_addr
("$controller_prefix$controller", $bridges, $arch, $machine_type);
3942 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ?
"virtio-scsi-pci" : $scsihw;
3945 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{iothread
}){
3946 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3947 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3948 } elsif ($drive->{iothread
}) {
3949 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3953 if($conf->{scsihw
} && $conf->{scsihw
} eq "virtio-scsi-single" && $drive->{queues
}){
3954 $queues = ",num_queues=$drive->{queues}";
3957 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3958 if !$scsicontroller->{$controller};
3959 $scsicontroller->{$controller}=1;
3962 if ($drive->{interface
} eq 'sata') {
3963 my $controller = int($drive->{index} / $PVE::QemuServer
::Drive
::MAX_SATA_DISKS
);
3964 my $pciaddr = print_pci_addr
("ahci$controller", $bridges, $arch, $machine_type);
3965 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3966 if !$ahcicontroller->{$controller};
3967 $ahcicontroller->{$controller}=1;
3970 my $pbs_conf = $pbs_backing->{$ds};
3971 my $pbs_name = undef;
3973 $pbs_name = "drive-$ds-pbs";
3974 push @$devices, '-blockdev', print_pbs_blockdev
($pbs_conf, $pbs_name);
3977 my $drive_cmd = print_drive_commandline_full
(
3978 $storecfg, $vmid, $drive, $pbs_name, min_version
($kvmver, 6, 0));
3980 # extra protection for templates, but SATA and IDE don't support it..
3981 $drive_cmd .= ',readonly=on' if drive_is_read_only
($conf, $drive);
3983 push @$devices, '-drive',$drive_cmd;
3984 push @$devices, '-device', print_drivedevice_full
(
3985 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3988 for (my $i = 0; $i < $MAX_NETS; $i++) {
3989 my $netname = "net$i";
3991 next if !$conf->{$netname};
3992 my $d = parse_net
($conf->{$netname});
3995 $use_virtio = 1 if $d->{model
} eq 'virtio';
3997 $d->{bootindex
} = $bootorder->{$netname} if $bootorder->{$netname};
3999 my $netdevfull = print_netdev_full
($vmid, $conf, $arch, $d, $netname);
4000 push @$devices, '-netdev', $netdevfull;
4002 my $netdevicefull = print_netdevice_full
(
4003 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
4005 push @$devices, '-device', $netdevicefull;
4008 if ($conf->{ivshmem
}) {
4009 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
4013 $bus = print_pcie_addr
("ivshmem");
4015 $bus = print_pci_addr
("ivshmem", $bridges, $arch, $machine_type);
4018 my $ivshmem_name = $ivshmem->{name
} // $vmid;
4019 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4021 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4022 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4023 .",size=$ivshmem->{size}M";
4026 # pci.4 is nested in pci.1
4027 $bridges->{1} = 1 if $bridges->{4};
4029 if (!$q35) { # add pci bridges
4030 if (min_version
($machine_version, 2, 3)) {
4034 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4037 for my $k (sort {$b cmp $a} keys %$bridges) {
4038 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4041 if ($k == 2 && $legacy_igd) {
4044 my $pciaddr = print_pci_addr
("pci.$k_name", undef, $arch, $machine_type);
4045 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4047 if ($q35) { # add after -readconfig pve-q35.cfg
4048 splice @$devices, 2, 0, '-device', $devstr;
4050 unshift @$devices, '-device', $devstr if $k > 0;
4055 push @$machineFlags, 'accel=tcg';
4058 push @$machineFlags, 'smm=off' if should_disable_smm
($conf, $vga);
4060 my $machine_type_min = $machine_type;
4061 if ($add_pve_version) {
4062 $machine_type_min =~ s/\+pve\d+$//;
4063 $machine_type_min .= "+pve$required_pve_version";
4065 push @$machineFlags, "type=${machine_type_min}";
4067 push @$cmd, @$devices;
4068 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4069 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4070 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4072 if (my $vmstate = $conf->{vmstate
}) {
4073 my $statepath = PVE
::Storage
::path
($storecfg, $vmstate);
4074 push @$vollist, $vmstate;
4075 push @$cmd, '-loadstate', $statepath;
4076 print "activating and using '$vmstate' as vmstate\n";
4079 if (PVE
::QemuConfig-
>is_template($conf)) {
4080 # needed to workaround base volumes being read-only
4081 push @$cmd, '-snapshot';
4085 if ($conf->{args
}) {
4086 my $aa = PVE
::Tools
::split_args
($conf->{args
});
4090 return wantarray ?
($cmd, $vollist, $spice_port) : $cmd;
4093 sub check_rng_source
{
4096 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4097 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4100 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4101 if ($source eq '/dev/hwrng' && file_read_firstline
($rng_current) eq 'none') {
4102 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4103 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4104 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4105 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4113 my $res = mon_cmd
($vmid, 'query-spice');
4115 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4118 sub vm_devices_list
{
4121 my $res = mon_cmd
($vmid, 'query-pci');
4122 my $devices_to_check = [];
4124 foreach my $pcibus (@$res) {
4125 push @$devices_to_check, @{$pcibus->{devices
}},
4128 while (@$devices_to_check) {
4130 for my $d (@$devices_to_check) {
4131 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4132 next if !$d->{'pci_bridge'};
4134 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices
}});
4135 push @$to_check, @{$d->{'pci_bridge'}->{devices
}};
4137 $devices_to_check = $to_check;
4140 my $resblock = mon_cmd
($vmid, 'query-block');
4141 foreach my $block (@$resblock) {
4142 if($block->{device
} =~ m/^drive-(\S+)/){
4147 my $resmice = mon_cmd
($vmid, 'query-mice');
4148 foreach my $mice (@$resmice) {
4149 if ($mice->{name
} eq 'QEMU HID Tablet') {
4150 $devices->{tablet
} = 1;
4155 # for usb devices there is no query-usb
4156 # but we can iterate over the entries in
4157 # qom-list path=/machine/peripheral
4158 my $resperipheral = mon_cmd
($vmid, 'qom-list', path
=> '/machine/peripheral');
4159 foreach my $per (@$resperipheral) {
4160 if ($per->{name
} =~ m/^usb\d+$/) {
4161 $devices->{$per->{name
}} = 1;
4169 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4171 my $q35 = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
4173 my $devices_list = vm_devices_list
($vmid);
4174 return 1 if defined($devices_list->{$deviceid});
4176 # add PCI bridge if we need it for the device
4177 qemu_add_pci_bridge
($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4179 if ($deviceid eq 'tablet') {
4180 qemu_deviceadd
($vmid, print_tabletdevice_full
($conf, $arch));
4181 } elsif ($deviceid eq 'keyboard') {
4182 qemu_deviceadd
($vmid, print_keyboarddevice_full
($conf, $arch));
4183 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4184 die "usb hotplug currently not reliable\n";
4185 # since we can't reliably hot unplug all added usb devices and usb
4186 # passthrough breaks live migration we disable usb hotplugging for now
4187 #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4188 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4189 qemu_iothread_add
($vmid, $deviceid, $device);
4191 qemu_driveadd
($storecfg, $vmid, $device);
4192 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4194 qemu_deviceadd
($vmid, $devicefull);
4195 eval { qemu_deviceaddverify
($vmid, $deviceid); };
4197 eval { qemu_drivedel
($vmid, $deviceid); };
4201 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4202 my $scsihw = defined($conf->{scsihw
}) ?
$conf->{scsihw
} : "lsi";
4203 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4204 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ?
"virtio-scsi-pci" : $scsihw;
4206 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4208 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread
}) {
4209 qemu_iothread_add
($vmid, $deviceid, $device);
4210 $devicefull .= ",iothread=iothread-$deviceid";
4213 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues
}) {
4214 $devicefull .= ",num_queues=$device->{queues}";
4217 qemu_deviceadd
($vmid, $devicefull);
4218 qemu_deviceaddverify
($vmid, $deviceid);
4219 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4220 qemu_findorcreatescsihw
($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4221 qemu_driveadd
($storecfg, $vmid, $device);
4223 my $devicefull = print_drivedevice_full
($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4224 eval { qemu_deviceadd
($vmid, $devicefull); };
4226 eval { qemu_drivedel
($vmid, $deviceid); };
4230 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4231 return if !qemu_netdevadd
($vmid, $conf, $arch, $device, $deviceid);
4233 my $machine_type = PVE
::QemuServer
::Machine
::qemu_machine_pxe
($vmid, $conf);
4234 my $use_old_bios_files = undef;
4235 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files
($machine_type);
4237 my $netdevicefull = print_netdevice_full
(
4238 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4239 qemu_deviceadd
($vmid, $netdevicefull);
4241 qemu_deviceaddverify
($vmid, $deviceid);
4242 qemu_set_link_status
($vmid, $deviceid, !$device->{link_down
});
4245 eval { qemu_netdevdel
($vmid, $deviceid); };
4249 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4251 my $pciaddr = print_pci_addr
($deviceid, undef, $arch, $machine_type);
4252 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4254 qemu_deviceadd
($vmid, $devicefull);
4255 qemu_deviceaddverify
($vmid, $deviceid);
4257 die "can't hotplug device '$deviceid'\n";
4263 # fixme: this should raise exceptions on error!
4264 sub vm_deviceunplug
{
4265 my ($vmid, $conf, $deviceid) = @_;
4267 my $devices_list = vm_devices_list
($vmid);
4268 return 1 if !defined($devices_list->{$deviceid});
4270 my $bootdisks = PVE
::QemuServer
::Drive
::get_bootdisks
($conf);
4271 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4273 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4274 qemu_devicedel
($vmid, $deviceid);
4275 } elsif ($deviceid =~ m/^usb\d+$/) {
4276 die "usb hotplug currently not reliable\n";
4277 # when unplugging usb devices this way, there may be remaining usb
4278 # controllers/hubs so we disable it for now
4279 #qemu_devicedel($vmid, $deviceid);
4280 #qemu_devicedelverify($vmid, $deviceid);
4281 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4282 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4284 qemu_devicedel
($vmid, $deviceid);
4285 qemu_devicedelverify
($vmid, $deviceid);
4286 qemu_drivedel
($vmid, $deviceid);
4287 qemu_iothread_del
($vmid, $deviceid, $device);
4288 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4289 qemu_devicedel
($vmid, $deviceid);
4290 qemu_devicedelverify
($vmid, $deviceid);
4291 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4292 my $device = parse_drive
($deviceid, $conf->{$deviceid});
4294 qemu_devicedel
($vmid, $deviceid);
4295 qemu_devicedelverify
($vmid, $deviceid);
4296 qemu_drivedel
($vmid, $deviceid);
4297 qemu_deletescsihw
($conf, $vmid, $deviceid);
4299 qemu_iothread_del
($vmid, "virtioscsi$device->{index}", $device)
4300 if $conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single');
4301 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4302 qemu_devicedel
($vmid, $deviceid);
4303 qemu_devicedelverify
($vmid, $deviceid);
4304 qemu_netdevdel
($vmid, $deviceid);
4306 die "can't unplug device '$deviceid'\n";
4312 sub qemu_deviceadd
{
4313 my ($vmid, $devicefull) = @_;
4315 $devicefull = "driver=".$devicefull;
4316 my %options = split(/[=,]/, $devicefull);
4318 mon_cmd
($vmid, "device_add" , %options);
4321 sub qemu_devicedel
{
4322 my ($vmid, $deviceid) = @_;
4324 my $ret = mon_cmd
($vmid, "device_del", id
=> $deviceid);
4327 sub qemu_iothread_add
{
4328 my ($vmid, $deviceid, $device) = @_;
4330 if ($device->{iothread
}) {
4331 my $iothreads = vm_iothreads_list
($vmid);
4332 qemu_objectadd
($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4336 sub qemu_iothread_del
{
4337 my ($vmid, $deviceid, $device) = @_;
4339 if ($device->{iothread
}) {
4340 my $iothreads = vm_iothreads_list
($vmid);
4341 qemu_objectdel
($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4345 sub qemu_objectadd
{
4346 my ($vmid, $objectid, $qomtype) = @_;
4348 mon_cmd
($vmid, "object-add", id
=> $objectid, "qom-type" => $qomtype);
4353 sub qemu_objectdel
{
4354 my ($vmid, $objectid) = @_;
4356 mon_cmd
($vmid, "object-del", id
=> $objectid);
4362 my ($storecfg, $vmid, $device) = @_;
4364 my $kvmver = get_running_qemu_version
($vmid);
4365 my $io_uring = min_version
($kvmver, 6, 0);
4366 my $drive = print_drive_commandline_full
($storecfg, $vmid, $device, undef, $io_uring);
4367 $drive =~ s/\\/\\\\/g;
4368 my $ret = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "drive_add auto \"$drive\"");
4370 # If the command succeeds qemu prints: "OK
"
4371 return 1 if $ret =~ m/OK/s;
4373 die "adding drive failed
: $ret\n";
4377 my ($vmid, $deviceid) = @_;
4379 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-
$deviceid");
4382 return 1 if $ret eq "";
4384 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4385 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4387 die "deleting drive
$deviceid failed
: $ret\n";
4390 sub qemu_deviceaddverify {
4391 my ($vmid, $deviceid) = @_;
4393 for (my $i = 0; $i <= 5; $i++) {
4394 my $devices_list = vm_devices_list($vmid);
4395 return 1 if defined($devices_list->{$deviceid});
4399 die "error on hotplug device
'$deviceid'\n";
4403 sub qemu_devicedelverify {
4404 my ($vmid, $deviceid) = @_;
4406 # need to verify that the device is correctly removed as device_del
4407 # is async and empty return is not reliable
4409 for (my $i = 0; $i <= 5; $i++) {
4410 my $devices_list = vm_devices_list($vmid);
4411 return 1 if !defined($devices_list->{$deviceid});
4415 die "error on hot-unplugging device
'$deviceid'\n";
4418 sub qemu_findorcreatescsihw {
4419 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4421 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4423 my $scsihwid="$controller_prefix$controller";
4424 my $devices_list = vm_devices_list($vmid);
4426 if (!defined($devices_list->{$scsihwid})) {
4427 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4433 sub qemu_deletescsihw {
4434 my ($conf, $vmid, $opt) = @_;
4436 my $device = parse_drive($opt, $conf->{$opt});
4438 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4439 vm_deviceunplug($vmid, $conf, "virtioscsi
$device->{index}");
4443 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4445 my $devices_list = vm_devices_list($vmid);
4446 foreach my $opt (keys %{$devices_list}) {
4447 if (is_valid_drivename($opt)) {
4448 my $drive = parse_drive($opt, $conf->{$opt});
4449 if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4455 my $scsihwid="scsihw
$controller";
4457 vm_deviceunplug($vmid, $conf, $scsihwid);
4462 sub qemu_add_pci_bridge {
4463 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4469 print_pci_addr($device, $bridges, $arch, $machine_type);
4471 while (my ($k, $v) = each %$bridges) {
4474 return 1 if !defined($bridgeid) || $bridgeid < 1;
4476 my $bridge = "pci
.$bridgeid";
4477 my $devices_list = vm_devices_list($vmid);
4479 if (!defined($devices_list->{$bridge})) {
4480 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4486 sub qemu_set_link_status {
4487 my ($vmid, $device, $up) = @_;
4489 mon_cmd($vmid, "set_link
", name => $device,
4490 up => $up ? JSON::true : JSON::false);
4493 sub qemu_netdevadd {
4494 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4496 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4497 my %options = split(/[=,]/, $netdev);
4499 if (defined(my $vhost = $options{vhost})) {
4500 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4503 if (defined(my $queues = $options{queues})) {
4504 $options{queues} = $queues + 0;
4507 mon_cmd($vmid, "netdev_add
", %options);
4511 sub qemu_netdevdel {
4512 my ($vmid, $deviceid) = @_;
4514 mon_cmd($vmid, "netdev_del
", id => $deviceid);
4517 sub qemu_usb_hotplug {
4518 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4522 # remove the old one first
4523 vm_deviceunplug($vmid, $conf, $deviceid);
4525 # check if xhci controller is necessary and available
4526 if ($device->{usb3}) {
4528 my $devicelist = vm_devices_list($vmid);
4530 if (!$devicelist->{xhci}) {
4531 my $pciaddr = print_pci_addr("xhci
", undef, $arch, $machine_type);
4532 qemu_deviceadd($vmid, "nec-usb-xhci
,id
=xhci
$pciaddr");
4535 my $d = parse_usb_device($device->{host});
4536 $d->{usb3} = $device->{usb3};
4539 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4542 sub qemu_cpu_hotplug {
4543 my ($vmid, $conf, $vcpus) = @_;
4545 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4548 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4549 $sockets = $conf->{sockets} if $conf->{sockets};
4550 my $cores = $conf->{cores} || 1;
4551 my $maxcpus = $sockets * $cores;
4553 $vcpus = $maxcpus if !$vcpus;
4555 die "you can
't add more vcpus than maxcpus\n"
4556 if $vcpus > $maxcpus;
4558 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4560 if ($vcpus < $currentvcpus) {
4562 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4564 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4565 qemu_devicedel($vmid, "cpu$i");
4567 my $currentrunningvcpus = undef;
4569 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4570 last if scalar(@{$currentrunningvcpus}) == $i-1;
4571 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4575 #update conf after each succesfull cpu unplug
4576 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4577 PVE::QemuConfig->write_config($vmid, $conf);
4580 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4586 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4587 die "vcpus in running vm does not match its configuration\n"
4588 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4590 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4592 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4593 my $cpustr = print_cpu_device($conf, $i);
4594 qemu_deviceadd($vmid, $cpustr);
4597 my $currentrunningvcpus = undef;
4599 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4600 last if scalar(@{$currentrunningvcpus}) == $i;
4601 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4605 #update conf after each succesfull cpu hotplug
4606 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4607 PVE::QemuConfig->write_config($vmid, $conf);
4611 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4612 mon_cmd($vmid, "cpu-add", id => int($i));
4617 sub qemu_block_set_io_throttle {
4618 my ($vmid, $deviceid,
4619 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4620 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4621 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4622 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4624 return if !check_running($vmid) ;
4626 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4628 bps_rd => int($bps_rd),
4629 bps_wr => int($bps_wr),
4631 iops_rd => int($iops_rd),
4632 iops_wr => int($iops_wr),
4633 bps_max => int($bps_max),
4634 bps_rd_max => int($bps_rd_max),
4635 bps_wr_max => int($bps_wr_max),
4636 iops_max => int($iops_max),
4637 iops_rd_max => int($iops_rd_max),
4638 iops_wr_max => int($iops_wr_max),
4639 bps_max_length => int($bps_max_length),
4640 bps_rd_max_length => int($bps_rd_max_length),
4641 bps_wr_max_length => int($bps_wr_max_length),
4642 iops_max_length => int($iops_max_length),
4643 iops_rd_max_length => int($iops_rd_max_length),
4644 iops_wr_max_length => int($iops_wr_max_length),
4649 sub qemu_block_resize {
4650 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4652 my $running = check_running($vmid);
4654 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4656 return if !$running;
4658 my $padding = (1024 - $size % 1024) % 1024;
4659 $size = $size + $padding;
4664 device => $deviceid,
4670 sub qemu_volume_snapshot {
4671 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4673 my $running = check_running($vmid);
4675 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4676 mon_cmd($vmid, 'blockdev-snapshot-internal-sync
', device => $deviceid, name => $snap);
4678 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4682 sub qemu_volume_snapshot_delete {
4683 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4685 my $running = check_running($vmid);
4690 my $conf = PVE::QemuConfig->load_config($vmid);
4691 PVE::QemuConfig->foreach_volume($conf, sub {
4692 my ($ds, $drive) = @_;
4693 $running = 1 if $drive->{file} eq $volid;
4697 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4698 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync
', device => $deviceid, name => $snap);
4700 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4704 sub set_migration_caps {
4705 my ($vmid, $savevm) = @_;
4707 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4709 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm
' : 'pbs-dirty-bitmap-migration
';
4710 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4715 "auto-converge" => 1,
4717 "x-rdma-pin-all" => 0,
4720 "dirty-bitmaps" => $dirty_bitmaps,
4723 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4725 for my $supported_capability (@$supported_capabilities) {
4727 capability => $supported_capability->{capability},
4728 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4732 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4736 my ($conf, $func, @param) = @_;
4740 my $test_volid = sub {
4741 my ($key, $drive, $snapname) = @_;
4743 my $volid = $drive->{file};
4746 $volhash->{$volid}->{cdrom} //= 1;
4747 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4749 my $replicate = $drive->{replicate} // 1;
4750 $volhash->{$volid}->{replicate} //= 0;
4751 $volhash->{$volid}->{replicate} = 1 if $replicate;
4753 $volhash->{$volid}->{shared} //= 0;
4754 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4756 $volhash->{$volid}->{referenced_in_config} //= 0;
4757 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4759 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4760 if defined($snapname);
4762 my $size = $drive->{size};
4763 $volhash->{$volid}->{size} //= $size if $size;
4765 $volhash->{$volid}->{is_vmstate} //= 0;
4766 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate
';
4768 $volhash->{$volid}->{is_tpmstate} //= 0;
4769 $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0
';
4771 $volhash->{$volid}->{is_unused} //= 0;
4772 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4774 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4777 my $include_opts = {
4778 extra_keys => ['vmstate
'],
4779 include_unused => 1,
4782 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4783 foreach my $snapname (keys %{$conf->{snapshots}}) {
4784 my $snap = $conf->{snapshots}->{$snapname};
4785 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4788 foreach my $volid (keys %$volhash) {
4789 &$func($volid, $volhash->{$volid}, @param);
4793 my $fast_plug_option = {
4801 'vmstatestorage
' => 1,
4806 # hotplug changes in [PENDING]
4807 # $selection hash can be used to only apply specified options, for
4808 # example: { cores => 1 } (only apply changed 'cores
')
4809 # $errors ref is used to return error messages
4810 sub vmconfig_hotplug_pending {
4811 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4813 my $defaults = load_defaults();
4814 my $arch = get_vm_arch($conf);
4815 my $machine_type = get_vm_machine($conf, undef, $arch);
4817 # commit values which do not have any impact on running VM first
4818 # Note: those option cannot raise errors, we we do not care about
4819 # $selection and always apply them.
4821 my $add_error = sub {
4822 my ($opt, $msg) = @_;
4823 $errors->{$opt} = "hotplug problem - $msg";
4827 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4828 if ($fast_plug_option->{$opt}) {
4829 $conf->{$opt} = $conf->{pending}->{$opt};
4830 delete $conf->{pending}->{$opt};
4836 PVE::QemuConfig->write_config($vmid, $conf);
4839 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4841 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4842 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4843 foreach my $opt (sort keys %$pending_delete_hash) {
4844 next if $selection && !$selection->{$opt};
4845 my $force = $pending_delete_hash->{$opt}->{force};
4847 if ($opt eq 'hotplug
') {
4848 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4849 } elsif ($opt eq 'tablet
') {
4850 die "skip\n" if !$hotplug_features->{usb};
4851 if ($defaults->{tablet}) {
4852 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4853 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4854 if $arch eq 'aarch64
';
4856 vm_deviceunplug($vmid, $conf, 'tablet
');
4857 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4859 } elsif ($opt =~ m/^usb\d+/) {
4861 # since we cannot reliably hot unplug usb devices we are disabling it
4862 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4863 #vm_deviceunplug($vmid, $conf, $opt);
4864 } elsif ($opt eq 'vcpus
') {
4865 die "skip\n" if !$hotplug_features->{cpu};
4866 qemu_cpu_hotplug($vmid, $conf, undef);
4867 } elsif ($opt eq 'balloon
') {
4868 # enable balloon device is not hotpluggable
4869 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4870 # here we reset the ballooning value to memory
4871 my $balloon = $conf->{memory} || $defaults->{memory};
4872 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4873 } elsif ($fast_plug_option->{$opt}) {
4875 } elsif ($opt =~ m/^net(\d+)$/) {
4876 die "skip\n" if !$hotplug_features->{network};
4877 vm_deviceunplug($vmid, $conf, $opt);
4878 } elsif (is_valid_drivename($opt)) {
4879 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4880 vm_deviceunplug($vmid, $conf, $opt);
4881 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4882 } elsif ($opt =~ m/^memory$/) {
4883 die "skip\n" if !$hotplug_features->{memory};
4884 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4885 } elsif ($opt eq 'cpuunits
') {
4886 $cgroup->change_cpu_shares(undef, 1024);
4887 } elsif ($opt eq 'cpulimit
') {
4888 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4894 &$add_error($opt, $err) if $err ne "skip\n";
4896 delete $conf->{$opt};
4897 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4901 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4902 $apply_pending_cloudinit = sub {
4903 return if $apply_pending_cloudinit_done; # once is enough
4904 $apply_pending_cloudinit_done = 1; # once is enough
4906 my ($key, $value) = @_;
4908 my @cloudinit_opts = keys %$confdesc_cloudinit;
4909 foreach my $opt (keys %{$conf->{pending}}) {
4910 next if !grep { $_ eq $opt } @cloudinit_opts;
4911 $conf->{$opt} = delete $conf->{pending}->{$opt};
4914 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4915 foreach my $opt (sort keys %$pending_delete_hash) {
4916 next if !grep { $_ eq $opt } @cloudinit_opts;
4917 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4918 delete $conf->{$opt};
4921 my $new_conf = { %$conf };
4922 $new_conf->{$key} = $value;
4923 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4926 foreach my $opt (keys %{$conf->{pending}}) {
4927 next if $selection && !$selection->{$opt};
4928 my $value = $conf->{pending}->{$opt};
4930 if ($opt eq 'hotplug
') {
4931 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4932 } elsif ($opt eq 'tablet
') {
4933 die "skip\n" if !$hotplug_features->{usb};
4935 vm_deviceplug($storecfg, $conf, $vmid, 'tablet
', $arch, $machine_type);
4936 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard
', $arch, $machine_type)
4937 if $arch eq 'aarch64
';
4938 } elsif ($value == 0) {
4939 vm_deviceunplug($vmid, $conf, 'tablet
');
4940 vm_deviceunplug($vmid, $conf, 'keyboard
') if $arch eq 'aarch64
';
4942 } elsif ($opt =~ m/^usb\d+$/) {
4944 # since we cannot reliably hot unplug usb devices we disable it for now
4945 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4946 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4947 #die "skip\n" if !$d;
4948 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4949 } elsif ($opt eq 'vcpus
') {
4950 die "skip\n" if !$hotplug_features->{cpu};
4951 qemu_cpu_hotplug($vmid, $conf, $value);
4952 } elsif ($opt eq 'balloon
') {
4953 # enable/disable balloning device is not hotpluggable
4954 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4955 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4956 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4958 # allow manual ballooning if shares is set to zero
4959 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4960 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4961 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4963 } elsif ($opt =~ m/^net(\d+)$/) {
4964 # some changes can be done without hotplug
4965 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4966 $vmid, $opt, $value, $arch, $machine_type);
4967 } elsif (is_valid_drivename($opt)) {
4968 die "skip\n" if $opt eq 'efidisk0
' || $opt eq 'tpmstate0
';
4969 # some changes can be done without hotplug
4970 my $drive = parse_drive($opt, $value);
4971 if (drive_is_cloudinit($drive)) {
4972 &$apply_pending_cloudinit($opt, $value);
4974 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4975 $vmid, $opt, $value, $arch, $machine_type);
4976 } elsif ($opt =~ m/^memory$/) { #dimms
4977 die "skip\n" if !$hotplug_features->{memory};
4978 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4979 } elsif ($opt eq 'cpuunits
') {
4980 my $new_cpuunits = get_cpuunits({ $opt => $conf->{pending}->{$opt} }); # to clamp
4981 $cgroup->change_cpu_shares($new_cpuunits, 1024);
4982 } elsif ($opt eq 'cpulimit
') {
4983 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4984 $cgroup->change_cpu_quota($cpulimit, 100000);
4985 } elsif ($opt eq 'agent
') {
4986 vmconfig_update_agent($conf, $opt, $value);
4988 die "skip\n"; # skip non-hot-pluggable options
4992 &$add_error($opt, $err) if $err ne "skip\n";
4994 $conf->{$opt} = $value;
4995 delete $conf->{pending}->{$opt};
4999 PVE::QemuConfig->write_config($vmid, $conf);
5002 sub try_deallocate_drive {
5003 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5005 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5006 my $volid = $drive->{file};
5007 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5008 my $sid = PVE::Storage::parse_volume_id($volid);
5009 $rpcenv->check($authuser, "/storage/$sid", ['Datastore
.AllocateSpace
']);
5011 # check if the disk is really unused
5012 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5013 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5014 PVE::Storage::vdisk_free($storecfg, $volid);
5017 # If vm is not owner of this disk remove from config
5025 sub vmconfig_delete_or_detach_drive {
5026 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5028 my $drive = parse_drive($opt, $conf->{$opt});
5030 my $rpcenv = PVE::RPCEnvironment::get();
5031 my $authuser = $rpcenv->get_user();
5034 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM
.Config
.Disk
']);
5035 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5037 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5043 sub vmconfig_apply_pending {
5044 my ($vmid, $conf, $storecfg, $errors) = @_;
5046 return if !scalar(keys %{$conf->{pending}});
5048 my $add_apply_error = sub {
5049 my ($opt, $msg) = @_;
5050 my $err_msg = "unable to apply pending change $opt : $msg";
5051 $errors->{$opt} = $err_msg;
5057 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5058 foreach my $opt (sort keys %$pending_delete_hash) {
5059 my $force = $pending_delete_hash->{$opt}->{force};
5061 if ($opt =~ m/^unused/) {
5062 die "internal error";
5063 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5064 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5068 $add_apply_error->($opt, $err);
5070 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5071 delete $conf->{$opt};
5075 PVE::QemuConfig->cleanup_pending($conf);
5077 foreach my $opt (keys %{$conf->{pending}}) { # add/change
5078 next if $opt eq 'delete'; # just to be sure
5080 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5081 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5085 $add_apply_error->($opt, $err);
5087 $conf->{$opt} = delete $conf->{pending}->{$opt};
5091 # write all changes at once to avoid unnecessary i/o
5092 PVE::QemuConfig->write_config($vmid, $conf);
5095 sub vmconfig_update_net {
5096 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5098 my $newnet = parse_net($value);
5100 if ($conf->{$opt}) {
5101 my $oldnet = parse_net($conf->{$opt});
5103 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5104 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5105 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5106 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5108 # for non online change, we try to hot-unplug
5109 die "skip\n" if !$hotplug;
5110 vm_deviceunplug($vmid, $conf, $opt);
5113 die "internal error" if $opt !~ m/net(\d+)/;
5114 my $iface = "tap${vmid}i$1";
5116 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5117 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5118 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5119 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5120 PVE::Network::tap_unplug($iface);
5123 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5125 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5127 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5128 # Rate can be applied on its own but any change above needs to
5129 # include the rate in tap_plug since OVS resets everything.
5130 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5133 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5134 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5142 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5148 sub vmconfig_update_agent {
5149 my ($conf, $opt, $value) = @_;
5151 die "skip\n" if !$conf->{$opt};
5153 my $hotplug_options = { fstrim_cloned_disks => 1 };
5155 my $old_agent = parse_guest_agent($conf);
5156 my $agent = parse_guest_agent({$opt => $value});
5158 for my $option (keys %$agent) { # added/changed options
5159 next if defined($hotplug_options->{$option});
5160 die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5163 for my $option (keys %$old_agent) { # removed options
5164 next if defined($hotplug_options->{$option});
5165 die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5168 return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5171 sub vmconfig_update_disk {
5172 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5174 my $drive = parse_drive($opt, $value);
5176 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5177 my $media = $drive->{media} || 'disk
';
5178 my $oldmedia = $old_drive->{media} || 'disk
';
5179 die "unable to change media type\n" if $media ne $oldmedia;
5181 if (!drive_is_cdrom($old_drive)) {
5183 if ($drive->{file} ne $old_drive->{file}) {
5185 die "skip\n" if !$hotplug;
5187 # unplug and register as unused
5188 vm_deviceunplug($vmid, $conf, $opt);
5189 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5192 # update existing disk
5194 # skip non hotpluggable value
5195 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5196 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5197 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5198 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5199 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5204 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5205 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5206 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5207 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5208 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5209 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5210 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5211 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5212 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5213 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5214 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5215 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5216 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5217 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5218 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5219 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5220 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5221 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5223 qemu_block_set_io_throttle(
5225 ($drive->{mbps} || 0)*1024*1024,
5226 ($drive->{mbps_rd} || 0)*1024*1024,
5227 ($drive->{mbps_wr} || 0)*1024*1024,
5228 $drive->{iops} || 0,
5229 $drive->{iops_rd} || 0,
5230 $drive->{iops_wr} || 0,
5231 ($drive->{mbps_max} || 0)*1024*1024,
5232 ($drive->{mbps_rd_max} || 0)*1024*1024,
5233 ($drive->{mbps_wr_max} || 0)*1024*1024,
5234 $drive->{iops_max} || 0,
5235 $drive->{iops_rd_max} || 0,
5236 $drive->{iops_wr_max} || 0,
5237 $drive->{bps_max_length} || 1,
5238 $drive->{bps_rd_max_length} || 1,
5239 $drive->{bps_wr_max_length} || 1,
5240 $drive->{iops_max_length} || 1,
5241 $drive->{iops_rd_max_length} || 1,
5242 $drive->{iops_wr_max_length} || 1,
5252 if ($drive->{file} eq 'none
') {
5253 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5254 if (drive_is_cloudinit($old_drive)) {
5255 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5258 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5260 # force eject if locked
5261 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5264 mon_cmd($vmid, "blockdev-change-medium",
5265 id => "$opt", filename => "$path");
5273 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5275 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5276 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5279 # called in locked context by incoming migration
5280 sub vm_migrate_get_nbd_disks {
5281 my ($storecfg, $conf, $replicated_volumes) = @_;
5283 my $local_volumes = {};
5284 PVE::QemuConfig->foreach_volume($conf, sub {
5285 my ($ds, $drive) = @_;
5287 return if drive_is_cdrom($drive);
5288 return if $ds eq 'tpmstate0
';
5290 my $volid = $drive->{file};
5294 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5296 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5297 return if $scfg->{shared};
5299 # replicated disks re-use existing state via bitmap
5300 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5301 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5303 return $local_volumes;
5306 # called in locked context by incoming migration
5307 sub vm_migrate_alloc_nbd_disks {
5308 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5311 foreach my $opt (sort keys %$source_volumes) {
5312 my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5314 if ($use_existing) {
5315 $nbd->{$opt}->{drivestr} = print_drive($drive);
5316 $nbd->{$opt}->{volid} = $volid;
5317 $nbd->{$opt}->{replicated} = 1;
5321 # storage mapping + volname = regular migration
5322 # storage mapping + format = remote migration
5323 # order of precedence, filtered by whether storage supports it:
5324 # 1. explicit requested format
5325 # 2. format of current volume
5326 # 3. default format of storage
5327 if (!$storagemap->{identity}) {
5328 $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5329 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5330 if (!$format || !grep { $format eq $_ } @$validFormats) {
5332 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5333 my $fileFormat = qemu_img_format($scfg, $volname);
5334 $format = $fileFormat
5335 if grep { $fileFormat eq $_ } @$validFormats;
5337 $format //= $defFormat;
5340 # can't happen
for remote migration
, so
$volname is always
defined
5341 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
5342 $format = qemu_img_format
($scfg, $volname);
5345 my $size = $drive->{size
} / 1024;
5346 my $newvolid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $format, undef, $size);
5347 my $newdrive = $drive;
5348 $newdrive->{format
} = $format;
5349 $newdrive->{file
} = $newvolid;
5350 my $drivestr = print_drive
($newdrive);
5351 $nbd->{$opt}->{drivestr
} = $drivestr;
5352 $nbd->{$opt}->{volid
} = $newvolid;
5358 # see vm_start_nolock for parameters, additionally:
5360 # storagemap = parsed storage map for allocating NBD disks
5362 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5364 return PVE
::QemuConfig-
>lock_config($vmid, sub {
5365 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migrate_opts->{migratedfrom
});
5367 die "you can't start a vm if it's a template\n"
5368 if !$params->{skiptemplate
} && PVE
::QemuConfig-
>is_template($conf);
5370 my $has_suspended_lock = PVE
::QemuConfig-
>has_lock($conf, 'suspended');
5371 my $has_backup_lock = PVE
::QemuConfig-
>has_lock($conf, 'backup');
5373 my $running = check_running
($vmid, undef, $migrate_opts->{migratedfrom
});
5375 if ($has_backup_lock && $running) {
5376 # a backup is currently running, attempt to start the guest in the
5377 # existing QEMU instance
5378 return vm_resume
($vmid);
5381 PVE
::QemuConfig-
>check_lock($conf)
5382 if !($params->{skiplock
} || $has_suspended_lock);
5384 $params->{resume
} = $has_suspended_lock || defined($conf->{vmstate
});
5386 die "VM $vmid already running\n" if $running;
5388 if (my $storagemap = $migrate_opts->{storagemap
}) {
5389 my $replicated = $migrate_opts->{replicated_volumes
};
5390 my $disks = vm_migrate_get_nbd_disks
($storecfg, $conf, $replicated);
5391 $migrate_opts->{nbd
} = vm_migrate_alloc_nbd_disks
($storecfg, $vmid, $disks, $storagemap);
5393 foreach my $opt (keys %{$migrate_opts->{nbd
}}) {
5394 $conf->{$opt} = $migrate_opts->{nbd
}->{$opt}->{drivestr
};
5398 return vm_start_nolock
($storecfg, $vmid, $conf, $params, $migrate_opts);
5404 # statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5405 # skiplock => 0/1, skip checking for config lock
5406 # skiptemplate => 0/1, skip checking whether VM is template
5407 # forcemachine => to force Qemu machine (rollback/migration)
5408 # forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5409 # timeout => in seconds
5410 # paused => start VM in paused state (backup)
5411 # resume => resume from hibernation
5422 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5423 # migratedfrom => source node
5424 # spice_ticket => used for spice migration, passed via tunnel/stdin
5425 # network => CIDR of migration network
5426 # type => secure/insecure - tunnel over encrypted connection or plain-text
5427 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
5428 # replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5429 # tpmstate_vol => new volid of tpmstate0, not yet contained in config
5430 sub vm_start_nolock
{
5431 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5433 my $statefile = $params->{statefile
};
5434 my $resume = $params->{resume
};
5436 my $migratedfrom = $migrate_opts->{migratedfrom
};
5437 my $migration_type = $migrate_opts->{type
};
5441 # clean up leftover reboot request files
5442 eval { clear_reboot_request
($vmid); };
5445 if (!$statefile && scalar(keys %{$conf->{pending
}})) {
5446 vmconfig_apply_pending
($vmid, $conf, $storecfg);
5447 $conf = PVE
::QemuConfig-
>load_config($vmid); # update/reload
5450 # don't regenerate the ISO if the VM is started as part of a live migration
5451 # this way we can reuse the old ISO with the correct config
5452 PVE
::QemuServer
::Cloudinit
::generate_cloudinitconfig
($conf, $vmid) if !$migratedfrom;
5454 # override TPM state vol if migrated, conf is out of date still
5455 if (my $tpmvol = $migrate_opts->{tpmstate_vol
}) {
5456 my $parsed = parse_drive
("tpmstate0", $conf->{tpmstate0
});
5457 $parsed->{file
} = $tpmvol;
5458 $conf->{tpmstate0
} = print_drive
($parsed);
5461 my $defaults = load_defaults
();
5463 # set environment variable useful inside network script
5464 $ENV{PVE_MIGRATED_FROM
} = $migratedfrom if $migratedfrom;
5466 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-start', 1);
5468 my $forcemachine = $params->{forcemachine
};
5469 my $forcecpu = $params->{forcecpu
};
5471 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5472 $forcemachine = $conf->{runningmachine
};
5473 $forcecpu = $conf->{runningcpu
};
5474 print "Resuming suspended VM\n";
5477 my ($cmd, $vollist, $spice_port) = config_to_command
($storecfg, $vmid,
5478 $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5481 my $get_migration_ip = sub {
5482 my ($nodename) = @_;
5484 return $migration_ip if defined($migration_ip);
5486 my $cidr = $migrate_opts->{network
};
5488 if (!defined($cidr)) {
5489 my $dc_conf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5490 $cidr = $dc_conf->{migration
}->{network
};
5493 if (defined($cidr)) {
5494 my $ips = PVE
::Network
::get_local_ip_from_cidr
($cidr);
5496 die "could not get IP: no address configured on local " .
5497 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5499 die "could not get IP: multiple addresses configured on local " .
5500 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5502 $migration_ip = @$ips[0];
5505 $migration_ip = PVE
::Cluster
::remote_node_ip
($nodename, 1)
5506 if !defined($migration_ip);
5508 return $migration_ip;
5513 if ($statefile eq 'tcp') {
5514 my $localip = "localhost";
5515 my $datacenterconf = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
5516 my $nodename = nodename
();
5518 if (!defined($migration_type)) {
5519 if (defined($datacenterconf->{migration
}->{type
})) {
5520 $migration_type = $datacenterconf->{migration
}->{type
};
5522 $migration_type = 'secure';
5526 if ($migration_type eq 'insecure') {
5527 $localip = $get_migration_ip->($nodename);
5528 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
5531 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5532 my $migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
5533 $migrate_uri = "tcp:${localip}:${migrate_port}";
5534 push @$cmd, '-incoming', $migrate_uri;
5537 } elsif ($statefile eq 'unix') {
5538 # should be default for secure migrations as a ssh TCP forward
5539 # tunnel is not deterministic reliable ready and fails regurarly
5540 # to set up in time, so use UNIX socket forwards
5541 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5542 unlink $socket_addr;
5544 $migrate_uri = "unix:$socket_addr";
5546 push @$cmd, '-incoming', $migrate_uri;
5549 } elsif (-e
$statefile) {
5550 push @$cmd, '-loadstate', $statefile;
5552 my $statepath = PVE
::Storage
::path
($storecfg, $statefile);
5553 push @$vollist, $statefile;
5554 push @$cmd, '-loadstate', $statepath;
5556 } elsif ($params->{paused
}) {
5560 my $start_timeout = $params->{timeout
} // config_aware_timeout
($conf, $resume);
5562 my $pci_devices = {}; # host pci devices
5563 for (my $i = 0; $i < $PVE::QemuServer
::PCI
::MAX_HOSTPCI_DEVICES
; $i++) {
5564 my $dev = $conf->{"hostpci$i"} or next;
5565 $pci_devices->{$i} = parse_hostpci
($dev);
5568 # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5569 my $real_pci_devices = [ grep { !(defined($_->{mdev
}) && scalar($_->{pciid
}->@*) == 1) } values $pci_devices->%* ];
5571 # map to a flat list of pci ids
5572 my $pci_id_list = [ map { $_->{id
} } map { $_->{pciid
}->@* } $real_pci_devices->@* ];
5574 # reserve all PCI IDs before actually doing anything with them
5575 PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_id_list, $vmid, $start_timeout);
5578 for my $id (sort keys %$pci_devices) {
5579 my $d = $pci_devices->{$id};
5580 for my $dev ($d->{pciid
}->@*) {
5581 PVE
::QemuServer
::PCI
::prepare_pci_device
($vmid, $dev->{id
}, $id, $d->{mdev
});
5586 eval { PVE
::QemuServer
::PCI
::remove_pci_reservation
($pci_id_list) };
5591 PVE
::Storage
::activate_volumes
($storecfg, $vollist);
5594 run_command
(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc
=> sub{}, errfunc
=> sub{});
5596 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5597 # timeout should be more than enough here...
5598 PVE
::Systemd
::wait_for_unit_removed
("$vmid.scope", 20);
5600 my $cpuunits = get_cpuunits
($conf);
5603 timeout
=> $statefile ?
undef : $start_timeout,
5608 # when migrating, prefix QEMU output so other side can pick up any
5609 # errors that might occur and show the user
5610 if ($migratedfrom) {
5611 $run_params{quiet
} = 1;
5612 $run_params{logfunc
} = sub { print "QEMU: $_[0]\n" };
5615 my %systemd_properties = (
5616 Slice
=> 'qemu.slice',
5617 KillMode
=> 'process',
5619 TimeoutStopUSec
=> ULONG_MAX
, # infinity
5622 if (PVE
::CGroup
::cgroup_mode
() == 2) {
5623 $systemd_properties{CPUWeight
} = $cpuunits;
5625 $systemd_properties{CPUShares
} = $cpuunits;
5628 if (my $cpulimit = $conf->{cpulimit
}) {
5629 $systemd_properties{CPUQuota
} = int($cpulimit * 100);
5631 $systemd_properties{timeout
} = 10 if $statefile; # setting up the scope shoul be quick
5633 my $run_qemu = sub {
5634 PVE
::Tools
::run_fork
sub {
5635 PVE
::Systemd
::enter_systemd_scope
($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5638 if (my $tpm = $conf->{tpmstate0
}) {
5639 # start the TPM emulator so QEMU can connect on start
5640 $tpmpid = start_swtpm
($storecfg, $vmid, $tpm, $migratedfrom);
5643 my $exitcode = run_command
($cmd, %run_params);
5646 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5647 kill 'TERM', $tpmpid;
5649 die "QEMU exited with code $exitcode\n";
5654 if ($conf->{hugepages
}) {
5657 my $hugepages_topology = PVE
::QemuServer
::Memory
::hugepages_topology
($conf);
5658 my $hugepages_host_topology = PVE
::QemuServer
::Memory
::hugepages_host_topology
();
5660 PVE
::QemuServer
::Memory
::hugepages_mount
();
5661 PVE
::QemuServer
::Memory
::hugepages_allocate
($hugepages_topology, $hugepages_host_topology);
5663 eval { $run_qemu->() };
5665 PVE
::QemuServer
::Memory
::hugepages_reset
($hugepages_host_topology)
5666 if !$conf->{keephugepages
};
5670 PVE
::QemuServer
::Memory
::hugepages_pre_deallocate
($hugepages_topology)
5671 if !$conf->{keephugepages
};
5673 eval { PVE
::QemuServer
::Memory
::hugepages_update_locked
($code); };
5676 eval { $run_qemu->() };
5680 # deactivate volumes if start fails
5681 eval { PVE
::Storage
::deactivate_volumes
($storecfg, $vollist); };
5682 eval { PVE
::QemuServer
::PCI
::remove_pci_reservation
($pci_id_list) };
5684 die "start failed: $err";
5687 # re-reserve all PCI IDs now that we can know the actual VM PID
5688 my $pid = PVE
::QemuServer
::Helpers
::vm_running_locally
($vmid);
5689 eval { PVE
::QemuServer
::PCI
::reserve_pci_usage
($pci_id_list, $vmid, undef, $pid) };
5692 print "migration listens on $migrate_uri\n" if $migrate_uri;
5693 $res->{migrate_uri
} = $migrate_uri;
5695 if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') {
5696 eval { mon_cmd
($vmid, "cont"); };
5700 #start nbd server for storage migration
5701 if (my $nbd = $migrate_opts->{nbd
}) {
5702 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version
} // 0;
5704 my $migrate_storage_uri;
5705 # nbd_protocol_version > 0 for unix socket support
5706 if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5707 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5708 mon_cmd
($vmid, "nbd-server-start", addr
=> { type
=> 'unix', data
=> { path
=> $socket_path } } );
5709 $migrate_storage_uri = "nbd:unix:$socket_path";
5711 my $nodename = nodename
();
5712 my $localip = $get_migration_ip->($nodename);
5713 my $pfamily = PVE
::Tools
::get_host_address_family
($nodename);
5714 my $storage_migrate_port = PVE
::Tools
::next_migrate_port
($pfamily);
5716 mon_cmd
($vmid, "nbd-server-start", addr
=> {
5719 host
=> "${localip}",
5720 port
=> "${storage_migrate_port}",
5723 $localip = "[$localip]" if Net
::IP
::ip_is_ipv6
($localip);
5724 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5727 $res->{migrate_storage_uri
} = $migrate_storage_uri;
5729 foreach my $opt (sort keys %$nbd) {
5730 my $drivestr = $nbd->{$opt}->{drivestr
};
5731 my $volid = $nbd->{$opt}->{volid
};
5732 mon_cmd
($vmid, "nbd-server-add", device
=> "drive-$opt", writable
=> JSON
::true
);
5733 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5734 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5735 print "re-using replicated volume: $opt - $volid\n"
5736 if $nbd->{$opt}->{replicated
};
5738 $res->{drives
}->{$opt} = $nbd->{$opt};
5739 $res->{drives
}->{$opt}->{nbd_uri
} = $nbd_uri;
5743 if ($migratedfrom) {
5745 set_migration_caps
($vmid);
5750 print "spice listens on port $spice_port\n";
5751 $res->{spice_port
} = $spice_port;
5752 if ($migrate_opts->{spice_ticket
}) {
5753 mon_cmd
($vmid, "set_password", protocol
=> 'spice', password
=>
5754 $migrate_opts->{spice_ticket
});
5755 mon_cmd
($vmid, "expire_password", protocol
=> 'spice', time => "+30");
5760 mon_cmd
($vmid, "balloon", value
=> $conf->{balloon
}*1024*1024)
5761 if !$statefile && $conf->{balloon
};
5763 foreach my $opt (keys %$conf) {
5764 next if $opt !~ m/^net\d+$/;
5765 my $nicconf = parse_net
($conf->{$opt});
5766 qemu_set_link_status
($vmid, $opt, 0) if $nicconf->{link_down
};
5770 mon_cmd
($vmid, 'qom-set',
5771 path
=> "machine/peripheral/balloon0",
5772 property
=> "guest-stats-polling-interval",
5773 value
=> 2) if (!defined($conf->{balloon
}) || $conf->{balloon
});
5776 print "Resumed VM, removing state\n";
5777 if (my $vmstate = $conf->{vmstate
}) {
5778 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
5779 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
5781 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5782 PVE
::QemuConfig-
>write_config($vmid, $conf);
5785 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'post-start');
5790 sub vm_commandline
{
5791 my ($storecfg, $vmid, $snapname) = @_;
5793 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5795 my ($forcemachine, $forcecpu);
5797 my $snapshot = $conf->{snapshots
}->{$snapname};
5798 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5800 # check for machine or CPU overrides in snapshot
5801 $forcemachine = $snapshot->{runningmachine
};
5802 $forcecpu = $snapshot->{runningcpu
};
5804 $snapshot->{digest
} = $conf->{digest
}; # keep file digest for API
5809 my $defaults = load_defaults
();
5811 my $cmd = config_to_command
($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5813 return PVE
::Tools
::cmd2string
($cmd);
5817 my ($vmid, $skiplock) = @_;
5819 PVE
::QemuConfig-
>lock_config($vmid, sub {
5821 my $conf = PVE
::QemuConfig-
>load_config($vmid);
5823 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5825 mon_cmd
($vmid, "system_reset");
5829 sub get_vm_volumes
{
5833 foreach_volid
($conf, sub {
5834 my ($volid, $attr) = @_;
5836 return if $volid =~ m
|^/|;
5838 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
5841 push @$vollist, $volid;
5847 sub vm_stop_cleanup
{
5848 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5853 my $vollist = get_vm_volumes
($conf);
5854 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
5856 if (my $tpmdrive = $conf->{tpmstate0
}) {
5857 my $tpm = parse_drive
("tpmstate0", $tpmdrive);
5858 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($tpm->{file
}, 1);
5860 PVE
::Storage
::unmap_volume
($storecfg, $tpm->{file
});
5865 foreach my $ext (qw(mon qmp pid vnc qga)) {
5866 unlink "/var/run/qemu-server/${vmid}.$ext";
5869 if ($conf->{ivshmem
}) {
5870 my $ivshmem = parse_property_string
($ivshmem_fmt, $conf->{ivshmem
});
5871 # just delete it for now, VMs which have this already open do not
5872 # are affected, but new VMs will get a separated one. If this
5873 # becomes an issue we either add some sort of ref-counting or just
5874 # add a "don't delete on stop" flag to the ivshmem format.
5875 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name
} // $vmid);
5879 foreach my $key (keys %$conf) {
5880 next if $key !~ m/^hostpci(\d+)$/;
5881 my $hostpciindex = $1;
5882 my $d = parse_hostpci
($conf->{$key});
5883 my $uuid = PVE
::SysFSTools
::generate_mdev_uuid
($vmid, $hostpciindex);
5885 foreach my $pci (@{$d->{pciid
}}) {
5886 my $pciid = $pci->{id
};
5887 push @$ids, $pci->{id
};
5888 PVE
::SysFSTools
::pci_cleanup_mdev_device
($pciid, $uuid);
5891 PVE
::QemuServer
::PCI
::remove_pci_reservation
($ids);
5893 vmconfig_apply_pending
($vmid, $conf, $storecfg) if $apply_pending_changes;
5895 warn $@ if $@; # avoid errors - just warn
5898 # call only in locked context
5900 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5902 my $pid = check_running
($vmid, $nocheck);
5907 $conf = PVE
::QemuConfig-
>load_config($vmid);
5908 PVE
::QemuConfig-
>check_lock($conf) if !$skiplock;
5909 if (!defined($timeout) && $shutdown && $conf->{startup
}) {
5910 my $opts = PVE
::JSONSchema
::pve_parse_startup_order
($conf->{startup
});
5911 $timeout = $opts->{down
} if $opts->{down
};
5913 PVE
::GuestHelpers
::exec_hookscript
($conf, $vmid, 'pre-stop');
5918 if (defined($conf) && get_qga_key
($conf, 'enabled')) {
5919 mon_cmd
($vmid, "guest-shutdown", timeout
=> $timeout);
5921 mon_cmd
($vmid, "system_powerdown");
5924 mon_cmd
($vmid, "quit");
5930 $timeout = 60 if !defined($timeout);
5933 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5938 if ($count >= $timeout) {
5940 warn "VM still running - terminating now with SIGTERM\n";
5943 die "VM quit/powerdown failed - got timeout\n";
5946 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5950 if (!check_running
($vmid, $nocheck)) {
5951 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5955 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5958 die "VM quit/powerdown failed\n";
5966 while (($count < $timeout) && check_running
($vmid, $nocheck)) {
5971 if ($count >= $timeout) {
5972 warn "VM still running - terminating now with SIGKILL\n";
5977 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5980 # Note: use $nocheck to skip tests if VM configuration file exists.
5981 # We need that when migration VMs to other nodes (files already moved)
5982 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5984 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5986 $force = 1 if !defined($force) && !$shutdown;
5989 my $pid = check_running
($vmid, $nocheck, $migratedfrom);
5990 kill 15, $pid if $pid;
5991 my $conf = PVE
::QemuConfig-
>load_config($vmid, $migratedfrom);
5992 vm_stop_cleanup
($storecfg, $vmid, $conf, $keepActive, 0);
5996 PVE
::QemuConfig-
>lock_config($vmid, sub {
5997 _do_vm_stop
($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6002 my ($vmid, $timeout) = @_;
6004 PVE
::QemuConfig-
>lock_config($vmid, sub {
6007 # only reboot if running, as qmeventd starts it again on a stop event
6008 return if !check_running
($vmid);
6010 create_reboot_request
($vmid);
6012 my $storecfg = PVE
::Storage
::config
();
6013 _do_vm_stop
($storecfg, $vmid, undef, undef, $timeout, 1);
6017 # avoid that the next normal shutdown will be confused for a reboot
6018 clear_reboot_request
($vmid);
6024 # note: if using the statestorage parameter, the caller has to check privileges
6026 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6033 PVE
::QemuConfig-
>lock_config($vmid, sub {
6035 $conf = PVE
::QemuConfig-
>load_config($vmid);
6037 my $is_backing_up = PVE
::QemuConfig-
>has_lock($conf, 'backup');
6038 PVE
::QemuConfig-
>check_lock($conf)
6039 if !($skiplock || $is_backing_up);
6041 die "cannot suspend to disk during backup\n"
6042 if $is_backing_up && $includestate;
6044 if ($includestate) {
6045 $conf->{lock} = 'suspending';
6046 my $date = strftime
("%Y-%m-%d", localtime(time()));
6047 $storecfg = PVE
::Storage
::config
();
6048 if (!$statestorage) {
6049 $statestorage = find_vmstate_storage
($conf, $storecfg);
6050 # check permissions for the storage
6051 my $rpcenv = PVE
::RPCEnvironment
::get
();
6052 if ($rpcenv->{type
} ne 'cli') {
6053 my $authuser = $rpcenv->get_user();
6054 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6059 $vmstate = PVE
::QemuConfig-
>__snapshot_save_vmstate(
6060 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6061 $path = PVE
::Storage
::path
($storecfg, $vmstate);
6062 PVE
::QemuConfig-
>write_config($vmid, $conf);
6064 mon_cmd
($vmid, "stop");
6068 if ($includestate) {
6070 PVE
::Storage
::activate_volumes
($storecfg, [$vmstate]);
6073 set_migration_caps
($vmid, 1);
6074 mon_cmd
($vmid, "savevm-start", statefile
=> $path);
6076 my $state = mon_cmd
($vmid, "query-savevm");
6077 if (!$state->{status
}) {
6078 die "savevm not active\n";
6079 } elsif ($state->{status
} eq 'active') {
6082 } elsif ($state->{status
} eq 'completed') {
6083 print "State saved, quitting\n";
6085 } elsif ($state->{status
} eq 'failed' && $state->{error
}) {
6086 die "query-savevm failed with error '$state->{error}'\n"
6088 die "query-savevm returned status '$state->{status}'\n";
6094 PVE
::QemuConfig-
>lock_config($vmid, sub {
6095 $conf = PVE
::QemuConfig-
>load_config($vmid);
6097 # cleanup, but leave suspending lock, to indicate something went wrong
6099 mon_cmd
($vmid, "savevm-end");
6100 PVE
::Storage
::deactivate_volumes
($storecfg, [$vmstate]);
6101 PVE
::Storage
::vdisk_free
($storecfg, $vmstate);
6102 delete $conf->@{qw(vmstate runningmachine runningcpu)};
6103 PVE
::QemuConfig-
>write_config($vmid, $conf);
6109 die "lock changed unexpectedly\n"
6110 if !PVE
::QemuConfig-
>has_lock($conf, 'suspending');
6112 mon_cmd
($vmid, "quit");
6113 $conf->{lock} = 'suspended';
6114 PVE
::QemuConfig-
>write_config($vmid, $conf);
6120 my ($vmid, $skiplock, $nocheck) = @_;
6122 PVE
::QemuConfig-
>lock_config($vmid, sub {
6123 my $res = mon_cmd
($vmid, 'query-status');
6124 my $resume_cmd = 'cont';
6127 if ($res->{status
}) {
6128 return if $res->{status
} eq 'running'; # job done, go home
6129 $resume_cmd = 'system_wakeup' if $res->{status
} eq 'suspended';
6130 $reset = 1 if $res->{status
} eq 'shutdown';
6135 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6137 PVE
::QemuConfig-
>check_lock($conf)
6138 if !($skiplock || PVE
::QemuConfig-
>has_lock($conf, 'backup'));
6142 # required if a VM shuts down during a backup and we get a resume
6143 # request before the backup finishes for example
6144 mon_cmd
($vmid, "system_reset");
6146 mon_cmd
($vmid, $resume_cmd);
6151 my ($vmid, $skiplock, $key) = @_;
6153 PVE
::QemuConfig-
>lock_config($vmid, sub {
6155 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6157 # there is no qmp command, so we use the human monitor command
6158 my $res = PVE
::QemuServer
::Monitor
::hmp_cmd
($vmid, "sendkey $key");
6159 die $res if $res ne '';
6163 # vzdump restore implementaion
6165 sub tar_archive_read_firstfile
{
6166 my $archive = shift;
6168 die "ERROR: file '$archive' does not exist\n" if ! -f
$archive;
6170 # try to detect archive type first
6171 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6172 die "unable to open file '$archive'\n";
6173 my $firstfile = <$fh>;
6177 die "ERROR: archive contaions no data\n" if !$firstfile;
6183 sub tar_restore_cleanup
{
6184 my ($storecfg, $statfile) = @_;
6186 print STDERR
"starting cleanup\n";
6188 if (my $fd = IO
::File-
>new($statfile, "r")) {
6189 while (defined(my $line = <$fd>)) {
6190 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6193 if ($volid =~ m
|^/|) {
6194 unlink $volid || die 'unlink failed\n';
6196 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6198 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6200 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6202 print STDERR
"unable to parse line in statfile - $line";
6209 sub restore_file_archive
{
6210 my ($archive, $vmid, $user, $opts) = @_;
6212 return restore_vma_archive
($archive, $vmid, $user, $opts)
6215 my $info = PVE
::Storage
::archive_info
($archive);
6216 my $format = $opts->{format
} // $info->{format
};
6217 my $comp = $info->{compression
};
6219 # try to detect archive format
6220 if ($format eq 'tar') {
6221 return restore_tar_archive
($archive, $vmid, $user, $opts);
6223 return restore_vma_archive
($archive, $vmid, $user, $opts, $comp);
6227 # hepler to remove disks that will not be used after restore
6228 my $restore_cleanup_oldconf = sub {
6229 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6231 PVE
::QemuConfig-
>foreach_volume($oldconf, sub {
6232 my ($ds, $drive) = @_;
6234 return if drive_is_cdrom
($drive, 1);
6236 my $volid = $drive->{file
};
6237 return if !$volid || $volid =~ m
|^/|;
6239 my ($path, $owner) = PVE
::Storage
::path
($storecfg, $volid);
6240 return if !$path || !$owner || ($owner != $vmid);
6242 # Note: only delete disk we want to restore
6243 # other volumes will become unused
6244 if ($virtdev_hash->{$ds}) {
6245 eval { PVE
::Storage
::vdisk_free
($storecfg, $volid); };
6252 # delete vmstate files, after the restore we have no snapshots anymore
6253 foreach my $snapname (keys %{$oldconf->{snapshots
}}) {
6254 my $snap = $oldconf->{snapshots
}->{$snapname};
6255 if ($snap->{vmstate
}) {
6256 eval { PVE
::Storage
::vdisk_free
($storecfg, $snap->{vmstate
}); };
6264 # Helper to parse vzdump backup device hints
6266 # $rpcenv: Environment, used to ckeck storage permissions
6267 # $user: User ID, to check storage permissions
6268 # $storecfg: Storage configuration
6269 # $fh: the file handle for reading the configuration
6270 # $devinfo: should contain device sizes for all backu-up'ed devices
6271 # $options: backup options (pool, default storage)
6273 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6274 my $parse_backup_hints = sub {
6275 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6277 my $check_storage = sub { # assert if an image can be allocate
6278 my ($storeid, $scfg) = @_;
6279 die "Content type 'images' is not available on storage '$storeid'\n"
6280 if !$scfg->{content
}->{images
};
6281 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6282 if $user ne 'root@pam';
6285 my $virtdev_hash = {};
6286 while (defined(my $line = <$fh>)) {
6287 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6288 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6289 die "archive does not contain data for drive '$virtdev'\n"
6290 if !$devinfo->{$devname};
6292 if (defined($options->{storage
})) {
6293 $storeid = $options->{storage
} || 'local';
6294 } elsif (!$storeid) {
6297 $format = 'raw' if !$format;
6298 $devinfo->{$devname}->{devname
} = $devname;
6299 $devinfo->{$devname}->{virtdev
} = $virtdev;
6300 $devinfo->{$devname}->{format
} = $format;
6301 $devinfo->{$devname}->{storeid
} = $storeid;
6303 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6304 $check_storage->($storeid, $scfg); # permission and content type check
6306 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6307 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6309 my $drive = parse_drive
($virtdev, $2);
6311 if (drive_is_cloudinit
($drive)) {
6312 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
6313 $storeid = $options->{storage
} if defined ($options->{storage
});
6314 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6315 my $format = qemu_img_format
($scfg, $volname); # has 'raw' fallback
6317 $check_storage->($storeid, $scfg); # permission and content type check
6319 $virtdev_hash->{$virtdev} = {
6321 storeid
=> $storeid,
6322 size
=> PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
,
6329 return $virtdev_hash;
6332 # Helper to allocate and activate all volumes required for a restore
6334 # $storecfg: Storage configuration
6335 # $virtdev_hash: as returned by parse_backup_hints()
6337 # Returns: { $virtdev => $volid }
6338 my $restore_allocate_devices = sub {
6339 my ($storecfg, $virtdev_hash, $vmid) = @_;
6342 foreach my $virtdev (sort keys %$virtdev_hash) {
6343 my $d = $virtdev_hash->{$virtdev};
6344 my $alloc_size = int(($d->{size
} + 1024 - 1)/1024);
6345 my $storeid = $d->{storeid
};
6346 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6348 # test if requested format is supported
6349 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
6350 my $supported = grep { $_ eq $d->{format
} } @$validFormats;
6351 $d->{format
} = $defFormat if !$supported;
6354 if ($d->{is_cloudinit
}) {
6355 $name = "vm-$vmid-cloudinit";
6356 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6357 if ($scfg->{path
}) {
6358 $name .= ".$d->{format}";
6362 my $volid = PVE
::Storage
::vdisk_alloc
(
6363 $storecfg, $storeid, $vmid, $d->{format
}, $name, $alloc_size);
6365 print STDERR
"new volume ID is '$volid'\n";
6366 $d->{volid
} = $volid;
6368 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
6370 $map->{$virtdev} = $volid;
6376 sub restore_update_config_line
{
6377 my ($cookie, $map, $line, $unique) = @_;
6379 return '' if $line =~ m/^\#qmdump\#/;
6380 return '' if $line =~ m/^\#vzdump\#/;
6381 return '' if $line =~ m/^lock:/;
6382 return '' if $line =~ m/^unused\d+:/;
6383 return '' if $line =~ m/^parent:/;
6387 my $dc = PVE
::Cluster
::cfs_read_file
('datacenter.cfg');
6388 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6389 # try to convert old 1.X settings
6390 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6391 foreach my $devconfig (PVE
::Tools
::split_list
($ethcfg)) {
6392 my ($model, $macaddr) = split(/\=/, $devconfig);
6393 $macaddr = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if !$macaddr || $unique;
6396 bridge
=> "vmbr$ind",
6397 macaddr
=> $macaddr,
6399 my $netstr = print_net
($net);
6401 $res .= "net$cookie->{netcount}: $netstr\n";
6402 $cookie->{netcount
}++;
6404 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6405 my ($id, $netstr) = ($1, $2);
6406 my $net = parse_net
($netstr);
6407 $net->{macaddr
} = PVE
::Tools
::random_ether_addr
($dc->{mac_prefix
}) if $net->{macaddr
};
6408 $netstr = print_net
($net);
6409 $res .= "$id: $netstr\n";
6410 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6413 my $di = parse_drive
($virtdev, $value);
6414 if (defined($di->{backup
}) && !$di->{backup
}) {
6416 } elsif ($map->{$virtdev}) {
6417 delete $di->{format
}; # format can change on restore
6418 $di->{file
} = $map->{$virtdev};
6419 $value = print_drive
($di);
6420 $res .= "$virtdev: $value\n";
6424 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6426 if ($vmgenid ne '0') {
6427 # always generate a new vmgenid if there was a valid one setup
6428 $vmgenid = generate_uuid
();
6430 $res .= "vmgenid: $vmgenid\n";
6431 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6432 my ($uuid, $uuid_str);
6433 UUID
::generate
($uuid);
6434 UUID
::unparse
($uuid, $uuid_str);
6435 my $smbios1 = parse_smbios1
($2);
6436 $smbios1->{uuid
} = $uuid_str;
6437 $res .= $1.print_smbios1
($smbios1)."\n";
6445 my $restore_deactivate_volumes = sub {
6446 my ($storecfg, $devinfo) = @_;
6449 foreach my $devname (keys %$devinfo) {
6450 my $volid = $devinfo->{$devname}->{volid
};
6451 push @$vollist, $volid if $volid;
6454 PVE
::Storage
::deactivate_volumes
($storecfg, $vollist);
6457 my $restore_destroy_volumes = sub {
6458 my ($storecfg, $devinfo) = @_;
6460 foreach my $devname (keys %$devinfo) {
6461 my $volid = $devinfo->{$devname}->{volid
};
6464 if ($volid =~ m
|^/|) {
6465 unlink $volid || die 'unlink failed\n';
6467 PVE
::Storage
::vdisk_free
($storecfg, $volid);
6469 print STDERR
"temporary volume '$volid' sucessfuly removed\n";
6471 print STDERR
"unable to cleanup '$volid' - $@" if $@;
6476 my ($cfg, $vmid) = @_;
6478 my $info = PVE
::Storage
::vdisk_list
($cfg, undef, $vmid, undef, 'images');
6480 my $volid_hash = {};
6481 foreach my $storeid (keys %$info) {
6482 foreach my $item (@{$info->{$storeid}}) {
6483 next if !($item->{volid
} && $item->{size
});
6484 $item->{path
} = PVE
::Storage
::path
($cfg, $item->{volid
});
6485 $volid_hash->{$item->{volid
}} = $item;
6492 sub update_disk_config
{
6493 my ($vmid, $conf, $volid_hash) = @_;
6496 my $prefix = "VM $vmid";
6498 # used and unused disks
6499 my $referenced = {};
6501 # Note: it is allowed to define multiple storages with same path (alias), so
6502 # we need to check both 'volid' and real 'path' (two different volid can point
6503 # to the same path).
6505 my $referencedpath = {};
6508 PVE
::QemuConfig-
>foreach_volume($conf, sub {
6509 my ($opt, $drive) = @_;
6511 my $volid = $drive->{file
};
6513 my $volume = $volid_hash->{$volid};
6515 # mark volid as "in-use" for next step
6516 $referenced->{$volid} = 1;
6517 if ($volume && (my $path = $volume->{path
})) {
6518 $referencedpath->{$path} = 1;
6521 return if drive_is_cdrom
($drive);
6524 my ($updated, $msg) = PVE
::QemuServer
::Drive
::update_disksize
($drive, $volume->{size
});
6525 if (defined($updated)) {
6527 $conf->{$opt} = print_drive
($updated);
6528 print "$prefix ($opt): $msg\n";
6532 # remove 'unusedX' entry if volume is used
6533 PVE
::QemuConfig-
>foreach_unused_volume($conf, sub {
6534 my ($opt, $drive) = @_;
6536 my $volid = $drive->{file
};
6540 $path = $volid_hash->{$volid}->{path
} if $volid_hash->{$volid};
6541 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6542 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6544 delete $conf->{$opt};
6547 $referenced->{$volid} = 1;
6548 $referencedpath->{$path} = 1 if $path;
6551 foreach my $volid (sort keys %$volid_hash) {
6552 next if $volid =~ m/vm-$vmid-state-/;
6553 next if $referenced->{$volid};
6554 my $path = $volid_hash->{$volid}->{path
};
6555 next if !$path; # just to be sure
6556 next if $referencedpath->{$path};
6558 my $key = PVE
::QemuConfig-
>add_unused_volume($conf, $volid);
6559 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6560 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6567 my ($vmid, $nolock, $dryrun) = @_;
6569 my $cfg = PVE
::Storage
::config
();
6571 print "rescan volumes...\n";
6572 my $volid_hash = scan_volids
($cfg, $vmid);
6574 my $updatefn = sub {
6577 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6579 PVE
::QemuConfig-
>check_lock($conf);
6582 foreach my $volid (keys %$volid_hash) {
6583 my $info = $volid_hash->{$volid};
6584 $vm_volids->{$volid} = $info if $info->{vmid
} && $info->{vmid
} == $vmid;
6587 my $changes = update_disk_config
($vmid, $conf, $vm_volids);
6589 PVE
::QemuConfig-
>write_config($vmid, $conf) if $changes && !$dryrun;
6592 if (defined($vmid)) {
6596 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6599 my $vmlist = config_list
();
6600 foreach my $vmid (keys %$vmlist) {
6604 PVE
::QemuConfig-
>lock_config($vmid, $updatefn, $vmid);
6610 sub restore_proxmox_backup_archive
{
6611 my ($archive, $vmid, $user, $options) = @_;
6613 my $storecfg = PVE
::Storage
::config
();
6615 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($archive);
6616 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
6618 my $fingerprint = $scfg->{fingerprint
};
6619 my $keyfile = PVE
::Storage
::PBSPlugin
::pbs_encryption_key_file_name
($storecfg, $storeid);
6621 my $repo = PVE
::PBSClient
::get_repository
($scfg);
6623 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6624 my $password = PVE
::Storage
::PBSPlugin
::pbs_get_password
($scfg, $storeid);
6625 local $ENV{PBS_PASSWORD
} = $password;
6626 local $ENV{PBS_FINGERPRINT
} = $fingerprint if defined($fingerprint);
6628 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6629 PVE
::Storage
::parse_volname
($storecfg, $archive);
6631 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6633 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6635 my $tmpdir = "/var/tmp/vzdumptmp$$";
6639 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6640 # disable interrupts (always do cleanups)
6644 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
6646 # Note: $oldconf is undef if VM does not exists
6647 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6648 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6649 my $new_conf_raw = '';
6651 my $rpcenv = PVE
::RPCEnvironment
::get
();
6660 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
6662 my $cfgfn = "$tmpdir/qemu-server.conf";
6663 my $firewall_config_fn = "$tmpdir/fw.conf";
6664 my $index_fn = "$tmpdir/index.json";
6666 my $cmd = "restore";
6668 my $param = [$pbs_backup_name, "index.json", $index_fn];
6669 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6670 my $index = PVE
::Tools
::file_get_contents
($index_fn);
6671 $index = decode_json
($index);
6673 foreach my $info (@{$index->{files
}}) {
6674 if ($info->{filename
} =~ m/^(drive-\S+).img.fidx$/) {
6676 if ($info->{size
} =~ m/^(\d+)$/) { # untaint size
6677 $devinfo->{$devname}->{size
} = $1;
6679 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6684 my $is_qemu_server_backup = scalar(
6685 grep { $_->{filename
} eq 'qemu-server.conf.blob' } @{$index->{files
}}
6687 if (!$is_qemu_server_backup) {
6688 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6690 my $has_firewall_config = scalar(grep { $_->{filename
} eq 'fw.conf.blob' } @{$index->{files
}});
6692 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6693 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6695 if ($has_firewall_config) {
6696 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6697 PVE
::Storage
::PBSPlugin
::run_raw_client_cmd
($scfg, $storeid, $cmd, $param);
6699 my $pve_firewall_dir = '/etc/pve/firewall';
6700 mkdir $pve_firewall_dir; # make sure the dir exists
6701 PVE
::Tools
::file_copy
($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6704 my $fh = IO
::File-
>new($cfgfn, "r") ||
6705 die "unable to read qemu-server.conf - $!\n";
6707 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6709 # fixme: rate limit?
6711 # create empty/temp config
6712 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\nlock: create");
6714 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6717 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6719 foreach my $virtdev (sort keys %$virtdev_hash) {
6720 my $d = $virtdev_hash->{$virtdev};
6721 next if $d->{is_cloudinit
}; # no need to restore cloudinit
6723 # this fails if storage is unavailable
6724 my $volid = $d->{volid
};
6725 my $path = PVE
::Storage
::path
($storecfg, $volid);
6727 # for live-restore we only want to preload the efidisk and TPM state
6728 next if $options->{live
} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6730 my $pbs_restore_cmd = [
6731 '/usr/bin/pbs-restore',
6732 '--repository', $repo,
6734 "$d->{devname}.img.fidx",
6739 push @$pbs_restore_cmd, '--format', $d->{format
} if $d->{format
};
6740 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e
$keyfile;
6742 if (PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $volid)) {
6743 push @$pbs_restore_cmd, '--skip-zero';
6746 my $dbg_cmdstring = PVE
::Tools
::cmd2string
($pbs_restore_cmd);
6747 print "restore proxmox backup image: $dbg_cmdstring\n";
6748 run_command
($pbs_restore_cmd);
6751 $fh->seek(0, 0) || die "seek failed - $!\n";
6753 my $cookie = { netcount
=> 0 };
6754 while (defined(my $line = <$fh>)) {
6755 $new_conf_raw .= restore_update_config_line
(
6767 if ($err || !$options->{live
}) {
6768 $restore_deactivate_volumes->($storecfg, $devinfo);
6774 $restore_destroy_volumes->($storecfg, $devinfo);
6778 if ($options->{live
}) {
6779 # keep lock during live-restore
6780 $new_conf_raw .= "\nlock: create";
6783 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
6785 PVE
::Cluster
::cfs_update
(); # make sure we read new file
6787 eval { rescan
($vmid, 1); };
6790 PVE
::AccessControl
::add_vm_to_pool
($vmid, $options->{pool
}) if $options->{pool
};
6792 if ($options->{live
}) {
6798 local $SIG{PIPE
} = sub { die "got signal ($!) - abort\n"; };
6800 my $conf = PVE
::QemuConfig-
>load_config($vmid);
6801 die "cannot do live-restore for template\n" if PVE
::QemuConfig-
>is_template($conf);
6803 # these special drives are already restored before start
6804 delete $devinfo->{'drive-efidisk0'};
6805 delete $devinfo->{'drive-tpmstate0-backup'};
6806 pbs_live_restore
($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6808 PVE
::QemuConfig-
>remove_lock($vmid, "create");
6812 sub pbs_live_restore
{
6813 my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6815 print "starting VM for live-restore\n";
6816 print "repository: '$repo', snapshot: '$snap'\n";
6818 my $pbs_backing = {};
6819 for my $ds (keys %$restored_disks) {
6820 $ds =~ m/^drive-(.*)$/;
6822 $pbs_backing->{$confname} = {
6823 repository
=> $repo,
6825 archive
=> "$ds.img.fidx",
6827 $pbs_backing->{$confname}->{keyfile
} = $keyfile if -e
$keyfile;
6829 my $drive = parse_drive
($confname, $conf->{$confname});
6830 print "restoring '$ds' to '$drive->{file}'\n";
6833 my $drives_streamed = 0;
6835 # make sure HA doesn't interrupt our restore by stopping the VM
6836 if (PVE
::HA
::Config
::vm_is_ha_managed
($vmid)) {
6837 run_command
(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
6840 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6841 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6842 vm_start_nolock
($storecfg, $vmid, $conf, {paused
=> 1, 'pbs-backing' => $pbs_backing}, {});
6844 my $qmeventd_fd = register_qmeventd_handle
($vmid);
6846 # begin streaming, i.e. data copy from PBS to target disk for every vol,
6847 # this will effectively collapse the backing image chain consisting of
6848 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6849 # removes itself once all backing images vanish with 'auto-remove=on')
6851 for my $ds (sort keys %$restored_disks) {
6852 my $job_id = "restore-$ds";
6853 mon_cmd
($vmid, 'block-stream',
6854 'job-id' => $job_id,
6857 $jobs->{$job_id} = {};
6860 mon_cmd
($vmid, 'cont');
6861 qemu_drive_mirror_monitor
($vmid, undef, $jobs, 'auto', 0, 'stream');
6863 print "restore-drive jobs finished successfully, removing all tracking block devices"
6864 ." to disconnect from Proxmox Backup Server\n";
6866 for my $ds (sort keys %$restored_disks) {
6867 mon_cmd
($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6870 close($qmeventd_fd);
6876 warn "An error occurred during live-restore: $err\n";
6877 _do_vm_stop
($storecfg, $vmid, 1, 1, 10, 0, 1);
6878 die "live-restore failed\n";
6882 sub restore_vma_archive
{
6883 my ($archive, $vmid, $user, $opts, $comp) = @_;
6885 my $readfrom = $archive;
6887 my $cfg = PVE
::Storage
::config
();
6889 my $bwlimit = $opts->{bwlimit
};
6891 my $dbg_cmdstring = '';
6892 my $add_pipe = sub {
6894 push @$commands, $cmd;
6895 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6896 $dbg_cmdstring .= PVE
::Tools
::cmd2string
($cmd);
6901 if ($archive eq '-') {
6904 # If we use a backup from a PVE defined storage we also consider that
6905 # storage's rate limit:
6906 my (undef, $volid) = PVE
::Storage
::path_to_volume_id
($cfg, $archive);
6907 if (defined($volid)) {
6908 my ($sid, undef) = PVE
::Storage
::parse_volume_id
($volid);
6909 my $readlimit = PVE
::Storage
::get_bandwidth_limit
('restore', [$sid], $bwlimit);
6911 print STDERR
"applying read rate limit: $readlimit\n";
6912 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6913 $add_pipe->($cstream);
6919 my $info = PVE
::Storage
::decompressor_info
('vma', $comp);
6920 my $cmd = $info->{decompressor
};
6921 push @$cmd, $readfrom;
6925 my $tmpdir = "/var/tmp/vzdumptmp$$";
6928 # disable interrupts (always do cleanups)
6932 local $SIG{HUP
} = sub { warn "got interrupt - ignored\n"; };
6934 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6935 POSIX
::mkfifo
($mapfifo, 0600);
6937 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6939 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6946 my $rpcenv = PVE
::RPCEnvironment
::get
();
6948 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
6950 # Note: $oldconf is undef if VM does not exist
6951 my $cfs_path = PVE
::QemuConfig-
>cfs_config_path($vmid);
6952 my $oldconf = PVE
::Cluster
::cfs_read_file
($cfs_path);
6953 my $new_conf_raw = '';
6957 my $print_devmap = sub {
6958 my $cfgfn = "$tmpdir/qemu-server.conf";
6960 # we can read the config - that is already extracted
6961 my $fh = IO
::File-
>new($cfgfn, "r") ||
6962 die "unable to read qemu-server.conf - $!\n";
6964 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6966 my $pve_firewall_dir = '/etc/pve/firewall';
6967 mkdir $pve_firewall_dir; # make sure the dir exists
6968 PVE
::Tools
::file_copy
($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6971 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6973 foreach my $info (values %{$virtdev_hash}) {
6974 my $storeid = $info->{storeid
};
6975 next if defined($storage_limits{$storeid});
6977 my $limit = PVE
::Storage
::get_bandwidth_limit
('restore', [$storeid], $bwlimit) // 0;
6978 print STDERR
"rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6979 $storage_limits{$storeid} = $limit * 1024;
6982 foreach my $devname (keys %$devinfo) {
6983 die "found no device mapping information for device '$devname'\n"
6984 if !$devinfo->{$devname}->{virtdev
};
6987 # create empty/temp config
6989 PVE
::Tools
::file_set_contents
($conffile, "memory: 128\n");
6990 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6994 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6996 # print restore information to $fifofh
6997 foreach my $virtdev (sort keys %$virtdev_hash) {
6998 my $d = $virtdev_hash->{$virtdev};
6999 next if $d->{is_cloudinit
}; # no need to restore cloudinit
7001 my $storeid = $d->{storeid
};
7002 my $volid = $d->{volid
};
7005 if (my $limit = $storage_limits{$storeid}) {
7006 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7009 my $write_zeros = 1;
7010 if (PVE
::Storage
::volume_has_feature
($cfg, 'sparseinit', $volid)) {
7014 my $path = PVE
::Storage
::path
($cfg, $volid);
7016 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7018 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7021 $fh->seek(0, 0) || die "seek failed - $!\n";
7023 my $cookie = { netcount
=> 0 };
7024 while (defined(my $line = <$fh>)) {
7025 $new_conf_raw .= restore_update_config_line
(
7042 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7043 local $SIG{ALRM
} = sub { die "got timeout\n"; };
7045 $oldtimeout = alarm($timeout);
7052 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7053 my ($dev_id, $size, $devname) = ($1, $2, $3);
7054 $devinfo->{$devname} = { size
=> $size, dev_id
=> $dev_id };
7055 } elsif ($line =~ m/^CTIME: /) {
7056 # we correctly received the vma config, so we can disable
7057 # the timeout now for disk allocation (set to 10 minutes, so
7058 # that we always timeout if something goes wrong)
7061 print $fifofh "done\n";
7062 my $tmp = $oldtimeout || 0;
7063 $oldtimeout = undef;
7070 print "restore vma archive: $dbg_cmdstring\n";
7071 run_command
($commands, input
=> $input, outfunc
=> $parser, afterfork
=> $openfifo);
7075 alarm($oldtimeout) if $oldtimeout;
7077 $restore_deactivate_volumes->($cfg, $devinfo);
7079 close($fifofh) if $fifofh;
7084 $restore_destroy_volumes->($cfg, $devinfo);
7088 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
7090 PVE
::Cluster
::cfs_update
(); # make sure we read new file
7092 eval { rescan
($vmid, 1); };
7095 PVE
::AccessControl
::add_vm_to_pool
($vmid, $opts->{pool
}) if $opts->{pool
};
7098 sub restore_tar_archive
{
7099 my ($archive, $vmid, $user, $opts) = @_;
7101 if ($archive ne '-') {
7102 my $firstfile = tar_archive_read_firstfile
($archive);
7103 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7104 if $firstfile ne 'qemu-server.conf';
7107 my $storecfg = PVE
::Storage
::config
();
7109 # avoid zombie disks when restoring over an existing VM -> cleanup first
7110 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7111 # skiplock=1 because qmrestore has set the 'create' lock itself already
7112 my $vmcfgfn = PVE
::QemuConfig-
>config_file($vmid);
7113 destroy_vm
($storecfg, $vmid, 1, { lock => 'restore' }) if -f
$vmcfgfn;
7115 my $tocmd = "/usr/lib/qemu-server/qmextract";
7117 $tocmd .= " --storage " . PVE
::Tools
::shellquote
($opts->{storage
}) if $opts->{storage
};
7118 $tocmd .= " --pool " . PVE
::Tools
::shellquote
($opts->{pool
}) if $opts->{pool
};
7119 $tocmd .= ' --prealloc' if $opts->{prealloc
};
7120 $tocmd .= ' --info' if $opts->{info
};
7122 # tar option "xf" does not autodetect compression when read from STDIN,
7123 # so we pipe to zcat
7124 my $cmd = "zcat -f|tar xf " . PVE
::Tools
::shellquote
($archive) . " " .
7125 PVE
::Tools
::shellquote
("--to-command=$tocmd");
7127 my $tmpdir = "/var/tmp/vzdumptmp$$";
7130 local $ENV{VZDUMP_TMPDIR
} = $tmpdir;
7131 local $ENV{VZDUMP_VMID
} = $vmid;
7132 local $ENV{VZDUMP_USER
} = $user;
7134 my $conffile = PVE
::QemuConfig-
>config_file($vmid);
7135 my $new_conf_raw = '';
7137 # disable interrupts (always do cleanups)
7141 local $SIG{HUP
} = sub { print STDERR
"got interrupt - ignored\n"; };
7149 local $SIG{PIPE
} = sub { die "interrupted by signal\n"; };
7151 if ($archive eq '-') {
7152 print "extracting archive from STDIN\n";
7153 run_command
($cmd, input
=> "<&STDIN");
7155 print "extracting archive '$archive'\n";
7159 return if $opts->{info
};
7163 my $statfile = "$tmpdir/qmrestore.stat";
7164 if (my $fd = IO
::File-
>new($statfile, "r")) {
7165 while (defined (my $line = <$fd>)) {
7166 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7167 $map->{$1} = $2 if $1;
7169 print STDERR
"unable to parse line in statfile - $line\n";
7175 my $confsrc = "$tmpdir/qemu-server.conf";
7177 my $srcfd = IO
::File-
>new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7179 my $cookie = { netcount
=> 0 };
7180 while (defined (my $line = <$srcfd>)) {
7181 $new_conf_raw .= restore_update_config_line
(
7192 tar_restore_cleanup
($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info
};
7198 PVE
::Tools
::file_set_contents
($conffile, $new_conf_raw);
7200 PVE
::Cluster
::cfs_update
(); # make sure we read new file
7202 eval { rescan
($vmid, 1); };
7206 sub foreach_storage_used_by_vm
{
7207 my ($conf, $func) = @_;
7211 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7212 my ($ds, $drive) = @_;
7213 return if drive_is_cdrom
($drive);
7215 my $volid = $drive->{file
};
7217 my ($sid, $volname) = PVE
::Storage
::parse_volume_id
($volid, 1);
7218 $sidhash->{$sid} = $sid if $sid;
7221 foreach my $sid (sort keys %$sidhash) {
7226 my $qemu_snap_storage = {
7229 sub do_snapshots_with_qemu
{
7230 my ($storecfg, $volid, $deviceid) = @_;
7232 return if $deviceid =~ m/tpmstate0/;
7234 my $storage_name = PVE
::Storage
::parse_volume_id
($volid);
7235 my $scfg = $storecfg->{ids
}->{$storage_name};
7236 die "could not find storage '$storage_name'\n" if !defined($scfg);
7238 if ($qemu_snap_storage->{$scfg->{type
}} && !$scfg->{krbd
}){
7242 if ($volid =~ m/\.(qcow2|qed)$/){
7249 sub qga_check_running
{
7250 my ($vmid, $nowarn) = @_;
7252 eval { mon_cmd
($vmid, "guest-ping", timeout
=> 3); };
7254 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7260 sub template_create
{
7261 my ($vmid, $conf, $disk) = @_;
7263 my $storecfg = PVE
::Storage
::config
();
7265 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7266 my ($ds, $drive) = @_;
7268 return if drive_is_cdrom
($drive);
7269 return if $disk && $ds ne $disk;
7271 my $volid = $drive->{file
};
7272 return if !PVE
::Storage
::volume_has_feature
($storecfg, 'template', $volid);
7274 my $voliddst = PVE
::Storage
::vdisk_create_base
($storecfg, $volid);
7275 $drive->{file
} = $voliddst;
7276 $conf->{$ds} = print_drive
($drive);
7277 PVE
::QemuConfig-
>write_config($vmid, $conf);
7281 sub convert_iscsi_path
{
7284 if ($path =~ m
|^iscsi
://([^/]+)/([^/]+)/(.+)$|) {
7289 my $initiator_name = get_initiator_name
();
7291 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7292 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7295 die "cannot convert iscsi path '$path', unkown format\n";
7298 sub qemu_img_convert
{
7299 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7301 my $storecfg = PVE
::Storage
::config
();
7302 my ($src_storeid, $src_volname) = PVE
::Storage
::parse_volume_id
($src_volid, 1);
7303 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid, 1);
7305 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7309 my $src_is_iscsi = 0;
7313 PVE
::Storage
::activate_volumes
($storecfg, [$src_volid], $snapname);
7314 my $src_scfg = PVE
::Storage
::storage_config
($storecfg, $src_storeid);
7315 $src_format = qemu_img_format
($src_scfg, $src_volname);
7316 $src_path = PVE
::Storage
::path
($storecfg, $src_volid, $snapname);
7317 $src_is_iscsi = ($src_path =~ m
|^iscsi
://|);
7318 $cachemode = 'none' if $src_scfg->{type
} eq 'zfspool';
7319 } elsif (-f
$src_volid || -b
$src_volid) {
7320 $src_path = $src_volid;
7321 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7326 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7328 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7329 my $dst_format = qemu_img_format
($dst_scfg, $dst_volname);
7330 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7331 my $dst_is_iscsi = ($dst_path =~ m
|^iscsi
://|);
7334 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7335 push @$cmd, '-l', "snapshot.name=$snapname"
7336 if $snapname && $src_format && $src_format eq "qcow2";
7337 push @$cmd, '-t', 'none' if $dst_scfg->{type
} eq 'zfspool';
7338 push @$cmd, '-T', $cachemode if defined($cachemode);
7340 if ($src_is_iscsi) {
7341 push @$cmd, '--image-opts';
7342 $src_path = convert_iscsi_path
($src_path);
7343 } elsif ($src_format) {
7344 push @$cmd, '-f', $src_format;
7347 if ($dst_is_iscsi) {
7348 push @$cmd, '--target-image-opts';
7349 $dst_path = convert_iscsi_path
($dst_path);
7351 push @$cmd, '-O', $dst_format;
7354 push @$cmd, $src_path;
7356 if (!$dst_is_iscsi && $is_zero_initialized) {
7357 push @$cmd, "zeroinit:$dst_path";
7359 push @$cmd, $dst_path;
7364 if($line =~ m/\((\S+)\/100\
%\)/){
7366 my $transferred = int($size * $percent / 100);
7367 my $total_h = render_bytes
($size, 1);
7368 my $transferred_h = render_bytes
($transferred, 1);
7370 print "transferred $transferred_h of $total_h ($percent%)\n";
7375 eval { run_command
($cmd, timeout
=> undef, outfunc
=> $parser); };
7377 die "copy failed: $err" if $err;
7380 sub qemu_img_format
{
7381 my ($scfg, $volname) = @_;
7383 if ($scfg->{path
} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7390 sub qemu_drive_mirror
{
7391 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7393 $jobs = {} if !$jobs;
7397 $jobs->{"drive-$drive"} = {};
7399 if ($dst_volid =~ /^nbd:/) {
7400 $qemu_target = $dst_volid;
7403 my $storecfg = PVE
::Storage
::config
();
7404 my ($dst_storeid, $dst_volname) = PVE
::Storage
::parse_volume_id
($dst_volid);
7406 my $dst_scfg = PVE
::Storage
::storage_config
($storecfg, $dst_storeid);
7408 $format = qemu_img_format
($dst_scfg, $dst_volname);
7410 my $dst_path = PVE
::Storage
::path
($storecfg, $dst_volid);
7412 $qemu_target = $is_zero_initialized ?
"zeroinit:$dst_path" : $dst_path;
7415 my $opts = { timeout
=> 10, device
=> "drive-$drive", mode
=> "existing", sync
=> "full", target
=> $qemu_target };
7416 $opts->{format
} = $format if $format;
7418 if (defined($src_bitmap)) {
7419 $opts->{sync
} = 'incremental';
7420 $opts->{bitmap
} = $src_bitmap;
7421 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7424 if (defined($bwlimit)) {
7425 $opts->{speed
} = $bwlimit * 1024;
7426 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7428 print "drive mirror is starting for drive-$drive\n";
7431 # if a job already runs for this device we get an error, catch it for cleanup
7432 eval { mon_cmd
($vmid, "drive-mirror", %$opts); };
7434 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7436 die "mirroring error: $err\n";
7439 qemu_drive_mirror_monitor
($vmid, $vmiddst, $jobs, $completion, $qga);
7442 # $completion can be either
7443 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7444 # 'cancel': wait until all jobs are ready, block-job-cancel them
7445 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7446 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7447 sub qemu_drive_mirror_monitor
{
7448 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7450 $completion //= 'complete';
7454 my $err_complete = 0;
7456 my $starttime = time ();
7458 die "block job ('$op') timed out\n" if $err_complete > 300;
7460 my $stats = mon_cmd
($vmid, "query-block-jobs");
7463 my $running_jobs = {};
7464 for my $stat (@$stats) {
7465 next if $stat->{type
} ne $op;
7466 $running_jobs->{$stat->{device
}} = $stat;
7469 my $readycounter = 0;
7471 for my $job_id (sort keys %$jobs) {
7472 my $job = $running_jobs->{$job_id};
7474 my $vanished = !defined($job);
7475 my $complete = defined($jobs->{$job_id}->{complete
}) && $vanished;
7476 if($complete || ($vanished && $completion eq 'auto')) {
7477 print "$job_id: $op-job finished\n";
7478 delete $jobs->{$job_id};
7482 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7484 my $busy = $job->{busy
};
7485 my $ready = $job->{ready
};
7486 if (my $total = $job->{len
}) {
7487 my $transferred = $job->{offset
} || 0;
7488 my $remaining = $total - $transferred;
7489 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7491 my $duration = $ctime - $starttime;
7492 my $total_h = render_bytes
($total, 1);
7493 my $transferred_h = render_bytes
($transferred, 1);
7495 my $status = sprintf(
7496 "transferred $transferred_h of $total_h ($percent%%) in %s",
7497 render_duration
($duration),
7502 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7504 $status .= ", ready";
7507 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready
};
7508 $jobs->{$job_id}->{ready
} = $ready;
7511 $readycounter++ if $job->{ready
};
7514 last if scalar(keys %$jobs) == 0;
7516 if ($readycounter == scalar(keys %$jobs)) {
7517 print "all '$op' jobs are ready\n";
7519 # do the complete later (or has already been done)
7520 last if $completion eq 'skip' || $completion eq 'auto';
7522 if ($vmiddst && $vmiddst != $vmid) {
7523 my $agent_running = $qga && qga_check_running
($vmid);
7524 if ($agent_running) {
7525 print "freeze filesystem\n";
7526 eval { mon_cmd
($vmid, "guest-fsfreeze-freeze"); };
7529 print "suspend vm\n";
7530 eval { PVE
::QemuServer
::vm_suspend
($vmid, 1); };
7534 # if we clone a disk for a new target vm, we don't switch the disk
7535 PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs);
7537 if ($agent_running) {
7538 print "unfreeze filesystem\n";
7539 eval { mon_cmd
($vmid, "guest-fsfreeze-thaw"); };
7542 print "resume vm\n";
7543 eval { PVE
::QemuServer
::vm_resume
($vmid, 1, 1); };
7550 for my $job_id (sort keys %$jobs) {
7551 # try to switch the disk if source and destination are on the same guest
7552 print "$job_id: Completing block job_id...\n";
7555 if ($completion eq 'complete') {
7556 $op = 'block-job-complete';
7557 } elsif ($completion eq 'cancel') {
7558 $op = 'block-job-cancel';
7560 die "invalid completion value: $completion\n";
7562 eval { mon_cmd
($vmid, $op, device
=> $job_id) };
7563 if ($@ =~ m/cannot be completed/) {
7564 print "$job_id: block job cannot be completed, trying again.\n";
7567 print "$job_id: Completed successfully.\n";
7568 $jobs->{$job_id}->{complete
} = 1;
7579 eval { PVE
::QemuServer
::qemu_blockjobs_cancel
($vmid, $jobs) };
7580 die "block job ($op) error: $err";
7584 sub qemu_blockjobs_cancel
{
7585 my ($vmid, $jobs) = @_;
7587 foreach my $job (keys %$jobs) {
7588 print "$job: Cancelling block job\n";
7589 eval { mon_cmd
($vmid, "block-job-cancel", device
=> $job); };
7590 $jobs->{$job}->{cancel
} = 1;
7594 my $stats = mon_cmd
($vmid, "query-block-jobs");
7596 my $running_jobs = {};
7597 foreach my $stat (@$stats) {
7598 $running_jobs->{$stat->{device
}} = $stat;
7601 foreach my $job (keys %$jobs) {
7603 if (defined($jobs->{$job}->{cancel
}) && !defined($running_jobs->{$job})) {
7604 print "$job: Done.\n";
7605 delete $jobs->{$job};
7609 last if scalar(keys %$jobs) == 0;
7616 my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7618 my ($vmid, $running) = $source->@{qw(vmid running)};
7619 my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7621 my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7622 my ($storage, $format) = $dest->@{qw(storage format)};
7624 my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7626 if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7627 die "cloning from/to EFI disk requires EFI disk\n"
7628 if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7629 die "cloning from/to TPM state requires TPM state\n"
7630 if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7632 # This would lead to two device nodes in QEMU pointing to the same backing image!
7633 die "cannot change drive name when cloning disk from/to the same VM\n"
7634 if $use_drive_mirror && $vmid == $newvmid;
7637 die "cannot move TPM state while VM is running\n"
7638 if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7642 print "create " . ($full ?
'full' : 'linked') . " clone of drive ";
7643 print "$src_drivename " if $src_drivename;
7644 print "($drive->{file})\n";
7647 $newvolid = PVE
::Storage
::vdisk_clone
($storecfg, $drive->{file
}, $newvmid, $snapname);
7648 push @$newvollist, $newvolid;
7651 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($drive->{file
});
7652 $storeid = $storage if $storage;
7654 my $dst_format = resolve_dst_disk_format
($storecfg, $storeid, $volname, $format);
7658 if (drive_is_cloudinit
($drive)) {
7659 $name = "vm-$newvmid-cloudinit";
7660 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7661 if ($scfg->{path
}) {
7662 $name .= ".$dst_format";
7665 $size = PVE
::QemuServer
::Cloudinit
::CLOUDINIT_DISK_SIZE
;
7666 } elsif ($dst_drivename eq 'efidisk0') {
7667 $size = $efisize or die "internal error - need to specify EFI disk size\n";
7668 } elsif ($dst_drivename eq 'tpmstate0') {
7669 $dst_format = 'raw';
7670 $size = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
7672 ($size) = PVE
::Storage
::volume_size_info
($storecfg, $drive->{file
}, 10);
7674 $newvolid = PVE
::Storage
::vdisk_alloc
(
7675 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7677 push @$newvollist, $newvolid;
7679 PVE
::Storage
::activate_volumes
($storecfg, [$newvolid]);
7681 if (drive_is_cloudinit
($drive)) {
7682 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7683 # if this is the case, we have to complete any block-jobs still there from
7684 # previous drive-mirrors
7685 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7686 qemu_drive_mirror_monitor
($vmid, $newvmid, $jobs, $completion, $qga);
7691 my $sparseinit = PVE
::Storage
::volume_has_feature
($storecfg, 'sparseinit', $newvolid);
7692 if ($use_drive_mirror) {
7693 qemu_drive_mirror
($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7694 $completion, $qga, $bwlimit);
7696 # TODO: handle bwlimits
7697 if ($dst_drivename eq 'efidisk0') {
7698 # the relevant data on the efidisk may be smaller than the source
7699 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7700 # that is given by the OVMF_VARS.fd
7701 my $src_path = PVE
::Storage
::path
($storecfg, $drive->{file
}, $snapname);
7702 my $dst_path = PVE
::Storage
::path
($storecfg, $newvolid);
7704 my $src_format = (PVE
::Storage
::parse_volname
($storecfg, $drive->{file
}))[6];
7706 # better for Ceph if block size is not too small, see bug #3324
7709 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
7711 if ($src_format eq 'qcow2' && $snapname) {
7712 die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
7713 if !min_version
(kvm_user_version
(), 6, 2);
7714 push $cmd->@*, '-l', $snapname;
7716 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
7719 qemu_img_convert
($drive->{file
}, $newvolid, $size, $snapname, $sparseinit);
7725 my ($size) = eval { PVE
::Storage
::volume_size_info
($storecfg, $newvolid, 10) };
7727 my $disk = dclone
($drive);
7728 delete $disk->{format
};
7729 $disk->{file
} = $newvolid;
7730 $disk->{size
} = $size if defined($size);
7735 sub get_running_qemu_version
{
7737 my $res = mon_cmd
($vmid, "query-version");
7738 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7741 sub qemu_use_old_bios_files
{
7742 my ($machine_type) = @_;
7744 return if !$machine_type;
7746 my $use_old_bios_files = undef;
7748 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7750 $use_old_bios_files = 1;
7752 my $version = extract_version
($machine_type, kvm_user_version
());
7753 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7754 # load new efi bios files on migration. So this hack is required to allow
7755 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7756 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7757 $use_old_bios_files = !min_version
($version, 2, 4);
7760 return ($use_old_bios_files, $machine_type);
7763 sub get_efivars_size
{
7764 my ($conf, $efidisk) = @_;
7766 my $arch = get_vm_arch
($conf);
7767 $efidisk //= $conf->{efidisk0
} ? parse_drive
('efidisk0', $conf->{efidisk0
}) : undef;
7768 my $smm = PVE
::QemuServer
::Machine
::machine_type_is_q35
($conf);
7769 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
7770 die "uefi vars image '$ovmf_vars' not found\n" if ! -f
$ovmf_vars;
7771 return -s
$ovmf_vars;
7774 sub update_efidisk_size
{
7777 return if !defined($conf->{efidisk0
});
7779 my $disk = PVE
::QemuServer
::parse_drive
('efidisk0', $conf->{efidisk0
});
7780 $disk->{size
} = get_efivars_size
($conf);
7781 $conf->{efidisk0
} = print_drive
($disk);
7786 sub update_tpmstate_size
{
7789 my $disk = PVE
::QemuServer
::parse_drive
('tpmstate0', $conf->{tpmstate0
});
7790 $disk->{size
} = PVE
::QemuServer
::Drive
::TPMSTATE_DISK_SIZE
;
7791 $conf->{tpmstate0
} = print_drive
($disk);
7794 sub create_efidisk
($$$$$$$) {
7795 my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7797 my (undef, $ovmf_vars) = get_ovmf_files
($arch, $efidisk, $smm);
7798 die "EFI vars default image not found\n" if ! -f
$ovmf_vars;
7800 my $vars_size_b = -s
$ovmf_vars;
7801 my $vars_size = PVE
::Tools
::convert_size
($vars_size_b, 'b' => 'kb');
7802 my $volid = PVE
::Storage
::vdisk_alloc
($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7803 PVE
::Storage
::activate_volumes
($storecfg, [$volid]);
7805 qemu_img_convert
($ovmf_vars, $volid, $vars_size_b, undef, 0);
7806 my ($size) = PVE
::Storage
::volume_size_info
($storecfg, $volid, 3);
7808 return ($volid, $size/1024);
7811 sub vm_iothreads_list
{
7814 my $res = mon_cmd
($vmid, 'query-iothreads');
7817 foreach my $iothread (@$res) {
7818 $iothreads->{ $iothread->{id
} } = $iothread->{"thread-id"};
7825 my ($conf, $drive) = @_;
7829 if (!$conf->{scsihw
} || ($conf->{scsihw
} =~ m/^lsi/)) {
7831 } elsif ($conf->{scsihw
} && ($conf->{scsihw
} eq 'virtio-scsi-single')) {
7837 my $controller = int($drive->{index} / $maxdev);
7838 my $controller_prefix = ($conf->{scsihw
} && $conf->{scsihw
} eq 'virtio-scsi-single')
7842 return ($maxdev, $controller, $controller_prefix);
7845 sub windows_version
{
7848 return 0 if !$ostype;
7852 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7854 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7856 } elsif ($ostype =~ m/^win(\d+)$/) {
7863 sub resolve_dst_disk_format
{
7864 my ($storecfg, $storeid, $src_volname, $format) = @_;
7865 my ($defFormat, $validFormats) = PVE
::Storage
::storage_default_format
($storecfg, $storeid);
7868 # if no target format is specified, use the source disk format as hint
7870 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
7871 $format = qemu_img_format
($scfg, $src_volname);
7877 # test if requested format is supported - else use default
7878 my $supported = grep { $_ eq $format } @$validFormats;
7879 $format = $defFormat if !$supported;
7883 # NOTE: if this logic changes, please update docs & possibly gui logic
7884 sub find_vmstate_storage
{
7885 my ($conf, $storecfg) = @_;
7887 # first, return storage from conf if set
7888 return $conf->{vmstatestorage
} if $conf->{vmstatestorage
};
7890 my ($target, $shared, $local);
7892 foreach_storage_used_by_vm
($conf, sub {
7894 my $scfg = PVE
::Storage
::storage_config
($storecfg, $sid);
7895 my $dst = $scfg->{shared
} ? \
$shared : \
$local;
7896 $$dst = $sid if !$$dst || $scfg->{path
}; # prefer file based storage
7899 # second, use shared storage where VM has at least one disk
7900 # third, use local storage where VM has at least one disk
7901 # fall back to local storage
7902 $target = $shared // $local // 'local';
7908 my ($uuid, $uuid_str);
7909 UUID
::generate
($uuid);
7910 UUID
::unparse
($uuid, $uuid_str);
7914 sub generate_smbios1_uuid
{
7915 return "uuid=".generate_uuid
();
7921 mon_cmd
($vmid, 'nbd-server-stop');
7924 sub create_reboot_request
{
7926 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7927 or die "failed to create reboot trigger file: $!\n";
7931 sub clear_reboot_request
{
7933 my $path = "/run/qemu-server/$vmid.reboot";
7936 $res = unlink($path);
7937 die "could not remove reboot request for $vmid: $!"
7938 if !$res && $! != POSIX
::ENOENT
;
7943 sub bootorder_from_legacy
{
7944 my ($conf, $bootcfg) = @_;
7946 my $boot = $bootcfg->{legacy
} || $boot_fmt->{legacy
}->{default};
7947 my $bootindex_hash = {};
7949 foreach my $o (split(//, $boot)) {
7950 $bootindex_hash->{$o} = $i*100;
7956 PVE
::QemuConfig-
>foreach_volume($conf, sub {
7957 my ($ds, $drive) = @_;
7959 if (drive_is_cdrom
($drive, 1)) {
7960 if ($bootindex_hash->{d
}) {
7961 $bootorder->{$ds} = $bootindex_hash->{d
};
7962 $bootindex_hash->{d
} += 1;
7964 } elsif ($bootindex_hash->{c
}) {
7965 $bootorder->{$ds} = $bootindex_hash->{c
}
7966 if $conf->{bootdisk
} && $conf->{bootdisk
} eq $ds;
7967 $bootindex_hash->{c
} += 1;
7971 if ($bootindex_hash->{n
}) {
7972 for (my $i = 0; $i < $MAX_NETS; $i++) {
7973 my $netname = "net$i";
7974 next if !$conf->{$netname};
7975 $bootorder->{$netname} = $bootindex_hash->{n
};
7976 $bootindex_hash->{n
} += 1;
7983 # Generate default device list for 'boot: order=' property. Matches legacy
7984 # default boot order, but with explicit device names. This is important, since
7985 # the fallback for when neither 'order' nor the old format is specified relies
7986 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7987 sub get_default_bootdevices
{
7993 my $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 0);
7994 push @ret, $first if $first;
7997 $first = PVE
::QemuServer
::Drive
::resolve_first_disk
($conf, 1);
7998 push @ret, $first if $first;
8001 for (my $i = 0; $i < $MAX_NETS; $i++) {
8002 my $netname = "net$i";
8003 next if !$conf->{$netname};
8004 push @ret, $netname;
8011 sub device_bootorder
{
8014 return bootorder_from_legacy
($conf) if !defined($conf->{boot
});
8016 my $boot = parse_property_string
($boot_fmt, $conf->{boot
});
8019 if (!defined($boot) || $boot->{legacy
}) {
8020 $bootorder = bootorder_from_legacy
($conf, $boot);
8021 } elsif ($boot->{order
}) {
8022 my $i = 100; # start at 100 to allow user to insert devices before us with -args
8023 for my $dev (PVE
::Tools
::split_list
($boot->{order
})) {
8024 $bootorder->{$dev} = $i++;
8031 sub register_qmeventd_handle
{
8035 my $peer = "/var/run/qmeventd.sock";
8040 $fh = IO
::Socket
::UNIX-
>new(Peer
=> $peer, Blocking
=> 0, Timeout
=> 1);
8042 if ($! != EINTR
&& $! != EAGAIN
) {
8043 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8046 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8047 . "after $count retries\n";
8052 # send handshake to mark VM as backing up
8053 print $fh to_json
({vzdump
=> {vmid
=> "$vmid"}});
8055 # return handle to be closed later when inhibit is no longer required
8059 # bash completion helper
8061 sub complete_backup_archives
{
8062 my ($cmdname, $pname, $cvalue) = @_;
8064 my $cfg = PVE
::Storage
::config
();
8068 if ($cvalue =~ m/^([^:]+):/) {
8072 my $data = PVE
::Storage
::template_list
($cfg, $storeid, 'backup');
8075 foreach my $id (keys %$data) {
8076 foreach my $item (@{$data->{$id}}) {
8077 next if $item->{format
} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8078 push @$res, $item->{volid
} if defined($item->{volid
});
8085 my $complete_vmid_full = sub {
8088 my $idlist = vmstatus
();
8092 foreach my $id (keys %$idlist) {
8093 my $d = $idlist->{$id};
8094 if (defined($running)) {
8095 next if $d->{template
};
8096 next if $running && $d->{status
} ne 'running';
8097 next if !$running && $d->{status
} eq 'running';
8106 return &$complete_vmid_full();
8109 sub complete_vmid_stopped
{
8110 return &$complete_vmid_full(0);
8113 sub complete_vmid_running
{
8114 return &$complete_vmid_full(1);
8117 sub complete_storage
{
8119 my $cfg = PVE
::Storage
::config
();
8120 my $ids = $cfg->{ids
};
8123 foreach my $sid (keys %$ids) {
8124 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, undef, 1);
8125 next if !$ids->{$sid}->{content
}->{images
};
8132 sub complete_migration_storage
{
8133 my ($cmd, $param, $current_value, $all_args) = @_;
8135 my $targetnode = @$all_args[1];
8137 my $cfg = PVE
::Storage
::config
();
8138 my $ids = $cfg->{ids
};
8141 foreach my $sid (keys %$ids) {
8142 next if !PVE
::Storage
::storage_check_enabled
($cfg, $sid, $targetnode, 1);
8143 next if !$ids->{$sid}->{content
}->{images
};
8152 my $qmpstatus = eval {
8153 PVE
::QemuConfig
::assert_config_exists_on_node
($vmid);
8154 mon_cmd
($vmid, "query-status");
8157 return $qmpstatus && $qmpstatus->{status
} eq "paused";
8160 sub check_volume_storage_type
{
8161 my ($storecfg, $vol) = @_;
8163 my ($storeid, $volname) = PVE
::Storage
::parse_volume_id
($vol);
8164 my $scfg = PVE
::Storage
::storage_config
($storecfg, $storeid);
8165 my ($vtype) = PVE
::Storage
::parse_volname
($storecfg, $vol);
8167 die "storage '$storeid' does not support content-type '$vtype'\n"
8168 if !$scfg->{content
}->{$vtype};